1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  26  */
  27 
  28 /*
  29  * VM - Hardware Address Translation management for Spitfire MMU.
  30  *
  31  * This file implements the machine specific hardware translation
  32  * needed by the VM system.  The machine independent interface is
  33  * described in <vm/hat.h> while the machine dependent interface
  34  * and data structures are described in <vm/hat_sfmmu.h>.
  35  *
  36  * The hat layer manages the address translation hardware as a cache
  37  * driven by calls from the higher levels in the VM system.
  38  */
  39 
  40 #include <sys/types.h>
  41 #include <sys/kstat.h>
  42 #include <vm/hat.h>
  43 #include <vm/hat_sfmmu.h>
  44 #include <vm/page.h>
  45 #include <sys/pte.h>
  46 #include <sys/systm.h>
  47 #include <sys/mman.h>
  48 #include <sys/sysmacros.h>
  49 #include <sys/machparam.h>
  50 #include <sys/vtrace.h>
  51 #include <sys/kmem.h>
  52 #include <sys/mmu.h>
  53 #include <sys/cmn_err.h>
  54 #include <sys/cpu.h>
  55 #include <sys/cpuvar.h>
  56 #include <sys/debug.h>
  57 #include <sys/lgrp.h>
  58 #include <sys/archsystm.h>
  59 #include <sys/machsystm.h>
  60 #include <sys/vmsystm.h>
  61 #include <vm/as.h>
  62 #include <vm/seg.h>
  63 #include <vm/seg_kp.h>
  64 #include <vm/seg_kmem.h>
  65 #include <vm/seg_kpm.h>
  66 #include <vm/rm.h>
  67 #include <sys/t_lock.h>
  68 #include <sys/obpdefs.h>
  69 #include <sys/vm_machparam.h>
  70 #include <sys/var.h>
  71 #include <sys/trap.h>
  72 #include <sys/machtrap.h>
  73 #include <sys/scb.h>
  74 #include <sys/bitmap.h>
  75 #include <sys/machlock.h>
  76 #include <sys/membar.h>
  77 #include <sys/atomic.h>
  78 #include <sys/cpu_module.h>
  79 #include <sys/prom_debug.h>
  80 #include <sys/ksynch.h>
  81 #include <sys/mem_config.h>
  82 #include <sys/mem_cage.h>
  83 #include <vm/vm_dep.h>
  84 #include <vm/xhat_sfmmu.h>
  85 #include <sys/fpu/fpusystm.h>
  86 #include <vm/mach_kpm.h>
  87 #include <sys/callb.h>
  88 
  89 #ifdef  DEBUG
  90 #define SFMMU_VALIDATE_HMERID(hat, rid, saddr, len)                     \
  91         if (SFMMU_IS_SHMERID_VALID(rid)) {                              \
  92                 caddr_t _eaddr = (saddr) + (len);                       \
  93                 sf_srd_t *_srdp;                                        \
  94                 sf_region_t *_rgnp;                                     \
  95                 ASSERT((rid) < SFMMU_MAX_HME_REGIONS);                       \
  96                 ASSERT(SF_RGNMAP_TEST(hat->sfmmu_hmeregion_map, rid));       \
  97                 ASSERT((hat) != ksfmmup);                               \
  98                 _srdp = (hat)->sfmmu_srdp;                           \
  99                 ASSERT(_srdp != NULL);                                  \
 100                 ASSERT(_srdp->srd_refcnt != 0);                              \
 101                 _rgnp = _srdp->srd_hmergnp[(rid)];                   \
 102                 ASSERT(_rgnp != NULL && _rgnp->rgn_id == rid);               \
 103                 ASSERT(_rgnp->rgn_refcnt != 0);                              \
 104                 ASSERT(!(_rgnp->rgn_flags & SFMMU_REGION_FREE)); \
 105                 ASSERT((_rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) ==    \
 106                     SFMMU_REGION_HME);                                  \
 107                 ASSERT((saddr) >= _rgnp->rgn_saddr);                      \
 108                 ASSERT((saddr) < _rgnp->rgn_saddr + _rgnp->rgn_size);  \
 109                 ASSERT(_eaddr > _rgnp->rgn_saddr);                        \
 110                 ASSERT(_eaddr <= _rgnp->rgn_saddr + _rgnp->rgn_size);  \
 111         }
 112 
 113 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid)              \
 114 {                                                                        \
 115                 caddr_t _hsva;                                           \
 116                 caddr_t _heva;                                           \
 117                 caddr_t _rsva;                                           \
 118                 caddr_t _reva;                                           \
 119                 int     _ttesz = get_hblk_ttesz(hmeblkp);                \
 120                 int     _flagtte;                                        \
 121                 ASSERT((srdp)->srd_refcnt != 0);                      \
 122                 ASSERT((rid) < SFMMU_MAX_HME_REGIONS);                        \
 123                 ASSERT((rgnp)->rgn_id == rid);                                \
 124                 ASSERT(!((rgnp)->rgn_flags & SFMMU_REGION_FREE));         \
 125                 ASSERT(((rgnp)->rgn_flags & SFMMU_REGION_TYPE_MASK) ==    \
 126                     SFMMU_REGION_HME);                                   \
 127                 ASSERT(_ttesz <= (rgnp)->rgn_pgszc);                       \
 128                 _hsva = (caddr_t)get_hblk_base(hmeblkp);                 \
 129                 _heva = get_hblk_endaddr(hmeblkp);                       \
 130                 _rsva = (caddr_t)P2ALIGN(                                \
 131                     (uintptr_t)(rgnp)->rgn_saddr, HBLK_MIN_BYTES);    \
 132                 _reva = (caddr_t)P2ROUNDUP(                              \
 133                     (uintptr_t)((rgnp)->rgn_saddr + (rgnp)->rgn_size),     \
 134                     HBLK_MIN_BYTES);                                     \
 135                 ASSERT(_hsva >= _rsva);                                       \
 136                 ASSERT(_hsva < _reva);                                        \
 137                 ASSERT(_heva > _rsva);                                        \
 138                 ASSERT(_heva <= _reva);                                       \
 139                 _flagtte = (_ttesz < HBLK_MIN_TTESZ) ? HBLK_MIN_TTESZ :  \
 140                         _ttesz;                                          \
 141                 ASSERT(rgnp->rgn_hmeflags & (0x1 << _flagtte));             \
 142 }
 143 
 144 #else /* DEBUG */
 145 #define SFMMU_VALIDATE_HMERID(hat, rid, addr, len)
 146 #define SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid)
 147 #endif /* DEBUG */
 148 
 149 #if defined(SF_ERRATA_57)
 150 extern caddr_t errata57_limit;
 151 #endif
 152 
 153 #define HME8BLK_SZ_RND          ((roundup(HME8BLK_SZ, sizeof (int64_t))) /  \
 154                                 (sizeof (int64_t)))
 155 #define HBLK_RESERVE            ((struct hme_blk *)hblk_reserve)
 156 
 157 #define HBLK_RESERVE_CNT        128
 158 #define HBLK_RESERVE_MIN        20
 159 
 160 static struct hme_blk           *freehblkp;
 161 static kmutex_t                 freehblkp_lock;
 162 static int                      freehblkcnt;
 163 
 164 static int64_t                  hblk_reserve[HME8BLK_SZ_RND];
 165 static kmutex_t                 hblk_reserve_lock;
 166 static kthread_t                *hblk_reserve_thread;
 167 
 168 static nucleus_hblk8_info_t     nucleus_hblk8;
 169 static nucleus_hblk1_info_t     nucleus_hblk1;
 170 
 171 /*
 172  * Data to manage per-cpu hmeblk pending queues, hmeblks are queued here
 173  * after the initial phase of removing an hmeblk from the hash chain, see
 174  * the detailed comment in sfmmu_hblk_hash_rm() for further details.
 175  */
 176 static cpu_hme_pend_t           *cpu_hme_pend;
 177 static uint_t                   cpu_hme_pend_thresh;
 178 /*
 179  * SFMMU specific hat functions
 180  */
 181 void    hat_pagecachectl(struct page *, int);
 182 
 183 /* flags for hat_pagecachectl */
 184 #define HAT_CACHE       0x1
 185 #define HAT_UNCACHE     0x2
 186 #define HAT_TMPNC       0x4
 187 
 188 /*
 189  * Flag to allow the creation of non-cacheable translations
 190  * to system memory. It is off by default. At the moment this
 191  * flag is used by the ecache error injector. The error injector
 192  * will turn it on when creating such a translation then shut it
 193  * off when it's finished.
 194  */
 195 
 196 int     sfmmu_allow_nc_trans = 0;
 197 
 198 /*
 199  * Flag to disable large page support.
 200  *      value of 1 => disable all large pages.
 201  *      bits 1, 2, and 3 are to disable 64K, 512K and 4M pages respectively.
 202  *
 203  * For example, use the value 0x4 to disable 512K pages.
 204  *
 205  */
 206 #define LARGE_PAGES_OFF         0x1
 207 
 208 /*
 209  * The disable_large_pages and disable_ism_large_pages variables control
 210  * hat_memload_array and the page sizes to be used by ISM and the kernel.
 211  *
 212  * The disable_auto_data_large_pages and disable_auto_text_large_pages variables
 213  * are only used to control which OOB pages to use at upper VM segment creation
 214  * time, and are set in hat_init_pagesizes and used in the map_pgsz* routines.
 215  * Their values may come from platform or CPU specific code to disable page
 216  * sizes that should not be used.
 217  *
 218  * WARNING: 512K pages are currently not supported for ISM/DISM.
 219  */
 220 uint_t  disable_large_pages = 0;
 221 uint_t  disable_ism_large_pages = (1 << TTE512K);
 222 uint_t  disable_auto_data_large_pages = 0;
 223 uint_t  disable_auto_text_large_pages = 0;
 224 
 225 /*
 226  * Private sfmmu data structures for hat management
 227  */
 228 static struct kmem_cache *sfmmuid_cache;
 229 static struct kmem_cache *mmuctxdom_cache;
 230 
 231 /*
 232  * Private sfmmu data structures for tsb management
 233  */
 234 static struct kmem_cache *sfmmu_tsbinfo_cache;
 235 static struct kmem_cache *sfmmu_tsb8k_cache;
 236 static struct kmem_cache *sfmmu_tsb_cache[NLGRPS_MAX];
 237 static vmem_t *kmem_bigtsb_arena;
 238 static vmem_t *kmem_tsb_arena;
 239 
 240 /*
 241  * sfmmu static variables for hmeblk resource management.
 242  */
 243 static vmem_t *hat_memload1_arena; /* HAT translation arena for sfmmu1_cache */
 244 static struct kmem_cache *sfmmu8_cache;
 245 static struct kmem_cache *sfmmu1_cache;
 246 static struct kmem_cache *pa_hment_cache;
 247 
 248 static kmutex_t         ism_mlist_lock; /* mutex for ism mapping list */
 249 /*
 250  * private data for ism
 251  */
 252 static struct kmem_cache *ism_blk_cache;
 253 static struct kmem_cache *ism_ment_cache;
 254 #define ISMID_STARTADDR NULL
 255 
 256 /*
 257  * Region management data structures and function declarations.
 258  */
 259 
 260 static void     sfmmu_leave_srd(sfmmu_t *);
 261 static int      sfmmu_srdcache_constructor(void *, void *, int);
 262 static void     sfmmu_srdcache_destructor(void *, void *);
 263 static int      sfmmu_rgncache_constructor(void *, void *, int);
 264 static void     sfmmu_rgncache_destructor(void *, void *);
 265 static int      sfrgnmap_isnull(sf_region_map_t *);
 266 static int      sfhmergnmap_isnull(sf_hmeregion_map_t *);
 267 static int      sfmmu_scdcache_constructor(void *, void *, int);
 268 static void     sfmmu_scdcache_destructor(void *, void *);
 269 static void     sfmmu_rgn_cb_noop(caddr_t, caddr_t, caddr_t,
 270     size_t, void *, u_offset_t);
 271 
 272 static uint_t srd_hashmask = SFMMU_MAX_SRD_BUCKETS - 1;
 273 static sf_srd_bucket_t *srd_buckets;
 274 static struct kmem_cache *srd_cache;
 275 static uint_t srd_rgn_hashmask = SFMMU_MAX_REGION_BUCKETS - 1;
 276 static struct kmem_cache *region_cache;
 277 static struct kmem_cache *scd_cache;
 278 
 279 #ifdef sun4v
 280 int use_bigtsb_arena = 1;
 281 #else
 282 int use_bigtsb_arena = 0;
 283 #endif
 284 
 285 /* External /etc/system tunable, for turning on&off the shctx support */
 286 int disable_shctx = 0;
 287 /* Internal variable, set by MD if the HW supports shctx feature */
 288 int shctx_on = 0;
 289 
 290 #ifdef DEBUG
 291 static void check_scd_sfmmu_list(sfmmu_t **, sfmmu_t *, int);
 292 #endif
 293 static void sfmmu_to_scd_list(sfmmu_t **, sfmmu_t *);
 294 static void sfmmu_from_scd_list(sfmmu_t **, sfmmu_t *);
 295 
 296 static sf_scd_t *sfmmu_alloc_scd(sf_srd_t *, sf_region_map_t *);
 297 static void sfmmu_find_scd(sfmmu_t *);
 298 static void sfmmu_join_scd(sf_scd_t *, sfmmu_t *);
 299 static void sfmmu_finish_join_scd(sfmmu_t *);
 300 static void sfmmu_leave_scd(sfmmu_t *, uchar_t);
 301 static void sfmmu_destroy_scd(sf_srd_t *, sf_scd_t *, sf_region_map_t *);
 302 static int sfmmu_alloc_scd_tsbs(sf_srd_t *, sf_scd_t *);
 303 static void sfmmu_free_scd_tsbs(sfmmu_t *);
 304 static void sfmmu_tsb_inv_ctx(sfmmu_t *);
 305 static int find_ism_rid(sfmmu_t *, sfmmu_t *, caddr_t, uint_t *);
 306 static void sfmmu_ism_hatflags(sfmmu_t *, int);
 307 static int sfmmu_srd_lock_held(sf_srd_t *);
 308 static void sfmmu_remove_scd(sf_scd_t **, sf_scd_t *);
 309 static void sfmmu_add_scd(sf_scd_t **headp, sf_scd_t *);
 310 static void sfmmu_link_scd_to_regions(sf_srd_t *, sf_scd_t *);
 311 static void sfmmu_unlink_scd_from_regions(sf_srd_t *, sf_scd_t *);
 312 static void sfmmu_link_to_hmeregion(sfmmu_t *, sf_region_t *);
 313 static void sfmmu_unlink_from_hmeregion(sfmmu_t *, sf_region_t *);
 314 
 315 /*
 316  * ``hat_lock'' is a hashed mutex lock for protecting sfmmu TSB lists,
 317  * HAT flags, synchronizing TLB/TSB coherency, and context management.
 318  * The lock is hashed on the sfmmup since the case where we need to lock
 319  * all processes is rare but does occur (e.g. we need to unload a shared
 320  * mapping from all processes using the mapping).  We have a lot of buckets,
 321  * and each slab of sfmmu_t's can use about a quarter of them, giving us
 322  * a fairly good distribution without wasting too much space and overhead
 323  * when we have to grab them all.
 324  */
 325 #define SFMMU_NUM_LOCK  128             /* must be power of two */
 326 hatlock_t       hat_lock[SFMMU_NUM_LOCK];
 327 
 328 /*
 329  * Hash algorithm optimized for a small number of slabs.
 330  *  7 is (highbit((sizeof sfmmu_t)) - 1)
 331  * This hash algorithm is based upon the knowledge that sfmmu_t's come from a
 332  * kmem_cache, and thus they will be sequential within that cache.  In
 333  * addition, each new slab will have a different "color" up to cache_maxcolor
 334  * which will skew the hashing for each successive slab which is allocated.
 335  * If the size of sfmmu_t changed to a larger size, this algorithm may need
 336  * to be revisited.
 337  */
 338 #define TSB_HASH_SHIFT_BITS (7)
 339 #define PTR_HASH(x) ((uintptr_t)x >> TSB_HASH_SHIFT_BITS)
 340 
 341 #ifdef DEBUG
 342 int tsb_hash_debug = 0;
 343 #define TSB_HASH(sfmmup)        \
 344         (tsb_hash_debug ? &hat_lock[0] : \
 345         &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)])
 346 #else   /* DEBUG */
 347 #define TSB_HASH(sfmmup)        &hat_lock[PTR_HASH(sfmmup) & (SFMMU_NUM_LOCK-1)]
 348 #endif  /* DEBUG */
 349 
 350 
 351 /* sfmmu_replace_tsb() return codes. */
 352 typedef enum tsb_replace_rc {
 353         TSB_SUCCESS,
 354         TSB_ALLOCFAIL,
 355         TSB_LOSTRACE,
 356         TSB_ALREADY_SWAPPED,
 357         TSB_CANTGROW
 358 } tsb_replace_rc_t;
 359 
 360 /*
 361  * Flags for TSB allocation routines.
 362  */
 363 #define TSB_ALLOC       0x01
 364 #define TSB_FORCEALLOC  0x02
 365 #define TSB_GROW        0x04
 366 #define TSB_SHRINK      0x08
 367 #define TSB_SWAPIN      0x10
 368 
 369 /*
 370  * Support for HAT callbacks.
 371  */
 372 #define SFMMU_MAX_RELOC_CALLBACKS       10
 373 int sfmmu_max_cb_id = SFMMU_MAX_RELOC_CALLBACKS;
 374 static id_t sfmmu_cb_nextid = 0;
 375 static id_t sfmmu_tsb_cb_id;
 376 struct sfmmu_callback *sfmmu_cb_table;
 377 
 378 kmutex_t        kpr_mutex;
 379 kmutex_t        kpr_suspendlock;
 380 kthread_t       *kreloc_thread;
 381 
 382 /*
 383  * Enable VA->PA translation sanity checking on DEBUG kernels.
 384  * Disabled by default.  This is incompatible with some
 385  * drivers (error injector, RSM) so if it breaks you get
 386  * to keep both pieces.
 387  */
 388 int hat_check_vtop = 0;
 389 
 390 /*
 391  * Private sfmmu routines (prototypes)
 392  */
 393 static struct hme_blk *sfmmu_shadow_hcreate(sfmmu_t *, caddr_t, int, uint_t);
 394 static struct   hme_blk *sfmmu_hblk_alloc(sfmmu_t *, caddr_t,
 395                         struct hmehash_bucket *, uint_t, hmeblk_tag, uint_t,
 396                         uint_t);
 397 static caddr_t  sfmmu_hblk_unload(struct hat *, struct hme_blk *, caddr_t,
 398                         caddr_t, demap_range_t *, uint_t);
 399 static caddr_t  sfmmu_hblk_sync(struct hat *, struct hme_blk *, caddr_t,
 400                         caddr_t, int);
 401 static void     sfmmu_hblk_free(struct hme_blk **);
 402 static void     sfmmu_hblks_list_purge(struct hme_blk **, int);
 403 static uint_t   sfmmu_get_free_hblk(struct hme_blk **, uint_t);
 404 static uint_t   sfmmu_put_free_hblk(struct hme_blk *, uint_t);
 405 static struct hme_blk *sfmmu_hblk_steal(int);
 406 static int      sfmmu_steal_this_hblk(struct hmehash_bucket *,
 407                         struct hme_blk *, uint64_t, struct hme_blk *);
 408 static caddr_t  sfmmu_hblk_unlock(struct hme_blk *, caddr_t, caddr_t);
 409 
 410 static void     hat_do_memload_array(struct hat *, caddr_t, size_t,
 411                     struct page **, uint_t, uint_t, uint_t);
 412 static void     hat_do_memload(struct hat *, caddr_t, struct page *,
 413                     uint_t, uint_t, uint_t);
 414 static void     sfmmu_memload_batchsmall(struct hat *, caddr_t, page_t **,
 415                     uint_t, uint_t, pgcnt_t, uint_t);
 416 void            sfmmu_tteload(struct hat *, tte_t *, caddr_t, page_t *,
 417                         uint_t);
 418 static int      sfmmu_tteload_array(sfmmu_t *, tte_t *, caddr_t, page_t **,
 419                         uint_t, uint_t);
 420 static struct hmehash_bucket *sfmmu_tteload_acquire_hashbucket(sfmmu_t *,
 421                                         caddr_t, int, uint_t);
 422 static struct hme_blk *sfmmu_tteload_find_hmeblk(sfmmu_t *,
 423                         struct hmehash_bucket *, caddr_t, uint_t, uint_t,
 424                         uint_t);
 425 static int      sfmmu_tteload_addentry(sfmmu_t *, struct hme_blk *, tte_t *,
 426                         caddr_t, page_t **, uint_t, uint_t);
 427 static void     sfmmu_tteload_release_hashbucket(struct hmehash_bucket *);
 428 
 429 static int      sfmmu_pagearray_setup(caddr_t, page_t **, tte_t *, int);
 430 static pfn_t    sfmmu_uvatopfn(caddr_t, sfmmu_t *, tte_t *);
 431 void            sfmmu_memtte(tte_t *, pfn_t, uint_t, int);
 432 #ifdef VAC
 433 static void     sfmmu_vac_conflict(struct hat *, caddr_t, page_t *);
 434 static int      sfmmu_vacconflict_array(caddr_t, page_t *, int *);
 435 int     tst_tnc(page_t *pp, pgcnt_t);
 436 void    conv_tnc(page_t *pp, int);
 437 #endif
 438 
 439 static void     sfmmu_get_ctx(sfmmu_t *);
 440 static void     sfmmu_free_sfmmu(sfmmu_t *);
 441 
 442 static void     sfmmu_ttesync(struct hat *, caddr_t, tte_t *, page_t *);
 443 static void     sfmmu_chgattr(struct hat *, caddr_t, size_t, uint_t, int);
 444 
 445 cpuset_t        sfmmu_pageunload(page_t *, struct sf_hment *, int);
 446 static void     hat_pagereload(struct page *, struct page *);
 447 static cpuset_t sfmmu_pagesync(page_t *, struct sf_hment *, uint_t);
 448 #ifdef VAC
 449 void    sfmmu_page_cache_array(page_t *, int, int, pgcnt_t);
 450 static void     sfmmu_page_cache(page_t *, int, int, int);
 451 #endif
 452 
 453 cpuset_t        sfmmu_rgntlb_demap(caddr_t, sf_region_t *,
 454     struct hme_blk *, int);
 455 static void     sfmmu_tlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *,
 456                         pfn_t, int, int, int, int);
 457 static void     sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *,
 458                         pfn_t, int);
 459 static void     sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int);
 460 static void     sfmmu_tlb_range_demap(demap_range_t *);
 461 static void     sfmmu_invalidate_ctx(sfmmu_t *);
 462 static void     sfmmu_sync_mmustate(sfmmu_t *);
 463 
 464 static void     sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t);
 465 static int      sfmmu_tsbinfo_alloc(struct tsb_info **, int, int, uint_t,
 466                         sfmmu_t *);
 467 static void     sfmmu_tsb_free(struct tsb_info *);
 468 static void     sfmmu_tsbinfo_free(struct tsb_info *);
 469 static int      sfmmu_init_tsbinfo(struct tsb_info *, int, int, uint_t,
 470                         sfmmu_t *);
 471 static void     sfmmu_tsb_chk_reloc(sfmmu_t *, hatlock_t *);
 472 static void     sfmmu_tsb_swapin(sfmmu_t *, hatlock_t *);
 473 static int      sfmmu_select_tsb_szc(pgcnt_t);
 474 static void     sfmmu_mod_tsb(sfmmu_t *, caddr_t, tte_t *, int);
 475 #define         sfmmu_load_tsb(sfmmup, vaddr, tte, szc) \
 476         sfmmu_mod_tsb(sfmmup, vaddr, tte, szc)
 477 #define         sfmmu_unload_tsb(sfmmup, vaddr, szc)    \
 478         sfmmu_mod_tsb(sfmmup, vaddr, NULL, szc)
 479 static void     sfmmu_copy_tsb(struct tsb_info *, struct tsb_info *);
 480 static tsb_replace_rc_t sfmmu_replace_tsb(sfmmu_t *, struct tsb_info *, uint_t,
 481     hatlock_t *, uint_t);
 482 static void     sfmmu_size_tsb(sfmmu_t *, int, uint64_t, uint64_t, int);
 483 
 484 #ifdef VAC
 485 void    sfmmu_cache_flush(pfn_t, int);
 486 void    sfmmu_cache_flushcolor(int, pfn_t);
 487 #endif
 488 static caddr_t  sfmmu_hblk_chgattr(sfmmu_t *, struct hme_blk *, caddr_t,
 489                         caddr_t, demap_range_t *, uint_t, int);
 490 
 491 static uint64_t sfmmu_vtop_attr(uint_t, int mode, tte_t *);
 492 static uint_t   sfmmu_ptov_attr(tte_t *);
 493 static caddr_t  sfmmu_hblk_chgprot(sfmmu_t *, struct hme_blk *, caddr_t,
 494                         caddr_t, demap_range_t *, uint_t);
 495 static uint_t   sfmmu_vtop_prot(uint_t, uint_t *);
 496 static int      sfmmu_idcache_constructor(void *, void *, int);
 497 static void     sfmmu_idcache_destructor(void *, void *);
 498 static int      sfmmu_hblkcache_constructor(void *, void *, int);
 499 static void     sfmmu_hblkcache_destructor(void *, void *);
 500 static void     sfmmu_hblkcache_reclaim(void *);
 501 static void     sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *,
 502                         struct hmehash_bucket *);
 503 static void     sfmmu_hblk_hash_rm(struct hmehash_bucket *, struct hme_blk *,
 504                         struct hme_blk *, struct hme_blk **, int);
 505 static void     sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *,
 506                         uint64_t);
 507 static struct hme_blk *sfmmu_check_pending_hblks(int);
 508 static void     sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int);
 509 static void     sfmmu_cleanup_rhblk(sf_srd_t *, caddr_t, uint_t, int);
 510 static void     sfmmu_unload_hmeregion_va(sf_srd_t *, uint_t, caddr_t, caddr_t,
 511                         int, caddr_t *);
 512 static void     sfmmu_unload_hmeregion(sf_srd_t *, sf_region_t *);
 513 
 514 static void     sfmmu_rm_large_mappings(page_t *, int);
 515 
 516 static void     hat_lock_init(void);
 517 static void     hat_kstat_init(void);
 518 static int      sfmmu_kstat_percpu_update(kstat_t *ksp, int rw);
 519 static void     sfmmu_set_scd_rttecnt(sf_srd_t *, sf_scd_t *);
 520 static  int     sfmmu_is_rgnva(sf_srd_t *, caddr_t, ulong_t, ulong_t);
 521 static void     sfmmu_check_page_sizes(sfmmu_t *, int);
 522 int     fnd_mapping_sz(page_t *);
 523 static void     iment_add(struct ism_ment *,  struct hat *);
 524 static void     iment_sub(struct ism_ment *, struct hat *);
 525 static pgcnt_t  ism_tsb_entries(sfmmu_t *, int szc);
 526 extern void     sfmmu_setup_tsbinfo(sfmmu_t *);
 527 extern void     sfmmu_clear_utsbinfo(void);
 528 
 529 static void             sfmmu_ctx_wrap_around(mmu_ctx_t *, boolean_t);
 530 
 531 extern int vpm_enable;
 532 
 533 /* kpm globals */
 534 #ifdef  DEBUG
 535 /*
 536  * Enable trap level tsbmiss handling
 537  */
 538 int     kpm_tsbmtl = 1;
 539 
 540 /*
 541  * Flush the TLB on kpm mapout. Note: Xcalls are used (again) for the
 542  * required TLB shootdowns in this case, so handle w/ care. Off by default.
 543  */
 544 int     kpm_tlb_flush;
 545 #endif  /* DEBUG */
 546 
 547 static void     *sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *, size_t, int);
 548 
 549 #ifdef DEBUG
 550 static void     sfmmu_check_hblk_flist();
 551 #endif
 552 
 553 /*
 554  * Semi-private sfmmu data structures.  Some of them are initialize in
 555  * startup or in hat_init. Some of them are private but accessed by
 556  * assembly code or mach_sfmmu.c
 557  */
 558 struct hmehash_bucket *uhme_hash;       /* user hmeblk hash table */
 559 struct hmehash_bucket *khme_hash;       /* kernel hmeblk hash table */
 560 uint64_t        uhme_hash_pa;           /* PA of uhme_hash */
 561 uint64_t        khme_hash_pa;           /* PA of khme_hash */
 562 int             uhmehash_num;           /* # of buckets in user hash table */
 563 int             khmehash_num;           /* # of buckets in kernel hash table */
 564 
 565 uint_t          max_mmu_ctxdoms = 0;    /* max context domains in the system */
 566 mmu_ctx_t       **mmu_ctxs_tbl;         /* global array of context domains */
 567 uint64_t        mmu_saved_gnum = 0;     /* to init incoming MMUs' gnums */
 568 
 569 #define DEFAULT_NUM_CTXS_PER_MMU 8192
 570 static uint_t   nctxs = DEFAULT_NUM_CTXS_PER_MMU;
 571 
 572 int             cache;                  /* describes system cache */
 573 
 574 caddr_t         ktsb_base;              /* kernel 8k-indexed tsb base address */
 575 uint64_t        ktsb_pbase;             /* kernel 8k-indexed tsb phys address */
 576 int             ktsb_szcode;            /* kernel 8k-indexed tsb size code */
 577 int             ktsb_sz;                /* kernel 8k-indexed tsb size */
 578 
 579 caddr_t         ktsb4m_base;            /* kernel 4m-indexed tsb base address */
 580 uint64_t        ktsb4m_pbase;           /* kernel 4m-indexed tsb phys address */
 581 int             ktsb4m_szcode;          /* kernel 4m-indexed tsb size code */
 582 int             ktsb4m_sz;              /* kernel 4m-indexed tsb size */
 583 
 584 uint64_t        kpm_tsbbase;            /* kernel seg_kpm 4M TSB base address */
 585 int             kpm_tsbsz;              /* kernel seg_kpm 4M TSB size code */
 586 uint64_t        kpmsm_tsbbase;          /* kernel seg_kpm 8K TSB base address */
 587 int             kpmsm_tsbsz;            /* kernel seg_kpm 8K TSB size code */
 588 
 589 #ifndef sun4v
 590 int             utsb_dtlb_ttenum = -1;  /* index in TLB for utsb locked TTE */
 591 int             utsb4m_dtlb_ttenum = -1; /* index in TLB for 4M TSB TTE */
 592 int             dtlb_resv_ttenum;       /* index in TLB of first reserved TTE */
 593 caddr_t         utsb_vabase;            /* reserved kernel virtual memory */
 594 caddr_t         utsb4m_vabase;          /* for trap handler TSB accesses */
 595 #endif /* sun4v */
 596 uint64_t        tsb_alloc_bytes = 0;    /* bytes allocated to TSBs */
 597 vmem_t          *kmem_tsb_default_arena[NLGRPS_MAX];    /* For dynamic TSBs */
 598 vmem_t          *kmem_bigtsb_default_arena[NLGRPS_MAX]; /* dynamic 256M TSBs */
 599 
 600 /*
 601  * Size to use for TSB slabs.  Future platforms that support page sizes
 602  * larger than 4M may wish to change these values, and provide their own
 603  * assembly macros for building and decoding the TSB base register contents.
 604  * Note disable_large_pages will override the value set here.
 605  */
 606 static  uint_t tsb_slab_ttesz = TTE4M;
 607 size_t  tsb_slab_size = MMU_PAGESIZE4M;
 608 uint_t  tsb_slab_shift = MMU_PAGESHIFT4M;
 609 /* PFN mask for TTE */
 610 size_t  tsb_slab_mask = MMU_PAGEOFFSET4M >> MMU_PAGESHIFT;
 611 
 612 /*
 613  * Size to use for TSB slabs.  These are used only when 256M tsb arenas
 614  * exist.
 615  */
 616 static uint_t   bigtsb_slab_ttesz = TTE256M;
 617 static size_t   bigtsb_slab_size = MMU_PAGESIZE256M;
 618 static uint_t   bigtsb_slab_shift = MMU_PAGESHIFT256M;
 619 /* 256M page alignment for 8K pfn */
 620 static size_t   bigtsb_slab_mask = MMU_PAGEOFFSET256M >> MMU_PAGESHIFT;
 621 
 622 /* largest TSB size to grow to, will be smaller on smaller memory systems */
 623 static int      tsb_max_growsize = 0;
 624 
 625 /*
 626  * Tunable parameters dealing with TSB policies.
 627  */
 628 
 629 /*
 630  * This undocumented tunable forces all 8K TSBs to be allocated from
 631  * the kernel heap rather than from the kmem_tsb_default_arena arenas.
 632  */
 633 #ifdef  DEBUG
 634 int     tsb_forceheap = 0;
 635 #endif  /* DEBUG */
 636 
 637 /*
 638  * Decide whether to use per-lgroup arenas, or one global set of
 639  * TSB arenas.  The default is not to break up per-lgroup, since
 640  * most platforms don't recognize any tangible benefit from it.
 641  */
 642 int     tsb_lgrp_affinity = 0;
 643 
 644 /*
 645  * Used for growing the TSB based on the process RSS.
 646  * tsb_rss_factor is based on the smallest TSB, and is
 647  * shifted by the TSB size to determine if we need to grow.
 648  * The default will grow the TSB if the number of TTEs for
 649  * this page size exceeds 75% of the number of TSB entries,
 650  * which should _almost_ eliminate all conflict misses
 651  * (at the expense of using up lots and lots of memory).
 652  */
 653 #define TSB_RSS_FACTOR          (TSB_ENTRIES(TSB_MIN_SZCODE) * 0.75)
 654 #define SFMMU_RSS_TSBSIZE(tsbszc)       (tsb_rss_factor << tsbszc)
 655 #define SELECT_TSB_SIZECODE(pgcnt) ( \
 656         (enable_tsb_rss_sizing)? sfmmu_select_tsb_szc(pgcnt) : \
 657         default_tsb_size)
 658 #define TSB_OK_SHRINK() \
 659         (tsb_alloc_bytes > tsb_alloc_hiwater || freemem < desfree)
 660 #define TSB_OK_GROW()   \
 661         (tsb_alloc_bytes < tsb_alloc_hiwater && freemem > desfree)
 662 
 663 int     enable_tsb_rss_sizing = 1;
 664 int     tsb_rss_factor  = (int)TSB_RSS_FACTOR;
 665 
 666 /* which TSB size code to use for new address spaces or if rss sizing off */
 667 int default_tsb_size = TSB_8K_SZCODE;
 668 
 669 static uint64_t tsb_alloc_hiwater; /* limit TSB reserved memory */
 670 uint64_t tsb_alloc_hiwater_factor; /* tsb_alloc_hiwater = physmem / this */
 671 #define TSB_ALLOC_HIWATER_FACTOR_DEFAULT        32
 672 
 673 #ifdef DEBUG
 674 static int tsb_random_size = 0; /* set to 1 to test random tsb sizes on alloc */
 675 static int tsb_grow_stress = 0; /* if set to 1, keep replacing TSB w/ random */
 676 static int tsb_alloc_mtbf = 0;  /* fail allocation every n attempts */
 677 static int tsb_alloc_fail_mtbf = 0;
 678 static int tsb_alloc_count = 0;
 679 #endif /* DEBUG */
 680 
 681 /* if set to 1, will remap valid TTEs when growing TSB. */
 682 int tsb_remap_ttes = 1;
 683 
 684 /*
 685  * If we have more than this many mappings, allocate a second TSB.
 686  * This default is chosen because the I/D fully associative TLBs are
 687  * assumed to have at least 8 available entries. Platforms with a
 688  * larger fully-associative TLB could probably override the default.
 689  */
 690 
 691 #ifdef sun4v
 692 int tsb_sectsb_threshold = 0;
 693 #else
 694 int tsb_sectsb_threshold = 8;
 695 #endif
 696 
 697 /*
 698  * kstat data
 699  */
 700 struct sfmmu_global_stat sfmmu_global_stat;
 701 struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat;
 702 
 703 /*
 704  * Global data
 705  */
 706 sfmmu_t         *ksfmmup;               /* kernel's hat id */
 707 
 708 #ifdef DEBUG
 709 static void     chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *);
 710 #endif
 711 
 712 /* sfmmu locking operations */
 713 static kmutex_t *sfmmu_mlspl_enter(struct page *, int);
 714 static int      sfmmu_mlspl_held(struct page *, int);
 715 
 716 kmutex_t *sfmmu_page_enter(page_t *);
 717 void    sfmmu_page_exit(kmutex_t *);
 718 int     sfmmu_page_spl_held(struct page *);
 719 
 720 /* sfmmu internal locking operations - accessed directly */
 721 static void     sfmmu_mlist_reloc_enter(page_t *, page_t *,
 722                                 kmutex_t **, kmutex_t **);
 723 static void     sfmmu_mlist_reloc_exit(kmutex_t *, kmutex_t *);
 724 static hatlock_t *
 725                 sfmmu_hat_enter(sfmmu_t *);
 726 static hatlock_t *
 727                 sfmmu_hat_tryenter(sfmmu_t *);
 728 static void     sfmmu_hat_exit(hatlock_t *);
 729 static void     sfmmu_hat_lock_all(void);
 730 static void     sfmmu_hat_unlock_all(void);
 731 static void     sfmmu_ismhat_enter(sfmmu_t *, int);
 732 static void     sfmmu_ismhat_exit(sfmmu_t *, int);
 733 
 734 kpm_hlk_t       *kpmp_table;
 735 uint_t          kpmp_table_sz;  /* must be a power of 2 */
 736 uchar_t         kpmp_shift;
 737 
 738 kpm_shlk_t      *kpmp_stable;
 739 uint_t          kpmp_stable_sz; /* must be a power of 2 */
 740 
 741 /*
 742  * SPL_TABLE_SIZE is 2 * NCPU, but no smaller than 128.
 743  * SPL_SHIFT is log2(SPL_TABLE_SIZE).
 744  */
 745 #if ((2*NCPU_P2) > 128)
 746 #define SPL_SHIFT       ((unsigned)(NCPU_LOG2 + 1))
 747 #else
 748 #define SPL_SHIFT       7U
 749 #endif
 750 #define SPL_TABLE_SIZE  (1U << SPL_SHIFT)
 751 #define SPL_MASK        (SPL_TABLE_SIZE - 1)
 752 
 753 /*
 754  * We shift by PP_SHIFT to take care of the low-order 0 bits of a page_t
 755  * and by multiples of SPL_SHIFT to get as many varied bits as we can.
 756  */
 757 #define SPL_INDEX(pp) \
 758         ((((uintptr_t)(pp) >> PP_SHIFT) ^ \
 759         ((uintptr_t)(pp) >> (PP_SHIFT + SPL_SHIFT)) ^ \
 760         ((uintptr_t)(pp) >> (PP_SHIFT + SPL_SHIFT * 2)) ^ \
 761         ((uintptr_t)(pp) >> (PP_SHIFT + SPL_SHIFT * 3))) & \
 762         SPL_MASK)
 763 
 764 #define SPL_HASH(pp)    \
 765         (&sfmmu_page_lock[SPL_INDEX(pp)].pad_mutex)
 766 
 767 static  pad_mutex_t     sfmmu_page_lock[SPL_TABLE_SIZE];
 768 
 769 /* Array of mutexes protecting a page's mapping list and p_nrm field. */
 770 
 771 #define MML_TABLE_SIZE  SPL_TABLE_SIZE
 772 #define MLIST_HASH(pp)  (&mml_table[SPL_INDEX(pp)].pad_mutex)
 773 
 774 static pad_mutex_t      mml_table[MML_TABLE_SIZE];
 775 
 776 /*
 777  * hat_unload_callback() will group together callbacks in order
 778  * to avoid xt_sync() calls.  This is the maximum size of the group.
 779  */
 780 #define MAX_CB_ADDR     32
 781 
 782 tte_t   hw_tte;
 783 static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT;
 784 
 785 static char     *mmu_ctx_kstat_names[] = {
 786         "mmu_ctx_tsb_exceptions",
 787         "mmu_ctx_tsb_raise_exception",
 788         "mmu_ctx_wrap_around",
 789 };
 790 
 791 /*
 792  * Wrapper for vmem_xalloc since vmem_create only allows limited
 793  * parameters for vm_source_alloc functions.  This function allows us
 794  * to specify alignment consistent with the size of the object being
 795  * allocated.
 796  */
 797 static void *
 798 sfmmu_vmem_xalloc_aligned_wrapper(vmem_t *vmp, size_t size, int vmflag)
 799 {
 800         return (vmem_xalloc(vmp, size, size, 0, 0, NULL, NULL, vmflag));
 801 }
 802 
 803 /* Common code for setting tsb_alloc_hiwater. */
 804 #define SFMMU_SET_TSB_ALLOC_HIWATER(pages)      tsb_alloc_hiwater = \
 805                 ptob(pages) / tsb_alloc_hiwater_factor
 806 
 807 /*
 808  * Set tsb_max_growsize to allow at most all of physical memory to be mapped by
 809  * a single TSB.  physmem is the number of physical pages so we need physmem 8K
 810  * TTEs to represent all those physical pages.  We round this up by using
 811  * 1<<highbit().  To figure out which size code to use, remember that the size
 812  * code is just an amount to shift the smallest TSB size to get the size of
 813  * this TSB.  So we subtract that size, TSB_START_SIZE, from highbit() (or
 814  * highbit() - 1) to get the size code for the smallest TSB that can represent
 815  * all of physical memory, while erring on the side of too much.
 816  *
 817  * Restrict tsb_max_growsize to make sure that:
 818  *      1) TSBs can't grow larger than the TSB slab size
 819  *      2) TSBs can't grow larger than UTSB_MAX_SZCODE.
 820  */
 821 #define SFMMU_SET_TSB_MAX_GROWSIZE(pages) {                             \
 822         int     _i, _szc, _slabszc, _tsbszc;                            \
 823                                                                         \
 824         _i = highbit(pages);                                            \
 825         if ((1 << (_i - 1)) == (pages))                                   \
 826                 _i--;           /* 2^n case, round down */              \
 827         _szc = _i - TSB_START_SIZE;                                     \
 828         _slabszc = bigtsb_slab_shift - (TSB_START_SIZE + TSB_ENTRY_SHIFT); \
 829         _tsbszc = MIN(_szc, _slabszc);                                  \
 830         tsb_max_growsize = MIN(_tsbszc, UTSB_MAX_SZCODE);               \
 831 }
 832 
 833 /*
 834  * Given a pointer to an sfmmu and a TTE size code, return a pointer to the
 835  * tsb_info which handles that TTE size.
 836  */
 837 #define SFMMU_GET_TSBINFO(tsbinfop, sfmmup, tte_szc) {                  \
 838         (tsbinfop) = (sfmmup)->sfmmu_tsb;                            \
 839         ASSERT(((tsbinfop)->tsb_flags & TSB_SHAREDCTX) ||                \
 840             sfmmu_hat_lock_held(sfmmup));                               \
 841         if ((tte_szc) >= TTE4M)      {                                       \
 842                 ASSERT((tsbinfop) != NULL);                             \
 843                 (tsbinfop) = (tsbinfop)->tsb_next;                   \
 844         }                                                               \
 845 }
 846 
 847 /*
 848  * Macro to use to unload entries from the TSB.
 849  * It has knowledge of which page sizes get replicated in the TSB
 850  * and will call the appropriate unload routine for the appropriate size.
 851  */
 852 #define SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, ismhat)         \
 853 {                                                                       \
 854         int ttesz = get_hblk_ttesz(hmeblkp);                            \
 855         if (ttesz == TTE8K || ttesz == TTE4M) {                         \
 856                 sfmmu_unload_tsb(sfmmup, addr, ttesz);                  \
 857         } else {                                                        \
 858                 caddr_t sva = ismhat ? addr :                           \
 859                     (caddr_t)get_hblk_base(hmeblkp);                    \
 860                 caddr_t eva = sva + get_hblk_span(hmeblkp);             \
 861                 ASSERT(addr >= sva && addr < eva);                        \
 862                 sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz);        \
 863         }                                                               \
 864 }
 865 
 866 
 867 /* Update tsb_alloc_hiwater after memory is configured. */
 868 /*ARGSUSED*/
 869 static void
 870 sfmmu_update_post_add(void *arg, pgcnt_t delta_pages)
 871 {
 872         /* Assumes physmem has already been updated. */
 873         SFMMU_SET_TSB_ALLOC_HIWATER(physmem);
 874         SFMMU_SET_TSB_MAX_GROWSIZE(physmem);
 875 }
 876 
 877 /*
 878  * Update tsb_alloc_hiwater before memory is deleted.  We'll do nothing here
 879  * and update tsb_alloc_hiwater and tsb_max_growsize after the memory is
 880  * deleted.
 881  */
 882 /*ARGSUSED*/
 883 static int
 884 sfmmu_update_pre_del(void *arg, pgcnt_t delta_pages)
 885 {
 886         return (0);
 887 }
 888 
 889 /* Update tsb_alloc_hiwater after memory fails to be unconfigured. */
 890 /*ARGSUSED*/
 891 static void
 892 sfmmu_update_post_del(void *arg, pgcnt_t delta_pages, int cancelled)
 893 {
 894         /*
 895          * Whether the delete was cancelled or not, just go ahead and update
 896          * tsb_alloc_hiwater and tsb_max_growsize.
 897          */
 898         SFMMU_SET_TSB_ALLOC_HIWATER(physmem);
 899         SFMMU_SET_TSB_MAX_GROWSIZE(physmem);
 900 }
 901 
 902 static kphysm_setup_vector_t sfmmu_update_vec = {
 903         KPHYSM_SETUP_VECTOR_VERSION,    /* version */
 904         sfmmu_update_post_add,          /* post_add */
 905         sfmmu_update_pre_del,           /* pre_del */
 906         sfmmu_update_post_del           /* post_del */
 907 };
 908 
 909 
 910 /*
 911  * HME_BLK HASH PRIMITIVES
 912  */
 913 
 914 /*
 915  * Enter a hme on the mapping list for page pp.
 916  * When large pages are more prevalent in the system we might want to
 917  * keep the mapping list in ascending order by the hment size. For now,
 918  * small pages are more frequent, so don't slow it down.
 919  */
 920 #define HME_ADD(hme, pp)                                        \
 921 {                                                               \
 922         ASSERT(sfmmu_mlist_held(pp));                           \
 923                                                                 \
 924         hme->hme_prev = NULL;                                        \
 925         hme->hme_next = pp->p_mapping;                            \
 926         hme->hme_page = pp;                                  \
 927         if (pp->p_mapping) {                                 \
 928                 ((struct sf_hment *)(pp->p_mapping))->hme_prev = hme;\
 929                 ASSERT(pp->p_share > 0);                  \
 930         } else  {                                               \
 931                 /* EMPTY */                                     \
 932                 ASSERT(pp->p_share == 0);                    \
 933         }                                                       \
 934         pp->p_mapping = hme;                                 \
 935         pp->p_share++;                                               \
 936 }
 937 
 938 /*
 939  * Enter a hme on the mapping list for page pp.
 940  * If we are unmapping a large translation, we need to make sure that the
 941  * change is reflect in the corresponding bit of the p_index field.
 942  */
 943 #define HME_SUB(hme, pp)                                        \
 944 {                                                               \
 945         ASSERT(sfmmu_mlist_held(pp));                           \
 946         ASSERT(hme->hme_page == pp || IS_PAHME(hme));                \
 947                                                                 \
 948         if (pp->p_mapping == NULL) {                         \
 949                 panic("hme_remove - no mappings");              \
 950         }                                                       \
 951                                                                 \
 952         membar_stst();  /* ensure previous stores finish */     \
 953                                                                 \
 954         ASSERT(pp->p_share > 0);                          \
 955         pp->p_share--;                                               \
 956                                                                 \
 957         if (hme->hme_prev) {                                 \
 958                 ASSERT(pp->p_mapping != hme);                        \
 959                 ASSERT(hme->hme_prev->hme_page == pp ||           \
 960                         IS_PAHME(hme->hme_prev));            \
 961                 hme->hme_prev->hme_next = hme->hme_next;       \
 962         } else {                                                \
 963                 ASSERT(pp->p_mapping == hme);                        \
 964                 pp->p_mapping = hme->hme_next;                    \
 965                 ASSERT((pp->p_mapping == NULL) ?             \
 966                         (pp->p_share == 0) : 1);             \
 967         }                                                       \
 968                                                                 \
 969         if (hme->hme_next) {                                 \
 970                 ASSERT(hme->hme_next->hme_page == pp ||           \
 971                         IS_PAHME(hme->hme_next));            \
 972                 hme->hme_next->hme_prev = hme->hme_prev;       \
 973         }                                                       \
 974                                                                 \
 975         /* zero out the entry */                                \
 976         hme->hme_next = NULL;                                        \
 977         hme->hme_prev = NULL;                                        \
 978         hme->hme_page = NULL;                                        \
 979                                                                 \
 980         if (hme_size(hme) > TTE8K) {                         \
 981                 /* remove mappings for remainder of large pg */ \
 982                 sfmmu_rm_large_mappings(pp, hme_size(hme));     \
 983         }                                                       \
 984 }
 985 
 986 /*
 987  * This function returns the hment given the hme_blk and a vaddr.
 988  * It assumes addr has already been checked to belong to hme_blk's
 989  * range.
 990  */
 991 #define HBLKTOHME(hment, hmeblkp, addr)                                 \
 992 {                                                                       \
 993         int index;                                                      \
 994         HBLKTOHME_IDX(hment, hmeblkp, addr, index)                      \
 995 }
 996 
 997 /*
 998  * Version of HBLKTOHME that also returns the index in hmeblkp
 999  * of the hment.
1000  */
1001 #define HBLKTOHME_IDX(hment, hmeblkp, addr, idx)                        \
1002 {                                                                       \
1003         ASSERT(in_hblk_range((hmeblkp), (addr)));                       \
1004                                                                         \
1005         if (get_hblk_ttesz(hmeblkp) == TTE8K) {                         \
1006                 idx = (((uintptr_t)(addr) >> MMU_PAGESHIFT) & (NHMENTS-1)); \
1007         } else                                                          \
1008                 idx = 0;                                                \
1009                                                                         \
1010         (hment) = &(hmeblkp)->hblk_hme[idx];                             \
1011 }
1012 
1013 /*
1014  * Disable any page sizes not supported by the CPU
1015  */
1016 void
1017 hat_init_pagesizes()
1018 {
1019         int             i;
1020 
1021         mmu_exported_page_sizes = 0;
1022         for (i = TTE8K; i < max_mmu_page_sizes; i++) {
1023 
1024                 szc_2_userszc[i] = (uint_t)-1;
1025                 userszc_2_szc[i] = (uint_t)-1;
1026 
1027                 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) {
1028                         disable_large_pages |= (1 << i);
1029                 } else {
1030                         szc_2_userszc[i] = mmu_exported_page_sizes;
1031                         userszc_2_szc[mmu_exported_page_sizes] = i;
1032                         mmu_exported_page_sizes++;
1033                 }
1034         }
1035 
1036         disable_ism_large_pages |= disable_large_pages;
1037         disable_auto_data_large_pages = disable_large_pages;
1038         disable_auto_text_large_pages = disable_large_pages;
1039 
1040         /*
1041          * Initialize mmu-specific large page sizes.
1042          */
1043         if (&mmu_large_pages_disabled) {
1044                 disable_large_pages |= mmu_large_pages_disabled(HAT_LOAD);
1045                 disable_ism_large_pages |=
1046                     mmu_large_pages_disabled(HAT_LOAD_SHARE);
1047                 disable_auto_data_large_pages |=
1048                     mmu_large_pages_disabled(HAT_AUTO_DATA);
1049                 disable_auto_text_large_pages |=
1050                     mmu_large_pages_disabled(HAT_AUTO_TEXT);
1051         }
1052 }
1053 
1054 /*
1055  * Initialize the hardware address translation structures.
1056  */
1057 void
1058 hat_init(void)
1059 {
1060         int             i;
1061         uint_t          sz;
1062         size_t          size;
1063 
1064         hat_lock_init();
1065         hat_kstat_init();
1066 
1067         /*
1068          * Hardware-only bits in a TTE
1069          */
1070         MAKE_TTE_MASK(&hw_tte);
1071 
1072         hat_init_pagesizes();
1073 
1074         /* Initialize the hash locks */
1075         for (i = 0; i < khmehash_num; i++) {
1076                 mutex_init(&khme_hash[i].hmehash_mutex, NULL,
1077                     MUTEX_DEFAULT, NULL);
1078                 khme_hash[i].hmeh_nextpa = HMEBLK_ENDPA;
1079         }
1080         for (i = 0; i < uhmehash_num; i++) {
1081                 mutex_init(&uhme_hash[i].hmehash_mutex, NULL,
1082                     MUTEX_DEFAULT, NULL);
1083                 uhme_hash[i].hmeh_nextpa = HMEBLK_ENDPA;
1084         }
1085         khmehash_num--;         /* make sure counter starts from 0 */
1086         uhmehash_num--;         /* make sure counter starts from 0 */
1087 
1088         /*
1089          * Allocate context domain structures.
1090          *
1091          * A platform may choose to modify max_mmu_ctxdoms in
1092          * set_platform_defaults(). If a platform does not define
1093          * a set_platform_defaults() or does not choose to modify
1094          * max_mmu_ctxdoms, it gets one MMU context domain for every CPU.
1095          *
1096          * For all platforms that have CPUs sharing MMUs, this
1097          * value must be defined.
1098          */
1099         if (max_mmu_ctxdoms == 0)
1100                 max_mmu_ctxdoms = max_ncpus;
1101 
1102         size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *);
1103         mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP);
1104 
1105         /* mmu_ctx_t is 64 bytes aligned */
1106         mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache",
1107             sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
1108         /*
1109          * MMU context domain initialization for the Boot CPU.
1110          * This needs the context domains array allocated above.
1111          */
1112         mutex_enter(&cpu_lock);
1113         sfmmu_cpu_init(CPU);
1114         mutex_exit(&cpu_lock);
1115 
1116         /*
1117          * Intialize ism mapping list lock.
1118          */
1119 
1120         mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL);
1121 
1122         /*
1123          * Each sfmmu structure carries an array of MMU context info
1124          * structures, one per context domain. The size of this array depends
1125          * on the maximum number of context domains. So, the size of the
1126          * sfmmu structure varies per platform.
1127          *
1128          * sfmmu is allocated from static arena, because trap
1129          * handler at TL > 0 is not allowed to touch kernel relocatable
1130          * memory. sfmmu's alignment is changed to 64 bytes from
1131          * default 8 bytes, as the lower 6 bits will be used to pass
1132          * pgcnt to vtag_flush_pgcnt_tl1.
1133          */
1134         size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1);
1135 
1136         sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size,
1137             64, sfmmu_idcache_constructor, sfmmu_idcache_destructor,
1138             NULL, NULL, static_arena, 0);
1139 
1140         sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache",
1141             sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0);
1142 
1143         /*
1144          * Since we only use the tsb8k cache to "borrow" pages for TSBs
1145          * from the heap when low on memory or when TSB_FORCEALLOC is
1146          * specified, don't use magazines to cache them--we want to return
1147          * them to the system as quickly as possible.
1148          */
1149         sfmmu_tsb8k_cache = kmem_cache_create("sfmmu_tsb8k_cache",
1150             MMU_PAGESIZE, MMU_PAGESIZE, NULL, NULL, NULL, NULL,
1151             static_arena, KMC_NOMAGAZINE);
1152 
1153         /*
1154          * Set tsb_alloc_hiwater to 1/tsb_alloc_hiwater_factor of physical
1155          * memory, which corresponds to the old static reserve for TSBs.
1156          * tsb_alloc_hiwater_factor defaults to 32.  This caps the amount of
1157          * memory we'll allocate for TSB slabs; beyond this point TSB
1158          * allocations will be taken from the kernel heap (via
1159          * sfmmu_tsb8k_cache) and will be throttled as would any other kmem
1160          * consumer.
1161          */
1162         if (tsb_alloc_hiwater_factor == 0) {
1163                 tsb_alloc_hiwater_factor = TSB_ALLOC_HIWATER_FACTOR_DEFAULT;
1164         }
1165         SFMMU_SET_TSB_ALLOC_HIWATER(physmem);
1166 
1167         for (sz = tsb_slab_ttesz; sz > 0; sz--) {
1168                 if (!(disable_large_pages & (1 << sz)))
1169                         break;
1170         }
1171 
1172         if (sz < tsb_slab_ttesz) {
1173                 tsb_slab_ttesz = sz;
1174                 tsb_slab_shift = MMU_PAGESHIFT + (sz << 1) + sz;
1175                 tsb_slab_size = 1 << tsb_slab_shift;
1176                 tsb_slab_mask = (1 << (tsb_slab_shift - MMU_PAGESHIFT)) - 1;
1177                 use_bigtsb_arena = 0;
1178         } else if (use_bigtsb_arena &&
1179             (disable_large_pages & (1 << bigtsb_slab_ttesz))) {
1180                 use_bigtsb_arena = 0;
1181         }
1182 
1183         if (!use_bigtsb_arena) {
1184                 bigtsb_slab_shift = tsb_slab_shift;
1185         }
1186         SFMMU_SET_TSB_MAX_GROWSIZE(physmem);
1187 
1188         /*
1189          * On smaller memory systems, allocate TSB memory in smaller chunks
1190          * than the default 4M slab size. We also honor disable_large_pages
1191          * here.
1192          *
1193          * The trap handlers need to be patched with the final slab shift,
1194          * since they need to be able to construct the TSB pointer at runtime.
1195          */
1196         if ((tsb_max_growsize <= TSB_512K_SZCODE) &&
1197             !(disable_large_pages & (1 << TTE512K))) {
1198                 tsb_slab_ttesz = TTE512K;
1199                 tsb_slab_shift = MMU_PAGESHIFT512K;
1200                 tsb_slab_size = MMU_PAGESIZE512K;
1201                 tsb_slab_mask = MMU_PAGEOFFSET512K >> MMU_PAGESHIFT;
1202                 use_bigtsb_arena = 0;
1203         }
1204 
1205         if (!use_bigtsb_arena) {
1206                 bigtsb_slab_ttesz = tsb_slab_ttesz;
1207                 bigtsb_slab_shift = tsb_slab_shift;
1208                 bigtsb_slab_size = tsb_slab_size;
1209                 bigtsb_slab_mask = tsb_slab_mask;
1210         }
1211 
1212 
1213         /*
1214          * Set up memory callback to update tsb_alloc_hiwater and
1215          * tsb_max_growsize.
1216          */
1217         i = kphysm_setup_func_register(&sfmmu_update_vec, (void *) 0);
1218         ASSERT(i == 0);
1219 
1220         /*
1221          * kmem_tsb_arena is the source from which large TSB slabs are
1222          * drawn.  The quantum of this arena corresponds to the largest
1223          * TSB size we can dynamically allocate for user processes.
1224          * Currently it must also be a supported page size since we
1225          * use exactly one translation entry to map each slab page.
1226          *
1227          * The per-lgroup kmem_tsb_default_arena arenas are the arenas from
1228          * which most TSBs are allocated.  Since most TSB allocations are
1229          * typically 8K we have a kmem cache we stack on top of each
1230          * kmem_tsb_default_arena to speed up those allocations.
1231          *
1232          * Note the two-level scheme of arenas is required only
1233          * because vmem_create doesn't allow us to specify alignment
1234          * requirements.  If this ever changes the code could be
1235          * simplified to use only one level of arenas.
1236          *
1237          * If 256M page support exists on sun4v, 256MB kmem_bigtsb_arena
1238          * will be provided in addition to the 4M kmem_tsb_arena.
1239          */
1240         if (use_bigtsb_arena) {
1241                 kmem_bigtsb_arena = vmem_create("kmem_bigtsb", NULL, 0,
1242                     bigtsb_slab_size, sfmmu_vmem_xalloc_aligned_wrapper,
1243                     vmem_xfree, heap_arena, 0, VM_SLEEP);
1244         }
1245 
1246         kmem_tsb_arena = vmem_create("kmem_tsb", NULL, 0, tsb_slab_size,
1247             sfmmu_vmem_xalloc_aligned_wrapper,
1248             vmem_xfree, heap_arena, 0, VM_SLEEP);
1249 
1250         if (tsb_lgrp_affinity) {
1251                 char s[50];
1252                 for (i = 0; i < NLGRPS_MAX; i++) {
1253                         if (use_bigtsb_arena) {
1254                                 (void) sprintf(s, "kmem_bigtsb_lgrp%d", i);
1255                                 kmem_bigtsb_default_arena[i] = vmem_create(s,
1256                                     NULL, 0, 2 * tsb_slab_size,
1257                                     sfmmu_tsb_segkmem_alloc,
1258                                     sfmmu_tsb_segkmem_free, kmem_bigtsb_arena,
1259                                     0, VM_SLEEP | VM_BESTFIT);
1260                         }
1261 
1262                         (void) sprintf(s, "kmem_tsb_lgrp%d", i);
1263                         kmem_tsb_default_arena[i] = vmem_create(s,
1264                             NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc,
1265                             sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0,
1266                             VM_SLEEP | VM_BESTFIT);
1267 
1268                         (void) sprintf(s, "sfmmu_tsb_lgrp%d_cache", i);
1269                         sfmmu_tsb_cache[i] = kmem_cache_create(s,
1270                             PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL,
1271                             kmem_tsb_default_arena[i], 0);
1272                 }
1273         } else {
1274                 if (use_bigtsb_arena) {
1275                         kmem_bigtsb_default_arena[0] =
1276                             vmem_create("kmem_bigtsb_default", NULL, 0,
1277                             2 * tsb_slab_size, sfmmu_tsb_segkmem_alloc,
1278                             sfmmu_tsb_segkmem_free, kmem_bigtsb_arena, 0,
1279                             VM_SLEEP | VM_BESTFIT);
1280                 }
1281 
1282                 kmem_tsb_default_arena[0] = vmem_create("kmem_tsb_default",
1283                     NULL, 0, PAGESIZE, sfmmu_tsb_segkmem_alloc,
1284                     sfmmu_tsb_segkmem_free, kmem_tsb_arena, 0,
1285                     VM_SLEEP | VM_BESTFIT);
1286                 sfmmu_tsb_cache[0] = kmem_cache_create("sfmmu_tsb_cache",
1287                     PAGESIZE, PAGESIZE, NULL, NULL, NULL, NULL,
1288                     kmem_tsb_default_arena[0], 0);
1289         }
1290 
1291         sfmmu8_cache = kmem_cache_create("sfmmu8_cache", HME8BLK_SZ,
1292             HMEBLK_ALIGN, sfmmu_hblkcache_constructor,
1293             sfmmu_hblkcache_destructor,
1294             sfmmu_hblkcache_reclaim, (void *)HME8BLK_SZ,
1295             hat_memload_arena, KMC_NOHASH);
1296 
1297         hat_memload1_arena = vmem_create("hat_memload1", NULL, 0, PAGESIZE,
1298             segkmem_alloc_permanent, segkmem_free, heap_arena, 0,
1299             VMC_DUMPSAFE | VM_SLEEP);
1300 
1301         sfmmu1_cache = kmem_cache_create("sfmmu1_cache", HME1BLK_SZ,
1302             HMEBLK_ALIGN, sfmmu_hblkcache_constructor,
1303             sfmmu_hblkcache_destructor,
1304             NULL, (void *)HME1BLK_SZ,
1305             hat_memload1_arena, KMC_NOHASH);
1306 
1307         pa_hment_cache = kmem_cache_create("pa_hment_cache", PAHME_SZ,
1308             0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH);
1309 
1310         ism_blk_cache = kmem_cache_create("ism_blk_cache",
1311             sizeof (ism_blk_t), ecache_alignsize, NULL, NULL,
1312             NULL, NULL, static_arena, KMC_NOHASH);
1313 
1314         ism_ment_cache = kmem_cache_create("ism_ment_cache",
1315             sizeof (ism_ment_t), 0, NULL, NULL,
1316             NULL, NULL, NULL, 0);
1317 
1318         /*
1319          * We grab the first hat for the kernel,
1320          */
1321         AS_LOCK_ENTER(&kas, &kas.a_lock, RW_WRITER);
1322         kas.a_hat = hat_alloc(&kas);
1323         AS_LOCK_EXIT(&kas, &kas.a_lock);
1324 
1325         /*
1326          * Initialize hblk_reserve.
1327          */
1328         ((struct hme_blk *)hblk_reserve)->hblk_nextpa =
1329             va_to_pa((caddr_t)hblk_reserve);
1330 
1331 #ifndef UTSB_PHYS
1332         /*
1333          * Reserve some kernel virtual address space for the locked TTEs
1334          * that allow us to probe the TSB from TL>0.
1335          */
1336         utsb_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size,
1337             0, 0, NULL, NULL, VM_SLEEP);
1338         utsb4m_vabase = vmem_xalloc(heap_arena, tsb_slab_size, tsb_slab_size,
1339             0, 0, NULL, NULL, VM_SLEEP);
1340 #endif
1341 
1342 #ifdef VAC
1343         /*
1344          * The big page VAC handling code assumes VAC
1345          * will not be bigger than the smallest big
1346          * page- which is 64K.
1347          */
1348         if (TTEPAGES(TTE64K) < CACHE_NUM_COLOR) {
1349                 cmn_err(CE_PANIC, "VAC too big!");
1350         }
1351 #endif
1352 
1353         (void) xhat_init();
1354 
1355         uhme_hash_pa = va_to_pa(uhme_hash);
1356         khme_hash_pa = va_to_pa(khme_hash);
1357 
1358         /*
1359          * Initialize relocation locks. kpr_suspendlock is held
1360          * at PIL_MAX to prevent interrupts from pinning the holder
1361          * of a suspended TTE which may access it leading to a
1362          * deadlock condition.
1363          */
1364         mutex_init(&kpr_mutex, NULL, MUTEX_DEFAULT, NULL);
1365         mutex_init(&kpr_suspendlock, NULL, MUTEX_SPIN, (void *)PIL_MAX);
1366 
1367         /*
1368          * If Shared context support is disabled via /etc/system
1369          * set shctx_on to 0 here if it was set to 1 earlier in boot
1370          * sequence by cpu module initialization code.
1371          */
1372         if (shctx_on && disable_shctx) {
1373                 shctx_on = 0;
1374         }
1375 
1376         if (shctx_on) {
1377                 srd_buckets = kmem_zalloc(SFMMU_MAX_SRD_BUCKETS *
1378                     sizeof (srd_buckets[0]), KM_SLEEP);
1379                 for (i = 0; i < SFMMU_MAX_SRD_BUCKETS; i++) {
1380                         mutex_init(&srd_buckets[i].srdb_lock, NULL,
1381                             MUTEX_DEFAULT, NULL);
1382                 }
1383 
1384                 srd_cache = kmem_cache_create("srd_cache", sizeof (sf_srd_t),
1385                     0, sfmmu_srdcache_constructor, sfmmu_srdcache_destructor,
1386                     NULL, NULL, NULL, 0);
1387                 region_cache = kmem_cache_create("region_cache",
1388                     sizeof (sf_region_t), 0, sfmmu_rgncache_constructor,
1389                     sfmmu_rgncache_destructor, NULL, NULL, NULL, 0);
1390                 scd_cache = kmem_cache_create("scd_cache", sizeof (sf_scd_t),
1391                     0, sfmmu_scdcache_constructor,  sfmmu_scdcache_destructor,
1392                     NULL, NULL, NULL, 0);
1393         }
1394 
1395         /*
1396          * Pre-allocate hrm_hashtab before enabling the collection of
1397          * refmod statistics.  Allocating on the fly would mean us
1398          * running the risk of suffering recursive mutex enters or
1399          * deadlocks.
1400          */
1401         hrm_hashtab = kmem_zalloc(HRM_HASHSIZE * sizeof (struct hrmstat *),
1402             KM_SLEEP);
1403 
1404         /* Allocate per-cpu pending freelist of hmeblks */
1405         cpu_hme_pend = kmem_zalloc((NCPU * sizeof (cpu_hme_pend_t)) + 64,
1406             KM_SLEEP);
1407         cpu_hme_pend = (cpu_hme_pend_t *)P2ROUNDUP(
1408             (uintptr_t)cpu_hme_pend, 64);
1409 
1410         for (i = 0; i < NCPU; i++) {
1411                 mutex_init(&cpu_hme_pend[i].chp_mutex, NULL, MUTEX_DEFAULT,
1412                     NULL);
1413         }
1414 
1415         if (cpu_hme_pend_thresh == 0) {
1416                 cpu_hme_pend_thresh = CPU_HME_PEND_THRESH;
1417         }
1418 }
1419 
1420 /*
1421  * Initialize locking for the hat layer, called early during boot.
1422  */
1423 static void
1424 hat_lock_init()
1425 {
1426         int i;
1427 
1428         /*
1429          * initialize the array of mutexes protecting a page's mapping
1430          * list and p_nrm field.
1431          */
1432         for (i = 0; i < MML_TABLE_SIZE; i++)
1433                 mutex_init(&mml_table[i].pad_mutex, NULL, MUTEX_DEFAULT, NULL);
1434 
1435         if (kpm_enable) {
1436                 for (i = 0; i < kpmp_table_sz; i++) {
1437                         mutex_init(&kpmp_table[i].khl_mutex, NULL,
1438                             MUTEX_DEFAULT, NULL);
1439                 }
1440         }
1441 
1442         /*
1443          * Initialize array of mutex locks that protects sfmmu fields and
1444          * TSB lists.
1445          */
1446         for (i = 0; i < SFMMU_NUM_LOCK; i++)
1447                 mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT,
1448                     NULL);
1449 }
1450 
1451 #define SFMMU_KERNEL_MAXVA \
1452         (kmem64_base ? (uintptr_t)kmem64_end : (SYSLIMIT))
1453 
1454 /*
1455  * Allocate a hat structure.
1456  * Called when an address space first uses a hat.
1457  */
1458 struct hat *
1459 hat_alloc(struct as *as)
1460 {
1461         sfmmu_t *sfmmup;
1462         int i;
1463         uint64_t cnum;
1464         extern uint_t get_color_start(struct as *);
1465 
1466         ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1467         sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP);
1468         sfmmup->sfmmu_as = as;
1469         sfmmup->sfmmu_flags = 0;
1470         sfmmup->sfmmu_tteflags = 0;
1471         sfmmup->sfmmu_rtteflags = 0;
1472         LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock);
1473 
1474         if (as == &kas) {
1475                 ksfmmup = sfmmup;
1476                 sfmmup->sfmmu_cext = 0;
1477                 cnum = KCONTEXT;
1478 
1479                 sfmmup->sfmmu_clrstart = 0;
1480                 sfmmup->sfmmu_tsb = NULL;
1481                 /*
1482                  * hat_kern_setup() will call sfmmu_init_ktsbinfo()
1483                  * to setup tsb_info for ksfmmup.
1484                  */
1485         } else {
1486 
1487                 /*
1488                  * Just set to invalid ctx. When it faults, it will
1489                  * get a valid ctx. This would avoid the situation
1490                  * where we get a ctx, but it gets stolen and then
1491                  * we fault when we try to run and so have to get
1492                  * another ctx.
1493                  */
1494                 sfmmup->sfmmu_cext = 0;
1495                 cnum = INVALID_CONTEXT;
1496 
1497                 /* initialize original physical page coloring bin */
1498                 sfmmup->sfmmu_clrstart = get_color_start(as);
1499 #ifdef DEBUG
1500                 if (tsb_random_size) {
1501                         uint32_t randval = (uint32_t)gettick() >> 4;
1502                         int size = randval % (tsb_max_growsize + 1);
1503 
1504                         /* chose a random tsb size for stress testing */
1505                         (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb, size,
1506                             TSB8K|TSB64K|TSB512K, 0, sfmmup);
1507                 } else
1508 #endif /* DEBUG */
1509                         (void) sfmmu_tsbinfo_alloc(&sfmmup->sfmmu_tsb,
1510                             default_tsb_size,
1511                             TSB8K|TSB64K|TSB512K, 0, sfmmup);
1512                 sfmmup->sfmmu_flags = HAT_SWAPPED | HAT_ALLCTX_INVALID;
1513                 ASSERT(sfmmup->sfmmu_tsb != NULL);
1514         }
1515 
1516         ASSERT(max_mmu_ctxdoms > 0);
1517         for (i = 0; i < max_mmu_ctxdoms; i++) {
1518                 sfmmup->sfmmu_ctxs[i].cnum = cnum;
1519                 sfmmup->sfmmu_ctxs[i].gnum = 0;
1520         }
1521 
1522         for (i = 0; i < max_mmu_page_sizes; i++) {
1523                 sfmmup->sfmmu_ttecnt[i] = 0;
1524                 sfmmup->sfmmu_scdrttecnt[i] = 0;
1525                 sfmmup->sfmmu_ismttecnt[i] = 0;
1526                 sfmmup->sfmmu_scdismttecnt[i] = 0;
1527                 sfmmup->sfmmu_pgsz[i] = TTE8K;
1528         }
1529         sfmmup->sfmmu_tsb0_4minflcnt = 0;
1530         sfmmup->sfmmu_iblk = NULL;
1531         sfmmup->sfmmu_ismhat = 0;
1532         sfmmup->sfmmu_scdhat = 0;
1533         sfmmup->sfmmu_ismblkpa = (uint64_t)-1;
1534         if (sfmmup == ksfmmup) {
1535                 CPUSET_ALL(sfmmup->sfmmu_cpusran);
1536         } else {
1537                 CPUSET_ZERO(sfmmup->sfmmu_cpusran);
1538         }
1539         sfmmup->sfmmu_free = 0;
1540         sfmmup->sfmmu_rmstat = 0;
1541         sfmmup->sfmmu_clrbin = sfmmup->sfmmu_clrstart;
1542         sfmmup->sfmmu_xhat_provider = NULL;
1543         cv_init(&sfmmup->sfmmu_tsb_cv, NULL, CV_DEFAULT, NULL);
1544         sfmmup->sfmmu_srdp = NULL;
1545         SF_RGNMAP_ZERO(sfmmup->sfmmu_region_map);
1546         bzero(sfmmup->sfmmu_hmeregion_links, SFMMU_L1_HMERLINKS_SIZE);
1547         sfmmup->sfmmu_scdp = NULL;
1548         sfmmup->sfmmu_scd_link.next = NULL;
1549         sfmmup->sfmmu_scd_link.prev = NULL;
1550         return (sfmmup);
1551 }
1552 
1553 /*
1554  * Create per-MMU context domain kstats for a given MMU ctx.
1555  */
1556 static void
1557 sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp)
1558 {
1559         mmu_ctx_stat_t  stat;
1560         kstat_t         *mmu_kstat;
1561 
1562         ASSERT(MUTEX_HELD(&cpu_lock));
1563         ASSERT(mmu_ctxp->mmu_kstat == NULL);
1564 
1565         mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx",
1566             "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL);
1567 
1568         if (mmu_kstat == NULL) {
1569                 cmn_err(CE_WARN, "kstat_create for MMU %d failed",
1570                     mmu_ctxp->mmu_idx);
1571         } else {
1572                 mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data;
1573                 for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++)
1574                         kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat],
1575                             mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64);
1576                 mmu_ctxp->mmu_kstat = mmu_kstat;
1577                 kstat_install(mmu_kstat);
1578         }
1579 }
1580 
1581 /*
1582  * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU
1583  * context domain information for a given CPU. If a platform does not
1584  * specify that interface, then the function below is used instead to return
1585  * default information. The defaults are as follows:
1586  *
1587  *      - The number of MMU context IDs supported on any CPU in the
1588  *        system is 8K.
1589  *      - There is one MMU context domain per CPU.
1590  */
1591 /*ARGSUSED*/
1592 static void
1593 sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop)
1594 {
1595         infop->mmu_nctxs = nctxs;
1596         infop->mmu_idx = cpu[cpuid]->cpu_seqid;
1597 }
1598 
1599 /*
1600  * Called during CPU initialization to set the MMU context-related information
1601  * for a CPU.
1602  *
1603  * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum.
1604  */
1605 void
1606 sfmmu_cpu_init(cpu_t *cp)
1607 {
1608         mmu_ctx_info_t  info;
1609         mmu_ctx_t       *mmu_ctxp;
1610 
1611         ASSERT(MUTEX_HELD(&cpu_lock));
1612 
1613         if (&plat_cpuid_to_mmu_ctx_info == NULL)
1614                 sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info);
1615         else
1616                 plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info);
1617 
1618         ASSERT(info.mmu_idx < max_mmu_ctxdoms);
1619 
1620         if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) {
1621                 /* Each mmu_ctx is cacheline aligned. */
1622                 mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP);
1623                 bzero(mmu_ctxp, sizeof (mmu_ctx_t));
1624 
1625                 mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN,
1626                     (void *)ipltospl(DISP_LEVEL));
1627                 mmu_ctxp->mmu_idx = info.mmu_idx;
1628                 mmu_ctxp->mmu_nctxs = info.mmu_nctxs;
1629                 /*
1630                  * Globally for lifetime of a system,
1631                  * gnum must always increase.
1632                  * mmu_saved_gnum is protected by the cpu_lock.
1633                  */
1634                 mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1;
1635                 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS;
1636 
1637                 sfmmu_mmu_kstat_create(mmu_ctxp);
1638 
1639                 mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp;
1640         } else {
1641                 ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx);
1642                 ASSERT(mmu_ctxp->mmu_nctxs <= info.mmu_nctxs);
1643         }
1644 
1645         /*
1646          * The mmu_lock is acquired here to prevent races with
1647          * the wrap-around code.
1648          */
1649         mutex_enter(&mmu_ctxp->mmu_lock);
1650 
1651 
1652         mmu_ctxp->mmu_ncpus++;
1653         CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id);
1654         CPU_MMU_IDX(cp) = info.mmu_idx;
1655         CPU_MMU_CTXP(cp) = mmu_ctxp;
1656 
1657         mutex_exit(&mmu_ctxp->mmu_lock);
1658 }
1659 
1660 static void
1661 sfmmu_ctxdom_free(mmu_ctx_t *mmu_ctxp)
1662 {
1663         ASSERT(MUTEX_HELD(&cpu_lock));
1664         ASSERT(!MUTEX_HELD(&mmu_ctxp->mmu_lock));
1665 
1666         mutex_destroy(&mmu_ctxp->mmu_lock);
1667 
1668         if (mmu_ctxp->mmu_kstat)
1669                 kstat_delete(mmu_ctxp->mmu_kstat);
1670 
1671         /* mmu_saved_gnum is protected by the cpu_lock. */
1672         if (mmu_saved_gnum < mmu_ctxp->mmu_gnum)
1673                 mmu_saved_gnum = mmu_ctxp->mmu_gnum;
1674 
1675         kmem_cache_free(mmuctxdom_cache, mmu_ctxp);
1676 }
1677 
1678 /*
1679  * Called to perform MMU context-related cleanup for a CPU.
1680  */
1681 void
1682 sfmmu_cpu_cleanup(cpu_t *cp)
1683 {
1684         mmu_ctx_t       *mmu_ctxp;
1685 
1686         ASSERT(MUTEX_HELD(&cpu_lock));
1687 
1688         mmu_ctxp = CPU_MMU_CTXP(cp);
1689         ASSERT(mmu_ctxp != NULL);
1690 
1691         /*
1692          * The mmu_lock is acquired here to prevent races with
1693          * the wrap-around code.
1694          */
1695         mutex_enter(&mmu_ctxp->mmu_lock);
1696 
1697         CPU_MMU_CTXP(cp) = NULL;
1698 
1699         CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id);
1700         if (--mmu_ctxp->mmu_ncpus == 0) {
1701                 mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL;
1702                 mutex_exit(&mmu_ctxp->mmu_lock);
1703                 sfmmu_ctxdom_free(mmu_ctxp);
1704                 return;
1705         }
1706 
1707         mutex_exit(&mmu_ctxp->mmu_lock);
1708 }
1709 
1710 uint_t
1711 sfmmu_ctxdom_nctxs(int idx)
1712 {
1713         return (mmu_ctxs_tbl[idx]->mmu_nctxs);
1714 }
1715 
1716 #ifdef sun4v
1717 /*
1718  * sfmmu_ctxdoms_* is an interface provided to help keep context domains
1719  * consistant after suspend/resume on system that can resume on a different
1720  * hardware than it was suspended.
1721  *
1722  * sfmmu_ctxdom_lock(void) locks all context domains and prevents new contexts
1723  * from being allocated.  It acquires all hat_locks, which blocks most access to
1724  * context data, except for a few cases that are handled separately or are
1725  * harmless.  It wraps each domain to increment gnum and invalidate on-CPU
1726  * contexts, and forces cnum to its max.  As a result of this call all user
1727  * threads that are running on CPUs trap and try to perform wrap around but
1728  * can't because hat_locks are taken.  Threads that were not on CPUs but started
1729  * by scheduler go to sfmmu_alloc_ctx() to aquire context without checking
1730  * hat_lock, but fail, because cnum == nctxs, and therefore also trap and block
1731  * on hat_lock trying to wrap.  sfmmu_ctxdom_lock() must be called before CPUs
1732  * are paused, else it could deadlock acquiring locks held by paused CPUs.
1733  *
1734  * sfmmu_ctxdoms_remove() removes context domains from every CPUs and records
1735  * the CPUs that had them.  It must be called after CPUs have been paused. This
1736  * ensures that no threads are in sfmmu_alloc_ctx() accessing domain data,
1737  * because pause_cpus sends a mondo interrupt to every CPU, and sfmmu_alloc_ctx
1738  * runs with interrupts disabled.  When CPUs are later resumed, they may enter
1739  * sfmmu_alloc_ctx, but it will check for CPU_MMU_CTXP = NULL and immediately
1740  * return failure.  Or, they will be blocked trying to acquire hat_lock. Thus
1741  * after sfmmu_ctxdoms_remove returns, we are guaranteed that no one is
1742  * accessing the old context domains.
1743  *
1744  * sfmmu_ctxdoms_update(void) frees space used by old context domains and
1745  * allocates new context domains based on hardware layout.  It initializes
1746  * every CPU that had context domain before migration to have one again.
1747  * sfmmu_ctxdoms_update must be called after CPUs are resumed, else it
1748  * could deadlock acquiring locks held by paused CPUs.
1749  *
1750  * sfmmu_ctxdoms_unlock(void) releases all hat_locks after which user threads
1751  * acquire new context ids and continue execution.
1752  *
1753  * Therefore functions should be called in the following order:
1754  *       suspend_routine()
1755  *              sfmmu_ctxdom_lock()
1756  *              pause_cpus()
1757  *              suspend()
1758  *                      if (suspend failed)
1759  *                              sfmmu_ctxdom_unlock()
1760  *              ...
1761  *              sfmmu_ctxdom_remove()
1762  *              resume_cpus()
1763  *              sfmmu_ctxdom_update()
1764  *              sfmmu_ctxdom_unlock()
1765  */
1766 static cpuset_t sfmmu_ctxdoms_pset;
1767 
1768 void
1769 sfmmu_ctxdoms_remove()
1770 {
1771         processorid_t   id;
1772         cpu_t           *cp;
1773 
1774         /*
1775          * Record the CPUs that have domains in sfmmu_ctxdoms_pset, so they can
1776          * be restored post-migration. A CPU may be powered off and not have a
1777          * domain, for example.
1778          */
1779         CPUSET_ZERO(sfmmu_ctxdoms_pset);
1780 
1781         for (id = 0; id < NCPU; id++) {
1782                 if ((cp = cpu[id]) != NULL && CPU_MMU_CTXP(cp) != NULL) {
1783                         CPUSET_ADD(sfmmu_ctxdoms_pset, id);
1784                         CPU_MMU_CTXP(cp) = NULL;
1785                 }
1786         }
1787 }
1788 
1789 void
1790 sfmmu_ctxdoms_lock(void)
1791 {
1792         int             idx;
1793         mmu_ctx_t       *mmu_ctxp;
1794 
1795         sfmmu_hat_lock_all();
1796 
1797         /*
1798          * At this point, no thread can be in sfmmu_ctx_wrap_around, because
1799          * hat_lock is always taken before calling it.
1800          *
1801          * For each domain, set mmu_cnum to max so no more contexts can be
1802          * allocated, and wrap to flush on-CPU contexts and force threads to
1803          * acquire a new context when we later drop hat_lock after migration.
1804          * Setting mmu_cnum may race with sfmmu_alloc_ctx which also sets cnum,
1805          * but the latter uses CAS and will miscompare and not overwrite it.
1806          */
1807         kpreempt_disable(); /* required by sfmmu_ctx_wrap_around */
1808         for (idx = 0; idx < max_mmu_ctxdoms; idx++) {
1809                 if ((mmu_ctxp = mmu_ctxs_tbl[idx]) != NULL) {
1810                         mutex_enter(&mmu_ctxp->mmu_lock);
1811                         mmu_ctxp->mmu_cnum = mmu_ctxp->mmu_nctxs;
1812                         /* make sure updated cnum visible */
1813                         membar_enter();
1814                         mutex_exit(&mmu_ctxp->mmu_lock);
1815                         sfmmu_ctx_wrap_around(mmu_ctxp, B_FALSE);
1816                 }
1817         }
1818         kpreempt_enable();
1819 }
1820 
1821 void
1822 sfmmu_ctxdoms_unlock(void)
1823 {
1824         sfmmu_hat_unlock_all();
1825 }
1826 
1827 void
1828 sfmmu_ctxdoms_update(void)
1829 {
1830         processorid_t   id;
1831         cpu_t           *cp;
1832         uint_t          idx;
1833         mmu_ctx_t       *mmu_ctxp;
1834 
1835         /*
1836          * Free all context domains.  As side effect, this increases
1837          * mmu_saved_gnum to the maximum gnum over all domains, which is used to
1838          * init gnum in the new domains, which therefore will be larger than the
1839          * sfmmu gnum for any process, guaranteeing that every process will see
1840          * a new generation and allocate a new context regardless of what new
1841          * domain it runs in.
1842          */
1843         mutex_enter(&cpu_lock);
1844 
1845         for (idx = 0; idx < max_mmu_ctxdoms; idx++) {
1846                 if (mmu_ctxs_tbl[idx] != NULL) {
1847                         mmu_ctxp = mmu_ctxs_tbl[idx];
1848                         mmu_ctxs_tbl[idx] = NULL;
1849                         sfmmu_ctxdom_free(mmu_ctxp);
1850                 }
1851         }
1852 
1853         for (id = 0; id < NCPU; id++) {
1854                 if (CPU_IN_SET(sfmmu_ctxdoms_pset, id) &&
1855                     (cp = cpu[id]) != NULL)
1856                         sfmmu_cpu_init(cp);
1857         }
1858         mutex_exit(&cpu_lock);
1859 }
1860 #endif
1861 
1862 /*
1863  * Hat_setup, makes an address space context the current active one.
1864  * In sfmmu this translates to setting the secondary context with the
1865  * corresponding context.
1866  */
1867 void
1868 hat_setup(struct hat *sfmmup, int allocflag)
1869 {
1870         hatlock_t *hatlockp;
1871 
1872         /* Init needs some special treatment. */
1873         if (allocflag == HAT_INIT) {
1874                 /*
1875                  * Make sure that we have
1876                  * 1. a TSB
1877                  * 2. a valid ctx that doesn't get stolen after this point.
1878                  */
1879                 hatlockp = sfmmu_hat_enter(sfmmup);
1880 
1881                 /*
1882                  * Swap in the TSB.  hat_init() allocates tsbinfos without
1883                  * TSBs, but we need one for init, since the kernel does some
1884                  * special things to set up its stack and needs the TSB to
1885                  * resolve page faults.
1886                  */
1887                 sfmmu_tsb_swapin(sfmmup, hatlockp);
1888 
1889                 sfmmu_get_ctx(sfmmup);
1890 
1891                 sfmmu_hat_exit(hatlockp);
1892         } else {
1893                 ASSERT(allocflag == HAT_ALLOC);
1894 
1895                 hatlockp = sfmmu_hat_enter(sfmmup);
1896                 kpreempt_disable();
1897 
1898                 CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id);
1899                 /*
1900                  * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter,
1901                  * pagesize bits don't matter in this case since we are passing
1902                  * INVALID_CONTEXT to it.
1903                  * Compatibility Note: hw takes care of MMU_SCONTEXT1
1904                  */
1905                 sfmmu_setctx_sec(INVALID_CONTEXT);
1906                 sfmmu_clear_utsbinfo();
1907 
1908                 kpreempt_enable();
1909                 sfmmu_hat_exit(hatlockp);
1910         }
1911 }
1912 
1913 /*
1914  * Free all the translation resources for the specified address space.
1915  * Called from as_free when an address space is being destroyed.
1916  */
1917 void
1918 hat_free_start(struct hat *sfmmup)
1919 {
1920         ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
1921         ASSERT(sfmmup != ksfmmup);
1922         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
1923 
1924         sfmmup->sfmmu_free = 1;
1925         if (sfmmup->sfmmu_scdp != NULL) {
1926                 sfmmu_leave_scd(sfmmup, 0);
1927         }
1928 
1929         ASSERT(sfmmup->sfmmu_scdp == NULL);
1930 }
1931 
1932 void
1933 hat_free_end(struct hat *sfmmup)
1934 {
1935         int i;
1936 
1937         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
1938         ASSERT(sfmmup->sfmmu_free == 1);
1939         ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0);
1940         ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0);
1941         ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0);
1942         ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0);
1943         ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
1944         ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
1945 
1946         if (sfmmup->sfmmu_rmstat) {
1947                 hat_freestat(sfmmup->sfmmu_as, NULL);
1948         }
1949 
1950         while (sfmmup->sfmmu_tsb != NULL) {
1951                 struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next;
1952                 sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb);
1953                 sfmmup->sfmmu_tsb = next;
1954         }
1955 
1956         if (sfmmup->sfmmu_srdp != NULL) {
1957                 sfmmu_leave_srd(sfmmup);
1958                 ASSERT(sfmmup->sfmmu_srdp == NULL);
1959                 for (i = 0; i < SFMMU_L1_HMERLINKS; i++) {
1960                         if (sfmmup->sfmmu_hmeregion_links[i] != NULL) {
1961                                 kmem_free(sfmmup->sfmmu_hmeregion_links[i],
1962                                     SFMMU_L2_HMERLINKS_SIZE);
1963                                 sfmmup->sfmmu_hmeregion_links[i] = NULL;
1964                         }
1965                 }
1966         }
1967         sfmmu_free_sfmmu(sfmmup);
1968 
1969 #ifdef DEBUG
1970         for (i = 0; i < SFMMU_L1_HMERLINKS; i++) {
1971                 ASSERT(sfmmup->sfmmu_hmeregion_links[i] == NULL);
1972         }
1973 #endif
1974 
1975         kmem_cache_free(sfmmuid_cache, sfmmup);
1976 }
1977 
1978 /*
1979  * Set up any translation structures, for the specified address space,
1980  * that are needed or preferred when the process is being swapped in.
1981  */
1982 /* ARGSUSED */
1983 void
1984 hat_swapin(struct hat *hat)
1985 {
1986         ASSERT(hat->sfmmu_xhat_provider == NULL);
1987 }
1988 
1989 /*
1990  * Free all of the translation resources, for the specified address space,
1991  * that can be freed while the process is swapped out. Called from as_swapout.
1992  * Also, free up the ctx that this process was using.
1993  */
1994 void
1995 hat_swapout(struct hat *sfmmup)
1996 {
1997         struct hmehash_bucket *hmebp;
1998         struct hme_blk *hmeblkp;
1999         struct hme_blk *pr_hblk = NULL;
2000         struct hme_blk *nx_hblk;
2001         int i;
2002         struct hme_blk *list = NULL;
2003         hatlock_t *hatlockp;
2004         struct tsb_info *tsbinfop;
2005         struct free_tsb {
2006                 struct free_tsb *next;
2007                 struct tsb_info *tsbinfop;
2008         };                      /* free list of TSBs */
2009         struct free_tsb *freelist, *last, *next;
2010 
2011         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
2012         SFMMU_STAT(sf_swapout);
2013 
2014         /*
2015          * There is no way to go from an as to all its translations in sfmmu.
2016          * Here is one of the times when we take the big hit and traverse
2017          * the hash looking for hme_blks to free up.  Not only do we free up
2018          * this as hme_blks but all those that are free.  We are obviously
2019          * swapping because we need memory so let's free up as much
2020          * as we can.
2021          *
2022          * Note that we don't flush TLB/TSB here -- it's not necessary
2023          * because:
2024          *  1) we free the ctx we're using and throw away the TSB(s);
2025          *  2) processes aren't runnable while being swapped out.
2026          */
2027         ASSERT(sfmmup != KHATID);
2028         for (i = 0; i <= UHMEHASH_SZ; i++) {
2029                 hmebp = &uhme_hash[i];
2030                 SFMMU_HASH_LOCK(hmebp);
2031                 hmeblkp = hmebp->hmeblkp;
2032                 pr_hblk = NULL;
2033                 while (hmeblkp) {
2034 
2035                         ASSERT(!hmeblkp->hblk_xhat_bit);
2036 
2037                         if ((hmeblkp->hblk_tag.htag_id == sfmmup) &&
2038                             !hmeblkp->hblk_shw_bit && !hmeblkp->hblk_lckcnt) {
2039                                 ASSERT(!hmeblkp->hblk_shared);
2040                                 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
2041                                     (caddr_t)get_hblk_base(hmeblkp),
2042                                     get_hblk_endaddr(hmeblkp),
2043                                     NULL, HAT_UNLOAD);
2044                         }
2045                         nx_hblk = hmeblkp->hblk_next;
2046                         if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
2047                                 ASSERT(!hmeblkp->hblk_lckcnt);
2048                                 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
2049                                     &list, 0);
2050                         } else {
2051                                 pr_hblk = hmeblkp;
2052                         }
2053                         hmeblkp = nx_hblk;
2054                 }
2055                 SFMMU_HASH_UNLOCK(hmebp);
2056         }
2057 
2058         sfmmu_hblks_list_purge(&list, 0);
2059 
2060         /*
2061          * Now free up the ctx so that others can reuse it.
2062          */
2063         hatlockp = sfmmu_hat_enter(sfmmup);
2064 
2065         sfmmu_invalidate_ctx(sfmmup);
2066 
2067         /*
2068          * Free TSBs, but not tsbinfos, and set SWAPPED flag.
2069          * If TSBs were never swapped in, just return.
2070          * This implies that we don't support partial swapping
2071          * of TSBs -- either all are swapped out, or none are.
2072          *
2073          * We must hold the HAT lock here to prevent racing with another
2074          * thread trying to unmap TTEs from the TSB or running the post-
2075          * relocator after relocating the TSB's memory.  Unfortunately, we
2076          * can't free memory while holding the HAT lock or we could
2077          * deadlock, so we build a list of TSBs to be freed after marking
2078          * the tsbinfos as swapped out and free them after dropping the
2079          * lock.
2080          */
2081         if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
2082                 sfmmu_hat_exit(hatlockp);
2083                 return;
2084         }
2085 
2086         SFMMU_FLAGS_SET(sfmmup, HAT_SWAPPED);
2087         last = freelist = NULL;
2088         for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
2089             tsbinfop = tsbinfop->tsb_next) {
2090                 ASSERT((tsbinfop->tsb_flags & TSB_SWAPPED) == 0);
2091 
2092                 /*
2093                  * Cast the TSB into a struct free_tsb and put it on the free
2094                  * list.
2095                  */
2096                 if (freelist == NULL) {
2097                         last = freelist = (struct free_tsb *)tsbinfop->tsb_va;
2098                 } else {
2099                         last->next = (struct free_tsb *)tsbinfop->tsb_va;
2100                         last = last->next;
2101                 }
2102                 last->next = NULL;
2103                 last->tsbinfop = tsbinfop;
2104                 tsbinfop->tsb_flags |= TSB_SWAPPED;
2105                 /*
2106                  * Zero out the TTE to clear the valid bit.
2107                  * Note we can't use a value like 0xbad because we want to
2108                  * ensure diagnostic bits are NEVER set on TTEs that might
2109                  * be loaded.  The intent is to catch any invalid access
2110                  * to the swapped TSB, such as a thread running with a valid
2111                  * context without first calling sfmmu_tsb_swapin() to
2112                  * allocate TSB memory.
2113                  */
2114                 tsbinfop->tsb_tte.ll = 0;
2115         }
2116 
2117         /* Now we can drop the lock and free the TSB memory. */
2118         sfmmu_hat_exit(hatlockp);
2119         for (; freelist != NULL; freelist = next) {
2120                 next = freelist->next;
2121                 sfmmu_tsb_free(freelist->tsbinfop);
2122         }
2123 }
2124 
2125 /*
2126  * Duplicate the translations of an as into another newas
2127  */
2128 /* ARGSUSED */
2129 int
2130 hat_dup(struct hat *hat, struct hat *newhat, caddr_t addr, size_t len,
2131         uint_t flag)
2132 {
2133         sf_srd_t *srdp;
2134         sf_scd_t *scdp;
2135         int i;
2136         extern uint_t get_color_start(struct as *);
2137 
2138         ASSERT(hat->sfmmu_xhat_provider == NULL);
2139         ASSERT((flag == 0) || (flag == HAT_DUP_ALL) || (flag == HAT_DUP_COW) ||
2140             (flag == HAT_DUP_SRD));
2141         ASSERT(hat != ksfmmup);
2142         ASSERT(newhat != ksfmmup);
2143         ASSERT(flag != HAT_DUP_ALL || hat->sfmmu_srdp == newhat->sfmmu_srdp);
2144 
2145         if (flag == HAT_DUP_COW) {
2146                 panic("hat_dup: HAT_DUP_COW not supported");
2147         }
2148 
2149         if (flag == HAT_DUP_SRD && ((srdp = hat->sfmmu_srdp) != NULL)) {
2150                 ASSERT(srdp->srd_evp != NULL);
2151                 VN_HOLD(srdp->srd_evp);
2152                 ASSERT(srdp->srd_refcnt > 0);
2153                 newhat->sfmmu_srdp = srdp;
2154                 atomic_add_32((volatile uint_t *)&srdp->srd_refcnt, 1);
2155         }
2156 
2157         /*
2158          * HAT_DUP_ALL flag is used after as duplication is done.
2159          */
2160         if (flag == HAT_DUP_ALL && ((srdp = newhat->sfmmu_srdp) != NULL)) {
2161                 ASSERT(newhat->sfmmu_srdp->srd_refcnt >= 2);
2162                 newhat->sfmmu_rtteflags = hat->sfmmu_rtteflags;
2163                 if (hat->sfmmu_flags & HAT_4MTEXT_FLAG) {
2164                         newhat->sfmmu_flags |= HAT_4MTEXT_FLAG;
2165                 }
2166 
2167                 /* check if need to join scd */
2168                 if ((scdp = hat->sfmmu_scdp) != NULL &&
2169                     newhat->sfmmu_scdp != scdp) {
2170                         int ret;
2171                         SF_RGNMAP_IS_SUBSET(&newhat->sfmmu_region_map,
2172                             &scdp->scd_region_map, ret);
2173                         ASSERT(ret);
2174                         sfmmu_join_scd(scdp, newhat);
2175                         ASSERT(newhat->sfmmu_scdp == scdp &&
2176                             scdp->scd_refcnt >= 2);
2177                         for (i = 0; i < max_mmu_page_sizes; i++) {
2178                                 newhat->sfmmu_ismttecnt[i] =
2179                                     hat->sfmmu_ismttecnt[i];
2180                                 newhat->sfmmu_scdismttecnt[i] =
2181                                     hat->sfmmu_scdismttecnt[i];
2182                         }
2183                 }
2184 
2185                 sfmmu_check_page_sizes(newhat, 1);
2186         }
2187 
2188         if (flag == HAT_DUP_ALL && consistent_coloring == 0 &&
2189             update_proc_pgcolorbase_after_fork != 0) {
2190                 hat->sfmmu_clrbin = get_color_start(hat->sfmmu_as);
2191         }
2192         return (0);
2193 }
2194 
2195 void
2196 hat_memload(struct hat *hat, caddr_t addr, struct page *pp,
2197         uint_t attr, uint_t flags)
2198 {
2199         hat_do_memload(hat, addr, pp, attr, flags,
2200             SFMMU_INVALID_SHMERID);
2201 }
2202 
2203 void
2204 hat_memload_region(struct hat *hat, caddr_t addr, struct page *pp,
2205         uint_t attr, uint_t flags, hat_region_cookie_t rcookie)
2206 {
2207         uint_t rid;
2208         if (rcookie == HAT_INVALID_REGION_COOKIE ||
2209             hat->sfmmu_xhat_provider != NULL) {
2210                 hat_do_memload(hat, addr, pp, attr, flags,
2211                     SFMMU_INVALID_SHMERID);
2212                 return;
2213         }
2214         rid = (uint_t)((uint64_t)rcookie);
2215         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
2216         hat_do_memload(hat, addr, pp, attr, flags, rid);
2217 }
2218 
2219 /*
2220  * Set up addr to map to page pp with protection prot.
2221  * As an optimization we also load the TSB with the
2222  * corresponding tte but it is no big deal if  the tte gets kicked out.
2223  */
2224 static void
2225 hat_do_memload(struct hat *hat, caddr_t addr, struct page *pp,
2226         uint_t attr, uint_t flags, uint_t rid)
2227 {
2228         tte_t tte;
2229 
2230 
2231         ASSERT(hat != NULL);
2232         ASSERT(PAGE_LOCKED(pp));
2233         ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET));
2234         ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG));
2235         ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR));
2236         SFMMU_VALIDATE_HMERID(hat, rid, addr, MMU_PAGESIZE);
2237 
2238         if (PP_ISFREE(pp)) {
2239                 panic("hat_memload: loading a mapping to free page %p",
2240                     (void *)pp);
2241         }
2242 
2243         if (hat->sfmmu_xhat_provider) {
2244                 /* no regions for xhats */
2245                 ASSERT(!SFMMU_IS_SHMERID_VALID(rid));
2246                 XHAT_MEMLOAD(hat, addr, pp, attr, flags);
2247                 return;
2248         }
2249 
2250         ASSERT((hat == ksfmmup) ||
2251             AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock));
2252 
2253         if (flags & ~SFMMU_LOAD_ALLFLAG)
2254                 cmn_err(CE_NOTE, "hat_memload: unsupported flags %d",
2255                     flags & ~SFMMU_LOAD_ALLFLAG);
2256 
2257         if (hat->sfmmu_rmstat)
2258                 hat_resvstat(MMU_PAGESIZE, hat->sfmmu_as, addr);
2259 
2260 #if defined(SF_ERRATA_57)
2261         if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) &&
2262             (addr < errata57_limit) && (attr & PROT_EXEC) &&
2263             !(flags & HAT_LOAD_SHARE)) {
2264                 cmn_err(CE_WARN, "hat_memload: illegal attempt to make user "
2265                     " page executable");
2266                 attr &= ~PROT_EXEC;
2267         }
2268 #endif
2269 
2270         sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K);
2271         (void) sfmmu_tteload_array(hat, &tte, addr, &pp, flags, rid);
2272 
2273         /*
2274          * Check TSB and TLB page sizes.
2275          */
2276         if ((flags & HAT_LOAD_SHARE) == 0) {
2277                 sfmmu_check_page_sizes(hat, 1);
2278         }
2279 }
2280 
2281 /*
2282  * hat_devload can be called to map real memory (e.g.
2283  * /dev/kmem) and even though hat_devload will determine pf is
2284  * for memory, it will be unable to get a shared lock on the
2285  * page (because someone else has it exclusively) and will
2286  * pass dp = NULL.  If tteload doesn't get a non-NULL
2287  * page pointer it can't cache memory.
2288  */
2289 void
2290 hat_devload(struct hat *hat, caddr_t addr, size_t len, pfn_t pfn,
2291         uint_t attr, int flags)
2292 {
2293         tte_t tte;
2294         struct page *pp = NULL;
2295         int use_lgpg = 0;
2296 
2297         ASSERT(hat != NULL);
2298 
2299         if (hat->sfmmu_xhat_provider) {
2300                 XHAT_DEVLOAD(hat, addr, len, pfn, attr, flags);
2301                 return;
2302         }
2303 
2304         ASSERT(!(flags & ~SFMMU_LOAD_ALLFLAG));
2305         ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR));
2306         ASSERT((hat == ksfmmup) ||
2307             AS_LOCK_HELD(hat->sfmmu_as, &hat->sfmmu_as->a_lock));
2308         if (len == 0)
2309                 panic("hat_devload: zero len");
2310         if (flags & ~SFMMU_LOAD_ALLFLAG)
2311                 cmn_err(CE_NOTE, "hat_devload: unsupported flags %d",
2312                     flags & ~SFMMU_LOAD_ALLFLAG);
2313 
2314 #if defined(SF_ERRATA_57)
2315         if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) &&
2316             (addr < errata57_limit) && (attr & PROT_EXEC) &&
2317             !(flags & HAT_LOAD_SHARE)) {
2318                 cmn_err(CE_WARN, "hat_devload: illegal attempt to make user "
2319                     " page executable");
2320                 attr &= ~PROT_EXEC;
2321         }
2322 #endif
2323 
2324         /*
2325          * If it's a memory page find its pp
2326          */
2327         if (!(flags & HAT_LOAD_NOCONSIST) && pf_is_memory(pfn)) {
2328                 pp = page_numtopp_nolock(pfn);
2329                 if (pp == NULL) {
2330                         flags |= HAT_LOAD_NOCONSIST;
2331                 } else {
2332                         if (PP_ISFREE(pp)) {
2333                                 panic("hat_memload: loading "
2334                                     "a mapping to free page %p",
2335                                     (void *)pp);
2336                         }
2337                         if (!PAGE_LOCKED(pp) && !PP_ISNORELOC(pp)) {
2338                                 panic("hat_memload: loading a mapping "
2339                                     "to unlocked relocatable page %p",
2340                                     (void *)pp);
2341                         }
2342                         ASSERT(len == MMU_PAGESIZE);
2343                 }
2344         }
2345 
2346         if (hat->sfmmu_rmstat)
2347                 hat_resvstat(len, hat->sfmmu_as, addr);
2348 
2349         if (flags & HAT_LOAD_NOCONSIST) {
2350                 attr |= SFMMU_UNCACHEVTTE;
2351                 use_lgpg = 1;
2352         }
2353         if (!pf_is_memory(pfn)) {
2354                 attr |= SFMMU_UNCACHEPTTE | HAT_NOSYNC;
2355                 use_lgpg = 1;
2356                 switch (attr & HAT_ORDER_MASK) {
2357                         case HAT_STRICTORDER:
2358                         case HAT_UNORDERED_OK:
2359                                 /*
2360                                  * we set the side effect bit for all non
2361                                  * memory mappings unless merging is ok
2362                                  */
2363                                 attr |= SFMMU_SIDEFFECT;
2364                                 break;
2365                         case HAT_MERGING_OK:
2366                         case HAT_LOADCACHING_OK:
2367                         case HAT_STORECACHING_OK:
2368                                 break;
2369                         default:
2370                                 panic("hat_devload: bad attr");
2371                                 break;
2372                 }
2373         }
2374         while (len) {
2375                 if (!use_lgpg) {
2376                         sfmmu_memtte(&tte, pfn, attr, TTE8K);
2377                         (void) sfmmu_tteload_array(hat, &tte, addr, &pp,
2378                             flags, SFMMU_INVALID_SHMERID);
2379                         len -= MMU_PAGESIZE;
2380                         addr += MMU_PAGESIZE;
2381                         pfn++;
2382                         continue;
2383                 }
2384                 /*
2385                  *  try to use large pages, check va/pa alignments
2386                  *  Note that 32M/256M page sizes are not (yet) supported.
2387                  */
2388                 if ((len >= MMU_PAGESIZE4M) &&
2389                     !((uintptr_t)addr & MMU_PAGEOFFSET4M) &&
2390                     !(disable_large_pages & (1 << TTE4M)) &&
2391                     !(mmu_ptob(pfn) & MMU_PAGEOFFSET4M)) {
2392                         sfmmu_memtte(&tte, pfn, attr, TTE4M);
2393                         (void) sfmmu_tteload_array(hat, &tte, addr, &pp,
2394                             flags, SFMMU_INVALID_SHMERID);
2395                         len -= MMU_PAGESIZE4M;
2396                         addr += MMU_PAGESIZE4M;
2397                         pfn += MMU_PAGESIZE4M / MMU_PAGESIZE;
2398                 } else if ((len >= MMU_PAGESIZE512K) &&
2399                     !((uintptr_t)addr & MMU_PAGEOFFSET512K) &&
2400                     !(disable_large_pages & (1 << TTE512K)) &&
2401                     !(mmu_ptob(pfn) & MMU_PAGEOFFSET512K)) {
2402                         sfmmu_memtte(&tte, pfn, attr, TTE512K);
2403                         (void) sfmmu_tteload_array(hat, &tte, addr, &pp,
2404                             flags, SFMMU_INVALID_SHMERID);
2405                         len -= MMU_PAGESIZE512K;
2406                         addr += MMU_PAGESIZE512K;
2407                         pfn += MMU_PAGESIZE512K / MMU_PAGESIZE;
2408                 } else if ((len >= MMU_PAGESIZE64K) &&
2409                     !((uintptr_t)addr & MMU_PAGEOFFSET64K) &&
2410                     !(disable_large_pages & (1 << TTE64K)) &&
2411                     !(mmu_ptob(pfn) & MMU_PAGEOFFSET64K)) {
2412                         sfmmu_memtte(&tte, pfn, attr, TTE64K);
2413                         (void) sfmmu_tteload_array(hat, &tte, addr, &pp,
2414                             flags, SFMMU_INVALID_SHMERID);
2415                         len -= MMU_PAGESIZE64K;
2416                         addr += MMU_PAGESIZE64K;
2417                         pfn += MMU_PAGESIZE64K / MMU_PAGESIZE;
2418                 } else {
2419                         sfmmu_memtte(&tte, pfn, attr, TTE8K);
2420                         (void) sfmmu_tteload_array(hat, &tte, addr, &pp,
2421                             flags, SFMMU_INVALID_SHMERID);
2422                         len -= MMU_PAGESIZE;
2423                         addr += MMU_PAGESIZE;
2424                         pfn++;
2425                 }
2426         }
2427 
2428         /*
2429          * Check TSB and TLB page sizes.
2430          */
2431         if ((flags & HAT_LOAD_SHARE) == 0) {
2432                 sfmmu_check_page_sizes(hat, 1);
2433         }
2434 }
2435 
2436 void
2437 hat_memload_array(struct hat *hat, caddr_t addr, size_t len,
2438         struct page **pps, uint_t attr, uint_t flags)
2439 {
2440         hat_do_memload_array(hat, addr, len, pps, attr, flags,
2441             SFMMU_INVALID_SHMERID);
2442 }
2443 
2444 void
2445 hat_memload_array_region(struct hat *hat, caddr_t addr, size_t len,
2446         struct page **pps, uint_t attr, uint_t flags,
2447         hat_region_cookie_t rcookie)
2448 {
2449         uint_t rid;
2450         if (rcookie == HAT_INVALID_REGION_COOKIE ||
2451             hat->sfmmu_xhat_provider != NULL) {
2452                 hat_do_memload_array(hat, addr, len, pps, attr, flags,
2453                     SFMMU_INVALID_SHMERID);
2454                 return;
2455         }
2456         rid = (uint_t)((uint64_t)rcookie);
2457         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
2458         hat_do_memload_array(hat, addr, len, pps, attr, flags, rid);
2459 }
2460 
2461 /*
2462  * Map the largest extend possible out of the page array. The array may NOT
2463  * be in order.  The largest possible mapping a page can have
2464  * is specified in the p_szc field.  The p_szc field
2465  * cannot change as long as there any mappings (large or small)
2466  * to any of the pages that make up the large page. (ie. any
2467  * promotion/demotion of page size is not up to the hat but up to
2468  * the page free list manager).  The array
2469  * should consist of properly aligned contigous pages that are
2470  * part of a big page for a large mapping to be created.
2471  */
2472 static void
2473 hat_do_memload_array(struct hat *hat, caddr_t addr, size_t len,
2474         struct page **pps, uint_t attr, uint_t flags, uint_t rid)
2475 {
2476         int  ttesz;
2477         size_t mapsz;
2478         pgcnt_t numpg, npgs;
2479         tte_t tte;
2480         page_t *pp;
2481         uint_t large_pages_disable;
2482 
2483         ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET));
2484         SFMMU_VALIDATE_HMERID(hat, rid, addr, len);
2485 
2486         if (hat->sfmmu_xhat_provider) {
2487                 ASSERT(!SFMMU_IS_SHMERID_VALID(rid));
2488                 XHAT_MEMLOAD_ARRAY(hat, addr, len, pps, attr, flags);
2489                 return;
2490         }
2491 
2492         if (hat->sfmmu_rmstat)
2493                 hat_resvstat(len, hat->sfmmu_as, addr);
2494 
2495 #if defined(SF_ERRATA_57)
2496         if ((hat != ksfmmup) && AS_TYPE_64BIT(hat->sfmmu_as) &&
2497             (addr < errata57_limit) && (attr & PROT_EXEC) &&
2498             !(flags & HAT_LOAD_SHARE)) {
2499                 cmn_err(CE_WARN, "hat_memload_array: illegal attempt to make "
2500                     "user page executable");
2501                 attr &= ~PROT_EXEC;
2502         }
2503 #endif
2504 
2505         /* Get number of pages */
2506         npgs = len >> MMU_PAGESHIFT;
2507 
2508         if (flags & HAT_LOAD_SHARE) {
2509                 large_pages_disable = disable_ism_large_pages;
2510         } else {
2511                 large_pages_disable = disable_large_pages;
2512         }
2513 
2514         if (npgs < NHMENTS || large_pages_disable == LARGE_PAGES_OFF) {
2515                 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs,
2516                     rid);
2517                 return;
2518         }
2519 
2520         while (npgs >= NHMENTS) {
2521                 pp = *pps;
2522                 for (ttesz = pp->p_szc; ttesz != TTE8K; ttesz--) {
2523                         /*
2524                          * Check if this page size is disabled.
2525                          */
2526                         if (large_pages_disable & (1 << ttesz))
2527                                 continue;
2528 
2529                         numpg = TTEPAGES(ttesz);
2530                         mapsz = numpg << MMU_PAGESHIFT;
2531                         if ((npgs >= numpg) &&
2532                             IS_P2ALIGNED(addr, mapsz) &&
2533                             IS_P2ALIGNED(pp->p_pagenum, numpg)) {
2534                                 /*
2535                                  * At this point we have enough pages and
2536                                  * we know the virtual address and the pfn
2537                                  * are properly aligned.  We still need
2538                                  * to check for physical contiguity but since
2539                                  * it is very likely that this is the case
2540                                  * we will assume they are so and undo
2541                                  * the request if necessary.  It would
2542                                  * be great if we could get a hint flag
2543                                  * like HAT_CONTIG which would tell us
2544                                  * the pages are contigous for sure.
2545                                  */
2546                                 sfmmu_memtte(&tte, (*pps)->p_pagenum,
2547                                     attr, ttesz);
2548                                 if (!sfmmu_tteload_array(hat, &tte, addr,
2549                                     pps, flags, rid)) {
2550                                         break;
2551                                 }
2552                         }
2553                 }
2554                 if (ttesz == TTE8K) {
2555                         /*
2556                          * We were not able to map array using a large page
2557                          * batch a hmeblk or fraction at a time.
2558                          */
2559                         numpg = ((uintptr_t)addr >> MMU_PAGESHIFT)
2560                             & (NHMENTS-1);
2561                         numpg = NHMENTS - numpg;
2562                         ASSERT(numpg <= npgs);
2563                         mapsz = numpg * MMU_PAGESIZE;
2564                         sfmmu_memload_batchsmall(hat, addr, pps, attr, flags,
2565                             numpg, rid);
2566                 }
2567                 addr += mapsz;
2568                 npgs -= numpg;
2569                 pps += numpg;
2570         }
2571 
2572         if (npgs) {
2573                 sfmmu_memload_batchsmall(hat, addr, pps, attr, flags, npgs,
2574                     rid);
2575         }
2576 
2577         /*
2578          * Check TSB and TLB page sizes.
2579          */
2580         if ((flags & HAT_LOAD_SHARE) == 0) {
2581                 sfmmu_check_page_sizes(hat, 1);
2582         }
2583 }
2584 
2585 /*
2586  * Function tries to batch 8K pages into the same hme blk.
2587  */
2588 static void
2589 sfmmu_memload_batchsmall(struct hat *hat, caddr_t vaddr, page_t **pps,
2590                     uint_t attr, uint_t flags, pgcnt_t npgs, uint_t rid)
2591 {
2592         tte_t   tte;
2593         page_t *pp;
2594         struct hmehash_bucket *hmebp;
2595         struct hme_blk *hmeblkp;
2596         int     index;
2597 
2598         while (npgs) {
2599                 /*
2600                  * Acquire the hash bucket.
2601                  */
2602                 hmebp = sfmmu_tteload_acquire_hashbucket(hat, vaddr, TTE8K,
2603                     rid);
2604                 ASSERT(hmebp);
2605 
2606                 /*
2607                  * Find the hment block.
2608                  */
2609                 hmeblkp = sfmmu_tteload_find_hmeblk(hat, hmebp, vaddr,
2610                     TTE8K, flags, rid);
2611                 ASSERT(hmeblkp);
2612 
2613                 do {
2614                         /*
2615                          * Make the tte.
2616                          */
2617                         pp = *pps;
2618                         sfmmu_memtte(&tte, pp->p_pagenum, attr, TTE8K);
2619 
2620                         /*
2621                          * Add the translation.
2622                          */
2623                         (void) sfmmu_tteload_addentry(hat, hmeblkp, &tte,
2624                             vaddr, pps, flags, rid);
2625 
2626                         /*
2627                          * Goto next page.
2628                          */
2629                         pps++;
2630                         npgs--;
2631 
2632                         /*
2633                          * Goto next address.
2634                          */
2635                         vaddr += MMU_PAGESIZE;
2636 
2637                         /*
2638                          * Don't crossover into a different hmentblk.
2639                          */
2640                         index = (int)(((uintptr_t)vaddr >> MMU_PAGESHIFT) &
2641                             (NHMENTS-1));
2642 
2643                 } while (index != 0 && npgs != 0);
2644 
2645                 /*
2646                  * Release the hash bucket.
2647                  */
2648 
2649                 sfmmu_tteload_release_hashbucket(hmebp);
2650         }
2651 }
2652 
2653 /*
2654  * Construct a tte for a page:
2655  *
2656  * tte_valid = 1
2657  * tte_size2 = size & TTE_SZ2_BITS (Panther and Olympus-C only)
2658  * tte_size = size
2659  * tte_nfo = attr & HAT_NOFAULT
2660  * tte_ie = attr & HAT_STRUCTURE_LE
2661  * tte_hmenum = hmenum
2662  * tte_pahi = pp->p_pagenum >> TTE_PASHIFT;
2663  * tte_palo = pp->p_pagenum & TTE_PALOMASK;
2664  * tte_ref = 1 (optimization)
2665  * tte_wr_perm = attr & PROT_WRITE;
2666  * tte_no_sync = attr & HAT_NOSYNC
2667  * tte_lock = attr & SFMMU_LOCKTTE
2668  * tte_cp = !(attr & SFMMU_UNCACHEPTTE)
2669  * tte_cv = !(attr & SFMMU_UNCACHEVTTE)
2670  * tte_e = attr & SFMMU_SIDEFFECT
2671  * tte_priv = !(attr & PROT_USER)
2672  * tte_hwwr = if nosync is set and it is writable we set the mod bit (opt)
2673  * tte_glb = 0
2674  */
2675 void
2676 sfmmu_memtte(tte_t *ttep, pfn_t pfn, uint_t attr, int tte_sz)
2677 {
2678         ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR));
2679 
2680         ttep->tte_inthi = MAKE_TTE_INTHI(pfn, attr, tte_sz, 0 /* hmenum */);
2681         ttep->tte_intlo = MAKE_TTE_INTLO(pfn, attr, tte_sz, 0 /* hmenum */);
2682 
2683         if (TTE_IS_NOSYNC(ttep)) {
2684                 TTE_SET_REF(ttep);
2685                 if (TTE_IS_WRITABLE(ttep)) {
2686                         TTE_SET_MOD(ttep);
2687                 }
2688         }
2689         if (TTE_IS_NFO(ttep) && TTE_IS_EXECUTABLE(ttep)) {
2690                 panic("sfmmu_memtte: can't set both NFO and EXEC bits");
2691         }
2692 }
2693 
2694 /*
2695  * This function will add a translation to the hme_blk and allocate the
2696  * hme_blk if one does not exist.
2697  * If a page structure is specified then it will add the
2698  * corresponding hment to the mapping list.
2699  * It will also update the hmenum field for the tte.
2700  *
2701  * Currently this function is only used for kernel mappings.
2702  * So pass invalid region to sfmmu_tteload_array().
2703  */
2704 void
2705 sfmmu_tteload(struct hat *sfmmup, tte_t *ttep, caddr_t vaddr, page_t *pp,
2706         uint_t flags)
2707 {
2708         ASSERT(sfmmup == ksfmmup);
2709         (void) sfmmu_tteload_array(sfmmup, ttep, vaddr, &pp, flags,
2710             SFMMU_INVALID_SHMERID);
2711 }
2712 
2713 /*
2714  * Load (ttep != NULL) or unload (ttep == NULL) one entry in the TSB.
2715  * Assumes that a particular page size may only be resident in one TSB.
2716  */
2717 static void
2718 sfmmu_mod_tsb(sfmmu_t *sfmmup, caddr_t vaddr, tte_t *ttep, int ttesz)
2719 {
2720         struct tsb_info *tsbinfop = NULL;
2721         uint64_t tag;
2722         struct tsbe *tsbe_addr;
2723         uint64_t tsb_base;
2724         uint_t tsb_size;
2725         int vpshift = MMU_PAGESHIFT;
2726         int phys = 0;
2727 
2728         if (sfmmup == ksfmmup) { /* No support for 32/256M ksfmmu pages */
2729                 phys = ktsb_phys;
2730                 if (ttesz >= TTE4M) {
2731 #ifndef sun4v
2732                         ASSERT((ttesz != TTE32M) && (ttesz != TTE256M));
2733 #endif
2734                         tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base;
2735                         tsb_size = ktsb4m_szcode;
2736                 } else {
2737                         tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base;
2738                         tsb_size = ktsb_szcode;
2739                 }
2740         } else {
2741                 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz);
2742 
2743                 /*
2744                  * If there isn't a TSB for this page size, or the TSB is
2745                  * swapped out, there is nothing to do.  Note that the latter
2746                  * case seems impossible but can occur if hat_pageunload()
2747                  * is called on an ISM mapping while the process is swapped
2748                  * out.
2749                  */
2750                 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED))
2751                         return;
2752 
2753                 /*
2754                  * If another thread is in the middle of relocating a TSB
2755                  * we can't unload the entry so set a flag so that the
2756                  * TSB will be flushed before it can be accessed by the
2757                  * process.
2758                  */
2759                 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) {
2760                         if (ttep == NULL)
2761                                 tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED;
2762                         return;
2763                 }
2764 #if defined(UTSB_PHYS)
2765                 phys = 1;
2766                 tsb_base = (uint64_t)tsbinfop->tsb_pa;
2767 #else
2768                 tsb_base = (uint64_t)tsbinfop->tsb_va;
2769 #endif
2770                 tsb_size = tsbinfop->tsb_szc;
2771         }
2772         if (ttesz >= TTE4M)
2773                 vpshift = MMU_PAGESHIFT4M;
2774 
2775         tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size);
2776         tag = sfmmu_make_tsbtag(vaddr);
2777 
2778         if (ttep == NULL) {
2779                 sfmmu_unload_tsbe(tsbe_addr, tag, phys);
2780         } else {
2781                 if (ttesz >= TTE4M) {
2782                         SFMMU_STAT(sf_tsb_load4m);
2783                 } else {
2784                         SFMMU_STAT(sf_tsb_load8k);
2785                 }
2786 
2787                 sfmmu_load_tsbe(tsbe_addr, tag, ttep, phys);
2788         }
2789 }
2790 
2791 /*
2792  * Unmap all entries from [start, end) matching the given page size.
2793  *
2794  * This function is used primarily to unmap replicated 64K or 512K entries
2795  * from the TSB that are inserted using the base page size TSB pointer, but
2796  * it may also be called to unmap a range of addresses from the TSB.
2797  */
2798 void
2799 sfmmu_unload_tsb_range(sfmmu_t *sfmmup, caddr_t start, caddr_t end, int ttesz)
2800 {
2801         struct tsb_info *tsbinfop;
2802         uint64_t tag;
2803         struct tsbe *tsbe_addr;
2804         caddr_t vaddr;
2805         uint64_t tsb_base;
2806         int vpshift, vpgsz;
2807         uint_t tsb_size;
2808         int phys = 0;
2809 
2810         /*
2811          * Assumptions:
2812          *  If ttesz == 8K, 64K or 512K, we walk through the range 8K
2813          *  at a time shooting down any valid entries we encounter.
2814          *
2815          *  If ttesz >= 4M we walk the range 4M at a time shooting
2816          *  down any valid mappings we find.
2817          */
2818         if (sfmmup == ksfmmup) {
2819                 phys = ktsb_phys;
2820                 if (ttesz >= TTE4M) {
2821 #ifndef sun4v
2822                         ASSERT((ttesz != TTE32M) && (ttesz != TTE256M));
2823 #endif
2824                         tsb_base = (phys)? ktsb4m_pbase : (uint64_t)ktsb4m_base;
2825                         tsb_size = ktsb4m_szcode;
2826                 } else {
2827                         tsb_base = (phys)? ktsb_pbase : (uint64_t)ktsb_base;
2828                         tsb_size = ktsb_szcode;
2829                 }
2830         } else {
2831                 SFMMU_GET_TSBINFO(tsbinfop, sfmmup, ttesz);
2832 
2833                 /*
2834                  * If there isn't a TSB for this page size, or the TSB is
2835                  * swapped out, there is nothing to do.  Note that the latter
2836                  * case seems impossible but can occur if hat_pageunload()
2837                  * is called on an ISM mapping while the process is swapped
2838                  * out.
2839                  */
2840                 if (tsbinfop == NULL || (tsbinfop->tsb_flags & TSB_SWAPPED))
2841                         return;
2842 
2843                 /*
2844                  * If another thread is in the middle of relocating a TSB
2845                  * we can't unload the entry so set a flag so that the
2846                  * TSB will be flushed before it can be accessed by the
2847                  * process.
2848                  */
2849                 if ((tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) {
2850                         tsbinfop->tsb_flags |= TSB_FLUSH_NEEDED;
2851                         return;
2852                 }
2853 #if defined(UTSB_PHYS)
2854                 phys = 1;
2855                 tsb_base = (uint64_t)tsbinfop->tsb_pa;
2856 #else
2857                 tsb_base = (uint64_t)tsbinfop->tsb_va;
2858 #endif
2859                 tsb_size = tsbinfop->tsb_szc;
2860         }
2861         if (ttesz >= TTE4M) {
2862                 vpshift = MMU_PAGESHIFT4M;
2863                 vpgsz = MMU_PAGESIZE4M;
2864         } else {
2865                 vpshift = MMU_PAGESHIFT;
2866                 vpgsz = MMU_PAGESIZE;
2867         }
2868 
2869         for (vaddr = start; vaddr < end; vaddr += vpgsz) {
2870                 tag = sfmmu_make_tsbtag(vaddr);
2871                 tsbe_addr = sfmmu_get_tsbe(tsb_base, vaddr, vpshift, tsb_size);
2872                 sfmmu_unload_tsbe(tsbe_addr, tag, phys);
2873         }
2874 }
2875 
2876 /*
2877  * Select the optimum TSB size given the number of mappings
2878  * that need to be cached.
2879  */
2880 static int
2881 sfmmu_select_tsb_szc(pgcnt_t pgcnt)
2882 {
2883         int szc = 0;
2884 
2885 #ifdef DEBUG
2886         if (tsb_grow_stress) {
2887                 uint32_t randval = (uint32_t)gettick() >> 4;
2888                 return (randval % (tsb_max_growsize + 1));
2889         }
2890 #endif  /* DEBUG */
2891 
2892         while ((szc < tsb_max_growsize) && (pgcnt > SFMMU_RSS_TSBSIZE(szc)))
2893                 szc++;
2894         return (szc);
2895 }
2896 
2897 /*
2898  * This function will add a translation to the hme_blk and allocate the
2899  * hme_blk if one does not exist.
2900  * If a page structure is specified then it will add the
2901  * corresponding hment to the mapping list.
2902  * It will also update the hmenum field for the tte.
2903  * Furthermore, it attempts to create a large page translation
2904  * for <addr,hat> at page array pps.  It assumes addr and first
2905  * pp is correctly aligned.  It returns 0 if successful and 1 otherwise.
2906  */
2907 static int
2908 sfmmu_tteload_array(sfmmu_t *sfmmup, tte_t *ttep, caddr_t vaddr,
2909         page_t **pps, uint_t flags, uint_t rid)
2910 {
2911         struct hmehash_bucket *hmebp;
2912         struct hme_blk *hmeblkp;
2913         int     ret;
2914         uint_t  size;
2915 
2916         /*
2917          * Get mapping size.
2918          */
2919         size = TTE_CSZ(ttep);
2920         ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size)));
2921 
2922         /*
2923          * Acquire the hash bucket.
2924          */
2925         hmebp = sfmmu_tteload_acquire_hashbucket(sfmmup, vaddr, size, rid);
2926         ASSERT(hmebp);
2927 
2928         /*
2929          * Find the hment block.
2930          */
2931         hmeblkp = sfmmu_tteload_find_hmeblk(sfmmup, hmebp, vaddr, size, flags,
2932             rid);
2933         ASSERT(hmeblkp);
2934 
2935         /*
2936          * Add the translation.
2937          */
2938         ret = sfmmu_tteload_addentry(sfmmup, hmeblkp, ttep, vaddr, pps, flags,
2939             rid);
2940 
2941         /*
2942          * Release the hash bucket.
2943          */
2944         sfmmu_tteload_release_hashbucket(hmebp);
2945 
2946         return (ret);
2947 }
2948 
2949 /*
2950  * Function locks and returns a pointer to the hash bucket for vaddr and size.
2951  */
2952 static struct hmehash_bucket *
2953 sfmmu_tteload_acquire_hashbucket(sfmmu_t *sfmmup, caddr_t vaddr, int size,
2954     uint_t rid)
2955 {
2956         struct hmehash_bucket *hmebp;
2957         int hmeshift;
2958         void *htagid = sfmmutohtagid(sfmmup, rid);
2959 
2960         ASSERT(htagid != NULL);
2961 
2962         hmeshift = HME_HASH_SHIFT(size);
2963 
2964         hmebp = HME_HASH_FUNCTION(htagid, vaddr, hmeshift);
2965 
2966         SFMMU_HASH_LOCK(hmebp);
2967 
2968         return (hmebp);
2969 }
2970 
2971 /*
2972  * Function returns a pointer to an hmeblk in the hash bucket, hmebp. If the
2973  * hmeblk doesn't exists for the [sfmmup, vaddr & size] signature, a hmeblk is
2974  * allocated.
2975  */
2976 static struct hme_blk *
2977 sfmmu_tteload_find_hmeblk(sfmmu_t *sfmmup, struct hmehash_bucket *hmebp,
2978         caddr_t vaddr, uint_t size, uint_t flags, uint_t rid)
2979 {
2980         hmeblk_tag hblktag;
2981         int hmeshift;
2982         struct hme_blk *hmeblkp, *pr_hblk, *list = NULL;
2983 
2984         SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
2985 
2986         hblktag.htag_id = sfmmutohtagid(sfmmup, rid);
2987         ASSERT(hblktag.htag_id != NULL);
2988         hmeshift = HME_HASH_SHIFT(size);
2989         hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift);
2990         hblktag.htag_rehash = HME_HASH_REHASH(size);
2991         hblktag.htag_rid = rid;
2992 
2993 ttearray_realloc:
2994 
2995         HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
2996 
2997         /*
2998          * We block until hblk_reserve_lock is released; it's held by
2999          * the thread, temporarily using hblk_reserve, until hblk_reserve is
3000          * replaced by a hblk from sfmmu8_cache.
3001          */
3002         if (hmeblkp == (struct hme_blk *)hblk_reserve &&
3003             hblk_reserve_thread != curthread) {
3004                 SFMMU_HASH_UNLOCK(hmebp);
3005                 mutex_enter(&hblk_reserve_lock);
3006                 mutex_exit(&hblk_reserve_lock);
3007                 SFMMU_STAT(sf_hblk_reserve_hit);
3008                 SFMMU_HASH_LOCK(hmebp);
3009                 goto ttearray_realloc;
3010         }
3011 
3012         if (hmeblkp == NULL) {
3013                 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size,
3014                     hblktag, flags, rid);
3015                 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared);
3016                 ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared);
3017         } else {
3018                 /*
3019                  * It is possible for 8k and 64k hblks to collide since they
3020                  * have the same rehash value. This is because we
3021                  * lazily free hblks and 8K/64K blks could be lingering.
3022                  * If we find size mismatch we free the block and & try again.
3023                  */
3024                 if (get_hblk_ttesz(hmeblkp) != size) {
3025                         ASSERT(!hmeblkp->hblk_vcnt);
3026                         ASSERT(!hmeblkp->hblk_hmecnt);
3027                         sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
3028                             &list, 0);
3029                         goto ttearray_realloc;
3030                 }
3031                 if (hmeblkp->hblk_shw_bit) {
3032                         /*
3033                          * if the hblk was previously used as a shadow hblk then
3034                          * we will change it to a normal hblk
3035                          */
3036                         ASSERT(!hmeblkp->hblk_shared);
3037                         if (hmeblkp->hblk_shw_mask) {
3038                                 sfmmu_shadow_hcleanup(sfmmup, hmeblkp, hmebp);
3039                                 ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
3040                                 goto ttearray_realloc;
3041                         } else {
3042                                 hmeblkp->hblk_shw_bit = 0;
3043                         }
3044                 }
3045                 SFMMU_STAT(sf_hblk_hit);
3046         }
3047 
3048         /*
3049          * hat_memload() should never call kmem_cache_free() for kernel hmeblks;
3050          * see block comment showing the stacktrace in sfmmu_hblk_alloc();
3051          * set the flag parameter to 1 so that sfmmu_hblks_list_purge() will
3052          * just add these hmeblks to the per-cpu pending queue.
3053          */
3054         sfmmu_hblks_list_purge(&list, 1);
3055 
3056         ASSERT(get_hblk_ttesz(hmeblkp) == size);
3057         ASSERT(!hmeblkp->hblk_shw_bit);
3058         ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared);
3059         ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared);
3060         ASSERT(hmeblkp->hblk_tag.htag_rid == rid);
3061 
3062         return (hmeblkp);
3063 }
3064 
3065 /*
3066  * Function adds a tte entry into the hmeblk. It returns 0 if successful and 1
3067  * otherwise.
3068  */
3069 static int
3070 sfmmu_tteload_addentry(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, tte_t *ttep,
3071         caddr_t vaddr, page_t **pps, uint_t flags, uint_t rid)
3072 {
3073         page_t *pp = *pps;
3074         int hmenum, size, remap;
3075         tte_t tteold, flush_tte;
3076 #ifdef DEBUG
3077         tte_t orig_old;
3078 #endif /* DEBUG */
3079         struct sf_hment *sfhme;
3080         kmutex_t *pml, *pmtx;
3081         hatlock_t *hatlockp;
3082         int myflt;
3083 
3084         /*
3085          * remove this panic when we decide to let user virtual address
3086          * space be >= USERLIMIT.
3087          */
3088         if (!TTE_IS_PRIVILEGED(ttep) && vaddr >= (caddr_t)USERLIMIT)
3089                 panic("user addr %p in kernel space", (void *)vaddr);
3090 #if defined(TTE_IS_GLOBAL)
3091         if (TTE_IS_GLOBAL(ttep))
3092                 panic("sfmmu_tteload: creating global tte");
3093 #endif
3094 
3095 #ifdef DEBUG
3096         if (pf_is_memory(sfmmu_ttetopfn(ttep, vaddr)) &&
3097             !TTE_IS_PCACHEABLE(ttep) && !sfmmu_allow_nc_trans)
3098                 panic("sfmmu_tteload: non cacheable memory tte");
3099 #endif /* DEBUG */
3100 
3101         /* don't simulate dirty bit for writeable ISM/DISM mappings */
3102         if ((flags & HAT_LOAD_SHARE) && TTE_IS_WRITABLE(ttep)) {
3103                 TTE_SET_REF(ttep);
3104                 TTE_SET_MOD(ttep);
3105         }
3106 
3107         if ((flags & HAT_LOAD_SHARE) || !TTE_IS_REF(ttep) ||
3108             !TTE_IS_MOD(ttep)) {
3109                 /*
3110                  * Don't load TSB for dummy as in ISM.  Also don't preload
3111                  * the TSB if the TTE isn't writable since we're likely to
3112                  * fault on it again -- preloading can be fairly expensive.
3113                  */
3114                 flags |= SFMMU_NO_TSBLOAD;
3115         }
3116 
3117         size = TTE_CSZ(ttep);
3118         switch (size) {
3119         case TTE8K:
3120                 SFMMU_STAT(sf_tteload8k);
3121                 break;
3122         case TTE64K:
3123                 SFMMU_STAT(sf_tteload64k);
3124                 break;
3125         case TTE512K:
3126                 SFMMU_STAT(sf_tteload512k);
3127                 break;
3128         case TTE4M:
3129                 SFMMU_STAT(sf_tteload4m);
3130                 break;
3131         case (TTE32M):
3132                 SFMMU_STAT(sf_tteload32m);
3133                 ASSERT(mmu_page_sizes == max_mmu_page_sizes);
3134                 break;
3135         case (TTE256M):
3136                 SFMMU_STAT(sf_tteload256m);
3137                 ASSERT(mmu_page_sizes == max_mmu_page_sizes);
3138                 break;
3139         }
3140 
3141         ASSERT(!((uintptr_t)vaddr & TTE_PAGE_OFFSET(size)));
3142         SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
3143         ASSERT(!SFMMU_IS_SHMERID_VALID(rid) || hmeblkp->hblk_shared);
3144         ASSERT(SFMMU_IS_SHMERID_VALID(rid) || !hmeblkp->hblk_shared);
3145 
3146         HBLKTOHME_IDX(sfhme, hmeblkp, vaddr, hmenum);
3147 
3148         /*
3149          * Need to grab mlist lock here so that pageunload
3150          * will not change tte behind us.
3151          */
3152         if (pp) {
3153                 pml = sfmmu_mlist_enter(pp);
3154         }
3155 
3156         sfmmu_copytte(&sfhme->hme_tte, &tteold);
3157         /*
3158          * Look for corresponding hment and if valid verify
3159          * pfns are equal.
3160          */
3161         remap = TTE_IS_VALID(&tteold);
3162         if (remap) {
3163                 pfn_t   new_pfn, old_pfn;
3164 
3165                 old_pfn = TTE_TO_PFN(vaddr, &tteold);
3166                 new_pfn = TTE_TO_PFN(vaddr, ttep);
3167 
3168                 if (flags & HAT_LOAD_REMAP) {
3169                         /* make sure we are remapping same type of pages */
3170                         if (pf_is_memory(old_pfn) != pf_is_memory(new_pfn)) {
3171                                 panic("sfmmu_tteload - tte remap io<->memory");
3172                         }
3173                         if (old_pfn != new_pfn &&
3174                             (pp != NULL || sfhme->hme_page != NULL)) {
3175                                 panic("sfmmu_tteload - tte remap pp != NULL");
3176                         }
3177                 } else if (old_pfn != new_pfn) {
3178                         panic("sfmmu_tteload - tte remap, hmeblkp 0x%p",
3179                             (void *)hmeblkp);
3180                 }
3181                 ASSERT(TTE_CSZ(&tteold) == TTE_CSZ(ttep));
3182         }
3183 
3184         if (pp) {
3185                 if (size == TTE8K) {
3186 #ifdef VAC
3187                         /*
3188                          * Handle VAC consistency
3189                          */
3190                         if (!remap && (cache & CACHE_VAC) && !PP_ISNC(pp)) {
3191                                 sfmmu_vac_conflict(sfmmup, vaddr, pp);
3192                         }
3193 #endif
3194 
3195                         if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) {
3196                                 pmtx = sfmmu_page_enter(pp);
3197                                 PP_CLRRO(pp);
3198                                 sfmmu_page_exit(pmtx);
3199                         } else if (!PP_ISMAPPED(pp) &&
3200                             (!TTE_IS_WRITABLE(ttep)) && !(PP_ISMOD(pp))) {
3201                                 pmtx = sfmmu_page_enter(pp);
3202                                 if (!(PP_ISMOD(pp))) {
3203                                         PP_SETRO(pp);
3204                                 }
3205                                 sfmmu_page_exit(pmtx);
3206                         }
3207 
3208                 } else if (sfmmu_pagearray_setup(vaddr, pps, ttep, remap)) {
3209                         /*
3210                          * sfmmu_pagearray_setup failed so return
3211                          */
3212                         sfmmu_mlist_exit(pml);
3213                         return (1);
3214                 }
3215         }
3216 
3217         /*
3218          * Make sure hment is not on a mapping list.
3219          */
3220         ASSERT(remap || (sfhme->hme_page == NULL));
3221 
3222         /* if it is not a remap then hme->next better be NULL */
3223         ASSERT((!remap) ? sfhme->hme_next == NULL : 1);
3224 
3225         if (flags & HAT_LOAD_LOCK) {
3226                 if ((hmeblkp->hblk_lckcnt + 1) >= MAX_HBLK_LCKCNT) {
3227                         panic("too high lckcnt-hmeblk %p",
3228                             (void *)hmeblkp);
3229                 }
3230                 atomic_add_32(&hmeblkp->hblk_lckcnt, 1);
3231 
3232                 HBLK_STACK_TRACE(hmeblkp, HBLK_LOCK);
3233         }
3234 
3235 #ifdef VAC
3236         if (pp && PP_ISNC(pp)) {
3237                 /*
3238                  * If the physical page is marked to be uncacheable, like
3239                  * by a vac conflict, make sure the new mapping is also
3240                  * uncacheable.
3241                  */
3242                 TTE_CLR_VCACHEABLE(ttep);
3243                 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR);
3244         }
3245 #endif
3246         ttep->tte_hmenum = hmenum;
3247 
3248 #ifdef DEBUG
3249         orig_old = tteold;
3250 #endif /* DEBUG */
3251 
3252         while (sfmmu_modifytte_try(&tteold, ttep, &sfhme->hme_tte) < 0) {
3253                 if ((sfmmup == KHATID) &&
3254                     (flags & (HAT_LOAD_LOCK | HAT_LOAD_REMAP))) {
3255                         sfmmu_copytte(&sfhme->hme_tte, &tteold);
3256                 }
3257 #ifdef DEBUG
3258                 chk_tte(&orig_old, &tteold, ttep, hmeblkp);
3259 #endif /* DEBUG */
3260         }
3261         ASSERT(TTE_IS_VALID(&sfhme->hme_tte));
3262 
3263         if (!TTE_IS_VALID(&tteold)) {
3264 
3265                 atomic_add_16(&hmeblkp->hblk_vcnt, 1);
3266                 if (rid == SFMMU_INVALID_SHMERID) {
3267                         atomic_add_long(&sfmmup->sfmmu_ttecnt[size], 1);
3268                 } else {
3269                         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
3270                         sf_region_t *rgnp = srdp->srd_hmergnp[rid];
3271                         /*
3272                          * We already accounted for region ttecnt's in sfmmu
3273                          * during hat_join_region() processing. Here we
3274                          * only update ttecnt's in region struture.
3275                          */
3276                         atomic_add_long(&rgnp->rgn_ttecnt[size], 1);
3277                 }
3278         }
3279 
3280         myflt = (astosfmmu(curthread->t_procp->p_as) == sfmmup);
3281         if (size > TTE8K && (flags & HAT_LOAD_SHARE) == 0 &&
3282             sfmmup != ksfmmup) {
3283                 uchar_t tteflag = 1 << size;
3284                 if (rid == SFMMU_INVALID_SHMERID) {
3285                         if (!(sfmmup->sfmmu_tteflags & tteflag)) {
3286                                 hatlockp = sfmmu_hat_enter(sfmmup);
3287                                 sfmmup->sfmmu_tteflags |= tteflag;
3288                                 sfmmu_hat_exit(hatlockp);
3289                         }
3290                 } else if (!(sfmmup->sfmmu_rtteflags & tteflag)) {
3291                         hatlockp = sfmmu_hat_enter(sfmmup);
3292                         sfmmup->sfmmu_rtteflags |= tteflag;
3293                         sfmmu_hat_exit(hatlockp);
3294                 }
3295                 /*
3296                  * Update the current CPU tsbmiss area, so the current thread
3297                  * won't need to take the tsbmiss for the new pagesize.
3298                  * The other threads in the process will update their tsb
3299                  * miss area lazily in sfmmu_tsbmiss_exception() when they
3300                  * fail to find the translation for a newly added pagesize.
3301                  */
3302                 if (size > TTE64K && myflt) {
3303                         struct tsbmiss *tsbmp;
3304                         kpreempt_disable();
3305                         tsbmp = &tsbmiss_area[CPU->cpu_id];
3306                         if (rid == SFMMU_INVALID_SHMERID) {
3307                                 if (!(tsbmp->uhat_tteflags & tteflag)) {
3308                                         tsbmp->uhat_tteflags |= tteflag;
3309                                 }
3310                         } else {
3311                                 if (!(tsbmp->uhat_rtteflags & tteflag)) {
3312                                         tsbmp->uhat_rtteflags |= tteflag;
3313                                 }
3314                         }
3315                         kpreempt_enable();
3316                 }
3317         }
3318 
3319         if (size >= TTE4M && (flags & HAT_LOAD_TEXT) &&
3320             !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) {
3321                 hatlockp = sfmmu_hat_enter(sfmmup);
3322                 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG);
3323                 sfmmu_hat_exit(hatlockp);
3324         }
3325 
3326         flush_tte.tte_intlo = (tteold.tte_intlo ^ ttep->tte_intlo) &
3327             hw_tte.tte_intlo;
3328         flush_tte.tte_inthi = (tteold.tte_inthi ^ ttep->tte_inthi) &
3329             hw_tte.tte_inthi;
3330 
3331         if (remap && (flush_tte.tte_inthi || flush_tte.tte_intlo)) {
3332                 /*
3333                  * If remap and new tte differs from old tte we need
3334                  * to sync the mod bit and flush TLB/TSB.  We don't
3335                  * need to sync ref bit because we currently always set
3336                  * ref bit in tteload.
3337                  */
3338                 ASSERT(TTE_IS_REF(ttep));
3339                 if (TTE_IS_MOD(&tteold)) {
3340                         sfmmu_ttesync(sfmmup, vaddr, &tteold, pp);
3341                 }
3342                 /*
3343                  * hwtte bits shouldn't change for SRD hmeblks as long as SRD
3344                  * hmes are only used for read only text. Adding this code for
3345                  * completeness and future use of shared hmeblks with writable
3346                  * mappings of VMODSORT vnodes.
3347                  */
3348                 if (hmeblkp->hblk_shared) {
3349                         cpuset_t cpuset = sfmmu_rgntlb_demap(vaddr,
3350                             sfmmup->sfmmu_srdp->srd_hmergnp[rid], hmeblkp, 1);
3351                         xt_sync(cpuset);
3352                         SFMMU_STAT_ADD(sf_region_remap_demap, 1);
3353                 } else {
3354                         sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 0);
3355                         xt_sync(sfmmup->sfmmu_cpusran);
3356                 }
3357         }
3358 
3359         if ((flags & SFMMU_NO_TSBLOAD) == 0) {
3360                 /*
3361                  * We only preload 8K and 4M mappings into the TSB, since
3362                  * 64K and 512K mappings are replicated and hence don't
3363                  * have a single, unique TSB entry. Ditto for 32M/256M.
3364                  */
3365                 if (size == TTE8K || size == TTE4M) {
3366                         sf_scd_t *scdp;
3367                         hatlockp = sfmmu_hat_enter(sfmmup);
3368                         /*
3369                          * Don't preload private TSB if the mapping is used
3370                          * by the shctx in the SCD.
3371                          */
3372                         scdp = sfmmup->sfmmu_scdp;
3373                         if (rid == SFMMU_INVALID_SHMERID || scdp == NULL ||
3374                             !SF_RGNMAP_TEST(scdp->scd_hmeregion_map, rid)) {
3375                                 sfmmu_load_tsb(sfmmup, vaddr, &sfhme->hme_tte,
3376                                     size);
3377                         }
3378                         sfmmu_hat_exit(hatlockp);
3379                 }
3380         }
3381         if (pp) {
3382                 if (!remap) {
3383                         HME_ADD(sfhme, pp);
3384                         atomic_add_16(&hmeblkp->hblk_hmecnt, 1);
3385                         ASSERT(hmeblkp->hblk_hmecnt > 0);
3386 
3387                         /*
3388                          * Cannot ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS)
3389                          * see pageunload() for comment.
3390                          */
3391                 }
3392                 sfmmu_mlist_exit(pml);
3393         }
3394 
3395         return (0);
3396 }
3397 /*
3398  * Function unlocks hash bucket.
3399  */
3400 static void
3401 sfmmu_tteload_release_hashbucket(struct hmehash_bucket *hmebp)
3402 {
3403         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
3404         SFMMU_HASH_UNLOCK(hmebp);
3405 }
3406 
3407 /*
3408  * function which checks and sets up page array for a large
3409  * translation.  Will set p_vcolor, p_index, p_ro fields.
3410  * Assumes addr and pfnum of first page are properly aligned.
3411  * Will check for physical contiguity. If check fails it return
3412  * non null.
3413  */
3414 static int
3415 sfmmu_pagearray_setup(caddr_t addr, page_t **pps, tte_t *ttep, int remap)
3416 {
3417         int     i, index, ttesz;
3418         pfn_t   pfnum;
3419         pgcnt_t npgs;
3420         page_t *pp, *pp1;
3421         kmutex_t *pmtx;
3422 #ifdef VAC
3423         int osz;
3424         int cflags = 0;
3425         int vac_err = 0;
3426 #endif
3427         int newidx = 0;
3428 
3429         ttesz = TTE_CSZ(ttep);
3430 
3431         ASSERT(ttesz > TTE8K);
3432 
3433         npgs = TTEPAGES(ttesz);
3434         index = PAGESZ_TO_INDEX(ttesz);
3435 
3436         pfnum = (*pps)->p_pagenum;
3437         ASSERT(IS_P2ALIGNED(pfnum, npgs));
3438 
3439         /*
3440          * Save the first pp so we can do HAT_TMPNC at the end.
3441          */
3442         pp1 = *pps;
3443 #ifdef VAC
3444         osz = fnd_mapping_sz(pp1);
3445 #endif
3446 
3447         for (i = 0; i < npgs; i++, pps++) {
3448                 pp = *pps;
3449                 ASSERT(PAGE_LOCKED(pp));
3450                 ASSERT(pp->p_szc >= ttesz);
3451                 ASSERT(pp->p_szc == pp1->p_szc);
3452                 ASSERT(sfmmu_mlist_held(pp));
3453 
3454                 /*
3455                  * XXX is it possible to maintain P_RO on the root only?
3456                  */
3457                 if (TTE_IS_WRITABLE(ttep) && PP_ISRO(pp)) {
3458                         pmtx = sfmmu_page_enter(pp);
3459                         PP_CLRRO(pp);
3460                         sfmmu_page_exit(pmtx);
3461                 } else if (!PP_ISMAPPED(pp) && !TTE_IS_WRITABLE(ttep) &&
3462                     !PP_ISMOD(pp)) {
3463                         pmtx = sfmmu_page_enter(pp);
3464                         if (!(PP_ISMOD(pp))) {
3465                                 PP_SETRO(pp);
3466                         }
3467                         sfmmu_page_exit(pmtx);
3468                 }
3469 
3470                 /*
3471                  * If this is a remap we skip vac & contiguity checks.
3472                  */
3473                 if (remap)
3474                         continue;
3475 
3476                 /*
3477                  * set p_vcolor and detect any vac conflicts.
3478                  */
3479 #ifdef VAC
3480                 if (vac_err == 0) {
3481                         vac_err = sfmmu_vacconflict_array(addr, pp, &cflags);
3482 
3483                 }
3484 #endif
3485 
3486                 /*
3487                  * Save current index in case we need to undo it.
3488                  * Note: "PAGESZ_TO_INDEX(sz)   (1 << (sz))"
3489                  *      "SFMMU_INDEX_SHIFT      6"
3490                  *       "SFMMU_INDEX_MASK      ((1 << SFMMU_INDEX_SHIFT) - 1)"
3491                  *       "PP_MAPINDEX(p_index)  (p_index & SFMMU_INDEX_MASK)"
3492                  *
3493                  * So:  index = PAGESZ_TO_INDEX(ttesz);
3494                  *      if ttesz == 1 then index = 0x2
3495                  *                  2 then index = 0x4
3496                  *                  3 then index = 0x8
3497                  *                  4 then index = 0x10
3498                  *                  5 then index = 0x20
3499                  * The code below checks if it's a new pagesize (ie, newidx)
3500                  * in case we need to take it back out of p_index,
3501                  * and then or's the new index into the existing index.
3502                  */
3503                 if ((PP_MAPINDEX(pp) & index) == 0)
3504                         newidx = 1;
3505                 pp->p_index = (PP_MAPINDEX(pp) | index);
3506 
3507                 /*
3508                  * contiguity check
3509                  */
3510                 if (pp->p_pagenum != pfnum) {
3511                         /*
3512                          * If we fail the contiguity test then
3513                          * the only thing we need to fix is the p_index field.
3514                          * We might get a few extra flushes but since this
3515                          * path is rare that is ok.  The p_ro field will
3516                          * get automatically fixed on the next tteload to
3517                          * the page.  NO TNC bit is set yet.
3518                          */
3519                         while (i >= 0) {
3520                                 pp = *pps;
3521                                 if (newidx)
3522                                         pp->p_index = (PP_MAPINDEX(pp) &
3523                                             ~index);
3524                                 pps--;
3525                                 i--;
3526                         }
3527                         return (1);
3528                 }
3529                 pfnum++;
3530                 addr += MMU_PAGESIZE;
3531         }
3532 
3533 #ifdef VAC
3534         if (vac_err) {
3535                 if (ttesz > osz) {
3536                         /*
3537                          * There are some smaller mappings that causes vac
3538                          * conflicts. Convert all existing small mappings to
3539                          * TNC.
3540                          */
3541                         SFMMU_STAT_ADD(sf_uncache_conflict, npgs);
3542                         sfmmu_page_cache_array(pp1, HAT_TMPNC, CACHE_FLUSH,
3543                             npgs);
3544                 } else {
3545                         /* EMPTY */
3546                         /*
3547                          * If there exists an big page mapping,
3548                          * that means the whole existing big page
3549                          * has TNC setting already. No need to covert to
3550                          * TNC again.
3551                          */
3552                         ASSERT(PP_ISTNC(pp1));
3553                 }
3554         }
3555 #endif  /* VAC */
3556 
3557         return (0);
3558 }
3559 
3560 #ifdef VAC
3561 /*
3562  * Routine that detects vac consistency for a large page. It also
3563  * sets virtual color for all pp's for this big mapping.
3564  */
3565 static int
3566 sfmmu_vacconflict_array(caddr_t addr, page_t *pp, int *cflags)
3567 {
3568         int vcolor, ocolor;
3569 
3570         ASSERT(sfmmu_mlist_held(pp));
3571 
3572         if (PP_ISNC(pp)) {
3573                 return (HAT_TMPNC);
3574         }
3575 
3576         vcolor = addr_to_vcolor(addr);
3577         if (PP_NEWPAGE(pp)) {
3578                 PP_SET_VCOLOR(pp, vcolor);
3579                 return (0);
3580         }
3581 
3582         ocolor = PP_GET_VCOLOR(pp);
3583         if (ocolor == vcolor) {
3584                 return (0);
3585         }
3586 
3587         if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) {
3588                 /*
3589                  * Previous user of page had a differnet color
3590                  * but since there are no current users
3591                  * we just flush the cache and change the color.
3592                  * As an optimization for large pages we flush the
3593                  * entire cache of that color and set a flag.
3594                  */
3595                 SFMMU_STAT(sf_pgcolor_conflict);
3596                 if (!CacheColor_IsFlushed(*cflags, ocolor)) {
3597                         CacheColor_SetFlushed(*cflags, ocolor);
3598                         sfmmu_cache_flushcolor(ocolor, pp->p_pagenum);
3599                 }
3600                 PP_SET_VCOLOR(pp, vcolor);
3601                 return (0);
3602         }
3603 
3604         /*
3605          * We got a real conflict with a current mapping.
3606          * set flags to start unencaching all mappings
3607          * and return failure so we restart looping
3608          * the pp array from the beginning.
3609          */
3610         return (HAT_TMPNC);
3611 }
3612 #endif  /* VAC */
3613 
3614 /*
3615  * creates a large page shadow hmeblk for a tte.
3616  * The purpose of this routine is to allow us to do quick unloads because
3617  * the vm layer can easily pass a very large but sparsely populated range.
3618  */
3619 static struct hme_blk *
3620 sfmmu_shadow_hcreate(sfmmu_t *sfmmup, caddr_t vaddr, int ttesz, uint_t flags)
3621 {
3622         struct hmehash_bucket *hmebp;
3623         hmeblk_tag hblktag;
3624         int hmeshift, size, vshift;
3625         uint_t shw_mask, newshw_mask;
3626         struct hme_blk *hmeblkp;
3627 
3628         ASSERT(sfmmup != KHATID);
3629         if (mmu_page_sizes == max_mmu_page_sizes) {
3630                 ASSERT(ttesz < TTE256M);
3631         } else {
3632                 ASSERT(ttesz < TTE4M);
3633                 ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
3634                 ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
3635         }
3636 
3637         if (ttesz == TTE8K) {
3638                 size = TTE512K;
3639         } else {
3640                 size = ++ttesz;
3641         }
3642 
3643         hblktag.htag_id = sfmmup;
3644         hmeshift = HME_HASH_SHIFT(size);
3645         hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift);
3646         hblktag.htag_rehash = HME_HASH_REHASH(size);
3647         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
3648         hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift);
3649 
3650         SFMMU_HASH_LOCK(hmebp);
3651 
3652         HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp);
3653         ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve);
3654         if (hmeblkp == NULL) {
3655                 hmeblkp = sfmmu_hblk_alloc(sfmmup, vaddr, hmebp, size,
3656                     hblktag, flags, SFMMU_INVALID_SHMERID);
3657         }
3658         ASSERT(hmeblkp);
3659         if (!hmeblkp->hblk_shw_mask) {
3660                 /*
3661                  * if this is a unused hblk it was just allocated or could
3662                  * potentially be a previous large page hblk so we need to
3663                  * set the shadow bit.
3664                  */
3665                 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt);
3666                 hmeblkp->hblk_shw_bit = 1;
3667         } else if (hmeblkp->hblk_shw_bit == 0) {
3668                 panic("sfmmu_shadow_hcreate: shw bit not set in hmeblkp 0x%p",
3669                     (void *)hmeblkp);
3670         }
3671         ASSERT(hmeblkp->hblk_shw_bit == 1);
3672         ASSERT(!hmeblkp->hblk_shared);
3673         vshift = vaddr_to_vshift(hblktag, vaddr, size);
3674         ASSERT(vshift < 8);
3675         /*
3676          * Atomically set shw mask bit
3677          */
3678         do {
3679                 shw_mask = hmeblkp->hblk_shw_mask;
3680                 newshw_mask = shw_mask | (1 << vshift);
3681                 newshw_mask = cas32(&hmeblkp->hblk_shw_mask, shw_mask,
3682                     newshw_mask);
3683         } while (newshw_mask != shw_mask);
3684 
3685         SFMMU_HASH_UNLOCK(hmebp);
3686 
3687         return (hmeblkp);
3688 }
3689 
3690 /*
3691  * This routine cleanup a previous shadow hmeblk and changes it to
3692  * a regular hblk.  This happens rarely but it is possible
3693  * when a process wants to use large pages and there are hblks still
3694  * lying around from the previous as that used these hmeblks.
3695  * The alternative was to cleanup the shadow hblks at unload time
3696  * but since so few user processes actually use large pages, it is
3697  * better to be lazy and cleanup at this time.
3698  */
3699 static void
3700 sfmmu_shadow_hcleanup(sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
3701         struct hmehash_bucket *hmebp)
3702 {
3703         caddr_t addr, endaddr;
3704         int hashno, size;
3705 
3706         ASSERT(hmeblkp->hblk_shw_bit);
3707         ASSERT(!hmeblkp->hblk_shared);
3708 
3709         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
3710 
3711         if (!hmeblkp->hblk_shw_mask) {
3712                 hmeblkp->hblk_shw_bit = 0;
3713                 return;
3714         }
3715         addr = (caddr_t)get_hblk_base(hmeblkp);
3716         endaddr = get_hblk_endaddr(hmeblkp);
3717         size = get_hblk_ttesz(hmeblkp);
3718         hashno = size - 1;
3719         ASSERT(hashno > 0);
3720         SFMMU_HASH_UNLOCK(hmebp);
3721 
3722         sfmmu_free_hblks(sfmmup, addr, endaddr, hashno);
3723 
3724         SFMMU_HASH_LOCK(hmebp);
3725 }
3726 
3727 static void
3728 sfmmu_free_hblks(sfmmu_t *sfmmup, caddr_t addr, caddr_t endaddr,
3729         int hashno)
3730 {
3731         int hmeshift, shadow = 0;
3732         hmeblk_tag hblktag;
3733         struct hmehash_bucket *hmebp;
3734         struct hme_blk *hmeblkp;
3735         struct hme_blk *nx_hblk, *pr_hblk, *list = NULL;
3736 
3737         ASSERT(hashno > 0);
3738         hblktag.htag_id = sfmmup;
3739         hblktag.htag_rehash = hashno;
3740         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
3741 
3742         hmeshift = HME_HASH_SHIFT(hashno);
3743 
3744         while (addr < endaddr) {
3745                 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
3746                 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
3747                 SFMMU_HASH_LOCK(hmebp);
3748                 /* inline HME_HASH_SEARCH */
3749                 hmeblkp = hmebp->hmeblkp;
3750                 pr_hblk = NULL;
3751                 while (hmeblkp) {
3752                         if (HTAGS_EQ(hmeblkp->hblk_tag, hblktag)) {
3753                                 /* found hme_blk */
3754                                 ASSERT(!hmeblkp->hblk_shared);
3755                                 if (hmeblkp->hblk_shw_bit) {
3756                                         if (hmeblkp->hblk_shw_mask) {
3757                                                 shadow = 1;
3758                                                 sfmmu_shadow_hcleanup(sfmmup,
3759                                                     hmeblkp, hmebp);
3760                                                 break;
3761                                         } else {
3762                                                 hmeblkp->hblk_shw_bit = 0;
3763                                         }
3764                                 }
3765 
3766                                 /*
3767                                  * Hblk_hmecnt and hblk_vcnt could be non zero
3768                                  * since hblk_unload() does not gurantee that.
3769                                  *
3770                                  * XXX - this could cause tteload() to spin
3771                                  * where sfmmu_shadow_hcleanup() is called.
3772                                  */
3773                         }
3774 
3775                         nx_hblk = hmeblkp->hblk_next;
3776                         if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
3777                                 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
3778                                     &list, 0);
3779                         } else {
3780                                 pr_hblk = hmeblkp;
3781                         }
3782                         hmeblkp = nx_hblk;
3783                 }
3784 
3785                 SFMMU_HASH_UNLOCK(hmebp);
3786 
3787                 if (shadow) {
3788                         /*
3789                          * We found another shadow hblk so cleaned its
3790                          * children.  We need to go back and cleanup
3791                          * the original hblk so we don't change the
3792                          * addr.
3793                          */
3794                         shadow = 0;
3795                 } else {
3796                         addr = (caddr_t)roundup((uintptr_t)addr + 1,
3797                             (1 << hmeshift));
3798                 }
3799         }
3800         sfmmu_hblks_list_purge(&list, 0);
3801 }
3802 
3803 /*
3804  * This routine's job is to delete stale invalid shared hmeregions hmeblks that
3805  * may still linger on after pageunload.
3806  */
3807 static void
3808 sfmmu_cleanup_rhblk(sf_srd_t *srdp, caddr_t addr, uint_t rid, int ttesz)
3809 {
3810         int hmeshift;
3811         hmeblk_tag hblktag;
3812         struct hmehash_bucket *hmebp;
3813         struct hme_blk *hmeblkp;
3814         struct hme_blk *pr_hblk;
3815         struct hme_blk *list = NULL;
3816 
3817         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
3818         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
3819 
3820         hmeshift = HME_HASH_SHIFT(ttesz);
3821         hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
3822         hblktag.htag_rehash = ttesz;
3823         hblktag.htag_rid = rid;
3824         hblktag.htag_id = srdp;
3825         hmebp = HME_HASH_FUNCTION(srdp, addr, hmeshift);
3826 
3827         SFMMU_HASH_LOCK(hmebp);
3828         HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
3829         if (hmeblkp != NULL) {
3830                 ASSERT(hmeblkp->hblk_shared);
3831                 ASSERT(!hmeblkp->hblk_shw_bit);
3832                 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) {
3833                         panic("sfmmu_cleanup_rhblk: valid hmeblk");
3834                 }
3835                 ASSERT(!hmeblkp->hblk_lckcnt);
3836                 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
3837                     &list, 0);
3838         }
3839         SFMMU_HASH_UNLOCK(hmebp);
3840         sfmmu_hblks_list_purge(&list, 0);
3841 }
3842 
3843 /* ARGSUSED */
3844 static void
3845 sfmmu_rgn_cb_noop(caddr_t saddr, caddr_t eaddr, caddr_t r_saddr,
3846     size_t r_size, void *r_obj, u_offset_t r_objoff)
3847 {
3848 }
3849 
3850 /*
3851  * Searches for an hmeblk which maps addr, then unloads this mapping
3852  * and updates *eaddrp, if the hmeblk is found.
3853  */
3854 static void
3855 sfmmu_unload_hmeregion_va(sf_srd_t *srdp, uint_t rid, caddr_t addr,
3856     caddr_t eaddr, int ttesz, caddr_t *eaddrp)
3857 {
3858         int hmeshift;
3859         hmeblk_tag hblktag;
3860         struct hmehash_bucket *hmebp;
3861         struct hme_blk *hmeblkp;
3862         struct hme_blk *pr_hblk;
3863         struct hme_blk *list = NULL;
3864 
3865         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
3866         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
3867         ASSERT(ttesz >= HBLK_MIN_TTESZ);
3868 
3869         hmeshift = HME_HASH_SHIFT(ttesz);
3870         hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
3871         hblktag.htag_rehash = ttesz;
3872         hblktag.htag_rid = rid;
3873         hblktag.htag_id = srdp;
3874         hmebp = HME_HASH_FUNCTION(srdp, addr, hmeshift);
3875 
3876         SFMMU_HASH_LOCK(hmebp);
3877         HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
3878         if (hmeblkp != NULL) {
3879                 ASSERT(hmeblkp->hblk_shared);
3880                 ASSERT(!hmeblkp->hblk_lckcnt);
3881                 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) {
3882                         *eaddrp = sfmmu_hblk_unload(NULL, hmeblkp, addr,
3883                             eaddr, NULL, HAT_UNLOAD);
3884                         ASSERT(*eaddrp > addr);
3885                 }
3886                 ASSERT(!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt);
3887                 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
3888                     &list, 0);
3889         }
3890         SFMMU_HASH_UNLOCK(hmebp);
3891         sfmmu_hblks_list_purge(&list, 0);
3892 }
3893 
3894 static void
3895 sfmmu_unload_hmeregion(sf_srd_t *srdp, sf_region_t *rgnp)
3896 {
3897         int ttesz = rgnp->rgn_pgszc;
3898         size_t rsz = rgnp->rgn_size;
3899         caddr_t rsaddr = rgnp->rgn_saddr;
3900         caddr_t readdr = rsaddr + rsz;
3901         caddr_t rhsaddr;
3902         caddr_t va;
3903         uint_t rid = rgnp->rgn_id;
3904         caddr_t cbsaddr;
3905         caddr_t cbeaddr;
3906         hat_rgn_cb_func_t rcbfunc;
3907         ulong_t cnt;
3908 
3909         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
3910         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
3911 
3912         ASSERT(IS_P2ALIGNED(rsaddr, TTEBYTES(ttesz)));
3913         ASSERT(IS_P2ALIGNED(rsz, TTEBYTES(ttesz)));
3914         if (ttesz < HBLK_MIN_TTESZ) {
3915                 ttesz = HBLK_MIN_TTESZ;
3916                 rhsaddr = (caddr_t)P2ALIGN((uintptr_t)rsaddr, HBLK_MIN_BYTES);
3917         } else {
3918                 rhsaddr = rsaddr;
3919         }
3920 
3921         if ((rcbfunc = rgnp->rgn_cb_function) == NULL) {
3922                 rcbfunc = sfmmu_rgn_cb_noop;
3923         }
3924 
3925         while (ttesz >= HBLK_MIN_TTESZ) {
3926                 cbsaddr = rsaddr;
3927                 cbeaddr = rsaddr;
3928                 if (!(rgnp->rgn_hmeflags & (1 << ttesz))) {
3929                         ttesz--;
3930                         continue;
3931                 }
3932                 cnt = 0;
3933                 va = rsaddr;
3934                 while (va < readdr) {
3935                         ASSERT(va >= rhsaddr);
3936                         if (va != cbeaddr) {
3937                                 if (cbeaddr != cbsaddr) {
3938                                         ASSERT(cbeaddr > cbsaddr);
3939                                         (*rcbfunc)(cbsaddr, cbeaddr,
3940                                             rsaddr, rsz, rgnp->rgn_obj,
3941                                             rgnp->rgn_objoff);
3942                                 }
3943                                 cbsaddr = va;
3944                                 cbeaddr = va;
3945                         }
3946                         sfmmu_unload_hmeregion_va(srdp, rid, va, readdr,
3947                             ttesz, &cbeaddr);
3948                         cnt++;
3949                         va = rhsaddr + (cnt << TTE_PAGE_SHIFT(ttesz));
3950                 }
3951                 if (cbeaddr != cbsaddr) {
3952                         ASSERT(cbeaddr > cbsaddr);
3953                         (*rcbfunc)(cbsaddr, cbeaddr, rsaddr,
3954                             rsz, rgnp->rgn_obj,
3955                             rgnp->rgn_objoff);
3956                 }
3957                 ttesz--;
3958         }
3959 }
3960 
3961 /*
3962  * Release one hardware address translation lock on the given address range.
3963  */
3964 void
3965 hat_unlock(struct hat *sfmmup, caddr_t addr, size_t len)
3966 {
3967         struct hmehash_bucket *hmebp;
3968         hmeblk_tag hblktag;
3969         int hmeshift, hashno = 1;
3970         struct hme_blk *hmeblkp, *list = NULL;
3971         caddr_t endaddr;
3972 
3973         ASSERT(sfmmup != NULL);
3974         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
3975 
3976         ASSERT((sfmmup == ksfmmup) ||
3977             AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
3978         ASSERT((len & MMU_PAGEOFFSET) == 0);
3979         endaddr = addr + len;
3980         hblktag.htag_id = sfmmup;
3981         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
3982 
3983         /*
3984          * Spitfire supports 4 page sizes.
3985          * Most pages are expected to be of the smallest page size (8K) and
3986          * these will not need to be rehashed. 64K pages also don't need to be
3987          * rehashed because an hmeblk spans 64K of address space. 512K pages
3988          * might need 1 rehash and and 4M pages might need 2 rehashes.
3989          */
3990         while (addr < endaddr) {
3991                 hmeshift = HME_HASH_SHIFT(hashno);
3992                 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
3993                 hblktag.htag_rehash = hashno;
3994                 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
3995 
3996                 SFMMU_HASH_LOCK(hmebp);
3997 
3998                 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list);
3999                 if (hmeblkp != NULL) {
4000                         ASSERT(!hmeblkp->hblk_shared);
4001                         /*
4002                          * If we encounter a shadow hmeblk then
4003                          * we know there are no valid hmeblks mapping
4004                          * this address at this size or larger.
4005                          * Just increment address by the smallest
4006                          * page size.
4007                          */
4008                         if (hmeblkp->hblk_shw_bit) {
4009                                 addr += MMU_PAGESIZE;
4010                         } else {
4011                                 addr = sfmmu_hblk_unlock(hmeblkp, addr,
4012                                     endaddr);
4013                         }
4014                         SFMMU_HASH_UNLOCK(hmebp);
4015                         hashno = 1;
4016                         continue;
4017                 }
4018                 SFMMU_HASH_UNLOCK(hmebp);
4019 
4020                 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
4021                         /*
4022                          * We have traversed the whole list and rehashed
4023                          * if necessary without finding the address to unlock
4024                          * which should never happen.
4025                          */
4026                         panic("sfmmu_unlock: addr not found. "
4027                             "addr %p hat %p", (void *)addr, (void *)sfmmup);
4028                 } else {
4029                         hashno++;
4030                 }
4031         }
4032 
4033         sfmmu_hblks_list_purge(&list, 0);
4034 }
4035 
4036 void
4037 hat_unlock_region(struct hat *sfmmup, caddr_t addr, size_t len,
4038     hat_region_cookie_t rcookie)
4039 {
4040         sf_srd_t *srdp;
4041         sf_region_t *rgnp;
4042         int ttesz;
4043         uint_t rid;
4044         caddr_t eaddr;
4045         caddr_t va;
4046         int hmeshift;
4047         hmeblk_tag hblktag;
4048         struct hmehash_bucket *hmebp;
4049         struct hme_blk *hmeblkp;
4050         struct hme_blk *pr_hblk;
4051         struct hme_blk *list;
4052 
4053         if (rcookie == HAT_INVALID_REGION_COOKIE) {
4054                 hat_unlock(sfmmup, addr, len);
4055                 return;
4056         }
4057 
4058         ASSERT(sfmmup != NULL);
4059         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
4060         ASSERT(sfmmup != ksfmmup);
4061 
4062         srdp = sfmmup->sfmmu_srdp;
4063         rid = (uint_t)((uint64_t)rcookie);
4064         VERIFY3U(rid, <, SFMMU_MAX_HME_REGIONS);
4065         eaddr = addr + len;
4066         va = addr;
4067         list = NULL;
4068         rgnp = srdp->srd_hmergnp[rid];
4069         SFMMU_VALIDATE_HMERID(sfmmup, rid, addr, len);
4070 
4071         ASSERT(IS_P2ALIGNED(addr, TTEBYTES(rgnp->rgn_pgszc)));
4072         ASSERT(IS_P2ALIGNED(len, TTEBYTES(rgnp->rgn_pgszc)));
4073         if (rgnp->rgn_pgszc < HBLK_MIN_TTESZ) {
4074                 ttesz = HBLK_MIN_TTESZ;
4075         } else {
4076                 ttesz = rgnp->rgn_pgszc;
4077         }
4078         while (va < eaddr) {
4079                 while (ttesz < rgnp->rgn_pgszc &&
4080                     IS_P2ALIGNED(va, TTEBYTES(ttesz + 1))) {
4081                         ttesz++;
4082                 }
4083                 while (ttesz >= HBLK_MIN_TTESZ) {
4084                         if (!(rgnp->rgn_hmeflags & (1 << ttesz))) {
4085                                 ttesz--;
4086                                 continue;
4087                         }
4088                         hmeshift = HME_HASH_SHIFT(ttesz);
4089                         hblktag.htag_bspage = HME_HASH_BSPAGE(va, hmeshift);
4090                         hblktag.htag_rehash = ttesz;
4091                         hblktag.htag_rid = rid;
4092                         hblktag.htag_id = srdp;
4093                         hmebp = HME_HASH_FUNCTION(srdp, va, hmeshift);
4094                         SFMMU_HASH_LOCK(hmebp);
4095                         HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk,
4096                             &list);
4097                         if (hmeblkp == NULL) {
4098                                 SFMMU_HASH_UNLOCK(hmebp);
4099                                 ttesz--;
4100                                 continue;
4101                         }
4102                         ASSERT(hmeblkp->hblk_shared);
4103                         va = sfmmu_hblk_unlock(hmeblkp, va, eaddr);
4104                         ASSERT(va >= eaddr ||
4105                             IS_P2ALIGNED((uintptr_t)va, TTEBYTES(ttesz)));
4106                         SFMMU_HASH_UNLOCK(hmebp);
4107                         break;
4108                 }
4109                 if (ttesz < HBLK_MIN_TTESZ) {
4110                         panic("hat_unlock_region: addr not found "
4111                             "addr %p hat %p", (void *)va, (void *)sfmmup);
4112                 }
4113         }
4114         sfmmu_hblks_list_purge(&list, 0);
4115 }
4116 
4117 /*
4118  * Function to unlock a range of addresses in an hmeblk.  It returns the
4119  * next address that needs to be unlocked.
4120  * Should be called with the hash lock held.
4121  */
4122 static caddr_t
4123 sfmmu_hblk_unlock(struct hme_blk *hmeblkp, caddr_t addr, caddr_t endaddr)
4124 {
4125         struct sf_hment *sfhme;
4126         tte_t tteold, ttemod;
4127         int ttesz, ret;
4128 
4129         ASSERT(in_hblk_range(hmeblkp, addr));
4130         ASSERT(hmeblkp->hblk_shw_bit == 0);
4131 
4132         endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp));
4133         ttesz = get_hblk_ttesz(hmeblkp);
4134 
4135         HBLKTOHME(sfhme, hmeblkp, addr);
4136         while (addr < endaddr) {
4137 readtte:
4138                 sfmmu_copytte(&sfhme->hme_tte, &tteold);
4139                 if (TTE_IS_VALID(&tteold)) {
4140 
4141                         ttemod = tteold;
4142 
4143                         ret = sfmmu_modifytte_try(&tteold, &ttemod,
4144                             &sfhme->hme_tte);
4145 
4146                         if (ret < 0)
4147                                 goto readtte;
4148 
4149                         if (hmeblkp->hblk_lckcnt == 0)
4150                                 panic("zero hblk lckcnt");
4151 
4152                         if (((uintptr_t)addr + TTEBYTES(ttesz)) >
4153                             (uintptr_t)endaddr)
4154                                 panic("can't unlock large tte");
4155 
4156                         ASSERT(hmeblkp->hblk_lckcnt > 0);
4157                         atomic_add_32(&hmeblkp->hblk_lckcnt, -1);
4158                         HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK);
4159                 } else {
4160                         panic("sfmmu_hblk_unlock: invalid tte");
4161                 }
4162                 addr += TTEBYTES(ttesz);
4163                 sfhme++;
4164         }
4165         return (addr);
4166 }
4167 
4168 /*
4169  * Physical Address Mapping Framework
4170  *
4171  * General rules:
4172  *
4173  * (1) Applies only to seg_kmem memory pages. To make things easier,
4174  *     seg_kpm addresses are also accepted by the routines, but nothing
4175  *     is done with them since by definition their PA mappings are static.
4176  * (2) hat_add_callback() may only be called while holding the page lock
4177  *     SE_SHARED or SE_EXCL of the underlying page (e.g., as_pagelock()),
4178  *     or passing HAC_PAGELOCK flag.
4179  * (3) prehandler() and posthandler() may not call hat_add_callback() or
4180  *     hat_delete_callback(), nor should they allocate memory. Post quiesce
4181  *     callbacks may not sleep or acquire adaptive mutex locks.
4182  * (4) Either prehandler() or posthandler() (but not both) may be specified
4183  *     as being NULL.  Specifying an errhandler() is optional.
4184  *
4185  * Details of using the framework:
4186  *
4187  * registering a callback (hat_register_callback())
4188  *
4189  *      Pass prehandler, posthandler, errhandler addresses
4190  *      as described below. If capture_cpus argument is nonzero,
4191  *      suspend callback to the prehandler will occur with CPUs
4192  *      captured and executing xc_loop() and CPUs will remain
4193  *      captured until after the posthandler suspend callback
4194  *      occurs.
4195  *
4196  * adding a callback (hat_add_callback())
4197  *
4198  *      as_pagelock();
4199  *      hat_add_callback();
4200  *      save returned pfn in private data structures or program registers;
4201  *      as_pageunlock();
4202  *
4203  * prehandler()
4204  *
4205  *      Stop all accesses by physical address to this memory page.
4206  *      Called twice: the first, PRESUSPEND, is a context safe to acquire
4207  *      adaptive locks. The second, SUSPEND, is called at high PIL with
4208  *      CPUs captured so adaptive locks may NOT be acquired (and all spin
4209  *      locks must be XCALL_PIL or higher locks).
4210  *
4211  *      May return the following errors:
4212  *              EIO:    A fatal error has occurred. This will result in panic.
4213  *              EAGAIN: The page cannot be suspended. This will fail the
4214  *                      relocation.
4215  *              0:      Success.
4216  *
4217  * posthandler()
4218  *
4219  *      Save new pfn in private data structures or program registers;
4220  *      not allowed to fail (non-zero return values will result in panic).
4221  *
4222  * errhandler()
4223  *
4224  *      called when an error occurs related to the callback.  Currently
4225  *      the only such error is HAT_CB_ERR_LEAKED which indicates that
4226  *      a page is being freed, but there are still outstanding callback(s)
4227  *      registered on the page.
4228  *
4229  * removing a callback (hat_delete_callback(); e.g., prior to freeing memory)
4230  *
4231  *      stop using physical address
4232  *      hat_delete_callback();
4233  *
4234  */
4235 
4236 /*
4237  * Register a callback class.  Each subsystem should do this once and
4238  * cache the id_t returned for use in setting up and tearing down callbacks.
4239  *
4240  * There is no facility for removing callback IDs once they are created;
4241  * the "key" should be unique for each module, so in case a module is unloaded
4242  * and subsequently re-loaded, we can recycle the module's previous entry.
4243  */
4244 id_t
4245 hat_register_callback(int key,
4246         int (*prehandler)(caddr_t, uint_t, uint_t, void *),
4247         int (*posthandler)(caddr_t, uint_t, uint_t, void *, pfn_t),
4248         int (*errhandler)(caddr_t, uint_t, uint_t, void *),
4249         int capture_cpus)
4250 {
4251         id_t id;
4252 
4253         /*
4254          * Search the table for a pre-existing callback associated with
4255          * the identifier "key".  If one exists, we re-use that entry in
4256          * the table for this instance, otherwise we assign the next
4257          * available table slot.
4258          */
4259         for (id = 0; id < sfmmu_max_cb_id; id++) {
4260                 if (sfmmu_cb_table[id].key == key)
4261                         break;
4262         }
4263 
4264         if (id == sfmmu_max_cb_id) {
4265                 id = sfmmu_cb_nextid++;
4266                 if (id >= sfmmu_max_cb_id)
4267                         panic("hat_register_callback: out of callback IDs");
4268         }
4269 
4270         ASSERT(prehandler != NULL || posthandler != NULL);
4271 
4272         sfmmu_cb_table[id].key = key;
4273         sfmmu_cb_table[id].prehandler = prehandler;
4274         sfmmu_cb_table[id].posthandler = posthandler;
4275         sfmmu_cb_table[id].errhandler = errhandler;
4276         sfmmu_cb_table[id].capture_cpus = capture_cpus;
4277 
4278         return (id);
4279 }
4280 
4281 #define HAC_COOKIE_NONE (void *)-1
4282 
4283 /*
4284  * Add relocation callbacks to the specified addr/len which will be called
4285  * when relocating the associated page. See the description of pre and
4286  * posthandler above for more details.
4287  *
4288  * If HAC_PAGELOCK is included in flags, the underlying memory page is
4289  * locked internally so the caller must be able to deal with the callback
4290  * running even before this function has returned.  If HAC_PAGELOCK is not
4291  * set, it is assumed that the underlying memory pages are locked.
4292  *
4293  * Since the caller must track the individual page boundaries anyway,
4294  * we only allow a callback to be added to a single page (large
4295  * or small).  Thus [addr, addr + len) MUST be contained within a single
4296  * page.
4297  *
4298  * Registering multiple callbacks on the same [addr, addr+len) is supported,
4299  * _provided_that_ a unique parameter is specified for each callback.
4300  * If multiple callbacks are registered on the same range the callback will
4301  * be invoked with each unique parameter. Registering the same callback with
4302  * the same argument more than once will result in corrupted kernel state.
4303  *
4304  * Returns the pfn of the underlying kernel page in *rpfn
4305  * on success, or PFN_INVALID on failure.
4306  *
4307  * cookiep (if passed) provides storage space for an opaque cookie
4308  * to return later to hat_delete_callback(). This cookie makes the callback
4309  * deletion significantly quicker by avoiding a potentially lengthy hash
4310  * search.
4311  *
4312  * Returns values:
4313  *    0:      success
4314  *    ENOMEM: memory allocation failure (e.g. flags was passed as HAC_NOSLEEP)
4315  *    EINVAL: callback ID is not valid
4316  *    ENXIO:  ["vaddr", "vaddr" + len) is not mapped in the kernel's address
4317  *            space
4318  *    ERANGE: ["vaddr", "vaddr" + len) crosses a page boundary
4319  */
4320 int
4321 hat_add_callback(id_t callback_id, caddr_t vaddr, uint_t len, uint_t flags,
4322         void *pvt, pfn_t *rpfn, void **cookiep)
4323 {
4324         struct          hmehash_bucket *hmebp;
4325         hmeblk_tag      hblktag;
4326         struct hme_blk  *hmeblkp;
4327         int             hmeshift, hashno;
4328         caddr_t         saddr, eaddr, baseaddr;
4329         struct pa_hment *pahmep;
4330         struct sf_hment *sfhmep, *osfhmep;
4331         kmutex_t        *pml;
4332         tte_t           tte;
4333         page_t          *pp;
4334         vnode_t         *vp;
4335         u_offset_t      off;
4336         pfn_t           pfn;
4337         int             kmflags = (flags & HAC_SLEEP)? KM_SLEEP : KM_NOSLEEP;
4338         int             locked = 0;
4339 
4340         /*
4341          * For KPM mappings, just return the physical address since we
4342          * don't need to register any callbacks.
4343          */
4344         if (IS_KPM_ADDR(vaddr)) {
4345                 uint64_t paddr;
4346                 SFMMU_KPM_VTOP(vaddr, paddr);
4347                 *rpfn = btop(paddr);
4348                 if (cookiep != NULL)
4349                         *cookiep = HAC_COOKIE_NONE;
4350                 return (0);
4351         }
4352 
4353         if (callback_id < (id_t)0 || callback_id >= sfmmu_cb_nextid) {
4354                 *rpfn = PFN_INVALID;
4355                 return (EINVAL);
4356         }
4357 
4358         if ((pahmep = kmem_cache_alloc(pa_hment_cache, kmflags)) == NULL) {
4359                 *rpfn = PFN_INVALID;
4360                 return (ENOMEM);
4361         }
4362 
4363         sfhmep = &pahmep->sfment;
4364 
4365         saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK);
4366         eaddr = saddr + len;
4367 
4368 rehash:
4369         /* Find the mapping(s) for this page */
4370         for (hashno = TTE64K, hmeblkp = NULL;
4371             hmeblkp == NULL && hashno <= mmu_hashcnt;
4372             hashno++) {
4373                 hmeshift = HME_HASH_SHIFT(hashno);
4374                 hblktag.htag_id = ksfmmup;
4375                 hblktag.htag_rid = SFMMU_INVALID_SHMERID;
4376                 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift);
4377                 hblktag.htag_rehash = hashno;
4378                 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift);
4379 
4380                 SFMMU_HASH_LOCK(hmebp);
4381 
4382                 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp);
4383 
4384                 if (hmeblkp == NULL)
4385                         SFMMU_HASH_UNLOCK(hmebp);
4386         }
4387 
4388         if (hmeblkp == NULL) {
4389                 kmem_cache_free(pa_hment_cache, pahmep);
4390                 *rpfn = PFN_INVALID;
4391                 return (ENXIO);
4392         }
4393 
4394         ASSERT(!hmeblkp->hblk_shared);
4395 
4396         HBLKTOHME(osfhmep, hmeblkp, saddr);
4397         sfmmu_copytte(&osfhmep->hme_tte, &tte);
4398 
4399         if (!TTE_IS_VALID(&tte)) {
4400                 SFMMU_HASH_UNLOCK(hmebp);
4401                 kmem_cache_free(pa_hment_cache, pahmep);
4402                 *rpfn = PFN_INVALID;
4403                 return (ENXIO);
4404         }
4405 
4406         /*
4407          * Make sure the boundaries for the callback fall within this
4408          * single mapping.
4409          */
4410         baseaddr = (caddr_t)get_hblk_base(hmeblkp);
4411         ASSERT(saddr >= baseaddr);
4412         if (eaddr > saddr + TTEBYTES(TTE_CSZ(&tte))) {
4413                 SFMMU_HASH_UNLOCK(hmebp);
4414                 kmem_cache_free(pa_hment_cache, pahmep);
4415                 *rpfn = PFN_INVALID;
4416                 return (ERANGE);
4417         }
4418 
4419         pfn = sfmmu_ttetopfn(&tte, vaddr);
4420 
4421         /*
4422          * The pfn may not have a page_t underneath in which case we
4423          * just return it. This can happen if we are doing I/O to a
4424          * static portion of the kernel's address space, for instance.
4425          */
4426         pp = osfhmep->hme_page;
4427         if (pp == NULL) {
4428                 SFMMU_HASH_UNLOCK(hmebp);
4429                 kmem_cache_free(pa_hment_cache, pahmep);
4430                 *rpfn = pfn;
4431                 if (cookiep)
4432                         *cookiep = HAC_COOKIE_NONE;
4433                 return (0);
4434         }
4435         ASSERT(pp == PP_PAGEROOT(pp));
4436 
4437         vp = pp->p_vnode;
4438         off = pp->p_offset;
4439 
4440         pml = sfmmu_mlist_enter(pp);
4441 
4442         if (flags & HAC_PAGELOCK) {
4443                 if (!page_trylock(pp, SE_SHARED)) {
4444                         /*
4445                          * Somebody is holding SE_EXCL lock. Might
4446                          * even be hat_page_relocate(). Drop all
4447                          * our locks, lookup the page in &kvp, and
4448                          * retry. If it doesn't exist in &kvp and &zvp,
4449                          * then we must be dealing with a kernel mapped
4450                          * page which doesn't actually belong to
4451                          * segkmem so we punt.
4452                          */
4453                         sfmmu_mlist_exit(pml);
4454                         SFMMU_HASH_UNLOCK(hmebp);
4455                         pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED);
4456 
4457                         /* check zvp before giving up */
4458                         if (pp == NULL)
4459                                 pp = page_lookup(&zvp, (u_offset_t)saddr,
4460                                     SE_SHARED);
4461 
4462                         /* Okay, we didn't find it, give up */
4463                         if (pp == NULL) {
4464                                 kmem_cache_free(pa_hment_cache, pahmep);
4465                                 *rpfn = pfn;
4466                                 if (cookiep)
4467                                         *cookiep = HAC_COOKIE_NONE;
4468                                 return (0);
4469                         }
4470                         page_unlock(pp);
4471                         goto rehash;
4472                 }
4473                 locked = 1;
4474         }
4475 
4476         if (!PAGE_LOCKED(pp) && !panicstr)
4477                 panic("hat_add_callback: page 0x%p not locked", (void *)pp);
4478 
4479         if (osfhmep->hme_page != pp || pp->p_vnode != vp ||
4480             pp->p_offset != off) {
4481                 /*
4482                  * The page moved before we got our hands on it.  Drop
4483                  * all the locks and try again.
4484                  */
4485                 ASSERT((flags & HAC_PAGELOCK) != 0);
4486                 sfmmu_mlist_exit(pml);
4487                 SFMMU_HASH_UNLOCK(hmebp);
4488                 page_unlock(pp);
4489                 locked = 0;
4490                 goto rehash;
4491         }
4492 
4493         if (!VN_ISKAS(vp)) {
4494                 /*
4495                  * This is not a segkmem page but another page which
4496                  * has been kernel mapped. It had better have at least
4497                  * a share lock on it. Return the pfn.
4498                  */
4499                 sfmmu_mlist_exit(pml);
4500                 SFMMU_HASH_UNLOCK(hmebp);
4501                 if (locked)
4502                         page_unlock(pp);
4503                 kmem_cache_free(pa_hment_cache, pahmep);
4504                 ASSERT(PAGE_LOCKED(pp));
4505                 *rpfn = pfn;
4506                 if (cookiep)
4507                         *cookiep = HAC_COOKIE_NONE;
4508                 return (0);
4509         }
4510 
4511         /*
4512          * Setup this pa_hment and link its embedded dummy sf_hment into
4513          * the mapping list.
4514          */
4515         pp->p_share++;
4516         pahmep->cb_id = callback_id;
4517         pahmep->addr = vaddr;
4518         pahmep->len = len;
4519         pahmep->refcnt = 1;
4520         pahmep->flags = 0;
4521         pahmep->pvt = pvt;
4522 
4523         sfhmep->hme_tte.ll = 0;
4524         sfhmep->hme_data = pahmep;
4525         sfhmep->hme_prev = osfhmep;
4526         sfhmep->hme_next = osfhmep->hme_next;
4527 
4528         if (osfhmep->hme_next)
4529                 osfhmep->hme_next->hme_prev = sfhmep;
4530 
4531         osfhmep->hme_next = sfhmep;
4532 
4533         sfmmu_mlist_exit(pml);
4534         SFMMU_HASH_UNLOCK(hmebp);
4535 
4536         if (locked)
4537                 page_unlock(pp);
4538 
4539         *rpfn = pfn;
4540         if (cookiep)
4541                 *cookiep = (void *)pahmep;
4542 
4543         return (0);
4544 }
4545 
4546 /*
4547  * Remove the relocation callbacks from the specified addr/len.
4548  */
4549 void
4550 hat_delete_callback(caddr_t vaddr, uint_t len, void *pvt, uint_t flags,
4551         void *cookie)
4552 {
4553         struct          hmehash_bucket *hmebp;
4554         hmeblk_tag      hblktag;
4555         struct hme_blk  *hmeblkp;
4556         int             hmeshift, hashno;
4557         caddr_t         saddr;
4558         struct pa_hment *pahmep;
4559         struct sf_hment *sfhmep, *osfhmep;
4560         kmutex_t        *pml;
4561         tte_t           tte;
4562         page_t          *pp;
4563         vnode_t         *vp;
4564         u_offset_t      off;
4565         int             locked = 0;
4566 
4567         /*
4568          * If the cookie is HAC_COOKIE_NONE then there is no pa_hment to
4569          * remove so just return.
4570          */
4571         if (cookie == HAC_COOKIE_NONE || IS_KPM_ADDR(vaddr))
4572                 return;
4573 
4574         saddr = (caddr_t)((uintptr_t)vaddr & MMU_PAGEMASK);
4575 
4576 rehash:
4577         /* Find the mapping(s) for this page */
4578         for (hashno = TTE64K, hmeblkp = NULL;
4579             hmeblkp == NULL && hashno <= mmu_hashcnt;
4580             hashno++) {
4581                 hmeshift = HME_HASH_SHIFT(hashno);
4582                 hblktag.htag_id = ksfmmup;
4583                 hblktag.htag_rid = SFMMU_INVALID_SHMERID;
4584                 hblktag.htag_bspage = HME_HASH_BSPAGE(saddr, hmeshift);
4585                 hblktag.htag_rehash = hashno;
4586                 hmebp = HME_HASH_FUNCTION(ksfmmup, saddr, hmeshift);
4587 
4588                 SFMMU_HASH_LOCK(hmebp);
4589 
4590                 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp);
4591 
4592                 if (hmeblkp == NULL)
4593                         SFMMU_HASH_UNLOCK(hmebp);
4594         }
4595 
4596         if (hmeblkp == NULL)
4597                 return;
4598 
4599         ASSERT(!hmeblkp->hblk_shared);
4600 
4601         HBLKTOHME(osfhmep, hmeblkp, saddr);
4602 
4603         sfmmu_copytte(&osfhmep->hme_tte, &tte);
4604         if (!TTE_IS_VALID(&tte)) {
4605                 SFMMU_HASH_UNLOCK(hmebp);
4606                 return;
4607         }
4608 
4609         pp = osfhmep->hme_page;
4610         if (pp == NULL) {
4611                 SFMMU_HASH_UNLOCK(hmebp);
4612                 ASSERT(cookie == NULL);
4613                 return;
4614         }
4615 
4616         vp = pp->p_vnode;
4617         off = pp->p_offset;
4618 
4619         pml = sfmmu_mlist_enter(pp);
4620 
4621         if (flags & HAC_PAGELOCK) {
4622                 if (!page_trylock(pp, SE_SHARED)) {
4623                         /*
4624                          * Somebody is holding SE_EXCL lock. Might
4625                          * even be hat_page_relocate(). Drop all
4626                          * our locks, lookup the page in &kvp, and
4627                          * retry. If it doesn't exist in &kvp and &zvp,
4628                          * then we must be dealing with a kernel mapped
4629                          * page which doesn't actually belong to
4630                          * segkmem so we punt.
4631                          */
4632                         sfmmu_mlist_exit(pml);
4633                         SFMMU_HASH_UNLOCK(hmebp);
4634                         pp = page_lookup(&kvp, (u_offset_t)saddr, SE_SHARED);
4635                         /* check zvp before giving up */
4636                         if (pp == NULL)
4637                                 pp = page_lookup(&zvp, (u_offset_t)saddr,
4638                                     SE_SHARED);
4639 
4640                         if (pp == NULL) {
4641                                 ASSERT(cookie == NULL);
4642                                 return;
4643                         }
4644                         page_unlock(pp);
4645                         goto rehash;
4646                 }
4647                 locked = 1;
4648         }
4649 
4650         ASSERT(PAGE_LOCKED(pp));
4651 
4652         if (osfhmep->hme_page != pp || pp->p_vnode != vp ||
4653             pp->p_offset != off) {
4654                 /*
4655                  * The page moved before we got our hands on it.  Drop
4656                  * all the locks and try again.
4657                  */
4658                 ASSERT((flags & HAC_PAGELOCK) != 0);
4659                 sfmmu_mlist_exit(pml);
4660                 SFMMU_HASH_UNLOCK(hmebp);
4661                 page_unlock(pp);
4662                 locked = 0;
4663                 goto rehash;
4664         }
4665 
4666         if (!VN_ISKAS(vp)) {
4667                 /*
4668                  * This is not a segkmem page but another page which
4669                  * has been kernel mapped.
4670                  */
4671                 sfmmu_mlist_exit(pml);
4672                 SFMMU_HASH_UNLOCK(hmebp);
4673                 if (locked)
4674                         page_unlock(pp);
4675                 ASSERT(cookie == NULL);
4676                 return;
4677         }
4678 
4679         if (cookie != NULL) {
4680                 pahmep = (struct pa_hment *)cookie;
4681                 sfhmep = &pahmep->sfment;
4682         } else {
4683                 for (sfhmep = pp->p_mapping; sfhmep != NULL;
4684                     sfhmep = sfhmep->hme_next) {
4685 
4686                         /*
4687                          * skip va<->pa mappings
4688                          */
4689                         if (!IS_PAHME(sfhmep))
4690                                 continue;
4691 
4692                         pahmep = sfhmep->hme_data;
4693                         ASSERT(pahmep != NULL);
4694 
4695                         /*
4696                          * if pa_hment matches, remove it
4697                          */
4698                         if ((pahmep->pvt == pvt) &&
4699                             (pahmep->addr == vaddr) &&
4700                             (pahmep->len == len)) {
4701                                 break;
4702                         }
4703                 }
4704         }
4705 
4706         if (sfhmep == NULL) {
4707                 if (!panicstr) {
4708                         panic("hat_delete_callback: pa_hment not found, pp %p",
4709                             (void *)pp);
4710                 }
4711                 return;
4712         }
4713 
4714         /*
4715          * Note: at this point a valid kernel mapping must still be
4716          * present on this page.
4717          */
4718         pp->p_share--;
4719         if (pp->p_share <= 0)
4720                 panic("hat_delete_callback: zero p_share");
4721 
4722         if (--pahmep->refcnt == 0) {
4723                 if (pahmep->flags != 0)
4724                         panic("hat_delete_callback: pa_hment is busy");
4725 
4726                 /*
4727                  * Remove sfhmep from the mapping list for the page.
4728                  */
4729                 if (sfhmep->hme_prev) {
4730                         sfhmep->hme_prev->hme_next = sfhmep->hme_next;
4731                 } else {
4732                         pp->p_mapping = sfhmep->hme_next;
4733                 }
4734 
4735                 if (sfhmep->hme_next)
4736                         sfhmep->hme_next->hme_prev = sfhmep->hme_prev;
4737 
4738                 sfmmu_mlist_exit(pml);
4739                 SFMMU_HASH_UNLOCK(hmebp);
4740 
4741                 if (locked)
4742                         page_unlock(pp);
4743 
4744                 kmem_cache_free(pa_hment_cache, pahmep);
4745                 return;
4746         }
4747 
4748         sfmmu_mlist_exit(pml);
4749         SFMMU_HASH_UNLOCK(hmebp);
4750         if (locked)
4751                 page_unlock(pp);
4752 }
4753 
4754 /*
4755  * hat_probe returns 1 if the translation for the address 'addr' is
4756  * loaded, zero otherwise.
4757  *
4758  * hat_probe should be used only for advisorary purposes because it may
4759  * occasionally return the wrong value. The implementation must guarantee that
4760  * returning the wrong value is a very rare event. hat_probe is used
4761  * to implement optimizations in the segment drivers.
4762  *
4763  */
4764 int
4765 hat_probe(struct hat *sfmmup, caddr_t addr)
4766 {
4767         pfn_t pfn;
4768         tte_t tte;
4769 
4770         ASSERT(sfmmup != NULL);
4771         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
4772 
4773         ASSERT((sfmmup == ksfmmup) ||
4774             AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
4775 
4776         if (sfmmup == ksfmmup) {
4777                 while ((pfn = sfmmu_vatopfn(addr, sfmmup, &tte))
4778                     == PFN_SUSPENDED) {
4779                         sfmmu_vatopfn_suspended(addr, sfmmup, &tte);
4780                 }
4781         } else {
4782                 pfn = sfmmu_uvatopfn(addr, sfmmup, NULL);
4783         }
4784 
4785         if (pfn != PFN_INVALID)
4786                 return (1);
4787         else
4788                 return (0);
4789 }
4790 
4791 ssize_t
4792 hat_getpagesize(struct hat *sfmmup, caddr_t addr)
4793 {
4794         tte_t tte;
4795 
4796         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
4797 
4798         if (sfmmup == ksfmmup) {
4799                 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4800                         return (-1);
4801                 }
4802         } else {
4803                 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4804                         return (-1);
4805                 }
4806         }
4807 
4808         ASSERT(TTE_IS_VALID(&tte));
4809         return (TTEBYTES(TTE_CSZ(&tte)));
4810 }
4811 
4812 uint_t
4813 hat_getattr(struct hat *sfmmup, caddr_t addr, uint_t *attr)
4814 {
4815         tte_t tte;
4816 
4817         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
4818 
4819         if (sfmmup == ksfmmup) {
4820                 if (sfmmu_vatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4821                         tte.ll = 0;
4822                 }
4823         } else {
4824                 if (sfmmu_uvatopfn(addr, sfmmup, &tte) == PFN_INVALID) {
4825                         tte.ll = 0;
4826                 }
4827         }
4828         if (TTE_IS_VALID(&tte)) {
4829                 *attr = sfmmu_ptov_attr(&tte);
4830                 return (0);
4831         }
4832         *attr = 0;
4833         return ((uint_t)0xffffffff);
4834 }
4835 
4836 /*
4837  * Enables more attributes on specified address range (ie. logical OR)
4838  */
4839 void
4840 hat_setattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr)
4841 {
4842         if (hat->sfmmu_xhat_provider) {
4843                 XHAT_SETATTR(hat, addr, len, attr);
4844                 return;
4845         } else {
4846                 /*
4847                  * This must be a CPU HAT. If the address space has
4848                  * XHATs attached, change attributes for all of them,
4849                  * just in case
4850                  */
4851                 ASSERT(hat->sfmmu_as != NULL);
4852                 if (hat->sfmmu_as->a_xhat != NULL)
4853                         xhat_setattr_all(hat->sfmmu_as, addr, len, attr);
4854         }
4855 
4856         sfmmu_chgattr(hat, addr, len, attr, SFMMU_SETATTR);
4857 }
4858 
4859 /*
4860  * Assigns attributes to the specified address range.  All the attributes
4861  * are specified.
4862  */
4863 void
4864 hat_chgattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr)
4865 {
4866         if (hat->sfmmu_xhat_provider) {
4867                 XHAT_CHGATTR(hat, addr, len, attr);
4868                 return;
4869         } else {
4870                 /*
4871                  * This must be a CPU HAT. If the address space has
4872                  * XHATs attached, change attributes for all of them,
4873                  * just in case
4874                  */
4875                 ASSERT(hat->sfmmu_as != NULL);
4876                 if (hat->sfmmu_as->a_xhat != NULL)
4877                         xhat_chgattr_all(hat->sfmmu_as, addr, len, attr);
4878         }
4879 
4880         sfmmu_chgattr(hat, addr, len, attr, SFMMU_CHGATTR);
4881 }
4882 
4883 /*
4884  * Remove attributes on the specified address range (ie. loginal NAND)
4885  */
4886 void
4887 hat_clrattr(struct hat *hat, caddr_t addr, size_t len, uint_t attr)
4888 {
4889         if (hat->sfmmu_xhat_provider) {
4890                 XHAT_CLRATTR(hat, addr, len, attr);
4891                 return;
4892         } else {
4893                 /*
4894                  * This must be a CPU HAT. If the address space has
4895                  * XHATs attached, change attributes for all of them,
4896                  * just in case
4897                  */
4898                 ASSERT(hat->sfmmu_as != NULL);
4899                 if (hat->sfmmu_as->a_xhat != NULL)
4900                         xhat_clrattr_all(hat->sfmmu_as, addr, len, attr);
4901         }
4902 
4903         sfmmu_chgattr(hat, addr, len, attr, SFMMU_CLRATTR);
4904 }
4905 
4906 /*
4907  * Change attributes on an address range to that specified by attr and mode.
4908  */
4909 static void
4910 sfmmu_chgattr(struct hat *sfmmup, caddr_t addr, size_t len, uint_t attr,
4911         int mode)
4912 {
4913         struct hmehash_bucket *hmebp;
4914         hmeblk_tag hblktag;
4915         int hmeshift, hashno = 1;
4916         struct hme_blk *hmeblkp, *list = NULL;
4917         caddr_t endaddr;
4918         cpuset_t cpuset;
4919         demap_range_t dmr;
4920 
4921         CPUSET_ZERO(cpuset);
4922 
4923         ASSERT((sfmmup == ksfmmup) ||
4924             AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
4925         ASSERT((len & MMU_PAGEOFFSET) == 0);
4926         ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0);
4927 
4928         if ((attr & PROT_USER) && (mode != SFMMU_CLRATTR) &&
4929             ((addr + len) > (caddr_t)USERLIMIT)) {
4930                 panic("user addr %p in kernel space",
4931                     (void *)addr);
4932         }
4933 
4934         endaddr = addr + len;
4935         hblktag.htag_id = sfmmup;
4936         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
4937         DEMAP_RANGE_INIT(sfmmup, &dmr);
4938 
4939         while (addr < endaddr) {
4940                 hmeshift = HME_HASH_SHIFT(hashno);
4941                 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
4942                 hblktag.htag_rehash = hashno;
4943                 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
4944 
4945                 SFMMU_HASH_LOCK(hmebp);
4946 
4947                 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list);
4948                 if (hmeblkp != NULL) {
4949                         ASSERT(!hmeblkp->hblk_shared);
4950                         /*
4951                          * We've encountered a shadow hmeblk so skip the range
4952                          * of the next smaller mapping size.
4953                          */
4954                         if (hmeblkp->hblk_shw_bit) {
4955                                 ASSERT(sfmmup != ksfmmup);
4956                                 ASSERT(hashno > 1);
4957                                 addr = (caddr_t)P2END((uintptr_t)addr,
4958                                     TTEBYTES(hashno - 1));
4959                         } else {
4960                                 addr = sfmmu_hblk_chgattr(sfmmup,
4961                                     hmeblkp, addr, endaddr, &dmr, attr, mode);
4962                         }
4963                         SFMMU_HASH_UNLOCK(hmebp);
4964                         hashno = 1;
4965                         continue;
4966                 }
4967                 SFMMU_HASH_UNLOCK(hmebp);
4968 
4969                 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
4970                         /*
4971                          * We have traversed the whole list and rehashed
4972                          * if necessary without finding the address to chgattr.
4973                          * This is ok, so we increment the address by the
4974                          * smallest hmeblk range for kernel mappings or for
4975                          * user mappings with no large pages, and the largest
4976                          * hmeblk range, to account for shadow hmeblks, for
4977                          * user mappings with large pages and continue.
4978                          */
4979                         if (sfmmup == ksfmmup)
4980                                 addr = (caddr_t)P2END((uintptr_t)addr,
4981                                     TTEBYTES(1));
4982                         else
4983                                 addr = (caddr_t)P2END((uintptr_t)addr,
4984                                     TTEBYTES(hashno));
4985                         hashno = 1;
4986                 } else {
4987                         hashno++;
4988                 }
4989         }
4990 
4991         sfmmu_hblks_list_purge(&list, 0);
4992         DEMAP_RANGE_FLUSH(&dmr);
4993         cpuset = sfmmup->sfmmu_cpusran;
4994         xt_sync(cpuset);
4995 }
4996 
4997 /*
4998  * This function chgattr on a range of addresses in an hmeblk.  It returns the
4999  * next addres that needs to be chgattr.
5000  * It should be called with the hash lock held.
5001  * XXX It should be possible to optimize chgattr by not flushing every time but
5002  * on the other hand:
5003  * 1. do one flush crosscall.
5004  * 2. only flush if we are increasing permissions (make sure this will work)
5005  */
5006 static caddr_t
5007 sfmmu_hblk_chgattr(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
5008         caddr_t endaddr, demap_range_t *dmrp, uint_t attr, int mode)
5009 {
5010         tte_t tte, tteattr, tteflags, ttemod;
5011         struct sf_hment *sfhmep;
5012         int ttesz;
5013         struct page *pp = NULL;
5014         kmutex_t *pml, *pmtx;
5015         int ret;
5016         int use_demap_range;
5017 #if defined(SF_ERRATA_57)
5018         int check_exec;
5019 #endif
5020 
5021         ASSERT(in_hblk_range(hmeblkp, addr));
5022         ASSERT(hmeblkp->hblk_shw_bit == 0);
5023         ASSERT(!hmeblkp->hblk_shared);
5024 
5025         endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp));
5026         ttesz = get_hblk_ttesz(hmeblkp);
5027 
5028         /*
5029          * Flush the current demap region if addresses have been
5030          * skipped or the page size doesn't match.
5031          */
5032         use_demap_range = (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp));
5033         if (use_demap_range) {
5034                 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr);
5035         } else if (dmrp != NULL) {
5036                 DEMAP_RANGE_FLUSH(dmrp);
5037         }
5038 
5039         tteattr.ll = sfmmu_vtop_attr(attr, mode, &tteflags);
5040 #if defined(SF_ERRATA_57)
5041         check_exec = (sfmmup != ksfmmup) &&
5042             AS_TYPE_64BIT(sfmmup->sfmmu_as) &&
5043             TTE_IS_EXECUTABLE(&tteattr);
5044 #endif
5045         HBLKTOHME(sfhmep, hmeblkp, addr);
5046         while (addr < endaddr) {
5047                 sfmmu_copytte(&sfhmep->hme_tte, &tte);
5048                 if (TTE_IS_VALID(&tte)) {
5049                         if ((tte.ll & tteflags.ll) == tteattr.ll) {
5050                                 /*
5051                                  * if the new attr is the same as old
5052                                  * continue
5053                                  */
5054                                 goto next_addr;
5055                         }
5056                         if (!TTE_IS_WRITABLE(&tteattr)) {
5057                                 /*
5058                                  * make sure we clear hw modify bit if we
5059                                  * removing write protections
5060                                  */
5061                                 tteflags.tte_intlo |= TTE_HWWR_INT;
5062                         }
5063 
5064                         pml = NULL;
5065                         pp = sfhmep->hme_page;
5066                         if (pp) {
5067                                 pml = sfmmu_mlist_enter(pp);
5068                         }
5069 
5070                         if (pp != sfhmep->hme_page) {
5071                                 /*
5072                                  * tte must have been unloaded.
5073                                  */
5074                                 ASSERT(pml);
5075                                 sfmmu_mlist_exit(pml);
5076                                 continue;
5077                         }
5078 
5079                         ASSERT(pp == NULL || sfmmu_mlist_held(pp));
5080 
5081                         ttemod = tte;
5082                         ttemod.ll = (ttemod.ll & ~tteflags.ll) | tteattr.ll;
5083                         ASSERT(TTE_TO_TTEPFN(&ttemod) == TTE_TO_TTEPFN(&tte));
5084 
5085 #if defined(SF_ERRATA_57)
5086                         if (check_exec && addr < errata57_limit)
5087                                 ttemod.tte_exec_perm = 0;
5088 #endif
5089                         ret = sfmmu_modifytte_try(&tte, &ttemod,
5090                             &sfhmep->hme_tte);
5091 
5092                         if (ret < 0) {
5093                                 /* tte changed underneath us */
5094                                 if (pml) {
5095                                         sfmmu_mlist_exit(pml);
5096                                 }
5097                                 continue;
5098                         }
5099 
5100                         if (tteflags.tte_intlo & TTE_HWWR_INT) {
5101                                 /*
5102                                  * need to sync if we are clearing modify bit.
5103                                  */
5104                                 sfmmu_ttesync(sfmmup, addr, &tte, pp);
5105                         }
5106 
5107                         if (pp && PP_ISRO(pp)) {
5108                                 if (tteattr.tte_intlo & TTE_WRPRM_INT) {
5109                                         pmtx = sfmmu_page_enter(pp);
5110                                         PP_CLRRO(pp);
5111                                         sfmmu_page_exit(pmtx);
5112                                 }
5113                         }
5114 
5115                         if (ret > 0 && use_demap_range) {
5116                                 DEMAP_RANGE_MARKPG(dmrp, addr);
5117                         } else if (ret > 0) {
5118                                 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
5119                         }
5120 
5121                         if (pml) {
5122                                 sfmmu_mlist_exit(pml);
5123                         }
5124                 }
5125 next_addr:
5126                 addr += TTEBYTES(ttesz);
5127                 sfhmep++;
5128                 DEMAP_RANGE_NEXTPG(dmrp);
5129         }
5130         return (addr);
5131 }
5132 
5133 /*
5134  * This routine converts virtual attributes to physical ones.  It will
5135  * update the tteflags field with the tte mask corresponding to the attributes
5136  * affected and it returns the new attributes.  It will also clear the modify
5137  * bit if we are taking away write permission.  This is necessary since the
5138  * modify bit is the hardware permission bit and we need to clear it in order
5139  * to detect write faults.
5140  */
5141 static uint64_t
5142 sfmmu_vtop_attr(uint_t attr, int mode, tte_t *ttemaskp)
5143 {
5144         tte_t ttevalue;
5145 
5146         ASSERT(!(attr & ~SFMMU_LOAD_ALLATTR));
5147 
5148         switch (mode) {
5149         case SFMMU_CHGATTR:
5150                 /* all attributes specified */
5151                 ttevalue.tte_inthi = MAKE_TTEATTR_INTHI(attr);
5152                 ttevalue.tte_intlo = MAKE_TTEATTR_INTLO(attr);
5153                 ttemaskp->tte_inthi = TTEINTHI_ATTR;
5154                 ttemaskp->tte_intlo = TTEINTLO_ATTR;
5155                 break;
5156         case SFMMU_SETATTR:
5157                 ASSERT(!(attr & ~HAT_PROT_MASK));
5158                 ttemaskp->ll = 0;
5159                 ttevalue.ll = 0;
5160                 /*
5161                  * a valid tte implies exec and read for sfmmu
5162                  * so no need to do anything about them.
5163                  * since priviledged access implies user access
5164                  * PROT_USER doesn't make sense either.
5165                  */
5166                 if (attr & PROT_WRITE) {
5167                         ttemaskp->tte_intlo |= TTE_WRPRM_INT;
5168                         ttevalue.tte_intlo |= TTE_WRPRM_INT;
5169                 }
5170                 break;
5171         case SFMMU_CLRATTR:
5172                 /* attributes will be nand with current ones */
5173                 if (attr & ~(PROT_WRITE | PROT_USER)) {
5174                         panic("sfmmu: attr %x not supported", attr);
5175                 }
5176                 ttemaskp->ll = 0;
5177                 ttevalue.ll = 0;
5178                 if (attr & PROT_WRITE) {
5179                         /* clear both writable and modify bit */
5180                         ttemaskp->tte_intlo |= TTE_WRPRM_INT | TTE_HWWR_INT;
5181                 }
5182                 if (attr & PROT_USER) {
5183                         ttemaskp->tte_intlo |= TTE_PRIV_INT;
5184                         ttevalue.tte_intlo |= TTE_PRIV_INT;
5185                 }
5186                 break;
5187         default:
5188                 panic("sfmmu_vtop_attr: bad mode %x", mode);
5189         }
5190         ASSERT(TTE_TO_TTEPFN(&ttevalue) == 0);
5191         return (ttevalue.ll);
5192 }
5193 
5194 static uint_t
5195 sfmmu_ptov_attr(tte_t *ttep)
5196 {
5197         uint_t attr;
5198 
5199         ASSERT(TTE_IS_VALID(ttep));
5200 
5201         attr = PROT_READ;
5202 
5203         if (TTE_IS_WRITABLE(ttep)) {
5204                 attr |= PROT_WRITE;
5205         }
5206         if (TTE_IS_EXECUTABLE(ttep)) {
5207                 attr |= PROT_EXEC;
5208         }
5209         if (!TTE_IS_PRIVILEGED(ttep)) {
5210                 attr |= PROT_USER;
5211         }
5212         if (TTE_IS_NFO(ttep)) {
5213                 attr |= HAT_NOFAULT;
5214         }
5215         if (TTE_IS_NOSYNC(ttep)) {
5216                 attr |= HAT_NOSYNC;
5217         }
5218         if (TTE_IS_SIDEFFECT(ttep)) {
5219                 attr |= SFMMU_SIDEFFECT;
5220         }
5221         if (!TTE_IS_VCACHEABLE(ttep)) {
5222                 attr |= SFMMU_UNCACHEVTTE;
5223         }
5224         if (!TTE_IS_PCACHEABLE(ttep)) {
5225                 attr |= SFMMU_UNCACHEPTTE;
5226         }
5227         return (attr);
5228 }
5229 
5230 /*
5231  * hat_chgprot is a deprecated hat call.  New segment drivers
5232  * should store all attributes and use hat_*attr calls.
5233  *
5234  * Change the protections in the virtual address range
5235  * given to the specified virtual protection.  If vprot is ~PROT_WRITE,
5236  * then remove write permission, leaving the other
5237  * permissions unchanged.  If vprot is ~PROT_USER, remove user permissions.
5238  *
5239  */
5240 void
5241 hat_chgprot(struct hat *sfmmup, caddr_t addr, size_t len, uint_t vprot)
5242 {
5243         struct hmehash_bucket *hmebp;
5244         hmeblk_tag hblktag;
5245         int hmeshift, hashno = 1;
5246         struct hme_blk *hmeblkp, *list = NULL;
5247         caddr_t endaddr;
5248         cpuset_t cpuset;
5249         demap_range_t dmr;
5250 
5251         ASSERT((len & MMU_PAGEOFFSET) == 0);
5252         ASSERT(((uintptr_t)addr & MMU_PAGEOFFSET) == 0);
5253 
5254         if (sfmmup->sfmmu_xhat_provider) {
5255                 XHAT_CHGPROT(sfmmup, addr, len, vprot);
5256                 return;
5257         } else {
5258                 /*
5259                  * This must be a CPU HAT. If the address space has
5260                  * XHATs attached, change attributes for all of them,
5261                  * just in case
5262                  */
5263                 ASSERT(sfmmup->sfmmu_as != NULL);
5264                 if (sfmmup->sfmmu_as->a_xhat != NULL)
5265                         xhat_chgprot_all(sfmmup->sfmmu_as, addr, len, vprot);
5266         }
5267 
5268         CPUSET_ZERO(cpuset);
5269 
5270         if ((vprot != (uint_t)~PROT_WRITE) && (vprot & PROT_USER) &&
5271             ((addr + len) > (caddr_t)USERLIMIT)) {
5272                 panic("user addr %p vprot %x in kernel space",
5273                     (void *)addr, vprot);
5274         }
5275         endaddr = addr + len;
5276         hblktag.htag_id = sfmmup;
5277         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
5278         DEMAP_RANGE_INIT(sfmmup, &dmr);
5279 
5280         while (addr < endaddr) {
5281                 hmeshift = HME_HASH_SHIFT(hashno);
5282                 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
5283                 hblktag.htag_rehash = hashno;
5284                 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
5285 
5286                 SFMMU_HASH_LOCK(hmebp);
5287 
5288                 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list);
5289                 if (hmeblkp != NULL) {
5290                         ASSERT(!hmeblkp->hblk_shared);
5291                         /*
5292                          * We've encountered a shadow hmeblk so skip the range
5293                          * of the next smaller mapping size.
5294                          */
5295                         if (hmeblkp->hblk_shw_bit) {
5296                                 ASSERT(sfmmup != ksfmmup);
5297                                 ASSERT(hashno > 1);
5298                                 addr = (caddr_t)P2END((uintptr_t)addr,
5299                                     TTEBYTES(hashno - 1));
5300                         } else {
5301                                 addr = sfmmu_hblk_chgprot(sfmmup, hmeblkp,
5302                                     addr, endaddr, &dmr, vprot);
5303                         }
5304                         SFMMU_HASH_UNLOCK(hmebp);
5305                         hashno = 1;
5306                         continue;
5307                 }
5308                 SFMMU_HASH_UNLOCK(hmebp);
5309 
5310                 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
5311                         /*
5312                          * We have traversed the whole list and rehashed
5313                          * if necessary without finding the address to chgprot.
5314                          * This is ok so we increment the address by the
5315                          * smallest hmeblk range for kernel mappings and the
5316                          * largest hmeblk range, to account for shadow hmeblks,
5317                          * for user mappings and continue.
5318                          */
5319                         if (sfmmup == ksfmmup)
5320                                 addr = (caddr_t)P2END((uintptr_t)addr,
5321                                     TTEBYTES(1));
5322                         else
5323                                 addr = (caddr_t)P2END((uintptr_t)addr,
5324                                     TTEBYTES(hashno));
5325                         hashno = 1;
5326                 } else {
5327                         hashno++;
5328                 }
5329         }
5330 
5331         sfmmu_hblks_list_purge(&list, 0);
5332         DEMAP_RANGE_FLUSH(&dmr);
5333         cpuset = sfmmup->sfmmu_cpusran;
5334         xt_sync(cpuset);
5335 }
5336 
5337 /*
5338  * This function chgprots a range of addresses in an hmeblk.  It returns the
5339  * next addres that needs to be chgprot.
5340  * It should be called with the hash lock held.
5341  * XXX It shold be possible to optimize chgprot by not flushing every time but
5342  * on the other hand:
5343  * 1. do one flush crosscall.
5344  * 2. only flush if we are increasing permissions (make sure this will work)
5345  */
5346 static caddr_t
5347 sfmmu_hblk_chgprot(sfmmu_t *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
5348         caddr_t endaddr, demap_range_t *dmrp, uint_t vprot)
5349 {
5350         uint_t pprot;
5351         tte_t tte, ttemod;
5352         struct sf_hment *sfhmep;
5353         uint_t tteflags;
5354         int ttesz;
5355         struct page *pp = NULL;
5356         kmutex_t *pml, *pmtx;
5357         int ret;
5358         int use_demap_range;
5359 #if defined(SF_ERRATA_57)
5360         int check_exec;
5361 #endif
5362 
5363         ASSERT(in_hblk_range(hmeblkp, addr));
5364         ASSERT(hmeblkp->hblk_shw_bit == 0);
5365         ASSERT(!hmeblkp->hblk_shared);
5366 
5367 #ifdef DEBUG
5368         if (get_hblk_ttesz(hmeblkp) != TTE8K &&
5369             (endaddr < get_hblk_endaddr(hmeblkp))) {
5370                 panic("sfmmu_hblk_chgprot: partial chgprot of large page");
5371         }
5372 #endif /* DEBUG */
5373 
5374         endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp));
5375         ttesz = get_hblk_ttesz(hmeblkp);
5376 
5377         pprot = sfmmu_vtop_prot(vprot, &tteflags);
5378 #if defined(SF_ERRATA_57)
5379         check_exec = (sfmmup != ksfmmup) &&
5380             AS_TYPE_64BIT(sfmmup->sfmmu_as) &&
5381             ((vprot & PROT_EXEC) == PROT_EXEC);
5382 #endif
5383         HBLKTOHME(sfhmep, hmeblkp, addr);
5384 
5385         /*
5386          * Flush the current demap region if addresses have been
5387          * skipped or the page size doesn't match.
5388          */
5389         use_demap_range = (TTEBYTES(ttesz) == MMU_PAGESIZE);
5390         if (use_demap_range) {
5391                 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr);
5392         } else if (dmrp != NULL) {
5393                 DEMAP_RANGE_FLUSH(dmrp);
5394         }
5395 
5396         while (addr < endaddr) {
5397                 sfmmu_copytte(&sfhmep->hme_tte, &tte);
5398                 if (TTE_IS_VALID(&tte)) {
5399                         if (TTE_GET_LOFLAGS(&tte, tteflags) == pprot) {
5400                                 /*
5401                                  * if the new protection is the same as old
5402                                  * continue
5403                                  */
5404                                 goto next_addr;
5405                         }
5406                         pml = NULL;
5407                         pp = sfhmep->hme_page;
5408                         if (pp) {
5409                                 pml = sfmmu_mlist_enter(pp);
5410                         }
5411                         if (pp != sfhmep->hme_page) {
5412                                 /*
5413                                  * tte most have been unloaded
5414                                  * underneath us.  Recheck
5415                                  */
5416                                 ASSERT(pml);
5417                                 sfmmu_mlist_exit(pml);
5418                                 continue;
5419                         }
5420 
5421                         ASSERT(pp == NULL || sfmmu_mlist_held(pp));
5422 
5423                         ttemod = tte;
5424                         TTE_SET_LOFLAGS(&ttemod, tteflags, pprot);
5425 #if defined(SF_ERRATA_57)
5426                         if (check_exec && addr < errata57_limit)
5427                                 ttemod.tte_exec_perm = 0;
5428 #endif
5429                         ret = sfmmu_modifytte_try(&tte, &ttemod,
5430                             &sfhmep->hme_tte);
5431 
5432                         if (ret < 0) {
5433                                 /* tte changed underneath us */
5434                                 if (pml) {
5435                                         sfmmu_mlist_exit(pml);
5436                                 }
5437                                 continue;
5438                         }
5439 
5440                         if (tteflags & TTE_HWWR_INT) {
5441                                 /*
5442                                  * need to sync if we are clearing modify bit.
5443                                  */
5444                                 sfmmu_ttesync(sfmmup, addr, &tte, pp);
5445                         }
5446 
5447                         if (pp && PP_ISRO(pp)) {
5448                                 if (pprot & TTE_WRPRM_INT) {
5449                                         pmtx = sfmmu_page_enter(pp);
5450                                         PP_CLRRO(pp);
5451                                         sfmmu_page_exit(pmtx);
5452                                 }
5453                         }
5454 
5455                         if (ret > 0 && use_demap_range) {
5456                                 DEMAP_RANGE_MARKPG(dmrp, addr);
5457                         } else if (ret > 0) {
5458                                 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
5459                         }
5460 
5461                         if (pml) {
5462                                 sfmmu_mlist_exit(pml);
5463                         }
5464                 }
5465 next_addr:
5466                 addr += TTEBYTES(ttesz);
5467                 sfhmep++;
5468                 DEMAP_RANGE_NEXTPG(dmrp);
5469         }
5470         return (addr);
5471 }
5472 
5473 /*
5474  * This routine is deprecated and should only be used by hat_chgprot.
5475  * The correct routine is sfmmu_vtop_attr.
5476  * This routine converts virtual page protections to physical ones.  It will
5477  * update the tteflags field with the tte mask corresponding to the protections
5478  * affected and it returns the new protections.  It will also clear the modify
5479  * bit if we are taking away write permission.  This is necessary since the
5480  * modify bit is the hardware permission bit and we need to clear it in order
5481  * to detect write faults.
5482  * It accepts the following special protections:
5483  * ~PROT_WRITE = remove write permissions.
5484  * ~PROT_USER = remove user permissions.
5485  */
5486 static uint_t
5487 sfmmu_vtop_prot(uint_t vprot, uint_t *tteflagsp)
5488 {
5489         if (vprot == (uint_t)~PROT_WRITE) {
5490                 *tteflagsp = TTE_WRPRM_INT | TTE_HWWR_INT;
5491                 return (0);             /* will cause wrprm to be cleared */
5492         }
5493         if (vprot == (uint_t)~PROT_USER) {
5494                 *tteflagsp = TTE_PRIV_INT;
5495                 return (0);             /* will cause privprm to be cleared */
5496         }
5497         if ((vprot == 0) || (vprot == PROT_USER) ||
5498             ((vprot & PROT_ALL) != vprot)) {
5499                 panic("sfmmu_vtop_prot -- bad prot %x", vprot);
5500         }
5501 
5502         switch (vprot) {
5503         case (PROT_READ):
5504         case (PROT_EXEC):
5505         case (PROT_EXEC | PROT_READ):
5506                 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT;
5507                 return (TTE_PRIV_INT);          /* set prv and clr wrt */
5508         case (PROT_WRITE):
5509         case (PROT_WRITE | PROT_READ):
5510         case (PROT_EXEC | PROT_WRITE):
5511         case (PROT_EXEC | PROT_WRITE | PROT_READ):
5512                 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT;
5513                 return (TTE_PRIV_INT | TTE_WRPRM_INT);  /* set prv and wrt */
5514         case (PROT_USER | PROT_READ):
5515         case (PROT_USER | PROT_EXEC):
5516         case (PROT_USER | PROT_EXEC | PROT_READ):
5517                 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT | TTE_HWWR_INT;
5518                 return (0);                     /* clr prv and wrt */
5519         case (PROT_USER | PROT_WRITE):
5520         case (PROT_USER | PROT_WRITE | PROT_READ):
5521         case (PROT_USER | PROT_EXEC | PROT_WRITE):
5522         case (PROT_USER | PROT_EXEC | PROT_WRITE | PROT_READ):
5523                 *tteflagsp = TTE_PRIV_INT | TTE_WRPRM_INT;
5524                 return (TTE_WRPRM_INT);         /* clr prv and set wrt */
5525         default:
5526                 panic("sfmmu_vtop_prot -- bad prot %x", vprot);
5527         }
5528         return (0);
5529 }
5530 
5531 /*
5532  * Alternate unload for very large virtual ranges. With a true 64 bit VA,
5533  * the normal algorithm would take too long for a very large VA range with
5534  * few real mappings. This routine just walks thru all HMEs in the global
5535  * hash table to find and remove mappings.
5536  */
5537 static void
5538 hat_unload_large_virtual(
5539         struct hat              *sfmmup,
5540         caddr_t                 startaddr,
5541         size_t                  len,
5542         uint_t                  flags,
5543         hat_callback_t          *callback)
5544 {
5545         struct hmehash_bucket *hmebp;
5546         struct hme_blk *hmeblkp;
5547         struct hme_blk *pr_hblk = NULL;
5548         struct hme_blk *nx_hblk;
5549         struct hme_blk *list = NULL;
5550         int i;
5551         demap_range_t dmr, *dmrp;
5552         cpuset_t cpuset;
5553         caddr_t endaddr = startaddr + len;
5554         caddr_t sa;
5555         caddr_t ea;
5556         caddr_t cb_sa[MAX_CB_ADDR];
5557         caddr_t cb_ea[MAX_CB_ADDR];
5558         int     addr_cnt = 0;
5559         int     a = 0;
5560 
5561         if (sfmmup->sfmmu_free) {
5562                 dmrp = NULL;
5563         } else {
5564                 dmrp = &dmr;
5565                 DEMAP_RANGE_INIT(sfmmup, dmrp);
5566         }
5567 
5568         /*
5569          * Loop through all the hash buckets of HME blocks looking for matches.
5570          */
5571         for (i = 0; i <= UHMEHASH_SZ; i++) {
5572                 hmebp = &uhme_hash[i];
5573                 SFMMU_HASH_LOCK(hmebp);
5574                 hmeblkp = hmebp->hmeblkp;
5575                 pr_hblk = NULL;
5576                 while (hmeblkp) {
5577                         nx_hblk = hmeblkp->hblk_next;
5578 
5579                         /*
5580                          * skip if not this context, if a shadow block or
5581                          * if the mapping is not in the requested range
5582                          */
5583                         if (hmeblkp->hblk_tag.htag_id != sfmmup ||
5584                             hmeblkp->hblk_shw_bit ||
5585                             (sa = (caddr_t)get_hblk_base(hmeblkp)) >= endaddr ||
5586                             (ea = get_hblk_endaddr(hmeblkp)) <= startaddr) {
5587                                 pr_hblk = hmeblkp;
5588                                 goto next_block;
5589                         }
5590 
5591                         ASSERT(!hmeblkp->hblk_shared);
5592                         /*
5593                          * unload if there are any current valid mappings
5594                          */
5595                         if (hmeblkp->hblk_vcnt != 0 ||
5596                             hmeblkp->hblk_hmecnt != 0)
5597                                 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
5598                                     sa, ea, dmrp, flags);
5599 
5600                         /*
5601                          * on unmap we also release the HME block itself, once
5602                          * all mappings are gone.
5603                          */
5604                         if ((flags & HAT_UNLOAD_UNMAP) != 0 &&
5605                             !hmeblkp->hblk_vcnt &&
5606                             !hmeblkp->hblk_hmecnt) {
5607                                 ASSERT(!hmeblkp->hblk_lckcnt);
5608                                 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
5609                                     &list, 0);
5610                         } else {
5611                                 pr_hblk = hmeblkp;
5612                         }
5613 
5614                         if (callback == NULL)
5615                                 goto next_block;
5616 
5617                         /*
5618                          * HME blocks may span more than one page, but we may be
5619                          * unmapping only one page, so check for a smaller range
5620                          * for the callback
5621                          */
5622                         if (sa < startaddr)
5623                                 sa = startaddr;
5624                         if (--ea > endaddr)
5625                                 ea = endaddr - 1;
5626 
5627                         cb_sa[addr_cnt] = sa;
5628                         cb_ea[addr_cnt] = ea;
5629                         if (++addr_cnt == MAX_CB_ADDR) {
5630                                 if (dmrp != NULL) {
5631                                         DEMAP_RANGE_FLUSH(dmrp);
5632                                         cpuset = sfmmup->sfmmu_cpusran;
5633                                         xt_sync(cpuset);
5634                                 }
5635 
5636                                 for (a = 0; a < MAX_CB_ADDR; ++a) {
5637                                         callback->hcb_start_addr = cb_sa[a];
5638                                         callback->hcb_end_addr = cb_ea[a];
5639                                         callback->hcb_function(callback);
5640                                 }
5641                                 addr_cnt = 0;
5642                         }
5643 
5644 next_block:
5645                         hmeblkp = nx_hblk;
5646                 }
5647                 SFMMU_HASH_UNLOCK(hmebp);
5648         }
5649 
5650         sfmmu_hblks_list_purge(&list, 0);
5651         if (dmrp != NULL) {
5652                 DEMAP_RANGE_FLUSH(dmrp);
5653                 cpuset = sfmmup->sfmmu_cpusran;
5654                 xt_sync(cpuset);
5655         }
5656 
5657         for (a = 0; a < addr_cnt; ++a) {
5658                 callback->hcb_start_addr = cb_sa[a];
5659                 callback->hcb_end_addr = cb_ea[a];
5660                 callback->hcb_function(callback);
5661         }
5662 
5663         /*
5664          * Check TSB and TLB page sizes if the process isn't exiting.
5665          */
5666         if (!sfmmup->sfmmu_free)
5667                 sfmmu_check_page_sizes(sfmmup, 0);
5668 }
5669 
5670 /*
5671  * Unload all the mappings in the range [addr..addr+len). addr and len must
5672  * be MMU_PAGESIZE aligned.
5673  */
5674 
5675 extern struct seg *segkmap;
5676 #define ISSEGKMAP(sfmmup, addr) (sfmmup == ksfmmup && \
5677 segkmap->s_base <= (addr) && (addr) < (segkmap->s_base + segkmap->s_size))
5678 
5679 
5680 void
5681 hat_unload_callback(
5682         struct hat *sfmmup,
5683         caddr_t addr,
5684         size_t len,
5685         uint_t flags,
5686         hat_callback_t *callback)
5687 {
5688         struct hmehash_bucket *hmebp;
5689         hmeblk_tag hblktag;
5690         int hmeshift, hashno, iskernel;
5691         struct hme_blk *hmeblkp, *pr_hblk, *list = NULL;
5692         caddr_t endaddr;
5693         cpuset_t cpuset;
5694         int addr_count = 0;
5695         int a;
5696         caddr_t cb_start_addr[MAX_CB_ADDR];
5697         caddr_t cb_end_addr[MAX_CB_ADDR];
5698         int issegkmap = ISSEGKMAP(sfmmup, addr);
5699         demap_range_t dmr, *dmrp;
5700 
5701         if (sfmmup->sfmmu_xhat_provider) {
5702                 XHAT_UNLOAD_CALLBACK(sfmmup, addr, len, flags, callback);
5703                 return;
5704         } else {
5705                 /*
5706                  * This must be a CPU HAT. If the address space has
5707                  * XHATs attached, unload the mappings for all of them,
5708                  * just in case
5709                  */
5710                 ASSERT(sfmmup->sfmmu_as != NULL);
5711                 if (sfmmup->sfmmu_as->a_xhat != NULL)
5712                         xhat_unload_callback_all(sfmmup->sfmmu_as, addr,
5713                             len, flags, callback);
5714         }
5715 
5716         ASSERT((sfmmup == ksfmmup) || (flags & HAT_UNLOAD_OTHER) || \
5717             AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
5718 
5719         ASSERT(sfmmup != NULL);
5720         ASSERT((len & MMU_PAGEOFFSET) == 0);
5721         ASSERT(!((uintptr_t)addr & MMU_PAGEOFFSET));
5722 
5723         /*
5724          * Probing through a large VA range (say 63 bits) will be slow, even
5725          * at 4 Meg steps between the probes. So, when the virtual address range
5726          * is very large, search the HME entries for what to unload.
5727          *
5728          *      len >> TTE_PAGE_SHIFT(TTE4M) is the # of 4Meg probes we'd need
5729          *
5730          *      UHMEHASH_SZ is number of hash buckets to examine
5731          *
5732          */
5733         if (sfmmup != KHATID && (len >> TTE_PAGE_SHIFT(TTE4M)) > UHMEHASH_SZ) {
5734                 hat_unload_large_virtual(sfmmup, addr, len, flags, callback);
5735                 return;
5736         }
5737 
5738         CPUSET_ZERO(cpuset);
5739 
5740         /*
5741          * If the process is exiting, we can save a lot of fuss since
5742          * we'll flush the TLB when we free the ctx anyway.
5743          */
5744         if (sfmmup->sfmmu_free) {
5745                 dmrp = NULL;
5746         } else {
5747                 dmrp = &dmr;
5748                 DEMAP_RANGE_INIT(sfmmup, dmrp);
5749         }
5750 
5751         endaddr = addr + len;
5752         hblktag.htag_id = sfmmup;
5753         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
5754 
5755         /*
5756          * It is likely for the vm to call unload over a wide range of
5757          * addresses that are actually very sparsely populated by
5758          * translations.  In order to speed this up the sfmmu hat supports
5759          * the concept of shadow hmeblks. Dummy large page hmeblks that
5760          * correspond to actual small translations are allocated at tteload
5761          * time and are referred to as shadow hmeblks.  Now, during unload
5762          * time, we first check if we have a shadow hmeblk for that
5763          * translation.  The absence of one means the corresponding address
5764          * range is empty and can be skipped.
5765          *
5766          * The kernel is an exception to above statement and that is why
5767          * we don't use shadow hmeblks and hash starting from the smallest
5768          * page size.
5769          */
5770         if (sfmmup == KHATID) {
5771                 iskernel = 1;
5772                 hashno = TTE64K;
5773         } else {
5774                 iskernel = 0;
5775                 if (mmu_page_sizes == max_mmu_page_sizes) {
5776                         hashno = TTE256M;
5777                 } else {
5778                         hashno = TTE4M;
5779                 }
5780         }
5781         while (addr < endaddr) {
5782                 hmeshift = HME_HASH_SHIFT(hashno);
5783                 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
5784                 hblktag.htag_rehash = hashno;
5785                 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
5786 
5787                 SFMMU_HASH_LOCK(hmebp);
5788 
5789                 HME_HASH_SEARCH_PREV(hmebp, hblktag, hmeblkp, pr_hblk, &list);
5790                 if (hmeblkp == NULL) {
5791                         /*
5792                          * didn't find an hmeblk. skip the appropiate
5793                          * address range.
5794                          */
5795                         SFMMU_HASH_UNLOCK(hmebp);
5796                         if (iskernel) {
5797                                 if (hashno < mmu_hashcnt) {
5798                                         hashno++;
5799                                         continue;
5800                                 } else {
5801                                         hashno = TTE64K;
5802                                         addr = (caddr_t)roundup((uintptr_t)addr
5803                                             + 1, MMU_PAGESIZE64K);
5804                                         continue;
5805                                 }
5806                         }
5807                         addr = (caddr_t)roundup((uintptr_t)addr + 1,
5808                             (1 << hmeshift));
5809                         if ((uintptr_t)addr & MMU_PAGEOFFSET512K) {
5810                                 ASSERT(hashno == TTE64K);
5811                                 continue;
5812                         }
5813                         if ((uintptr_t)addr & MMU_PAGEOFFSET4M) {
5814                                 hashno = TTE512K;
5815                                 continue;
5816                         }
5817                         if (mmu_page_sizes == max_mmu_page_sizes) {
5818                                 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) {
5819                                         hashno = TTE4M;
5820                                         continue;
5821                                 }
5822                                 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) {
5823                                         hashno = TTE32M;
5824                                         continue;
5825                                 }
5826                                 hashno = TTE256M;
5827                                 continue;
5828                         } else {
5829                                 hashno = TTE4M;
5830                                 continue;
5831                         }
5832                 }
5833                 ASSERT(hmeblkp);
5834                 ASSERT(!hmeblkp->hblk_shared);
5835                 if (!hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
5836                         /*
5837                          * If the valid count is zero we can skip the range
5838                          * mapped by this hmeblk.
5839                          * We free hblks in the case of HAT_UNMAP.  HAT_UNMAP
5840                          * is used by segment drivers as a hint
5841                          * that the mapping resource won't be used any longer.
5842                          * The best example of this is during exit().
5843                          */
5844                         addr = (caddr_t)roundup((uintptr_t)addr + 1,
5845                             get_hblk_span(hmeblkp));
5846                         if ((flags & HAT_UNLOAD_UNMAP) ||
5847                             (iskernel && !issegkmap)) {
5848                                 sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk,
5849                                     &list, 0);
5850                         }
5851                         SFMMU_HASH_UNLOCK(hmebp);
5852 
5853                         if (iskernel) {
5854                                 hashno = TTE64K;
5855                                 continue;
5856                         }
5857                         if ((uintptr_t)addr & MMU_PAGEOFFSET512K) {
5858                                 ASSERT(hashno == TTE64K);
5859                                 continue;
5860                         }
5861                         if ((uintptr_t)addr & MMU_PAGEOFFSET4M) {
5862                                 hashno = TTE512K;
5863                                 continue;
5864                         }
5865                         if (mmu_page_sizes == max_mmu_page_sizes) {
5866                                 if ((uintptr_t)addr & MMU_PAGEOFFSET32M) {
5867                                         hashno = TTE4M;
5868                                         continue;
5869                                 }
5870                                 if ((uintptr_t)addr & MMU_PAGEOFFSET256M) {
5871                                         hashno = TTE32M;
5872                                         continue;
5873                                 }
5874                                 hashno = TTE256M;
5875                                 continue;
5876                         } else {
5877                                 hashno = TTE4M;
5878                                 continue;
5879                         }
5880                 }
5881                 if (hmeblkp->hblk_shw_bit) {
5882                         /*
5883                          * If we encounter a shadow hmeblk we know there is
5884                          * smaller sized hmeblks mapping the same address space.
5885                          * Decrement the hash size and rehash.
5886                          */
5887                         ASSERT(sfmmup != KHATID);
5888                         hashno--;
5889                         SFMMU_HASH_UNLOCK(hmebp);
5890                         continue;
5891                 }
5892 
5893                 /*
5894                  * track callback address ranges.
5895                  * only start a new range when it's not contiguous
5896                  */
5897                 if (callback != NULL) {
5898                         if (addr_count > 0 &&
5899                             addr == cb_end_addr[addr_count - 1])
5900                                 --addr_count;
5901                         else
5902                                 cb_start_addr[addr_count] = addr;
5903                 }
5904 
5905                 addr = sfmmu_hblk_unload(sfmmup, hmeblkp, addr, endaddr,
5906                     dmrp, flags);
5907 
5908                 if (callback != NULL)
5909                         cb_end_addr[addr_count++] = addr;
5910 
5911                 if (((flags & HAT_UNLOAD_UNMAP) || (iskernel && !issegkmap)) &&
5912                     !hmeblkp->hblk_vcnt && !hmeblkp->hblk_hmecnt) {
5913                         sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 0);
5914                 }
5915                 SFMMU_HASH_UNLOCK(hmebp);
5916 
5917                 /*
5918                  * Notify our caller as to exactly which pages
5919                  * have been unloaded. We do these in clumps,
5920                  * to minimize the number of xt_sync()s that need to occur.
5921                  */
5922                 if (callback != NULL && addr_count == MAX_CB_ADDR) {
5923                         if (dmrp != NULL) {
5924                                 DEMAP_RANGE_FLUSH(dmrp);
5925                                 cpuset = sfmmup->sfmmu_cpusran;
5926                                 xt_sync(cpuset);
5927                         }
5928 
5929                         for (a = 0; a < MAX_CB_ADDR; ++a) {
5930                                 callback->hcb_start_addr = cb_start_addr[a];
5931                                 callback->hcb_end_addr = cb_end_addr[a];
5932                                 callback->hcb_function(callback);
5933                         }
5934                         addr_count = 0;
5935                 }
5936                 if (iskernel) {
5937                         hashno = TTE64K;
5938                         continue;
5939                 }
5940                 if ((uintptr_t)addr & MMU_PAGEOFFSET512K) {
5941                         ASSERT(hashno == TTE64K);
5942                         continue;
5943                 }
5944                 if ((uintptr_t)addr & MMU_PAGEOFFSET4M) {
5945                         hashno = TTE512K;
5946                         continue;
5947                 }
5948                 if (mmu_page_sizes == max_mmu_page_sizes) {
5949                         if ((uintptr_t)addr & MMU_PAGEOFFSET32M) {
5950                                 hashno = TTE4M;
5951                                 continue;
5952                         }
5953                         if ((uintptr_t)addr & MMU_PAGEOFFSET256M) {
5954                                 hashno = TTE32M;
5955                                 continue;
5956                         }
5957                         hashno = TTE256M;
5958                 } else {
5959                         hashno = TTE4M;
5960                 }
5961         }
5962 
5963         sfmmu_hblks_list_purge(&list, 0);
5964         if (dmrp != NULL) {
5965                 DEMAP_RANGE_FLUSH(dmrp);
5966                 cpuset = sfmmup->sfmmu_cpusran;
5967                 xt_sync(cpuset);
5968         }
5969         if (callback && addr_count != 0) {
5970                 for (a = 0; a < addr_count; ++a) {
5971                         callback->hcb_start_addr = cb_start_addr[a];
5972                         callback->hcb_end_addr = cb_end_addr[a];
5973                         callback->hcb_function(callback);
5974                 }
5975         }
5976 
5977         /*
5978          * Check TSB and TLB page sizes if the process isn't exiting.
5979          */
5980         if (!sfmmup->sfmmu_free)
5981                 sfmmu_check_page_sizes(sfmmup, 0);
5982 }
5983 
5984 /*
5985  * Unload all the mappings in the range [addr..addr+len). addr and len must
5986  * be MMU_PAGESIZE aligned.
5987  */
5988 void
5989 hat_unload(struct hat *sfmmup, caddr_t addr, size_t len, uint_t flags)
5990 {
5991         if (sfmmup->sfmmu_xhat_provider) {
5992                 XHAT_UNLOAD(sfmmup, addr, len, flags);
5993                 return;
5994         }
5995         hat_unload_callback(sfmmup, addr, len, flags, NULL);
5996 }
5997 
5998 
5999 /*
6000  * Find the largest mapping size for this page.
6001  */
6002 int
6003 fnd_mapping_sz(page_t *pp)
6004 {
6005         int sz;
6006         int p_index;
6007 
6008         p_index = PP_MAPINDEX(pp);
6009 
6010         sz = 0;
6011         p_index >>= 1;    /* don't care about 8K bit */
6012         for (; p_index; p_index >>= 1) {
6013                 sz++;
6014         }
6015 
6016         return (sz);
6017 }
6018 
6019 /*
6020  * This function unloads a range of addresses for an hmeblk.
6021  * It returns the next address to be unloaded.
6022  * It should be called with the hash lock held.
6023  */
6024 static caddr_t
6025 sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
6026         caddr_t endaddr, demap_range_t *dmrp, uint_t flags)
6027 {
6028         tte_t   tte, ttemod;
6029         struct  sf_hment *sfhmep;
6030         int     ttesz;
6031         long    ttecnt;
6032         page_t *pp;
6033         kmutex_t *pml;
6034         int ret;
6035         int use_demap_range;
6036 
6037         ASSERT(in_hblk_range(hmeblkp, addr));
6038         ASSERT(!hmeblkp->hblk_shw_bit);
6039         ASSERT(sfmmup != NULL || hmeblkp->hblk_shared);
6040         ASSERT(sfmmup == NULL || !hmeblkp->hblk_shared);
6041         ASSERT(dmrp == NULL || !hmeblkp->hblk_shared);
6042 
6043 #ifdef DEBUG
6044         if (get_hblk_ttesz(hmeblkp) != TTE8K &&
6045             (endaddr < get_hblk_endaddr(hmeblkp))) {
6046                 panic("sfmmu_hblk_unload: partial unload of large page");
6047         }
6048 #endif /* DEBUG */
6049 
6050         endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp));
6051         ttesz = get_hblk_ttesz(hmeblkp);
6052 
6053         use_demap_range = ((dmrp == NULL) ||
6054             (TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)));
6055 
6056         if (use_demap_range) {
6057                 DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr);
6058         } else if (dmrp != NULL) {
6059                 DEMAP_RANGE_FLUSH(dmrp);
6060         }
6061         ttecnt = 0;
6062         HBLKTOHME(sfhmep, hmeblkp, addr);
6063 
6064         while (addr < endaddr) {
6065                 pml = NULL;
6066                 sfmmu_copytte(&sfhmep->hme_tte, &tte);
6067                 if (TTE_IS_VALID(&tte)) {
6068                         pp = sfhmep->hme_page;
6069                         if (pp != NULL) {
6070                                 pml = sfmmu_mlist_enter(pp);
6071                         }
6072 
6073                         /*
6074                          * Verify if hme still points to 'pp' now that
6075                          * we have p_mapping lock.
6076                          */
6077                         if (sfhmep->hme_page != pp) {
6078                                 if (pp != NULL && sfhmep->hme_page != NULL) {
6079                                         ASSERT(pml != NULL);
6080                                         sfmmu_mlist_exit(pml);
6081                                         /* Re-start this iteration. */
6082                                         continue;
6083                                 }
6084                                 ASSERT((pp != NULL) &&
6085                                     (sfhmep->hme_page == NULL));
6086                                 goto tte_unloaded;
6087                         }
6088 
6089                         /*
6090                          * This point on we have both HASH and p_mapping
6091                          * lock.
6092                          */
6093                         ASSERT(pp == sfhmep->hme_page);
6094                         ASSERT(pp == NULL || sfmmu_mlist_held(pp));
6095 
6096                         /*
6097                          * We need to loop on modify tte because it is
6098                          * possible for pagesync to come along and
6099                          * change the software bits beneath us.
6100                          *
6101                          * Page_unload can also invalidate the tte after
6102                          * we read tte outside of p_mapping lock.
6103                          */
6104 again:
6105                         ttemod = tte;
6106 
6107                         TTE_SET_INVALID(&ttemod);
6108                         ret = sfmmu_modifytte_try(&tte, &ttemod,
6109                             &sfhmep->hme_tte);
6110 
6111                         if (ret <= 0) {
6112                                 if (TTE_IS_VALID(&tte)) {
6113                                         ASSERT(ret < 0);
6114                                         goto again;
6115                                 }
6116                                 if (pp != NULL) {
6117                                         panic("sfmmu_hblk_unload: pp = 0x%p "
6118                                             "tte became invalid under mlist"
6119                                             " lock = 0x%p", (void *)pp,
6120                                             (void *)pml);
6121                                 }
6122                                 continue;
6123                         }
6124 
6125                         if (!(flags & HAT_UNLOAD_NOSYNC)) {
6126                                 sfmmu_ttesync(sfmmup, addr, &tte, pp);
6127                         }
6128 
6129                         /*
6130                          * Ok- we invalidated the tte. Do the rest of the job.
6131                          */
6132                         ttecnt++;
6133 
6134                         if (flags & HAT_UNLOAD_UNLOCK) {
6135                                 ASSERT(hmeblkp->hblk_lckcnt > 0);
6136                                 atomic_add_32(&hmeblkp->hblk_lckcnt, -1);
6137                                 HBLK_STACK_TRACE(hmeblkp, HBLK_UNLOCK);
6138                         }
6139 
6140                         /*
6141                          * Normally we would need to flush the page
6142                          * from the virtual cache at this point in
6143                          * order to prevent a potential cache alias
6144                          * inconsistency.
6145                          * The particular scenario we need to worry
6146                          * about is:
6147                          * Given:  va1 and va2 are two virtual address
6148                          * that alias and map the same physical
6149                          * address.
6150                          * 1.   mapping exists from va1 to pa and data
6151                          * has been read into the cache.
6152                          * 2.   unload va1.
6153                          * 3.   load va2 and modify data using va2.
6154                          * 4    unload va2.
6155                          * 5.   load va1 and reference data.  Unless we
6156                          * flush the data cache when we unload we will
6157                          * get stale data.
6158                          * Fortunately, page coloring eliminates the
6159                          * above scenario by remembering the color a
6160                          * physical page was last or is currently
6161                          * mapped to.  Now, we delay the flush until
6162                          * the loading of translations.  Only when the
6163                          * new translation is of a different color
6164                          * are we forced to flush.
6165                          */
6166                         if (use_demap_range) {
6167                                 /*
6168                                  * Mark this page as needing a demap.
6169                                  */
6170                                 DEMAP_RANGE_MARKPG(dmrp, addr);
6171                         } else {
6172                                 ASSERT(sfmmup != NULL);
6173                                 ASSERT(!hmeblkp->hblk_shared);
6174                                 sfmmu_tlb_demap(addr, sfmmup, hmeblkp,
6175                                     sfmmup->sfmmu_free, 0);
6176                         }
6177 
6178                         if (pp) {
6179                                 /*
6180                                  * Remove the hment from the mapping list
6181                                  */
6182                                 ASSERT(hmeblkp->hblk_hmecnt > 0);
6183 
6184                                 /*
6185                                  * Again, we cannot
6186                                  * ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS);
6187                                  */
6188                                 HME_SUB(sfhmep, pp);
6189                                 membar_stst();
6190                                 atomic_add_16(&hmeblkp->hblk_hmecnt, -1);
6191                         }
6192 
6193                         ASSERT(hmeblkp->hblk_vcnt > 0);
6194                         atomic_add_16(&hmeblkp->hblk_vcnt, -1);
6195 
6196                         ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt ||
6197                             !hmeblkp->hblk_lckcnt);
6198 
6199 #ifdef VAC
6200                         if (pp && (pp->p_nrm & (P_KPMC | P_KPMS | P_TNC))) {
6201                                 if (PP_ISTNC(pp)) {
6202                                         /*
6203                                          * If page was temporary
6204                                          * uncached, try to recache
6205                                          * it. Note that HME_SUB() was
6206                                          * called above so p_index and
6207                                          * mlist had been updated.
6208                                          */
6209                                         conv_tnc(pp, ttesz);
6210                                 } else if (pp->p_mapping == NULL) {
6211                                         ASSERT(kpm_enable);
6212                                         /*
6213                                          * Page is marked to be in VAC conflict
6214                                          * to an existing kpm mapping and/or is
6215                                          * kpm mapped using only the regular
6216                                          * pagesize.
6217                                          */
6218                                         sfmmu_kpm_hme_unload(pp);
6219                                 }
6220                         }
6221 #endif  /* VAC */
6222                 } else if ((pp = sfhmep->hme_page) != NULL) {
6223                                 /*
6224                                  * TTE is invalid but the hme
6225                                  * still exists. let pageunload
6226                                  * complete its job.
6227                                  */
6228                                 ASSERT(pml == NULL);
6229                                 pml = sfmmu_mlist_enter(pp);
6230                                 if (sfhmep->hme_page != NULL) {
6231                                         sfmmu_mlist_exit(pml);
6232                                         continue;
6233                                 }
6234                                 ASSERT(sfhmep->hme_page == NULL);
6235                 } else if (hmeblkp->hblk_hmecnt != 0) {
6236                         /*
6237                          * pageunload may have not finished decrementing
6238                          * hblk_vcnt and hblk_hmecnt. Find page_t if any and
6239                          * wait for pageunload to finish. Rely on pageunload
6240                          * to decrement hblk_hmecnt after hblk_vcnt.
6241                          */
6242                         pfn_t pfn = TTE_TO_TTEPFN(&tte);
6243                         ASSERT(pml == NULL);
6244                         if (pf_is_memory(pfn)) {
6245                                 pp = page_numtopp_nolock(pfn);
6246                                 if (pp != NULL) {
6247                                         pml = sfmmu_mlist_enter(pp);
6248                                         sfmmu_mlist_exit(pml);
6249                                         pml = NULL;
6250                                 }
6251                         }
6252                 }
6253 
6254 tte_unloaded:
6255                 /*
6256                  * At this point, the tte we are looking at
6257                  * should be unloaded, and hme has been unlinked
6258                  * from page too. This is important because in
6259                  * pageunload, it does ttesync() then HME_SUB.
6260                  * We need to make sure HME_SUB has been completed
6261                  * so we know ttesync() has been completed. Otherwise,
6262                  * at exit time, after return from hat layer, VM will
6263                  * release as structure which hat_setstat() (called
6264                  * by ttesync()) needs.
6265                  */
6266 #ifdef DEBUG
6267                 {
6268                         tte_t   dtte;
6269 
6270                         ASSERT(sfhmep->hme_page == NULL);
6271 
6272                         sfmmu_copytte(&sfhmep->hme_tte, &dtte);
6273                         ASSERT(!TTE_IS_VALID(&dtte));
6274                 }
6275 #endif
6276 
6277                 if (pml) {
6278                         sfmmu_mlist_exit(pml);
6279                 }
6280 
6281                 addr += TTEBYTES(ttesz);
6282                 sfhmep++;
6283                 DEMAP_RANGE_NEXTPG(dmrp);
6284         }
6285         /*
6286          * For shared hmeblks this routine is only called when region is freed
6287          * and no longer referenced.  So no need to decrement ttecnt
6288          * in the region structure here.
6289          */
6290         if (ttecnt > 0 && sfmmup != NULL) {
6291                 atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -ttecnt);
6292         }
6293         return (addr);
6294 }
6295 
6296 /*
6297  * Invalidate a virtual address range for the local CPU.
6298  * For best performance ensure that the va range is completely
6299  * mapped, otherwise the entire TLB will be flushed.
6300  */
6301 void
6302 hat_flush_range(struct hat *sfmmup, caddr_t va, size_t size)
6303 {
6304         ssize_t sz;
6305         caddr_t endva = va + size;
6306 
6307         while (va < endva) {
6308                 sz = hat_getpagesize(sfmmup, va);
6309                 if (sz < 0) {
6310                         vtag_flushall();
6311                         break;
6312                 }
6313                 vtag_flushpage(va, (uint64_t)sfmmup);
6314                 va += sz;
6315         }
6316 }
6317 
6318 /*
6319  * Synchronize all the mappings in the range [addr..addr+len).
6320  * Can be called with clearflag having two states:
6321  * HAT_SYNC_DONTZERO means just return the rm stats
6322  * HAT_SYNC_ZERORM means zero rm bits in the tte and return the stats
6323  */
6324 void
6325 hat_sync(struct hat *sfmmup, caddr_t addr, size_t len, uint_t clearflag)
6326 {
6327         struct hmehash_bucket *hmebp;
6328         hmeblk_tag hblktag;
6329         int hmeshift, hashno = 1;
6330         struct hme_blk *hmeblkp, *list = NULL;
6331         caddr_t endaddr;
6332         cpuset_t cpuset;
6333 
6334         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
6335         ASSERT((sfmmup == ksfmmup) ||
6336             AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
6337         ASSERT((len & MMU_PAGEOFFSET) == 0);
6338         ASSERT((clearflag == HAT_SYNC_DONTZERO) ||
6339             (clearflag == HAT_SYNC_ZERORM));
6340 
6341         CPUSET_ZERO(cpuset);
6342 
6343         endaddr = addr + len;
6344         hblktag.htag_id = sfmmup;
6345         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
6346 
6347         /*
6348          * Spitfire supports 4 page sizes.
6349          * Most pages are expected to be of the smallest page
6350          * size (8K) and these will not need to be rehashed. 64K
6351          * pages also don't need to be rehashed because the an hmeblk
6352          * spans 64K of address space. 512K pages might need 1 rehash and
6353          * and 4M pages 2 rehashes.
6354          */
6355         while (addr < endaddr) {
6356                 hmeshift = HME_HASH_SHIFT(hashno);
6357                 hblktag.htag_bspage = HME_HASH_BSPAGE(addr, hmeshift);
6358                 hblktag.htag_rehash = hashno;
6359                 hmebp = HME_HASH_FUNCTION(sfmmup, addr, hmeshift);
6360 
6361                 SFMMU_HASH_LOCK(hmebp);
6362 
6363                 HME_HASH_SEARCH(hmebp, hblktag, hmeblkp, &list);
6364                 if (hmeblkp != NULL) {
6365                         ASSERT(!hmeblkp->hblk_shared);
6366                         /*
6367                          * We've encountered a shadow hmeblk so skip the range
6368                          * of the next smaller mapping size.
6369                          */
6370                         if (hmeblkp->hblk_shw_bit) {
6371                                 ASSERT(sfmmup != ksfmmup);
6372                                 ASSERT(hashno > 1);
6373                                 addr = (caddr_t)P2END((uintptr_t)addr,
6374                                     TTEBYTES(hashno - 1));
6375                         } else {
6376                                 addr = sfmmu_hblk_sync(sfmmup, hmeblkp,
6377                                     addr, endaddr, clearflag);
6378                         }
6379                         SFMMU_HASH_UNLOCK(hmebp);
6380                         hashno = 1;
6381                         continue;
6382                 }
6383                 SFMMU_HASH_UNLOCK(hmebp);
6384 
6385                 if (!HME_REHASH(sfmmup) || (hashno >= mmu_hashcnt)) {
6386                         /*
6387                          * We have traversed the whole list and rehashed
6388                          * if necessary without finding the address to sync.
6389                          * This is ok so we increment the address by the
6390                          * smallest hmeblk range for kernel mappings and the
6391                          * largest hmeblk range, to account for shadow hmeblks,
6392                          * for user mappings and continue.
6393                          */
6394                         if (sfmmup == ksfmmup)
6395                                 addr = (caddr_t)P2END((uintptr_t)addr,
6396                                     TTEBYTES(1));
6397                         else
6398                                 addr = (caddr_t)P2END((uintptr_t)addr,
6399                                     TTEBYTES(hashno));
6400                         hashno = 1;
6401                 } else {
6402                         hashno++;
6403                 }
6404         }
6405         sfmmu_hblks_list_purge(&list, 0);
6406         cpuset = sfmmup->sfmmu_cpusran;
6407         xt_sync(cpuset);
6408 }
6409 
6410 static caddr_t
6411 sfmmu_hblk_sync(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr,
6412         caddr_t endaddr, int clearflag)
6413 {
6414         tte_t   tte, ttemod;
6415         struct sf_hment *sfhmep;
6416         int ttesz;
6417         struct page *pp;
6418         kmutex_t *pml;
6419         int ret;
6420 
6421         ASSERT(hmeblkp->hblk_shw_bit == 0);
6422         ASSERT(!hmeblkp->hblk_shared);
6423 
6424         endaddr = MIN(endaddr, get_hblk_endaddr(hmeblkp));
6425 
6426         ttesz = get_hblk_ttesz(hmeblkp);
6427         HBLKTOHME(sfhmep, hmeblkp, addr);
6428 
6429         while (addr < endaddr) {
6430                 sfmmu_copytte(&sfhmep->hme_tte, &tte);
6431                 if (TTE_IS_VALID(&tte)) {
6432                         pml = NULL;
6433                         pp = sfhmep->hme_page;
6434                         if (pp) {
6435                                 pml = sfmmu_mlist_enter(pp);
6436                         }
6437                         if (pp != sfhmep->hme_page) {
6438                                 /*
6439                                  * tte most have been unloaded
6440                                  * underneath us.  Recheck
6441                                  */
6442                                 ASSERT(pml);
6443                                 sfmmu_mlist_exit(pml);
6444                                 continue;
6445                         }
6446 
6447                         ASSERT(pp == NULL || sfmmu_mlist_held(pp));
6448 
6449                         if (clearflag == HAT_SYNC_ZERORM) {
6450                                 ttemod = tte;
6451                                 TTE_CLR_RM(&ttemod);
6452                                 ret = sfmmu_modifytte_try(&tte, &ttemod,
6453                                     &sfhmep->hme_tte);
6454                                 if (ret < 0) {
6455                                         if (pml) {
6456                                                 sfmmu_mlist_exit(pml);
6457                                         }
6458                                         continue;
6459                                 }
6460 
6461                                 if (ret > 0) {
6462                                         sfmmu_tlb_demap(addr, sfmmup,
6463                                             hmeblkp, 0, 0);
6464                                 }
6465                         }
6466                         sfmmu_ttesync(sfmmup, addr, &tte, pp);
6467                         if (pml) {
6468                                 sfmmu_mlist_exit(pml);
6469                         }
6470                 }
6471                 addr += TTEBYTES(ttesz);
6472                 sfhmep++;
6473         }
6474         return (addr);
6475 }
6476 
6477 /*
6478  * This function will sync a tte to the page struct and it will
6479  * update the hat stats. Currently it allows us to pass a NULL pp
6480  * and we will simply update the stats.  We may want to change this
6481  * so we only keep stats for pages backed by pp's.
6482  */
6483 static void
6484 sfmmu_ttesync(struct hat *sfmmup, caddr_t addr, tte_t *ttep, page_t *pp)
6485 {
6486         uint_t rm = 0;
6487         int     sz;
6488         pgcnt_t npgs;
6489 
6490         ASSERT(TTE_IS_VALID(ttep));
6491 
6492         if (TTE_IS_NOSYNC(ttep)) {
6493                 return;
6494         }
6495 
6496         if (TTE_IS_REF(ttep))  {
6497                 rm = P_REF;
6498         }
6499         if (TTE_IS_MOD(ttep))  {
6500                 rm |= P_MOD;
6501         }
6502 
6503         if (rm == 0) {
6504                 return;
6505         }
6506 
6507         sz = TTE_CSZ(ttep);
6508         if (sfmmup != NULL && sfmmup->sfmmu_rmstat) {
6509                 int i;
6510                 caddr_t vaddr = addr;
6511 
6512                 for (i = 0; i < TTEPAGES(sz); i++, vaddr += MMU_PAGESIZE) {
6513                         hat_setstat(sfmmup->sfmmu_as, vaddr, MMU_PAGESIZE, rm);
6514                 }
6515 
6516         }
6517 
6518         /*
6519          * XXX I want to use cas to update nrm bits but they
6520          * currently belong in common/vm and not in hat where
6521          * they should be.
6522          * The nrm bits are protected by the same mutex as
6523          * the one that protects the page's mapping list.
6524          */
6525         if (!pp)
6526                 return;
6527         ASSERT(sfmmu_mlist_held(pp));
6528         /*
6529          * If the tte is for a large page, we need to sync all the
6530          * pages covered by the tte.
6531          */
6532         if (sz != TTE8K) {
6533                 ASSERT(pp->p_szc != 0);
6534                 pp = PP_GROUPLEADER(pp, sz);
6535                 ASSERT(sfmmu_mlist_held(pp));
6536         }
6537 
6538         /* Get number of pages from tte size. */
6539         npgs = TTEPAGES(sz);
6540 
6541         do {
6542                 ASSERT(pp);
6543                 ASSERT(sfmmu_mlist_held(pp));
6544                 if (((rm & P_REF) != 0 && !PP_ISREF(pp)) ||
6545                     ((rm & P_MOD) != 0 && !PP_ISMOD(pp)))
6546                         hat_page_setattr(pp, rm);
6547 
6548                 /*
6549                  * Are we done? If not, we must have a large mapping.
6550                  * For large mappings we need to sync the rest of the pages
6551                  * covered by this tte; goto the next page.
6552                  */
6553         } while (--npgs > 0 && (pp = PP_PAGENEXT(pp)));
6554 }
6555 
6556 /*
6557  * Execute pre-callback handler of each pa_hment linked to pp
6558  *
6559  * Inputs:
6560  *   flag: either HAT_PRESUSPEND or HAT_SUSPEND.
6561  *   capture_cpus: pointer to return value (below)
6562  *
6563  * Returns:
6564  *   Propagates the subsystem callback return values back to the caller;
6565  *   returns 0 on success.  If capture_cpus is non-NULL, the value returned
6566  *   is zero if all of the pa_hments are of a type that do not require
6567  *   capturing CPUs prior to suspending the mapping, else it is 1.
6568  */
6569 static int
6570 hat_pageprocess_precallbacks(struct page *pp, uint_t flag, int *capture_cpus)
6571 {
6572         struct sf_hment *sfhmep;
6573         struct pa_hment *pahmep;
6574         int (*f)(caddr_t, uint_t, uint_t, void *);
6575         int             ret;
6576         id_t            id;
6577         int             locked = 0;
6578         kmutex_t        *pml;
6579 
6580         ASSERT(PAGE_EXCL(pp));
6581         if (!sfmmu_mlist_held(pp)) {
6582                 pml = sfmmu_mlist_enter(pp);
6583                 locked = 1;
6584         }
6585 
6586         if (capture_cpus)
6587                 *capture_cpus = 0;
6588 
6589 top:
6590         for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) {
6591                 /*
6592                  * skip sf_hments corresponding to VA<->PA mappings;
6593                  * for pa_hment's, hme_tte.ll is zero
6594                  */
6595                 if (!IS_PAHME(sfhmep))
6596                         continue;
6597 
6598                 pahmep = sfhmep->hme_data;
6599                 ASSERT(pahmep != NULL);
6600 
6601                 /*
6602                  * skip if pre-handler has been called earlier in this loop
6603                  */
6604                 if (pahmep->flags & flag)
6605                         continue;
6606 
6607                 id = pahmep->cb_id;
6608                 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid);
6609                 if (capture_cpus && sfmmu_cb_table[id].capture_cpus != 0)
6610                         *capture_cpus = 1;
6611                 if ((f = sfmmu_cb_table[id].prehandler) == NULL) {
6612                         pahmep->flags |= flag;
6613                         continue;
6614                 }
6615 
6616                 /*
6617                  * Drop the mapping list lock to avoid locking order issues.
6618                  */
6619                 if (locked)
6620                         sfmmu_mlist_exit(pml);
6621 
6622                 ret = f(pahmep->addr, pahmep->len, flag, pahmep->pvt);
6623                 if (ret != 0)
6624                         return (ret);   /* caller must do the cleanup */
6625 
6626                 if (locked) {
6627                         pml = sfmmu_mlist_enter(pp);
6628                         pahmep->flags |= flag;
6629                         goto top;
6630                 }
6631 
6632                 pahmep->flags |= flag;
6633         }
6634 
6635         if (locked)
6636                 sfmmu_mlist_exit(pml);
6637 
6638         return (0);
6639 }
6640 
6641 /*
6642  * Execute post-callback handler of each pa_hment linked to pp
6643  *
6644  * Same overall assumptions and restrictions apply as for
6645  * hat_pageprocess_precallbacks().
6646  */
6647 static void
6648 hat_pageprocess_postcallbacks(struct page *pp, uint_t flag)
6649 {
6650         pfn_t pgpfn = pp->p_pagenum;
6651         pfn_t pgmask = btop(page_get_pagesize(pp->p_szc)) - 1;
6652         pfn_t newpfn;
6653         struct sf_hment *sfhmep;
6654         struct pa_hment *pahmep;
6655         int (*f)(caddr_t, uint_t, uint_t, void *, pfn_t);
6656         id_t    id;
6657         int     locked = 0;
6658         kmutex_t *pml;
6659 
6660         ASSERT(PAGE_EXCL(pp));
6661         if (!sfmmu_mlist_held(pp)) {
6662                 pml = sfmmu_mlist_enter(pp);
6663                 locked = 1;
6664         }
6665 
6666 top:
6667         for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) {
6668                 /*
6669                  * skip sf_hments corresponding to VA<->PA mappings;
6670                  * for pa_hment's, hme_tte.ll is zero
6671                  */
6672                 if (!IS_PAHME(sfhmep))
6673                         continue;
6674 
6675                 pahmep = sfhmep->hme_data;
6676                 ASSERT(pahmep != NULL);
6677 
6678                 if ((pahmep->flags & flag) == 0)
6679                         continue;
6680 
6681                 pahmep->flags &= ~flag;
6682 
6683                 id = pahmep->cb_id;
6684                 ASSERT(id >= (id_t)0 && id < sfmmu_cb_nextid);
6685                 if ((f = sfmmu_cb_table[id].posthandler) == NULL)
6686                         continue;
6687 
6688                 /*
6689                  * Convert the base page PFN into the constituent PFN
6690                  * which is needed by the callback handler.
6691                  */
6692                 newpfn = pgpfn | (btop((uintptr_t)pahmep->addr) & pgmask);
6693 
6694                 /*
6695                  * Drop the mapping list lock to avoid locking order issues.
6696                  */
6697                 if (locked)
6698                         sfmmu_mlist_exit(pml);
6699 
6700                 if (f(pahmep->addr, pahmep->len, flag, pahmep->pvt, newpfn)
6701                     != 0)
6702                         panic("sfmmu: posthandler failed");
6703 
6704                 if (locked) {
6705                         pml = sfmmu_mlist_enter(pp);
6706                         goto top;
6707                 }
6708         }
6709 
6710         if (locked)
6711                 sfmmu_mlist_exit(pml);
6712 }
6713 
6714 /*
6715  * Suspend locked kernel mapping
6716  */
6717 void
6718 hat_pagesuspend(struct page *pp)
6719 {
6720         struct sf_hment *sfhmep;
6721         sfmmu_t *sfmmup;
6722         tte_t tte, ttemod;
6723         struct hme_blk *hmeblkp;
6724         caddr_t addr;
6725         int index, cons;
6726         cpuset_t cpuset;
6727 
6728         ASSERT(PAGE_EXCL(pp));
6729         ASSERT(sfmmu_mlist_held(pp));
6730 
6731         mutex_enter(&kpr_suspendlock);
6732 
6733         /*
6734          * We're about to suspend a kernel mapping so mark this thread as
6735          * non-traceable by DTrace. This prevents us from running into issues
6736          * with probe context trying to touch a suspended page
6737          * in the relocation codepath itself.
6738          */
6739         curthread->t_flag |= T_DONTDTRACE;
6740 
6741         index = PP_MAPINDEX(pp);
6742         cons = TTE8K;
6743 
6744 retry:
6745         for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) {
6746 
6747                 if (IS_PAHME(sfhmep))
6748                         continue;
6749 
6750                 if (get_hblk_ttesz(sfmmu_hmetohblk(sfhmep)) != cons)
6751                         continue;
6752 
6753                 /*
6754                  * Loop until we successfully set the suspend bit in
6755                  * the TTE.
6756                  */
6757 again:
6758                 sfmmu_copytte(&sfhmep->hme_tte, &tte);
6759                 ASSERT(TTE_IS_VALID(&tte));
6760 
6761                 ttemod = tte;
6762                 TTE_SET_SUSPEND(&ttemod);
6763                 if (sfmmu_modifytte_try(&tte, &ttemod,
6764                     &sfhmep->hme_tte) < 0)
6765                         goto again;
6766 
6767                 /*
6768                  * Invalidate TSB entry
6769                  */
6770                 hmeblkp = sfmmu_hmetohblk(sfhmep);
6771 
6772                 sfmmup = hblktosfmmu(hmeblkp);
6773                 ASSERT(sfmmup == ksfmmup);
6774                 ASSERT(!hmeblkp->hblk_shared);
6775 
6776                 addr = tte_to_vaddr(hmeblkp, tte);
6777 
6778                 /*
6779                  * No need to make sure that the TSB for this sfmmu is
6780                  * not being relocated since it is ksfmmup and thus it
6781                  * will never be relocated.
6782                  */
6783                 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
6784 
6785                 /*
6786                  * Update xcall stats
6787                  */
6788                 cpuset = cpu_ready_set;
6789                 CPUSET_DEL(cpuset, CPU->cpu_id);
6790 
6791                 /* LINTED: constant in conditional context */
6792                 SFMMU_XCALL_STATS(ksfmmup);
6793 
6794                 /*
6795                  * Flush TLB entry on remote CPU's
6796                  */
6797                 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr,
6798                     (uint64_t)ksfmmup);
6799                 xt_sync(cpuset);
6800 
6801                 /*
6802                  * Flush TLB entry on local CPU
6803                  */
6804                 vtag_flushpage(addr, (uint64_t)ksfmmup);
6805         }
6806 
6807         while (index != 0) {
6808                 index = index >> 1;
6809                 if (index != 0)
6810                         cons++;
6811                 if (index & 0x1) {
6812                         pp = PP_GROUPLEADER(pp, cons);
6813                         goto retry;
6814                 }
6815         }
6816 }
6817 
6818 #ifdef  DEBUG
6819 
6820 #define N_PRLE  1024
6821 struct prle {
6822         page_t *targ;
6823         page_t *repl;
6824         int status;
6825         int pausecpus;
6826         hrtime_t whence;
6827 };
6828 
6829 static struct prle page_relocate_log[N_PRLE];
6830 static int prl_entry;
6831 static kmutex_t prl_mutex;
6832 
6833 #define PAGE_RELOCATE_LOG(t, r, s, p)                                   \
6834         mutex_enter(&prl_mutex);                                    \
6835         page_relocate_log[prl_entry].targ = *(t);                       \
6836         page_relocate_log[prl_entry].repl = *(r);                       \
6837         page_relocate_log[prl_entry].status = (s);                      \
6838         page_relocate_log[prl_entry].pausecpus = (p);                   \
6839         page_relocate_log[prl_entry].whence = gethrtime();              \
6840         prl_entry = (prl_entry == (N_PRLE - 1))? 0 : prl_entry + 1;     \
6841         mutex_exit(&prl_mutex);
6842 
6843 #else   /* !DEBUG */
6844 #define PAGE_RELOCATE_LOG(t, r, s, p)
6845 #endif
6846 
6847 /*
6848  * Core Kernel Page Relocation Algorithm
6849  *
6850  * Input:
6851  *
6852  * target :     constituent pages are SE_EXCL locked.
6853  * replacement: constituent pages are SE_EXCL locked.
6854  *
6855  * Output:
6856  *
6857  * nrelocp:     number of pages relocated
6858  */
6859 int
6860 hat_page_relocate(page_t **target, page_t **replacement, spgcnt_t *nrelocp)
6861 {
6862         page_t          *targ, *repl;
6863         page_t          *tpp, *rpp;
6864         kmutex_t        *low, *high;
6865         spgcnt_t        npages, i;
6866         page_t          *pl = NULL;
6867         int             old_pil;
6868         cpuset_t        cpuset;
6869         int             cap_cpus;
6870         int             ret;
6871 #ifdef VAC
6872         int             cflags = 0;
6873 #endif
6874 
6875         if (!kcage_on || PP_ISNORELOC(*target)) {
6876                 PAGE_RELOCATE_LOG(target, replacement, EAGAIN, -1);
6877                 return (EAGAIN);
6878         }
6879 
6880         mutex_enter(&kpr_mutex);
6881         kreloc_thread = curthread;
6882 
6883         targ = *target;
6884         repl = *replacement;
6885         ASSERT(repl != NULL);
6886         ASSERT(targ->p_szc == repl->p_szc);
6887 
6888         npages = page_get_pagecnt(targ->p_szc);
6889 
6890         /*
6891          * unload VA<->PA mappings that are not locked
6892          */
6893         tpp = targ;
6894         for (i = 0; i < npages; i++) {
6895                 (void) hat_pageunload(tpp, SFMMU_KERNEL_RELOC);
6896                 tpp++;
6897         }
6898 
6899         /*
6900          * Do "presuspend" callbacks, in a context from which we can still
6901          * block as needed. Note that we don't hold the mapping list lock
6902          * of "targ" at this point due to potential locking order issues;
6903          * we assume that between the hat_pageunload() above and holding
6904          * the SE_EXCL lock that the mapping list *cannot* change at this
6905          * point.
6906          */
6907         ret = hat_pageprocess_precallbacks(targ, HAT_PRESUSPEND, &cap_cpus);
6908         if (ret != 0) {
6909                 /*
6910                  * EIO translates to fatal error, for all others cleanup
6911                  * and return EAGAIN.
6912                  */
6913                 ASSERT(ret != EIO);
6914                 hat_pageprocess_postcallbacks(targ, HAT_POSTUNSUSPEND);
6915                 PAGE_RELOCATE_LOG(target, replacement, ret, -1);
6916                 kreloc_thread = NULL;
6917                 mutex_exit(&kpr_mutex);
6918                 return (EAGAIN);
6919         }
6920 
6921         /*
6922          * acquire p_mapping list lock for both the target and replacement
6923          * root pages.
6924          *
6925          * low and high refer to the need to grab the mlist locks in a
6926          * specific order in order to prevent race conditions.  Thus the
6927          * lower lock must be grabbed before the higher lock.
6928          *
6929          * This will block hat_unload's accessing p_mapping list.  Since
6930          * we have SE_EXCL lock, hat_memload and hat_pageunload will be
6931          * blocked.  Thus, no one else will be accessing the p_mapping list
6932          * while we suspend and reload the locked mapping below.
6933          */
6934         tpp = targ;
6935         rpp = repl;
6936         sfmmu_mlist_reloc_enter(tpp, rpp, &low, &high);
6937 
6938         kpreempt_disable();
6939 
6940         /*
6941          * We raise our PIL to 13 so that we don't get captured by
6942          * another CPU or pinned by an interrupt thread.  We can't go to
6943          * PIL 14 since the nexus driver(s) may need to interrupt at
6944          * that level in the case of IOMMU pseudo mappings.
6945          */
6946         cpuset = cpu_ready_set;
6947         CPUSET_DEL(cpuset, CPU->cpu_id);
6948         if (!cap_cpus || CPUSET_ISNULL(cpuset)) {
6949                 old_pil = splr(XCALL_PIL);
6950         } else {
6951                 old_pil = -1;
6952                 xc_attention(cpuset);
6953         }
6954         ASSERT(getpil() == XCALL_PIL);
6955 
6956         /*
6957          * Now do suspend callbacks. In the case of an IOMMU mapping
6958          * this will suspend all DMA activity to the page while it is
6959          * being relocated. Since we are well above LOCK_LEVEL and CPUs
6960          * may be captured at this point we should have acquired any needed
6961          * locks in the presuspend callback.
6962          */
6963         ret = hat_pageprocess_precallbacks(targ, HAT_SUSPEND, NULL);
6964         if (ret != 0) {
6965                 repl = targ;
6966                 goto suspend_fail;
6967         }
6968 
6969         /*
6970          * Raise the PIL yet again, this time to block all high-level
6971          * interrupts on this CPU. This is necessary to prevent an
6972          * interrupt routine from pinning the thread which holds the
6973          * mapping suspended and then touching the suspended page.
6974          *
6975          * Once the page is suspended we also need to be careful to
6976          * avoid calling any functions which touch any seg_kmem memory
6977          * since that memory may be backed by the very page we are
6978          * relocating in here!
6979          */
6980         hat_pagesuspend(targ);
6981 
6982         /*
6983          * Now that we are confident everybody has stopped using this page,
6984          * copy the page contents.  Note we use a physical copy to prevent
6985          * locking issues and to avoid fpRAS because we can't handle it in
6986          * this context.
6987          */
6988         for (i = 0; i < npages; i++, tpp++, rpp++) {
6989 #ifdef VAC
6990                 /*
6991                  * If the replacement has a different vcolor than
6992                  * the one being replacd, we need to handle VAC
6993                  * consistency for it just as we were setting up
6994                  * a new mapping to it.
6995                  */
6996                 if ((PP_GET_VCOLOR(rpp) != NO_VCOLOR) &&
6997                     (tpp->p_vcolor != rpp->p_vcolor) &&
6998                     !CacheColor_IsFlushed(cflags, PP_GET_VCOLOR(rpp))) {
6999                         CacheColor_SetFlushed(cflags, PP_GET_VCOLOR(rpp));
7000                         sfmmu_cache_flushcolor(PP_GET_VCOLOR(rpp),
7001                             rpp->p_pagenum);
7002                 }
7003 #endif
7004                 /*
7005                  * Copy the contents of the page.
7006                  */
7007                 ppcopy_kernel(tpp, rpp);
7008         }
7009 
7010         tpp = targ;
7011         rpp = repl;
7012         for (i = 0; i < npages; i++, tpp++, rpp++) {
7013                 /*
7014                  * Copy attributes.  VAC consistency was handled above,
7015                  * if required.
7016                  */
7017                 rpp->p_nrm = tpp->p_nrm;
7018                 tpp->p_nrm = 0;
7019                 rpp->p_index = tpp->p_index;
7020                 tpp->p_index = 0;
7021 #ifdef VAC
7022                 rpp->p_vcolor = tpp->p_vcolor;
7023 #endif
7024         }
7025 
7026         /*
7027          * First, unsuspend the page, if we set the suspend bit, and transfer
7028          * the mapping list from the target page to the replacement page.
7029          * Next process postcallbacks; since pa_hment's are linked only to the
7030          * p_mapping list of root page, we don't iterate over the constituent
7031          * pages.
7032          */
7033         hat_pagereload(targ, repl);
7034 
7035 suspend_fail:
7036         hat_pageprocess_postcallbacks(repl, HAT_UNSUSPEND);
7037 
7038         /*
7039          * Now lower our PIL and release any captured CPUs since we
7040          * are out of the "danger zone".  After this it will again be
7041          * safe to acquire adaptive mutex locks, or to drop them...
7042          */
7043         if (old_pil != -1) {
7044                 splx(old_pil);
7045         } else {
7046                 xc_dismissed(cpuset);
7047         }
7048 
7049         kpreempt_enable();
7050 
7051         sfmmu_mlist_reloc_exit(low, high);
7052 
7053         /*
7054          * Postsuspend callbacks should drop any locks held across
7055          * the suspend callbacks.  As before, we don't hold the mapping
7056          * list lock at this point.. our assumption is that the mapping
7057          * list still can't change due to our holding SE_EXCL lock and
7058          * there being no unlocked mappings left. Hence the restriction
7059          * on calling context to hat_delete_callback()
7060          */
7061         hat_pageprocess_postcallbacks(repl, HAT_POSTUNSUSPEND);
7062         if (ret != 0) {
7063                 /*
7064                  * The second presuspend call failed: we got here through
7065                  * the suspend_fail label above.
7066                  */
7067                 ASSERT(ret != EIO);
7068                 PAGE_RELOCATE_LOG(target, replacement, ret, cap_cpus);
7069                 kreloc_thread = NULL;
7070                 mutex_exit(&kpr_mutex);
7071                 return (EAGAIN);
7072         }
7073 
7074         /*
7075          * Now that we're out of the performance critical section we can
7076          * take care of updating the hash table, since we still
7077          * hold all the pages locked SE_EXCL at this point we
7078          * needn't worry about things changing out from under us.
7079          */
7080         tpp = targ;
7081         rpp = repl;
7082         for (i = 0; i < npages; i++, tpp++, rpp++) {
7083 
7084                 /*
7085                  * replace targ with replacement in page_hash table
7086                  */
7087                 targ = tpp;
7088                 page_relocate_hash(rpp, targ);
7089 
7090                 /*
7091                  * concatenate target; caller of platform_page_relocate()
7092                  * expects target to be concatenated after returning.
7093                  */
7094                 ASSERT(targ->p_next == targ);
7095                 ASSERT(targ->p_prev == targ);
7096                 page_list_concat(&pl, &targ);
7097         }
7098 
7099         ASSERT(*target == pl);
7100         *nrelocp = npages;
7101         PAGE_RELOCATE_LOG(target, replacement, 0, cap_cpus);
7102         kreloc_thread = NULL;
7103         mutex_exit(&kpr_mutex);
7104         return (0);
7105 }
7106 
7107 /*
7108  * Called when stray pa_hments are found attached to a page which is
7109  * being freed.  Notify the subsystem which attached the pa_hment of
7110  * the error if it registered a suitable handler, else panic.
7111  */
7112 static void
7113 sfmmu_pahment_leaked(struct pa_hment *pahmep)
7114 {
7115         id_t cb_id = pahmep->cb_id;
7116 
7117         ASSERT(cb_id >= (id_t)0 && cb_id < sfmmu_cb_nextid);
7118         if (sfmmu_cb_table[cb_id].errhandler != NULL) {
7119                 if (sfmmu_cb_table[cb_id].errhandler(pahmep->addr, pahmep->len,
7120                     HAT_CB_ERR_LEAKED, pahmep->pvt) == 0)
7121                         return;         /* non-fatal */
7122         }
7123         panic("pa_hment leaked: 0x%p", (void *)pahmep);
7124 }
7125 
7126 /*
7127  * Remove all mappings to page 'pp'.
7128  */
7129 int
7130 hat_pageunload(struct page *pp, uint_t forceflag)
7131 {
7132         struct page *origpp = pp;
7133         struct sf_hment *sfhme, *tmphme;
7134         struct hme_blk *hmeblkp;
7135         kmutex_t *pml;
7136 #ifdef VAC
7137         kmutex_t *pmtx;
7138 #endif
7139         cpuset_t cpuset, tset;
7140         int index, cons;
7141         int xhme_blks;
7142         int pa_hments;
7143 
7144         ASSERT(PAGE_EXCL(pp));
7145 
7146 retry_xhat:
7147         tmphme = NULL;
7148         xhme_blks = 0;
7149         pa_hments = 0;
7150         CPUSET_ZERO(cpuset);
7151 
7152         pml = sfmmu_mlist_enter(pp);
7153 
7154 #ifdef VAC
7155         if (pp->p_kpmref)
7156                 sfmmu_kpm_pageunload(pp);
7157         ASSERT(!PP_ISMAPPED_KPM(pp));
7158 #endif
7159         /*
7160          * Clear vpm reference. Since the page is exclusively locked
7161          * vpm cannot be referencing it.
7162          */
7163         if (vpm_enable) {
7164                 pp->p_vpmref = 0;
7165         }
7166 
7167         index = PP_MAPINDEX(pp);
7168         cons = TTE8K;
7169 retry:
7170         for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) {
7171                 tmphme = sfhme->hme_next;
7172 
7173                 if (IS_PAHME(sfhme)) {
7174                         ASSERT(sfhme->hme_data != NULL);
7175                         pa_hments++;
7176                         continue;
7177                 }
7178 
7179                 hmeblkp = sfmmu_hmetohblk(sfhme);
7180                 if (hmeblkp->hblk_xhat_bit) {
7181                         struct xhat_hme_blk *xblk =
7182                             (struct xhat_hme_blk *)hmeblkp;
7183 
7184                         (void) XHAT_PAGEUNLOAD(xblk->xhat_hme_blk_hat,
7185                             pp, forceflag, XBLK2PROVBLK(xblk));
7186 
7187                         xhme_blks = 1;
7188                         continue;
7189                 }
7190 
7191                 /*
7192                  * If there are kernel mappings don't unload them, they will
7193                  * be suspended.
7194                  */
7195                 if (forceflag == SFMMU_KERNEL_RELOC && hmeblkp->hblk_lckcnt &&
7196                     hmeblkp->hblk_tag.htag_id == ksfmmup)
7197                         continue;
7198 
7199                 tset = sfmmu_pageunload(pp, sfhme, cons);
7200                 CPUSET_OR(cpuset, tset);
7201         }
7202 
7203         while (index != 0) {
7204                 index = index >> 1;
7205                 if (index != 0)
7206                         cons++;
7207                 if (index & 0x1) {
7208                         /* Go to leading page */
7209                         pp = PP_GROUPLEADER(pp, cons);
7210                         ASSERT(sfmmu_mlist_held(pp));
7211                         goto retry;
7212                 }
7213         }
7214 
7215         /*
7216          * cpuset may be empty if the page was only mapped by segkpm,
7217          * in which case we won't actually cross-trap.
7218          */
7219         xt_sync(cpuset);
7220 
7221         /*
7222          * The page should have no mappings at this point, unless
7223          * we were called from hat_page_relocate() in which case we
7224          * leave the locked mappings which will be suspended later.
7225          */
7226         ASSERT(!PP_ISMAPPED(origpp) || xhme_blks || pa_hments ||
7227             (forceflag == SFMMU_KERNEL_RELOC));
7228 
7229 #ifdef VAC
7230         if (PP_ISTNC(pp)) {
7231                 if (cons == TTE8K) {
7232                         pmtx = sfmmu_page_enter(pp);
7233                         PP_CLRTNC(pp);
7234                         sfmmu_page_exit(pmtx);
7235                 } else {
7236                         conv_tnc(pp, cons);
7237                 }
7238         }
7239 #endif  /* VAC */
7240 
7241         if (pa_hments && forceflag != SFMMU_KERNEL_RELOC) {
7242                 /*
7243                  * Unlink any pa_hments and free them, calling back
7244                  * the responsible subsystem to notify it of the error.
7245                  * This can occur in situations such as drivers leaking
7246                  * DMA handles: naughty, but common enough that we'd like
7247                  * to keep the system running rather than bringing it
7248                  * down with an obscure error like "pa_hment leaked"
7249                  * which doesn't aid the user in debugging their driver.
7250                  */
7251                 for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) {
7252                         tmphme = sfhme->hme_next;
7253                         if (IS_PAHME(sfhme)) {
7254                                 struct pa_hment *pahmep = sfhme->hme_data;
7255                                 sfmmu_pahment_leaked(pahmep);
7256                                 HME_SUB(sfhme, pp);
7257                                 kmem_cache_free(pa_hment_cache, pahmep);
7258                         }
7259                 }
7260 
7261                 ASSERT(!PP_ISMAPPED(origpp) || xhme_blks);
7262         }
7263 
7264         sfmmu_mlist_exit(pml);
7265 
7266         /*
7267          * XHAT may not have finished unloading pages
7268          * because some other thread was waiting for
7269          * mlist lock and XHAT_PAGEUNLOAD let it do
7270          * the job.
7271          */
7272         if (xhme_blks) {
7273                 pp = origpp;
7274                 goto retry_xhat;
7275         }
7276 
7277         return (0);
7278 }
7279 
7280 cpuset_t
7281 sfmmu_pageunload(page_t *pp, struct sf_hment *sfhme, int cons)
7282 {
7283         struct hme_blk *hmeblkp;
7284         sfmmu_t *sfmmup;
7285         tte_t tte, ttemod;
7286 #ifdef DEBUG
7287         tte_t orig_old;
7288 #endif /* DEBUG */
7289         caddr_t addr;
7290         int ttesz;
7291         int ret;
7292         cpuset_t cpuset;
7293 
7294         ASSERT(pp != NULL);
7295         ASSERT(sfmmu_mlist_held(pp));
7296         ASSERT(!PP_ISKAS(pp));
7297 
7298         CPUSET_ZERO(cpuset);
7299 
7300         hmeblkp = sfmmu_hmetohblk(sfhme);
7301 
7302 readtte:
7303         sfmmu_copytte(&sfhme->hme_tte, &tte);
7304         if (TTE_IS_VALID(&tte)) {
7305                 sfmmup = hblktosfmmu(hmeblkp);
7306                 ttesz = get_hblk_ttesz(hmeblkp);
7307                 /*
7308                  * Only unload mappings of 'cons' size.
7309                  */
7310                 if (ttesz != cons)
7311                         return (cpuset);
7312 
7313                 /*
7314                  * Note that we have p_mapping lock, but no hash lock here.
7315                  * hblk_unload() has to have both hash lock AND p_mapping
7316                  * lock before it tries to modify tte. So, the tte could
7317                  * not become invalid in the sfmmu_modifytte_try() below.
7318                  */
7319                 ttemod = tte;
7320 #ifdef DEBUG
7321                 orig_old = tte;
7322 #endif /* DEBUG */
7323 
7324                 TTE_SET_INVALID(&ttemod);
7325                 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte);
7326                 if (ret < 0) {
7327 #ifdef DEBUG
7328                         /* only R/M bits can change. */
7329                         chk_tte(&orig_old, &tte, &ttemod, hmeblkp);
7330 #endif /* DEBUG */
7331                         goto readtte;
7332                 }
7333 
7334                 if (ret == 0) {
7335                         panic("pageunload: cas failed?");
7336                 }
7337 
7338                 addr = tte_to_vaddr(hmeblkp, tte);
7339 
7340                 if (hmeblkp->hblk_shared) {
7341                         sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7342                         uint_t rid = hmeblkp->hblk_tag.htag_rid;
7343                         sf_region_t *rgnp;
7344                         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
7345                         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
7346                         ASSERT(srdp != NULL);
7347                         rgnp = srdp->srd_hmergnp[rid];
7348                         SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
7349                         cpuset = sfmmu_rgntlb_demap(addr, rgnp, hmeblkp, 1);
7350                         sfmmu_ttesync(NULL, addr, &tte, pp);
7351                         ASSERT(rgnp->rgn_ttecnt[ttesz] > 0);
7352                         atomic_add_long(&rgnp->rgn_ttecnt[ttesz], -1);
7353                 } else {
7354                         sfmmu_ttesync(sfmmup, addr, &tte, pp);
7355                         atomic_add_long(&sfmmup->sfmmu_ttecnt[ttesz], -1);
7356 
7357                         /*
7358                          * We need to flush the page from the virtual cache
7359                          * in order to prevent a virtual cache alias
7360                          * inconsistency. The particular scenario we need
7361                          * to worry about is:
7362                          * Given:  va1 and va2 are two virtual address that
7363                          * alias and will map the same physical address.
7364                          * 1.   mapping exists from va1 to pa and data has
7365                          *      been read into the cache.
7366                          * 2.   unload va1.
7367                          * 3.   load va2 and modify data using va2.
7368                          * 4    unload va2.
7369                          * 5.   load va1 and reference data.  Unless we flush
7370                          *      the data cache when we unload we will get
7371                          *      stale data.
7372                          * This scenario is taken care of by using virtual
7373                          * page coloring.
7374                          */
7375                         if (sfmmup->sfmmu_ismhat) {
7376                                 /*
7377                                  * Flush TSBs, TLBs and caches
7378                                  * of every process
7379                                  * sharing this ism segment.
7380                                  */
7381                                 sfmmu_hat_lock_all();
7382                                 mutex_enter(&ism_mlist_lock);
7383                                 kpreempt_disable();
7384                                 sfmmu_ismtlbcache_demap(addr, sfmmup, hmeblkp,
7385                                     pp->p_pagenum, CACHE_NO_FLUSH);
7386                                 kpreempt_enable();
7387                                 mutex_exit(&ism_mlist_lock);
7388                                 sfmmu_hat_unlock_all();
7389                                 cpuset = cpu_ready_set;
7390                         } else {
7391                                 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
7392                                 cpuset = sfmmup->sfmmu_cpusran;
7393                         }
7394                 }
7395 
7396                 /*
7397                  * Hme_sub has to run after ttesync() and a_rss update.
7398                  * See hblk_unload().
7399                  */
7400                 HME_SUB(sfhme, pp);
7401                 membar_stst();
7402 
7403                 /*
7404                  * We can not make ASSERT(hmeblkp->hblk_hmecnt <= NHMENTS)
7405                  * since pteload may have done a HME_ADD() right after
7406                  * we did the HME_SUB() above. Hmecnt is now maintained
7407                  * by cas only. no lock guranteed its value. The only
7408                  * gurantee we have is the hmecnt should not be less than
7409                  * what it should be so the hblk will not be taken away.
7410                  * It's also important that we decremented the hmecnt after
7411                  * we are done with hmeblkp so that this hmeblk won't be
7412                  * stolen.
7413                  */
7414                 ASSERT(hmeblkp->hblk_hmecnt > 0);
7415                 ASSERT(hmeblkp->hblk_vcnt > 0);
7416                 atomic_add_16(&hmeblkp->hblk_vcnt, -1);
7417                 atomic_add_16(&hmeblkp->hblk_hmecnt, -1);
7418                 /*
7419                  * This is bug 4063182.
7420                  * XXX: fixme
7421                  * ASSERT(hmeblkp->hblk_hmecnt || hmeblkp->hblk_vcnt ||
7422                  *      !hmeblkp->hblk_lckcnt);
7423                  */
7424         } else {
7425                 panic("invalid tte? pp %p &tte %p",
7426                     (void *)pp, (void *)&tte);
7427         }
7428 
7429         return (cpuset);
7430 }
7431 
7432 /*
7433  * While relocating a kernel page, this function will move the mappings
7434  * from tpp to dpp and modify any associated data with these mappings.
7435  * It also unsuspends the suspended kernel mapping.
7436  */
7437 static void
7438 hat_pagereload(struct page *tpp, struct page *dpp)
7439 {
7440         struct sf_hment *sfhme;
7441         tte_t tte, ttemod;
7442         int index, cons;
7443 
7444         ASSERT(getpil() == PIL_MAX);
7445         ASSERT(sfmmu_mlist_held(tpp));
7446         ASSERT(sfmmu_mlist_held(dpp));
7447 
7448         index = PP_MAPINDEX(tpp);
7449         cons = TTE8K;
7450 
7451         /* Update real mappings to the page */
7452 retry:
7453         for (sfhme = tpp->p_mapping; sfhme != NULL; sfhme = sfhme->hme_next) {
7454                 if (IS_PAHME(sfhme))
7455                         continue;
7456                 sfmmu_copytte(&sfhme->hme_tte, &tte);
7457                 ttemod = tte;
7458 
7459                 /*
7460                  * replace old pfn with new pfn in TTE
7461                  */
7462                 PFN_TO_TTE(ttemod, dpp->p_pagenum);
7463 
7464                 /*
7465                  * clear suspend bit
7466                  */
7467                 ASSERT(TTE_IS_SUSPEND(&ttemod));
7468                 TTE_CLR_SUSPEND(&ttemod);
7469 
7470                 if (sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte) < 0)
7471                         panic("hat_pagereload(): sfmmu_modifytte_try() failed");
7472 
7473                 /*
7474                  * set hme_page point to new page
7475                  */
7476                 sfhme->hme_page = dpp;
7477         }
7478 
7479         /*
7480          * move p_mapping list from old page to new page
7481          */
7482         dpp->p_mapping = tpp->p_mapping;
7483         tpp->p_mapping = NULL;
7484         dpp->p_share = tpp->p_share;
7485         tpp->p_share = 0;
7486 
7487         while (index != 0) {
7488                 index = index >> 1;
7489                 if (index != 0)
7490                         cons++;
7491                 if (index & 0x1) {
7492                         tpp = PP_GROUPLEADER(tpp, cons);
7493                         dpp = PP_GROUPLEADER(dpp, cons);
7494                         goto retry;
7495                 }
7496         }
7497 
7498         curthread->t_flag &= ~T_DONTDTRACE;
7499         mutex_exit(&kpr_suspendlock);
7500 }
7501 
7502 uint_t
7503 hat_pagesync(struct page *pp, uint_t clearflag)
7504 {
7505         struct sf_hment *sfhme, *tmphme = NULL;
7506         struct hme_blk *hmeblkp;
7507         kmutex_t *pml;
7508         cpuset_t cpuset, tset;
7509         int     index, cons;
7510         extern  ulong_t po_share;
7511         page_t  *save_pp = pp;
7512         int     stop_on_sh = 0;
7513         uint_t  shcnt;
7514 
7515         CPUSET_ZERO(cpuset);
7516 
7517         if (PP_ISRO(pp) && (clearflag & HAT_SYNC_STOPON_MOD)) {
7518                 return (PP_GENERIC_ATTR(pp));
7519         }
7520 
7521         if ((clearflag & HAT_SYNC_ZERORM) == 0) {
7522                 if ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(pp)) {
7523                         return (PP_GENERIC_ATTR(pp));
7524                 }
7525                 if ((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(pp)) {
7526                         return (PP_GENERIC_ATTR(pp));
7527                 }
7528                 if (clearflag & HAT_SYNC_STOPON_SHARED) {
7529                         if (pp->p_share > po_share) {
7530                                 hat_page_setattr(pp, P_REF);
7531                                 return (PP_GENERIC_ATTR(pp));
7532                         }
7533                         stop_on_sh = 1;
7534                         shcnt = 0;
7535                 }
7536         }
7537 
7538         clearflag &= ~HAT_SYNC_STOPON_SHARED;
7539         pml = sfmmu_mlist_enter(pp);
7540         index = PP_MAPINDEX(pp);
7541         cons = TTE8K;
7542 retry:
7543         for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) {
7544                 /*
7545                  * We need to save the next hment on the list since
7546                  * it is possible for pagesync to remove an invalid hment
7547                  * from the list.
7548                  */
7549                 tmphme = sfhme->hme_next;
7550                 if (IS_PAHME(sfhme))
7551                         continue;
7552                 /*
7553                  * If we are looking for large mappings and this hme doesn't
7554                  * reach the range we are seeking, just ignore it.
7555                  */
7556                 hmeblkp = sfmmu_hmetohblk(sfhme);
7557                 if (hmeblkp->hblk_xhat_bit)
7558                         continue;
7559 
7560                 if (hme_size(sfhme) < cons)
7561                         continue;
7562 
7563                 if (stop_on_sh) {
7564                         if (hmeblkp->hblk_shared) {
7565                                 sf_srd_t *srdp = hblktosrd(hmeblkp);
7566                                 uint_t rid = hmeblkp->hblk_tag.htag_rid;
7567                                 sf_region_t *rgnp;
7568                                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
7569                                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
7570                                 ASSERT(srdp != NULL);
7571                                 rgnp = srdp->srd_hmergnp[rid];
7572                                 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp,
7573                                     rgnp, rid);
7574                                 shcnt += rgnp->rgn_refcnt;
7575                         } else {
7576                                 shcnt++;
7577                         }
7578                         if (shcnt > po_share) {
7579                                 /*
7580                                  * tell the pager to spare the page this time
7581                                  * around.
7582                                  */
7583                                 hat_page_setattr(save_pp, P_REF);
7584                                 index = 0;
7585                                 break;
7586                         }
7587                 }
7588                 tset = sfmmu_pagesync(pp, sfhme,
7589                     clearflag & ~HAT_SYNC_STOPON_RM);
7590                 CPUSET_OR(cpuset, tset);
7591 
7592                 /*
7593                  * If clearflag is HAT_SYNC_DONTZERO, break out as soon
7594                  * as the "ref" or "mod" is set or share cnt exceeds po_share.
7595                  */
7596                 if ((clearflag & ~HAT_SYNC_STOPON_RM) == HAT_SYNC_DONTZERO &&
7597                     (((clearflag & HAT_SYNC_STOPON_MOD) && PP_ISMOD(save_pp)) ||
7598                     ((clearflag & HAT_SYNC_STOPON_REF) && PP_ISREF(save_pp)))) {
7599                         index = 0;
7600                         break;
7601                 }
7602         }
7603 
7604         while (index) {
7605                 index = index >> 1;
7606                 cons++;
7607                 if (index & 0x1) {
7608                         /* Go to leading page */
7609                         pp = PP_GROUPLEADER(pp, cons);
7610                         goto retry;
7611                 }
7612         }
7613 
7614         xt_sync(cpuset);
7615         sfmmu_mlist_exit(pml);
7616         return (PP_GENERIC_ATTR(save_pp));
7617 }
7618 
7619 /*
7620  * Get all the hardware dependent attributes for a page struct
7621  */
7622 static cpuset_t
7623 sfmmu_pagesync(struct page *pp, struct sf_hment *sfhme,
7624         uint_t clearflag)
7625 {
7626         caddr_t addr;
7627         tte_t tte, ttemod;
7628         struct hme_blk *hmeblkp;
7629         int ret;
7630         sfmmu_t *sfmmup;
7631         cpuset_t cpuset;
7632 
7633         ASSERT(pp != NULL);
7634         ASSERT(sfmmu_mlist_held(pp));
7635         ASSERT((clearflag == HAT_SYNC_DONTZERO) ||
7636             (clearflag == HAT_SYNC_ZERORM));
7637 
7638         SFMMU_STAT(sf_pagesync);
7639 
7640         CPUSET_ZERO(cpuset);
7641 
7642 sfmmu_pagesync_retry:
7643 
7644         sfmmu_copytte(&sfhme->hme_tte, &tte);
7645         if (TTE_IS_VALID(&tte)) {
7646                 hmeblkp = sfmmu_hmetohblk(sfhme);
7647                 sfmmup = hblktosfmmu(hmeblkp);
7648                 addr = tte_to_vaddr(hmeblkp, tte);
7649                 if (clearflag == HAT_SYNC_ZERORM) {
7650                         ttemod = tte;
7651                         TTE_CLR_RM(&ttemod);
7652                         ret = sfmmu_modifytte_try(&tte, &ttemod,
7653                             &sfhme->hme_tte);
7654                         if (ret < 0) {
7655                                 /*
7656                                  * cas failed and the new value is not what
7657                                  * we want.
7658                                  */
7659                                 goto sfmmu_pagesync_retry;
7660                         }
7661 
7662                         if (ret > 0) {
7663                                 /* we win the cas */
7664                                 if (hmeblkp->hblk_shared) {
7665                                         sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7666                                         uint_t rid =
7667                                             hmeblkp->hblk_tag.htag_rid;
7668                                         sf_region_t *rgnp;
7669                                         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
7670                                         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
7671                                         ASSERT(srdp != NULL);
7672                                         rgnp = srdp->srd_hmergnp[rid];
7673                                         SFMMU_VALIDATE_SHAREDHBLK(hmeblkp,
7674                                             srdp, rgnp, rid);
7675                                         cpuset = sfmmu_rgntlb_demap(addr,
7676                                             rgnp, hmeblkp, 1);
7677                                 } else {
7678                                         sfmmu_tlb_demap(addr, sfmmup, hmeblkp,
7679                                             0, 0);
7680                                         cpuset = sfmmup->sfmmu_cpusran;
7681                                 }
7682                         }
7683                 }
7684                 sfmmu_ttesync(hmeblkp->hblk_shared ? NULL : sfmmup, addr,
7685                     &tte, pp);
7686         }
7687         return (cpuset);
7688 }
7689 
7690 /*
7691  * Remove write permission from a mappings to a page, so that
7692  * we can detect the next modification of it. This requires modifying
7693  * the TTE then invalidating (demap) any TLB entry using that TTE.
7694  * This code is similar to sfmmu_pagesync().
7695  */
7696 static cpuset_t
7697 sfmmu_pageclrwrt(struct page *pp, struct sf_hment *sfhme)
7698 {
7699         caddr_t addr;
7700         tte_t tte;
7701         tte_t ttemod;
7702         struct hme_blk *hmeblkp;
7703         int ret;
7704         sfmmu_t *sfmmup;
7705         cpuset_t cpuset;
7706 
7707         ASSERT(pp != NULL);
7708         ASSERT(sfmmu_mlist_held(pp));
7709 
7710         CPUSET_ZERO(cpuset);
7711         SFMMU_STAT(sf_clrwrt);
7712 
7713 retry:
7714 
7715         sfmmu_copytte(&sfhme->hme_tte, &tte);
7716         if (TTE_IS_VALID(&tte) && TTE_IS_WRITABLE(&tte)) {
7717                 hmeblkp = sfmmu_hmetohblk(sfhme);
7718 
7719                 /*
7720                  * xhat mappings should never be to a VMODSORT page.
7721                  */
7722                 ASSERT(hmeblkp->hblk_xhat_bit == 0);
7723 
7724                 sfmmup = hblktosfmmu(hmeblkp);
7725                 addr = tte_to_vaddr(hmeblkp, tte);
7726 
7727                 ttemod = tte;
7728                 TTE_CLR_WRT(&ttemod);
7729                 TTE_CLR_MOD(&ttemod);
7730                 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte);
7731 
7732                 /*
7733                  * if cas failed and the new value is not what
7734                  * we want retry
7735                  */
7736                 if (ret < 0)
7737                         goto retry;
7738 
7739                 /* we win the cas */
7740                 if (ret > 0) {
7741                         if (hmeblkp->hblk_shared) {
7742                                 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
7743                                 uint_t rid = hmeblkp->hblk_tag.htag_rid;
7744                                 sf_region_t *rgnp;
7745                                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
7746                                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
7747                                 ASSERT(srdp != NULL);
7748                                 rgnp = srdp->srd_hmergnp[rid];
7749                                 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp,
7750                                     srdp, rgnp, rid);
7751                                 cpuset = sfmmu_rgntlb_demap(addr,
7752                                     rgnp, hmeblkp, 1);
7753                         } else {
7754                                 sfmmu_tlb_demap(addr, sfmmup, hmeblkp, 0, 0);
7755                                 cpuset = sfmmup->sfmmu_cpusran;
7756                         }
7757                 }
7758         }
7759 
7760         return (cpuset);
7761 }
7762 
7763 /*
7764  * Walk all mappings of a page, removing write permission and clearing the
7765  * ref/mod bits. This code is similar to hat_pagesync()
7766  */
7767 static void
7768 hat_page_clrwrt(page_t *pp)
7769 {
7770         struct sf_hment *sfhme;
7771         struct sf_hment *tmphme = NULL;
7772         kmutex_t *pml;
7773         cpuset_t cpuset;
7774         cpuset_t tset;
7775         int     index;
7776         int      cons;
7777 
7778         CPUSET_ZERO(cpuset);
7779 
7780         pml = sfmmu_mlist_enter(pp);
7781         index = PP_MAPINDEX(pp);
7782         cons = TTE8K;
7783 retry:
7784         for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) {
7785                 tmphme = sfhme->hme_next;
7786 
7787                 /*
7788                  * If we are looking for large mappings and this hme doesn't
7789                  * reach the range we are seeking, just ignore its.
7790                  */
7791 
7792                 if (hme_size(sfhme) < cons)
7793                         continue;
7794 
7795                 tset = sfmmu_pageclrwrt(pp, sfhme);
7796                 CPUSET_OR(cpuset, tset);
7797         }
7798 
7799         while (index) {
7800                 index = index >> 1;
7801                 cons++;
7802                 if (index & 0x1) {
7803                         /* Go to leading page */
7804                         pp = PP_GROUPLEADER(pp, cons);
7805                         goto retry;
7806                 }
7807         }
7808 
7809         xt_sync(cpuset);
7810         sfmmu_mlist_exit(pml);
7811 }
7812 
7813 /*
7814  * Set the given REF/MOD/RO bits for the given page.
7815  * For a vnode with a sorted v_pages list, we need to change
7816  * the attributes and the v_pages list together under page_vnode_mutex.
7817  */
7818 void
7819 hat_page_setattr(page_t *pp, uint_t flag)
7820 {
7821         vnode_t         *vp = pp->p_vnode;
7822         page_t          **listp;
7823         kmutex_t        *pmtx;
7824         kmutex_t        *vphm = NULL;
7825         int             noshuffle;
7826 
7827         noshuffle = flag & P_NSH;
7828         flag &= ~P_NSH;
7829 
7830         ASSERT(!(flag & ~(P_MOD | P_REF | P_RO)));
7831 
7832         /*
7833          * nothing to do if attribute already set
7834          */
7835         if ((pp->p_nrm & flag) == flag)
7836                 return;
7837 
7838         if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp) &&
7839             !noshuffle) {
7840                 vphm = page_vnode_mutex(vp);
7841                 mutex_enter(vphm);
7842         }
7843 
7844         pmtx = sfmmu_page_enter(pp);
7845         pp->p_nrm |= flag;
7846         sfmmu_page_exit(pmtx);
7847 
7848         if (vphm != NULL) {
7849                 /*
7850                  * Some File Systems examine v_pages for NULL w/o
7851                  * grabbing the vphm mutex. Must not let it become NULL when
7852                  * pp is the only page on the list.
7853                  */
7854                 if (pp->p_vpnext != pp) {
7855                         page_vpsub(&vp->v_pages, pp);
7856                         if (vp->v_pages != NULL)
7857                                 listp = &vp->v_pages->p_vpprev->p_vpnext;
7858                         else
7859                                 listp = &vp->v_pages;
7860                         page_vpadd(listp, pp);
7861                 }
7862                 mutex_exit(vphm);
7863         }
7864 }
7865 
7866 void
7867 hat_page_clrattr(page_t *pp, uint_t flag)
7868 {
7869         vnode_t         *vp = pp->p_vnode;
7870         kmutex_t        *pmtx;
7871 
7872         ASSERT(!(flag & ~(P_MOD | P_REF | P_RO)));
7873 
7874         pmtx = sfmmu_page_enter(pp);
7875 
7876         /*
7877          * Caller is expected to hold page's io lock for VMODSORT to work
7878          * correctly with pvn_vplist_dirty() and pvn_getdirty() when mod
7879          * bit is cleared.
7880          * We don't have assert to avoid tripping some existing third party
7881          * code. The dirty page is moved back to top of the v_page list
7882          * after IO is done in pvn_write_done().
7883          */
7884         pp->p_nrm &= ~flag;
7885         sfmmu_page_exit(pmtx);
7886 
7887         if ((flag & P_MOD) != 0 && vp != NULL && IS_VMODSORT(vp)) {
7888 
7889                 /*
7890                  * VMODSORT works by removing write permissions and getting
7891                  * a fault when a page is made dirty. At this point
7892                  * we need to remove write permission from all mappings
7893                  * to this page.
7894                  */
7895                 hat_page_clrwrt(pp);
7896         }
7897 }
7898 
7899 uint_t
7900 hat_page_getattr(page_t *pp, uint_t flag)
7901 {
7902         ASSERT(!(flag & ~(P_MOD | P_REF | P_RO)));
7903         return ((uint_t)(pp->p_nrm & flag));
7904 }
7905 
7906 /*
7907  * DEBUG kernels: verify that a kernel va<->pa translation
7908  * is safe by checking the underlying page_t is in a page
7909  * relocation-safe state.
7910  */
7911 #ifdef  DEBUG
7912 void
7913 sfmmu_check_kpfn(pfn_t pfn)
7914 {
7915         page_t *pp;
7916         int index, cons;
7917 
7918         if (hat_check_vtop == 0)
7919                 return;
7920 
7921         if (kvseg.s_base == NULL || panicstr)
7922                 return;
7923 
7924         pp = page_numtopp_nolock(pfn);
7925         if (!pp)
7926                 return;
7927 
7928         if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp))
7929                 return;
7930 
7931         /*
7932          * Handed a large kernel page, we dig up the root page since we
7933          * know the root page might have the lock also.
7934          */
7935         if (pp->p_szc != 0) {
7936                 index = PP_MAPINDEX(pp);
7937                 cons = TTE8K;
7938 again:
7939                 while (index != 0) {
7940                         index >>= 1;
7941                         if (index != 0)
7942                                 cons++;
7943                         if (index & 0x1) {
7944                                 pp = PP_GROUPLEADER(pp, cons);
7945                                 goto again;
7946                         }
7947                 }
7948         }
7949 
7950         if (PAGE_LOCKED(pp) || PP_ISNORELOC(pp))
7951                 return;
7952 
7953         /*
7954          * Pages need to be locked or allocated "permanent" (either from
7955          * static_arena arena or explicitly setting PG_NORELOC when calling
7956          * page_create_va()) for VA->PA translations to be valid.
7957          */
7958         if (!PP_ISNORELOC(pp))
7959                 panic("Illegal VA->PA translation, pp 0x%p not permanent",
7960                     (void *)pp);
7961         else
7962                 panic("Illegal VA->PA translation, pp 0x%p not locked",
7963                     (void *)pp);
7964 }
7965 #endif  /* DEBUG */
7966 
7967 /*
7968  * Returns a page frame number for a given virtual address.
7969  * Returns PFN_INVALID to indicate an invalid mapping
7970  */
7971 pfn_t
7972 hat_getpfnum(struct hat *hat, caddr_t addr)
7973 {
7974         pfn_t pfn;
7975         tte_t tte;
7976 
7977         /*
7978          * We would like to
7979          * ASSERT(AS_LOCK_HELD(as, &as->a_lock));
7980          * but we can't because the iommu driver will call this
7981          * routine at interrupt time and it can't grab the as lock
7982          * or it will deadlock: A thread could have the as lock
7983          * and be waiting for io.  The io can't complete
7984          * because the interrupt thread is blocked trying to grab
7985          * the as lock.
7986          */
7987 
7988         ASSERT(hat->sfmmu_xhat_provider == NULL);
7989 
7990         if (hat == ksfmmup) {
7991                 if (IS_KMEM_VA_LARGEPAGE(addr)) {
7992                         ASSERT(segkmem_lpszc > 0);
7993                         pfn = sfmmu_kvaszc2pfn(addr, segkmem_lpszc);
7994                         if (pfn != PFN_INVALID) {
7995                                 sfmmu_check_kpfn(pfn);
7996                                 return (pfn);
7997                         }
7998                 } else if (segkpm && IS_KPM_ADDR(addr)) {
7999                         return (sfmmu_kpm_vatopfn(addr));
8000                 }
8001                 while ((pfn = sfmmu_vatopfn(addr, ksfmmup, &tte))
8002                     == PFN_SUSPENDED) {
8003                         sfmmu_vatopfn_suspended(addr, ksfmmup, &tte);
8004                 }
8005                 sfmmu_check_kpfn(pfn);
8006                 return (pfn);
8007         } else {
8008                 return (sfmmu_uvatopfn(addr, hat, NULL));
8009         }
8010 }
8011 
8012 /*
8013  * This routine will return both pfn and tte for the vaddr.
8014  */
8015 static pfn_t
8016 sfmmu_uvatopfn(caddr_t vaddr, struct hat *sfmmup, tte_t *ttep)
8017 {
8018         struct hmehash_bucket *hmebp;
8019         hmeblk_tag hblktag;
8020         int hmeshift, hashno = 1;
8021         struct hme_blk *hmeblkp = NULL;
8022         tte_t tte;
8023 
8024         struct sf_hment *sfhmep;
8025         pfn_t pfn;
8026 
8027         /* support for ISM */
8028         ism_map_t       *ism_map;
8029         ism_blk_t       *ism_blkp;
8030         int             i;
8031         sfmmu_t *ism_hatid = NULL;
8032         sfmmu_t *locked_hatid = NULL;
8033         sfmmu_t *sv_sfmmup = sfmmup;
8034         caddr_t sv_vaddr = vaddr;
8035         sf_srd_t *srdp;
8036 
8037         if (ttep == NULL) {
8038                 ttep = &tte;
8039         } else {
8040                 ttep->ll = 0;
8041         }
8042 
8043         ASSERT(sfmmup != ksfmmup);
8044         SFMMU_STAT(sf_user_vtop);
8045         /*
8046          * Set ism_hatid if vaddr falls in a ISM segment.
8047          */
8048         ism_blkp = sfmmup->sfmmu_iblk;
8049         if (ism_blkp != NULL) {
8050                 sfmmu_ismhat_enter(sfmmup, 0);
8051                 locked_hatid = sfmmup;
8052         }
8053         while (ism_blkp != NULL && ism_hatid == NULL) {
8054                 ism_map = ism_blkp->iblk_maps;
8055                 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) {
8056                         if (vaddr >= ism_start(ism_map[i]) &&
8057                             vaddr < ism_end(ism_map[i])) {
8058                                 sfmmup = ism_hatid = ism_map[i].imap_ismhat;
8059                                 vaddr = (caddr_t)(vaddr -
8060                                     ism_start(ism_map[i]));
8061                                 break;
8062                         }
8063                 }
8064                 ism_blkp = ism_blkp->iblk_next;
8065         }
8066         if (locked_hatid) {
8067                 sfmmu_ismhat_exit(locked_hatid, 0);
8068         }
8069 
8070         hblktag.htag_id = sfmmup;
8071         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
8072         do {
8073                 hmeshift = HME_HASH_SHIFT(hashno);
8074                 hblktag.htag_bspage = HME_HASH_BSPAGE(vaddr, hmeshift);
8075                 hblktag.htag_rehash = hashno;
8076                 hmebp = HME_HASH_FUNCTION(sfmmup, vaddr, hmeshift);
8077 
8078                 SFMMU_HASH_LOCK(hmebp);
8079 
8080                 HME_HASH_FAST_SEARCH(hmebp, hblktag, hmeblkp);
8081                 if (hmeblkp != NULL) {
8082                         ASSERT(!hmeblkp->hblk_shared);
8083                         HBLKTOHME(sfhmep, hmeblkp, vaddr);
8084                         sfmmu_copytte(&sfhmep->hme_tte, ttep);
8085                         SFMMU_HASH_UNLOCK(hmebp);
8086                         if (TTE_IS_VALID(ttep)) {
8087                                 pfn = TTE_TO_PFN(vaddr, ttep);
8088                                 return (pfn);
8089                         }
8090                         break;
8091                 }
8092                 SFMMU_HASH_UNLOCK(hmebp);
8093                 hashno++;
8094         } while (HME_REHASH(sfmmup) && (hashno <= mmu_hashcnt));
8095 
8096         if (SF_HMERGNMAP_ISNULL(sv_sfmmup)) {
8097                 return (PFN_INVALID);
8098         }
8099         srdp = sv_sfmmup->sfmmu_srdp;
8100         ASSERT(srdp != NULL);
8101         ASSERT(srdp->srd_refcnt != 0);
8102         hblktag.htag_id = srdp;
8103         hashno = 1;
8104         do {
8105                 hmeshift = HME_HASH_SHIFT(hashno);
8106                 hblktag.htag_bspage = HME_HASH_BSPAGE(sv_vaddr, hmeshift);
8107                 hblktag.htag_rehash = hashno;
8108                 hmebp = HME_HASH_FUNCTION(srdp, sv_vaddr, hmeshift);
8109 
8110                 SFMMU_HASH_LOCK(hmebp);
8111                 for (hmeblkp = hmebp->hmeblkp; hmeblkp != NULL;
8112                     hmeblkp = hmeblkp->hblk_next) {
8113                         uint_t rid;
8114                         sf_region_t *rgnp;
8115                         caddr_t rsaddr;
8116                         caddr_t readdr;
8117 
8118                         if (!HTAGS_EQ_SHME(hmeblkp->hblk_tag, hblktag,
8119                             sv_sfmmup->sfmmu_hmeregion_map)) {
8120                                 continue;
8121                         }
8122                         ASSERT(hmeblkp->hblk_shared);
8123                         rid = hmeblkp->hblk_tag.htag_rid;
8124                         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
8125                         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
8126                         rgnp = srdp->srd_hmergnp[rid];
8127                         SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
8128                         HBLKTOHME(sfhmep, hmeblkp, sv_vaddr);
8129                         sfmmu_copytte(&sfhmep->hme_tte, ttep);
8130                         rsaddr = rgnp->rgn_saddr;
8131                         readdr = rsaddr + rgnp->rgn_size;
8132 #ifdef DEBUG
8133                         if (TTE_IS_VALID(ttep) ||
8134                             get_hblk_ttesz(hmeblkp) > TTE8K) {
8135                                 caddr_t eva = tte_to_evaddr(hmeblkp, ttep);
8136                                 ASSERT(eva > sv_vaddr);
8137                                 ASSERT(sv_vaddr >= rsaddr);
8138                                 ASSERT(sv_vaddr < readdr);
8139                                 ASSERT(eva <= readdr);
8140                         }
8141 #endif /* DEBUG */
8142                         /*
8143                          * Continue the search if we
8144                          * found an invalid 8K tte outside of the area
8145                          * covered by this hmeblk's region.
8146                          */
8147                         if (TTE_IS_VALID(ttep)) {
8148                                 SFMMU_HASH_UNLOCK(hmebp);
8149                                 pfn = TTE_TO_PFN(sv_vaddr, ttep);
8150                                 return (pfn);
8151                         } else if (get_hblk_ttesz(hmeblkp) > TTE8K ||
8152                             (sv_vaddr >= rsaddr && sv_vaddr < readdr)) {
8153                                 SFMMU_HASH_UNLOCK(hmebp);
8154                                 pfn = PFN_INVALID;
8155                                 return (pfn);
8156                         }
8157                 }
8158                 SFMMU_HASH_UNLOCK(hmebp);
8159                 hashno++;
8160         } while (hashno <= mmu_hashcnt);
8161         return (PFN_INVALID);
8162 }
8163 
8164 
8165 /*
8166  * For compatability with AT&T and later optimizations
8167  */
8168 /* ARGSUSED */
8169 void
8170 hat_map(struct hat *hat, caddr_t addr, size_t len, uint_t flags)
8171 {
8172         ASSERT(hat != NULL);
8173         ASSERT(hat->sfmmu_xhat_provider == NULL);
8174 }
8175 
8176 /*
8177  * Return the number of mappings to a particular page.  This number is an
8178  * approximation of the number of people sharing the page.
8179  *
8180  * shared hmeblks or ism hmeblks are counted as 1 mapping here.
8181  * hat_page_checkshare() can be used to compare threshold to share
8182  * count that reflects the number of region sharers albeit at higher cost.
8183  */
8184 ulong_t
8185 hat_page_getshare(page_t *pp)
8186 {
8187         page_t *spp = pp;       /* start page */
8188         kmutex_t *pml;
8189         ulong_t cnt;
8190         int index, sz = TTE64K;
8191 
8192         /*
8193          * We need to grab the mlist lock to make sure any outstanding
8194          * load/unloads complete.  Otherwise we could return zero
8195          * even though the unload(s) hasn't finished yet.
8196          */
8197         pml = sfmmu_mlist_enter(spp);
8198         cnt = spp->p_share;
8199 
8200 #ifdef VAC
8201         if (kpm_enable)
8202                 cnt += spp->p_kpmref;
8203 #endif
8204         if (vpm_enable && pp->p_vpmref) {
8205                 cnt += 1;
8206         }
8207 
8208         /*
8209          * If we have any large mappings, we count the number of
8210          * mappings that this large page is part of.
8211          */
8212         index = PP_MAPINDEX(spp);
8213         index >>= 1;
8214         while (index) {
8215                 pp = PP_GROUPLEADER(spp, sz);
8216                 if ((index & 0x1) && pp != spp) {
8217                         cnt += pp->p_share;
8218                         spp = pp;
8219                 }
8220                 index >>= 1;
8221                 sz++;
8222         }
8223         sfmmu_mlist_exit(pml);
8224         return (cnt);
8225 }
8226 
8227 /*
8228  * Return 1 if the number of mappings exceeds sh_thresh. Return 0
8229  * otherwise. Count shared hmeblks by region's refcnt.
8230  */
8231 int
8232 hat_page_checkshare(page_t *pp, ulong_t sh_thresh)
8233 {
8234         kmutex_t *pml;
8235         ulong_t cnt = 0;
8236         int index, sz = TTE8K;
8237         struct sf_hment *sfhme, *tmphme = NULL;
8238         struct hme_blk *hmeblkp;
8239 
8240         pml = sfmmu_mlist_enter(pp);
8241 
8242 #ifdef VAC
8243         if (kpm_enable)
8244                 cnt = pp->p_kpmref;
8245 #endif
8246 
8247         if (vpm_enable && pp->p_vpmref) {
8248                 cnt += 1;
8249         }
8250 
8251         if (pp->p_share + cnt > sh_thresh) {
8252                 sfmmu_mlist_exit(pml);
8253                 return (1);
8254         }
8255 
8256         index = PP_MAPINDEX(pp);
8257 
8258 again:
8259         for (sfhme = pp->p_mapping; sfhme; sfhme = tmphme) {
8260                 tmphme = sfhme->hme_next;
8261                 if (IS_PAHME(sfhme)) {
8262                         continue;
8263                 }
8264 
8265                 hmeblkp = sfmmu_hmetohblk(sfhme);
8266                 if (hmeblkp->hblk_xhat_bit) {
8267                         cnt++;
8268                         if (cnt > sh_thresh) {
8269                                 sfmmu_mlist_exit(pml);
8270                                 return (1);
8271                         }
8272                         continue;
8273                 }
8274                 if (hme_size(sfhme) != sz) {
8275                         continue;
8276                 }
8277 
8278                 if (hmeblkp->hblk_shared) {
8279                         sf_srd_t *srdp = hblktosrd(hmeblkp);
8280                         uint_t rid = hmeblkp->hblk_tag.htag_rid;
8281                         sf_region_t *rgnp;
8282                         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
8283                         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
8284                         ASSERT(srdp != NULL);
8285                         rgnp = srdp->srd_hmergnp[rid];
8286                         SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp,
8287                             rgnp, rid);
8288                         cnt += rgnp->rgn_refcnt;
8289                 } else {
8290                         cnt++;
8291                 }
8292                 if (cnt > sh_thresh) {
8293                         sfmmu_mlist_exit(pml);
8294                         return (1);
8295                 }
8296         }
8297 
8298         index >>= 1;
8299         sz++;
8300         while (index) {
8301                 pp = PP_GROUPLEADER(pp, sz);
8302                 ASSERT(sfmmu_mlist_held(pp));
8303                 if (index & 0x1) {
8304                         goto again;
8305                 }
8306                 index >>= 1;
8307                 sz++;
8308         }
8309         sfmmu_mlist_exit(pml);
8310         return (0);
8311 }
8312 
8313 /*
8314  * Unload all large mappings to the pp and reset the p_szc field of every
8315  * constituent page according to the remaining mappings.
8316  *
8317  * pp must be locked SE_EXCL. Even though no other constituent pages are
8318  * locked it's legal to unload the large mappings to the pp because all
8319  * constituent pages of large locked mappings have to be locked SE_SHARED.
8320  * This means if we have SE_EXCL lock on one of constituent pages none of the
8321  * large mappings to pp are locked.
8322  *
8323  * Decrease p_szc field starting from the last constituent page and ending
8324  * with the root page. This method is used because other threads rely on the
8325  * root's p_szc to find the lock to syncronize on. After a root page_t's p_szc
8326  * is demoted then other threads will succeed in sfmmu_mlspl_enter(). This
8327  * ensures that p_szc changes of the constituent pages appears atomic for all
8328  * threads that use sfmmu_mlspl_enter() to examine p_szc field.
8329  *
8330  * This mechanism is only used for file system pages where it's not always
8331  * possible to get SE_EXCL locks on all constituent pages to demote the size
8332  * code (as is done for anonymous or kernel large pages).
8333  *
8334  * See more comments in front of sfmmu_mlspl_enter().
8335  */
8336 void
8337 hat_page_demote(page_t *pp)
8338 {
8339         int index;
8340         int sz;
8341         cpuset_t cpuset;
8342         int sync = 0;
8343         page_t *rootpp;
8344         struct sf_hment *sfhme;
8345         struct sf_hment *tmphme = NULL;
8346         struct hme_blk *hmeblkp;
8347         uint_t pszc;
8348         page_t *lastpp;
8349         cpuset_t tset;
8350         pgcnt_t npgs;
8351         kmutex_t *pml;
8352         kmutex_t *pmtx = NULL;
8353 
8354         ASSERT(PAGE_EXCL(pp));
8355         ASSERT(!PP_ISFREE(pp));
8356         ASSERT(!PP_ISKAS(pp));
8357         ASSERT(page_szc_lock_assert(pp));
8358         pml = sfmmu_mlist_enter(pp);
8359 
8360         pszc = pp->p_szc;
8361         if (pszc == 0) {
8362                 goto out;
8363         }
8364 
8365         index = PP_MAPINDEX(pp) >> 1;
8366 
8367         if (index) {
8368                 CPUSET_ZERO(cpuset);
8369                 sz = TTE64K;
8370                 sync = 1;
8371         }
8372 
8373         while (index) {
8374                 if (!(index & 0x1)) {
8375                         index >>= 1;
8376                         sz++;
8377                         continue;
8378                 }
8379                 ASSERT(sz <= pszc);
8380                 rootpp = PP_GROUPLEADER(pp, sz);
8381                 for (sfhme = rootpp->p_mapping; sfhme; sfhme = tmphme) {
8382                         tmphme = sfhme->hme_next;
8383                         ASSERT(!IS_PAHME(sfhme));
8384                         hmeblkp = sfmmu_hmetohblk(sfhme);
8385                         if (hme_size(sfhme) != sz) {
8386                                 continue;
8387                         }
8388                         if (hmeblkp->hblk_xhat_bit) {
8389                                 cmn_err(CE_PANIC,
8390                                     "hat_page_demote: xhat hmeblk");
8391                         }
8392                         tset = sfmmu_pageunload(rootpp, sfhme, sz);
8393                         CPUSET_OR(cpuset, tset);
8394                 }
8395                 if (index >>= 1) {
8396                         sz++;
8397                 }
8398         }
8399 
8400         ASSERT(!PP_ISMAPPED_LARGE(pp));
8401 
8402         if (sync) {
8403                 xt_sync(cpuset);
8404 #ifdef VAC
8405                 if (PP_ISTNC(pp)) {
8406                         conv_tnc(rootpp, sz);
8407                 }
8408 #endif  /* VAC */
8409         }
8410 
8411         pmtx = sfmmu_page_enter(pp);
8412 
8413         ASSERT(pp->p_szc == pszc);
8414         rootpp = PP_PAGEROOT(pp);
8415         ASSERT(rootpp->p_szc == pszc);
8416         lastpp = PP_PAGENEXT_N(rootpp, TTEPAGES(pszc) - 1);
8417 
8418         while (lastpp != rootpp) {
8419                 sz = PP_MAPINDEX(lastpp) ? fnd_mapping_sz(lastpp) : 0;
8420                 ASSERT(sz < pszc);
8421                 npgs = (sz == 0) ? 1 : TTEPAGES(sz);
8422                 ASSERT(P2PHASE(lastpp->p_pagenum, npgs) == npgs - 1);
8423                 while (--npgs > 0) {
8424                         lastpp->p_szc = (uchar_t)sz;
8425                         lastpp = PP_PAGEPREV(lastpp);
8426                 }
8427                 if (sz) {
8428                         /*
8429                          * make sure before current root's pszc
8430                          * is updated all updates to constituent pages pszc
8431                          * fields are globally visible.
8432                          */
8433                         membar_producer();
8434                 }
8435                 lastpp->p_szc = sz;
8436                 ASSERT(IS_P2ALIGNED(lastpp->p_pagenum, TTEPAGES(sz)));
8437                 if (lastpp != rootpp) {
8438                         lastpp = PP_PAGEPREV(lastpp);
8439                 }
8440         }
8441         if (sz == 0) {
8442                 /* the loop above doesn't cover this case */
8443                 rootpp->p_szc = 0;
8444         }
8445 out:
8446         ASSERT(pp->p_szc == 0);
8447         if (pmtx != NULL) {
8448                 sfmmu_page_exit(pmtx);
8449         }
8450         sfmmu_mlist_exit(pml);
8451 }
8452 
8453 /*
8454  * Refresh the HAT ismttecnt[] element for size szc.
8455  * Caller must have set ISM busy flag to prevent mapping
8456  * lists from changing while we're traversing them.
8457  */
8458 pgcnt_t
8459 ism_tsb_entries(sfmmu_t *sfmmup, int szc)
8460 {
8461         ism_blk_t       *ism_blkp = sfmmup->sfmmu_iblk;
8462         ism_map_t       *ism_map;
8463         pgcnt_t         npgs = 0;
8464         pgcnt_t         npgs_scd = 0;
8465         int             j;
8466         sf_scd_t        *scdp;
8467         uchar_t         rid;
8468 
8469         ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
8470         scdp = sfmmup->sfmmu_scdp;
8471 
8472         for (; ism_blkp != NULL; ism_blkp = ism_blkp->iblk_next) {
8473                 ism_map = ism_blkp->iblk_maps;
8474                 for (j = 0; ism_map[j].imap_ismhat && j < ISM_MAP_SLOTS; j++) {
8475                         rid = ism_map[j].imap_rid;
8476                         ASSERT(rid == SFMMU_INVALID_ISMRID ||
8477                             rid < sfmmup->sfmmu_srdp->srd_next_ismrid);
8478 
8479                         if (scdp != NULL && rid != SFMMU_INVALID_ISMRID &&
8480                             SF_RGNMAP_TEST(scdp->scd_ismregion_map, rid)) {
8481                                 /* ISM is in sfmmup's SCD */
8482                                 npgs_scd +=
8483                                     ism_map[j].imap_ismhat->sfmmu_ttecnt[szc];
8484                         } else {
8485                                 /* ISMs is not in SCD */
8486                                 npgs +=
8487                                     ism_map[j].imap_ismhat->sfmmu_ttecnt[szc];
8488                         }
8489                 }
8490         }
8491         sfmmup->sfmmu_ismttecnt[szc] = npgs;
8492         sfmmup->sfmmu_scdismttecnt[szc] = npgs_scd;
8493         return (npgs);
8494 }
8495 
8496 /*
8497  * Yield the memory claim requirement for an address space.
8498  *
8499  * This is currently implemented as the number of bytes that have active
8500  * hardware translations that have page structures.  Therefore, it can
8501  * underestimate the traditional resident set size, eg, if the
8502  * physical page is present and the hardware translation is missing;
8503  * and it can overestimate the rss, eg, if there are active
8504  * translations to a frame buffer with page structs.
8505  * Also, it does not take sharing into account.
8506  *
8507  * Note that we don't acquire locks here since this function is most often
8508  * called from the clock thread.
8509  */
8510 size_t
8511 hat_get_mapped_size(struct hat *hat)
8512 {
8513         size_t          assize = 0;
8514         int             i;
8515 
8516         if (hat == NULL)
8517                 return (0);
8518 
8519         ASSERT(hat->sfmmu_xhat_provider == NULL);
8520 
8521         for (i = 0; i < mmu_page_sizes; i++)
8522                 assize += ((pgcnt_t)hat->sfmmu_ttecnt[i] +
8523                     (pgcnt_t)hat->sfmmu_scdrttecnt[i]) * TTEBYTES(i);
8524 
8525         if (hat->sfmmu_iblk == NULL)
8526                 return (assize);
8527 
8528         for (i = 0; i < mmu_page_sizes; i++)
8529                 assize += ((pgcnt_t)hat->sfmmu_ismttecnt[i] +
8530                     (pgcnt_t)hat->sfmmu_scdismttecnt[i]) * TTEBYTES(i);
8531 
8532         return (assize);
8533 }
8534 
8535 int
8536 hat_stats_enable(struct hat *hat)
8537 {
8538         hatlock_t       *hatlockp;
8539 
8540         ASSERT(hat->sfmmu_xhat_provider == NULL);
8541 
8542         hatlockp = sfmmu_hat_enter(hat);
8543         hat->sfmmu_rmstat++;
8544         sfmmu_hat_exit(hatlockp);
8545         return (1);
8546 }
8547 
8548 void
8549 hat_stats_disable(struct hat *hat)
8550 {
8551         hatlock_t       *hatlockp;
8552 
8553         ASSERT(hat->sfmmu_xhat_provider == NULL);
8554 
8555         hatlockp = sfmmu_hat_enter(hat);
8556         hat->sfmmu_rmstat--;
8557         sfmmu_hat_exit(hatlockp);
8558 }
8559 
8560 /*
8561  * Routines for entering or removing  ourselves from the
8562  * ism_hat's mapping list. This is used for both private and
8563  * SCD hats.
8564  */
8565 static void
8566 iment_add(struct ism_ment *iment,  struct hat *ism_hat)
8567 {
8568         ASSERT(MUTEX_HELD(&ism_mlist_lock));
8569 
8570         iment->iment_prev = NULL;
8571         iment->iment_next = ism_hat->sfmmu_iment;
8572         if (ism_hat->sfmmu_iment) {
8573                 ism_hat->sfmmu_iment->iment_prev = iment;
8574         }
8575         ism_hat->sfmmu_iment = iment;
8576 }
8577 
8578 static void
8579 iment_sub(struct ism_ment *iment, struct hat *ism_hat)
8580 {
8581         ASSERT(MUTEX_HELD(&ism_mlist_lock));
8582 
8583         if (ism_hat->sfmmu_iment == NULL) {
8584                 panic("ism map entry remove - no entries");
8585         }
8586 
8587         if (iment->iment_prev) {
8588                 ASSERT(ism_hat->sfmmu_iment != iment);
8589                 iment->iment_prev->iment_next = iment->iment_next;
8590         } else {
8591                 ASSERT(ism_hat->sfmmu_iment == iment);
8592                 ism_hat->sfmmu_iment = iment->iment_next;
8593         }
8594 
8595         if (iment->iment_next) {
8596                 iment->iment_next->iment_prev = iment->iment_prev;
8597         }
8598 
8599         /*
8600          * zero out the entry
8601          */
8602         iment->iment_next = NULL;
8603         iment->iment_prev = NULL;
8604         iment->iment_hat =  NULL;
8605         iment->iment_base_va = 0;
8606 }
8607 
8608 /*
8609  * Hat_share()/unshare() return an (non-zero) error
8610  * when saddr and daddr are not properly aligned.
8611  *
8612  * The top level mapping element determines the alignment
8613  * requirement for saddr and daddr, depending on different
8614  * architectures.
8615  *
8616  * When hat_share()/unshare() are not supported,
8617  * HATOP_SHARE()/UNSHARE() return 0
8618  */
8619 int
8620 hat_share(struct hat *sfmmup, caddr_t addr,
8621         struct hat *ism_hatid, caddr_t sptaddr, size_t len, uint_t ismszc)
8622 {
8623         ism_blk_t       *ism_blkp;
8624         ism_blk_t       *new_iblk;
8625         ism_map_t       *ism_map;
8626         ism_ment_t      *ism_ment;
8627         int             i, added;
8628         hatlock_t       *hatlockp;
8629         int             reload_mmu = 0;
8630         uint_t          ismshift = page_get_shift(ismszc);
8631         size_t          ismpgsz = page_get_pagesize(ismszc);
8632         uint_t          ismmask = (uint_t)ismpgsz - 1;
8633         size_t          sh_size = ISM_SHIFT(ismshift, len);
8634         ushort_t        ismhatflag;
8635         hat_region_cookie_t rcookie;
8636         sf_scd_t        *old_scdp;
8637 
8638 #ifdef DEBUG
8639         caddr_t         eaddr = addr + len;
8640 #endif /* DEBUG */
8641 
8642         ASSERT(ism_hatid != NULL && sfmmup != NULL);
8643         ASSERT(sptaddr == ISMID_STARTADDR);
8644         /*
8645          * Check the alignment.
8646          */
8647         if (!ISM_ALIGNED(ismshift, addr) || !ISM_ALIGNED(ismshift, sptaddr))
8648                 return (EINVAL);
8649 
8650         /*
8651          * Check size alignment.
8652          */
8653         if (!ISM_ALIGNED(ismshift, len))
8654                 return (EINVAL);
8655 
8656         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
8657 
8658         /*
8659          * Allocate ism_ment for the ism_hat's mapping list, and an
8660          * ism map blk in case we need one.  We must do our
8661          * allocations before acquiring locks to prevent a deadlock
8662          * in the kmem allocator on the mapping list lock.
8663          */
8664         new_iblk = kmem_cache_alloc(ism_blk_cache, KM_SLEEP);
8665         ism_ment = kmem_cache_alloc(ism_ment_cache, KM_SLEEP);
8666 
8667         /*
8668          * Serialize ISM mappings with the ISM busy flag, and also the
8669          * trap handlers.
8670          */
8671         sfmmu_ismhat_enter(sfmmup, 0);
8672 
8673         /*
8674          * Allocate an ism map blk if necessary.
8675          */
8676         if (sfmmup->sfmmu_iblk == NULL) {
8677                 sfmmup->sfmmu_iblk = new_iblk;
8678                 bzero(new_iblk, sizeof (*new_iblk));
8679                 new_iblk->iblk_nextpa = (uint64_t)-1;
8680                 membar_stst();  /* make sure next ptr visible to all CPUs */
8681                 sfmmup->sfmmu_ismblkpa = va_to_pa((caddr_t)new_iblk);
8682                 reload_mmu = 1;
8683                 new_iblk = NULL;
8684         }
8685 
8686 #ifdef DEBUG
8687         /*
8688          * Make sure mapping does not already exist.
8689          */
8690         ism_blkp = sfmmup->sfmmu_iblk;
8691         while (ism_blkp != NULL) {
8692                 ism_map = ism_blkp->iblk_maps;
8693                 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) {
8694                         if ((addr >= ism_start(ism_map[i]) &&
8695                             addr < ism_end(ism_map[i])) ||
8696                             eaddr > ism_start(ism_map[i]) &&
8697                             eaddr <= ism_end(ism_map[i])) {
8698                                 panic("sfmmu_share: Already mapped!");
8699                         }
8700                 }
8701                 ism_blkp = ism_blkp->iblk_next;
8702         }
8703 #endif /* DEBUG */
8704 
8705         ASSERT(ismszc >= TTE4M);
8706         if (ismszc == TTE4M) {
8707                 ismhatflag = HAT_4M_FLAG;
8708         } else if (ismszc == TTE32M) {
8709                 ismhatflag = HAT_32M_FLAG;
8710         } else if (ismszc == TTE256M) {
8711                 ismhatflag = HAT_256M_FLAG;
8712         }
8713         /*
8714          * Add mapping to first available mapping slot.
8715          */
8716         ism_blkp = sfmmup->sfmmu_iblk;
8717         added = 0;
8718         while (!added) {
8719                 ism_map = ism_blkp->iblk_maps;
8720                 for (i = 0; i < ISM_MAP_SLOTS; i++)  {
8721                         if (ism_map[i].imap_ismhat == NULL) {
8722 
8723                                 ism_map[i].imap_ismhat = ism_hatid;
8724                                 ism_map[i].imap_vb_shift = (uchar_t)ismshift;
8725                                 ism_map[i].imap_rid = SFMMU_INVALID_ISMRID;
8726                                 ism_map[i].imap_hatflags = ismhatflag;
8727                                 ism_map[i].imap_sz_mask = ismmask;
8728                                 /*
8729                                  * imap_seg is checked in ISM_CHECK to see if
8730                                  * non-NULL, then other info assumed valid.
8731                                  */
8732                                 membar_stst();
8733                                 ism_map[i].imap_seg = (uintptr_t)addr | sh_size;
8734                                 ism_map[i].imap_ment = ism_ment;
8735 
8736                                 /*
8737                                  * Now add ourselves to the ism_hat's
8738                                  * mapping list.
8739                                  */
8740                                 ism_ment->iment_hat = sfmmup;
8741                                 ism_ment->iment_base_va = addr;
8742                                 ism_hatid->sfmmu_ismhat = 1;
8743                                 mutex_enter(&ism_mlist_lock);
8744                                 iment_add(ism_ment, ism_hatid);
8745                                 mutex_exit(&ism_mlist_lock);
8746                                 added = 1;
8747                                 break;
8748                         }
8749                 }
8750                 if (!added && ism_blkp->iblk_next == NULL) {
8751                         ism_blkp->iblk_next = new_iblk;
8752                         new_iblk = NULL;
8753                         bzero(ism_blkp->iblk_next,
8754                             sizeof (*ism_blkp->iblk_next));
8755                         ism_blkp->iblk_next->iblk_nextpa = (uint64_t)-1;
8756                         membar_stst();
8757                         ism_blkp->iblk_nextpa =
8758                             va_to_pa((caddr_t)ism_blkp->iblk_next);
8759                 }
8760                 ism_blkp = ism_blkp->iblk_next;
8761         }
8762 
8763         /*
8764          * After calling hat_join_region, sfmmup may join a new SCD or
8765          * move from the old scd to a new scd, in which case, we want to
8766          * shrink the sfmmup's private tsb size, i.e., pass shrink to
8767          * sfmmu_check_page_sizes at the end of this routine.
8768          */
8769         old_scdp = sfmmup->sfmmu_scdp;
8770 
8771         rcookie = hat_join_region(sfmmup, addr, len, (void *)ism_hatid, 0,
8772             PROT_ALL, ismszc, NULL, HAT_REGION_ISM);
8773         if (rcookie != HAT_INVALID_REGION_COOKIE) {
8774                 ism_map[i].imap_rid = (uchar_t)((uint64_t)rcookie);
8775         }
8776         /*
8777          * Update our counters for this sfmmup's ism mappings.
8778          */
8779         for (i = 0; i <= ismszc; i++) {
8780                 if (!(disable_ism_large_pages & (1 << i)))
8781                         (void) ism_tsb_entries(sfmmup, i);
8782         }
8783 
8784         /*
8785          * For ISM and DISM we do not support 512K pages, so we only only
8786          * search the 4M and 8K/64K hashes for 4 pagesize cpus, and search the
8787          * 256M or 32M, and 4M and 8K/64K hashes for 6 pagesize cpus.
8788          *
8789          * Need to set 32M/256M ISM flags to make sure
8790          * sfmmu_check_page_sizes() enables them on Panther.
8791          */
8792         ASSERT((disable_ism_large_pages & (1 << TTE512K)) != 0);
8793 
8794         switch (ismszc) {
8795         case TTE256M:
8796                 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_256M_ISM)) {
8797                         hatlockp = sfmmu_hat_enter(sfmmup);
8798                         SFMMU_FLAGS_SET(sfmmup, HAT_256M_ISM);
8799                         sfmmu_hat_exit(hatlockp);
8800                 }
8801                 break;
8802         case TTE32M:
8803                 if (!SFMMU_FLAGS_ISSET(sfmmup, HAT_32M_ISM)) {
8804                         hatlockp = sfmmu_hat_enter(sfmmup);
8805                         SFMMU_FLAGS_SET(sfmmup, HAT_32M_ISM);
8806                         sfmmu_hat_exit(hatlockp);
8807                 }
8808                 break;
8809         default:
8810                 break;
8811         }
8812 
8813         /*
8814          * If we updated the ismblkpa for this HAT we must make
8815          * sure all CPUs running this process reload their tsbmiss area.
8816          * Otherwise they will fail to load the mappings in the tsbmiss
8817          * handler and will loop calling pagefault().
8818          */
8819         if (reload_mmu) {
8820                 hatlockp = sfmmu_hat_enter(sfmmup);
8821                 sfmmu_sync_mmustate(sfmmup);
8822                 sfmmu_hat_exit(hatlockp);
8823         }
8824 
8825         sfmmu_ismhat_exit(sfmmup, 0);
8826 
8827         /*
8828          * Free up ismblk if we didn't use it.
8829          */
8830         if (new_iblk != NULL)
8831                 kmem_cache_free(ism_blk_cache, new_iblk);
8832 
8833         /*
8834          * Check TSB and TLB page sizes.
8835          */
8836         if (sfmmup->sfmmu_scdp != NULL && old_scdp != sfmmup->sfmmu_scdp) {
8837                 sfmmu_check_page_sizes(sfmmup, 0);
8838         } else {
8839                 sfmmu_check_page_sizes(sfmmup, 1);
8840         }
8841         return (0);
8842 }
8843 
8844 /*
8845  * hat_unshare removes exactly one ism_map from
8846  * this process's as.  It expects multiple calls
8847  * to hat_unshare for multiple shm segments.
8848  */
8849 void
8850 hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc)
8851 {
8852         ism_map_t       *ism_map;
8853         ism_ment_t      *free_ment = NULL;
8854         ism_blk_t       *ism_blkp;
8855         struct hat      *ism_hatid;
8856         int             found, i;
8857         hatlock_t       *hatlockp;
8858         struct tsb_info *tsbinfo;
8859         uint_t          ismshift = page_get_shift(ismszc);
8860         size_t          sh_size = ISM_SHIFT(ismshift, len);
8861         uchar_t         ism_rid;
8862         sf_scd_t        *old_scdp;
8863 
8864         ASSERT(ISM_ALIGNED(ismshift, addr));
8865         ASSERT(ISM_ALIGNED(ismshift, len));
8866         ASSERT(sfmmup != NULL);
8867         ASSERT(sfmmup != ksfmmup);
8868 
8869         if (sfmmup->sfmmu_xhat_provider) {
8870                 XHAT_UNSHARE(sfmmup, addr, len);
8871                 return;
8872         } else {
8873                 /*
8874                  * This must be a CPU HAT. If the address space has
8875                  * XHATs attached, inform all XHATs that ISM segment
8876                  * is going away
8877                  */
8878                 ASSERT(sfmmup->sfmmu_as != NULL);
8879                 if (sfmmup->sfmmu_as->a_xhat != NULL)
8880                         xhat_unshare_all(sfmmup->sfmmu_as, addr, len);
8881         }
8882 
8883         /*
8884          * Make sure that during the entire time ISM mappings are removed,
8885          * the trap handlers serialize behind us, and that no one else
8886          * can be mucking with ISM mappings.  This also lets us get away
8887          * with not doing expensive cross calls to flush the TLB -- we
8888          * just discard the context, flush the entire TSB, and call it
8889          * a day.
8890          */
8891         sfmmu_ismhat_enter(sfmmup, 0);
8892 
8893         /*
8894          * Remove the mapping.
8895          *
8896          * We can't have any holes in the ism map.
8897          * The tsb miss code while searching the ism map will
8898          * stop on an empty map slot.  So we must move
8899          * everyone past the hole up 1 if any.
8900          *
8901          * Also empty ism map blks are not freed until the
8902          * process exits. This is to prevent a MT race condition
8903          * between sfmmu_unshare() and sfmmu_tsbmiss_exception().
8904          */
8905         found = 0;
8906         ism_blkp = sfmmup->sfmmu_iblk;
8907         while (!found && ism_blkp != NULL) {
8908                 ism_map = ism_blkp->iblk_maps;
8909                 for (i = 0; i < ISM_MAP_SLOTS; i++) {
8910                         if (addr == ism_start(ism_map[i]) &&
8911                             sh_size == (size_t)(ism_size(ism_map[i]))) {
8912                                 found = 1;
8913                                 break;
8914                         }
8915                 }
8916                 if (!found)
8917                         ism_blkp = ism_blkp->iblk_next;
8918         }
8919 
8920         if (found) {
8921                 ism_hatid = ism_map[i].imap_ismhat;
8922                 ism_rid = ism_map[i].imap_rid;
8923                 ASSERT(ism_hatid != NULL);
8924                 ASSERT(ism_hatid->sfmmu_ismhat == 1);
8925 
8926                 /*
8927                  * After hat_leave_region, the sfmmup may leave SCD,
8928                  * in which case, we want to grow the private tsb size when
8929                  * calling sfmmu_check_page_sizes at the end of the routine.
8930                  */
8931                 old_scdp = sfmmup->sfmmu_scdp;
8932                 /*
8933                  * Then remove ourselves from the region.
8934                  */
8935                 if (ism_rid != SFMMU_INVALID_ISMRID) {
8936                         hat_leave_region(sfmmup, (void *)((uint64_t)ism_rid),
8937                             HAT_REGION_ISM);
8938                 }
8939 
8940                 /*
8941                  * And now guarantee that any other cpu
8942                  * that tries to process an ISM miss
8943                  * will go to tl=0.
8944                  */
8945                 hatlockp = sfmmu_hat_enter(sfmmup);
8946                 sfmmu_invalidate_ctx(sfmmup);
8947                 sfmmu_hat_exit(hatlockp);
8948 
8949                 /*
8950                  * Remove ourselves from the ism mapping list.
8951                  */
8952                 mutex_enter(&ism_mlist_lock);
8953                 iment_sub(ism_map[i].imap_ment, ism_hatid);
8954                 mutex_exit(&ism_mlist_lock);
8955                 free_ment = ism_map[i].imap_ment;
8956 
8957                 /*
8958                  * We delete the ism map by copying
8959                  * the next map over the current one.
8960                  * We will take the next one in the maps
8961                  * array or from the next ism_blk.
8962                  */
8963                 while (ism_blkp != NULL) {
8964                         ism_map = ism_blkp->iblk_maps;
8965                         while (i < (ISM_MAP_SLOTS - 1)) {
8966                                 ism_map[i] = ism_map[i + 1];
8967                                 i++;
8968                         }
8969                         /* i == (ISM_MAP_SLOTS - 1) */
8970                         ism_blkp = ism_blkp->iblk_next;
8971                         if (ism_blkp != NULL) {
8972                                 ism_map[i] = ism_blkp->iblk_maps[0];
8973                                 i = 0;
8974                         } else {
8975                                 ism_map[i].imap_seg = 0;
8976                                 ism_map[i].imap_vb_shift = 0;
8977                                 ism_map[i].imap_rid = SFMMU_INVALID_ISMRID;
8978                                 ism_map[i].imap_hatflags = 0;
8979                                 ism_map[i].imap_sz_mask = 0;
8980                                 ism_map[i].imap_ismhat = NULL;
8981                                 ism_map[i].imap_ment = NULL;
8982                         }
8983                 }
8984 
8985                 /*
8986                  * Now flush entire TSB for the process, since
8987                  * demapping page by page can be too expensive.
8988                  * We don't have to flush the TLB here anymore
8989                  * since we switch to a new TLB ctx instead.
8990                  * Also, there is no need to flush if the process
8991                  * is exiting since the TSB will be freed later.
8992                  */
8993                 if (!sfmmup->sfmmu_free) {
8994                         hatlockp = sfmmu_hat_enter(sfmmup);
8995                         for (tsbinfo = sfmmup->sfmmu_tsb; tsbinfo != NULL;
8996                             tsbinfo = tsbinfo->tsb_next) {
8997                                 if (tsbinfo->tsb_flags & TSB_SWAPPED)
8998                                         continue;
8999                                 if (tsbinfo->tsb_flags & TSB_RELOC_FLAG) {
9000                                         tsbinfo->tsb_flags |=
9001                                             TSB_FLUSH_NEEDED;
9002                                         continue;
9003                                 }
9004 
9005                                 sfmmu_inv_tsb(tsbinfo->tsb_va,
9006                                     TSB_BYTES(tsbinfo->tsb_szc));
9007                         }
9008                         sfmmu_hat_exit(hatlockp);
9009                 }
9010         }
9011 
9012         /*
9013          * Update our counters for this sfmmup's ism mappings.
9014          */
9015         for (i = 0; i <= ismszc; i++) {
9016                 if (!(disable_ism_large_pages & (1 << i)))
9017                         (void) ism_tsb_entries(sfmmup, i);
9018         }
9019 
9020         sfmmu_ismhat_exit(sfmmup, 0);
9021 
9022         /*
9023          * We must do our freeing here after dropping locks
9024          * to prevent a deadlock in the kmem allocator on the
9025          * mapping list lock.
9026          */
9027         if (free_ment != NULL)
9028                 kmem_cache_free(ism_ment_cache, free_ment);
9029 
9030         /*
9031          * Check TSB and TLB page sizes if the process isn't exiting.
9032          */
9033         if (!sfmmup->sfmmu_free) {
9034                 if (found && old_scdp != NULL && sfmmup->sfmmu_scdp == NULL) {
9035                         sfmmu_check_page_sizes(sfmmup, 1);
9036                 } else {
9037                         sfmmu_check_page_sizes(sfmmup, 0);
9038                 }
9039         }
9040 }
9041 
9042 /* ARGSUSED */
9043 static int
9044 sfmmu_idcache_constructor(void *buf, void *cdrarg, int kmflags)
9045 {
9046         /* void *buf is sfmmu_t pointer */
9047         bzero(buf, sizeof (sfmmu_t));
9048 
9049         return (0);
9050 }
9051 
9052 /* ARGSUSED */
9053 static void
9054 sfmmu_idcache_destructor(void *buf, void *cdrarg)
9055 {
9056         /* void *buf is sfmmu_t pointer */
9057 }
9058 
9059 /*
9060  * setup kmem hmeblks by bzeroing all members and initializing the nextpa
9061  * field to be the pa of this hmeblk
9062  */
9063 /* ARGSUSED */
9064 static int
9065 sfmmu_hblkcache_constructor(void *buf, void *cdrarg, int kmflags)
9066 {
9067         struct hme_blk *hmeblkp;
9068 
9069         bzero(buf, (size_t)cdrarg);
9070         hmeblkp = (struct hme_blk *)buf;
9071         hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp);
9072 
9073 #ifdef  HBLK_TRACE
9074         mutex_init(&hmeblkp->hblk_audit_lock, NULL, MUTEX_DEFAULT, NULL);
9075 #endif  /* HBLK_TRACE */
9076 
9077         return (0);
9078 }
9079 
9080 /* ARGSUSED */
9081 static void
9082 sfmmu_hblkcache_destructor(void *buf, void *cdrarg)
9083 {
9084 
9085 #ifdef  HBLK_TRACE
9086 
9087         struct hme_blk *hmeblkp;
9088 
9089         hmeblkp = (struct hme_blk *)buf;
9090         mutex_destroy(&hmeblkp->hblk_audit_lock);
9091 
9092 #endif  /* HBLK_TRACE */
9093 }
9094 
9095 #define SFMMU_CACHE_RECLAIM_SCAN_RATIO 8
9096 static int sfmmu_cache_reclaim_scan_ratio = SFMMU_CACHE_RECLAIM_SCAN_RATIO;
9097 /*
9098  * The kmem allocator will callback into our reclaim routine when the system
9099  * is running low in memory.  We traverse the hash and free up all unused but
9100  * still cached hme_blks.  We also traverse the free list and free them up
9101  * as well.
9102  */
9103 /*ARGSUSED*/
9104 static void
9105 sfmmu_hblkcache_reclaim(void *cdrarg)
9106 {
9107         int i;
9108         struct hmehash_bucket *hmebp;
9109         struct hme_blk *hmeblkp, *nx_hblk, *pr_hblk = NULL;
9110         static struct hmehash_bucket *uhmehash_reclaim_hand;
9111         static struct hmehash_bucket *khmehash_reclaim_hand;
9112         struct hme_blk *list = NULL, *last_hmeblkp;
9113         cpuset_t cpuset = cpu_ready_set;
9114         cpu_hme_pend_t *cpuhp;
9115 
9116         /* Free up hmeblks on the cpu pending lists */
9117         for (i = 0; i < NCPU; i++) {
9118                 cpuhp = &cpu_hme_pend[i];
9119                 if (cpuhp->chp_listp != NULL)  {
9120                         mutex_enter(&cpuhp->chp_mutex);
9121                         if (cpuhp->chp_listp == NULL) {
9122                                 mutex_exit(&cpuhp->chp_mutex);
9123                                 continue;
9124                         }
9125                         for (last_hmeblkp = cpuhp->chp_listp;
9126                             last_hmeblkp->hblk_next != NULL;
9127                             last_hmeblkp = last_hmeblkp->hblk_next)
9128                                 ;
9129                         last_hmeblkp->hblk_next = list;
9130                         list = cpuhp->chp_listp;
9131                         cpuhp->chp_listp = NULL;
9132                         cpuhp->chp_count = 0;
9133                         mutex_exit(&cpuhp->chp_mutex);
9134                 }
9135 
9136         }
9137 
9138         if (list != NULL) {
9139                 kpreempt_disable();
9140                 CPUSET_DEL(cpuset, CPU->cpu_id);
9141                 xt_sync(cpuset);
9142                 xt_sync(cpuset);
9143                 kpreempt_enable();
9144                 sfmmu_hblk_free(&list);
9145                 list = NULL;
9146         }
9147 
9148         hmebp = uhmehash_reclaim_hand;
9149         if (hmebp == NULL || hmebp > &uhme_hash[UHMEHASH_SZ])
9150                 uhmehash_reclaim_hand = hmebp = uhme_hash;
9151         uhmehash_reclaim_hand += UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio;
9152 
9153         for (i = UHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) {
9154                 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) {
9155                         hmeblkp = hmebp->hmeblkp;
9156                         pr_hblk = NULL;
9157                         while (hmeblkp) {
9158                                 nx_hblk = hmeblkp->hblk_next;
9159                                 if (!hmeblkp->hblk_vcnt &&
9160                                     !hmeblkp->hblk_hmecnt) {
9161                                         sfmmu_hblk_hash_rm(hmebp, hmeblkp,
9162                                             pr_hblk, &list, 0);
9163                                 } else {
9164                                         pr_hblk = hmeblkp;
9165                                 }
9166                                 hmeblkp = nx_hblk;
9167                         }
9168                         SFMMU_HASH_UNLOCK(hmebp);
9169                 }
9170                 if (hmebp++ == &uhme_hash[UHMEHASH_SZ])
9171                         hmebp = uhme_hash;
9172         }
9173 
9174         hmebp = khmehash_reclaim_hand;
9175         if (hmebp == NULL || hmebp > &khme_hash[KHMEHASH_SZ])
9176                 khmehash_reclaim_hand = hmebp = khme_hash;
9177         khmehash_reclaim_hand += KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio;
9178 
9179         for (i = KHMEHASH_SZ / sfmmu_cache_reclaim_scan_ratio; i; i--) {
9180                 if (SFMMU_HASH_LOCK_TRYENTER(hmebp) != 0) {
9181                         hmeblkp = hmebp->hmeblkp;
9182                         pr_hblk = NULL;
9183                         while (hmeblkp) {
9184                                 nx_hblk = hmeblkp->hblk_next;
9185                                 if (!hmeblkp->hblk_vcnt &&
9186                                     !hmeblkp->hblk_hmecnt) {
9187                                         sfmmu_hblk_hash_rm(hmebp, hmeblkp,
9188                                             pr_hblk, &list, 0);
9189                                 } else {
9190                                         pr_hblk = hmeblkp;
9191                                 }
9192                                 hmeblkp = nx_hblk;
9193                         }
9194                         SFMMU_HASH_UNLOCK(hmebp);
9195                 }
9196                 if (hmebp++ == &khme_hash[KHMEHASH_SZ])
9197                         hmebp = khme_hash;
9198         }
9199         sfmmu_hblks_list_purge(&list, 0);
9200 }
9201 
9202 /*
9203  * sfmmu_get_ppvcolor should become a vm_machdep or hatop interface.
9204  * same goes for sfmmu_get_addrvcolor().
9205  *
9206  * This function will return the virtual color for the specified page. The
9207  * virtual color corresponds to this page current mapping or its last mapping.
9208  * It is used by memory allocators to choose addresses with the correct
9209  * alignment so vac consistency is automatically maintained.  If the page
9210  * has no color it returns -1.
9211  */
9212 /*ARGSUSED*/
9213 int
9214 sfmmu_get_ppvcolor(struct page *pp)
9215 {
9216 #ifdef VAC
9217         int color;
9218 
9219         if (!(cache & CACHE_VAC) || PP_NEWPAGE(pp)) {
9220                 return (-1);
9221         }
9222         color = PP_GET_VCOLOR(pp);
9223         ASSERT(color < mmu_btop(shm_alignment));
9224         return (color);
9225 #else
9226         return (-1);
9227 #endif  /* VAC */
9228 }
9229 
9230 /*
9231  * This function will return the desired alignment for vac consistency
9232  * (vac color) given a virtual address.  If no vac is present it returns -1.
9233  */
9234 /*ARGSUSED*/
9235 int
9236 sfmmu_get_addrvcolor(caddr_t vaddr)
9237 {
9238 #ifdef VAC
9239         if (cache & CACHE_VAC) {
9240                 return (addr_to_vcolor(vaddr));
9241         } else {
9242                 return (-1);
9243         }
9244 #else
9245         return (-1);
9246 #endif  /* VAC */
9247 }
9248 
9249 #ifdef VAC
9250 /*
9251  * Check for conflicts.
9252  * A conflict exists if the new and existent mappings do not match in
9253  * their "shm_alignment fields. If conflicts exist, the existant mappings
9254  * are flushed unless one of them is locked. If one of them is locked, then
9255  * the mappings are flushed and converted to non-cacheable mappings.
9256  */
9257 static void
9258 sfmmu_vac_conflict(struct hat *hat, caddr_t addr, page_t *pp)
9259 {
9260         struct hat *tmphat;
9261         struct sf_hment *sfhmep, *tmphme = NULL;
9262         struct hme_blk *hmeblkp;
9263         int vcolor;
9264         tte_t tte;
9265 
9266         ASSERT(sfmmu_mlist_held(pp));
9267         ASSERT(!PP_ISNC(pp));           /* page better be cacheable */
9268 
9269         vcolor = addr_to_vcolor(addr);
9270         if (PP_NEWPAGE(pp)) {
9271                 PP_SET_VCOLOR(pp, vcolor);
9272                 return;
9273         }
9274 
9275         if (PP_GET_VCOLOR(pp) == vcolor) {
9276                 return;
9277         }
9278 
9279         if (!PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp)) {
9280                 /*
9281                  * Previous user of page had a different color
9282                  * but since there are no current users
9283                  * we just flush the cache and change the color.
9284                  */
9285                 SFMMU_STAT(sf_pgcolor_conflict);
9286                 sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp));
9287                 PP_SET_VCOLOR(pp, vcolor);
9288                 return;
9289         }
9290 
9291         /*
9292          * If we get here we have a vac conflict with a current
9293          * mapping.  VAC conflict policy is as follows.
9294          * - The default is to unload the other mappings unless:
9295          * - If we have a large mapping we uncache the page.
9296          * We need to uncache the rest of the large page too.
9297          * - If any of the mappings are locked we uncache the page.
9298          * - If the requested mapping is inconsistent
9299          * with another mapping and that mapping
9300          * is in the same address space we have to
9301          * make it non-cached.  The default thing
9302          * to do is unload the inconsistent mapping
9303          * but if they are in the same address space
9304          * we run the risk of unmapping the pc or the
9305          * stack which we will use as we return to the user,
9306          * in which case we can then fault on the thing
9307          * we just unloaded and get into an infinite loop.
9308          */
9309         if (PP_ISMAPPED_LARGE(pp)) {
9310                 int sz;
9311 
9312                 /*
9313                  * Existing mapping is for big pages. We don't unload
9314                  * existing big mappings to satisfy new mappings.
9315                  * Always convert all mappings to TNC.
9316                  */
9317                 sz = fnd_mapping_sz(pp);
9318                 pp = PP_GROUPLEADER(pp, sz);
9319                 SFMMU_STAT_ADD(sf_uncache_conflict, TTEPAGES(sz));
9320                 sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH,
9321                     TTEPAGES(sz));
9322 
9323                 return;
9324         }
9325 
9326         /*
9327          * check if any mapping is in same as or if it is locked
9328          * since in that case we need to uncache.
9329          */
9330         for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) {
9331                 tmphme = sfhmep->hme_next;
9332                 if (IS_PAHME(sfhmep))
9333                         continue;
9334                 hmeblkp = sfmmu_hmetohblk(sfhmep);
9335                 if (hmeblkp->hblk_xhat_bit)
9336                         continue;
9337                 tmphat = hblktosfmmu(hmeblkp);
9338                 sfmmu_copytte(&sfhmep->hme_tte, &tte);
9339                 ASSERT(TTE_IS_VALID(&tte));
9340                 if (hmeblkp->hblk_shared || tmphat == hat ||
9341                     hmeblkp->hblk_lckcnt) {
9342                         /*
9343                          * We have an uncache conflict
9344                          */
9345                         SFMMU_STAT(sf_uncache_conflict);
9346                         sfmmu_page_cache_array(pp, HAT_TMPNC, CACHE_FLUSH, 1);
9347                         return;
9348                 }
9349         }
9350 
9351         /*
9352          * We have an unload conflict
9353          * We have already checked for LARGE mappings, therefore
9354          * the remaining mapping(s) must be TTE8K.
9355          */
9356         SFMMU_STAT(sf_unload_conflict);
9357 
9358         for (sfhmep = pp->p_mapping; sfhmep; sfhmep = tmphme) {
9359                 tmphme = sfhmep->hme_next;
9360                 if (IS_PAHME(sfhmep))
9361                         continue;
9362                 hmeblkp = sfmmu_hmetohblk(sfhmep);
9363                 if (hmeblkp->hblk_xhat_bit)
9364                         continue;
9365                 ASSERT(!hmeblkp->hblk_shared);
9366                 (void) sfmmu_pageunload(pp, sfhmep, TTE8K);
9367         }
9368 
9369         if (PP_ISMAPPED_KPM(pp))
9370                 sfmmu_kpm_vac_unload(pp, addr);
9371 
9372         /*
9373          * Unloads only do TLB flushes so we need to flush the
9374          * cache here.
9375          */
9376         sfmmu_cache_flush(pp->p_pagenum, PP_GET_VCOLOR(pp));
9377         PP_SET_VCOLOR(pp, vcolor);
9378 }
9379 
9380 /*
9381  * Whenever a mapping is unloaded and the page is in TNC state,
9382  * we see if the page can be made cacheable again. 'pp' is
9383  * the page that we just unloaded a mapping from, the size
9384  * of mapping that was unloaded is 'ottesz'.
9385  * Remark:
9386  * The recache policy for mpss pages can leave a performance problem
9387  * under the following circumstances:
9388  * . A large page in uncached mode has just been unmapped.
9389  * . All constituent pages are TNC due to a conflicting small mapping.
9390  * . There are many other, non conflicting, small mappings around for
9391  *   a lot of the constituent pages.
9392  * . We're called w/ the "old" groupleader page and the old ottesz,
9393  *   but this is irrelevant, since we're no more "PP_ISMAPPED_LARGE", so
9394  *   we end up w/ TTE8K or npages == 1.
9395  * . We call tst_tnc w/ the old groupleader only, and if there is no
9396  *   conflict, we re-cache only this page.
9397  * . All other small mappings are not checked and will be left in TNC mode.
9398  * The problem is not very serious because:
9399  * . mpss is actually only defined for heap and stack, so the probability
9400  *   is not very high that a large page mapping exists in parallel to a small
9401  *   one (this is possible, but seems to be bad programming style in the
9402  *   appl).
9403  * . The problem gets a little bit more serious, when those TNC pages
9404  *   have to be mapped into kernel space, e.g. for networking.
9405  * . When VAC alias conflicts occur in applications, this is regarded
9406  *   as an application bug. So if kstat's show them, the appl should
9407  *   be changed anyway.
9408  */
9409 void
9410 conv_tnc(page_t *pp, int ottesz)
9411 {
9412         int cursz, dosz;
9413         pgcnt_t curnpgs, dopgs;
9414         pgcnt_t pg64k;
9415         page_t *pp2;
9416 
9417         /*
9418          * Determine how big a range we check for TNC and find
9419          * leader page. cursz is the size of the biggest
9420          * mapping that still exist on 'pp'.
9421          */
9422         if (PP_ISMAPPED_LARGE(pp)) {
9423                 cursz = fnd_mapping_sz(pp);
9424         } else {
9425                 cursz = TTE8K;
9426         }
9427 
9428         if (ottesz >= cursz) {
9429                 dosz = ottesz;
9430                 pp2 = pp;
9431         } else {
9432                 dosz = cursz;
9433                 pp2 = PP_GROUPLEADER(pp, dosz);
9434         }
9435 
9436         pg64k = TTEPAGES(TTE64K);
9437         dopgs = TTEPAGES(dosz);
9438 
9439         ASSERT(dopgs == 1 || ((dopgs & (pg64k - 1)) == 0));
9440 
9441         while (dopgs != 0) {
9442                 curnpgs = TTEPAGES(cursz);
9443                 if (tst_tnc(pp2, curnpgs)) {
9444                         SFMMU_STAT_ADD(sf_recache, curnpgs);
9445                         sfmmu_page_cache_array(pp2, HAT_CACHE, CACHE_NO_FLUSH,
9446                             curnpgs);
9447                 }
9448 
9449                 ASSERT(dopgs >= curnpgs);
9450                 dopgs -= curnpgs;
9451 
9452                 if (dopgs == 0) {
9453                         break;
9454                 }
9455 
9456                 pp2 = PP_PAGENEXT_N(pp2, curnpgs);
9457                 if (((dopgs & (pg64k - 1)) == 0) && PP_ISMAPPED_LARGE(pp2)) {
9458                         cursz = fnd_mapping_sz(pp2);
9459                 } else {
9460                         cursz = TTE8K;
9461                 }
9462         }
9463 }
9464 
9465 /*
9466  * Returns 1 if page(s) can be converted from TNC to cacheable setting,
9467  * returns 0 otherwise. Note that oaddr argument is valid for only
9468  * 8k pages.
9469  */
9470 int
9471 tst_tnc(page_t *pp, pgcnt_t npages)
9472 {
9473         struct  sf_hment *sfhme;
9474         struct  hme_blk *hmeblkp;
9475         tte_t   tte;
9476         caddr_t vaddr;
9477         int     clr_valid = 0;
9478         int     color, color1, bcolor;
9479         int     i, ncolors;
9480 
9481         ASSERT(pp != NULL);
9482         ASSERT(!(cache & CACHE_WRITEBACK));
9483 
9484         if (npages > 1) {
9485                 ncolors = CACHE_NUM_COLOR;
9486         }
9487 
9488         for (i = 0; i < npages; i++) {
9489                 ASSERT(sfmmu_mlist_held(pp));
9490                 ASSERT(PP_ISTNC(pp));
9491                 ASSERT(PP_GET_VCOLOR(pp) == NO_VCOLOR);
9492 
9493                 if (PP_ISPNC(pp)) {
9494                         return (0);
9495                 }
9496 
9497                 clr_valid = 0;
9498                 if (PP_ISMAPPED_KPM(pp)) {
9499                         caddr_t kpmvaddr;
9500 
9501                         ASSERT(kpm_enable);
9502                         kpmvaddr = hat_kpm_page2va(pp, 1);
9503                         ASSERT(!(npages > 1 && IS_KPM_ALIAS_RANGE(kpmvaddr)));
9504                         color1 = addr_to_vcolor(kpmvaddr);
9505                         clr_valid = 1;
9506                 }
9507 
9508                 for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) {
9509                         if (IS_PAHME(sfhme))
9510                                 continue;
9511                         hmeblkp = sfmmu_hmetohblk(sfhme);
9512                         if (hmeblkp->hblk_xhat_bit)
9513                                 continue;
9514 
9515                         sfmmu_copytte(&sfhme->hme_tte, &tte);
9516                         ASSERT(TTE_IS_VALID(&tte));
9517 
9518                         vaddr = tte_to_vaddr(hmeblkp, tte);
9519                         color = addr_to_vcolor(vaddr);
9520 
9521                         if (npages > 1) {
9522                                 /*
9523                                  * If there is a big mapping, make sure
9524                                  * 8K mapping is consistent with the big
9525                                  * mapping.
9526                                  */
9527                                 bcolor = i % ncolors;
9528                                 if (color != bcolor) {
9529                                         return (0);
9530                                 }
9531                         }
9532                         if (!clr_valid) {
9533                                 clr_valid = 1;
9534                                 color1 = color;
9535                         }
9536 
9537                         if (color1 != color) {
9538                                 return (0);
9539                         }
9540                 }
9541 
9542                 pp = PP_PAGENEXT(pp);
9543         }
9544 
9545         return (1);
9546 }
9547 
9548 void
9549 sfmmu_page_cache_array(page_t *pp, int flags, int cache_flush_flag,
9550         pgcnt_t npages)
9551 {
9552         kmutex_t *pmtx;
9553         int i, ncolors, bcolor;
9554         kpm_hlk_t *kpmp;
9555         cpuset_t cpuset;
9556 
9557         ASSERT(pp != NULL);
9558         ASSERT(!(cache & CACHE_WRITEBACK));
9559 
9560         kpmp = sfmmu_kpm_kpmp_enter(pp, npages);
9561         pmtx = sfmmu_page_enter(pp);
9562 
9563         /*
9564          * Fast path caching single unmapped page
9565          */
9566         if (npages == 1 && !PP_ISMAPPED(pp) && !PP_ISMAPPED_KPM(pp) &&
9567             flags == HAT_CACHE) {
9568                 PP_CLRTNC(pp);
9569                 PP_CLRPNC(pp);
9570                 sfmmu_page_exit(pmtx);
9571                 sfmmu_kpm_kpmp_exit(kpmp);
9572                 return;
9573         }
9574 
9575         /*
9576          * We need to capture all cpus in order to change cacheability
9577          * because we can't allow one cpu to access the same physical
9578          * page using a cacheable and a non-cachebale mapping at the same
9579          * time. Since we may end up walking the ism mapping list
9580          * have to grab it's lock now since we can't after all the
9581          * cpus have been captured.
9582          */
9583         sfmmu_hat_lock_all();
9584         mutex_enter(&ism_mlist_lock);
9585         kpreempt_disable();
9586         cpuset = cpu_ready_set;
9587         xc_attention(cpuset);
9588 
9589         if (npages > 1) {
9590                 /*
9591                  * Make sure all colors are flushed since the
9592                  * sfmmu_page_cache() only flushes one color-
9593                  * it does not know big pages.
9594                  */
9595                 ncolors = CACHE_NUM_COLOR;
9596                 if (flags & HAT_TMPNC) {
9597                         for (i = 0; i < ncolors; i++) {
9598                                 sfmmu_cache_flushcolor(i, pp->p_pagenum);
9599                         }
9600                         cache_flush_flag = CACHE_NO_FLUSH;
9601                 }
9602         }
9603 
9604         for (i = 0; i < npages; i++) {
9605 
9606                 ASSERT(sfmmu_mlist_held(pp));
9607 
9608                 if (!(flags == HAT_TMPNC && PP_ISTNC(pp))) {
9609 
9610                         if (npages > 1) {
9611                                 bcolor = i % ncolors;
9612                         } else {
9613                                 bcolor = NO_VCOLOR;
9614                         }
9615 
9616                         sfmmu_page_cache(pp, flags, cache_flush_flag,
9617                             bcolor);
9618                 }
9619 
9620                 pp = PP_PAGENEXT(pp);
9621         }
9622 
9623         xt_sync(cpuset);
9624         xc_dismissed(cpuset);
9625         mutex_exit(&ism_mlist_lock);
9626         sfmmu_hat_unlock_all();
9627         sfmmu_page_exit(pmtx);
9628         sfmmu_kpm_kpmp_exit(kpmp);
9629         kpreempt_enable();
9630 }
9631 
9632 /*
9633  * This function changes the virtual cacheability of all mappings to a
9634  * particular page.  When changing from uncache to cacheable the mappings will
9635  * only be changed if all of them have the same virtual color.
9636  * We need to flush the cache in all cpus.  It is possible that
9637  * a process referenced a page as cacheable but has sinced exited
9638  * and cleared the mapping list.  We still to flush it but have no
9639  * state so all cpus is the only alternative.
9640  */
9641 static void
9642 sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor)
9643 {
9644         struct  sf_hment *sfhme;
9645         struct  hme_blk *hmeblkp;
9646         sfmmu_t *sfmmup;
9647         tte_t   tte, ttemod;
9648         caddr_t vaddr;
9649         int     ret, color;
9650         pfn_t   pfn;
9651 
9652         color = bcolor;
9653         pfn = pp->p_pagenum;
9654 
9655         for (sfhme = pp->p_mapping; sfhme; sfhme = sfhme->hme_next) {
9656 
9657                 if (IS_PAHME(sfhme))
9658                         continue;
9659                 hmeblkp = sfmmu_hmetohblk(sfhme);
9660 
9661                 if (hmeblkp->hblk_xhat_bit)
9662                         continue;
9663 
9664                 sfmmu_copytte(&sfhme->hme_tte, &tte);
9665                 ASSERT(TTE_IS_VALID(&tte));
9666                 vaddr = tte_to_vaddr(hmeblkp, tte);
9667                 color = addr_to_vcolor(vaddr);
9668 
9669 #ifdef DEBUG
9670                 if ((flags & HAT_CACHE) && bcolor != NO_VCOLOR) {
9671                         ASSERT(color == bcolor);
9672                 }
9673 #endif
9674 
9675                 ASSERT(flags != HAT_TMPNC || color == PP_GET_VCOLOR(pp));
9676 
9677                 ttemod = tte;
9678                 if (flags & (HAT_UNCACHE | HAT_TMPNC)) {
9679                         TTE_CLR_VCACHEABLE(&ttemod);
9680                 } else {        /* flags & HAT_CACHE */
9681                         TTE_SET_VCACHEABLE(&ttemod);
9682                 }
9683                 ret = sfmmu_modifytte_try(&tte, &ttemod, &sfhme->hme_tte);
9684                 if (ret < 0) {
9685                         /*
9686                          * Since all cpus are captured modifytte should not
9687                          * fail.
9688                          */
9689                         panic("sfmmu_page_cache: write to tte failed");
9690                 }
9691 
9692                 sfmmup = hblktosfmmu(hmeblkp);
9693                 if (cache_flush_flag == CACHE_FLUSH) {
9694                         /*
9695                          * Flush TSBs, TLBs and caches
9696                          */
9697                         if (hmeblkp->hblk_shared) {
9698                                 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
9699                                 uint_t rid = hmeblkp->hblk_tag.htag_rid;
9700                                 sf_region_t *rgnp;
9701                                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
9702                                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
9703                                 ASSERT(srdp != NULL);
9704                                 rgnp = srdp->srd_hmergnp[rid];
9705                                 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp,
9706                                     srdp, rgnp, rid);
9707                                 (void) sfmmu_rgntlb_demap(vaddr, rgnp,
9708                                     hmeblkp, 0);
9709                                 sfmmu_cache_flush(pfn, addr_to_vcolor(vaddr));
9710                         } else if (sfmmup->sfmmu_ismhat) {
9711                                 if (flags & HAT_CACHE) {
9712                                         SFMMU_STAT(sf_ism_recache);
9713                                 } else {
9714                                         SFMMU_STAT(sf_ism_uncache);
9715                                 }
9716                                 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp,
9717                                     pfn, CACHE_FLUSH);
9718                         } else {
9719                                 sfmmu_tlbcache_demap(vaddr, sfmmup, hmeblkp,
9720                                     pfn, 0, FLUSH_ALL_CPUS, CACHE_FLUSH, 1);
9721                         }
9722 
9723                         /*
9724                          * all cache entries belonging to this pfn are
9725                          * now flushed.
9726                          */
9727                         cache_flush_flag = CACHE_NO_FLUSH;
9728                 } else {
9729                         /*
9730                          * Flush only TSBs and TLBs.
9731                          */
9732                         if (hmeblkp->hblk_shared) {
9733                                 sf_srd_t *srdp = (sf_srd_t *)sfmmup;
9734                                 uint_t rid = hmeblkp->hblk_tag.htag_rid;
9735                                 sf_region_t *rgnp;
9736                                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
9737                                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
9738                                 ASSERT(srdp != NULL);
9739                                 rgnp = srdp->srd_hmergnp[rid];
9740                                 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp,
9741                                     srdp, rgnp, rid);
9742                                 (void) sfmmu_rgntlb_demap(vaddr, rgnp,
9743                                     hmeblkp, 0);
9744                         } else if (sfmmup->sfmmu_ismhat) {
9745                                 if (flags & HAT_CACHE) {
9746                                         SFMMU_STAT(sf_ism_recache);
9747                                 } else {
9748                                         SFMMU_STAT(sf_ism_uncache);
9749                                 }
9750                                 sfmmu_ismtlbcache_demap(vaddr, sfmmup, hmeblkp,
9751                                     pfn, CACHE_NO_FLUSH);
9752                         } else {
9753                                 sfmmu_tlb_demap(vaddr, sfmmup, hmeblkp, 0, 1);
9754                         }
9755                 }
9756         }
9757 
9758         if (PP_ISMAPPED_KPM(pp))
9759                 sfmmu_kpm_page_cache(pp, flags, cache_flush_flag);
9760 
9761         switch (flags) {
9762 
9763                 default:
9764                         panic("sfmmu_pagecache: unknown flags");
9765                         break;
9766 
9767                 case HAT_CACHE:
9768                         PP_CLRTNC(pp);
9769                         PP_CLRPNC(pp);
9770                         PP_SET_VCOLOR(pp, color);
9771                         break;
9772 
9773                 case HAT_TMPNC:
9774                         PP_SETTNC(pp);
9775                         PP_SET_VCOLOR(pp, NO_VCOLOR);
9776                         break;
9777 
9778                 case HAT_UNCACHE:
9779                         PP_SETPNC(pp);
9780                         PP_CLRTNC(pp);
9781                         PP_SET_VCOLOR(pp, NO_VCOLOR);
9782                         break;
9783         }
9784 }
9785 #endif  /* VAC */
9786 
9787 
9788 /*
9789  * Wrapper routine used to return a context.
9790  *
9791  * It's the responsibility of the caller to guarantee that the
9792  * process serializes on calls here by taking the HAT lock for
9793  * the hat.
9794  *
9795  */
9796 static void
9797 sfmmu_get_ctx(sfmmu_t *sfmmup)
9798 {
9799         mmu_ctx_t *mmu_ctxp;
9800         uint_t pstate_save;
9801         int ret;
9802 
9803         ASSERT(sfmmu_hat_lock_held(sfmmup));
9804         ASSERT(sfmmup != ksfmmup);
9805 
9806         if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID)) {
9807                 sfmmu_setup_tsbinfo(sfmmup);
9808                 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ALLCTX_INVALID);
9809         }
9810 
9811         kpreempt_disable();
9812 
9813         mmu_ctxp = CPU_MMU_CTXP(CPU);
9814         ASSERT(mmu_ctxp);
9815         ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms);
9816         ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]);
9817 
9818         /*
9819          * Do a wrap-around if cnum reaches the max # cnum supported by a MMU.
9820          */
9821         if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs)
9822                 sfmmu_ctx_wrap_around(mmu_ctxp, B_TRUE);
9823 
9824         /*
9825          * Let the MMU set up the page sizes to use for
9826          * this context in the TLB. Don't program 2nd dtlb for ism hat.
9827          */
9828         if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) {
9829                 mmu_set_ctx_page_sizes(sfmmup);
9830         }
9831 
9832         /*
9833          * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with
9834          * interrupts disabled to prevent race condition with wrap-around
9835          * ctx invalidatation. In sun4v, ctx invalidation also involves
9836          * a HV call to set the number of TSBs to 0. If interrupts are not
9837          * disabled until after sfmmu_load_mmustate is complete TSBs may
9838          * become assigned to INVALID_CONTEXT. This is not allowed.
9839          */
9840         pstate_save = sfmmu_disable_intrs();
9841 
9842         if (sfmmu_alloc_ctx(sfmmup, 1, CPU, SFMMU_PRIVATE) &&
9843             sfmmup->sfmmu_scdp != NULL) {
9844                 sf_scd_t *scdp = sfmmup->sfmmu_scdp;
9845                 sfmmu_t *scsfmmup = scdp->scd_sfmmup;
9846                 ret = sfmmu_alloc_ctx(scsfmmup, 1, CPU, SFMMU_SHARED);
9847                 /* debug purpose only */
9848                 ASSERT(!ret || scsfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum
9849                     != INVALID_CONTEXT);
9850         }
9851         sfmmu_load_mmustate(sfmmup);
9852 
9853         sfmmu_enable_intrs(pstate_save);
9854 
9855         kpreempt_enable();
9856 }
9857 
9858 /*
9859  * When all cnums are used up in a MMU, cnum will wrap around to the
9860  * next generation and start from 2.
9861  */
9862 static void
9863 sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp, boolean_t reset_cnum)
9864 {
9865 
9866         /* caller must have disabled the preemption */
9867         ASSERT(curthread->t_preempt >= 1);
9868         ASSERT(mmu_ctxp != NULL);
9869 
9870         /* acquire Per-MMU (PM) spin lock */
9871         mutex_enter(&mmu_ctxp->mmu_lock);
9872 
9873         /* re-check to see if wrap-around is needed */
9874         if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs)
9875                 goto done;
9876 
9877         SFMMU_MMU_STAT(mmu_wrap_around);
9878 
9879         /* update gnum */
9880         ASSERT(mmu_ctxp->mmu_gnum != 0);
9881         mmu_ctxp->mmu_gnum++;
9882         if (mmu_ctxp->mmu_gnum == 0 ||
9883             mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) {
9884                 cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.",
9885                     (void *)mmu_ctxp);
9886         }
9887 
9888         if (mmu_ctxp->mmu_ncpus > 1) {
9889                 cpuset_t cpuset;
9890 
9891                 membar_enter(); /* make sure updated gnum visible */
9892 
9893                 SFMMU_XCALL_STATS(NULL);
9894 
9895                 /* xcall to others on the same MMU to invalidate ctx */
9896                 cpuset = mmu_ctxp->mmu_cpuset;
9897                 ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id) || !reset_cnum);
9898                 CPUSET_DEL(cpuset, CPU->cpu_id);
9899                 CPUSET_AND(cpuset, cpu_ready_set);
9900 
9901                 /*
9902                  * Pass in INVALID_CONTEXT as the first parameter to
9903                  * sfmmu_raise_tsb_exception, which invalidates the context
9904                  * of any process running on the CPUs in the MMU.
9905                  */
9906                 xt_some(cpuset, sfmmu_raise_tsb_exception,
9907                     INVALID_CONTEXT, INVALID_CONTEXT);
9908                 xt_sync(cpuset);
9909 
9910                 SFMMU_MMU_STAT(mmu_tsb_raise_exception);
9911         }
9912 
9913         if (sfmmu_getctx_sec() != INVALID_CONTEXT) {
9914                 sfmmu_setctx_sec(INVALID_CONTEXT);
9915                 sfmmu_clear_utsbinfo();
9916         }
9917 
9918         /*
9919          * No xcall is needed here. For sun4u systems all CPUs in context
9920          * domain share a single physical MMU therefore it's enough to flush
9921          * TLB on local CPU. On sun4v systems we use 1 global context
9922          * domain and flush all remote TLBs in sfmmu_raise_tsb_exception
9923          * handler. Note that vtag_flushall_uctxs() is called
9924          * for Ultra II machine, where the equivalent flushall functionality
9925          * is implemented in SW, and only user ctx TLB entries are flushed.
9926          */
9927         if (&vtag_flushall_uctxs != NULL) {
9928                 vtag_flushall_uctxs();
9929         } else {
9930                 vtag_flushall();
9931         }
9932 
9933         /* reset mmu cnum, skips cnum 0 and 1 */
9934         if (reset_cnum == B_TRUE)
9935                 mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS;
9936 
9937 done:
9938         mutex_exit(&mmu_ctxp->mmu_lock);
9939 }
9940 
9941 
9942 /*
9943  * For multi-threaded process, set the process context to INVALID_CONTEXT
9944  * so that it faults and reloads the MMU state from TL=0. For single-threaded
9945  * process, we can just load the MMU state directly without having to
9946  * set context invalid. Caller must hold the hat lock since we don't
9947  * acquire it here.
9948  */
9949 static void
9950 sfmmu_sync_mmustate(sfmmu_t *sfmmup)
9951 {
9952         uint_t cnum;
9953         uint_t pstate_save;
9954 
9955         ASSERT(sfmmup != ksfmmup);
9956         ASSERT(sfmmu_hat_lock_held(sfmmup));
9957 
9958         kpreempt_disable();
9959 
9960         /*
9961          * We check whether the pass'ed-in sfmmup is the same as the
9962          * current running proc. This is to makes sure the current proc
9963          * stays single-threaded if it already is.
9964          */
9965         if ((sfmmup == curthread->t_procp->p_as->a_hat) &&
9966             (curthread->t_procp->p_lwpcnt == 1)) {
9967                 /* single-thread */
9968                 cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum;
9969                 if (cnum != INVALID_CONTEXT) {
9970                         uint_t curcnum;
9971                         /*
9972                          * Disable interrupts to prevent race condition
9973                          * with sfmmu_ctx_wrap_around ctx invalidation.
9974                          * In sun4v, ctx invalidation involves setting
9975                          * TSB to NULL, hence, interrupts should be disabled
9976                          * untill after sfmmu_load_mmustate is completed.
9977                          */
9978                         pstate_save = sfmmu_disable_intrs();
9979                         curcnum = sfmmu_getctx_sec();
9980                         if (curcnum == cnum)
9981                                 sfmmu_load_mmustate(sfmmup);
9982                         sfmmu_enable_intrs(pstate_save);
9983                         ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT);
9984                 }
9985         } else {
9986                 /*
9987                  * multi-thread
9988                  * or when sfmmup is not the same as the curproc.
9989                  */
9990                 sfmmu_invalidate_ctx(sfmmup);
9991         }
9992 
9993         kpreempt_enable();
9994 }
9995 
9996 
9997 /*
9998  * Replace the specified TSB with a new TSB.  This function gets called when
9999  * we grow, shrink or swapin a TSB.  When swapping in a TSB (TSB_SWAPIN), the
10000  * TSB_FORCEALLOC flag may be used to force allocation of a minimum-sized TSB
10001  * (8K).
10002  *
10003  * Caller must hold the HAT lock, but should assume any tsb_info
10004  * pointers it has are no longer valid after calling this function.
10005  *
10006  * Return values:
10007  *      TSB_ALLOCFAIL   Failed to allocate a TSB, due to memory constraints
10008  *      TSB_LOSTRACE    HAT is busy, i.e. another thread is already doing
10009  *                      something to this tsbinfo/TSB
10010  *      TSB_SUCCESS     Operation succeeded
10011  */
10012 static tsb_replace_rc_t
10013 sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc,
10014     hatlock_t *hatlockp, uint_t flags)
10015 {
10016         struct tsb_info *new_tsbinfo = NULL;
10017         struct tsb_info *curtsb, *prevtsb;
10018         uint_t tte_sz_mask;
10019         int i;
10020 
10021         ASSERT(sfmmup != ksfmmup);
10022         ASSERT(sfmmup->sfmmu_ismhat == 0);
10023         ASSERT(sfmmu_hat_lock_held(sfmmup));
10024         ASSERT(szc <= tsb_max_growsize);
10025 
10026         if (SFMMU_FLAGS_ISSET(sfmmup, HAT_BUSY))
10027                 return (TSB_LOSTRACE);
10028 
10029         /*
10030          * Find the tsb_info ahead of this one in the list, and
10031          * also make sure that the tsb_info passed in really
10032          * exists!
10033          */
10034         for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb;
10035             curtsb != old_tsbinfo && curtsb != NULL;
10036             prevtsb = curtsb, curtsb = curtsb->tsb_next)
10037                 ;
10038         ASSERT(curtsb != NULL);
10039 
10040         if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
10041                 /*
10042                  * The process is swapped out, so just set the new size
10043                  * code.  When it swaps back in, we'll allocate a new one
10044                  * of the new chosen size.
10045                  */
10046                 curtsb->tsb_szc = szc;
10047                 return (TSB_SUCCESS);
10048         }
10049         SFMMU_FLAGS_SET(sfmmup, HAT_BUSY);
10050 
10051         tte_sz_mask = old_tsbinfo->tsb_ttesz_mask;
10052 
10053         /*
10054          * All initialization is done inside of sfmmu_tsbinfo_alloc().
10055          * If we fail to allocate a TSB, exit.
10056          *
10057          * If tsb grows with new tsb size > 4M and old tsb size < 4M,
10058          * then try 4M slab after the initial alloc fails.
10059          *
10060          * If tsb swapin with tsb size > 4M, then try 4M after the
10061          * initial alloc fails.
10062          */
10063         sfmmu_hat_exit(hatlockp);
10064         if (sfmmu_tsbinfo_alloc(&new_tsbinfo, szc,
10065             tte_sz_mask, flags, sfmmup) &&
10066             (!(flags & (TSB_GROW | TSB_SWAPIN)) || (szc <= TSB_4M_SZCODE) ||
10067             (!(flags & TSB_SWAPIN) &&
10068             (old_tsbinfo->tsb_szc >= TSB_4M_SZCODE)) ||
10069             sfmmu_tsbinfo_alloc(&new_tsbinfo, TSB_4M_SZCODE,
10070             tte_sz_mask, flags, sfmmup))) {
10071                 (void) sfmmu_hat_enter(sfmmup);
10072                 if (!(flags & TSB_SWAPIN))
10073                         SFMMU_STAT(sf_tsb_resize_failures);
10074                 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
10075                 return (TSB_ALLOCFAIL);
10076         }
10077         (void) sfmmu_hat_enter(sfmmup);
10078 
10079         /*
10080          * Re-check to make sure somebody else didn't muck with us while we
10081          * didn't hold the HAT lock.  If the process swapped out, fine, just
10082          * exit; this can happen if we try to shrink the TSB from the context
10083          * of another process (such as on an ISM unmap), though it is rare.
10084          */
10085         if (!(flags & TSB_SWAPIN) && SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
10086                 SFMMU_STAT(sf_tsb_resize_failures);
10087                 SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
10088                 sfmmu_hat_exit(hatlockp);
10089                 sfmmu_tsbinfo_free(new_tsbinfo);
10090                 (void) sfmmu_hat_enter(sfmmup);
10091                 return (TSB_LOSTRACE);
10092         }
10093 
10094 #ifdef  DEBUG
10095         /* Reverify that the tsb_info still exists.. for debugging only */
10096         for (prevtsb = NULL, curtsb = sfmmup->sfmmu_tsb;
10097             curtsb != old_tsbinfo && curtsb != NULL;
10098             prevtsb = curtsb, curtsb = curtsb->tsb_next)
10099                 ;
10100         ASSERT(curtsb != NULL);
10101 #endif  /* DEBUG */
10102 
10103         /*
10104          * Quiesce any CPUs running this process on their next TLB miss
10105          * so they atomically see the new tsb_info.  We temporarily set the
10106          * context to invalid context so new threads that come on processor
10107          * after we do the xcall to cpusran will also serialize behind the
10108          * HAT lock on TLB miss and will see the new TSB.  Since this short
10109          * race with a new thread coming on processor is relatively rare,
10110          * this synchronization mechanism should be cheaper than always
10111          * pausing all CPUs for the duration of the setup, which is what
10112          * the old implementation did.  This is particuarly true if we are
10113          * copying a huge chunk of memory around during that window.
10114          *
10115          * The memory barriers are to make sure things stay consistent
10116          * with resume() since it does not hold the HAT lock while
10117          * walking the list of tsb_info structures.
10118          */
10119         if ((flags & TSB_SWAPIN) != TSB_SWAPIN) {
10120                 /* The TSB is either growing or shrinking. */
10121                 sfmmu_invalidate_ctx(sfmmup);
10122         } else {
10123                 /*
10124                  * It is illegal to swap in TSBs from a process other
10125                  * than a process being swapped in.  This in turn
10126                  * implies we do not have a valid MMU context here
10127                  * since a process needs one to resolve translation
10128                  * misses.
10129                  */
10130                 ASSERT(curthread->t_procp->p_as->a_hat == sfmmup);
10131         }
10132 
10133 #ifdef DEBUG
10134         ASSERT(max_mmu_ctxdoms > 0);
10135 
10136         /*
10137          * Process should have INVALID_CONTEXT on all MMUs
10138          */
10139         for (i = 0; i < max_mmu_ctxdoms; i++) {
10140 
10141                 ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT);
10142         }
10143 #endif
10144 
10145         new_tsbinfo->tsb_next = old_tsbinfo->tsb_next;
10146         membar_stst();  /* strict ordering required */
10147         if (prevtsb)
10148                 prevtsb->tsb_next = new_tsbinfo;
10149         else
10150                 sfmmup->sfmmu_tsb = new_tsbinfo;
10151         membar_enter(); /* make sure new TSB globally visible */
10152 
10153         /*
10154          * We need to migrate TSB entries from the old TSB to the new TSB
10155          * if tsb_remap_ttes is set and the TSB is growing.
10156          */
10157         if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW))
10158                 sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo);
10159 
10160         SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY);
10161 
10162         /*
10163          * Drop the HAT lock to free our old tsb_info.
10164          */
10165         sfmmu_hat_exit(hatlockp);
10166 
10167         if ((flags & TSB_GROW) == TSB_GROW) {
10168                 SFMMU_STAT(sf_tsb_grow);
10169         } else if ((flags & TSB_SHRINK) == TSB_SHRINK) {
10170                 SFMMU_STAT(sf_tsb_shrink);
10171         }
10172 
10173         sfmmu_tsbinfo_free(old_tsbinfo);
10174 
10175         (void) sfmmu_hat_enter(sfmmup);
10176         return (TSB_SUCCESS);
10177 }
10178 
10179 /*
10180  * This function will re-program hat pgsz array, and invalidate the
10181  * process' context, forcing the process to switch to another
10182  * context on the next TLB miss, and therefore start using the
10183  * TLB that is reprogrammed for the new page sizes.
10184  */
10185 void
10186 sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz)
10187 {
10188         int i;
10189         hatlock_t *hatlockp = NULL;
10190 
10191         hatlockp = sfmmu_hat_enter(sfmmup);
10192         /* USIII+-IV+ optimization, requires hat lock */
10193         if (tmp_pgsz) {
10194                 for (i = 0; i < mmu_page_sizes; i++)
10195                         sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i];
10196         }
10197         SFMMU_STAT(sf_tlb_reprog_pgsz);
10198 
10199         sfmmu_invalidate_ctx(sfmmup);
10200 
10201         sfmmu_hat_exit(hatlockp);
10202 }
10203 
10204 /*
10205  * The scd_rttecnt field in the SCD must be updated to take account of the
10206  * regions which it contains.
10207  */
10208 static void
10209 sfmmu_set_scd_rttecnt(sf_srd_t *srdp, sf_scd_t *scdp)
10210 {
10211         uint_t rid;
10212         uint_t i, j;
10213         ulong_t w;
10214         sf_region_t *rgnp;
10215 
10216         ASSERT(srdp != NULL);
10217 
10218         for (i = 0; i < SFMMU_HMERGNMAP_WORDS; i++) {
10219                 if ((w = scdp->scd_region_map.bitmap[i]) == 0) {
10220                         continue;
10221                 }
10222 
10223                 j = 0;
10224                 while (w) {
10225                         if (!(w & 0x1)) {
10226                                 j++;
10227                                 w >>= 1;
10228                                 continue;
10229                         }
10230                         rid = (i << BT_ULSHIFT) | j;
10231                         j++;
10232                         w >>= 1;
10233 
10234                         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
10235                         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
10236                         rgnp = srdp->srd_hmergnp[rid];
10237                         ASSERT(rgnp->rgn_refcnt > 0);
10238                         ASSERT(rgnp->rgn_id == rid);
10239 
10240                         scdp->scd_rttecnt[rgnp->rgn_pgszc] +=
10241                             rgnp->rgn_size >> TTE_PAGE_SHIFT(rgnp->rgn_pgszc);
10242 
10243                         /*
10244                          * Maintain the tsb0 inflation cnt for the regions
10245                          * in the SCD.
10246                          */
10247                         if (rgnp->rgn_pgszc >= TTE4M) {
10248                                 scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt +=
10249                                     rgnp->rgn_size >>
10250                                     (TTE_PAGE_SHIFT(TTE8K) + 2);
10251                         }
10252                 }
10253         }
10254 }
10255 
10256 /*
10257  * This function assumes that there are either four or six supported page
10258  * sizes and at most two programmable TLBs, so we need to decide which
10259  * page sizes are most important and then tell the MMU layer so it
10260  * can adjust the TLB page sizes accordingly (if supported).
10261  *
10262  * If these assumptions change, this function will need to be
10263  * updated to support whatever the new limits are.
10264  *
10265  * The growing flag is nonzero if we are growing the address space,
10266  * and zero if it is shrinking.  This allows us to decide whether
10267  * to grow or shrink our TSB, depending upon available memory
10268  * conditions.
10269  */
10270 static void
10271 sfmmu_check_page_sizes(sfmmu_t *sfmmup, int growing)
10272 {
10273         uint64_t ttecnt[MMU_PAGE_SIZES];
10274         uint64_t tte8k_cnt, tte4m_cnt;
10275         uint8_t i;
10276         int sectsb_thresh;
10277 
10278         /*
10279          * Kernel threads, processes with small address spaces not using
10280          * large pages, and dummy ISM HATs need not apply.
10281          */
10282         if (sfmmup == ksfmmup || sfmmup->sfmmu_ismhat != NULL)
10283                 return;
10284 
10285         if (!SFMMU_LGPGS_INUSE(sfmmup) &&
10286             sfmmup->sfmmu_ttecnt[TTE8K] <= tsb_rss_factor)
10287                 return;
10288 
10289         for (i = 0; i < mmu_page_sizes; i++) {
10290                 ttecnt[i] = sfmmup->sfmmu_ttecnt[i] +
10291                     sfmmup->sfmmu_ismttecnt[i];
10292         }
10293 
10294         /* Check pagesizes in use, and possibly reprogram DTLB. */
10295         if (&mmu_check_page_sizes)
10296                 mmu_check_page_sizes(sfmmup, ttecnt);
10297 
10298         /*
10299          * Calculate the number of 8k ttes to represent the span of these
10300          * pages.
10301          */
10302         tte8k_cnt = ttecnt[TTE8K] +
10303             (ttecnt[TTE64K] << (MMU_PAGESHIFT64K - MMU_PAGESHIFT)) +
10304             (ttecnt[TTE512K] << (MMU_PAGESHIFT512K - MMU_PAGESHIFT));
10305         if (mmu_page_sizes == max_mmu_page_sizes) {
10306                 tte4m_cnt = ttecnt[TTE4M] +
10307                     (ttecnt[TTE32M] << (MMU_PAGESHIFT32M - MMU_PAGESHIFT4M)) +
10308                     (ttecnt[TTE256M] << (MMU_PAGESHIFT256M - MMU_PAGESHIFT4M));
10309         } else {
10310                 tte4m_cnt = ttecnt[TTE4M];
10311         }
10312 
10313         /*
10314          * Inflate tte8k_cnt to allow for region large page allocation failure.
10315          */
10316         tte8k_cnt += sfmmup->sfmmu_tsb0_4minflcnt;
10317 
10318         /*
10319          * Inflate TSB sizes by a factor of 2 if this process
10320          * uses 4M text pages to minimize extra conflict misses
10321          * in the first TSB since without counting text pages
10322          * 8K TSB may become too small.
10323          *
10324          * Also double the size of the second TSB to minimize
10325          * extra conflict misses due to competition between 4M text pages
10326          * and data pages.
10327          *
10328          * We need to adjust the second TSB allocation threshold by the
10329          * inflation factor, since there is no point in creating a second
10330          * TSB when we know all the mappings can fit in the I/D TLBs.
10331          */
10332         sectsb_thresh = tsb_sectsb_threshold;
10333         if (sfmmup->sfmmu_flags & HAT_4MTEXT_FLAG) {
10334                 tte8k_cnt <<= 1;
10335                 tte4m_cnt <<= 1;
10336                 sectsb_thresh <<= 1;
10337         }
10338 
10339         /*
10340          * Check to see if our TSB is the right size; we may need to
10341          * grow or shrink it.  If the process is small, our work is
10342          * finished at this point.
10343          */
10344         if (tte8k_cnt <= tsb_rss_factor && tte4m_cnt <= sectsb_thresh) {
10345                 return;
10346         }
10347         sfmmu_size_tsb(sfmmup, growing, tte8k_cnt, tte4m_cnt, sectsb_thresh);
10348 }
10349 
10350 static void
10351 sfmmu_size_tsb(sfmmu_t *sfmmup, int growing, uint64_t tte8k_cnt,
10352         uint64_t tte4m_cnt, int sectsb_thresh)
10353 {
10354         int tsb_bits;
10355         uint_t tsb_szc;
10356         struct tsb_info *tsbinfop;
10357         hatlock_t *hatlockp = NULL;
10358 
10359         hatlockp = sfmmu_hat_enter(sfmmup);
10360         ASSERT(hatlockp != NULL);
10361         tsbinfop = sfmmup->sfmmu_tsb;
10362         ASSERT(tsbinfop != NULL);
10363 
10364         /*
10365          * If we're growing, select the size based on RSS.  If we're
10366          * shrinking, leave some room so we don't have to turn around and
10367          * grow again immediately.
10368          */
10369         if (growing)
10370                 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt);
10371         else
10372                 tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt << 1);
10373 
10374         if (!growing && (tsb_szc < tsbinfop->tsb_szc) &&
10375             (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) {
10376                 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc,
10377                     hatlockp, TSB_SHRINK);
10378         } else if (growing && tsb_szc > tsbinfop->tsb_szc && TSB_OK_GROW()) {
10379                 (void) sfmmu_replace_tsb(sfmmup, tsbinfop, tsb_szc,
10380                     hatlockp, TSB_GROW);
10381         }
10382         tsbinfop = sfmmup->sfmmu_tsb;
10383 
10384         /*
10385          * With the TLB and first TSB out of the way, we need to see if
10386          * we need a second TSB for 4M pages.  If we managed to reprogram
10387          * the TLB page sizes above, the process will start using this new
10388          * TSB right away; otherwise, it will start using it on the next
10389          * context switch.  Either way, it's no big deal so there's no
10390          * synchronization with the trap handlers here unless we grow the
10391          * TSB (in which case it's required to prevent using the old one
10392          * after it's freed). Note: second tsb is required for 32M/256M
10393          * page sizes.
10394          */
10395         if (tte4m_cnt > sectsb_thresh) {
10396                 /*
10397                  * If we're growing, select the size based on RSS.  If we're
10398                  * shrinking, leave some room so we don't have to turn
10399                  * around and grow again immediately.
10400                  */
10401                 if (growing)
10402                         tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt);
10403                 else
10404                         tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt << 1);
10405                 if (tsbinfop->tsb_next == NULL) {
10406                         struct tsb_info *newtsb;
10407                         int allocflags = SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)?
10408                             0 : TSB_ALLOC;
10409 
10410                         sfmmu_hat_exit(hatlockp);
10411 
10412                         /*
10413                          * Try to allocate a TSB for 4[32|256]M pages.  If we
10414                          * can't get the size we want, retry w/a minimum sized
10415                          * TSB.  If that still didn't work, give up; we can
10416                          * still run without one.
10417                          */
10418                         tsb_bits = (mmu_page_sizes == max_mmu_page_sizes)?
10419                             TSB4M|TSB32M|TSB256M:TSB4M;
10420                         if ((sfmmu_tsbinfo_alloc(&newtsb, tsb_szc, tsb_bits,
10421                             allocflags, sfmmup)) &&
10422                             (tsb_szc <= TSB_4M_SZCODE ||
10423                             sfmmu_tsbinfo_alloc(&newtsb, TSB_4M_SZCODE,
10424                             tsb_bits, allocflags, sfmmup)) &&
10425                             sfmmu_tsbinfo_alloc(&newtsb, TSB_MIN_SZCODE,
10426                             tsb_bits, allocflags, sfmmup)) {
10427                                 return;
10428                         }
10429 
10430                         hatlockp = sfmmu_hat_enter(sfmmup);
10431 
10432                         sfmmu_invalidate_ctx(sfmmup);
10433 
10434                         if (sfmmup->sfmmu_tsb->tsb_next == NULL) {
10435                                 sfmmup->sfmmu_tsb->tsb_next = newtsb;
10436                                 SFMMU_STAT(sf_tsb_sectsb_create);
10437                                 sfmmu_hat_exit(hatlockp);
10438                                 return;
10439                         } else {
10440                                 /*
10441                                  * It's annoying, but possible for us
10442                                  * to get here.. we dropped the HAT lock
10443                                  * because of locking order in the kmem
10444                                  * allocator, and while we were off getting
10445                                  * our memory, some other thread decided to
10446                                  * do us a favor and won the race to get a
10447                                  * second TSB for this process.  Sigh.
10448                                  */
10449                                 sfmmu_hat_exit(hatlockp);
10450                                 sfmmu_tsbinfo_free(newtsb);
10451                                 return;
10452                         }
10453                 }
10454 
10455                 /*
10456                  * We have a second TSB, see if it's big enough.
10457                  */
10458                 tsbinfop = tsbinfop->tsb_next;
10459 
10460                 /*
10461                  * Check to see if our second TSB is the right size;
10462                  * we may need to grow or shrink it.
10463                  * To prevent thrashing (e.g. growing the TSB on a
10464                  * subsequent map operation), only try to shrink if
10465                  * the TSB reach exceeds twice the virtual address
10466                  * space size.
10467                  */
10468                 if (!growing && (tsb_szc < tsbinfop->tsb_szc) &&
10469                     (tsb_szc >= default_tsb_size) && TSB_OK_SHRINK()) {
10470                         (void) sfmmu_replace_tsb(sfmmup, tsbinfop,
10471                             tsb_szc, hatlockp, TSB_SHRINK);
10472                 } else if (growing && tsb_szc > tsbinfop->tsb_szc &&
10473                     TSB_OK_GROW()) {
10474                         (void) sfmmu_replace_tsb(sfmmup, tsbinfop,
10475                             tsb_szc, hatlockp, TSB_GROW);
10476                 }
10477         }
10478 
10479         sfmmu_hat_exit(hatlockp);
10480 }
10481 
10482 /*
10483  * Free up a sfmmu
10484  * Since the sfmmu is currently embedded in the hat struct we simply zero
10485  * out our fields and free up the ism map blk list if any.
10486  */
10487 static void
10488 sfmmu_free_sfmmu(sfmmu_t *sfmmup)
10489 {
10490         ism_blk_t       *blkp, *nx_blkp;
10491 #ifdef  DEBUG
10492         ism_map_t       *map;
10493         int             i;
10494 #endif
10495 
10496         ASSERT(sfmmup->sfmmu_ttecnt[TTE8K] == 0);
10497         ASSERT(sfmmup->sfmmu_ttecnt[TTE64K] == 0);
10498         ASSERT(sfmmup->sfmmu_ttecnt[TTE512K] == 0);
10499         ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0);
10500         ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0);
10501         ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0);
10502         ASSERT(SF_RGNMAP_ISNULL(sfmmup));
10503 
10504         sfmmup->sfmmu_free = 0;
10505         sfmmup->sfmmu_ismhat = 0;
10506 
10507         blkp = sfmmup->sfmmu_iblk;
10508         sfmmup->sfmmu_iblk = NULL;
10509 
10510         while (blkp) {
10511 #ifdef  DEBUG
10512                 map = blkp->iblk_maps;
10513                 for (i = 0; i < ISM_MAP_SLOTS; i++) {
10514                         ASSERT(map[i].imap_seg == 0);
10515                         ASSERT(map[i].imap_ismhat == NULL);
10516                         ASSERT(map[i].imap_ment == NULL);
10517                 }
10518 #endif
10519                 nx_blkp = blkp->iblk_next;
10520                 blkp->iblk_next = NULL;
10521                 blkp->iblk_nextpa = (uint64_t)-1;
10522                 kmem_cache_free(ism_blk_cache, blkp);
10523                 blkp = nx_blkp;
10524         }
10525 }
10526 
10527 /*
10528  * Locking primitves accessed by HATLOCK macros
10529  */
10530 
10531 #define SFMMU_SPL_MTX   (0x0)
10532 #define SFMMU_ML_MTX    (0x1)
10533 
10534 #define SFMMU_MLSPL_MTX(type, pg)       (((type) == SFMMU_SPL_MTX) ? \
10535                                             SPL_HASH(pg) : MLIST_HASH(pg))
10536 
10537 kmutex_t *
10538 sfmmu_page_enter(struct page *pp)
10539 {
10540         return (sfmmu_mlspl_enter(pp, SFMMU_SPL_MTX));
10541 }
10542 
10543 void
10544 sfmmu_page_exit(kmutex_t *spl)
10545 {
10546         mutex_exit(spl);
10547 }
10548 
10549 int
10550 sfmmu_page_spl_held(struct page *pp)
10551 {
10552         return (sfmmu_mlspl_held(pp, SFMMU_SPL_MTX));
10553 }
10554 
10555 kmutex_t *
10556 sfmmu_mlist_enter(struct page *pp)
10557 {
10558         return (sfmmu_mlspl_enter(pp, SFMMU_ML_MTX));
10559 }
10560 
10561 void
10562 sfmmu_mlist_exit(kmutex_t *mml)
10563 {
10564         mutex_exit(mml);
10565 }
10566 
10567 int
10568 sfmmu_mlist_held(struct page *pp)
10569 {
10570 
10571         return (sfmmu_mlspl_held(pp, SFMMU_ML_MTX));
10572 }
10573 
10574 /*
10575  * Common code for sfmmu_mlist_enter() and sfmmu_page_enter().  For
10576  * sfmmu_mlist_enter() case mml_table lock array is used and for
10577  * sfmmu_page_enter() sfmmu_page_lock lock array is used.
10578  *
10579  * The lock is taken on a root page so that it protects an operation on all
10580  * constituent pages of a large page pp belongs to.
10581  *
10582  * The routine takes a lock from the appropriate array. The lock is determined
10583  * by hashing the root page. After taking the lock this routine checks if the
10584  * root page has the same size code that was used to determine the root (i.e
10585  * that root hasn't changed).  If root page has the expected p_szc field we
10586  * have the right lock and it's returned to the caller. If root's p_szc
10587  * decreased we release the lock and retry from the beginning.  This case can
10588  * happen due to hat_page_demote() decreasing p_szc between our load of p_szc
10589  * value and taking the lock. The number of retries due to p_szc decrease is
10590  * limited by the maximum p_szc value. If p_szc is 0 we return the lock
10591  * determined by hashing pp itself.
10592  *
10593  * If our caller doesn't hold a SE_SHARED or SE_EXCL lock on pp it's also
10594  * possible that p_szc can increase. To increase p_szc a thread has to lock
10595  * all constituent pages EXCL and do hat_pageunload() on all of them. All the
10596  * callers that don't hold a page locked recheck if hmeblk through which pp
10597  * was found still maps this pp.  If it doesn't map it anymore returned lock
10598  * is immediately dropped. Therefore if sfmmu_mlspl_enter() hits the case of
10599  * p_szc increase after taking the lock it returns this lock without further
10600  * retries because in this case the caller doesn't care about which lock was
10601  * taken. The caller will drop it right away.
10602  *
10603  * After the routine returns it's guaranteed that hat_page_demote() can't
10604  * change p_szc field of any of constituent pages of a large page pp belongs
10605  * to as long as pp was either locked at least SHARED prior to this call or
10606  * the caller finds that hment that pointed to this pp still references this
10607  * pp (this also assumes that the caller holds hme hash bucket lock so that
10608  * the same pp can't be remapped into the same hmeblk after it was unmapped by
10609  * hat_pageunload()).
10610  */
10611 static kmutex_t *
10612 sfmmu_mlspl_enter(struct page *pp, int type)
10613 {
10614         kmutex_t        *mtx;
10615         uint_t          prev_rszc = UINT_MAX;
10616         page_t          *rootpp;
10617         uint_t          szc;
10618         uint_t          rszc;
10619         uint_t          pszc = pp->p_szc;
10620 
10621         ASSERT(pp != NULL);
10622 
10623 again:
10624         if (pszc == 0) {
10625                 mtx = SFMMU_MLSPL_MTX(type, pp);
10626                 mutex_enter(mtx);
10627                 return (mtx);
10628         }
10629 
10630         /* The lock lives in the root page */
10631         rootpp = PP_GROUPLEADER(pp, pszc);
10632         mtx = SFMMU_MLSPL_MTX(type, rootpp);
10633         mutex_enter(mtx);
10634 
10635         /*
10636          * Return mml in the following 3 cases:
10637          *
10638          * 1) If pp itself is root since if its p_szc decreased before we took
10639          * the lock pp is still the root of smaller szc page. And if its p_szc
10640          * increased it doesn't matter what lock we return (see comment in
10641          * front of this routine).
10642          *
10643          * 2) If pp's not root but rootpp is the root of a rootpp->p_szc size
10644          * large page we have the right lock since any previous potential
10645          * hat_page_demote() is done demoting from greater than current root's
10646          * p_szc because hat_page_demote() changes root's p_szc last. No
10647          * further hat_page_demote() can start or be in progress since it
10648          * would need the same lock we currently hold.
10649          *
10650          * 3) If rootpp's p_szc increased since previous iteration it doesn't
10651          * matter what lock we return (see comment in front of this routine).
10652          */
10653         if (pp == rootpp || (rszc = rootpp->p_szc) == pszc ||
10654             rszc >= prev_rszc) {
10655                 return (mtx);
10656         }
10657 
10658         /*
10659          * hat_page_demote() could have decreased root's p_szc.
10660          * In this case pp's p_szc must also be smaller than pszc.
10661          * Retry.
10662          */
10663         if (rszc < pszc) {
10664                 szc = pp->p_szc;
10665                 if (szc < pszc) {
10666                         mutex_exit(mtx);
10667                         pszc = szc;
10668                         goto again;
10669                 }
10670                 /*
10671                  * pp's p_szc increased after it was decreased.
10672                  * page cannot be mapped. Return current lock. The caller
10673                  * will drop it right away.
10674                  */
10675                 return (mtx);
10676         }
10677 
10678         /*
10679          * root's p_szc is greater than pp's p_szc.
10680          * hat_page_demote() is not done with all pages
10681          * yet. Wait for it to complete.
10682          */
10683         mutex_exit(mtx);
10684         rootpp = PP_GROUPLEADER(rootpp, rszc);
10685         mtx = SFMMU_MLSPL_MTX(type, rootpp);
10686         mutex_enter(mtx);
10687         mutex_exit(mtx);
10688         prev_rszc = rszc;
10689         goto again;
10690 }
10691 
10692 static int
10693 sfmmu_mlspl_held(struct page *pp, int type)
10694 {
10695         kmutex_t        *mtx;
10696 
10697         ASSERT(pp != NULL);
10698         /* The lock lives in the root page */
10699         pp = PP_PAGEROOT(pp);
10700         ASSERT(pp != NULL);
10701 
10702         mtx = SFMMU_MLSPL_MTX(type, pp);
10703         return (MUTEX_HELD(mtx));
10704 }
10705 
10706 static uint_t
10707 sfmmu_get_free_hblk(struct hme_blk **hmeblkpp, uint_t critical)
10708 {
10709         struct  hme_blk *hblkp;
10710 
10711 
10712         if (freehblkp != NULL) {
10713                 mutex_enter(&freehblkp_lock);
10714                 if (freehblkp != NULL) {
10715                         /*
10716                          * If the current thread is owning hblk_reserve OR
10717                          * critical request from sfmmu_hblk_steal()
10718                          * let it succeed even if freehblkcnt is really low.
10719                          */
10720                         if (freehblkcnt <= HBLK_RESERVE_MIN && !critical) {
10721                                 SFMMU_STAT(sf_get_free_throttle);
10722                                 mutex_exit(&freehblkp_lock);
10723                                 return (0);
10724                         }
10725                         freehblkcnt--;
10726                         *hmeblkpp = freehblkp;
10727                         hblkp = *hmeblkpp;
10728                         freehblkp = hblkp->hblk_next;
10729                         mutex_exit(&freehblkp_lock);
10730                         hblkp->hblk_next = NULL;
10731                         SFMMU_STAT(sf_get_free_success);
10732 
10733                         ASSERT(hblkp->hblk_hmecnt == 0);
10734                         ASSERT(hblkp->hblk_vcnt == 0);
10735                         ASSERT(hblkp->hblk_nextpa == va_to_pa((caddr_t)hblkp));
10736 
10737                         return (1);
10738                 }
10739                 mutex_exit(&freehblkp_lock);
10740         }
10741 
10742         /* Check cpu hblk pending queues */
10743         if ((*hmeblkpp = sfmmu_check_pending_hblks(TTE8K)) != NULL) {
10744                 hblkp = *hmeblkpp;
10745                 hblkp->hblk_next = NULL;
10746                 hblkp->hblk_nextpa = va_to_pa((caddr_t)hblkp);
10747 
10748                 ASSERT(hblkp->hblk_hmecnt == 0);
10749                 ASSERT(hblkp->hblk_vcnt == 0);
10750 
10751                 return (1);
10752         }
10753 
10754         SFMMU_STAT(sf_get_free_fail);
10755         return (0);
10756 }
10757 
10758 static uint_t
10759 sfmmu_put_free_hblk(struct hme_blk *hmeblkp, uint_t critical)
10760 {
10761         struct  hme_blk *hblkp;
10762 
10763         ASSERT(hmeblkp->hblk_hmecnt == 0);
10764         ASSERT(hmeblkp->hblk_vcnt == 0);
10765         ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp));
10766 
10767         /*
10768          * If the current thread is mapping into kernel space,
10769          * let it succede even if freehblkcnt is max
10770          * so that it will avoid freeing it to kmem.
10771          * This will prevent stack overflow due to
10772          * possible recursion since kmem_cache_free()
10773          * might require creation of a slab which
10774          * in turn needs an hmeblk to map that slab;
10775          * let's break this vicious chain at the first
10776          * opportunity.
10777          */
10778         if (freehblkcnt < HBLK_RESERVE_CNT || critical) {
10779                 mutex_enter(&freehblkp_lock);
10780                 if (freehblkcnt < HBLK_RESERVE_CNT || critical) {
10781                         SFMMU_STAT(sf_put_free_success);
10782                         freehblkcnt++;
10783                         hmeblkp->hblk_next = freehblkp;
10784                         freehblkp = hmeblkp;
10785                         mutex_exit(&freehblkp_lock);
10786                         return (1);
10787                 }
10788                 mutex_exit(&freehblkp_lock);
10789         }
10790 
10791         /*
10792          * Bring down freehblkcnt to HBLK_RESERVE_CNT. We are here
10793          * only if freehblkcnt is at least HBLK_RESERVE_CNT *and*
10794          * we are not in the process of mapping into kernel space.
10795          */
10796         ASSERT(!critical);
10797         while (freehblkcnt > HBLK_RESERVE_CNT) {
10798                 mutex_enter(&freehblkp_lock);
10799                 if (freehblkcnt > HBLK_RESERVE_CNT) {
10800                         freehblkcnt--;
10801                         hblkp = freehblkp;
10802                         freehblkp = hblkp->hblk_next;
10803                         mutex_exit(&freehblkp_lock);
10804                         ASSERT(get_hblk_cache(hblkp) == sfmmu8_cache);
10805                         kmem_cache_free(sfmmu8_cache, hblkp);
10806                         continue;
10807                 }
10808                 mutex_exit(&freehblkp_lock);
10809         }
10810         SFMMU_STAT(sf_put_free_fail);
10811         return (0);
10812 }
10813 
10814 static void
10815 sfmmu_hblk_swap(struct hme_blk *new)
10816 {
10817         struct hme_blk *old, *hblkp, *prev;
10818         uint64_t newpa;
10819         caddr_t base, vaddr, endaddr;
10820         struct hmehash_bucket *hmebp;
10821         struct sf_hment *osfhme, *nsfhme;
10822         page_t *pp;
10823         kmutex_t *pml;
10824         tte_t tte;
10825         struct hme_blk *list = NULL;
10826 
10827 #ifdef  DEBUG
10828         hmeblk_tag              hblktag;
10829         struct hme_blk          *found;
10830 #endif
10831         old = HBLK_RESERVE;
10832         ASSERT(!old->hblk_shared);
10833 
10834         /*
10835          * save pa before bcopy clobbers it
10836          */
10837         newpa = new->hblk_nextpa;
10838 
10839         base = (caddr_t)get_hblk_base(old);
10840         endaddr = base + get_hblk_span(old);
10841 
10842         /*
10843          * acquire hash bucket lock.
10844          */
10845         hmebp = sfmmu_tteload_acquire_hashbucket(ksfmmup, base, TTE8K,
10846             SFMMU_INVALID_SHMERID);
10847 
10848         /*
10849          * copy contents from old to new
10850          */
10851         bcopy((void *)old, (void *)new, HME8BLK_SZ);
10852 
10853         /*
10854          * add new to hash chain
10855          */
10856         sfmmu_hblk_hash_add(hmebp, new, newpa);
10857 
10858         /*
10859          * search hash chain for hblk_reserve; this needs to be performed
10860          * after adding new, otherwise prev won't correspond to the hblk which
10861          * is prior to old in hash chain when we call sfmmu_hblk_hash_rm to
10862          * remove old later.
10863          */
10864         for (prev = NULL,
10865             hblkp = hmebp->hmeblkp; hblkp != NULL && hblkp != old;
10866             prev = hblkp, hblkp = hblkp->hblk_next)
10867                 ;
10868 
10869         if (hblkp != old)
10870                 panic("sfmmu_hblk_swap: hblk_reserve not found");
10871 
10872         /*
10873          * p_mapping list is still pointing to hments in hblk_reserve;
10874          * fix up p_mapping list so that they point to hments in new.
10875          *
10876          * Since all these mappings are created by hblk_reserve_thread
10877          * on the way and it's using at least one of the buffers from each of
10878          * the newly minted slabs, there is no danger of any of these
10879          * mappings getting unloaded by another thread.
10880          *
10881          * tsbmiss could only modify ref/mod bits of hments in old/new.
10882          * Since all of these hments hold mappings established by segkmem
10883          * and mappings in segkmem are setup with HAT_NOSYNC, ref/mod bits
10884          * have no meaning for the mappings in hblk_reserve.  hments in
10885          * old and new are identical except for ref/mod bits.
10886          */
10887         for (vaddr = base; vaddr < endaddr; vaddr += TTEBYTES(TTE8K)) {
10888 
10889                 HBLKTOHME(osfhme, old, vaddr);
10890                 sfmmu_copytte(&osfhme->hme_tte, &tte);
10891 
10892                 if (TTE_IS_VALID(&tte)) {
10893                         if ((pp = osfhme->hme_page) == NULL)
10894                                 panic("sfmmu_hblk_swap: page not mapped");
10895 
10896                         pml = sfmmu_mlist_enter(pp);
10897 
10898                         if (pp != osfhme->hme_page)
10899                                 panic("sfmmu_hblk_swap: mapping changed");
10900 
10901                         HBLKTOHME(nsfhme, new, vaddr);
10902 
10903                         HME_ADD(nsfhme, pp);
10904                         HME_SUB(osfhme, pp);
10905 
10906                         sfmmu_mlist_exit(pml);
10907                 }
10908         }
10909 
10910         /*
10911          * remove old from hash chain
10912          */
10913         sfmmu_hblk_hash_rm(hmebp, old, prev, &list, 1);
10914 
10915 #ifdef  DEBUG
10916 
10917         hblktag.htag_id = ksfmmup;
10918         hblktag.htag_rid = SFMMU_INVALID_SHMERID;
10919         hblktag.htag_bspage = HME_HASH_BSPAGE(base, HME_HASH_SHIFT(TTE8K));
10920         hblktag.htag_rehash = HME_HASH_REHASH(TTE8K);
10921         HME_HASH_FAST_SEARCH(hmebp, hblktag, found);
10922 
10923         if (found != new)
10924                 panic("sfmmu_hblk_swap: new hblk not found");
10925 #endif
10926 
10927         SFMMU_HASH_UNLOCK(hmebp);
10928 
10929         /*
10930          * Reset hblk_reserve
10931          */
10932         bzero((void *)old, HME8BLK_SZ);
10933         old->hblk_nextpa = va_to_pa((caddr_t)old);
10934 }
10935 
10936 /*
10937  * Grab the mlist mutex for both pages passed in.
10938  *
10939  * low and high will be returned as pointers to the mutexes for these pages.
10940  * low refers to the mutex residing in the lower bin of the mlist hash, while
10941  * high refers to the mutex residing in the higher bin of the mlist hash.  This
10942  * is due to the locking order restrictions on the same thread grabbing
10943  * multiple mlist mutexes.  The low lock must be acquired before the high lock.
10944  *
10945  * If both pages hash to the same mutex, only grab that single mutex, and
10946  * high will be returned as NULL
10947  * If the pages hash to different bins in the hash, grab the lower addressed
10948  * lock first and then the higher addressed lock in order to follow the locking
10949  * rules involved with the same thread grabbing multiple mlist mutexes.
10950  * low and high will both have non-NULL values.
10951  */
10952 static void
10953 sfmmu_mlist_reloc_enter(struct page *targ, struct page *repl,
10954     kmutex_t **low, kmutex_t **high)
10955 {
10956         kmutex_t        *mml_targ, *mml_repl;
10957 
10958         /*
10959          * no need to do the dance around szc as in sfmmu_mlist_enter()
10960          * because this routine is only called by hat_page_relocate() and all
10961          * targ and repl pages are already locked EXCL so szc can't change.
10962          */
10963 
10964         mml_targ = MLIST_HASH(PP_PAGEROOT(targ));
10965         mml_repl = MLIST_HASH(PP_PAGEROOT(repl));
10966 
10967         if (mml_targ == mml_repl) {
10968                 *low = mml_targ;
10969                 *high = NULL;
10970         } else {
10971                 if (mml_targ < mml_repl) {
10972                         *low = mml_targ;
10973                         *high = mml_repl;
10974                 } else {
10975                         *low = mml_repl;
10976                         *high = mml_targ;
10977                 }
10978         }
10979 
10980         mutex_enter(*low);
10981         if (*high)
10982                 mutex_enter(*high);
10983 }
10984 
10985 static void
10986 sfmmu_mlist_reloc_exit(kmutex_t *low, kmutex_t *high)
10987 {
10988         if (high)
10989                 mutex_exit(high);
10990         mutex_exit(low);
10991 }
10992 
10993 static hatlock_t *
10994 sfmmu_hat_enter(sfmmu_t *sfmmup)
10995 {
10996         hatlock_t       *hatlockp;
10997 
10998         if (sfmmup != ksfmmup) {
10999                 hatlockp = TSB_HASH(sfmmup);
11000                 mutex_enter(HATLOCK_MUTEXP(hatlockp));
11001                 return (hatlockp);
11002         }
11003         return (NULL);
11004 }
11005 
11006 static hatlock_t *
11007 sfmmu_hat_tryenter(sfmmu_t *sfmmup)
11008 {
11009         hatlock_t       *hatlockp;
11010 
11011         if (sfmmup != ksfmmup) {
11012                 hatlockp = TSB_HASH(sfmmup);
11013                 if (mutex_tryenter(HATLOCK_MUTEXP(hatlockp)) == 0)
11014                         return (NULL);
11015                 return (hatlockp);
11016         }
11017         return (NULL);
11018 }
11019 
11020 static void
11021 sfmmu_hat_exit(hatlock_t *hatlockp)
11022 {
11023         if (hatlockp != NULL)
11024                 mutex_exit(HATLOCK_MUTEXP(hatlockp));
11025 }
11026 
11027 static void
11028 sfmmu_hat_lock_all(void)
11029 {
11030         int i;
11031         for (i = 0; i < SFMMU_NUM_LOCK; i++)
11032                 mutex_enter(HATLOCK_MUTEXP(&hat_lock[i]));
11033 }
11034 
11035 static void
11036 sfmmu_hat_unlock_all(void)
11037 {
11038         int i;
11039         for (i = SFMMU_NUM_LOCK - 1; i >= 0; i--)
11040                 mutex_exit(HATLOCK_MUTEXP(&hat_lock[i]));
11041 }
11042 
11043 int
11044 sfmmu_hat_lock_held(sfmmu_t *sfmmup)
11045 {
11046         ASSERT(sfmmup != ksfmmup);
11047         return (MUTEX_HELD(HATLOCK_MUTEXP(TSB_HASH(sfmmup))));
11048 }
11049 
11050 /*
11051  * Locking primitives to provide consistency between ISM unmap
11052  * and other operations.  Since ISM unmap can take a long time, we
11053  * use HAT_ISMBUSY flag (protected by the hatlock) to avoid creating
11054  * contention on the hatlock buckets while ISM segments are being
11055  * unmapped.  The tradeoff is that the flags don't prevent priority
11056  * inversion from occurring, so we must request kernel priority in
11057  * case we have to sleep to keep from getting buried while holding
11058  * the HAT_ISMBUSY flag set, which in turn could block other kernel
11059  * threads from running (for example, in sfmmu_uvatopfn()).
11060  */
11061 static void
11062 sfmmu_ismhat_enter(sfmmu_t *sfmmup, int hatlock_held)
11063 {
11064         hatlock_t *hatlockp;
11065 
11066         THREAD_KPRI_REQUEST();
11067         if (!hatlock_held)
11068                 hatlockp = sfmmu_hat_enter(sfmmup);
11069         while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY))
11070                 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp));
11071         SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY);
11072         if (!hatlock_held)
11073                 sfmmu_hat_exit(hatlockp);
11074 }
11075 
11076 static void
11077 sfmmu_ismhat_exit(sfmmu_t *sfmmup, int hatlock_held)
11078 {
11079         hatlock_t *hatlockp;
11080 
11081         if (!hatlock_held)
11082                 hatlockp = sfmmu_hat_enter(sfmmup);
11083         ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
11084         SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY);
11085         cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11086         if (!hatlock_held)
11087                 sfmmu_hat_exit(hatlockp);
11088         THREAD_KPRI_RELEASE();
11089 }
11090 
11091 /*
11092  *
11093  * Algorithm:
11094  *
11095  * (1) if segkmem is not ready, allocate hblk from an array of pre-alloc'ed
11096  *      hblks.
11097  *
11098  * (2) if we are allocating an hblk for mapping a slab in sfmmu_cache,
11099  *
11100  *              (a) try to return an hblk from reserve pool of free hblks;
11101  *              (b) if the reserve pool is empty, acquire hblk_reserve_lock
11102  *                  and return hblk_reserve.
11103  *
11104  * (3) call kmem_cache_alloc() to allocate hblk;
11105  *
11106  *              (a) if hblk_reserve_lock is held by the current thread,
11107  *                  atomically replace hblk_reserve by the hblk that is
11108  *                  returned by kmem_cache_alloc; release hblk_reserve_lock
11109  *                  and call kmem_cache_alloc() again.
11110  *              (b) if reserve pool is not full, add the hblk that is
11111  *                  returned by kmem_cache_alloc to reserve pool and
11112  *                  call kmem_cache_alloc again.
11113  *
11114  */
11115 static struct hme_blk *
11116 sfmmu_hblk_alloc(sfmmu_t *sfmmup, caddr_t vaddr,
11117         struct hmehash_bucket *hmebp, uint_t size, hmeblk_tag hblktag,
11118         uint_t flags, uint_t rid)
11119 {
11120         struct hme_blk *hmeblkp = NULL;
11121         struct hme_blk *newhblkp;
11122         struct hme_blk *shw_hblkp = NULL;
11123         struct kmem_cache *sfmmu_cache = NULL;
11124         uint64_t hblkpa;
11125         ulong_t index;
11126         uint_t owner;           /* set to 1 if using hblk_reserve */
11127         uint_t forcefree;
11128         int sleep;
11129         sf_srd_t *srdp;
11130         sf_region_t *rgnp;
11131 
11132         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
11133         ASSERT(hblktag.htag_rid == rid);
11134         SFMMU_VALIDATE_HMERID(sfmmup, rid, vaddr, TTEBYTES(size));
11135         ASSERT(!SFMMU_IS_SHMERID_VALID(rid) ||
11136             IS_P2ALIGNED(vaddr, TTEBYTES(size)));
11137 
11138         /*
11139          * If segkmem is not created yet, allocate from static hmeblks
11140          * created at the end of startup_modules().  See the block comment
11141          * in startup_modules() describing how we estimate the number of
11142          * static hmeblks that will be needed during re-map.
11143          */
11144         if (!hblk_alloc_dynamic) {
11145 
11146                 ASSERT(!SFMMU_IS_SHMERID_VALID(rid));
11147 
11148                 if (size == TTE8K) {
11149                         index = nucleus_hblk8.index;
11150                         if (index >= nucleus_hblk8.len) {
11151                                 /*
11152                                  * If we panic here, see startup_modules() to
11153                                  * make sure that we are calculating the
11154                                  * number of hblk8's that we need correctly.
11155                                  */
11156                                 prom_panic("no nucleus hblk8 to allocate");
11157                         }
11158                         hmeblkp =
11159                             (struct hme_blk *)&nucleus_hblk8.list[index];
11160                         nucleus_hblk8.index++;
11161                         SFMMU_STAT(sf_hblk8_nalloc);
11162                 } else {
11163                         index = nucleus_hblk1.index;
11164                         if (nucleus_hblk1.index >= nucleus_hblk1.len) {
11165                                 /*
11166                                  * If we panic here, see startup_modules().
11167                                  * Most likely you need to update the
11168                                  * calculation of the number of hblk1 elements
11169                                  * that the kernel needs to boot.
11170                                  */
11171                                 prom_panic("no nucleus hblk1 to allocate");
11172                         }
11173                         hmeblkp =
11174                             (struct hme_blk *)&nucleus_hblk1.list[index];
11175                         nucleus_hblk1.index++;
11176                         SFMMU_STAT(sf_hblk1_nalloc);
11177                 }
11178 
11179                 goto hblk_init;
11180         }
11181 
11182         SFMMU_HASH_UNLOCK(hmebp);
11183 
11184         if (sfmmup != KHATID && !SFMMU_IS_SHMERID_VALID(rid)) {
11185                 if (mmu_page_sizes == max_mmu_page_sizes) {
11186                         if (size < TTE256M)
11187                                 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr,
11188                                     size, flags);
11189                 } else {
11190                         if (size < TTE4M)
11191                                 shw_hblkp = sfmmu_shadow_hcreate(sfmmup, vaddr,
11192                                     size, flags);
11193                 }
11194         } else if (SFMMU_IS_SHMERID_VALID(rid)) {
11195                 /*
11196                  * Shared hmes use per region bitmaps in rgn_hmeflag
11197                  * rather than shadow hmeblks to keep track of the
11198                  * mapping sizes which have been allocated for the region.
11199                  * Here we cleanup old invalid hmeblks with this rid,
11200                  * which may be left around by pageunload().
11201                  */
11202                 int ttesz;
11203                 caddr_t va;
11204                 caddr_t eva = vaddr + TTEBYTES(size);
11205 
11206                 ASSERT(sfmmup != KHATID);
11207 
11208                 srdp = sfmmup->sfmmu_srdp;
11209                 ASSERT(srdp != NULL && srdp->srd_refcnt != 0);
11210                 rgnp = srdp->srd_hmergnp[rid];
11211                 ASSERT(rgnp != NULL && rgnp->rgn_id == rid);
11212                 ASSERT(rgnp->rgn_refcnt != 0);
11213                 ASSERT(size <= rgnp->rgn_pgszc);
11214 
11215                 ttesz = HBLK_MIN_TTESZ;
11216                 do {
11217                         if (!(rgnp->rgn_hmeflags & (0x1 << ttesz))) {
11218                                 continue;
11219                         }
11220 
11221                         if (ttesz > size && ttesz != HBLK_MIN_TTESZ) {
11222                                 sfmmu_cleanup_rhblk(srdp, vaddr, rid, ttesz);
11223                         } else if (ttesz < size) {
11224                                 for (va = vaddr; va < eva;
11225                                     va += TTEBYTES(ttesz)) {
11226                                         sfmmu_cleanup_rhblk(srdp, va, rid,
11227                                             ttesz);
11228                                 }
11229                         }
11230                 } while (++ttesz <= rgnp->rgn_pgszc);
11231         }
11232 
11233 fill_hblk:
11234         owner = (hblk_reserve_thread == curthread) ? 1 : 0;
11235 
11236         if (owner && size == TTE8K) {
11237 
11238                 ASSERT(!SFMMU_IS_SHMERID_VALID(rid));
11239                 /*
11240                  * We are really in a tight spot. We already own
11241                  * hblk_reserve and we need another hblk.  In anticipation
11242                  * of this kind of scenario, we specifically set aside
11243                  * HBLK_RESERVE_MIN number of hblks to be used exclusively
11244                  * by owner of hblk_reserve.
11245                  */
11246                 SFMMU_STAT(sf_hblk_recurse_cnt);
11247 
11248                 if (!sfmmu_get_free_hblk(&hmeblkp, 1))
11249                         panic("sfmmu_hblk_alloc: reserve list is empty");
11250 
11251                 goto hblk_verify;
11252         }
11253 
11254         ASSERT(!owner);
11255 
11256         if ((flags & HAT_NO_KALLOC) == 0) {
11257 
11258                 sfmmu_cache = ((size == TTE8K) ? sfmmu8_cache : sfmmu1_cache);
11259                 sleep = ((sfmmup == KHATID) ? KM_NOSLEEP : KM_SLEEP);
11260 
11261                 if ((hmeblkp = kmem_cache_alloc(sfmmu_cache, sleep)) == NULL) {
11262                         hmeblkp = sfmmu_hblk_steal(size);
11263                 } else {
11264                         /*
11265                          * if we are the owner of hblk_reserve,
11266                          * swap hblk_reserve with hmeblkp and
11267                          * start a fresh life.  Hope things go
11268                          * better this time.
11269                          */
11270                         if (hblk_reserve_thread == curthread) {
11271                                 ASSERT(sfmmu_cache == sfmmu8_cache);
11272                                 sfmmu_hblk_swap(hmeblkp);
11273                                 hblk_reserve_thread = NULL;
11274                                 mutex_exit(&hblk_reserve_lock);
11275                                 goto fill_hblk;
11276                         }
11277                         /*
11278                          * let's donate this hblk to our reserve list if
11279                          * we are not mapping kernel range
11280                          */
11281                         if (size == TTE8K && sfmmup != KHATID) {
11282                                 if (sfmmu_put_free_hblk(hmeblkp, 0))
11283                                         goto fill_hblk;
11284                         }
11285                 }
11286         } else {
11287                 /*
11288                  * We are here to map the slab in sfmmu8_cache; let's
11289                  * check if we could tap our reserve list; if successful,
11290                  * this will avoid the pain of going thru sfmmu_hblk_swap
11291                  */
11292                 SFMMU_STAT(sf_hblk_slab_cnt);
11293                 if (!sfmmu_get_free_hblk(&hmeblkp, 0)) {
11294                         /*
11295                          * let's start hblk_reserve dance
11296                          */
11297                         SFMMU_STAT(sf_hblk_reserve_cnt);
11298                         owner = 1;
11299                         mutex_enter(&hblk_reserve_lock);
11300                         hmeblkp = HBLK_RESERVE;
11301                         hblk_reserve_thread = curthread;
11302                 }
11303         }
11304 
11305 hblk_verify:
11306         ASSERT(hmeblkp != NULL);
11307         set_hblk_sz(hmeblkp, size);
11308         ASSERT(hmeblkp->hblk_nextpa == va_to_pa((caddr_t)hmeblkp));
11309         SFMMU_HASH_LOCK(hmebp);
11310         HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp);
11311         if (newhblkp != NULL) {
11312                 SFMMU_HASH_UNLOCK(hmebp);
11313                 if (hmeblkp != HBLK_RESERVE) {
11314                         /*
11315                          * This is really tricky!
11316                          *
11317                          * vmem_alloc(vmem_seg_arena)
11318                          *  vmem_alloc(vmem_internal_arena)
11319                          *   segkmem_alloc(heap_arena)
11320                          *    vmem_alloc(heap_arena)
11321                          *    page_create()
11322                          *    hat_memload()
11323                          *      kmem_cache_free()
11324                          *       kmem_cache_alloc()
11325                          *        kmem_slab_create()
11326                          *         vmem_alloc(kmem_internal_arena)
11327                          *          segkmem_alloc(heap_arena)
11328                          *              vmem_alloc(heap_arena)
11329                          *              page_create()
11330                          *              hat_memload()
11331                          *                kmem_cache_free()
11332                          *              ...
11333                          *
11334                          * Thus, hat_memload() could call kmem_cache_free
11335                          * for enough number of times that we could easily
11336                          * hit the bottom of the stack or run out of reserve
11337                          * list of vmem_seg structs.  So, we must donate
11338                          * this hblk to reserve list if it's allocated
11339                          * from sfmmu8_cache *and* mapping kernel range.
11340                          * We don't need to worry about freeing hmeblk1's
11341                          * to kmem since they don't map any kmem slabs.
11342                          *
11343                          * Note: When segkmem supports largepages, we must
11344                          * free hmeblk1's to reserve list as well.
11345                          */
11346                         forcefree = (sfmmup == KHATID) ? 1 : 0;
11347                         if (size == TTE8K &&
11348                             sfmmu_put_free_hblk(hmeblkp, forcefree)) {
11349                                 goto re_verify;
11350                         }
11351                         ASSERT(sfmmup != KHATID);
11352                         kmem_cache_free(get_hblk_cache(hmeblkp), hmeblkp);
11353                 } else {
11354                         /*
11355                          * Hey! we don't need hblk_reserve any more.
11356                          */
11357                         ASSERT(owner);
11358                         hblk_reserve_thread = NULL;
11359                         mutex_exit(&hblk_reserve_lock);
11360                         owner = 0;
11361                 }
11362 re_verify:
11363                 /*
11364                  * let's check if the goodies are still present
11365                  */
11366                 SFMMU_HASH_LOCK(hmebp);
11367                 HME_HASH_FAST_SEARCH(hmebp, hblktag, newhblkp);
11368                 if (newhblkp != NULL) {
11369                         /*
11370                          * return newhblkp if it's not hblk_reserve;
11371                          * if newhblkp is hblk_reserve, return it
11372                          * _only if_ we are the owner of hblk_reserve.
11373                          */
11374                         if (newhblkp != HBLK_RESERVE || owner) {
11375                                 ASSERT(!SFMMU_IS_SHMERID_VALID(rid) ||
11376                                     newhblkp->hblk_shared);
11377                                 ASSERT(SFMMU_IS_SHMERID_VALID(rid) ||
11378                                     !newhblkp->hblk_shared);
11379                                 return (newhblkp);
11380                         } else {
11381                                 /*
11382                                  * we just hit hblk_reserve in the hash and
11383                                  * we are not the owner of that;
11384                                  *
11385                                  * block until hblk_reserve_thread completes
11386                                  * swapping hblk_reserve and try the dance
11387                                  * once again.
11388                                  */
11389                                 SFMMU_HASH_UNLOCK(hmebp);
11390                                 mutex_enter(&hblk_reserve_lock);
11391                                 mutex_exit(&hblk_reserve_lock);
11392                                 SFMMU_STAT(sf_hblk_reserve_hit);
11393                                 goto fill_hblk;
11394                         }
11395                 } else {
11396                         /*
11397                          * it's no more! try the dance once again.
11398                          */
11399                         SFMMU_HASH_UNLOCK(hmebp);
11400                         goto fill_hblk;
11401                 }
11402         }
11403 
11404 hblk_init:
11405         if (SFMMU_IS_SHMERID_VALID(rid)) {
11406                 uint16_t tteflag = 0x1 <<
11407                     ((size < HBLK_MIN_TTESZ) ? HBLK_MIN_TTESZ : size);
11408 
11409                 if (!(rgnp->rgn_hmeflags & tteflag)) {
11410                         atomic_or_16(&rgnp->rgn_hmeflags, tteflag);
11411                 }
11412                 hmeblkp->hblk_shared = 1;
11413         } else {
11414                 hmeblkp->hblk_shared = 0;
11415         }
11416         set_hblk_sz(hmeblkp, size);
11417         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
11418         hmeblkp->hblk_next = (struct hme_blk *)NULL;
11419         hmeblkp->hblk_tag = hblktag;
11420         hmeblkp->hblk_shadow = shw_hblkp;
11421         hblkpa = hmeblkp->hblk_nextpa;
11422         hmeblkp->hblk_nextpa = HMEBLK_ENDPA;
11423 
11424         ASSERT(get_hblk_ttesz(hmeblkp) == size);
11425         ASSERT(get_hblk_span(hmeblkp) == HMEBLK_SPAN(size));
11426         ASSERT(hmeblkp->hblk_hmecnt == 0);
11427         ASSERT(hmeblkp->hblk_vcnt == 0);
11428         ASSERT(hmeblkp->hblk_lckcnt == 0);
11429         ASSERT(hblkpa == va_to_pa((caddr_t)hmeblkp));
11430         sfmmu_hblk_hash_add(hmebp, hmeblkp, hblkpa);
11431         return (hmeblkp);
11432 }
11433 
11434 /*
11435  * This function cleans up the hme_blk and returns it to the free list.
11436  */
11437 /* ARGSUSED */
11438 static void
11439 sfmmu_hblk_free(struct hme_blk **listp)
11440 {
11441         struct hme_blk *hmeblkp, *next_hmeblkp;
11442         int             size;
11443         uint_t          critical;
11444         uint64_t        hblkpa;
11445 
11446         ASSERT(*listp != NULL);
11447 
11448         hmeblkp = *listp;
11449         while (hmeblkp != NULL) {
11450                 next_hmeblkp = hmeblkp->hblk_next;
11451                 ASSERT(!hmeblkp->hblk_hmecnt);
11452                 ASSERT(!hmeblkp->hblk_vcnt);
11453                 ASSERT(!hmeblkp->hblk_lckcnt);
11454                 ASSERT(hmeblkp != (struct hme_blk *)hblk_reserve);
11455                 ASSERT(hmeblkp->hblk_shared == 0);
11456                 ASSERT(hmeblkp->hblk_shw_bit == 0);
11457                 ASSERT(hmeblkp->hblk_shadow == NULL);
11458 
11459                 hblkpa = va_to_pa((caddr_t)hmeblkp);
11460                 ASSERT(hblkpa != (uint64_t)-1);
11461                 critical = (hblktosfmmu(hmeblkp) == KHATID) ? 1 : 0;
11462 
11463                 size = get_hblk_ttesz(hmeblkp);
11464                 hmeblkp->hblk_next = NULL;
11465                 hmeblkp->hblk_nextpa = hblkpa;
11466 
11467                 if (hmeblkp->hblk_nuc_bit == 0) {
11468 
11469                         if (size != TTE8K ||
11470                             !sfmmu_put_free_hblk(hmeblkp, critical))
11471                                 kmem_cache_free(get_hblk_cache(hmeblkp),
11472                                     hmeblkp);
11473                 }
11474                 hmeblkp = next_hmeblkp;
11475         }
11476 }
11477 
11478 #define BUCKETS_TO_SEARCH_BEFORE_UNLOAD 30
11479 #define SFMMU_HBLK_STEAL_THRESHOLD 5
11480 
11481 static uint_t sfmmu_hblk_steal_twice;
11482 static uint_t sfmmu_hblk_steal_count, sfmmu_hblk_steal_unload_count;
11483 
11484 /*
11485  * Steal a hmeblk from user or kernel hme hash lists.
11486  * For 8K tte grab one from reserve pool (freehblkp) before proceeding to
11487  * steal and if we fail to steal after SFMMU_HBLK_STEAL_THRESHOLD attempts
11488  * tap into critical reserve of freehblkp.
11489  * Note: We remain looping in this routine until we find one.
11490  */
11491 static struct hme_blk *
11492 sfmmu_hblk_steal(int size)
11493 {
11494         static struct hmehash_bucket *uhmehash_steal_hand = NULL;
11495         struct hmehash_bucket *hmebp;
11496         struct hme_blk *hmeblkp = NULL, *pr_hblk;
11497         uint64_t hblkpa;
11498         int i;
11499         uint_t loop_cnt = 0, critical;
11500 
11501         for (;;) {
11502                 /* Check cpu hblk pending queues */
11503                 if ((hmeblkp = sfmmu_check_pending_hblks(size)) != NULL) {
11504                         hmeblkp->hblk_nextpa = va_to_pa((caddr_t)hmeblkp);
11505                         ASSERT(hmeblkp->hblk_hmecnt == 0);
11506                         ASSERT(hmeblkp->hblk_vcnt == 0);
11507                         return (hmeblkp);
11508                 }
11509 
11510                 if (size == TTE8K) {
11511                         critical =
11512                             (++loop_cnt > SFMMU_HBLK_STEAL_THRESHOLD) ? 1 : 0;
11513                         if (sfmmu_get_free_hblk(&hmeblkp, critical))
11514                                 return (hmeblkp);
11515                 }
11516 
11517                 hmebp = (uhmehash_steal_hand == NULL) ? uhme_hash :
11518                     uhmehash_steal_hand;
11519                 ASSERT(hmebp >= uhme_hash && hmebp <= &uhme_hash[UHMEHASH_SZ]);
11520 
11521                 for (i = 0; hmeblkp == NULL && i <= UHMEHASH_SZ +
11522                     BUCKETS_TO_SEARCH_BEFORE_UNLOAD; i++) {
11523                         SFMMU_HASH_LOCK(hmebp);
11524                         hmeblkp = hmebp->hmeblkp;
11525                         hblkpa = hmebp->hmeh_nextpa;
11526                         pr_hblk = NULL;
11527                         while (hmeblkp) {
11528                                 /*
11529                                  * check if it is a hmeblk that is not locked
11530                                  * and not shared. skip shadow hmeblks with
11531                                  * shadow_mask set i.e valid count non zero.
11532                                  */
11533                                 if ((get_hblk_ttesz(hmeblkp) == size) &&
11534                                     (hmeblkp->hblk_shw_bit == 0 ||
11535                                     hmeblkp->hblk_vcnt == 0) &&
11536                                     (hmeblkp->hblk_lckcnt == 0)) {
11537                                         /*
11538                                          * there is a high probability that we
11539                                          * will find a free one. search some
11540                                          * buckets for a free hmeblk initially
11541                                          * before unloading a valid hmeblk.
11542                                          */
11543                                         if ((hmeblkp->hblk_vcnt == 0 &&
11544                                             hmeblkp->hblk_hmecnt == 0) || (i >=
11545                                             BUCKETS_TO_SEARCH_BEFORE_UNLOAD)) {
11546                                                 if (sfmmu_steal_this_hblk(hmebp,
11547                                                     hmeblkp, hblkpa, pr_hblk)) {
11548                                                         /*
11549                                                          * Hblk is unloaded
11550                                                          * successfully
11551                                                          */
11552                                                         break;
11553                                                 }
11554                                         }
11555                                 }
11556                                 pr_hblk = hmeblkp;
11557                                 hblkpa = hmeblkp->hblk_nextpa;
11558                                 hmeblkp = hmeblkp->hblk_next;
11559                         }
11560 
11561                         SFMMU_HASH_UNLOCK(hmebp);
11562                         if (hmebp++ == &uhme_hash[UHMEHASH_SZ])
11563                                 hmebp = uhme_hash;
11564                 }
11565                 uhmehash_steal_hand = hmebp;
11566 
11567                 if (hmeblkp != NULL)
11568                         break;
11569 
11570                 /*
11571                  * in the worst case, look for a free one in the kernel
11572                  * hash table.
11573                  */
11574                 for (i = 0, hmebp = khme_hash; i <= KHMEHASH_SZ; i++) {
11575                         SFMMU_HASH_LOCK(hmebp);
11576                         hmeblkp = hmebp->hmeblkp;
11577                         hblkpa = hmebp->hmeh_nextpa;
11578                         pr_hblk = NULL;
11579                         while (hmeblkp) {
11580                                 /*
11581                                  * check if it is free hmeblk
11582                                  */
11583                                 if ((get_hblk_ttesz(hmeblkp) == size) &&
11584                                     (hmeblkp->hblk_lckcnt == 0) &&
11585                                     (hmeblkp->hblk_vcnt == 0) &&
11586                                     (hmeblkp->hblk_hmecnt == 0)) {
11587                                         if (sfmmu_steal_this_hblk(hmebp,
11588                                             hmeblkp, hblkpa, pr_hblk)) {
11589                                                 break;
11590                                         } else {
11591                                                 /*
11592                                                  * Cannot fail since we have
11593                                                  * hash lock.
11594                                                  */
11595                                                 panic("fail to steal?");
11596                                         }
11597                                 }
11598 
11599                                 pr_hblk = hmeblkp;
11600                                 hblkpa = hmeblkp->hblk_nextpa;
11601                                 hmeblkp = hmeblkp->hblk_next;
11602                         }
11603 
11604                         SFMMU_HASH_UNLOCK(hmebp);
11605                         if (hmebp++ == &khme_hash[KHMEHASH_SZ])
11606                                 hmebp = khme_hash;
11607                 }
11608 
11609                 if (hmeblkp != NULL)
11610                         break;
11611                 sfmmu_hblk_steal_twice++;
11612         }
11613         return (hmeblkp);
11614 }
11615 
11616 /*
11617  * This routine does real work to prepare a hblk to be "stolen" by
11618  * unloading the mappings, updating shadow counts ....
11619  * It returns 1 if the block is ready to be reused (stolen), or 0
11620  * means the block cannot be stolen yet- pageunload is still working
11621  * on this hblk.
11622  */
11623 static int
11624 sfmmu_steal_this_hblk(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
11625         uint64_t hblkpa, struct hme_blk *pr_hblk)
11626 {
11627         int shw_size, vshift;
11628         struct hme_blk *shw_hblkp;
11629         caddr_t vaddr;
11630         uint_t shw_mask, newshw_mask;
11631         struct hme_blk *list = NULL;
11632 
11633         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
11634 
11635         /*
11636          * check if the hmeblk is free, unload if necessary
11637          */
11638         if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) {
11639                 sfmmu_t *sfmmup;
11640                 demap_range_t dmr;
11641 
11642                 sfmmup = hblktosfmmu(hmeblkp);
11643                 if (hmeblkp->hblk_shared || sfmmup->sfmmu_ismhat) {
11644                         return (0);
11645                 }
11646                 DEMAP_RANGE_INIT(sfmmup, &dmr);
11647                 (void) sfmmu_hblk_unload(sfmmup, hmeblkp,
11648                     (caddr_t)get_hblk_base(hmeblkp),
11649                     get_hblk_endaddr(hmeblkp), &dmr, HAT_UNLOAD);
11650                 DEMAP_RANGE_FLUSH(&dmr);
11651                 if (hmeblkp->hblk_vcnt || hmeblkp->hblk_hmecnt) {
11652                         /*
11653                          * Pageunload is working on the same hblk.
11654                          */
11655                         return (0);
11656                 }
11657 
11658                 sfmmu_hblk_steal_unload_count++;
11659         }
11660 
11661         ASSERT(hmeblkp->hblk_lckcnt == 0);
11662         ASSERT(hmeblkp->hblk_vcnt == 0 && hmeblkp->hblk_hmecnt == 0);
11663 
11664         sfmmu_hblk_hash_rm(hmebp, hmeblkp, pr_hblk, &list, 1);
11665         hmeblkp->hblk_nextpa = hblkpa;
11666 
11667         shw_hblkp = hmeblkp->hblk_shadow;
11668         if (shw_hblkp) {
11669                 ASSERT(!hmeblkp->hblk_shared);
11670                 shw_size = get_hblk_ttesz(shw_hblkp);
11671                 vaddr = (caddr_t)get_hblk_base(hmeblkp);
11672                 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size);
11673                 ASSERT(vshift < 8);
11674                 /*
11675                  * Atomically clear shadow mask bit
11676                  */
11677                 do {
11678                         shw_mask = shw_hblkp->hblk_shw_mask;
11679                         ASSERT(shw_mask & (1 << vshift));
11680                         newshw_mask = shw_mask & ~(1 << vshift);
11681                         newshw_mask = cas32(&shw_hblkp->hblk_shw_mask,
11682                             shw_mask, newshw_mask);
11683                 } while (newshw_mask != shw_mask);
11684                 hmeblkp->hblk_shadow = NULL;
11685         }
11686 
11687         /*
11688          * remove shadow bit if we are stealing an unused shadow hmeblk.
11689          * sfmmu_hblk_alloc needs it that way, will set shadow bit later if
11690          * we are indeed allocating a shadow hmeblk.
11691          */
11692         hmeblkp->hblk_shw_bit = 0;
11693 
11694         if (hmeblkp->hblk_shared) {
11695                 sf_srd_t        *srdp;
11696                 sf_region_t     *rgnp;
11697                 uint_t          rid;
11698 
11699                 srdp = hblktosrd(hmeblkp);
11700                 ASSERT(srdp != NULL && srdp->srd_refcnt != 0);
11701                 rid = hmeblkp->hblk_tag.htag_rid;
11702                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
11703                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
11704                 rgnp = srdp->srd_hmergnp[rid];
11705                 ASSERT(rgnp != NULL);
11706                 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
11707                 hmeblkp->hblk_shared = 0;
11708         }
11709 
11710         sfmmu_hblk_steal_count++;
11711         SFMMU_STAT(sf_steal_count);
11712 
11713         return (1);
11714 }
11715 
11716 struct hme_blk *
11717 sfmmu_hmetohblk(struct sf_hment *sfhme)
11718 {
11719         struct hme_blk *hmeblkp;
11720         struct sf_hment *sfhme0;
11721         struct hme_blk *hblk_dummy = 0;
11722 
11723         /*
11724          * No dummy sf_hments, please.
11725          */
11726         ASSERT(sfhme->hme_tte.ll != 0);
11727 
11728         sfhme0 = sfhme - sfhme->hme_tte.tte_hmenum;
11729         hmeblkp = (struct hme_blk *)((uintptr_t)sfhme0 -
11730             (uintptr_t)&hblk_dummy->hblk_hme[0]);
11731 
11732         return (hmeblkp);
11733 }
11734 
11735 /*
11736  * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag.
11737  * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using
11738  * KM_SLEEP allocation.
11739  *
11740  * Return 0 on success, -1 otherwise.
11741  */
11742 static void
11743 sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp)
11744 {
11745         struct tsb_info *tsbinfop, *next;
11746         tsb_replace_rc_t rc;
11747         boolean_t gotfirst = B_FALSE;
11748 
11749         ASSERT(sfmmup != ksfmmup);
11750         ASSERT(sfmmu_hat_lock_held(sfmmup));
11751 
11752         while (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPIN)) {
11753                 cv_wait(&sfmmup->sfmmu_tsb_cv, HATLOCK_MUTEXP(hatlockp));
11754         }
11755 
11756         if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
11757                 SFMMU_FLAGS_SET(sfmmup, HAT_SWAPIN);
11758         } else {
11759                 return;
11760         }
11761 
11762         ASSERT(sfmmup->sfmmu_tsb != NULL);
11763 
11764         /*
11765          * Loop over all tsbinfo's replacing them with ones that actually have
11766          * a TSB.  If any of the replacements ever fail, bail out of the loop.
11767          */
11768         for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; tsbinfop = next) {
11769                 ASSERT(tsbinfop->tsb_flags & TSB_SWAPPED);
11770                 next = tsbinfop->tsb_next;
11771                 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, tsbinfop->tsb_szc,
11772                     hatlockp, TSB_SWAPIN);
11773                 if (rc != TSB_SUCCESS) {
11774                         break;
11775                 }
11776                 gotfirst = B_TRUE;
11777         }
11778 
11779         switch (rc) {
11780         case TSB_SUCCESS:
11781                 SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN);
11782                 cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11783                 return;
11784         case TSB_LOSTRACE:
11785                 break;
11786         case TSB_ALLOCFAIL:
11787                 break;
11788         default:
11789                 panic("sfmmu_replace_tsb returned unrecognized failure code "
11790                     "%d", rc);
11791         }
11792 
11793         /*
11794          * In this case, we failed to get one of our TSBs.  If we failed to
11795          * get the first TSB, get one of minimum size (8KB).  Walk the list
11796          * and throw away the tsbinfos, starting where the allocation failed;
11797          * we can get by with just one TSB as long as we don't leave the
11798          * SWAPPED tsbinfo structures lying around.
11799          */
11800         tsbinfop = sfmmup->sfmmu_tsb;
11801         next = tsbinfop->tsb_next;
11802         tsbinfop->tsb_next = NULL;
11803 
11804         sfmmu_hat_exit(hatlockp);
11805         for (tsbinfop = next; tsbinfop != NULL; tsbinfop = next) {
11806                 next = tsbinfop->tsb_next;
11807                 sfmmu_tsbinfo_free(tsbinfop);
11808         }
11809         hatlockp = sfmmu_hat_enter(sfmmup);
11810 
11811         /*
11812          * If we don't have any TSBs, get a single 8K TSB for 8K, 64K and 512K
11813          * pages.
11814          */
11815         if (!gotfirst) {
11816                 tsbinfop = sfmmup->sfmmu_tsb;
11817                 rc = sfmmu_replace_tsb(sfmmup, tsbinfop, TSB_MIN_SZCODE,
11818                     hatlockp, TSB_SWAPIN | TSB_FORCEALLOC);
11819                 ASSERT(rc == TSB_SUCCESS);
11820         }
11821 
11822         SFMMU_FLAGS_CLEAR(sfmmup, HAT_SWAPPED|HAT_SWAPIN);
11823         cv_broadcast(&sfmmup->sfmmu_tsb_cv);
11824 }
11825 
11826 static int
11827 sfmmu_is_rgnva(sf_srd_t *srdp, caddr_t addr, ulong_t w, ulong_t bmw)
11828 {
11829         ulong_t bix = 0;
11830         uint_t rid;
11831         sf_region_t *rgnp;
11832 
11833         ASSERT(srdp != NULL);
11834         ASSERT(srdp->srd_refcnt != 0);
11835 
11836         w <<= BT_ULSHIFT;
11837         while (bmw) {
11838                 if (!(bmw & 0x1)) {
11839                         bix++;
11840                         bmw >>= 1;
11841                         continue;
11842                 }
11843                 rid = w | bix;
11844                 rgnp = srdp->srd_hmergnp[rid];
11845                 ASSERT(rgnp->rgn_refcnt > 0);
11846                 ASSERT(rgnp->rgn_id == rid);
11847                 if (addr < rgnp->rgn_saddr ||
11848                     addr >= (rgnp->rgn_saddr + rgnp->rgn_size)) {
11849                         bix++;
11850                         bmw >>= 1;
11851                 } else {
11852                         return (1);
11853                 }
11854         }
11855         return (0);
11856 }
11857 
11858 /*
11859  * Handle exceptions for low level tsb_handler.
11860  *
11861  * There are many scenarios that could land us here:
11862  *
11863  * If the context is invalid we land here. The context can be invalid
11864  * for 3 reasons: 1) we couldn't allocate a new context and now need to
11865  * perform a wrap around operation in order to allocate a new context.
11866  * 2) Context was invalidated to change pagesize programming 3) ISMs or
11867  * TSBs configuration is changeing for this process and we are forced into
11868  * here to do a syncronization operation. If the context is valid we can
11869  * be here from window trap hanlder. In this case just call trap to handle
11870  * the fault.
11871  *
11872  * Note that the process will run in INVALID_CONTEXT before
11873  * faulting into here and subsequently loading the MMU registers
11874  * (including the TSB base register) associated with this process.
11875  * For this reason, the trap handlers must all test for
11876  * INVALID_CONTEXT before attempting to access any registers other
11877  * than the context registers.
11878  */
11879 void
11880 sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype)
11881 {
11882         sfmmu_t *sfmmup, *shsfmmup;
11883         uint_t ctxtype;
11884         klwp_id_t lwp;
11885         char lwp_save_state;
11886         hatlock_t *hatlockp, *shatlockp;
11887         struct tsb_info *tsbinfop;
11888         struct tsbmiss *tsbmp;
11889         sf_scd_t *scdp;
11890 
11891         SFMMU_STAT(sf_tsb_exceptions);
11892         SFMMU_MMU_STAT(mmu_tsb_exceptions);
11893         sfmmup = astosfmmu(curthread->t_procp->p_as);
11894         /*
11895          * note that in sun4u, tagacces register contains ctxnum
11896          * while sun4v passes ctxtype in the tagaccess register.
11897          */
11898         ctxtype = tagaccess & TAGACC_CTX_MASK;
11899 
11900         ASSERT(sfmmup != ksfmmup && ctxtype != KCONTEXT);
11901         ASSERT(sfmmup->sfmmu_ismhat == 0);
11902         ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) ||
11903             ctxtype == INVALID_CONTEXT);
11904 
11905         if (ctxtype != INVALID_CONTEXT && traptype != T_DATA_PROT) {
11906                 /*
11907                  * We may land here because shme bitmap and pagesize
11908                  * flags are updated lazily in tsbmiss area on other cpus.
11909                  * If we detect here that tsbmiss area is out of sync with
11910                  * sfmmu update it and retry the trapped instruction.
11911                  * Otherwise call trap().
11912                  */
11913                 int ret = 0;
11914                 uchar_t tteflag_mask = (1 << TTE64K) | (1 << TTE8K);
11915                 caddr_t addr = (caddr_t)(tagaccess & TAGACC_VADDR_MASK);
11916 
11917                 /*
11918                  * Must set lwp state to LWP_SYS before
11919                  * trying to acquire any adaptive lock
11920                  */
11921                 lwp = ttolwp(curthread);
11922                 ASSERT(lwp);
11923                 lwp_save_state = lwp->lwp_state;
11924                 lwp->lwp_state = LWP_SYS;
11925 
11926                 hatlockp = sfmmu_hat_enter(sfmmup);
11927                 kpreempt_disable();
11928                 tsbmp = &tsbmiss_area[CPU->cpu_id];
11929                 ASSERT(sfmmup == tsbmp->usfmmup);
11930                 if (((tsbmp->uhat_tteflags ^ sfmmup->sfmmu_tteflags) &
11931                     ~tteflag_mask) ||
11932                     ((tsbmp->uhat_rtteflags ^  sfmmup->sfmmu_rtteflags) &
11933                     ~tteflag_mask)) {
11934                         tsbmp->uhat_tteflags = sfmmup->sfmmu_tteflags;
11935                         tsbmp->uhat_rtteflags = sfmmup->sfmmu_rtteflags;
11936                         ret = 1;
11937                 }
11938                 if (sfmmup->sfmmu_srdp != NULL) {
11939                         ulong_t *sm = sfmmup->sfmmu_hmeregion_map.bitmap;
11940                         ulong_t *tm = tsbmp->shmermap;
11941                         ulong_t i;
11942                         for (i = 0; i < SFMMU_HMERGNMAP_WORDS; i++) {
11943                                 ulong_t d = tm[i] ^ sm[i];
11944                                 if (d) {
11945                                         if (d & sm[i]) {
11946                                                 if (!ret && sfmmu_is_rgnva(
11947                                                     sfmmup->sfmmu_srdp,
11948                                                     addr, i, d & sm[i])) {
11949                                                         ret = 1;
11950                                                 }
11951                                         }
11952                                         tm[i] = sm[i];
11953                                 }
11954                         }
11955                 }
11956                 kpreempt_enable();
11957                 sfmmu_hat_exit(hatlockp);
11958                 lwp->lwp_state = lwp_save_state;
11959                 if (ret) {
11960                         return;
11961                 }
11962         } else if (ctxtype == INVALID_CONTEXT) {
11963                 /*
11964                  * First, make sure we come out of here with a valid ctx,
11965                  * since if we don't get one we'll simply loop on the
11966                  * faulting instruction.
11967                  *
11968                  * If the ISM mappings are changing, the TSB is relocated,
11969                  * the process is swapped, the process is joining SCD or
11970                  * leaving SCD or shared regions we serialize behind the
11971                  * controlling thread with hat lock, sfmmu_flags and
11972                  * sfmmu_tsb_cv condition variable.
11973                  */
11974 
11975                 /*
11976                  * Must set lwp state to LWP_SYS before
11977                  * trying to acquire any adaptive lock
11978                  */
11979                 lwp = ttolwp(curthread);
11980                 ASSERT(lwp);
11981                 lwp_save_state = lwp->lwp_state;
11982                 lwp->lwp_state = LWP_SYS;
11983 
11984                 hatlockp = sfmmu_hat_enter(sfmmup);
11985 retry:
11986                 if ((scdp = sfmmup->sfmmu_scdp) != NULL) {
11987                         shsfmmup = scdp->scd_sfmmup;
11988                         ASSERT(shsfmmup != NULL);
11989 
11990                         for (tsbinfop = shsfmmup->sfmmu_tsb; tsbinfop != NULL;
11991                             tsbinfop = tsbinfop->tsb_next) {
11992                                 if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) {
11993                                         /* drop the private hat lock */
11994                                         sfmmu_hat_exit(hatlockp);
11995                                         /* acquire the shared hat lock */
11996                                         shatlockp = sfmmu_hat_enter(shsfmmup);
11997                                         /*
11998                                          * recheck to see if anything changed
11999                                          * after we drop the private hat lock.
12000                                          */
12001                                         if (sfmmup->sfmmu_scdp == scdp &&
12002                                             shsfmmup == scdp->scd_sfmmup) {
12003                                                 sfmmu_tsb_chk_reloc(shsfmmup,
12004                                                     shatlockp);
12005                                         }
12006                                         sfmmu_hat_exit(shatlockp);
12007                                         hatlockp = sfmmu_hat_enter(sfmmup);
12008                                         goto retry;
12009                                 }
12010                         }
12011                 }
12012 
12013                 for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
12014                     tsbinfop = tsbinfop->tsb_next) {
12015                         if (tsbinfop->tsb_flags & TSB_RELOC_FLAG) {
12016                                 cv_wait(&sfmmup->sfmmu_tsb_cv,
12017                                     HATLOCK_MUTEXP(hatlockp));
12018                                 goto retry;
12019                         }
12020                 }
12021 
12022                 /*
12023                  * Wait for ISM maps to be updated.
12024                  */
12025                 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) {
12026                         cv_wait(&sfmmup->sfmmu_tsb_cv,
12027                             HATLOCK_MUTEXP(hatlockp));
12028                         goto retry;
12029                 }
12030 
12031                 /* Is this process joining an SCD? */
12032                 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
12033                         /*
12034                          * Flush private TSB and setup shared TSB.
12035                          * sfmmu_finish_join_scd() does not drop the
12036                          * hat lock.
12037                          */
12038                         sfmmu_finish_join_scd(sfmmup);
12039                         SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD);
12040                 }
12041 
12042                 /*
12043                  * If we're swapping in, get TSB(s).  Note that we must do
12044                  * this before we get a ctx or load the MMU state.  Once
12045                  * we swap in we have to recheck to make sure the TSB(s) and
12046                  * ISM mappings didn't change while we slept.
12047                  */
12048                 if (SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) {
12049                         sfmmu_tsb_swapin(sfmmup, hatlockp);
12050                         goto retry;
12051                 }
12052 
12053                 sfmmu_get_ctx(sfmmup);
12054 
12055                 sfmmu_hat_exit(hatlockp);
12056                 /*
12057                  * Must restore lwp_state if not calling
12058                  * trap() for further processing. Restore
12059                  * it anyway.
12060                  */
12061                 lwp->lwp_state = lwp_save_state;
12062                 return;
12063         }
12064         trap(rp, (caddr_t)tagaccess, traptype, 0);
12065 }
12066 
12067 static void
12068 sfmmu_tsb_chk_reloc(sfmmu_t *sfmmup, hatlock_t *hatlockp)
12069 {
12070         struct tsb_info *tp;
12071 
12072         ASSERT(sfmmu_hat_lock_held(sfmmup));
12073 
12074         for (tp = sfmmup->sfmmu_tsb; tp != NULL; tp = tp->tsb_next) {
12075                 if (tp->tsb_flags & TSB_RELOC_FLAG) {
12076                         cv_wait(&sfmmup->sfmmu_tsb_cv,
12077                             HATLOCK_MUTEXP(hatlockp));
12078                         break;
12079                 }
12080         }
12081 }
12082 
12083 /*
12084  * sfmmu_vatopfn_suspended is called from GET_TTE when TL=0 and
12085  * TTE_SUSPENDED bit set in tte we block on aquiring a page lock
12086  * rather than spinning to avoid send mondo timeouts with
12087  * interrupts enabled. When the lock is acquired it is immediately
12088  * released and we return back to sfmmu_vatopfn just after
12089  * the GET_TTE call.
12090  */
12091 void
12092 sfmmu_vatopfn_suspended(caddr_t vaddr, sfmmu_t *sfmmu, tte_t *ttep)
12093 {
12094         struct page     **pp;
12095 
12096         (void) as_pagelock(sfmmu->sfmmu_as, &pp, vaddr, TTE_CSZ(ttep), S_WRITE);
12097         as_pageunlock(sfmmu->sfmmu_as, pp, vaddr, TTE_CSZ(ttep), S_WRITE);
12098 }
12099 
12100 /*
12101  * sfmmu_tsbmiss_suspended is called from GET_TTE when TL>0 and
12102  * TTE_SUSPENDED bit set in tte. We do this so that we can handle
12103  * cross traps which cannot be handled while spinning in the
12104  * trap handlers. Simply enter and exit the kpr_suspendlock spin
12105  * mutex, which is held by the holder of the suspend bit, and then
12106  * retry the trapped instruction after unwinding.
12107  */
12108 /*ARGSUSED*/
12109 void
12110 sfmmu_tsbmiss_suspended(struct regs *rp, uintptr_t tagacc, uint_t traptype)
12111 {
12112         ASSERT(curthread != kreloc_thread);
12113         mutex_enter(&kpr_suspendlock);
12114         mutex_exit(&kpr_suspendlock);
12115 }
12116 
12117 /*
12118  * This routine could be optimized to reduce the number of xcalls by flushing
12119  * the entire TLBs if region reference count is above some threshold but the
12120  * tradeoff will depend on the size of the TLB. So for now flush the specific
12121  * page a context at a time.
12122  *
12123  * If uselocks is 0 then it's called after all cpus were captured and all the
12124  * hat locks were taken. In this case don't take the region lock by relying on
12125  * the order of list region update operations in hat_join_region(),
12126  * hat_leave_region() and hat_dup_region(). The ordering in those routines
12127  * guarantees that list is always forward walkable and reaches active sfmmus
12128  * regardless of where xc_attention() captures a cpu.
12129  */
12130 cpuset_t
12131 sfmmu_rgntlb_demap(caddr_t addr, sf_region_t *rgnp,
12132     struct hme_blk *hmeblkp, int uselocks)
12133 {
12134         sfmmu_t *sfmmup;
12135         cpuset_t cpuset;
12136         cpuset_t rcpuset;
12137         hatlock_t *hatlockp;
12138         uint_t rid = rgnp->rgn_id;
12139         sf_rgn_link_t *rlink;
12140         sf_scd_t *scdp;
12141 
12142         ASSERT(hmeblkp->hblk_shared);
12143         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
12144         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
12145 
12146         CPUSET_ZERO(rcpuset);
12147         if (uselocks) {
12148                 mutex_enter(&rgnp->rgn_mutex);
12149         }
12150         sfmmup = rgnp->rgn_sfmmu_head;
12151         while (sfmmup != NULL) {
12152                 if (uselocks) {
12153                         hatlockp = sfmmu_hat_enter(sfmmup);
12154                 }
12155 
12156                 /*
12157                  * When an SCD is created the SCD hat is linked on the sfmmu
12158                  * region lists for each hme region which is part of the
12159                  * SCD. If we find an SCD hat, when walking these lists,
12160                  * then we flush the shared TSBs, if we find a private hat,
12161                  * which is part of an SCD, but where the region
12162                  * is not part of the SCD then we flush the private TSBs.
12163                  */
12164                 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL &&
12165                     !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
12166                         scdp = sfmmup->sfmmu_scdp;
12167                         if (SF_RGNMAP_TEST(scdp->scd_hmeregion_map, rid)) {
12168                                 if (uselocks) {
12169                                         sfmmu_hat_exit(hatlockp);
12170                                 }
12171                                 goto next;
12172                         }
12173                 }
12174 
12175                 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12176 
12177                 kpreempt_disable();
12178                 cpuset = sfmmup->sfmmu_cpusran;
12179                 CPUSET_AND(cpuset, cpu_ready_set);
12180                 CPUSET_DEL(cpuset, CPU->cpu_id);
12181                 SFMMU_XCALL_STATS(sfmmup);
12182                 xt_some(cpuset, vtag_flushpage_tl1,
12183                     (uint64_t)addr, (uint64_t)sfmmup);
12184                 vtag_flushpage(addr, (uint64_t)sfmmup);
12185                 if (uselocks) {
12186                         sfmmu_hat_exit(hatlockp);
12187                 }
12188                 kpreempt_enable();
12189                 CPUSET_OR(rcpuset, cpuset);
12190 
12191 next:
12192                 /* LINTED: constant in conditional context */
12193                 SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0);
12194                 ASSERT(rlink != NULL);
12195                 sfmmup = rlink->next;
12196         }
12197         if (uselocks) {
12198                 mutex_exit(&rgnp->rgn_mutex);
12199         }
12200         return (rcpuset);
12201 }
12202 
12203 /*
12204  * This routine takes an sfmmu pointer and the va for an adddress in an
12205  * ISM region as input and returns the corresponding region id in ism_rid.
12206  * The return value of 1 indicates that a region has been found and ism_rid
12207  * is valid, otherwise 0 is returned.
12208  */
12209 static int
12210 find_ism_rid(sfmmu_t *sfmmup, sfmmu_t *ism_sfmmup, caddr_t va, uint_t *ism_rid)
12211 {
12212         ism_blk_t       *ism_blkp;
12213         int             i;
12214         ism_map_t       *ism_map;
12215 #ifdef DEBUG
12216         struct hat      *ism_hatid;
12217 #endif
12218         ASSERT(sfmmu_hat_lock_held(sfmmup));
12219 
12220         ism_blkp = sfmmup->sfmmu_iblk;
12221         while (ism_blkp != NULL) {
12222                 ism_map = ism_blkp->iblk_maps;
12223                 for (i = 0; i < ISM_MAP_SLOTS && ism_map[i].imap_ismhat; i++) {
12224                         if ((va >= ism_start(ism_map[i])) &&
12225                             (va < ism_end(ism_map[i]))) {
12226 
12227                                 *ism_rid = ism_map[i].imap_rid;
12228 #ifdef DEBUG
12229                                 ism_hatid = ism_map[i].imap_ismhat;
12230                                 ASSERT(ism_hatid == ism_sfmmup);
12231                                 ASSERT(ism_hatid->sfmmu_ismhat);
12232 #endif
12233                                 return (1);
12234                         }
12235                 }
12236                 ism_blkp = ism_blkp->iblk_next;
12237         }
12238         return (0);
12239 }
12240 
12241 /*
12242  * Special routine to flush out ism mappings- TSBs, TLBs and D-caches.
12243  * This routine may be called with all cpu's captured. Therefore, the
12244  * caller is responsible for holding all locks and disabling kernel
12245  * preemption.
12246  */
12247 /* ARGSUSED */
12248 static void
12249 sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup,
12250         struct hme_blk *hmeblkp, pfn_t pfnum, int cache_flush_flag)
12251 {
12252         cpuset_t        cpuset;
12253         caddr_t         va;
12254         ism_ment_t      *ment;
12255         sfmmu_t         *sfmmup;
12256 #ifdef VAC
12257         int             vcolor;
12258 #endif
12259 
12260         sf_scd_t        *scdp;
12261         uint_t          ism_rid;
12262 
12263         ASSERT(!hmeblkp->hblk_shared);
12264         /*
12265          * Walk the ism_hat's mapping list and flush the page
12266          * from every hat sharing this ism_hat. This routine
12267          * may be called while all cpu's have been captured.
12268          * Therefore we can't attempt to grab any locks. For now
12269          * this means we will protect the ism mapping list under
12270          * a single lock which will be grabbed by the caller.
12271          * If hat_share/unshare scalibility becomes a performance
12272          * problem then we may need to re-think ism mapping list locking.
12273          */
12274         ASSERT(ism_sfmmup->sfmmu_ismhat);
12275         ASSERT(MUTEX_HELD(&ism_mlist_lock));
12276         addr = addr - ISMID_STARTADDR;
12277 
12278         for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) {
12279 
12280                 sfmmup = ment->iment_hat;
12281 
12282                 va = ment->iment_base_va;
12283                 va = (caddr_t)((uintptr_t)va  + (uintptr_t)addr);
12284 
12285                 /*
12286                  * When an SCD is created the SCD hat is linked on the ism
12287                  * mapping lists for each ISM segment which is part of the
12288                  * SCD. If we find an SCD hat, when walking these lists,
12289                  * then we flush the shared TSBs, if we find a private hat,
12290                  * which is part of an SCD, but where the region
12291                  * corresponding to this va is not part of the SCD then we
12292                  * flush the private TSBs.
12293                  */
12294                 if (!sfmmup->sfmmu_scdhat && sfmmup->sfmmu_scdp != NULL &&
12295                     !SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD) &&
12296                     !SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) {
12297                         if (!find_ism_rid(sfmmup, ism_sfmmup, va,
12298                             &ism_rid)) {
12299                                 cmn_err(CE_PANIC,
12300                                     "can't find matching ISM rid!");
12301                         }
12302 
12303                         scdp = sfmmup->sfmmu_scdp;
12304                         if (SFMMU_IS_ISMRID_VALID(ism_rid) &&
12305                             SF_RGNMAP_TEST(scdp->scd_ismregion_map,
12306                             ism_rid)) {
12307                                 continue;
12308                         }
12309                 }
12310                 SFMMU_UNLOAD_TSB(va, sfmmup, hmeblkp, 1);
12311 
12312                 cpuset = sfmmup->sfmmu_cpusran;
12313                 CPUSET_AND(cpuset, cpu_ready_set);
12314                 CPUSET_DEL(cpuset, CPU->cpu_id);
12315                 SFMMU_XCALL_STATS(sfmmup);
12316                 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va,
12317                     (uint64_t)sfmmup);
12318                 vtag_flushpage(va, (uint64_t)sfmmup);
12319 
12320 #ifdef VAC
12321                 /*
12322                  * Flush D$
12323                  * When flushing D$ we must flush all
12324                  * cpu's. See sfmmu_cache_flush().
12325                  */
12326                 if (cache_flush_flag == CACHE_FLUSH) {
12327                         cpuset = cpu_ready_set;
12328                         CPUSET_DEL(cpuset, CPU->cpu_id);
12329 
12330                         SFMMU_XCALL_STATS(sfmmup);
12331                         vcolor = addr_to_vcolor(va);
12332                         xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor);
12333                         vac_flushpage(pfnum, vcolor);
12334                 }
12335 #endif  /* VAC */
12336         }
12337 }
12338 
12339 /*
12340  * Demaps the TSB, CPU caches, and flushes all TLBs on all CPUs of
12341  * a particular virtual address and ctx.  If noflush is set we do not
12342  * flush the TLB/TSB.  This function may or may not be called with the
12343  * HAT lock held.
12344  */
12345 static void
12346 sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
12347         pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag,
12348         int hat_lock_held)
12349 {
12350 #ifdef VAC
12351         int vcolor;
12352 #endif
12353         cpuset_t cpuset;
12354         hatlock_t *hatlockp;
12355 
12356         ASSERT(!hmeblkp->hblk_shared);
12357 
12358 #if defined(lint) && !defined(VAC)
12359         pfnum = pfnum;
12360         cpu_flag = cpu_flag;
12361         cache_flush_flag = cache_flush_flag;
12362 #endif
12363 
12364         /*
12365          * There is no longer a need to protect against ctx being
12366          * stolen here since we don't store the ctx in the TSB anymore.
12367          */
12368 #ifdef VAC
12369         vcolor = addr_to_vcolor(addr);
12370 #endif
12371 
12372         /*
12373          * We must hold the hat lock during the flush of TLB,
12374          * to avoid a race with sfmmu_invalidate_ctx(), where
12375          * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT,
12376          * causing TLB demap routine to skip flush on that MMU.
12377          * If the context on a MMU has already been set to
12378          * INVALID_CONTEXT, we just get an extra flush on
12379          * that MMU.
12380          */
12381         if (!hat_lock_held && !tlb_noflush)
12382                 hatlockp = sfmmu_hat_enter(sfmmup);
12383 
12384         kpreempt_disable();
12385         if (!tlb_noflush) {
12386                 /*
12387                  * Flush the TSB and TLB.
12388                  */
12389                 SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12390 
12391                 cpuset = sfmmup->sfmmu_cpusran;
12392                 CPUSET_AND(cpuset, cpu_ready_set);
12393                 CPUSET_DEL(cpuset, CPU->cpu_id);
12394 
12395                 SFMMU_XCALL_STATS(sfmmup);
12396 
12397                 xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr,
12398                     (uint64_t)sfmmup);
12399 
12400                 vtag_flushpage(addr, (uint64_t)sfmmup);
12401         }
12402 
12403         if (!hat_lock_held && !tlb_noflush)
12404                 sfmmu_hat_exit(hatlockp);
12405 
12406 #ifdef VAC
12407         /*
12408          * Flush the D$
12409          *
12410          * Even if the ctx is stolen, we need to flush the
12411          * cache. Our ctx stealer only flushes the TLBs.
12412          */
12413         if (cache_flush_flag == CACHE_FLUSH) {
12414                 if (cpu_flag & FLUSH_ALL_CPUS) {
12415                         cpuset = cpu_ready_set;
12416                 } else {
12417                         cpuset = sfmmup->sfmmu_cpusran;
12418                         CPUSET_AND(cpuset, cpu_ready_set);
12419                 }
12420                 CPUSET_DEL(cpuset, CPU->cpu_id);
12421                 SFMMU_XCALL_STATS(sfmmup);
12422                 xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor);
12423                 vac_flushpage(pfnum, vcolor);
12424         }
12425 #endif  /* VAC */
12426         kpreempt_enable();
12427 }
12428 
12429 /*
12430  * Demaps the TSB and flushes all TLBs on all cpus for a particular virtual
12431  * address and ctx.  If noflush is set we do not currently do anything.
12432  * This function may or may not be called with the HAT lock held.
12433  */
12434 static void
12435 sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp,
12436         int tlb_noflush, int hat_lock_held)
12437 {
12438         cpuset_t cpuset;
12439         hatlock_t *hatlockp;
12440 
12441         ASSERT(!hmeblkp->hblk_shared);
12442 
12443         /*
12444          * If the process is exiting we have nothing to do.
12445          */
12446         if (tlb_noflush)
12447                 return;
12448 
12449         /*
12450          * Flush TSB.
12451          */
12452         if (!hat_lock_held)
12453                 hatlockp = sfmmu_hat_enter(sfmmup);
12454         SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp, 0);
12455 
12456         kpreempt_disable();
12457 
12458         cpuset = sfmmup->sfmmu_cpusran;
12459         CPUSET_AND(cpuset, cpu_ready_set);
12460         CPUSET_DEL(cpuset, CPU->cpu_id);
12461 
12462         SFMMU_XCALL_STATS(sfmmup);
12463         xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup);
12464 
12465         vtag_flushpage(addr, (uint64_t)sfmmup);
12466 
12467         if (!hat_lock_held)
12468                 sfmmu_hat_exit(hatlockp);
12469 
12470         kpreempt_enable();
12471 
12472 }
12473 
12474 /*
12475  * Special case of sfmmu_tlb_demap for MMU_PAGESIZE hblks. Use the xcall
12476  * call handler that can flush a range of pages to save on xcalls.
12477  */
12478 static int sfmmu_xcall_save;
12479 
12480 /*
12481  * this routine is never used for demaping addresses backed by SRD hmeblks.
12482  */
12483 static void
12484 sfmmu_tlb_range_demap(demap_range_t *dmrp)
12485 {
12486         sfmmu_t *sfmmup = dmrp->dmr_sfmmup;
12487         hatlock_t *hatlockp;
12488         cpuset_t cpuset;
12489         uint64_t sfmmu_pgcnt;
12490         pgcnt_t pgcnt = 0;
12491         int pgunload = 0;
12492         int dirtypg = 0;
12493         caddr_t addr = dmrp->dmr_addr;
12494         caddr_t eaddr;
12495         uint64_t bitvec = dmrp->dmr_bitvec;
12496 
12497         ASSERT(bitvec & 1);
12498 
12499         /*
12500          * Flush TSB and calculate number of pages to flush.
12501          */
12502         while (bitvec != 0) {
12503                 dirtypg = 0;
12504                 /*
12505                  * Find the first page to flush and then count how many
12506                  * pages there are after it that also need to be flushed.
12507                  * This way the number of TSB flushes is minimized.
12508                  */
12509                 while ((bitvec & 1) == 0) {
12510                         pgcnt++;
12511                         addr += MMU_PAGESIZE;
12512                         bitvec >>= 1;
12513                 }
12514                 while (bitvec & 1) {
12515                         dirtypg++;
12516                         bitvec >>= 1;
12517                 }
12518                 eaddr = addr + ptob(dirtypg);
12519                 hatlockp = sfmmu_hat_enter(sfmmup);
12520                 sfmmu_unload_tsb_range(sfmmup, addr, eaddr, TTE8K);
12521                 sfmmu_hat_exit(hatlockp);
12522                 pgunload += dirtypg;
12523                 addr = eaddr;
12524                 pgcnt += dirtypg;
12525         }
12526 
12527         ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr);
12528         if (sfmmup->sfmmu_free == 0) {
12529                 addr = dmrp->dmr_addr;
12530                 bitvec = dmrp->dmr_bitvec;
12531 
12532                 /*
12533                  * make sure it has SFMMU_PGCNT_SHIFT bits only,
12534                  * as it will be used to pack argument for xt_some
12535                  */
12536                 ASSERT((pgcnt > 0) &&
12537                     (pgcnt <= (1 << SFMMU_PGCNT_SHIFT)));
12538 
12539                 /*
12540                  * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in
12541                  * the low 6 bits of sfmmup. This is doable since pgcnt
12542                  * always >= 1.
12543                  */
12544                 ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK));
12545                 sfmmu_pgcnt = (uint64_t)sfmmup |
12546                     ((pgcnt - 1) & SFMMU_PGCNT_MASK);
12547 
12548                 /*
12549                  * We must hold the hat lock during the flush of TLB,
12550                  * to avoid a race with sfmmu_invalidate_ctx(), where
12551                  * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT,
12552                  * causing TLB demap routine to skip flush on that MMU.
12553                  * If the context on a MMU has already been set to
12554                  * INVALID_CONTEXT, we just get an extra flush on
12555                  * that MMU.
12556                  */
12557                 hatlockp = sfmmu_hat_enter(sfmmup);
12558                 kpreempt_disable();
12559 
12560                 cpuset = sfmmup->sfmmu_cpusran;
12561                 CPUSET_AND(cpuset, cpu_ready_set);
12562                 CPUSET_DEL(cpuset, CPU->cpu_id);
12563 
12564                 SFMMU_XCALL_STATS(sfmmup);
12565                 xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr,
12566                     sfmmu_pgcnt);
12567 
12568                 for (; bitvec != 0; bitvec >>= 1) {
12569                         if (bitvec & 1)
12570                                 vtag_flushpage(addr, (uint64_t)sfmmup);
12571                         addr += MMU_PAGESIZE;
12572                 }
12573                 kpreempt_enable();
12574                 sfmmu_hat_exit(hatlockp);
12575 
12576                 sfmmu_xcall_save += (pgunload-1);
12577         }
12578         dmrp->dmr_bitvec = 0;
12579 }
12580 
12581 /*
12582  * In cases where we need to synchronize with TLB/TSB miss trap
12583  * handlers, _and_ need to flush the TLB, it's a lot easier to
12584  * throw away the context from the process than to do a
12585  * special song and dance to keep things consistent for the
12586  * handlers.
12587  *
12588  * Since the process suddenly ends up without a context and our caller
12589  * holds the hat lock, threads that fault after this function is called
12590  * will pile up on the lock.  We can then do whatever we need to
12591  * atomically from the context of the caller.  The first blocked thread
12592  * to resume executing will get the process a new context, and the
12593  * process will resume executing.
12594  *
12595  * One added advantage of this approach is that on MMUs that
12596  * support a "flush all" operation, we will delay the flush until
12597  * cnum wrap-around, and then flush the TLB one time.  This
12598  * is rather rare, so it's a lot less expensive than making 8000
12599  * x-calls to flush the TLB 8000 times.
12600  *
12601  * A per-process (PP) lock is used to synchronize ctx allocations in
12602  * resume() and ctx invalidations here.
12603  */
12604 static void
12605 sfmmu_invalidate_ctx(sfmmu_t *sfmmup)
12606 {
12607         cpuset_t cpuset;
12608         int cnum, currcnum;
12609         mmu_ctx_t *mmu_ctxp;
12610         int i;
12611         uint_t pstate_save;
12612 
12613         SFMMU_STAT(sf_ctx_inv);
12614 
12615         ASSERT(sfmmu_hat_lock_held(sfmmup));
12616         ASSERT(sfmmup != ksfmmup);
12617 
12618         kpreempt_disable();
12619 
12620         mmu_ctxp = CPU_MMU_CTXP(CPU);
12621         ASSERT(mmu_ctxp);
12622         ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms);
12623         ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]);
12624 
12625         currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum;
12626 
12627         pstate_save = sfmmu_disable_intrs();
12628 
12629         lock_set(&sfmmup->sfmmu_ctx_lock);       /* acquire PP lock */
12630         /* set HAT cnum invalid across all context domains. */
12631         for (i = 0; i < max_mmu_ctxdoms; i++) {
12632 
12633                 cnum =  sfmmup->sfmmu_ctxs[i].cnum;
12634                 if (cnum == INVALID_CONTEXT) {
12635                         continue;
12636                 }
12637 
12638                 sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT;
12639         }
12640         membar_enter(); /* make sure globally visible to all CPUs */
12641         lock_clear(&sfmmup->sfmmu_ctx_lock);     /* release PP lock */
12642 
12643         sfmmu_enable_intrs(pstate_save);
12644 
12645         cpuset = sfmmup->sfmmu_cpusran;
12646         CPUSET_DEL(cpuset, CPU->cpu_id);
12647         CPUSET_AND(cpuset, cpu_ready_set);
12648         if (!CPUSET_ISNULL(cpuset)) {
12649                 SFMMU_XCALL_STATS(sfmmup);
12650                 xt_some(cpuset, sfmmu_raise_tsb_exception,
12651                     (uint64_t)sfmmup, INVALID_CONTEXT);
12652                 xt_sync(cpuset);
12653                 SFMMU_STAT(sf_tsb_raise_exception);
12654                 SFMMU_MMU_STAT(mmu_tsb_raise_exception);
12655         }
12656 
12657         /*
12658          * If the hat to-be-invalidated is the same as the current
12659          * process on local CPU we need to invalidate
12660          * this CPU context as well.
12661          */
12662         if ((sfmmu_getctx_sec() == currcnum) &&
12663             (currcnum != INVALID_CONTEXT)) {
12664                 /* sets shared context to INVALID too */
12665                 sfmmu_setctx_sec(INVALID_CONTEXT);
12666                 sfmmu_clear_utsbinfo();
12667         }
12668 
12669         SFMMU_FLAGS_SET(sfmmup, HAT_ALLCTX_INVALID);
12670 
12671         kpreempt_enable();
12672 
12673         /*
12674          * we hold the hat lock, so nobody should allocate a context
12675          * for us yet
12676          */
12677         ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT);
12678 }
12679 
12680 #ifdef VAC
12681 /*
12682  * We need to flush the cache in all cpus.  It is possible that
12683  * a process referenced a page as cacheable but has sinced exited
12684  * and cleared the mapping list.  We still to flush it but have no
12685  * state so all cpus is the only alternative.
12686  */
12687 void
12688 sfmmu_cache_flush(pfn_t pfnum, int vcolor)
12689 {
12690         cpuset_t cpuset;
12691 
12692         kpreempt_disable();
12693         cpuset = cpu_ready_set;
12694         CPUSET_DEL(cpuset, CPU->cpu_id);
12695         SFMMU_XCALL_STATS(NULL);        /* account to any ctx */
12696         xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor);
12697         xt_sync(cpuset);
12698         vac_flushpage(pfnum, vcolor);
12699         kpreempt_enable();
12700 }
12701 
12702 void
12703 sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum)
12704 {
12705         cpuset_t cpuset;
12706 
12707         ASSERT(vcolor >= 0);
12708 
12709         kpreempt_disable();
12710         cpuset = cpu_ready_set;
12711         CPUSET_DEL(cpuset, CPU->cpu_id);
12712         SFMMU_XCALL_STATS(NULL);        /* account to any ctx */
12713         xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum);
12714         xt_sync(cpuset);
12715         vac_flushcolor(vcolor, pfnum);
12716         kpreempt_enable();
12717 }
12718 #endif  /* VAC */
12719 
12720 /*
12721  * We need to prevent processes from accessing the TSB using a cached physical
12722  * address.  It's alright if they try to access the TSB via virtual address
12723  * since they will just fault on that virtual address once the mapping has
12724  * been suspended.
12725  */
12726 #pragma weak sendmondo_in_recover
12727 
12728 /* ARGSUSED */
12729 static int
12730 sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo)
12731 {
12732         struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo;
12733         sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu;
12734         hatlock_t *hatlockp;
12735         sf_scd_t *scdp;
12736 
12737         if (flags != HAT_PRESUSPEND)
12738                 return (0);
12739 
12740         /*
12741          * If tsb is a shared TSB with TSB_SHAREDCTX set, sfmmup must
12742          * be a shared hat, then set SCD's tsbinfo's flag.
12743          * If tsb is not shared, sfmmup is a private hat, then set
12744          * its private tsbinfo's flag.
12745          */
12746         hatlockp = sfmmu_hat_enter(sfmmup);
12747         tsbinfop->tsb_flags |= TSB_RELOC_FLAG;
12748 
12749         if (!(tsbinfop->tsb_flags & TSB_SHAREDCTX)) {
12750                 sfmmu_tsb_inv_ctx(sfmmup);
12751                 sfmmu_hat_exit(hatlockp);
12752         } else {
12753                 /* release lock on the shared hat */
12754                 sfmmu_hat_exit(hatlockp);
12755                 /* sfmmup is a shared hat */
12756                 ASSERT(sfmmup->sfmmu_scdhat);
12757                 scdp = sfmmup->sfmmu_scdp;
12758                 ASSERT(scdp != NULL);
12759                 /* get private hat from the scd list */
12760                 mutex_enter(&scdp->scd_mutex);
12761                 sfmmup = scdp->scd_sf_list;
12762                 while (sfmmup != NULL) {
12763                         hatlockp = sfmmu_hat_enter(sfmmup);
12764                         /*
12765                          * We do not call sfmmu_tsb_inv_ctx here because
12766                          * sendmondo_in_recover check is only needed for
12767                          * sun4u.
12768                          */
12769                         sfmmu_invalidate_ctx(sfmmup);
12770                         sfmmu_hat_exit(hatlockp);
12771                         sfmmup = sfmmup->sfmmu_scd_link.next;
12772 
12773                 }
12774                 mutex_exit(&scdp->scd_mutex);
12775         }
12776         return (0);
12777 }
12778 
12779 static void
12780 sfmmu_tsb_inv_ctx(sfmmu_t *sfmmup)
12781 {
12782         extern uint32_t sendmondo_in_recover;
12783 
12784         ASSERT(sfmmu_hat_lock_held(sfmmup));
12785 
12786         /*
12787          * For Cheetah+ Erratum 25:
12788          * Wait for any active recovery to finish.  We can't risk
12789          * relocating the TSB of the thread running mondo_recover_proc()
12790          * since, if we did that, we would deadlock.  The scenario we are
12791          * trying to avoid is as follows:
12792          *
12793          * THIS CPU                     RECOVER CPU
12794          * --------                     -----------
12795          *                              Begins recovery, walking through TSB
12796          * hat_pagesuspend() TSB TTE
12797          *                              TLB miss on TSB TTE, spins at TL1
12798          * xt_sync()
12799          *      send_mondo_timeout()
12800          *      mondo_recover_proc()
12801          *      ((deadlocked))
12802          *
12803          * The second half of the workaround is that mondo_recover_proc()
12804          * checks to see if the tsb_info has the RELOC flag set, and if it
12805          * does, it skips over that TSB without ever touching tsbinfop->tsb_va
12806          * and hence avoiding the TLB miss that could result in a deadlock.
12807          */
12808         if (&sendmondo_in_recover) {
12809                 membar_enter(); /* make sure RELOC flag visible */
12810                 while (sendmondo_in_recover) {
12811                         drv_usecwait(1);
12812                         membar_consumer();
12813                 }
12814         }
12815 
12816         sfmmu_invalidate_ctx(sfmmup);
12817 }
12818 
12819 /* ARGSUSED */
12820 static int
12821 sfmmu_tsb_post_relocator(caddr_t va, uint_t tsbsz, uint_t flags,
12822         void *tsbinfo, pfn_t newpfn)
12823 {
12824         hatlock_t *hatlockp;
12825         struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo;
12826         sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu;
12827 
12828         if (flags != HAT_POSTUNSUSPEND)
12829                 return (0);
12830 
12831         hatlockp = sfmmu_hat_enter(sfmmup);
12832 
12833         SFMMU_STAT(sf_tsb_reloc);
12834 
12835         /*
12836          * The process may have swapped out while we were relocating one
12837          * of its TSBs.  If so, don't bother doing the setup since the
12838          * process can't be using the memory anymore.
12839          */
12840         if ((tsbinfop->tsb_flags & TSB_SWAPPED) == 0) {
12841                 ASSERT(va == tsbinfop->tsb_va);
12842                 sfmmu_tsbinfo_setup_phys(tsbinfop, newpfn);
12843 
12844                 if (tsbinfop->tsb_flags & TSB_FLUSH_NEEDED) {
12845                         sfmmu_inv_tsb(tsbinfop->tsb_va,
12846                             TSB_BYTES(tsbinfop->tsb_szc));
12847                         tsbinfop->tsb_flags &= ~TSB_FLUSH_NEEDED;
12848                 }
12849         }
12850 
12851         membar_exit();
12852         tsbinfop->tsb_flags &= ~TSB_RELOC_FLAG;
12853         cv_broadcast(&sfmmup->sfmmu_tsb_cv);
12854 
12855         sfmmu_hat_exit(hatlockp);
12856 
12857         return (0);
12858 }
12859 
12860 /*
12861  * Allocate and initialize a tsb_info structure.  Note that we may or may not
12862  * allocate a TSB here, depending on the flags passed in.
12863  */
12864 static int
12865 sfmmu_tsbinfo_alloc(struct tsb_info **tsbinfopp, int tsb_szc, int tte_sz_mask,
12866         uint_t flags, sfmmu_t *sfmmup)
12867 {
12868         int err;
12869 
12870         *tsbinfopp = (struct tsb_info *)kmem_cache_alloc(
12871             sfmmu_tsbinfo_cache, KM_SLEEP);
12872 
12873         if ((err = sfmmu_init_tsbinfo(*tsbinfopp, tte_sz_mask,
12874             tsb_szc, flags, sfmmup)) != 0) {
12875                 kmem_cache_free(sfmmu_tsbinfo_cache, *tsbinfopp);
12876                 SFMMU_STAT(sf_tsb_allocfail);
12877                 *tsbinfopp = NULL;
12878                 return (err);
12879         }
12880         SFMMU_STAT(sf_tsb_alloc);
12881 
12882         /*
12883          * Bump the TSB size counters for this TSB size.
12884          */
12885         (*(((int *)&sfmmu_tsbsize_stat) + tsb_szc))++;
12886         return (0);
12887 }
12888 
12889 static void
12890 sfmmu_tsb_free(struct tsb_info *tsbinfo)
12891 {
12892         caddr_t tsbva = tsbinfo->tsb_va;
12893         uint_t tsb_size = TSB_BYTES(tsbinfo->tsb_szc);
12894         struct kmem_cache *kmem_cachep = tsbinfo->tsb_cache;
12895         vmem_t  *vmp = tsbinfo->tsb_vmp;
12896 
12897         /*
12898          * If we allocated this TSB from relocatable kernel memory, then we
12899          * need to uninstall the callback handler.
12900          */
12901         if (tsbinfo->tsb_cache != sfmmu_tsb8k_cache) {
12902                 uintptr_t slab_mask;
12903                 caddr_t slab_vaddr;
12904                 page_t **ppl;
12905                 int ret;
12906 
12907                 ASSERT(tsb_size <= MMU_PAGESIZE4M || use_bigtsb_arena);
12908                 if (tsb_size > MMU_PAGESIZE4M)
12909                         slab_mask = ~((uintptr_t)bigtsb_slab_mask) << PAGESHIFT;
12910                 else
12911                         slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT;
12912                 slab_vaddr = (caddr_t)((uintptr_t)tsbva & slab_mask);
12913 
12914                 ret = as_pagelock(&kas, &ppl, slab_vaddr, PAGESIZE, S_WRITE);
12915                 ASSERT(ret == 0);
12916                 hat_delete_callback(tsbva, (uint_t)tsb_size, (void *)tsbinfo,
12917                     0, NULL);
12918                 as_pageunlock(&kas, ppl, slab_vaddr, PAGESIZE, S_WRITE);
12919         }
12920 
12921         if (kmem_cachep != NULL) {
12922                 kmem_cache_free(kmem_cachep, tsbva);
12923         } else {
12924                 vmem_xfree(vmp, (void *)tsbva, tsb_size);
12925         }
12926         tsbinfo->tsb_va = (caddr_t)0xbad00bad;
12927         atomic_add_64(&tsb_alloc_bytes, -(int64_t)tsb_size);
12928 }
12929 
12930 static void
12931 sfmmu_tsbinfo_free(struct tsb_info *tsbinfo)
12932 {
12933         if ((tsbinfo->tsb_flags & TSB_SWAPPED) == 0) {
12934                 sfmmu_tsb_free(tsbinfo);
12935         }
12936         kmem_cache_free(sfmmu_tsbinfo_cache, tsbinfo);
12937 
12938 }
12939 
12940 /*
12941  * Setup all the references to physical memory for this tsbinfo.
12942  * The underlying page(s) must be locked.
12943  */
12944 static void
12945 sfmmu_tsbinfo_setup_phys(struct tsb_info *tsbinfo, pfn_t pfn)
12946 {
12947         ASSERT(pfn != PFN_INVALID);
12948         ASSERT(pfn == va_to_pfn(tsbinfo->tsb_va));
12949 
12950 #ifndef sun4v
12951         if (tsbinfo->tsb_szc == 0) {
12952                 sfmmu_memtte(&tsbinfo->tsb_tte, pfn,
12953                     PROT_WRITE|PROT_READ, TTE8K);
12954         } else {
12955                 /*
12956                  * Round down PA and use a large mapping; the handlers will
12957                  * compute the TSB pointer at the correct offset into the
12958                  * big virtual page.  NOTE: this assumes all TSBs larger
12959                  * than 8K must come from physically contiguous slabs of
12960                  * size tsb_slab_size.
12961                  */
12962                 sfmmu_memtte(&tsbinfo->tsb_tte, pfn & ~tsb_slab_mask,
12963                     PROT_WRITE|PROT_READ, tsb_slab_ttesz);
12964         }
12965         tsbinfo->tsb_pa = ptob(pfn);
12966 
12967         TTE_SET_LOCKED(&tsbinfo->tsb_tte); /* lock the tte into dtlb */
12968         TTE_SET_MOD(&tsbinfo->tsb_tte);    /* enable writes */
12969 
12970         ASSERT(TTE_IS_PRIVILEGED(&tsbinfo->tsb_tte));
12971         ASSERT(TTE_IS_LOCKED(&tsbinfo->tsb_tte));
12972 #else /* sun4v */
12973         tsbinfo->tsb_pa = ptob(pfn);
12974 #endif /* sun4v */
12975 }
12976 
12977 
12978 /*
12979  * Returns zero on success, ENOMEM if over the high water mark,
12980  * or EAGAIN if the caller needs to retry with a smaller TSB
12981  * size (or specify TSB_FORCEALLOC if the allocation can't fail).
12982  *
12983  * This call cannot fail to allocate a TSB if TSB_FORCEALLOC
12984  * is specified and the TSB requested is PAGESIZE, though it
12985  * may sleep waiting for memory if sufficient memory is not
12986  * available.
12987  */
12988 static int
12989 sfmmu_init_tsbinfo(struct tsb_info *tsbinfo, int tteszmask,
12990     int tsbcode, uint_t flags, sfmmu_t *sfmmup)
12991 {
12992         caddr_t vaddr = NULL;
12993         caddr_t slab_vaddr;
12994         uintptr_t slab_mask;
12995         int tsbbytes = TSB_BYTES(tsbcode);
12996         int lowmem = 0;
12997         struct kmem_cache *kmem_cachep = NULL;
12998         vmem_t *vmp = NULL;
12999         lgrp_id_t lgrpid = LGRP_NONE;
13000         pfn_t pfn;
13001         uint_t cbflags = HAC_SLEEP;
13002         page_t **pplist;
13003         int ret;
13004 
13005         ASSERT(tsbbytes <= MMU_PAGESIZE4M || use_bigtsb_arena);
13006         if (tsbbytes > MMU_PAGESIZE4M)
13007                 slab_mask = ~((uintptr_t)bigtsb_slab_mask) << PAGESHIFT;
13008         else
13009                 slab_mask = ~((uintptr_t)tsb_slab_mask) << PAGESHIFT;
13010 
13011         if (flags & (TSB_FORCEALLOC | TSB_SWAPIN | TSB_GROW | TSB_SHRINK))
13012                 flags |= TSB_ALLOC;
13013 
13014         ASSERT((flags & TSB_FORCEALLOC) == 0 || tsbcode == TSB_MIN_SZCODE);
13015 
13016         tsbinfo->tsb_sfmmu = sfmmup;
13017 
13018         /*
13019          * If not allocating a TSB, set up the tsbinfo, set TSB_SWAPPED, and
13020          * return.
13021          */
13022         if ((flags & TSB_ALLOC) == 0) {
13023                 tsbinfo->tsb_szc = tsbcode;
13024                 tsbinfo->tsb_ttesz_mask = tteszmask;
13025                 tsbinfo->tsb_va = (caddr_t)0xbadbadbeef;
13026                 tsbinfo->tsb_pa = -1;
13027                 tsbinfo->tsb_tte.ll = 0;
13028                 tsbinfo->tsb_next = NULL;
13029                 tsbinfo->tsb_flags = TSB_SWAPPED;
13030                 tsbinfo->tsb_cache = NULL;
13031                 tsbinfo->tsb_vmp = NULL;
13032                 return (0);
13033         }
13034 
13035 #ifdef DEBUG
13036         /*
13037          * For debugging:
13038          * Randomly force allocation failures every tsb_alloc_mtbf
13039          * tries if TSB_FORCEALLOC is not specified.  This will
13040          * return ENOMEM if tsb_alloc_mtbf is odd, or EAGAIN if
13041          * it is even, to allow testing of both failure paths...
13042          */
13043         if (tsb_alloc_mtbf && ((flags & TSB_FORCEALLOC) == 0) &&
13044             (tsb_alloc_count++ == tsb_alloc_mtbf)) {
13045                 tsb_alloc_count = 0;
13046                 tsb_alloc_fail_mtbf++;
13047                 return ((tsb_alloc_mtbf & 1)? ENOMEM : EAGAIN);
13048         }
13049 #endif  /* DEBUG */
13050 
13051         /*
13052          * Enforce high water mark if we are not doing a forced allocation
13053          * and are not shrinking a process' TSB.
13054          */
13055         if ((flags & TSB_SHRINK) == 0 &&
13056             (tsbbytes + tsb_alloc_bytes) > tsb_alloc_hiwater) {
13057                 if ((flags & TSB_FORCEALLOC) == 0)
13058                         return (ENOMEM);
13059                 lowmem = 1;
13060         }
13061 
13062         /*
13063          * Allocate from the correct location based upon the size of the TSB
13064          * compared to the base page size, and what memory conditions dictate.
13065          * Note we always do nonblocking allocations from the TSB arena since
13066          * we don't want memory fragmentation to cause processes to block
13067          * indefinitely waiting for memory; until the kernel algorithms that
13068          * coalesce large pages are improved this is our best option.
13069          *
13070          * Algorithm:
13071          *      If allocating a "large" TSB (>8K), allocate from the
13072          *              appropriate kmem_tsb_default_arena vmem arena
13073          *      else if low on memory or the TSB_FORCEALLOC flag is set or
13074          *      tsb_forceheap is set
13075          *              Allocate from kernel heap via sfmmu_tsb8k_cache with
13076          *              KM_SLEEP (never fails)
13077          *      else
13078          *              Allocate from appropriate sfmmu_tsb_cache with
13079          *              KM_NOSLEEP
13080          *      endif
13081          */
13082         if (tsb_lgrp_affinity)
13083                 lgrpid = lgrp_home_id(curthread);
13084         if (lgrpid == LGRP_NONE)
13085                 lgrpid = 0;     /* use lgrp of boot CPU */
13086 
13087         if (tsbbytes > MMU_PAGESIZE) {
13088                 if (tsbbytes > MMU_PAGESIZE4M) {
13089                         vmp = kmem_bigtsb_default_arena[lgrpid];
13090                         vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes,
13091                             0, 0, NULL, NULL, VM_NOSLEEP);
13092                 } else {
13093                         vmp = kmem_tsb_default_arena[lgrpid];
13094                         vaddr = (caddr_t)vmem_xalloc(vmp, tsbbytes, tsbbytes,
13095                             0, 0, NULL, NULL, VM_NOSLEEP);
13096                 }
13097 #ifdef  DEBUG
13098         } else if (lowmem || (flags & TSB_FORCEALLOC) || tsb_forceheap) {
13099 #else   /* !DEBUG */
13100         } else if (lowmem || (flags & TSB_FORCEALLOC)) {
13101 #endif  /* DEBUG */
13102                 kmem_cachep = sfmmu_tsb8k_cache;
13103                 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_SLEEP);
13104                 ASSERT(vaddr != NULL);
13105         } else {
13106                 kmem_cachep = sfmmu_tsb_cache[lgrpid];
13107                 vaddr = (caddr_t)kmem_cache_alloc(kmem_cachep, KM_NOSLEEP);
13108         }
13109 
13110         tsbinfo->tsb_cache = kmem_cachep;
13111         tsbinfo->tsb_vmp = vmp;
13112 
13113         if (vaddr == NULL) {
13114                 return (EAGAIN);
13115         }
13116 
13117         atomic_add_64(&tsb_alloc_bytes, (int64_t)tsbbytes);
13118         kmem_cachep = tsbinfo->tsb_cache;
13119 
13120         /*
13121          * If we are allocating from outside the cage, then we need to
13122          * register a relocation callback handler.  Note that for now
13123          * since pseudo mappings always hang off of the slab's root page,
13124          * we need only lock the first 8K of the TSB slab.  This is a bit
13125          * hacky but it is good for performance.
13126          */
13127         if (kmem_cachep != sfmmu_tsb8k_cache) {
13128                 slab_vaddr = (caddr_t)((uintptr_t)vaddr & slab_mask);
13129                 ret = as_pagelock(&kas, &pplist, slab_vaddr, PAGESIZE, S_WRITE);
13130                 ASSERT(ret == 0);
13131                 ret = hat_add_callback(sfmmu_tsb_cb_id, vaddr, (uint_t)tsbbytes,
13132                     cbflags, (void *)tsbinfo, &pfn, NULL);
13133 
13134                 /*
13135                  * Need to free up resources if we could not successfully
13136                  * add the callback function and return an error condition.
13137                  */
13138                 if (ret != 0) {
13139                         if (kmem_cachep) {
13140                                 kmem_cache_free(kmem_cachep, vaddr);
13141                         } else {
13142                                 vmem_xfree(vmp, (void *)vaddr, tsbbytes);
13143                         }
13144                         as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE,
13145                             S_WRITE);
13146                         return (EAGAIN);
13147                 }
13148         } else {
13149                 /*
13150                  * Since allocation of 8K TSBs from heap is rare and occurs
13151                  * during memory pressure we allocate them from permanent
13152                  * memory rather than using callbacks to get the PFN.
13153                  */
13154                 pfn = hat_getpfnum(kas.a_hat, vaddr);
13155         }
13156 
13157         tsbinfo->tsb_va = vaddr;
13158         tsbinfo->tsb_szc = tsbcode;
13159         tsbinfo->tsb_ttesz_mask = tteszmask;
13160         tsbinfo->tsb_next = NULL;
13161         tsbinfo->tsb_flags = 0;
13162 
13163         sfmmu_tsbinfo_setup_phys(tsbinfo, pfn);
13164 
13165         sfmmu_inv_tsb(vaddr, tsbbytes);
13166 
13167         if (kmem_cachep != sfmmu_tsb8k_cache) {
13168                 as_pageunlock(&kas, pplist, slab_vaddr, PAGESIZE, S_WRITE);
13169         }
13170 
13171         return (0);
13172 }
13173 
13174 /*
13175  * Initialize per cpu tsb and per cpu tsbmiss_area
13176  */
13177 void
13178 sfmmu_init_tsbs(void)
13179 {
13180         int i;
13181         struct tsbmiss  *tsbmissp;
13182         struct kpmtsbm  *kpmtsbmp;
13183 #ifndef sun4v
13184         extern int      dcache_line_mask;
13185 #endif /* sun4v */
13186         extern uint_t   vac_colors;
13187 
13188         /*
13189          * Init. tsb miss area.
13190          */
13191         tsbmissp = tsbmiss_area;
13192 
13193         for (i = 0; i < NCPU; tsbmissp++, i++) {
13194                 /*
13195                  * initialize the tsbmiss area.
13196                  * Do this for all possible CPUs as some may be added
13197                  * while the system is running. There is no cost to this.
13198                  */
13199                 tsbmissp->ksfmmup = ksfmmup;
13200 #ifndef sun4v
13201                 tsbmissp->dcache_line_mask = (uint16_t)dcache_line_mask;
13202 #endif /* sun4v */
13203                 tsbmissp->khashstart =
13204                     (struct hmehash_bucket *)va_to_pa((caddr_t)khme_hash);
13205                 tsbmissp->uhashstart =
13206                     (struct hmehash_bucket *)va_to_pa((caddr_t)uhme_hash);
13207                 tsbmissp->khashsz = khmehash_num;
13208                 tsbmissp->uhashsz = uhmehash_num;
13209         }
13210 
13211         sfmmu_tsb_cb_id = hat_register_callback('T'<<16 | 'S' << 8 | 'B',
13212             sfmmu_tsb_pre_relocator, sfmmu_tsb_post_relocator, NULL, 0);
13213 
13214         if (kpm_enable == 0)
13215                 return;
13216 
13217         /* -- Begin KPM specific init -- */
13218 
13219         if (kpm_smallpages) {
13220                 /*
13221                  * If we're using base pagesize pages for seg_kpm
13222                  * mappings, we use the kernel TSB since we can't afford
13223                  * to allocate a second huge TSB for these mappings.
13224                  */
13225                 kpm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base;
13226                 kpm_tsbsz = ktsb_szcode;
13227                 kpmsm_tsbbase = kpm_tsbbase;
13228                 kpmsm_tsbsz = kpm_tsbsz;
13229         } else {
13230                 /*
13231                  * In VAC conflict case, just put the entries in the
13232                  * kernel 8K indexed TSB for now so we can find them.
13233                  * This could really be changed in the future if we feel
13234                  * the need...
13235                  */
13236                 kpmsm_tsbbase = ktsb_phys? ktsb_pbase : (uint64_t)ktsb_base;
13237                 kpmsm_tsbsz = ktsb_szcode;
13238                 kpm_tsbbase = ktsb_phys? ktsb4m_pbase : (uint64_t)ktsb4m_base;
13239                 kpm_tsbsz = ktsb4m_szcode;
13240         }
13241 
13242         kpmtsbmp = kpmtsbm_area;
13243         for (i = 0; i < NCPU; kpmtsbmp++, i++) {
13244                 /*
13245                  * Initialize the kpmtsbm area.
13246                  * Do this for all possible CPUs as some may be added
13247                  * while the system is running. There is no cost to this.
13248                  */
13249                 kpmtsbmp->vbase = kpm_vbase;
13250                 kpmtsbmp->vend = kpm_vbase + kpm_size * vac_colors;
13251                 kpmtsbmp->sz_shift = kpm_size_shift;
13252                 kpmtsbmp->kpmp_shift = kpmp_shift;
13253                 kpmtsbmp->kpmp2pshft = (uchar_t)kpmp2pshft;
13254                 if (kpm_smallpages == 0) {
13255                         kpmtsbmp->kpmp_table_sz = kpmp_table_sz;
13256                         kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_table);
13257                 } else {
13258                         kpmtsbmp->kpmp_table_sz = kpmp_stable_sz;
13259                         kpmtsbmp->kpmp_tablepa = va_to_pa(kpmp_stable);
13260                 }
13261                 kpmtsbmp->msegphashpa = va_to_pa(memseg_phash);
13262                 kpmtsbmp->flags = KPMTSBM_ENABLE_FLAG;
13263 #ifdef  DEBUG
13264                 kpmtsbmp->flags |= (kpm_tsbmtl) ?  KPMTSBM_TLTSBM_FLAG : 0;
13265 #endif  /* DEBUG */
13266                 if (ktsb_phys)
13267                         kpmtsbmp->flags |= KPMTSBM_TSBPHYS_FLAG;
13268         }
13269 
13270         /* -- End KPM specific init -- */
13271 }
13272 
13273 /* Avoid using sfmmu_tsbinfo_alloc() to avoid kmem_alloc - no real reason */
13274 struct tsb_info ktsb_info[2];
13275 
13276 /*
13277  * Called from hat_kern_setup() to setup the tsb_info for ksfmmup.
13278  */
13279 void
13280 sfmmu_init_ktsbinfo()
13281 {
13282         ASSERT(ksfmmup != NULL);
13283         ASSERT(ksfmmup->sfmmu_tsb == NULL);
13284         /*
13285          * Allocate tsbinfos for kernel and copy in data
13286          * to make debug easier and sun4v setup easier.
13287          */
13288         ktsb_info[0].tsb_sfmmu = ksfmmup;
13289         ktsb_info[0].tsb_szc = ktsb_szcode;
13290         ktsb_info[0].tsb_ttesz_mask = TSB8K|TSB64K|TSB512K;
13291         ktsb_info[0].tsb_va = ktsb_base;
13292         ktsb_info[0].tsb_pa = ktsb_pbase;
13293         ktsb_info[0].tsb_flags = 0;
13294         ktsb_info[0].tsb_tte.ll = 0;
13295         ktsb_info[0].tsb_cache = NULL;
13296 
13297         ktsb_info[1].tsb_sfmmu = ksfmmup;
13298         ktsb_info[1].tsb_szc = ktsb4m_szcode;
13299         ktsb_info[1].tsb_ttesz_mask = TSB4M;
13300         ktsb_info[1].tsb_va = ktsb4m_base;
13301         ktsb_info[1].tsb_pa = ktsb4m_pbase;
13302         ktsb_info[1].tsb_flags = 0;
13303         ktsb_info[1].tsb_tte.ll = 0;
13304         ktsb_info[1].tsb_cache = NULL;
13305 
13306         /* Link them into ksfmmup. */
13307         ktsb_info[0].tsb_next = &ktsb_info[1];
13308         ktsb_info[1].tsb_next = NULL;
13309         ksfmmup->sfmmu_tsb = &ktsb_info[0];
13310 
13311         sfmmu_setup_tsbinfo(ksfmmup);
13312 }
13313 
13314 /*
13315  * Cache the last value returned from va_to_pa().  If the VA specified
13316  * in the current call to cached_va_to_pa() maps to the same Page (as the
13317  * previous call to cached_va_to_pa()), then compute the PA using
13318  * cached info, else call va_to_pa().
13319  *
13320  * Note: this function is neither MT-safe nor consistent in the presence
13321  * of multiple, interleaved threads.  This function was created to enable
13322  * an optimization used during boot (at a point when there's only one thread
13323  * executing on the "boot CPU", and before startup_vm() has been called).
13324  */
13325 static uint64_t
13326 cached_va_to_pa(void *vaddr)
13327 {
13328         static uint64_t prev_vaddr_base = 0;
13329         static uint64_t prev_pfn = 0;
13330 
13331         if ((((uint64_t)vaddr) & MMU_PAGEMASK) == prev_vaddr_base) {
13332                 return (prev_pfn | ((uint64_t)vaddr & MMU_PAGEOFFSET));
13333         } else {
13334                 uint64_t pa = va_to_pa(vaddr);
13335 
13336                 if (pa != ((uint64_t)-1)) {
13337                         /*
13338                          * Computed physical address is valid.  Cache its
13339                          * related info for the next cached_va_to_pa() call.
13340                          */
13341                         prev_pfn = pa & MMU_PAGEMASK;
13342                         prev_vaddr_base = ((uint64_t)vaddr) & MMU_PAGEMASK;
13343                 }
13344 
13345                 return (pa);
13346         }
13347 }
13348 
13349 /*
13350  * Carve up our nucleus hblk region.  We may allocate more hblks than
13351  * asked due to rounding errors but we are guaranteed to have at least
13352  * enough space to allocate the requested number of hblk8's and hblk1's.
13353  */
13354 void
13355 sfmmu_init_nucleus_hblks(caddr_t addr, size_t size, int nhblk8, int nhblk1)
13356 {
13357         struct hme_blk *hmeblkp;
13358         size_t hme8blk_sz, hme1blk_sz;
13359         size_t i;
13360         size_t hblk8_bound;
13361         ulong_t j = 0, k = 0;
13362 
13363         ASSERT(addr != NULL && size != 0);
13364 
13365         /* Need to use proper structure alignment */
13366         hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
13367         hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
13368 
13369         nucleus_hblk8.list = (void *)addr;
13370         nucleus_hblk8.index = 0;
13371 
13372         /*
13373          * Use as much memory as possible for hblk8's since we
13374          * expect all bop_alloc'ed memory to be allocated in 8k chunks.
13375          * We need to hold back enough space for the hblk1's which
13376          * we'll allocate next.
13377          */
13378         hblk8_bound = size - (nhblk1 * hme1blk_sz) - hme8blk_sz;
13379         for (i = 0; i <= hblk8_bound; i += hme8blk_sz, j++) {
13380                 hmeblkp = (struct hme_blk *)addr;
13381                 addr += hme8blk_sz;
13382                 hmeblkp->hblk_nuc_bit = 1;
13383                 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp);
13384         }
13385         nucleus_hblk8.len = j;
13386         ASSERT(j >= nhblk8);
13387         SFMMU_STAT_ADD(sf_hblk8_ncreate, j);
13388 
13389         nucleus_hblk1.list = (void *)addr;
13390         nucleus_hblk1.index = 0;
13391         for (; i <= (size - hme1blk_sz); i += hme1blk_sz, k++) {
13392                 hmeblkp = (struct hme_blk *)addr;
13393                 addr += hme1blk_sz;
13394                 hmeblkp->hblk_nuc_bit = 1;
13395                 hmeblkp->hblk_nextpa = cached_va_to_pa((caddr_t)hmeblkp);
13396         }
13397         ASSERT(k >= nhblk1);
13398         nucleus_hblk1.len = k;
13399         SFMMU_STAT_ADD(sf_hblk1_ncreate, k);
13400 }
13401 
13402 /*
13403  * This function is currently not supported on this platform. For what
13404  * it's supposed to do, see hat.c and hat_srmmu.c
13405  */
13406 /* ARGSUSED */
13407 faultcode_t
13408 hat_softlock(struct hat *hat, caddr_t addr, size_t *lenp, page_t **ppp,
13409     uint_t flags)
13410 {
13411         ASSERT(hat->sfmmu_xhat_provider == NULL);
13412         return (FC_NOSUPPORT);
13413 }
13414 
13415 /*
13416  * Searchs the mapping list of the page for a mapping of the same size. If not
13417  * found the corresponding bit is cleared in the p_index field. When large
13418  * pages are more prevalent in the system, we can maintain the mapping list
13419  * in order and we don't have to traverse the list each time. Just check the
13420  * next and prev entries, and if both are of different size, we clear the bit.
13421  */
13422 static void
13423 sfmmu_rm_large_mappings(page_t *pp, int ttesz)
13424 {
13425         struct sf_hment *sfhmep;
13426         struct hme_blk *hmeblkp;
13427         int     index;
13428         pgcnt_t npgs;
13429 
13430         ASSERT(ttesz > TTE8K);
13431 
13432         ASSERT(sfmmu_mlist_held(pp));
13433 
13434         ASSERT(PP_ISMAPPED_LARGE(pp));
13435 
13436         /*
13437          * Traverse mapping list looking for another mapping of same size.
13438          * since we only want to clear index field if all mappings of
13439          * that size are gone.
13440          */
13441 
13442         for (sfhmep = pp->p_mapping; sfhmep; sfhmep = sfhmep->hme_next) {
13443                 if (IS_PAHME(sfhmep))
13444                         continue;
13445                 hmeblkp = sfmmu_hmetohblk(sfhmep);
13446                 if (hmeblkp->hblk_xhat_bit)
13447                         continue;
13448                 if (hme_size(sfhmep) == ttesz) {
13449                         /*
13450                          * another mapping of the same size. don't clear index.
13451                          */
13452                         return;
13453                 }
13454         }
13455 
13456         /*
13457          * Clear the p_index bit for large page.
13458          */
13459         index = PAGESZ_TO_INDEX(ttesz);
13460         npgs = TTEPAGES(ttesz);
13461         while (npgs-- > 0) {
13462                 ASSERT(pp->p_index & index);
13463                 pp->p_index &= ~index;
13464                 pp = PP_PAGENEXT(pp);
13465         }
13466 }
13467 
13468 /*
13469  * return supported features
13470  */
13471 /* ARGSUSED */
13472 int
13473 hat_supported(enum hat_features feature, void *arg)
13474 {
13475         switch (feature) {
13476         case    HAT_SHARED_PT:
13477         case    HAT_DYNAMIC_ISM_UNMAP:
13478         case    HAT_VMODSORT:
13479                 return (1);
13480         case    HAT_SHARED_REGIONS:
13481                 if (shctx_on)
13482                         return (1);
13483                 else
13484                         return (0);
13485         default:
13486                 return (0);
13487         }
13488 }
13489 
13490 void
13491 hat_enter(struct hat *hat)
13492 {
13493         hatlock_t       *hatlockp;
13494 
13495         if (hat != ksfmmup) {
13496                 hatlockp = TSB_HASH(hat);
13497                 mutex_enter(HATLOCK_MUTEXP(hatlockp));
13498         }
13499 }
13500 
13501 void
13502 hat_exit(struct hat *hat)
13503 {
13504         hatlock_t       *hatlockp;
13505 
13506         if (hat != ksfmmup) {
13507                 hatlockp = TSB_HASH(hat);
13508                 mutex_exit(HATLOCK_MUTEXP(hatlockp));
13509         }
13510 }
13511 
13512 /*ARGSUSED*/
13513 void
13514 hat_reserve(struct as *as, caddr_t addr, size_t len)
13515 {
13516 }
13517 
13518 static void
13519 hat_kstat_init(void)
13520 {
13521         kstat_t *ksp;
13522 
13523         ksp = kstat_create("unix", 0, "sfmmu_global_stat", "hat",
13524             KSTAT_TYPE_RAW, sizeof (struct sfmmu_global_stat),
13525             KSTAT_FLAG_VIRTUAL);
13526         if (ksp) {
13527                 ksp->ks_data = (void *) &sfmmu_global_stat;
13528                 kstat_install(ksp);
13529         }
13530         ksp = kstat_create("unix", 0, "sfmmu_tsbsize_stat", "hat",
13531             KSTAT_TYPE_RAW, sizeof (struct sfmmu_tsbsize_stat),
13532             KSTAT_FLAG_VIRTUAL);
13533         if (ksp) {
13534                 ksp->ks_data = (void *) &sfmmu_tsbsize_stat;
13535                 kstat_install(ksp);
13536         }
13537         ksp = kstat_create("unix", 0, "sfmmu_percpu_stat", "hat",
13538             KSTAT_TYPE_RAW, sizeof (struct sfmmu_percpu_stat) * NCPU,
13539             KSTAT_FLAG_WRITABLE);
13540         if (ksp) {
13541                 ksp->ks_update = sfmmu_kstat_percpu_update;
13542                 kstat_install(ksp);
13543         }
13544 }
13545 
13546 /* ARGSUSED */
13547 static int
13548 sfmmu_kstat_percpu_update(kstat_t *ksp, int rw)
13549 {
13550         struct sfmmu_percpu_stat *cpu_kstat = ksp->ks_data;
13551         struct tsbmiss *tsbm = tsbmiss_area;
13552         struct kpmtsbm *kpmtsbm = kpmtsbm_area;
13553         int i;
13554 
13555         ASSERT(cpu_kstat);
13556         if (rw == KSTAT_READ) {
13557                 for (i = 0; i < NCPU; cpu_kstat++, tsbm++, kpmtsbm++, i++) {
13558                         cpu_kstat->sf_itlb_misses = 0;
13559                         cpu_kstat->sf_dtlb_misses = 0;
13560                         cpu_kstat->sf_utsb_misses = tsbm->utsb_misses -
13561                             tsbm->uprot_traps;
13562                         cpu_kstat->sf_ktsb_misses = tsbm->ktsb_misses +
13563                             kpmtsbm->kpm_tsb_misses - tsbm->kprot_traps;
13564                         cpu_kstat->sf_tsb_hits = 0;
13565                         cpu_kstat->sf_umod_faults = tsbm->uprot_traps;
13566                         cpu_kstat->sf_kmod_faults = tsbm->kprot_traps;
13567                 }
13568         } else {
13569                 /* KSTAT_WRITE is used to clear stats */
13570                 for (i = 0; i < NCPU; tsbm++, kpmtsbm++, i++) {
13571                         tsbm->utsb_misses = 0;
13572                         tsbm->ktsb_misses = 0;
13573                         tsbm->uprot_traps = 0;
13574                         tsbm->kprot_traps = 0;
13575                         kpmtsbm->kpm_dtlb_misses = 0;
13576                         kpmtsbm->kpm_tsb_misses = 0;
13577                 }
13578         }
13579         return (0);
13580 }
13581 
13582 #ifdef  DEBUG
13583 
13584 tte_t  *gorig[NCPU], *gcur[NCPU], *gnew[NCPU];
13585 
13586 /*
13587  * A tte checker. *orig_old is the value we read before cas.
13588  *      *cur is the value returned by cas.
13589  *      *new is the desired value when we do the cas.
13590  *
13591  *      *hmeblkp is currently unused.
13592  */
13593 
13594 /* ARGSUSED */
13595 void
13596 chk_tte(tte_t *orig_old, tte_t *cur, tte_t *new, struct hme_blk *hmeblkp)
13597 {
13598         pfn_t i, j, k;
13599         int cpuid = CPU->cpu_id;
13600 
13601         gorig[cpuid] = orig_old;
13602         gcur[cpuid] = cur;
13603         gnew[cpuid] = new;
13604 
13605 #ifdef lint
13606         hmeblkp = hmeblkp;
13607 #endif
13608 
13609         if (TTE_IS_VALID(orig_old)) {
13610                 if (TTE_IS_VALID(cur)) {
13611                         i = TTE_TO_TTEPFN(orig_old);
13612                         j = TTE_TO_TTEPFN(cur);
13613                         k = TTE_TO_TTEPFN(new);
13614                         if (i != j) {
13615                                 /* remap error? */
13616                                 panic("chk_tte: bad pfn, 0x%lx, 0x%lx", i, j);
13617                         }
13618 
13619                         if (i != k) {
13620                                 /* remap error? */
13621                                 panic("chk_tte: bad pfn2, 0x%lx, 0x%lx", i, k);
13622                         }
13623                 } else {
13624                         if (TTE_IS_VALID(new)) {
13625                                 panic("chk_tte: invalid cur? ");
13626                         }
13627 
13628                         i = TTE_TO_TTEPFN(orig_old);
13629                         k = TTE_TO_TTEPFN(new);
13630                         if (i != k) {
13631                                 panic("chk_tte: bad pfn3, 0x%lx, 0x%lx", i, k);
13632                         }
13633                 }
13634         } else {
13635                 if (TTE_IS_VALID(cur)) {
13636                         j = TTE_TO_TTEPFN(cur);
13637                         if (TTE_IS_VALID(new)) {
13638                                 k = TTE_TO_TTEPFN(new);
13639                                 if (j != k) {
13640                                         panic("chk_tte: bad pfn4, 0x%lx, 0x%lx",
13641                                             j, k);
13642                                 }
13643                         } else {
13644                                 panic("chk_tte: why here?");
13645                         }
13646                 } else {
13647                         if (!TTE_IS_VALID(new)) {
13648                                 panic("chk_tte: why here2 ?");
13649                         }
13650                 }
13651         }
13652 }
13653 
13654 #endif /* DEBUG */
13655 
13656 extern void prefetch_tsbe_read(struct tsbe *);
13657 extern void prefetch_tsbe_write(struct tsbe *);
13658 
13659 
13660 /*
13661  * We want to prefetch 7 cache lines ahead for our read prefetch.  This gives
13662  * us optimal performance on Cheetah+.  You can only have 8 outstanding
13663  * prefetches at any one time, so we opted for 7 read prefetches and 1 write
13664  * prefetch to make the most utilization of the prefetch capability.
13665  */
13666 #define TSBE_PREFETCH_STRIDE (7)
13667 
13668 void
13669 sfmmu_copy_tsb(struct tsb_info *old_tsbinfo, struct tsb_info *new_tsbinfo)
13670 {
13671         int old_bytes = TSB_BYTES(old_tsbinfo->tsb_szc);
13672         int new_bytes = TSB_BYTES(new_tsbinfo->tsb_szc);
13673         int old_entries = TSB_ENTRIES(old_tsbinfo->tsb_szc);
13674         int new_entries = TSB_ENTRIES(new_tsbinfo->tsb_szc);
13675         struct tsbe *old;
13676         struct tsbe *new;
13677         struct tsbe *new_base = (struct tsbe *)new_tsbinfo->tsb_va;
13678         uint64_t va;
13679         int new_offset;
13680         int i;
13681         int vpshift;
13682         int last_prefetch;
13683 
13684         if (old_bytes == new_bytes) {
13685                 bcopy(old_tsbinfo->tsb_va, new_tsbinfo->tsb_va, new_bytes);
13686         } else {
13687 
13688                 /*
13689                  * A TSBE is 16 bytes which means there are four TSBE's per
13690                  * P$ line (64 bytes), thus every 4 TSBE's we prefetch.
13691                  */
13692                 old = (struct tsbe *)old_tsbinfo->tsb_va;
13693                 last_prefetch = old_entries - (4*(TSBE_PREFETCH_STRIDE+1));
13694                 for (i = 0; i < old_entries; i++, old++) {
13695                         if (((i & (4-1)) == 0) && (i < last_prefetch))
13696                                 prefetch_tsbe_read(old);
13697                         if (!old->tte_tag.tag_invalid) {
13698                                 /*
13699                                  * We have a valid TTE to remap.  Check the
13700                                  * size.  We won't remap 64K or 512K TTEs
13701                                  * because they span more than one TSB entry
13702                                  * and are indexed using an 8K virt. page.
13703                                  * Ditto for 32M and 256M TTEs.
13704                                  */
13705                                 if (TTE_CSZ(&old->tte_data) == TTE64K ||
13706                                     TTE_CSZ(&old->tte_data) == TTE512K)
13707                                         continue;
13708                                 if (mmu_page_sizes == max_mmu_page_sizes) {
13709                                         if (TTE_CSZ(&old->tte_data) == TTE32M ||
13710                                             TTE_CSZ(&old->tte_data) == TTE256M)
13711                                                 continue;
13712                                 }
13713 
13714                                 /* clear the lower 22 bits of the va */
13715                                 va = *(uint64_t *)old << 22;
13716                                 /* turn va into a virtual pfn */
13717                                 va >>= 22 - TSB_START_SIZE;
13718                                 /*
13719                                  * or in bits from the offset in the tsb
13720                                  * to get the real virtual pfn. These
13721                                  * correspond to bits [21:13] in the va
13722                                  */
13723                                 vpshift =
13724                                     TTE_BSZS_SHIFT(TTE_CSZ(&old->tte_data)) &
13725                                     0x1ff;
13726                                 va |= (i << vpshift);
13727                                 va >>= vpshift;
13728                                 new_offset = va & (new_entries - 1);
13729                                 new = new_base + new_offset;
13730                                 prefetch_tsbe_write(new);
13731                                 *new = *old;
13732                         }
13733                 }
13734         }
13735 }
13736 
13737 /*
13738  * unused in sfmmu
13739  */
13740 void
13741 hat_dump(void)
13742 {
13743 }
13744 
13745 /*
13746  * Called when a thread is exiting and we have switched to the kernel address
13747  * space.  Perform the same VM initialization resume() uses when switching
13748  * processes.
13749  *
13750  * Note that sfmmu_load_mmustate() is currently a no-op for kernel threads, but
13751  * we call it anyway in case the semantics change in the future.
13752  */
13753 /*ARGSUSED*/
13754 void
13755 hat_thread_exit(kthread_t *thd)
13756 {
13757         uint_t pgsz_cnum;
13758         uint_t pstate_save;
13759 
13760         ASSERT(thd->t_procp->p_as == &kas);
13761 
13762         pgsz_cnum = KCONTEXT;
13763 #ifdef sun4u
13764         pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT);
13765 #endif
13766 
13767         /*
13768          * Note that sfmmu_load_mmustate() is currently a no-op for
13769          * kernel threads. We need to disable interrupts here,
13770          * simply because otherwise sfmmu_load_mmustate() would panic
13771          * if the caller does not disable interrupts.
13772          */
13773         pstate_save = sfmmu_disable_intrs();
13774 
13775         /* Compatibility Note: hw takes care of MMU_SCONTEXT1 */
13776         sfmmu_setctx_sec(pgsz_cnum);
13777         sfmmu_load_mmustate(ksfmmup);
13778         sfmmu_enable_intrs(pstate_save);
13779 }
13780 
13781 
13782 /*
13783  * SRD support
13784  */
13785 #define SRD_HASH_FUNCTION(vp)   (((((uintptr_t)(vp)) >> 4) ^ \
13786                                     (((uintptr_t)(vp)) >> 11)) & \
13787                                     srd_hashmask)
13788 
13789 /*
13790  * Attach the process to the srd struct associated with the exec vnode
13791  * from which the process is started.
13792  */
13793 void
13794 hat_join_srd(struct hat *sfmmup, vnode_t *evp)
13795 {
13796         uint_t hash = SRD_HASH_FUNCTION(evp);
13797         sf_srd_t *srdp;
13798         sf_srd_t *newsrdp;
13799 
13800         ASSERT(sfmmup != ksfmmup);
13801         ASSERT(sfmmup->sfmmu_srdp == NULL);
13802 
13803         if (!shctx_on) {
13804                 return;
13805         }
13806 
13807         VN_HOLD(evp);
13808 
13809         if (srd_buckets[hash].srdb_srdp != NULL) {
13810                 mutex_enter(&srd_buckets[hash].srdb_lock);
13811                 for (srdp = srd_buckets[hash].srdb_srdp; srdp != NULL;
13812                     srdp = srdp->srd_hash) {
13813                         if (srdp->srd_evp == evp) {
13814                                 ASSERT(srdp->srd_refcnt >= 0);
13815                                 sfmmup->sfmmu_srdp = srdp;
13816                                 atomic_add_32(
13817                                     (volatile uint_t *)&srdp->srd_refcnt, 1);
13818                                 mutex_exit(&srd_buckets[hash].srdb_lock);
13819                                 return;
13820                         }
13821                 }
13822                 mutex_exit(&srd_buckets[hash].srdb_lock);
13823         }
13824         newsrdp = kmem_cache_alloc(srd_cache, KM_SLEEP);
13825         ASSERT(newsrdp->srd_next_ismrid == 0 && newsrdp->srd_next_hmerid == 0);
13826 
13827         newsrdp->srd_evp = evp;
13828         newsrdp->srd_refcnt = 1;
13829         newsrdp->srd_hmergnfree = NULL;
13830         newsrdp->srd_ismrgnfree = NULL;
13831 
13832         mutex_enter(&srd_buckets[hash].srdb_lock);
13833         for (srdp = srd_buckets[hash].srdb_srdp; srdp != NULL;
13834             srdp = srdp->srd_hash) {
13835                 if (srdp->srd_evp == evp) {
13836                         ASSERT(srdp->srd_refcnt >= 0);
13837                         sfmmup->sfmmu_srdp = srdp;
13838                         atomic_add_32((volatile uint_t *)&srdp->srd_refcnt, 1);
13839                         mutex_exit(&srd_buckets[hash].srdb_lock);
13840                         kmem_cache_free(srd_cache, newsrdp);
13841                         return;
13842                 }
13843         }
13844         newsrdp->srd_hash = srd_buckets[hash].srdb_srdp;
13845         srd_buckets[hash].srdb_srdp = newsrdp;
13846         sfmmup->sfmmu_srdp = newsrdp;
13847 
13848         mutex_exit(&srd_buckets[hash].srdb_lock);
13849 
13850 }
13851 
13852 static void
13853 sfmmu_leave_srd(sfmmu_t *sfmmup)
13854 {
13855         vnode_t *evp;
13856         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
13857         uint_t hash;
13858         sf_srd_t **prev_srdpp;
13859         sf_region_t *rgnp;
13860         sf_region_t *nrgnp;
13861 #ifdef DEBUG
13862         int rgns = 0;
13863 #endif
13864         int i;
13865 
13866         ASSERT(sfmmup != ksfmmup);
13867         ASSERT(srdp != NULL);
13868         ASSERT(srdp->srd_refcnt > 0);
13869         ASSERT(sfmmup->sfmmu_scdp == NULL);
13870         ASSERT(sfmmup->sfmmu_free == 1);
13871 
13872         sfmmup->sfmmu_srdp = NULL;
13873         evp = srdp->srd_evp;
13874         ASSERT(evp != NULL);
13875         if (atomic_add_32_nv(
13876             (volatile uint_t *)&srdp->srd_refcnt, -1)) {
13877                 VN_RELE(evp);
13878                 return;
13879         }
13880 
13881         hash = SRD_HASH_FUNCTION(evp);
13882         mutex_enter(&srd_buckets[hash].srdb_lock);
13883         for (prev_srdpp = &srd_buckets[hash].srdb_srdp;
13884             (srdp = *prev_srdpp) != NULL; prev_srdpp = &srdp->srd_hash) {
13885                 if (srdp->srd_evp == evp) {
13886                         break;
13887                 }
13888         }
13889         if (srdp == NULL || srdp->srd_refcnt) {
13890                 mutex_exit(&srd_buckets[hash].srdb_lock);
13891                 VN_RELE(evp);
13892                 return;
13893         }
13894         *prev_srdpp = srdp->srd_hash;
13895         mutex_exit(&srd_buckets[hash].srdb_lock);
13896 
13897         ASSERT(srdp->srd_refcnt == 0);
13898         VN_RELE(evp);
13899 
13900 #ifdef DEBUG
13901         for (i = 0; i < SFMMU_MAX_REGION_BUCKETS; i++) {
13902                 ASSERT(srdp->srd_rgnhash[i] == NULL);
13903         }
13904 #endif /* DEBUG */
13905 
13906         /* free each hme regions in the srd */
13907         for (rgnp = srdp->srd_hmergnfree; rgnp != NULL; rgnp = nrgnp) {
13908                 nrgnp = rgnp->rgn_next;
13909                 ASSERT(rgnp->rgn_id < srdp->srd_next_hmerid);
13910                 ASSERT(rgnp->rgn_refcnt == 0);
13911                 ASSERT(rgnp->rgn_sfmmu_head == NULL);
13912                 ASSERT(rgnp->rgn_flags & SFMMU_REGION_FREE);
13913                 ASSERT(rgnp->rgn_hmeflags == 0);
13914                 ASSERT(srdp->srd_hmergnp[rgnp->rgn_id] == rgnp);
13915 #ifdef DEBUG
13916                 for (i = 0; i < MMU_PAGE_SIZES; i++) {
13917                         ASSERT(rgnp->rgn_ttecnt[i] == 0);
13918                 }
13919                 rgns++;
13920 #endif /* DEBUG */
13921                 kmem_cache_free(region_cache, rgnp);
13922         }
13923         ASSERT(rgns == srdp->srd_next_hmerid);
13924 
13925 #ifdef DEBUG
13926         rgns = 0;
13927 #endif
13928         /* free each ism rgns in the srd */
13929         for (rgnp = srdp->srd_ismrgnfree; rgnp != NULL; rgnp = nrgnp) {
13930                 nrgnp = rgnp->rgn_next;
13931                 ASSERT(rgnp->rgn_id < srdp->srd_next_ismrid);
13932                 ASSERT(rgnp->rgn_refcnt == 0);
13933                 ASSERT(rgnp->rgn_sfmmu_head == NULL);
13934                 ASSERT(rgnp->rgn_flags & SFMMU_REGION_FREE);
13935                 ASSERT(srdp->srd_ismrgnp[rgnp->rgn_id] == rgnp);
13936 #ifdef DEBUG
13937                 for (i = 0; i < MMU_PAGE_SIZES; i++) {
13938                         ASSERT(rgnp->rgn_ttecnt[i] == 0);
13939                 }
13940                 rgns++;
13941 #endif /* DEBUG */
13942                 kmem_cache_free(region_cache, rgnp);
13943         }
13944         ASSERT(rgns == srdp->srd_next_ismrid);
13945         ASSERT(srdp->srd_ismbusyrgns == 0);
13946         ASSERT(srdp->srd_hmebusyrgns == 0);
13947 
13948         srdp->srd_next_ismrid = 0;
13949         srdp->srd_next_hmerid = 0;
13950 
13951         bzero((void *)srdp->srd_ismrgnp,
13952             sizeof (sf_region_t *) * SFMMU_MAX_ISM_REGIONS);
13953         bzero((void *)srdp->srd_hmergnp,
13954             sizeof (sf_region_t *) * SFMMU_MAX_HME_REGIONS);
13955 
13956         ASSERT(srdp->srd_scdp == NULL);
13957         kmem_cache_free(srd_cache, srdp);
13958 }
13959 
13960 /* ARGSUSED */
13961 static int
13962 sfmmu_srdcache_constructor(void *buf, void *cdrarg, int kmflags)
13963 {
13964         sf_srd_t *srdp = (sf_srd_t *)buf;
13965         bzero(buf, sizeof (*srdp));
13966 
13967         mutex_init(&srdp->srd_mutex, NULL, MUTEX_DEFAULT, NULL);
13968         mutex_init(&srdp->srd_scd_mutex, NULL, MUTEX_DEFAULT, NULL);
13969         return (0);
13970 }
13971 
13972 /* ARGSUSED */
13973 static void
13974 sfmmu_srdcache_destructor(void *buf, void *cdrarg)
13975 {
13976         sf_srd_t *srdp = (sf_srd_t *)buf;
13977 
13978         mutex_destroy(&srdp->srd_mutex);
13979         mutex_destroy(&srdp->srd_scd_mutex);
13980 }
13981 
13982 /*
13983  * The caller makes sure hat_join_region()/hat_leave_region() can't be called
13984  * at the same time for the same process and address range. This is ensured by
13985  * the fact that address space is locked as writer when a process joins the
13986  * regions. Therefore there's no need to hold an srd lock during the entire
13987  * execution of hat_join_region()/hat_leave_region().
13988  */
13989 
13990 #define RGN_HASH_FUNCTION(obj)  (((((uintptr_t)(obj)) >> 4) ^ \
13991                                     (((uintptr_t)(obj)) >> 11)) & \
13992                                         srd_rgn_hashmask)
13993 /*
13994  * This routine implements the shared context functionality required when
13995  * attaching a segment to an address space. It must be called from
13996  * hat_share() for D(ISM) segments and from segvn_create() for segments
13997  * with the MAP_PRIVATE and MAP_TEXT flags set. It returns a region_cookie
13998  * which is saved in the private segment data for hme segments and
13999  * the ism_map structure for ism segments.
14000  */
14001 hat_region_cookie_t
14002 hat_join_region(struct hat *sfmmup,
14003         caddr_t r_saddr,
14004         size_t r_size,
14005         void *r_obj,
14006         u_offset_t r_objoff,
14007         uchar_t r_perm,
14008         uchar_t r_pgszc,
14009         hat_rgn_cb_func_t r_cb_function,
14010         uint_t flags)
14011 {
14012         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14013         uint_t rhash;
14014         uint_t rid;
14015         hatlock_t *hatlockp;
14016         sf_region_t *rgnp;
14017         sf_region_t *new_rgnp = NULL;
14018         int i;
14019         uint16_t *nextidp;
14020         sf_region_t **freelistp;
14021         int maxids;
14022         sf_region_t **rarrp;
14023         uint16_t *busyrgnsp;
14024         ulong_t rttecnt;
14025         uchar_t tteflag;
14026         uchar_t r_type = flags & HAT_REGION_TYPE_MASK;
14027         int text = (r_type == HAT_REGION_TEXT);
14028 
14029         if (srdp == NULL || r_size == 0) {
14030                 return (HAT_INVALID_REGION_COOKIE);
14031         }
14032 
14033         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
14034         ASSERT(sfmmup != ksfmmup);
14035         ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
14036         ASSERT(srdp->srd_refcnt > 0);
14037         ASSERT(!(flags & ~HAT_REGION_TYPE_MASK));
14038         ASSERT(flags == HAT_REGION_TEXT || flags == HAT_REGION_ISM);
14039         ASSERT(r_pgszc < mmu_page_sizes);
14040         if (!IS_P2ALIGNED(r_saddr, TTEBYTES(r_pgszc)) ||
14041             !IS_P2ALIGNED(r_size, TTEBYTES(r_pgszc))) {
14042                 panic("hat_join_region: region addr or size is not aligned\n");
14043         }
14044 
14045 
14046         r_type = (r_type == HAT_REGION_ISM) ? SFMMU_REGION_ISM :
14047             SFMMU_REGION_HME;
14048         /*
14049          * Currently only support shared hmes for the read only main text
14050          * region.
14051          */
14052         if (r_type == SFMMU_REGION_HME && ((r_obj != srdp->srd_evp) ||
14053             (r_perm & PROT_WRITE))) {
14054                 return (HAT_INVALID_REGION_COOKIE);
14055         }
14056 
14057         rhash = RGN_HASH_FUNCTION(r_obj);
14058 
14059         if (r_type == SFMMU_REGION_ISM) {
14060                 nextidp = &srdp->srd_next_ismrid;
14061                 freelistp = &srdp->srd_ismrgnfree;
14062                 maxids = SFMMU_MAX_ISM_REGIONS;
14063                 rarrp = srdp->srd_ismrgnp;
14064                 busyrgnsp = &srdp->srd_ismbusyrgns;
14065         } else {
14066                 nextidp = &srdp->srd_next_hmerid;
14067                 freelistp = &srdp->srd_hmergnfree;
14068                 maxids = SFMMU_MAX_HME_REGIONS;
14069                 rarrp = srdp->srd_hmergnp;
14070                 busyrgnsp = &srdp->srd_hmebusyrgns;
14071         }
14072 
14073         mutex_enter(&srdp->srd_mutex);
14074 
14075         for (rgnp = srdp->srd_rgnhash[rhash]; rgnp != NULL;
14076             rgnp = rgnp->rgn_hash) {
14077                 if (rgnp->rgn_saddr == r_saddr && rgnp->rgn_size == r_size &&
14078                     rgnp->rgn_obj == r_obj && rgnp->rgn_objoff == r_objoff &&
14079                     rgnp->rgn_perm == r_perm && rgnp->rgn_pgszc == r_pgszc) {
14080                         break;
14081                 }
14082         }
14083 
14084 rfound:
14085         if (rgnp != NULL) {
14086                 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == r_type);
14087                 ASSERT(rgnp->rgn_cb_function == r_cb_function);
14088                 ASSERT(rgnp->rgn_refcnt >= 0);
14089                 rid = rgnp->rgn_id;
14090                 ASSERT(rid < maxids);
14091                 ASSERT(rarrp[rid] == rgnp);
14092                 ASSERT(rid < *nextidp);
14093                 atomic_add_32((volatile uint_t *)&rgnp->rgn_refcnt, 1);
14094                 mutex_exit(&srdp->srd_mutex);
14095                 if (new_rgnp != NULL) {
14096                         kmem_cache_free(region_cache, new_rgnp);
14097                 }
14098                 if (r_type == SFMMU_REGION_HME) {
14099                         int myjoin =
14100                             (sfmmup == astosfmmu(curthread->t_procp->p_as));
14101 
14102                         sfmmu_link_to_hmeregion(sfmmup, rgnp);
14103                         /*
14104                          * bitmap should be updated after linking sfmmu on
14105                          * region list so that pageunload() doesn't skip
14106                          * TSB/TLB flush. As soon as bitmap is updated another
14107                          * thread in this process can already start accessing
14108                          * this region.
14109                          */
14110                         /*
14111                          * Normally ttecnt accounting is done as part of
14112                          * pagefault handling. But a process may not take any
14113                          * pagefaults on shared hmeblks created by some other
14114                          * process. To compensate for this assume that the
14115                          * entire region will end up faulted in using
14116                          * the region's pagesize.
14117                          *
14118                          */
14119                         if (r_pgszc > TTE8K) {
14120                                 tteflag = 1 << r_pgszc;
14121                                 if (disable_large_pages & tteflag) {
14122                                         tteflag = 0;
14123                                 }
14124                         } else {
14125                                 tteflag = 0;
14126                         }
14127                         if (tteflag && !(sfmmup->sfmmu_rtteflags & tteflag)) {
14128                                 hatlockp = sfmmu_hat_enter(sfmmup);
14129                                 sfmmup->sfmmu_rtteflags |= tteflag;
14130                                 sfmmu_hat_exit(hatlockp);
14131                         }
14132                         hatlockp = sfmmu_hat_enter(sfmmup);
14133 
14134                         /*
14135                          * Preallocate 1/4 of ttecnt's in 8K TSB for >= 4M
14136                          * region to allow for large page allocation failure.
14137                          */
14138                         if (r_pgszc >= TTE4M) {
14139                                 sfmmup->sfmmu_tsb0_4minflcnt +=
14140                                     r_size >> (TTE_PAGE_SHIFT(TTE8K) + 2);
14141                         }
14142 
14143                         /* update sfmmu_ttecnt with the shme rgn ttecnt */
14144                         rttecnt = r_size >> TTE_PAGE_SHIFT(r_pgszc);
14145                         atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc],
14146                             rttecnt);
14147 
14148                         if (text && r_pgszc >= TTE4M &&
14149                             (tteflag || ((disable_large_pages >> TTE4M) &
14150                             ((1 << (r_pgszc - TTE4M + 1)) - 1))) &&
14151                             !SFMMU_FLAGS_ISSET(sfmmup, HAT_4MTEXT_FLAG)) {
14152                                 SFMMU_FLAGS_SET(sfmmup, HAT_4MTEXT_FLAG);
14153                         }
14154 
14155                         sfmmu_hat_exit(hatlockp);
14156                         /*
14157                          * On Panther we need to make sure TLB is programmed
14158                          * to accept 32M/256M pages.  Call
14159                          * sfmmu_check_page_sizes() now to make sure TLB is
14160                          * setup before making hmeregions visible to other
14161                          * threads.
14162                          */
14163                         sfmmu_check_page_sizes(sfmmup, 1);
14164                         hatlockp = sfmmu_hat_enter(sfmmup);
14165                         SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid);
14166 
14167                         /*
14168                          * if context is invalid tsb miss exception code will
14169                          * call sfmmu_check_page_sizes() and update tsbmiss
14170                          * area later.
14171                          */
14172                         kpreempt_disable();
14173                         if (myjoin &&
14174                             (sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum
14175                             != INVALID_CONTEXT)) {
14176                                 struct tsbmiss *tsbmp;
14177 
14178                                 tsbmp = &tsbmiss_area[CPU->cpu_id];
14179                                 ASSERT(sfmmup == tsbmp->usfmmup);
14180                                 BT_SET(tsbmp->shmermap, rid);
14181                                 if (r_pgszc > TTE64K) {
14182                                         tsbmp->uhat_rtteflags |= tteflag;
14183                                 }
14184 
14185                         }
14186                         kpreempt_enable();
14187 
14188                         sfmmu_hat_exit(hatlockp);
14189                         ASSERT((hat_region_cookie_t)((uint64_t)rid) !=
14190                             HAT_INVALID_REGION_COOKIE);
14191                 } else {
14192                         hatlockp = sfmmu_hat_enter(sfmmup);
14193                         SF_RGNMAP_ADD(sfmmup->sfmmu_ismregion_map, rid);
14194                         sfmmu_hat_exit(hatlockp);
14195                 }
14196                 ASSERT(rid < maxids);
14197 
14198                 if (r_type == SFMMU_REGION_ISM) {
14199                         sfmmu_find_scd(sfmmup);
14200                 }
14201                 return ((hat_region_cookie_t)((uint64_t)rid));
14202         }
14203 
14204         ASSERT(new_rgnp == NULL);
14205 
14206         if (*busyrgnsp >= maxids) {
14207                 mutex_exit(&srdp->srd_mutex);
14208                 return (HAT_INVALID_REGION_COOKIE);
14209         }
14210 
14211         ASSERT(MUTEX_HELD(&srdp->srd_mutex));
14212         if (*freelistp != NULL) {
14213                 rgnp = *freelistp;
14214                 *freelistp = rgnp->rgn_next;
14215                 ASSERT(rgnp->rgn_id < *nextidp);
14216                 ASSERT(rgnp->rgn_id < maxids);
14217                 ASSERT(rgnp->rgn_flags & SFMMU_REGION_FREE);
14218                 ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK)
14219                     == r_type);
14220                 ASSERT(rarrp[rgnp->rgn_id] == rgnp);
14221                 ASSERT(rgnp->rgn_hmeflags == 0);
14222         } else {
14223                 /*
14224                  * release local locks before memory allocation.
14225                  */
14226                 mutex_exit(&srdp->srd_mutex);
14227 
14228                 new_rgnp = kmem_cache_alloc(region_cache, KM_SLEEP);
14229 
14230                 mutex_enter(&srdp->srd_mutex);
14231                 for (rgnp = srdp->srd_rgnhash[rhash]; rgnp != NULL;
14232                     rgnp = rgnp->rgn_hash) {
14233                         if (rgnp->rgn_saddr == r_saddr &&
14234                             rgnp->rgn_size == r_size &&
14235                             rgnp->rgn_obj == r_obj &&
14236                             rgnp->rgn_objoff == r_objoff &&
14237                             rgnp->rgn_perm == r_perm &&
14238                             rgnp->rgn_pgszc == r_pgszc) {
14239                                 break;
14240                         }
14241                 }
14242                 if (rgnp != NULL) {
14243                         goto rfound;
14244                 }
14245 
14246                 if (*nextidp >= maxids) {
14247                         mutex_exit(&srdp->srd_mutex);
14248                         goto fail;
14249                 }
14250                 rgnp = new_rgnp;
14251                 new_rgnp = NULL;
14252                 rgnp->rgn_id = (*nextidp)++;
14253                 ASSERT(rgnp->rgn_id < maxids);
14254                 ASSERT(rarrp[rgnp->rgn_id] == NULL);
14255                 rarrp[rgnp->rgn_id] = rgnp;
14256         }
14257 
14258         ASSERT(rgnp->rgn_sfmmu_head == NULL);
14259         ASSERT(rgnp->rgn_hmeflags == 0);
14260 #ifdef DEBUG
14261         for (i = 0; i < MMU_PAGE_SIZES; i++) {
14262                 ASSERT(rgnp->rgn_ttecnt[i] == 0);
14263         }
14264 #endif
14265         rgnp->rgn_saddr = r_saddr;
14266         rgnp->rgn_size = r_size;
14267         rgnp->rgn_obj = r_obj;
14268         rgnp->rgn_objoff = r_objoff;
14269         rgnp->rgn_perm = r_perm;
14270         rgnp->rgn_pgszc = r_pgszc;
14271         rgnp->rgn_flags = r_type;
14272         rgnp->rgn_refcnt = 0;
14273         rgnp->rgn_cb_function = r_cb_function;
14274         rgnp->rgn_hash = srdp->srd_rgnhash[rhash];
14275         srdp->srd_rgnhash[rhash] = rgnp;
14276         (*busyrgnsp)++;
14277         ASSERT(*busyrgnsp <= maxids);
14278         goto rfound;
14279 
14280 fail:
14281         ASSERT(new_rgnp != NULL);
14282         kmem_cache_free(region_cache, new_rgnp);
14283         return (HAT_INVALID_REGION_COOKIE);
14284 }
14285 
14286 /*
14287  * This function implements the shared context functionality required
14288  * when detaching a segment from an address space. It must be called
14289  * from hat_unshare() for all D(ISM) segments and from segvn_unmap(),
14290  * for segments with a valid region_cookie.
14291  * It will also be called from all seg_vn routines which change a
14292  * segment's attributes such as segvn_setprot(), segvn_setpagesize(),
14293  * segvn_clrszc() & segvn_advise(), as well as in the case of COW fault
14294  * from segvn_fault().
14295  */
14296 void
14297 hat_leave_region(struct hat *sfmmup, hat_region_cookie_t rcookie, uint_t flags)
14298 {
14299         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14300         sf_scd_t *scdp;
14301         uint_t rhash;
14302         uint_t rid = (uint_t)((uint64_t)rcookie);
14303         hatlock_t *hatlockp = NULL;
14304         sf_region_t *rgnp;
14305         sf_region_t **prev_rgnpp;
14306         sf_region_t *cur_rgnp;
14307         void *r_obj;
14308         int i;
14309         caddr_t r_saddr;
14310         caddr_t r_eaddr;
14311         size_t  r_size;
14312         uchar_t r_pgszc;
14313         uchar_t r_type = flags & HAT_REGION_TYPE_MASK;
14314 
14315         ASSERT(sfmmup != ksfmmup);
14316         ASSERT(srdp != NULL);
14317         ASSERT(srdp->srd_refcnt > 0);
14318         ASSERT(!(flags & ~HAT_REGION_TYPE_MASK));
14319         ASSERT(flags == HAT_REGION_TEXT || flags == HAT_REGION_ISM);
14320         ASSERT(!sfmmup->sfmmu_free || sfmmup->sfmmu_scdp == NULL);
14321 
14322         r_type = (r_type == HAT_REGION_ISM) ? SFMMU_REGION_ISM :
14323             SFMMU_REGION_HME;
14324 
14325         if (r_type == SFMMU_REGION_ISM) {
14326                 ASSERT(SFMMU_IS_ISMRID_VALID(rid));
14327                 ASSERT(rid < SFMMU_MAX_ISM_REGIONS);
14328                 rgnp = srdp->srd_ismrgnp[rid];
14329         } else {
14330                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
14331                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
14332                 rgnp = srdp->srd_hmergnp[rid];
14333         }
14334         ASSERT(rgnp != NULL);
14335         ASSERT(rgnp->rgn_id == rid);
14336         ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == r_type);
14337         ASSERT(!(rgnp->rgn_flags & SFMMU_REGION_FREE));
14338         ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
14339 
14340         ASSERT(sfmmup->sfmmu_xhat_provider == NULL);
14341         if (r_type == SFMMU_REGION_HME && sfmmup->sfmmu_as->a_xhat != NULL) {
14342                 xhat_unload_callback_all(sfmmup->sfmmu_as, rgnp->rgn_saddr,
14343                     rgnp->rgn_size, 0, NULL);
14344         }
14345 
14346         if (sfmmup->sfmmu_free) {
14347                 ulong_t rttecnt;
14348                 r_pgszc = rgnp->rgn_pgszc;
14349                 r_size = rgnp->rgn_size;
14350 
14351                 ASSERT(sfmmup->sfmmu_scdp == NULL);
14352                 if (r_type == SFMMU_REGION_ISM) {
14353                         SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid);
14354                 } else {
14355                         /* update shme rgns ttecnt in sfmmu_ttecnt */
14356                         rttecnt = r_size >> TTE_PAGE_SHIFT(r_pgszc);
14357                         ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt);
14358 
14359                         atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc],
14360                             -rttecnt);
14361 
14362                         SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid);
14363                 }
14364         } else if (r_type == SFMMU_REGION_ISM) {
14365                 hatlockp = sfmmu_hat_enter(sfmmup);
14366                 ASSERT(rid < srdp->srd_next_ismrid);
14367                 SF_RGNMAP_DEL(sfmmup->sfmmu_ismregion_map, rid);
14368                 scdp = sfmmup->sfmmu_scdp;
14369                 if (scdp != NULL &&
14370                     SF_RGNMAP_TEST(scdp->scd_ismregion_map, rid)) {
14371                         sfmmu_leave_scd(sfmmup, r_type);
14372                         ASSERT(sfmmu_hat_lock_held(sfmmup));
14373                 }
14374                 sfmmu_hat_exit(hatlockp);
14375         } else {
14376                 ulong_t rttecnt;
14377                 r_pgszc = rgnp->rgn_pgszc;
14378                 r_saddr = rgnp->rgn_saddr;
14379                 r_size = rgnp->rgn_size;
14380                 r_eaddr = r_saddr + r_size;
14381 
14382                 ASSERT(r_type == SFMMU_REGION_HME);
14383                 hatlockp = sfmmu_hat_enter(sfmmup);
14384                 ASSERT(rid < srdp->srd_next_hmerid);
14385                 SF_RGNMAP_DEL(sfmmup->sfmmu_hmeregion_map, rid);
14386 
14387                 /*
14388                  * If region is part of an SCD call sfmmu_leave_scd().
14389                  * Otherwise if process is not exiting and has valid context
14390                  * just drop the context on the floor to lose stale TLB
14391                  * entries and force the update of tsb miss area to reflect
14392                  * the new region map. After that clean our TSB entries.
14393                  */
14394                 scdp = sfmmup->sfmmu_scdp;
14395                 if (scdp != NULL &&
14396                     SF_RGNMAP_TEST(scdp->scd_hmeregion_map, rid)) {
14397                         sfmmu_leave_scd(sfmmup, r_type);
14398                         ASSERT(sfmmu_hat_lock_held(sfmmup));
14399                 }
14400                 sfmmu_invalidate_ctx(sfmmup);
14401 
14402                 i = TTE8K;
14403                 while (i < mmu_page_sizes) {
14404                         if (rgnp->rgn_ttecnt[i] != 0) {
14405                                 sfmmu_unload_tsb_range(sfmmup, r_saddr,
14406                                     r_eaddr, i);
14407                                 if (i < TTE4M) {
14408                                         i = TTE4M;
14409                                         continue;
14410                                 } else {
14411                                         break;
14412                                 }
14413                         }
14414                         i++;
14415                 }
14416                 /* Remove the preallocated 1/4 8k ttecnt for 4M regions. */
14417                 if (r_pgszc >= TTE4M) {
14418                         rttecnt = r_size >> (TTE_PAGE_SHIFT(TTE8K) + 2);
14419                         ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >=
14420                             rttecnt);
14421                         sfmmup->sfmmu_tsb0_4minflcnt -= rttecnt;
14422                 }
14423 
14424                 /* update shme rgns ttecnt in sfmmu_ttecnt */
14425                 rttecnt = r_size >> TTE_PAGE_SHIFT(r_pgszc);
14426                 ASSERT(sfmmup->sfmmu_ttecnt[r_pgszc] >= rttecnt);
14427                 atomic_add_long(&sfmmup->sfmmu_ttecnt[r_pgszc], -rttecnt);
14428 
14429                 sfmmu_hat_exit(hatlockp);
14430                 if (scdp != NULL && sfmmup->sfmmu_scdp == NULL) {
14431                         /* sfmmup left the scd, grow private tsb */
14432                         sfmmu_check_page_sizes(sfmmup, 1);
14433                 } else {
14434                         sfmmu_check_page_sizes(sfmmup, 0);
14435                 }
14436         }
14437 
14438         if (r_type == SFMMU_REGION_HME) {
14439                 sfmmu_unlink_from_hmeregion(sfmmup, rgnp);
14440         }
14441 
14442         r_obj = rgnp->rgn_obj;
14443         if (atomic_add_32_nv((volatile uint_t *)&rgnp->rgn_refcnt, -1)) {
14444                 return;
14445         }
14446 
14447         /*
14448          * looks like nobody uses this region anymore. Free it.
14449          */
14450         rhash = RGN_HASH_FUNCTION(r_obj);
14451         mutex_enter(&srdp->srd_mutex);
14452         for (prev_rgnpp = &srdp->srd_rgnhash[rhash];
14453             (cur_rgnp = *prev_rgnpp) != NULL;
14454             prev_rgnpp = &cur_rgnp->rgn_hash) {
14455                 if (cur_rgnp == rgnp && cur_rgnp->rgn_refcnt == 0) {
14456                         break;
14457                 }
14458         }
14459 
14460         if (cur_rgnp == NULL) {
14461                 mutex_exit(&srdp->srd_mutex);
14462                 return;
14463         }
14464 
14465         ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == r_type);
14466         *prev_rgnpp = rgnp->rgn_hash;
14467         if (r_type == SFMMU_REGION_ISM) {
14468                 rgnp->rgn_flags |= SFMMU_REGION_FREE;
14469                 ASSERT(rid < srdp->srd_next_ismrid);
14470                 rgnp->rgn_next = srdp->srd_ismrgnfree;
14471                 srdp->srd_ismrgnfree = rgnp;
14472                 ASSERT(srdp->srd_ismbusyrgns > 0);
14473                 srdp->srd_ismbusyrgns--;
14474                 mutex_exit(&srdp->srd_mutex);
14475                 return;
14476         }
14477         mutex_exit(&srdp->srd_mutex);
14478 
14479         /*
14480          * Destroy region's hmeblks.
14481          */
14482         sfmmu_unload_hmeregion(srdp, rgnp);
14483 
14484         rgnp->rgn_hmeflags = 0;
14485 
14486         ASSERT(rgnp->rgn_sfmmu_head == NULL);
14487         ASSERT(rgnp->rgn_id == rid);
14488         for (i = 0; i < MMU_PAGE_SIZES; i++) {
14489                 rgnp->rgn_ttecnt[i] = 0;
14490         }
14491         rgnp->rgn_flags |= SFMMU_REGION_FREE;
14492         mutex_enter(&srdp->srd_mutex);
14493         ASSERT(rid < srdp->srd_next_hmerid);
14494         rgnp->rgn_next = srdp->srd_hmergnfree;
14495         srdp->srd_hmergnfree = rgnp;
14496         ASSERT(srdp->srd_hmebusyrgns > 0);
14497         srdp->srd_hmebusyrgns--;
14498         mutex_exit(&srdp->srd_mutex);
14499 }
14500 
14501 /*
14502  * For now only called for hmeblk regions and not for ISM regions.
14503  */
14504 void
14505 hat_dup_region(struct hat *sfmmup, hat_region_cookie_t rcookie)
14506 {
14507         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14508         uint_t rid = (uint_t)((uint64_t)rcookie);
14509         sf_region_t *rgnp;
14510         sf_rgn_link_t *rlink;
14511         sf_rgn_link_t *hrlink;
14512         ulong_t rttecnt;
14513 
14514         ASSERT(sfmmup != ksfmmup);
14515         ASSERT(srdp != NULL);
14516         ASSERT(srdp->srd_refcnt > 0);
14517 
14518         ASSERT(rid < srdp->srd_next_hmerid);
14519         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
14520         ASSERT(rid < SFMMU_MAX_HME_REGIONS);
14521 
14522         rgnp = srdp->srd_hmergnp[rid];
14523         ASSERT(rgnp->rgn_refcnt > 0);
14524         ASSERT(rgnp->rgn_id == rid);
14525         ASSERT((rgnp->rgn_flags & SFMMU_REGION_TYPE_MASK) == SFMMU_REGION_HME);
14526         ASSERT(!(rgnp->rgn_flags & SFMMU_REGION_FREE));
14527 
14528         atomic_add_32((volatile uint_t *)&rgnp->rgn_refcnt, 1);
14529 
14530         /* LINTED: constant in conditional context */
14531         SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 0);
14532         ASSERT(rlink != NULL);
14533         mutex_enter(&rgnp->rgn_mutex);
14534         ASSERT(rgnp->rgn_sfmmu_head != NULL);
14535         /* LINTED: constant in conditional context */
14536         SFMMU_HMERID2RLINKP(rgnp->rgn_sfmmu_head, rid, hrlink, 0, 0);
14537         ASSERT(hrlink != NULL);
14538         ASSERT(hrlink->prev == NULL);
14539         rlink->next = rgnp->rgn_sfmmu_head;
14540         rlink->prev = NULL;
14541         hrlink->prev = sfmmup;
14542         /*
14543          * make sure rlink's next field is correct
14544          * before making this link visible.
14545          */
14546         membar_stst();
14547         rgnp->rgn_sfmmu_head = sfmmup;
14548         mutex_exit(&rgnp->rgn_mutex);
14549 
14550         /* update sfmmu_ttecnt with the shme rgn ttecnt */
14551         rttecnt = rgnp->rgn_size >> TTE_PAGE_SHIFT(rgnp->rgn_pgszc);
14552         atomic_add_long(&sfmmup->sfmmu_ttecnt[rgnp->rgn_pgszc], rttecnt);
14553         /* update tsb0 inflation count */
14554         if (rgnp->rgn_pgszc >= TTE4M) {
14555                 sfmmup->sfmmu_tsb0_4minflcnt +=
14556                     rgnp->rgn_size >> (TTE_PAGE_SHIFT(TTE8K) + 2);
14557         }
14558         /*
14559          * Update regionid bitmask without hat lock since no other thread
14560          * can update this region bitmask right now.
14561          */
14562         SF_RGNMAP_ADD(sfmmup->sfmmu_hmeregion_map, rid);
14563 }
14564 
14565 /* ARGSUSED */
14566 static int
14567 sfmmu_rgncache_constructor(void *buf, void *cdrarg, int kmflags)
14568 {
14569         sf_region_t *rgnp = (sf_region_t *)buf;
14570         bzero(buf, sizeof (*rgnp));
14571 
14572         mutex_init(&rgnp->rgn_mutex, NULL, MUTEX_DEFAULT, NULL);
14573 
14574         return (0);
14575 }
14576 
14577 /* ARGSUSED */
14578 static void
14579 sfmmu_rgncache_destructor(void *buf, void *cdrarg)
14580 {
14581         sf_region_t *rgnp = (sf_region_t *)buf;
14582         mutex_destroy(&rgnp->rgn_mutex);
14583 }
14584 
14585 static int
14586 sfrgnmap_isnull(sf_region_map_t *map)
14587 {
14588         int i;
14589 
14590         for (i = 0; i < SFMMU_RGNMAP_WORDS; i++) {
14591                 if (map->bitmap[i] != 0) {
14592                         return (0);
14593                 }
14594         }
14595         return (1);
14596 }
14597 
14598 static int
14599 sfhmergnmap_isnull(sf_hmeregion_map_t *map)
14600 {
14601         int i;
14602 
14603         for (i = 0; i < SFMMU_HMERGNMAP_WORDS; i++) {
14604                 if (map->bitmap[i] != 0) {
14605                         return (0);
14606                 }
14607         }
14608         return (1);
14609 }
14610 
14611 #ifdef DEBUG
14612 static void
14613 check_scd_sfmmu_list(sfmmu_t **headp, sfmmu_t *sfmmup, int onlist)
14614 {
14615         sfmmu_t *sp;
14616         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
14617 
14618         for (sp = *headp; sp != NULL; sp = sp->sfmmu_scd_link.next) {
14619                 ASSERT(srdp == sp->sfmmu_srdp);
14620                 if (sp == sfmmup) {
14621                         if (onlist) {
14622                                 return;
14623                         } else {
14624                                 panic("shctx: sfmmu 0x%p found on scd"
14625                                     "list 0x%p", (void *)sfmmup,
14626                                     (void *)*headp);
14627                         }
14628                 }
14629         }
14630         if (onlist) {
14631                 panic("shctx: sfmmu 0x%p not found on scd list 0x%p",
14632                     (void *)sfmmup, (void *)*headp);
14633         } else {
14634                 return;
14635         }
14636 }
14637 #else /* DEBUG */
14638 #define check_scd_sfmmu_list(headp, sfmmup, onlist)
14639 #endif /* DEBUG */
14640 
14641 /*
14642  * Removes an sfmmu from the SCD sfmmu list.
14643  */
14644 static void
14645 sfmmu_from_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup)
14646 {
14647         ASSERT(sfmmup->sfmmu_srdp != NULL);
14648         check_scd_sfmmu_list(headp, sfmmup, 1);
14649         if (sfmmup->sfmmu_scd_link.prev != NULL) {
14650                 ASSERT(*headp != sfmmup);
14651                 sfmmup->sfmmu_scd_link.prev->sfmmu_scd_link.next =
14652                     sfmmup->sfmmu_scd_link.next;
14653         } else {
14654                 ASSERT(*headp == sfmmup);
14655                 *headp = sfmmup->sfmmu_scd_link.next;
14656         }
14657         if (sfmmup->sfmmu_scd_link.next != NULL) {
14658                 sfmmup->sfmmu_scd_link.next->sfmmu_scd_link.prev =
14659                     sfmmup->sfmmu_scd_link.prev;
14660         }
14661 }
14662 
14663 
14664 /*
14665  * Adds an sfmmu to the start of the queue.
14666  */
14667 static void
14668 sfmmu_to_scd_list(sfmmu_t **headp, sfmmu_t *sfmmup)
14669 {
14670         check_scd_sfmmu_list(headp, sfmmup, 0);
14671         sfmmup->sfmmu_scd_link.prev = NULL;
14672         sfmmup->sfmmu_scd_link.next = *headp;
14673         if (*headp != NULL)
14674                 (*headp)->sfmmu_scd_link.prev = sfmmup;
14675         *headp = sfmmup;
14676 }
14677 
14678 /*
14679  * Remove an scd from the start of the queue.
14680  */
14681 static void
14682 sfmmu_remove_scd(sf_scd_t **headp, sf_scd_t *scdp)
14683 {
14684         if (scdp->scd_prev != NULL) {
14685                 ASSERT(*headp != scdp);
14686                 scdp->scd_prev->scd_next = scdp->scd_next;
14687         } else {
14688                 ASSERT(*headp == scdp);
14689                 *headp = scdp->scd_next;
14690         }
14691 
14692         if (scdp->scd_next != NULL) {
14693                 scdp->scd_next->scd_prev = scdp->scd_prev;
14694         }
14695 }
14696 
14697 /*
14698  * Add an scd to the start of the queue.
14699  */
14700 static void
14701 sfmmu_add_scd(sf_scd_t **headp, sf_scd_t *scdp)
14702 {
14703         scdp->scd_prev = NULL;
14704         scdp->scd_next = *headp;
14705         if (*headp != NULL) {
14706                 (*headp)->scd_prev = scdp;
14707         }
14708         *headp = scdp;
14709 }
14710 
14711 static int
14712 sfmmu_alloc_scd_tsbs(sf_srd_t *srdp, sf_scd_t *scdp)
14713 {
14714         uint_t rid;
14715         uint_t i;
14716         uint_t j;
14717         ulong_t w;
14718         sf_region_t *rgnp;
14719         ulong_t tte8k_cnt = 0;
14720         ulong_t tte4m_cnt = 0;
14721         uint_t tsb_szc;
14722         sfmmu_t *scsfmmup = scdp->scd_sfmmup;
14723         sfmmu_t *ism_hatid;
14724         struct tsb_info *newtsb;
14725         int szc;
14726 
14727         ASSERT(srdp != NULL);
14728 
14729         for (i = 0; i < SFMMU_RGNMAP_WORDS; i++) {
14730                 if ((w = scdp->scd_region_map.bitmap[i]) == 0) {
14731                         continue;
14732                 }
14733                 j = 0;
14734                 while (w) {
14735                         if (!(w & 0x1)) {
14736                                 j++;
14737                                 w >>= 1;
14738                                 continue;
14739                         }
14740                         rid = (i << BT_ULSHIFT) | j;
14741                         j++;
14742                         w >>= 1;
14743 
14744                         if (rid < SFMMU_MAX_HME_REGIONS) {
14745                                 rgnp = srdp->srd_hmergnp[rid];
14746                                 ASSERT(rgnp->rgn_id == rid);
14747                                 ASSERT(rgnp->rgn_refcnt > 0);
14748 
14749                                 if (rgnp->rgn_pgszc < TTE4M) {
14750                                         tte8k_cnt += rgnp->rgn_size >>
14751                                             TTE_PAGE_SHIFT(TTE8K);
14752                                 } else {
14753                                         ASSERT(rgnp->rgn_pgszc >= TTE4M);
14754                                         tte4m_cnt += rgnp->rgn_size >>
14755                                             TTE_PAGE_SHIFT(TTE4M);
14756                                         /*
14757                                          * Inflate SCD tsb0 by preallocating
14758                                          * 1/4 8k ttecnt for 4M regions to
14759                                          * allow for lgpg alloc failure.
14760                                          */
14761                                         tte8k_cnt += rgnp->rgn_size >>
14762                                             (TTE_PAGE_SHIFT(TTE8K) + 2);
14763                                 }
14764                         } else {
14765                                 rid -= SFMMU_MAX_HME_REGIONS;
14766                                 rgnp = srdp->srd_ismrgnp[rid];
14767                                 ASSERT(rgnp->rgn_id == rid);
14768                                 ASSERT(rgnp->rgn_refcnt > 0);
14769 
14770                                 ism_hatid = (sfmmu_t *)rgnp->rgn_obj;
14771                                 ASSERT(ism_hatid->sfmmu_ismhat);
14772 
14773                                 for (szc = 0; szc < TTE4M; szc++) {
14774                                         tte8k_cnt +=
14775                                             ism_hatid->sfmmu_ttecnt[szc] <<
14776                                             TTE_BSZS_SHIFT(szc);
14777                                 }
14778 
14779                                 ASSERT(rgnp->rgn_pgszc >= TTE4M);
14780                                 if (rgnp->rgn_pgszc >= TTE4M) {
14781                                         tte4m_cnt += rgnp->rgn_size >>
14782                                             TTE_PAGE_SHIFT(TTE4M);
14783                                 }
14784                         }
14785                 }
14786         }
14787 
14788         tsb_szc = SELECT_TSB_SIZECODE(tte8k_cnt);
14789 
14790         /* Allocate both the SCD TSBs here. */
14791         if (sfmmu_tsbinfo_alloc(&scsfmmup->sfmmu_tsb,
14792             tsb_szc, TSB8K|TSB64K|TSB512K, TSB_ALLOC, scsfmmup) &&
14793             (tsb_szc <= TSB_4M_SZCODE ||
14794             sfmmu_tsbinfo_alloc(&scsfmmup->sfmmu_tsb,
14795             TSB_4M_SZCODE, TSB8K|TSB64K|TSB512K,
14796             TSB_ALLOC, scsfmmup))) {
14797 
14798                 SFMMU_STAT(sf_scd_1sttsb_allocfail);
14799                 return (TSB_ALLOCFAIL);
14800         } else {
14801                 scsfmmup->sfmmu_tsb->tsb_flags |= TSB_SHAREDCTX;
14802 
14803                 if (tte4m_cnt) {
14804                         tsb_szc = SELECT_TSB_SIZECODE(tte4m_cnt);
14805                         if (sfmmu_tsbinfo_alloc(&newtsb, tsb_szc,
14806                             TSB4M|TSB32M|TSB256M, TSB_ALLOC, scsfmmup) &&
14807                             (tsb_szc <= TSB_4M_SZCODE ||
14808                             sfmmu_tsbinfo_alloc(&newtsb, TSB_4M_SZCODE,
14809                             TSB4M|TSB32M|TSB256M,
14810                             TSB_ALLOC, scsfmmup))) {
14811                                 /*
14812                                  * If we fail to allocate the 2nd shared tsb,
14813                                  * just free the 1st tsb, return failure.
14814                                  */
14815                                 sfmmu_tsbinfo_free(scsfmmup->sfmmu_tsb);
14816                                 SFMMU_STAT(sf_scd_2ndtsb_allocfail);
14817                                 return (TSB_ALLOCFAIL);
14818                         } else {
14819                                 ASSERT(scsfmmup->sfmmu_tsb->tsb_next == NULL);
14820                                 newtsb->tsb_flags |= TSB_SHAREDCTX;
14821                                 scsfmmup->sfmmu_tsb->tsb_next = newtsb;
14822                                 SFMMU_STAT(sf_scd_2ndtsb_alloc);
14823                         }
14824                 }
14825                 SFMMU_STAT(sf_scd_1sttsb_alloc);
14826         }
14827         return (TSB_SUCCESS);
14828 }
14829 
14830 static void
14831 sfmmu_free_scd_tsbs(sfmmu_t *scd_sfmmu)
14832 {
14833         while (scd_sfmmu->sfmmu_tsb != NULL) {
14834                 struct tsb_info *next = scd_sfmmu->sfmmu_tsb->tsb_next;
14835                 sfmmu_tsbinfo_free(scd_sfmmu->sfmmu_tsb);
14836                 scd_sfmmu->sfmmu_tsb = next;
14837         }
14838 }
14839 
14840 /*
14841  * Link the sfmmu onto the hme region list.
14842  */
14843 void
14844 sfmmu_link_to_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14845 {
14846         uint_t rid;
14847         sf_rgn_link_t *rlink;
14848         sfmmu_t *head;
14849         sf_rgn_link_t *hrlink;
14850 
14851         rid = rgnp->rgn_id;
14852         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
14853 
14854         /* LINTED: constant in conditional context */
14855         SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 1, 1);
14856         ASSERT(rlink != NULL);
14857         mutex_enter(&rgnp->rgn_mutex);
14858         if ((head = rgnp->rgn_sfmmu_head) == NULL) {
14859                 rlink->next = NULL;
14860                 rlink->prev = NULL;
14861                 /*
14862                  * make sure rlink's next field is NULL
14863                  * before making this link visible.
14864                  */
14865                 membar_stst();
14866                 rgnp->rgn_sfmmu_head = sfmmup;
14867         } else {
14868                 /* LINTED: constant in conditional context */
14869                 SFMMU_HMERID2RLINKP(head, rid, hrlink, 0, 0);
14870                 ASSERT(hrlink != NULL);
14871                 ASSERT(hrlink->prev == NULL);
14872                 rlink->next = head;
14873                 rlink->prev = NULL;
14874                 hrlink->prev = sfmmup;
14875                 /*
14876                  * make sure rlink's next field is correct
14877                  * before making this link visible.
14878                  */
14879                 membar_stst();
14880                 rgnp->rgn_sfmmu_head = sfmmup;
14881         }
14882         mutex_exit(&rgnp->rgn_mutex);
14883 }
14884 
14885 /*
14886  * Unlink the sfmmu from the hme region list.
14887  */
14888 void
14889 sfmmu_unlink_from_hmeregion(sfmmu_t *sfmmup, sf_region_t *rgnp)
14890 {
14891         uint_t rid;
14892         sf_rgn_link_t *rlink;
14893 
14894         rid = rgnp->rgn_id;
14895         ASSERT(SFMMU_IS_SHMERID_VALID(rid));
14896 
14897         /* LINTED: constant in conditional context */
14898         SFMMU_HMERID2RLINKP(sfmmup, rid, rlink, 0, 0);
14899         ASSERT(rlink != NULL);
14900         mutex_enter(&rgnp->rgn_mutex);
14901         if (rgnp->rgn_sfmmu_head == sfmmup) {
14902                 sfmmu_t *next = rlink->next;
14903                 rgnp->rgn_sfmmu_head = next;
14904                 /*
14905                  * if we are stopped by xc_attention() after this
14906                  * point the forward link walking in
14907                  * sfmmu_rgntlb_demap() will work correctly since the
14908                  * head correctly points to the next element.
14909                  */
14910                 membar_stst();
14911                 rlink->next = NULL;
14912                 ASSERT(rlink->prev == NULL);
14913                 if (next != NULL) {
14914                         sf_rgn_link_t *nrlink;
14915                         /* LINTED: constant in conditional context */
14916                         SFMMU_HMERID2RLINKP(next, rid, nrlink, 0, 0);
14917                         ASSERT(nrlink != NULL);
14918                         ASSERT(nrlink->prev == sfmmup);
14919                         nrlink->prev = NULL;
14920                 }
14921         } else {
14922                 sfmmu_t *next = rlink->next;
14923                 sfmmu_t *prev = rlink->prev;
14924                 sf_rgn_link_t *prlink;
14925 
14926                 ASSERT(prev != NULL);
14927                 /* LINTED: constant in conditional context */
14928                 SFMMU_HMERID2RLINKP(prev, rid, prlink, 0, 0);
14929                 ASSERT(prlink != NULL);
14930                 ASSERT(prlink->next == sfmmup);
14931                 prlink->next = next;
14932                 /*
14933                  * if we are stopped by xc_attention()
14934                  * after this point the forward link walking
14935                  * will work correctly since the prev element
14936                  * correctly points to the next element.
14937                  */
14938                 membar_stst();
14939                 rlink->next = NULL;
14940                 rlink->prev = NULL;
14941                 if (next != NULL) {
14942                         sf_rgn_link_t *nrlink;
14943                         /* LINTED: constant in conditional context */
14944                         SFMMU_HMERID2RLINKP(next, rid, nrlink, 0, 0);
14945                         ASSERT(nrlink != NULL);
14946                         ASSERT(nrlink->prev == sfmmup);
14947                         nrlink->prev = prev;
14948                 }
14949         }
14950         mutex_exit(&rgnp->rgn_mutex);
14951 }
14952 
14953 /*
14954  * Link scd sfmmu onto ism or hme region list for each region in the
14955  * scd region map.
14956  */
14957 void
14958 sfmmu_link_scd_to_regions(sf_srd_t *srdp, sf_scd_t *scdp)
14959 {
14960         uint_t rid;
14961         uint_t i;
14962         uint_t j;
14963         ulong_t w;
14964         sf_region_t *rgnp;
14965         sfmmu_t *scsfmmup;
14966 
14967         scsfmmup = scdp->scd_sfmmup;
14968         ASSERT(scsfmmup->sfmmu_scdhat);
14969         for (i = 0; i < SFMMU_RGNMAP_WORDS; i++) {
14970                 if ((w = scdp->scd_region_map.bitmap[i]) == 0) {
14971                         continue;
14972                 }
14973                 j = 0;
14974                 while (w) {
14975                         if (!(w & 0x1)) {
14976                                 j++;
14977                                 w >>= 1;
14978                                 continue;
14979                         }
14980                         rid = (i << BT_ULSHIFT) | j;
14981                         j++;
14982                         w >>= 1;
14983 
14984                         if (rid < SFMMU_MAX_HME_REGIONS) {
14985                                 rgnp = srdp->srd_hmergnp[rid];
14986                                 ASSERT(rgnp->rgn_id == rid);
14987                                 ASSERT(rgnp->rgn_refcnt > 0);
14988                                 sfmmu_link_to_hmeregion(scsfmmup, rgnp);
14989                         } else {
14990                                 sfmmu_t *ism_hatid = NULL;
14991                                 ism_ment_t *ism_ment;
14992                                 rid -= SFMMU_MAX_HME_REGIONS;
14993                                 rgnp = srdp->srd_ismrgnp[rid];
14994                                 ASSERT(rgnp->rgn_id == rid);
14995                                 ASSERT(rgnp->rgn_refcnt > 0);
14996 
14997                                 ism_hatid = (sfmmu_t *)rgnp->rgn_obj;
14998                                 ASSERT(ism_hatid->sfmmu_ismhat);
14999                                 ism_ment = &scdp->scd_ism_links[rid];
15000                                 ism_ment->iment_hat = scsfmmup;
15001                                 ism_ment->iment_base_va = rgnp->rgn_saddr;
15002                                 mutex_enter(&ism_mlist_lock);
15003                                 iment_add(ism_ment, ism_hatid);
15004                                 mutex_exit(&ism_mlist_lock);
15005 
15006                         }
15007                 }
15008         }
15009 }
15010 /*
15011  * Unlink scd sfmmu from ism or hme region list for each region in the
15012  * scd region map.
15013  */
15014 void
15015 sfmmu_unlink_scd_from_regions(sf_srd_t *srdp, sf_scd_t *scdp)
15016 {
15017         uint_t rid;
15018         uint_t i;
15019         uint_t j;
15020         ulong_t w;
15021         sf_region_t *rgnp;
15022         sfmmu_t *scsfmmup;
15023 
15024         scsfmmup = scdp->scd_sfmmup;
15025         for (i = 0; i < SFMMU_RGNMAP_WORDS; i++) {
15026                 if ((w = scdp->scd_region_map.bitmap[i]) == 0) {
15027                         continue;
15028                 }
15029                 j = 0;
15030                 while (w) {
15031                         if (!(w & 0x1)) {
15032                                 j++;
15033                                 w >>= 1;
15034                                 continue;
15035                         }
15036                         rid = (i << BT_ULSHIFT) | j;
15037                         j++;
15038                         w >>= 1;
15039 
15040                         if (rid < SFMMU_MAX_HME_REGIONS) {
15041                                 rgnp = srdp->srd_hmergnp[rid];
15042                                 ASSERT(rgnp->rgn_id == rid);
15043                                 ASSERT(rgnp->rgn_refcnt > 0);
15044                                 sfmmu_unlink_from_hmeregion(scsfmmup,
15045                                     rgnp);
15046 
15047                         } else {
15048                                 sfmmu_t *ism_hatid = NULL;
15049                                 ism_ment_t *ism_ment;
15050                                 rid -= SFMMU_MAX_HME_REGIONS;
15051                                 rgnp = srdp->srd_ismrgnp[rid];
15052                                 ASSERT(rgnp->rgn_id == rid);
15053                                 ASSERT(rgnp->rgn_refcnt > 0);
15054 
15055                                 ism_hatid = (sfmmu_t *)rgnp->rgn_obj;
15056                                 ASSERT(ism_hatid->sfmmu_ismhat);
15057                                 ism_ment = &scdp->scd_ism_links[rid];
15058                                 ASSERT(ism_ment->iment_hat == scdp->scd_sfmmup);
15059                                 ASSERT(ism_ment->iment_base_va ==
15060                                     rgnp->rgn_saddr);
15061                                 mutex_enter(&ism_mlist_lock);
15062                                 iment_sub(ism_ment, ism_hatid);
15063                                 mutex_exit(&ism_mlist_lock);
15064 
15065                         }
15066                 }
15067         }
15068 }
15069 /*
15070  * Allocates and initialises a new SCD structure, this is called with
15071  * the srd_scd_mutex held and returns with the reference count
15072  * initialised to 1.
15073  */
15074 static sf_scd_t *
15075 sfmmu_alloc_scd(sf_srd_t *srdp, sf_region_map_t *new_map)
15076 {
15077         sf_scd_t *new_scdp;
15078         sfmmu_t *scsfmmup;
15079         int i;
15080 
15081         ASSERT(MUTEX_HELD(&srdp->srd_scd_mutex));
15082         new_scdp = kmem_cache_alloc(scd_cache, KM_SLEEP);
15083 
15084         scsfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP);
15085         new_scdp->scd_sfmmup = scsfmmup;
15086         scsfmmup->sfmmu_srdp = srdp;
15087         scsfmmup->sfmmu_scdp = new_scdp;
15088         scsfmmup->sfmmu_tsb0_4minflcnt = 0;
15089         scsfmmup->sfmmu_scdhat = 1;
15090         CPUSET_ALL(scsfmmup->sfmmu_cpusran);
15091         bzero(scsfmmup->sfmmu_hmeregion_links, SFMMU_L1_HMERLINKS_SIZE);
15092 
15093         ASSERT(max_mmu_ctxdoms > 0);
15094         for (i = 0; i < max_mmu_ctxdoms; i++) {
15095                 scsfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT;
15096                 scsfmmup->sfmmu_ctxs[i].gnum = 0;
15097         }
15098 
15099         for (i = 0; i < MMU_PAGE_SIZES; i++) {
15100                 new_scdp->scd_rttecnt[i] = 0;
15101         }
15102 
15103         new_scdp->scd_region_map = *new_map;
15104         new_scdp->scd_refcnt = 1;
15105         if (sfmmu_alloc_scd_tsbs(srdp, new_scdp) != TSB_SUCCESS) {
15106                 kmem_cache_free(scd_cache, new_scdp);
15107                 kmem_cache_free(sfmmuid_cache, scsfmmup);
15108                 return (NULL);
15109         }
15110         if (&mmu_init_scd) {
15111                 mmu_init_scd(new_scdp);
15112         }
15113         return (new_scdp);
15114 }
15115 
15116 /*
15117  * The first phase of a process joining an SCD. The hat structure is
15118  * linked to the SCD queue and then the HAT_JOIN_SCD sfmmu flag is set
15119  * and a cross-call with context invalidation is used to cause the
15120  * remaining work to be carried out in the sfmmu_tsbmiss_exception()
15121  * routine.
15122  */
15123 static void
15124 sfmmu_join_scd(sf_scd_t *scdp, sfmmu_t *sfmmup)
15125 {
15126         hatlock_t *hatlockp;
15127         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
15128         int i;
15129         sf_scd_t *old_scdp;
15130 
15131         ASSERT(srdp != NULL);
15132         ASSERT(scdp != NULL);
15133         ASSERT(scdp->scd_refcnt > 0);
15134         ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
15135 
15136         if ((old_scdp = sfmmup->sfmmu_scdp) != NULL) {
15137                 ASSERT(old_scdp != scdp);
15138 
15139                 mutex_enter(&old_scdp->scd_mutex);
15140                 sfmmu_from_scd_list(&old_scdp->scd_sf_list, sfmmup);
15141                 mutex_exit(&old_scdp->scd_mutex);
15142                 /*
15143                  * sfmmup leaves the old scd. Update sfmmu_ttecnt to
15144                  * include the shme rgn ttecnt for rgns that
15145                  * were in the old SCD
15146                  */
15147                 for (i = 0; i < mmu_page_sizes; i++) {
15148                         ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15149                             old_scdp->scd_rttecnt[i]);
15150                         atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15151                             sfmmup->sfmmu_scdrttecnt[i]);
15152                 }
15153         }
15154 
15155         /*
15156          * Move sfmmu to the scd lists.
15157          */
15158         mutex_enter(&scdp->scd_mutex);
15159         sfmmu_to_scd_list(&scdp->scd_sf_list, sfmmup);
15160         mutex_exit(&scdp->scd_mutex);
15161         SF_SCD_INCR_REF(scdp);
15162 
15163         hatlockp = sfmmu_hat_enter(sfmmup);
15164         /*
15165          * For a multi-thread process, we must stop
15166          * all the other threads before joining the scd.
15167          */
15168 
15169         SFMMU_FLAGS_SET(sfmmup, HAT_JOIN_SCD);
15170 
15171         sfmmu_invalidate_ctx(sfmmup);
15172         sfmmup->sfmmu_scdp = scdp;
15173 
15174         /*
15175          * Copy scd_rttecnt into sfmmup's sfmmu_scdrttecnt, and update
15176          * sfmmu_ttecnt to not include the rgn ttecnt just joined in SCD.
15177          */
15178         for (i = 0; i < mmu_page_sizes; i++) {
15179                 sfmmup->sfmmu_scdrttecnt[i] = scdp->scd_rttecnt[i];
15180                 ASSERT(sfmmup->sfmmu_ttecnt[i] >= scdp->scd_rttecnt[i]);
15181                 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15182                     -sfmmup->sfmmu_scdrttecnt[i]);
15183         }
15184         /* update tsb0 inflation count */
15185         if (old_scdp != NULL) {
15186                 sfmmup->sfmmu_tsb0_4minflcnt +=
15187                     old_scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
15188         }
15189         ASSERT(sfmmup->sfmmu_tsb0_4minflcnt >=
15190             scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt);
15191         sfmmup->sfmmu_tsb0_4minflcnt -= scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
15192 
15193         sfmmu_hat_exit(hatlockp);
15194 
15195         if (old_scdp != NULL) {
15196                 SF_SCD_DECR_REF(srdp, old_scdp);
15197         }
15198 
15199 }
15200 
15201 /*
15202  * This routine is called by a process to become part of an SCD. It is called
15203  * from sfmmu_tsbmiss_exception() once most of the initial work has been
15204  * done by sfmmu_join_scd(). This routine must not drop the hat lock.
15205  */
15206 static void
15207 sfmmu_finish_join_scd(sfmmu_t *sfmmup)
15208 {
15209         struct tsb_info *tsbinfop;
15210 
15211         ASSERT(sfmmu_hat_lock_held(sfmmup));
15212         ASSERT(sfmmup->sfmmu_scdp != NULL);
15213         ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD));
15214         ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15215         ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ALLCTX_INVALID));
15216 
15217         for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL;
15218             tsbinfop = tsbinfop->tsb_next) {
15219                 if (tsbinfop->tsb_flags & TSB_SWAPPED) {
15220                         continue;
15221                 }
15222                 ASSERT(!(tsbinfop->tsb_flags & TSB_RELOC_FLAG));
15223 
15224                 sfmmu_inv_tsb(tsbinfop->tsb_va,
15225                     TSB_BYTES(tsbinfop->tsb_szc));
15226         }
15227 
15228         /* Set HAT_CTX1_FLAG for all SCD ISMs */
15229         sfmmu_ism_hatflags(sfmmup, 1);
15230 
15231         SFMMU_STAT(sf_join_scd);
15232 }
15233 
15234 /*
15235  * This routine is called in order to check if there is an SCD which matches
15236  * the process's region map if not then a new SCD may be created.
15237  */
15238 static void
15239 sfmmu_find_scd(sfmmu_t *sfmmup)
15240 {
15241         sf_srd_t *srdp = sfmmup->sfmmu_srdp;
15242         sf_scd_t *scdp, *new_scdp;
15243         int ret;
15244 
15245         ASSERT(srdp != NULL);
15246         ASSERT(AS_WRITE_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
15247 
15248         mutex_enter(&srdp->srd_scd_mutex);
15249         for (scdp = srdp->srd_scdp; scdp != NULL;
15250             scdp = scdp->scd_next) {
15251                 SF_RGNMAP_EQUAL(&scdp->scd_region_map,
15252                     &sfmmup->sfmmu_region_map, ret);
15253                 if (ret == 1) {
15254                         SF_SCD_INCR_REF(scdp);
15255                         mutex_exit(&srdp->srd_scd_mutex);
15256                         sfmmu_join_scd(scdp, sfmmup);
15257                         ASSERT(scdp->scd_refcnt >= 2);
15258                         atomic_add_32((volatile uint32_t *)
15259                             &scdp->scd_refcnt, -1);
15260                         return;
15261                 } else {
15262                         /*
15263                          * If the sfmmu region map is a subset of the scd
15264                          * region map, then the assumption is that this process
15265                          * will continue attaching to ISM segments until the
15266                          * region maps are equal.
15267                          */
15268                         SF_RGNMAP_IS_SUBSET(&scdp->scd_region_map,
15269                             &sfmmup->sfmmu_region_map, ret);
15270                         if (ret == 1) {
15271                                 mutex_exit(&srdp->srd_scd_mutex);
15272                                 return;
15273                         }
15274                 }
15275         }
15276 
15277         ASSERT(scdp == NULL);
15278         /*
15279          * No matching SCD has been found, create a new one.
15280          */
15281         if ((new_scdp = sfmmu_alloc_scd(srdp, &sfmmup->sfmmu_region_map)) ==
15282             NULL) {
15283                 mutex_exit(&srdp->srd_scd_mutex);
15284                 return;
15285         }
15286 
15287         /*
15288          * sfmmu_alloc_scd() returns with a ref count of 1 on the scd.
15289          */
15290 
15291         /* Set scd_rttecnt for shme rgns in SCD */
15292         sfmmu_set_scd_rttecnt(srdp, new_scdp);
15293 
15294         /*
15295          * Link scd onto srd_scdp list and scd sfmmu onto region/iment lists.
15296          */
15297         sfmmu_link_scd_to_regions(srdp, new_scdp);
15298         sfmmu_add_scd(&srdp->srd_scdp, new_scdp);
15299         SFMMU_STAT_ADD(sf_create_scd, 1);
15300 
15301         mutex_exit(&srdp->srd_scd_mutex);
15302         sfmmu_join_scd(new_scdp, sfmmup);
15303         ASSERT(new_scdp->scd_refcnt >= 2);
15304         atomic_add_32((volatile uint32_t *)&new_scdp->scd_refcnt, -1);
15305 }
15306 
15307 /*
15308  * This routine is called by a process to remove itself from an SCD. It is
15309  * either called when the processes has detached from a segment or from
15310  * hat_free_start() as a result of calling exit.
15311  */
15312 static void
15313 sfmmu_leave_scd(sfmmu_t *sfmmup, uchar_t r_type)
15314 {
15315         sf_scd_t *scdp = sfmmup->sfmmu_scdp;
15316         sf_srd_t *srdp =  sfmmup->sfmmu_srdp;
15317         hatlock_t *hatlockp = TSB_HASH(sfmmup);
15318         int i;
15319 
15320         ASSERT(scdp != NULL);
15321         ASSERT(srdp != NULL);
15322 
15323         if (sfmmup->sfmmu_free) {
15324                 /*
15325                  * If the process is part of an SCD the sfmmu is unlinked
15326                  * from scd_sf_list.
15327                  */
15328                 mutex_enter(&scdp->scd_mutex);
15329                 sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup);
15330                 mutex_exit(&scdp->scd_mutex);
15331                 /*
15332                  * Update sfmmu_ttecnt to include the rgn ttecnt for rgns that
15333                  * are about to leave the SCD
15334                  */
15335                 for (i = 0; i < mmu_page_sizes; i++) {
15336                         ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15337                             scdp->scd_rttecnt[i]);
15338                         atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15339                             sfmmup->sfmmu_scdrttecnt[i]);
15340                         sfmmup->sfmmu_scdrttecnt[i] = 0;
15341                 }
15342                 sfmmup->sfmmu_scdp = NULL;
15343 
15344                 SF_SCD_DECR_REF(srdp, scdp);
15345                 return;
15346         }
15347 
15348         ASSERT(r_type != SFMMU_REGION_ISM ||
15349             SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15350         ASSERT(scdp->scd_refcnt);
15351         ASSERT(!sfmmup->sfmmu_free);
15352         ASSERT(sfmmu_hat_lock_held(sfmmup));
15353         ASSERT(AS_LOCK_HELD(sfmmup->sfmmu_as, &sfmmup->sfmmu_as->a_lock));
15354 
15355         /*
15356          * Wait for ISM maps to be updated.
15357          */
15358         if (r_type != SFMMU_REGION_ISM) {
15359                 while (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY) &&
15360                     sfmmup->sfmmu_scdp != NULL) {
15361                         cv_wait(&sfmmup->sfmmu_tsb_cv,
15362                             HATLOCK_MUTEXP(hatlockp));
15363                 }
15364 
15365                 if (sfmmup->sfmmu_scdp == NULL) {
15366                         sfmmu_hat_exit(hatlockp);
15367                         return;
15368                 }
15369                 SFMMU_FLAGS_SET(sfmmup, HAT_ISMBUSY);
15370         }
15371 
15372         if (SFMMU_FLAGS_ISSET(sfmmup, HAT_JOIN_SCD)) {
15373                 SFMMU_FLAGS_CLEAR(sfmmup, HAT_JOIN_SCD);
15374                 /*
15375                  * Since HAT_JOIN_SCD was set our context
15376                  * is still invalid.
15377                  */
15378         } else {
15379                 /*
15380                  * For a multi-thread process, we must stop
15381                  * all the other threads before leaving the scd.
15382                  */
15383 
15384                 sfmmu_invalidate_ctx(sfmmup);
15385         }
15386 
15387         /* Clear all the rid's for ISM, delete flags, etc */
15388         ASSERT(SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY));
15389         sfmmu_ism_hatflags(sfmmup, 0);
15390 
15391         /*
15392          * Update sfmmu_ttecnt to include the rgn ttecnt for rgns that
15393          * are in SCD before this sfmmup leaves the SCD.
15394          */
15395         for (i = 0; i < mmu_page_sizes; i++) {
15396                 ASSERT(sfmmup->sfmmu_scdrttecnt[i] ==
15397                     scdp->scd_rttecnt[i]);
15398                 atomic_add_long(&sfmmup->sfmmu_ttecnt[i],
15399                     sfmmup->sfmmu_scdrttecnt[i]);
15400                 sfmmup->sfmmu_scdrttecnt[i] = 0;
15401                 /* update ismttecnt to include SCD ism before hat leaves SCD */
15402                 sfmmup->sfmmu_ismttecnt[i] += sfmmup->sfmmu_scdismttecnt[i];
15403                 sfmmup->sfmmu_scdismttecnt[i] = 0;
15404         }
15405         /* update tsb0 inflation count */
15406         sfmmup->sfmmu_tsb0_4minflcnt += scdp->scd_sfmmup->sfmmu_tsb0_4minflcnt;
15407 
15408         if (r_type != SFMMU_REGION_ISM) {
15409                 SFMMU_FLAGS_CLEAR(sfmmup, HAT_ISMBUSY);
15410         }
15411         sfmmup->sfmmu_scdp = NULL;
15412 
15413         sfmmu_hat_exit(hatlockp);
15414 
15415         /*
15416          * Unlink sfmmu from scd_sf_list this can be done without holding
15417          * the hat lock as we hold the sfmmu_as lock which prevents
15418          * hat_join_region from adding this thread to the scd again. Other
15419          * threads check if sfmmu_scdp is NULL under hat lock and if it's NULL
15420          * they won't get here, since sfmmu_leave_scd() clears sfmmu_scdp
15421          * while holding the hat lock.
15422          */
15423         mutex_enter(&scdp->scd_mutex);
15424         sfmmu_from_scd_list(&scdp->scd_sf_list, sfmmup);
15425         mutex_exit(&scdp->scd_mutex);
15426         SFMMU_STAT(sf_leave_scd);
15427 
15428         SF_SCD_DECR_REF(srdp, scdp);
15429         hatlockp = sfmmu_hat_enter(sfmmup);
15430 
15431 }
15432 
15433 /*
15434  * Unlink and free up an SCD structure with a reference count of 0.
15435  */
15436 static void
15437 sfmmu_destroy_scd(sf_srd_t *srdp, sf_scd_t *scdp, sf_region_map_t *scd_rmap)
15438 {
15439         sfmmu_t *scsfmmup;
15440         sf_scd_t *sp;
15441         hatlock_t *shatlockp;
15442         int i, ret;
15443 
15444         mutex_enter(&srdp->srd_scd_mutex);
15445         for (sp = srdp->srd_scdp; sp != NULL; sp = sp->scd_next) {
15446                 if (sp == scdp)
15447                         break;
15448         }
15449         if (sp == NULL || sp->scd_refcnt) {
15450                 mutex_exit(&srdp->srd_scd_mutex);
15451                 return;
15452         }
15453 
15454         /*
15455          * It is possible that the scd has been freed and reallocated with a
15456          * different region map while we've been waiting for the srd_scd_mutex.
15457          */
15458         SF_RGNMAP_EQUAL(scd_rmap, &sp->scd_region_map, ret);
15459         if (ret != 1) {
15460                 mutex_exit(&srdp->srd_scd_mutex);
15461                 return;
15462         }
15463 
15464         ASSERT(scdp->scd_sf_list == NULL);
15465         /*
15466          * Unlink scd from srd_scdp list.
15467          */
15468         sfmmu_remove_scd(&srdp->srd_scdp, scdp);
15469         mutex_exit(&srdp->srd_scd_mutex);
15470 
15471         sfmmu_unlink_scd_from_regions(srdp, scdp);
15472 
15473         /* Clear shared context tsb and release ctx */
15474         scsfmmup = scdp->scd_sfmmup;
15475 
15476         /*
15477          * create a barrier so that scd will not be destroyed
15478          * if other thread still holds the same shared hat lock.
15479          * E.g., sfmmu_tsbmiss_exception() needs to acquire the
15480          * shared hat lock before checking the shared tsb reloc flag.
15481          */
15482         shatlockp = sfmmu_hat_enter(scsfmmup);
15483         sfmmu_hat_exit(shatlockp);
15484 
15485         sfmmu_free_scd_tsbs(scsfmmup);
15486 
15487         for (i = 0; i < SFMMU_L1_HMERLINKS; i++) {
15488                 if (scsfmmup->sfmmu_hmeregion_links[i] != NULL) {
15489                         kmem_free(scsfmmup->sfmmu_hmeregion_links[i],
15490                             SFMMU_L2_HMERLINKS_SIZE);
15491                         scsfmmup->sfmmu_hmeregion_links[i] = NULL;
15492                 }
15493         }
15494         kmem_cache_free(sfmmuid_cache, scsfmmup);
15495         kmem_cache_free(scd_cache, scdp);
15496         SFMMU_STAT(sf_destroy_scd);
15497 }
15498 
15499 /*
15500  * Modifies the HAT_CTX1_FLAG for each of the ISM segments which correspond to
15501  * bits which are set in the ism_region_map parameter. This flag indicates to
15502  * the tsbmiss handler that mapping for these segments should be loaded using
15503  * the shared context.
15504  */
15505 static void
15506 sfmmu_ism_hatflags(sfmmu_t *sfmmup, int addflag)
15507 {
15508         sf_scd_t *scdp = sfmmup->sfmmu_scdp;
15509         ism_blk_t *ism_blkp;
15510         ism_map_t *ism_map;
15511         int i, rid;
15512 
15513         ASSERT(sfmmup->sfmmu_iblk != NULL);
15514         ASSERT(scdp != NULL);
15515         /*
15516          * Note that the caller either set HAT_ISMBUSY flag or checked
15517          * under hat lock that HAT_ISMBUSY was not set by another thread.
15518          */
15519         ASSERT(sfmmu_hat_lock_held(sfmmup));
15520 
15521         ism_blkp = sfmmup->sfmmu_iblk;
15522         while (ism_blkp != NULL) {
15523                 ism_map = ism_blkp->iblk_maps;
15524                 for (i = 0; ism_map[i].imap_ismhat && i < ISM_MAP_SLOTS; i++) {
15525                         rid = ism_map[i].imap_rid;
15526                         if (rid == SFMMU_INVALID_ISMRID) {
15527                                 continue;
15528                         }
15529                         ASSERT(rid >= 0 && rid < SFMMU_MAX_ISM_REGIONS);
15530                         if (SF_RGNMAP_TEST(scdp->scd_ismregion_map, rid) &&
15531                             addflag) {
15532                                 ism_map[i].imap_hatflags |=
15533                                     HAT_CTX1_FLAG;
15534                         } else {
15535                                 ism_map[i].imap_hatflags &=
15536                                     ~HAT_CTX1_FLAG;
15537                         }
15538                 }
15539                 ism_blkp = ism_blkp->iblk_next;
15540         }
15541 }
15542 
15543 static int
15544 sfmmu_srd_lock_held(sf_srd_t *srdp)
15545 {
15546         return (MUTEX_HELD(&srdp->srd_mutex));
15547 }
15548 
15549 /* ARGSUSED */
15550 static int
15551 sfmmu_scdcache_constructor(void *buf, void *cdrarg, int kmflags)
15552 {
15553         sf_scd_t *scdp = (sf_scd_t *)buf;
15554 
15555         bzero(buf, sizeof (sf_scd_t));
15556         mutex_init(&scdp->scd_mutex, NULL, MUTEX_DEFAULT, NULL);
15557         return (0);
15558 }
15559 
15560 /* ARGSUSED */
15561 static void
15562 sfmmu_scdcache_destructor(void *buf, void *cdrarg)
15563 {
15564         sf_scd_t *scdp = (sf_scd_t *)buf;
15565 
15566         mutex_destroy(&scdp->scd_mutex);
15567 }
15568 
15569 /*
15570  * The listp parameter is a pointer to a list of hmeblks which are partially
15571  * freed as result of calling sfmmu_hblk_hash_rm(), the last phase of the
15572  * freeing process is to cross-call all cpus to ensure that there are no
15573  * remaining cached references.
15574  *
15575  * If the local generation number is less than the global then we can free
15576  * hmeblks which are already on the pending queue as another cpu has completed
15577  * the cross-call.
15578  *
15579  * We cross-call to make sure that there are no threads on other cpus accessing
15580  * these hmblks and then complete the process of freeing them under the
15581  * following conditions:
15582  *      The total number of pending hmeblks is greater than the threshold
15583  *      The reserve list has fewer than HBLK_RESERVE_CNT hmeblks
15584  *      It is at least 1 second since the last time we cross-called
15585  *
15586  * Otherwise, we add the hmeblks to the per-cpu pending queue.
15587  */
15588 static void
15589 sfmmu_hblks_list_purge(struct hme_blk **listp, int dontfree)
15590 {
15591         struct hme_blk *hblkp, *pr_hblkp = NULL;
15592         int             count = 0;
15593         cpuset_t        cpuset = cpu_ready_set;
15594         cpu_hme_pend_t  *cpuhp;
15595         timestruc_t     now;
15596         int             one_second_expired = 0;
15597 
15598         gethrestime_lasttick(&now);
15599 
15600         for (hblkp = *listp; hblkp != NULL; hblkp = hblkp->hblk_next) {
15601                 ASSERT(hblkp->hblk_shw_bit == 0);
15602                 ASSERT(hblkp->hblk_shared == 0);
15603                 count++;
15604                 pr_hblkp = hblkp;
15605         }
15606 
15607         cpuhp = &cpu_hme_pend[CPU->cpu_seqid];
15608         mutex_enter(&cpuhp->chp_mutex);
15609 
15610         if ((cpuhp->chp_count + count) == 0) {
15611                 mutex_exit(&cpuhp->chp_mutex);
15612                 return;
15613         }
15614 
15615         if ((now.tv_sec - cpuhp->chp_timestamp) > 1) {
15616                 one_second_expired  = 1;
15617         }
15618 
15619         if (!dontfree && (freehblkcnt < HBLK_RESERVE_CNT ||
15620             (cpuhp->chp_count + count) > cpu_hme_pend_thresh ||
15621             one_second_expired)) {
15622                 /* Append global list to local */
15623                 if (pr_hblkp == NULL) {
15624                         *listp = cpuhp->chp_listp;
15625                 } else {
15626                         pr_hblkp->hblk_next = cpuhp->chp_listp;
15627                 }
15628                 cpuhp->chp_listp = NULL;
15629                 cpuhp->chp_count = 0;
15630                 cpuhp->chp_timestamp = now.tv_sec;
15631                 mutex_exit(&cpuhp->chp_mutex);
15632 
15633                 kpreempt_disable();
15634                 CPUSET_DEL(cpuset, CPU->cpu_id);
15635                 xt_sync(cpuset);
15636                 xt_sync(cpuset);
15637                 kpreempt_enable();
15638 
15639                 /*
15640                  * At this stage we know that no trap handlers on other
15641                  * cpus can have references to hmeblks on the list.
15642                  */
15643                 sfmmu_hblk_free(listp);
15644         } else if (*listp != NULL) {
15645                 pr_hblkp->hblk_next = cpuhp->chp_listp;
15646                 cpuhp->chp_listp = *listp;
15647                 cpuhp->chp_count += count;
15648                 *listp = NULL;
15649                 mutex_exit(&cpuhp->chp_mutex);
15650         } else {
15651                 mutex_exit(&cpuhp->chp_mutex);
15652         }
15653 }
15654 
15655 /*
15656  * Add an hmeblk to the the hash list.
15657  */
15658 void
15659 sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
15660         uint64_t hblkpa)
15661 {
15662         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
15663 #ifdef  DEBUG
15664         if (hmebp->hmeblkp == NULL) {
15665                 ASSERT(hmebp->hmeh_nextpa == HMEBLK_ENDPA);
15666         }
15667 #endif /* DEBUG */
15668 
15669         hmeblkp->hblk_nextpa = hmebp->hmeh_nextpa;
15670         /*
15671          * Since the TSB miss handler now does not lock the hash chain before
15672          * walking it, make sure that the hmeblks nextpa is globally visible
15673          * before we make the hmeblk globally visible by updating the chain root
15674          * pointer in the hash bucket.
15675          */
15676         membar_producer();
15677         hmebp->hmeh_nextpa = hblkpa;
15678         hmeblkp->hblk_next = hmebp->hmeblkp;
15679         hmebp->hmeblkp = hmeblkp;
15680 
15681 }
15682 
15683 /*
15684  * This function is the first part of a 2 part process to remove an hmeblk
15685  * from the hash chain. In this phase we unlink the hmeblk from the hash chain
15686  * but leave the next physical pointer unchanged. The hmeblk is then linked onto
15687  * a per-cpu pending list using the virtual address pointer.
15688  *
15689  * TSB miss trap handlers that start after this phase will no longer see
15690  * this hmeblk. TSB miss handlers that still cache this hmeblk in a register
15691  * can still use it for further chain traversal because we haven't yet modifed
15692  * the next physical pointer or freed it.
15693  *
15694  * In the second phase of hmeblk removal we'll issue a barrier xcall before
15695  * we reuse or free this hmeblk. This will make sure all lingering references to
15696  * the hmeblk after first phase disappear before we finally reclaim it.
15697  * This scheme eliminates the need for TSB miss handlers to lock hmeblk chains
15698  * during their traversal.
15699  *
15700  * The hmehash_mutex must be held when calling this function.
15701  *
15702  * Input:
15703  *       hmebp - hme hash bucket pointer
15704  *       hmeblkp - address of hmeblk to be removed
15705  *       pr_hblk - virtual address of previous hmeblkp
15706  *       listp - pointer to list of hmeblks linked by virtual address
15707  *       free_now flag - indicates that a complete removal from the hash chains
15708  *                       is necessary.
15709  *
15710  * It is inefficient to use the free_now flag as a cross-call is required to
15711  * remove a single hmeblk from the hash chain but is necessary when hmeblks are
15712  * in short supply.
15713  */
15714 void
15715 sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
15716     struct hme_blk *pr_hblk, struct hme_blk **listp,
15717     int free_now)
15718 {
15719         int shw_size, vshift;
15720         struct hme_blk *shw_hblkp;
15721         uint_t          shw_mask, newshw_mask;
15722         caddr_t         vaddr;
15723         int             size;
15724         cpuset_t cpuset = cpu_ready_set;
15725 
15726         ASSERT(SFMMU_HASH_LOCK_ISHELD(hmebp));
15727 
15728         if (hmebp->hmeblkp == hmeblkp) {
15729                 hmebp->hmeh_nextpa = hmeblkp->hblk_nextpa;
15730                 hmebp->hmeblkp = hmeblkp->hblk_next;
15731         } else {
15732                 pr_hblk->hblk_nextpa = hmeblkp->hblk_nextpa;
15733                 pr_hblk->hblk_next = hmeblkp->hblk_next;
15734         }
15735 
15736         size = get_hblk_ttesz(hmeblkp);
15737         shw_hblkp = hmeblkp->hblk_shadow;
15738         if (shw_hblkp) {
15739                 ASSERT(hblktosfmmu(hmeblkp) != KHATID);
15740                 ASSERT(!hmeblkp->hblk_shared);
15741 #ifdef  DEBUG
15742                 if (mmu_page_sizes == max_mmu_page_sizes) {
15743                         ASSERT(size < TTE256M);
15744                 } else {
15745                         ASSERT(size < TTE4M);
15746                 }
15747 #endif /* DEBUG */
15748 
15749                 shw_size = get_hblk_ttesz(shw_hblkp);
15750                 vaddr = (caddr_t)get_hblk_base(hmeblkp);
15751                 vshift = vaddr_to_vshift(shw_hblkp->hblk_tag, vaddr, shw_size);
15752                 ASSERT(vshift < 8);
15753                 /*
15754                  * Atomically clear shadow mask bit
15755                  */
15756                 do {
15757                         shw_mask = shw_hblkp->hblk_shw_mask;
15758                         ASSERT(shw_mask & (1 << vshift));
15759                         newshw_mask = shw_mask & ~(1 << vshift);
15760                         newshw_mask = cas32(&shw_hblkp->hblk_shw_mask,
15761                             shw_mask, newshw_mask);
15762                 } while (newshw_mask != shw_mask);
15763                 hmeblkp->hblk_shadow = NULL;
15764         }
15765         hmeblkp->hblk_shw_bit = 0;
15766 
15767         if (hmeblkp->hblk_shared) {
15768 #ifdef  DEBUG
15769                 sf_srd_t        *srdp;
15770                 sf_region_t     *rgnp;
15771                 uint_t          rid;
15772 
15773                 srdp = hblktosrd(hmeblkp);
15774                 ASSERT(srdp != NULL && srdp->srd_refcnt != 0);
15775                 rid = hmeblkp->hblk_tag.htag_rid;
15776                 ASSERT(SFMMU_IS_SHMERID_VALID(rid));
15777                 ASSERT(rid < SFMMU_MAX_HME_REGIONS);
15778                 rgnp = srdp->srd_hmergnp[rid];
15779                 ASSERT(rgnp != NULL);
15780                 SFMMU_VALIDATE_SHAREDHBLK(hmeblkp, srdp, rgnp, rid);
15781 #endif /* DEBUG */
15782                 hmeblkp->hblk_shared = 0;
15783         }
15784         if (free_now) {
15785                 kpreempt_disable();
15786                 CPUSET_DEL(cpuset, CPU->cpu_id);
15787                 xt_sync(cpuset);
15788                 xt_sync(cpuset);
15789                 kpreempt_enable();
15790 
15791                 hmeblkp->hblk_nextpa = HMEBLK_ENDPA;
15792                 hmeblkp->hblk_next = NULL;
15793         } else {
15794                 /* Append hmeblkp to listp for processing later. */
15795                 hmeblkp->hblk_next = *listp;
15796                 *listp = hmeblkp;
15797         }
15798 }
15799 
15800 /*
15801  * This routine is called when memory is in short supply and returns a free
15802  * hmeblk of the requested size from the cpu pending lists.
15803  */
15804 static struct hme_blk *
15805 sfmmu_check_pending_hblks(int size)
15806 {
15807         int i;
15808         struct hme_blk *hmeblkp = NULL, *last_hmeblkp;
15809         int found_hmeblk;
15810         cpuset_t cpuset = cpu_ready_set;
15811         cpu_hme_pend_t *cpuhp;
15812 
15813         /* Flush cpu hblk pending queues */
15814         for (i = 0; i < NCPU; i++) {
15815                 cpuhp = &cpu_hme_pend[i];
15816                 if (cpuhp->chp_listp != NULL)  {
15817                         mutex_enter(&cpuhp->chp_mutex);
15818                         if (cpuhp->chp_listp == NULL)  {
15819                                 mutex_exit(&cpuhp->chp_mutex);
15820                                 continue;
15821                         }
15822                         found_hmeblk = 0;
15823                         last_hmeblkp = NULL;
15824                         for (hmeblkp = cpuhp->chp_listp; hmeblkp != NULL;
15825                             hmeblkp = hmeblkp->hblk_next) {
15826                                 if (get_hblk_ttesz(hmeblkp) == size) {
15827                                         if (last_hmeblkp == NULL) {
15828                                                 cpuhp->chp_listp =
15829                                                     hmeblkp->hblk_next;
15830                                         } else {
15831                                                 last_hmeblkp->hblk_next =
15832                                                     hmeblkp->hblk_next;
15833                                         }
15834                                         ASSERT(cpuhp->chp_count > 0);
15835                                         cpuhp->chp_count--;
15836                                         found_hmeblk = 1;
15837                                         break;
15838                                 } else {
15839                                         last_hmeblkp = hmeblkp;
15840                                 }
15841                         }
15842                         mutex_exit(&cpuhp->chp_mutex);
15843 
15844                         if (found_hmeblk) {
15845                                 kpreempt_disable();
15846                                 CPUSET_DEL(cpuset, CPU->cpu_id);
15847                                 xt_sync(cpuset);
15848                                 xt_sync(cpuset);
15849                                 kpreempt_enable();
15850                                 return (hmeblkp);
15851                         }
15852                 }
15853         }
15854         return (NULL);
15855 }