1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/param.h> 26 #include <sys/user.h> 27 #include <sys/mman.h> 28 #include <sys/kmem.h> 29 #include <sys/sysmacros.h> 30 #include <sys/cmn_err.h> 31 #include <sys/systm.h> 32 #include <sys/tuneable.h> 33 #include <vm/hat.h> 34 #include <vm/seg.h> 35 #include <vm/as.h> 36 #include <vm/anon.h> 37 #include <vm/page.h> 38 #include <sys/buf.h> 39 #include <sys/swap.h> 40 #include <sys/atomic.h> 41 #include <vm/seg_spt.h> 42 #include <sys/debug.h> 43 #include <sys/vtrace.h> 44 #include <sys/shm.h> 45 #include <sys/shm_impl.h> 46 #include <sys/lgrp.h> 47 #include <sys/vmsystm.h> 48 #include <sys/policy.h> 49 #include <sys/project.h> 50 #include <sys/tnf_probe.h> 51 #include <sys/zone.h> 52 53 #define SEGSPTADDR (caddr_t)0x0 54 55 /* 56 * # pages used for spt 57 */ 58 size_t spt_used; 59 60 /* 61 * segspt_minfree is the memory left for system after ISM 62 * locked its pages; it is set up to 5% of availrmem in 63 * sptcreate when ISM is created. ISM should not use more 64 * than ~90% of availrmem; if it does, then the performance 65 * of the system may decrease. Machines with large memories may 66 * be able to use up more memory for ISM so we set the default 67 * segspt_minfree to 5% (which gives ISM max 95% of availrmem. 68 * If somebody wants even more memory for ISM (risking hanging 69 * the system) they can patch the segspt_minfree to smaller number. 70 */ 71 pgcnt_t segspt_minfree = 0; 72 73 static int segspt_create(struct seg *seg, caddr_t argsp); 74 static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize); 75 static void segspt_free(struct seg *seg); 76 static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len); 77 static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr); 78 79 static void 80 segspt_badop() 81 { 82 panic("segspt_badop called"); 83 /*NOTREACHED*/ 84 } 85 86 #define SEGSPT_BADOP(t) (t(*)())segspt_badop 87 88 struct seg_ops segspt_ops = { 89 SEGSPT_BADOP(int), /* dup */ 90 segspt_unmap, 91 segspt_free, 92 SEGSPT_BADOP(int), /* fault */ 93 SEGSPT_BADOP(faultcode_t), /* faulta */ 94 SEGSPT_BADOP(int), /* setprot */ 95 SEGSPT_BADOP(int), /* checkprot */ 96 SEGSPT_BADOP(int), /* kluster */ 97 SEGSPT_BADOP(int), /* sync */ 98 SEGSPT_BADOP(size_t), /* incore */ 99 SEGSPT_BADOP(int), /* lockop */ 100 SEGSPT_BADOP(int), /* getprot */ 101 SEGSPT_BADOP(u_offset_t), /* getoffset */ 102 SEGSPT_BADOP(int), /* gettype */ 103 SEGSPT_BADOP(int), /* getvp */ 104 SEGSPT_BADOP(int), /* advise */ 105 SEGSPT_BADOP(void), /* dump */ 106 SEGSPT_BADOP(int), /* pagelock */ 107 SEGSPT_BADOP(int), /* setpgsz */ 108 SEGSPT_BADOP(int), /* getmemid */ 109 segspt_getpolicy, /* getpolicy */ 110 SEGSPT_BADOP(int), /* capable */ 111 }; 112 113 static int segspt_shmdup(struct seg *seg, struct seg *newseg); 114 static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize); 115 static void segspt_shmfree(struct seg *seg); 116 static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg, 117 caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); 118 static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr); 119 static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr, 120 register size_t len, register uint_t prot); 121 static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, 122 uint_t prot); 123 static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta); 124 static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, 125 register char *vec); 126 static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len, 127 int attr, uint_t flags); 128 static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 129 int attr, int op, ulong_t *lockmap, size_t pos); 130 static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, 131 uint_t *protv); 132 static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr); 133 static int segspt_shmgettype(struct seg *seg, caddr_t addr); 134 static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 135 static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, 136 uint_t behav); 137 static void segspt_shmdump(struct seg *seg); 138 static int segspt_shmpagelock(struct seg *, caddr_t, size_t, 139 struct page ***, enum lock_type, enum seg_rw); 140 static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t); 141 static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *); 142 static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t); 143 static int segspt_shmcapable(struct seg *, segcapability_t); 144 145 struct seg_ops segspt_shmops = { 146 segspt_shmdup, 147 segspt_shmunmap, 148 segspt_shmfree, 149 segspt_shmfault, 150 segspt_shmfaulta, 151 segspt_shmsetprot, 152 segspt_shmcheckprot, 153 segspt_shmkluster, 154 segspt_shmsync, 155 segspt_shmincore, 156 segspt_shmlockop, 157 segspt_shmgetprot, 158 segspt_shmgetoffset, 159 segspt_shmgettype, 160 segspt_shmgetvp, 161 segspt_shmadvise, /* advise */ 162 segspt_shmdump, 163 segspt_shmpagelock, 164 segspt_shmsetpgsz, 165 segspt_shmgetmemid, 166 segspt_shmgetpolicy, 167 segspt_shmcapable, 168 }; 169 170 static void segspt_purge(struct seg *seg); 171 static int segspt_reclaim(void *, caddr_t, size_t, struct page **, 172 enum seg_rw, int); 173 static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len, 174 page_t **ppa); 175 176 177 178 /*ARGSUSED*/ 179 int 180 sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp, 181 uint_t prot, uint_t flags, uint_t share_szc) 182 { 183 int err; 184 struct as *newas; 185 struct segspt_crargs sptcargs; 186 187 #ifdef DEBUG 188 TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */, 189 tnf_ulong, size, size ); 190 #endif 191 if (segspt_minfree == 0) /* leave min 5% of availrmem for */ 192 segspt_minfree = availrmem/20; /* for the system */ 193 194 if (!hat_supported(HAT_SHARED_PT, (void *)0)) 195 return (EINVAL); 196 197 /* 198 * get a new as for this shared memory segment 199 */ 200 newas = as_alloc(); 201 newas->a_proc = NULL; 202 sptcargs.amp = amp; 203 sptcargs.prot = prot; 204 sptcargs.flags = flags; 205 sptcargs.szc = share_szc; 206 /* 207 * create a shared page table (spt) segment 208 */ 209 210 if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) { 211 as_free(newas); 212 return (err); 213 } 214 *sptseg = sptcargs.seg_spt; 215 return (0); 216 } 217 218 void 219 sptdestroy(struct as *as, struct anon_map *amp) 220 { 221 222 #ifdef DEBUG 223 TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */); 224 #endif 225 (void) as_unmap(as, SEGSPTADDR, amp->size); 226 as_free(as); 227 } 228 229 /* 230 * called from seg_free(). 231 * free (i.e., unlock, unmap, return to free list) 232 * all the pages in the given seg. 233 */ 234 void 235 segspt_free(struct seg *seg) 236 { 237 struct spt_data *sptd = (struct spt_data *)seg->s_data; 238 239 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 240 241 if (sptd != NULL) { 242 if (sptd->spt_realsize) 243 segspt_free_pages(seg, seg->s_base, sptd->spt_realsize); 244 245 if (sptd->spt_ppa_lckcnt) 246 kmem_free(sptd->spt_ppa_lckcnt, 247 sizeof (*sptd->spt_ppa_lckcnt) 248 * btopr(sptd->spt_amp->size)); 249 kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp)); 250 cv_destroy(&sptd->spt_cv); 251 mutex_destroy(&sptd->spt_lock); 252 kmem_free(sptd, sizeof (*sptd)); 253 } 254 } 255 256 /*ARGSUSED*/ 257 static int 258 segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr, 259 uint_t flags) 260 { 261 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 262 263 return (0); 264 } 265 266 /*ARGSUSED*/ 267 static size_t 268 segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec) 269 { 270 caddr_t eo_seg; 271 pgcnt_t npages; 272 struct shm_data *shmd = (struct shm_data *)seg->s_data; 273 struct seg *sptseg; 274 struct spt_data *sptd; 275 276 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 277 #ifdef lint 278 seg = seg; 279 #endif 280 sptseg = shmd->shm_sptseg; 281 sptd = sptseg->s_data; 282 283 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 284 eo_seg = addr + len; 285 while (addr < eo_seg) { 286 /* page exists, and it's locked. */ 287 *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED | 288 SEG_PAGE_ANON; 289 addr += PAGESIZE; 290 } 291 return (len); 292 } else { 293 struct anon_map *amp = shmd->shm_amp; 294 struct anon *ap; 295 page_t *pp; 296 pgcnt_t anon_index; 297 struct vnode *vp; 298 u_offset_t off; 299 ulong_t i; 300 int ret; 301 anon_sync_obj_t cookie; 302 303 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 304 anon_index = seg_page(seg, addr); 305 npages = btopr(len); 306 if (anon_index + npages > btopr(shmd->shm_amp->size)) { 307 return (EINVAL); 308 } 309 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 310 for (i = 0; i < npages; i++, anon_index++) { 311 ret = 0; 312 anon_array_enter(amp, anon_index, &cookie); 313 ap = anon_get_ptr(amp->ahp, anon_index); 314 if (ap != NULL) { 315 swap_xlate(ap, &vp, &off); 316 anon_array_exit(&cookie); 317 pp = page_lookup_nowait(vp, off, SE_SHARED); 318 if (pp != NULL) { 319 ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON; 320 page_unlock(pp); 321 } 322 } else { 323 anon_array_exit(&cookie); 324 } 325 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 326 ret |= SEG_PAGE_LOCKED; 327 } 328 *vec++ = (char)ret; 329 } 330 ANON_LOCK_EXIT(&->a_rwlock); 331 return (len); 332 } 333 } 334 335 static int 336 segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize) 337 { 338 size_t share_size; 339 340 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 341 342 /* 343 * seg.s_size may have been rounded up to the largest page size 344 * in shmat(). 345 * XXX This should be cleanedup. sptdestroy should take a length 346 * argument which should be the same as sptcreate. Then 347 * this rounding would not be needed (or is done in shm.c) 348 * Only the check for full segment will be needed. 349 * 350 * XXX -- shouldn't raddr == 0 always? These tests don't seem 351 * to be useful at all. 352 */ 353 share_size = page_get_pagesize(seg->s_szc); 354 ssize = P2ROUNDUP(ssize, share_size); 355 356 if (raddr == seg->s_base && ssize == seg->s_size) { 357 seg_free(seg); 358 return (0); 359 } else 360 return (EINVAL); 361 } 362 363 int 364 segspt_create(struct seg *seg, caddr_t argsp) 365 { 366 int err; 367 caddr_t addr = seg->s_base; 368 struct spt_data *sptd; 369 struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp; 370 struct anon_map *amp = sptcargs->amp; 371 struct kshmid *sp = amp->a_sp; 372 struct cred *cred = CRED(); 373 ulong_t i, j, anon_index = 0; 374 pgcnt_t npages = btopr(amp->size); 375 struct vnode *vp; 376 page_t **ppa; 377 uint_t hat_flags; 378 size_t pgsz; 379 pgcnt_t pgcnt; 380 caddr_t a; 381 pgcnt_t pidx; 382 size_t sz; 383 proc_t *procp = curproc; 384 rctl_qty_t lockedbytes = 0; 385 kproject_t *proj; 386 387 /* 388 * We are holding the a_lock on the underlying dummy as, 389 * so we can make calls to the HAT layer. 390 */ 391 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 392 ASSERT(sp != NULL); 393 394 #ifdef DEBUG 395 TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */, 396 tnf_opaque, addr, addr, tnf_ulong, len, seg->s_size); 397 #endif 398 if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 399 if (err = anon_swap_adjust(npages)) 400 return (err); 401 } 402 err = ENOMEM; 403 404 if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL) 405 goto out1; 406 407 if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 408 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages), 409 KM_NOSLEEP)) == NULL) 410 goto out2; 411 } 412 413 mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL); 414 415 if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL) 416 goto out3; 417 418 seg->s_ops = &segspt_ops; 419 sptd->spt_vp = vp; 420 sptd->spt_amp = amp; 421 sptd->spt_prot = sptcargs->prot; 422 sptd->spt_flags = sptcargs->flags; 423 seg->s_data = (caddr_t)sptd; 424 sptd->spt_ppa = NULL; 425 sptd->spt_ppa_lckcnt = NULL; 426 seg->s_szc = sptcargs->szc; 427 cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL); 428 sptd->spt_gen = 0; 429 430 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 431 if (seg->s_szc > amp->a_szc) { 432 amp->a_szc = seg->s_szc; 433 } 434 ANON_LOCK_EXIT(&->a_rwlock); 435 436 /* 437 * Set policy to affect initial allocation of pages in 438 * anon_map_createpages() 439 */ 440 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index, 441 NULL, 0, ptob(npages)); 442 443 if (sptcargs->flags & SHM_PAGEABLE) { 444 size_t share_sz; 445 pgcnt_t new_npgs, more_pgs; 446 struct anon_hdr *nahp; 447 zone_t *zone; 448 449 share_sz = page_get_pagesize(seg->s_szc); 450 if (!IS_P2ALIGNED(amp->size, share_sz)) { 451 /* 452 * We are rounding up the size of the anon array 453 * on 4 M boundary because we always create 4 M 454 * of page(s) when locking, faulting pages and we 455 * don't have to check for all corner cases e.g. 456 * if there is enough space to allocate 4 M 457 * page. 458 */ 459 new_npgs = btop(P2ROUNDUP(amp->size, share_sz)); 460 more_pgs = new_npgs - npages; 461 462 /* 463 * The zone will never be NULL, as a fully created 464 * shm always has an owning zone. 465 */ 466 zone = sp->shm_perm.ipc_zone_ref.zref_zone; 467 ASSERT(zone != NULL); 468 if (anon_resv_zone(ptob(more_pgs), zone) == 0) { 469 err = ENOMEM; 470 goto out4; 471 } 472 473 nahp = anon_create(new_npgs, ANON_SLEEP); 474 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 475 (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages, 476 ANON_SLEEP); 477 anon_release(amp->ahp, npages); 478 amp->ahp = nahp; 479 ASSERT(amp->swresv == ptob(npages)); 480 amp->swresv = amp->size = ptob(new_npgs); 481 ANON_LOCK_EXIT(&->a_rwlock); 482 npages = new_npgs; 483 } 484 485 sptd->spt_ppa_lckcnt = kmem_zalloc(npages * 486 sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP); 487 sptd->spt_pcachecnt = 0; 488 sptd->spt_realsize = ptob(npages); 489 sptcargs->seg_spt = seg; 490 return (0); 491 } 492 493 /* 494 * get array of pages for each anon slot in amp 495 */ 496 if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa, 497 seg, addr, S_CREATE, cred)) != 0) 498 goto out4; 499 500 mutex_enter(&sp->shm_mlock); 501 502 /* May be partially locked, so, count bytes to charge for locking */ 503 for (i = 0; i < npages; i++) 504 if (ppa[i]->p_lckcnt == 0) 505 lockedbytes += PAGESIZE; 506 507 proj = sp->shm_perm.ipc_proj; 508 509 if (lockedbytes > 0) { 510 mutex_enter(&procp->p_lock); 511 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) { 512 mutex_exit(&procp->p_lock); 513 mutex_exit(&sp->shm_mlock); 514 for (i = 0; i < npages; i++) 515 page_unlock(ppa[i]); 516 err = ENOMEM; 517 goto out4; 518 } 519 mutex_exit(&procp->p_lock); 520 } 521 522 /* 523 * addr is initial address corresponding to the first page on ppa list 524 */ 525 for (i = 0; i < npages; i++) { 526 /* attempt to lock all pages */ 527 if (page_pp_lock(ppa[i], 0, 1) == 0) { 528 /* 529 * if unable to lock any page, unlock all 530 * of them and return error 531 */ 532 for (j = 0; j < i; j++) 533 page_pp_unlock(ppa[j], 0, 1); 534 for (i = 0; i < npages; i++) 535 page_unlock(ppa[i]); 536 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0); 537 mutex_exit(&sp->shm_mlock); 538 err = ENOMEM; 539 goto out4; 540 } 541 } 542 mutex_exit(&sp->shm_mlock); 543 544 /* 545 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 546 * for the entire life of the segment. For example platforms 547 * that do not support Dynamic Reconfiguration. 548 */ 549 hat_flags = HAT_LOAD_SHARE; 550 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) 551 hat_flags |= HAT_LOAD_LOCK; 552 553 /* 554 * Load translations one lare page at a time 555 * to make sure we don't create mappings bigger than 556 * segment's size code in case underlying pages 557 * are shared with segvn's segment that uses bigger 558 * size code than we do. 559 */ 560 pgsz = page_get_pagesize(seg->s_szc); 561 pgcnt = page_get_pagecnt(seg->s_szc); 562 for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) { 563 sz = MIN(pgsz, ptob(npages - pidx)); 564 hat_memload_array(seg->s_as->a_hat, a, sz, 565 &ppa[pidx], sptd->spt_prot, hat_flags); 566 } 567 568 /* 569 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 570 * we will leave the pages locked SE_SHARED for the life 571 * of the ISM segment. This will prevent any calls to 572 * hat_pageunload() on this ISM segment for those platforms. 573 */ 574 if (!(hat_flags & HAT_LOAD_LOCK)) { 575 /* 576 * On platforms that support HAT_DYNAMIC_ISM_UNMAP, 577 * we no longer need to hold the SE_SHARED lock on the pages, 578 * since L_PAGELOCK and F_SOFTLOCK calls will grab the 579 * SE_SHARED lock on the pages as necessary. 580 */ 581 for (i = 0; i < npages; i++) 582 page_unlock(ppa[i]); 583 } 584 sptd->spt_pcachecnt = 0; 585 kmem_free(ppa, ((sizeof (page_t *)) * npages)); 586 sptd->spt_realsize = ptob(npages); 587 atomic_add_long(&spt_used, npages); 588 sptcargs->seg_spt = seg; 589 return (0); 590 591 out4: 592 seg->s_data = NULL; 593 kmem_free(vp, sizeof (*vp)); 594 cv_destroy(&sptd->spt_cv); 595 out3: 596 mutex_destroy(&sptd->spt_lock); 597 if ((sptcargs->flags & SHM_PAGEABLE) == 0) 598 kmem_free(ppa, (sizeof (*ppa) * npages)); 599 out2: 600 kmem_free(sptd, sizeof (*sptd)); 601 out1: 602 if ((sptcargs->flags & SHM_PAGEABLE) == 0) 603 anon_swap_restore(npages); 604 return (err); 605 } 606 607 /*ARGSUSED*/ 608 void 609 segspt_free_pages(struct seg *seg, caddr_t addr, size_t len) 610 { 611 struct page *pp; 612 struct spt_data *sptd = (struct spt_data *)seg->s_data; 613 pgcnt_t npages; 614 ulong_t anon_idx; 615 struct anon_map *amp; 616 struct anon *ap; 617 struct vnode *vp; 618 u_offset_t off; 619 uint_t hat_flags; 620 int root = 0; 621 pgcnt_t pgs, curnpgs = 0; 622 page_t *rootpp; 623 rctl_qty_t unlocked_bytes = 0; 624 kproject_t *proj; 625 kshmid_t *sp; 626 627 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 628 629 len = P2ROUNDUP(len, PAGESIZE); 630 631 npages = btop(len); 632 633 hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP; 634 if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) || 635 (sptd->spt_flags & SHM_PAGEABLE)) { 636 hat_flags = HAT_UNLOAD_UNMAP; 637 } 638 639 hat_unload(seg->s_as->a_hat, addr, len, hat_flags); 640 641 amp = sptd->spt_amp; 642 if (sptd->spt_flags & SHM_PAGEABLE) 643 npages = btop(amp->size); 644 645 ASSERT(amp != NULL); 646 647 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 648 sp = amp->a_sp; 649 proj = sp->shm_perm.ipc_proj; 650 mutex_enter(&sp->shm_mlock); 651 } 652 for (anon_idx = 0; anon_idx < npages; anon_idx++) { 653 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 654 if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) { 655 panic("segspt_free_pages: null app"); 656 /*NOTREACHED*/ 657 } 658 } else { 659 if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx)) 660 == NULL) 661 continue; 662 } 663 ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0); 664 swap_xlate(ap, &vp, &off); 665 666 /* 667 * If this platform supports HAT_DYNAMIC_ISM_UNMAP, 668 * the pages won't be having SE_SHARED lock at this 669 * point. 670 * 671 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 672 * the pages are still held SE_SHARED locked from the 673 * original segspt_create() 674 * 675 * Our goal is to get SE_EXCL lock on each page, remove 676 * permanent lock on it and invalidate the page. 677 */ 678 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 679 if (hat_flags == HAT_UNLOAD_UNMAP) 680 pp = page_lookup(vp, off, SE_EXCL); 681 else { 682 if ((pp = page_find(vp, off)) == NULL) { 683 panic("segspt_free_pages: " 684 "page not locked"); 685 /*NOTREACHED*/ 686 } 687 if (!page_tryupgrade(pp)) { 688 page_unlock(pp); 689 pp = page_lookup(vp, off, SE_EXCL); 690 } 691 } 692 if (pp == NULL) { 693 panic("segspt_free_pages: " 694 "page not in the system"); 695 /*NOTREACHED*/ 696 } 697 ASSERT(pp->p_lckcnt > 0); 698 page_pp_unlock(pp, 0, 1); 699 if (pp->p_lckcnt == 0) 700 unlocked_bytes += PAGESIZE; 701 } else { 702 if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL) 703 continue; 704 } 705 /* 706 * It's logical to invalidate the pages here as in most cases 707 * these were created by segspt. 708 */ 709 if (pp->p_szc != 0) { 710 if (root == 0) { 711 ASSERT(curnpgs == 0); 712 root = 1; 713 rootpp = pp; 714 pgs = curnpgs = page_get_pagecnt(pp->p_szc); 715 ASSERT(pgs > 1); 716 ASSERT(IS_P2ALIGNED(pgs, pgs)); 717 ASSERT(!(page_pptonum(pp) & (pgs - 1))); 718 curnpgs--; 719 } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) { 720 ASSERT(curnpgs == 1); 721 ASSERT(page_pptonum(pp) == 722 page_pptonum(rootpp) + (pgs - 1)); 723 page_destroy_pages(rootpp); 724 root = 0; 725 curnpgs = 0; 726 } else { 727 ASSERT(curnpgs > 1); 728 ASSERT(page_pptonum(pp) == 729 page_pptonum(rootpp) + (pgs - curnpgs)); 730 curnpgs--; 731 } 732 } else { 733 if (root != 0 || curnpgs != 0) { 734 panic("segspt_free_pages: bad large page"); 735 /*NOTREACHED*/ 736 } 737 /* 738 * Before destroying the pages, we need to take care 739 * of the rctl locked memory accounting. For that 740 * we need to calculte the unlocked_bytes. 741 */ 742 if (pp->p_lckcnt > 0) 743 unlocked_bytes += PAGESIZE; 744 /*LINTED: constant in conditional context */ 745 VN_DISPOSE(pp, B_INVAL, 0, kcred); 746 } 747 } 748 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 749 if (unlocked_bytes > 0) 750 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0); 751 mutex_exit(&sp->shm_mlock); 752 } 753 if (root != 0 || curnpgs != 0) { 754 panic("segspt_free_pages: bad large page"); 755 /*NOTREACHED*/ 756 } 757 758 /* 759 * mark that pages have been released 760 */ 761 sptd->spt_realsize = 0; 762 763 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 764 atomic_add_long(&spt_used, -npages); 765 anon_swap_restore(npages); 766 } 767 } 768 769 /* 770 * Get memory allocation policy info for specified address in given segment 771 */ 772 static lgrp_mem_policy_info_t * 773 segspt_getpolicy(struct seg *seg, caddr_t addr) 774 { 775 struct anon_map *amp; 776 ulong_t anon_index; 777 lgrp_mem_policy_info_t *policy_info; 778 struct spt_data *spt_data; 779 780 ASSERT(seg != NULL); 781 782 /* 783 * Get anon_map from segspt 784 * 785 * Assume that no lock needs to be held on anon_map, since 786 * it should be protected by its reference count which must be 787 * nonzero for an existing segment 788 * Need to grab readers lock on policy tree though 789 */ 790 spt_data = (struct spt_data *)seg->s_data; 791 if (spt_data == NULL) 792 return (NULL); 793 amp = spt_data->spt_amp; 794 ASSERT(amp->refcnt != 0); 795 796 /* 797 * Get policy info 798 * 799 * Assume starting anon index of 0 800 */ 801 anon_index = seg_page(seg, addr); 802 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 803 804 return (policy_info); 805 } 806 807 /* 808 * DISM only. 809 * Return locked pages over a given range. 810 * 811 * We will cache all DISM locked pages and save the pplist for the 812 * entire segment in the ppa field of the underlying DISM segment structure. 813 * Later, during a call to segspt_reclaim() we will use this ppa array 814 * to page_unlock() all of the pages and then we will free this ppa list. 815 */ 816 /*ARGSUSED*/ 817 static int 818 segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len, 819 struct page ***ppp, enum lock_type type, enum seg_rw rw) 820 { 821 struct shm_data *shmd = (struct shm_data *)seg->s_data; 822 struct seg *sptseg = shmd->shm_sptseg; 823 struct spt_data *sptd = sptseg->s_data; 824 pgcnt_t pg_idx, npages, tot_npages, npgs; 825 struct page **pplist, **pl, **ppa, *pp; 826 struct anon_map *amp; 827 spgcnt_t an_idx; 828 int ret = ENOTSUP; 829 uint_t pl_built = 0; 830 struct anon *ap; 831 struct vnode *vp; 832 u_offset_t off; 833 pgcnt_t claim_availrmem = 0; 834 uint_t szc; 835 836 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 837 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK); 838 839 /* 840 * We want to lock/unlock the entire ISM segment. Therefore, 841 * we will be using the underlying sptseg and it's base address 842 * and length for the caching arguments. 843 */ 844 ASSERT(sptseg); 845 ASSERT(sptd); 846 847 pg_idx = seg_page(seg, addr); 848 npages = btopr(len); 849 850 /* 851 * check if the request is larger than number of pages covered 852 * by amp 853 */ 854 if (pg_idx + npages > btopr(sptd->spt_amp->size)) { 855 *ppp = NULL; 856 return (ENOTSUP); 857 } 858 859 if (type == L_PAGEUNLOCK) { 860 ASSERT(sptd->spt_ppa != NULL); 861 862 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size, 863 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 864 865 /* 866 * If someone is blocked while unmapping, we purge 867 * segment page cache and thus reclaim pplist synchronously 868 * without waiting for seg_pasync_thread. This speeds up 869 * unmapping in cases where munmap(2) is called, while 870 * raw async i/o is still in progress or where a thread 871 * exits on data fault in a multithreaded application. 872 */ 873 if ((sptd->spt_flags & DISM_PPA_CHANGED) || 874 (AS_ISUNMAPWAIT(seg->s_as) && 875 shmd->shm_softlockcnt > 0)) { 876 segspt_purge(seg); 877 } 878 return (0); 879 } 880 881 /* The L_PAGELOCK case ... */ 882 883 if (sptd->spt_flags & DISM_PPA_CHANGED) { 884 segspt_purge(seg); 885 /* 886 * for DISM ppa needs to be rebuild since 887 * number of locked pages could be changed 888 */ 889 *ppp = NULL; 890 return (ENOTSUP); 891 } 892 893 /* 894 * First try to find pages in segment page cache, without 895 * holding the segment lock. 896 */ 897 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 898 S_WRITE, SEGP_FORCE_WIRED); 899 if (pplist != NULL) { 900 ASSERT(sptd->spt_ppa != NULL); 901 ASSERT(sptd->spt_ppa == pplist); 902 ppa = sptd->spt_ppa; 903 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 904 if (ppa[an_idx] == NULL) { 905 seg_pinactive(seg, NULL, seg->s_base, 906 sptd->spt_amp->size, ppa, 907 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 908 *ppp = NULL; 909 return (ENOTSUP); 910 } 911 if ((szc = ppa[an_idx]->p_szc) != 0) { 912 npgs = page_get_pagecnt(szc); 913 an_idx = P2ROUNDUP(an_idx + 1, npgs); 914 } else { 915 an_idx++; 916 } 917 } 918 /* 919 * Since we cache the entire DISM segment, we want to 920 * set ppp to point to the first slot that corresponds 921 * to the requested addr, i.e. pg_idx. 922 */ 923 *ppp = &(sptd->spt_ppa[pg_idx]); 924 return (0); 925 } 926 927 mutex_enter(&sptd->spt_lock); 928 /* 929 * try to find pages in segment page cache with mutex 930 */ 931 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 932 S_WRITE, SEGP_FORCE_WIRED); 933 if (pplist != NULL) { 934 ASSERT(sptd->spt_ppa != NULL); 935 ASSERT(sptd->spt_ppa == pplist); 936 ppa = sptd->spt_ppa; 937 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 938 if (ppa[an_idx] == NULL) { 939 mutex_exit(&sptd->spt_lock); 940 seg_pinactive(seg, NULL, seg->s_base, 941 sptd->spt_amp->size, ppa, 942 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 943 *ppp = NULL; 944 return (ENOTSUP); 945 } 946 if ((szc = ppa[an_idx]->p_szc) != 0) { 947 npgs = page_get_pagecnt(szc); 948 an_idx = P2ROUNDUP(an_idx + 1, npgs); 949 } else { 950 an_idx++; 951 } 952 } 953 /* 954 * Since we cache the entire DISM segment, we want to 955 * set ppp to point to the first slot that corresponds 956 * to the requested addr, i.e. pg_idx. 957 */ 958 mutex_exit(&sptd->spt_lock); 959 *ppp = &(sptd->spt_ppa[pg_idx]); 960 return (0); 961 } 962 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size, 963 SEGP_FORCE_WIRED) == SEGP_FAIL) { 964 mutex_exit(&sptd->spt_lock); 965 *ppp = NULL; 966 return (ENOTSUP); 967 } 968 969 /* 970 * No need to worry about protections because DISM pages are always rw. 971 */ 972 pl = pplist = NULL; 973 amp = sptd->spt_amp; 974 975 /* 976 * Do we need to build the ppa array? 977 */ 978 if (sptd->spt_ppa == NULL) { 979 pgcnt_t lpg_cnt = 0; 980 981 pl_built = 1; 982 tot_npages = btopr(sptd->spt_amp->size); 983 984 ASSERT(sptd->spt_pcachecnt == 0); 985 pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP); 986 pl = pplist; 987 988 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 989 for (an_idx = 0; an_idx < tot_npages; ) { 990 ap = anon_get_ptr(amp->ahp, an_idx); 991 /* 992 * Cache only mlocked pages. For large pages 993 * if one (constituent) page is mlocked 994 * all pages for that large page 995 * are cached also. This is for quick 996 * lookups of ppa array; 997 */ 998 if ((ap != NULL) && (lpg_cnt != 0 || 999 (sptd->spt_ppa_lckcnt[an_idx] != 0))) { 1000 1001 swap_xlate(ap, &vp, &off); 1002 pp = page_lookup(vp, off, SE_SHARED); 1003 ASSERT(pp != NULL); 1004 if (lpg_cnt == 0) { 1005 lpg_cnt++; 1006 /* 1007 * For a small page, we are done -- 1008 * lpg_count is reset to 0 below. 1009 * 1010 * For a large page, we are guaranteed 1011 * to find the anon structures of all 1012 * constituent pages and a non-zero 1013 * lpg_cnt ensures that we don't test 1014 * for mlock for these. We are done 1015 * when lpg_count reaches (npgs + 1). 1016 * If we are not the first constituent 1017 * page, restart at the first one. 1018 */ 1019 npgs = page_get_pagecnt(pp->p_szc); 1020 if (!IS_P2ALIGNED(an_idx, npgs)) { 1021 an_idx = P2ALIGN(an_idx, npgs); 1022 page_unlock(pp); 1023 continue; 1024 } 1025 } 1026 if (++lpg_cnt > npgs) 1027 lpg_cnt = 0; 1028 1029 /* 1030 * availrmem is decremented only 1031 * for unlocked pages 1032 */ 1033 if (sptd->spt_ppa_lckcnt[an_idx] == 0) 1034 claim_availrmem++; 1035 pplist[an_idx] = pp; 1036 } 1037 an_idx++; 1038 } 1039 ANON_LOCK_EXIT(&->a_rwlock); 1040 1041 if (claim_availrmem) { 1042 mutex_enter(&freemem_lock); 1043 if (availrmem < tune.t_minarmem + claim_availrmem) { 1044 mutex_exit(&freemem_lock); 1045 ret = ENOTSUP; 1046 claim_availrmem = 0; 1047 goto insert_fail; 1048 } else { 1049 availrmem -= claim_availrmem; 1050 } 1051 mutex_exit(&freemem_lock); 1052 } 1053 1054 sptd->spt_ppa = pl; 1055 } else { 1056 /* 1057 * We already have a valid ppa[]. 1058 */ 1059 pl = sptd->spt_ppa; 1060 } 1061 1062 ASSERT(pl != NULL); 1063 1064 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size, 1065 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED, 1066 segspt_reclaim); 1067 if (ret == SEGP_FAIL) { 1068 /* 1069 * seg_pinsert failed. We return 1070 * ENOTSUP, so that the as_pagelock() code will 1071 * then try the slower F_SOFTLOCK path. 1072 */ 1073 if (pl_built) { 1074 /* 1075 * No one else has referenced the ppa[]. 1076 * We created it and we need to destroy it. 1077 */ 1078 sptd->spt_ppa = NULL; 1079 } 1080 ret = ENOTSUP; 1081 goto insert_fail; 1082 } 1083 1084 /* 1085 * In either case, we increment softlockcnt on the 'real' segment. 1086 */ 1087 sptd->spt_pcachecnt++; 1088 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 1089 1090 ppa = sptd->spt_ppa; 1091 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 1092 if (ppa[an_idx] == NULL) { 1093 mutex_exit(&sptd->spt_lock); 1094 seg_pinactive(seg, NULL, seg->s_base, 1095 sptd->spt_amp->size, 1096 pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 1097 *ppp = NULL; 1098 return (ENOTSUP); 1099 } 1100 if ((szc = ppa[an_idx]->p_szc) != 0) { 1101 npgs = page_get_pagecnt(szc); 1102 an_idx = P2ROUNDUP(an_idx + 1, npgs); 1103 } else { 1104 an_idx++; 1105 } 1106 } 1107 /* 1108 * We can now drop the sptd->spt_lock since the ppa[] 1109 * exists and he have incremented pacachecnt. 1110 */ 1111 mutex_exit(&sptd->spt_lock); 1112 1113 /* 1114 * Since we cache the entire segment, we want to 1115 * set ppp to point to the first slot that corresponds 1116 * to the requested addr, i.e. pg_idx. 1117 */ 1118 *ppp = &(sptd->spt_ppa[pg_idx]); 1119 return (0); 1120 1121 insert_fail: 1122 /* 1123 * We will only reach this code if we tried and failed. 1124 * 1125 * And we can drop the lock on the dummy seg, once we've failed 1126 * to set up a new ppa[]. 1127 */ 1128 mutex_exit(&sptd->spt_lock); 1129 1130 if (pl_built) { 1131 if (claim_availrmem) { 1132 mutex_enter(&freemem_lock); 1133 availrmem += claim_availrmem; 1134 mutex_exit(&freemem_lock); 1135 } 1136 1137 /* 1138 * We created pl and we need to destroy it. 1139 */ 1140 pplist = pl; 1141 for (an_idx = 0; an_idx < tot_npages; an_idx++) { 1142 if (pplist[an_idx] != NULL) 1143 page_unlock(pplist[an_idx]); 1144 } 1145 kmem_free(pl, sizeof (page_t *) * tot_npages); 1146 } 1147 1148 if (shmd->shm_softlockcnt <= 0) { 1149 if (AS_ISUNMAPWAIT(seg->s_as)) { 1150 mutex_enter(&seg->s_as->a_contents); 1151 if (AS_ISUNMAPWAIT(seg->s_as)) { 1152 AS_CLRUNMAPWAIT(seg->s_as); 1153 cv_broadcast(&seg->s_as->a_cv); 1154 } 1155 mutex_exit(&seg->s_as->a_contents); 1156 } 1157 } 1158 *ppp = NULL; 1159 return (ret); 1160 } 1161 1162 1163 1164 /* 1165 * return locked pages over a given range. 1166 * 1167 * We will cache the entire ISM segment and save the pplist for the 1168 * entire segment in the ppa field of the underlying ISM segment structure. 1169 * Later, during a call to segspt_reclaim() we will use this ppa array 1170 * to page_unlock() all of the pages and then we will free this ppa list. 1171 */ 1172 /*ARGSUSED*/ 1173 static int 1174 segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len, 1175 struct page ***ppp, enum lock_type type, enum seg_rw rw) 1176 { 1177 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1178 struct seg *sptseg = shmd->shm_sptseg; 1179 struct spt_data *sptd = sptseg->s_data; 1180 pgcnt_t np, page_index, npages; 1181 caddr_t a, spt_base; 1182 struct page **pplist, **pl, *pp; 1183 struct anon_map *amp; 1184 ulong_t anon_index; 1185 int ret = ENOTSUP; 1186 uint_t pl_built = 0; 1187 struct anon *ap; 1188 struct vnode *vp; 1189 u_offset_t off; 1190 1191 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1192 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK); 1193 1194 1195 /* 1196 * We want to lock/unlock the entire ISM segment. Therefore, 1197 * we will be using the underlying sptseg and it's base address 1198 * and length for the caching arguments. 1199 */ 1200 ASSERT(sptseg); 1201 ASSERT(sptd); 1202 1203 if (sptd->spt_flags & SHM_PAGEABLE) { 1204 return (segspt_dismpagelock(seg, addr, len, ppp, type, rw)); 1205 } 1206 1207 page_index = seg_page(seg, addr); 1208 npages = btopr(len); 1209 1210 /* 1211 * check if the request is larger than number of pages covered 1212 * by amp 1213 */ 1214 if (page_index + npages > btopr(sptd->spt_amp->size)) { 1215 *ppp = NULL; 1216 return (ENOTSUP); 1217 } 1218 1219 if (type == L_PAGEUNLOCK) { 1220 1221 ASSERT(sptd->spt_ppa != NULL); 1222 1223 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size, 1224 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 1225 1226 /* 1227 * If someone is blocked while unmapping, we purge 1228 * segment page cache and thus reclaim pplist synchronously 1229 * without waiting for seg_pasync_thread. This speeds up 1230 * unmapping in cases where munmap(2) is called, while 1231 * raw async i/o is still in progress or where a thread 1232 * exits on data fault in a multithreaded application. 1233 */ 1234 if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 1235 segspt_purge(seg); 1236 } 1237 return (0); 1238 } 1239 1240 /* The L_PAGELOCK case... */ 1241 1242 /* 1243 * First try to find pages in segment page cache, without 1244 * holding the segment lock. 1245 */ 1246 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 1247 S_WRITE, SEGP_FORCE_WIRED); 1248 if (pplist != NULL) { 1249 ASSERT(sptd->spt_ppa == pplist); 1250 ASSERT(sptd->spt_ppa[page_index]); 1251 /* 1252 * Since we cache the entire ISM segment, we want to 1253 * set ppp to point to the first slot that corresponds 1254 * to the requested addr, i.e. page_index. 1255 */ 1256 *ppp = &(sptd->spt_ppa[page_index]); 1257 return (0); 1258 } 1259 1260 mutex_enter(&sptd->spt_lock); 1261 1262 /* 1263 * try to find pages in segment page cache 1264 */ 1265 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 1266 S_WRITE, SEGP_FORCE_WIRED); 1267 if (pplist != NULL) { 1268 ASSERT(sptd->spt_ppa == pplist); 1269 /* 1270 * Since we cache the entire segment, we want to 1271 * set ppp to point to the first slot that corresponds 1272 * to the requested addr, i.e. page_index. 1273 */ 1274 mutex_exit(&sptd->spt_lock); 1275 *ppp = &(sptd->spt_ppa[page_index]); 1276 return (0); 1277 } 1278 1279 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size, 1280 SEGP_FORCE_WIRED) == SEGP_FAIL) { 1281 mutex_exit(&sptd->spt_lock); 1282 *ppp = NULL; 1283 return (ENOTSUP); 1284 } 1285 1286 /* 1287 * No need to worry about protections because ISM pages 1288 * are always rw. 1289 */ 1290 pl = pplist = NULL; 1291 1292 /* 1293 * Do we need to build the ppa array? 1294 */ 1295 if (sptd->spt_ppa == NULL) { 1296 ASSERT(sptd->spt_ppa == pplist); 1297 1298 spt_base = sptseg->s_base; 1299 pl_built = 1; 1300 1301 /* 1302 * availrmem is decremented once during anon_swap_adjust() 1303 * and is incremented during the anon_unresv(), which is 1304 * called from shm_rm_amp() when the segment is destroyed. 1305 */ 1306 amp = sptd->spt_amp; 1307 ASSERT(amp != NULL); 1308 1309 /* pcachecnt is protected by sptd->spt_lock */ 1310 ASSERT(sptd->spt_pcachecnt == 0); 1311 pplist = kmem_zalloc(sizeof (page_t *) 1312 * btopr(sptd->spt_amp->size), KM_SLEEP); 1313 pl = pplist; 1314 1315 anon_index = seg_page(sptseg, spt_base); 1316 1317 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 1318 for (a = spt_base; a < (spt_base + sptd->spt_amp->size); 1319 a += PAGESIZE, anon_index++, pplist++) { 1320 ap = anon_get_ptr(amp->ahp, anon_index); 1321 ASSERT(ap != NULL); 1322 swap_xlate(ap, &vp, &off); 1323 pp = page_lookup(vp, off, SE_SHARED); 1324 ASSERT(pp != NULL); 1325 *pplist = pp; 1326 } 1327 ANON_LOCK_EXIT(&->a_rwlock); 1328 1329 if (a < (spt_base + sptd->spt_amp->size)) { 1330 ret = ENOTSUP; 1331 goto insert_fail; 1332 } 1333 sptd->spt_ppa = pl; 1334 } else { 1335 /* 1336 * We already have a valid ppa[]. 1337 */ 1338 pl = sptd->spt_ppa; 1339 } 1340 1341 ASSERT(pl != NULL); 1342 1343 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size, 1344 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED, 1345 segspt_reclaim); 1346 if (ret == SEGP_FAIL) { 1347 /* 1348 * seg_pinsert failed. We return 1349 * ENOTSUP, so that the as_pagelock() code will 1350 * then try the slower F_SOFTLOCK path. 1351 */ 1352 if (pl_built) { 1353 /* 1354 * No one else has referenced the ppa[]. 1355 * We created it and we need to destroy it. 1356 */ 1357 sptd->spt_ppa = NULL; 1358 } 1359 ret = ENOTSUP; 1360 goto insert_fail; 1361 } 1362 1363 /* 1364 * In either case, we increment softlockcnt on the 'real' segment. 1365 */ 1366 sptd->spt_pcachecnt++; 1367 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 1368 1369 /* 1370 * We can now drop the sptd->spt_lock since the ppa[] 1371 * exists and he have incremented pacachecnt. 1372 */ 1373 mutex_exit(&sptd->spt_lock); 1374 1375 /* 1376 * Since we cache the entire segment, we want to 1377 * set ppp to point to the first slot that corresponds 1378 * to the requested addr, i.e. page_index. 1379 */ 1380 *ppp = &(sptd->spt_ppa[page_index]); 1381 return (0); 1382 1383 insert_fail: 1384 /* 1385 * We will only reach this code if we tried and failed. 1386 * 1387 * And we can drop the lock on the dummy seg, once we've failed 1388 * to set up a new ppa[]. 1389 */ 1390 mutex_exit(&sptd->spt_lock); 1391 1392 if (pl_built) { 1393 /* 1394 * We created pl and we need to destroy it. 1395 */ 1396 pplist = pl; 1397 np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT); 1398 while (np) { 1399 page_unlock(*pplist); 1400 np--; 1401 pplist++; 1402 } 1403 kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size)); 1404 } 1405 if (shmd->shm_softlockcnt <= 0) { 1406 if (AS_ISUNMAPWAIT(seg->s_as)) { 1407 mutex_enter(&seg->s_as->a_contents); 1408 if (AS_ISUNMAPWAIT(seg->s_as)) { 1409 AS_CLRUNMAPWAIT(seg->s_as); 1410 cv_broadcast(&seg->s_as->a_cv); 1411 } 1412 mutex_exit(&seg->s_as->a_contents); 1413 } 1414 } 1415 *ppp = NULL; 1416 return (ret); 1417 } 1418 1419 /* 1420 * purge any cached pages in the I/O page cache 1421 */ 1422 static void 1423 segspt_purge(struct seg *seg) 1424 { 1425 seg_ppurge(seg, NULL, SEGP_FORCE_WIRED); 1426 } 1427 1428 static int 1429 segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist, 1430 enum seg_rw rw, int async) 1431 { 1432 struct seg *seg = (struct seg *)ptag; 1433 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1434 struct seg *sptseg; 1435 struct spt_data *sptd; 1436 pgcnt_t npages, i, free_availrmem = 0; 1437 int done = 0; 1438 1439 #ifdef lint 1440 addr = addr; 1441 #endif 1442 sptseg = shmd->shm_sptseg; 1443 sptd = sptseg->s_data; 1444 npages = (len >> PAGESHIFT); 1445 ASSERT(npages); 1446 ASSERT(sptd->spt_pcachecnt != 0); 1447 ASSERT(sptd->spt_ppa == pplist); 1448 ASSERT(npages == btopr(sptd->spt_amp->size)); 1449 ASSERT(async || AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1450 1451 /* 1452 * Acquire the lock on the dummy seg and destroy the 1453 * ppa array IF this is the last pcachecnt. 1454 */ 1455 mutex_enter(&sptd->spt_lock); 1456 if (--sptd->spt_pcachecnt == 0) { 1457 for (i = 0; i < npages; i++) { 1458 if (pplist[i] == NULL) { 1459 continue; 1460 } 1461 if (rw == S_WRITE) { 1462 hat_setrefmod(pplist[i]); 1463 } else { 1464 hat_setref(pplist[i]); 1465 } 1466 if ((sptd->spt_flags & SHM_PAGEABLE) && 1467 (sptd->spt_ppa_lckcnt[i] == 0)) 1468 free_availrmem++; 1469 page_unlock(pplist[i]); 1470 } 1471 if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) { 1472 mutex_enter(&freemem_lock); 1473 availrmem += free_availrmem; 1474 mutex_exit(&freemem_lock); 1475 } 1476 /* 1477 * Since we want to cach/uncache the entire ISM segment, 1478 * we will track the pplist in a segspt specific field 1479 * ppa, that is initialized at the time we add an entry to 1480 * the cache. 1481 */ 1482 ASSERT(sptd->spt_pcachecnt == 0); 1483 kmem_free(pplist, sizeof (page_t *) * npages); 1484 sptd->spt_ppa = NULL; 1485 sptd->spt_flags &= ~DISM_PPA_CHANGED; 1486 sptd->spt_gen++; 1487 cv_broadcast(&sptd->spt_cv); 1488 done = 1; 1489 } 1490 mutex_exit(&sptd->spt_lock); 1491 1492 /* 1493 * If we are pcache async thread or called via seg_ppurge_wiredpp() we 1494 * may not hold AS lock (in this case async argument is not 0). This 1495 * means if softlockcnt drops to 0 after the decrement below address 1496 * space may get freed. We can't allow it since after softlock 1497 * derement to 0 we still need to access as structure for possible 1498 * wakeup of unmap waiters. To prevent the disappearance of as we take 1499 * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes 1500 * this mutex as a barrier to make sure this routine completes before 1501 * segment is freed. 1502 * 1503 * The second complication we have to deal with in async case is a 1504 * possibility of missed wake up of unmap wait thread. When we don't 1505 * hold as lock here we may take a_contents lock before unmap wait 1506 * thread that was first to see softlockcnt was still not 0. As a 1507 * result we'll fail to wake up an unmap wait thread. To avoid this 1508 * race we set nounmapwait flag in as structure if we drop softlockcnt 1509 * to 0 if async is not 0. unmapwait thread 1510 * will not block if this flag is set. 1511 */ 1512 if (async) 1513 mutex_enter(&shmd->shm_segfree_syncmtx); 1514 1515 /* 1516 * Now decrement softlockcnt. 1517 */ 1518 ASSERT(shmd->shm_softlockcnt > 0); 1519 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 1520 1521 if (shmd->shm_softlockcnt <= 0) { 1522 if (async || AS_ISUNMAPWAIT(seg->s_as)) { 1523 mutex_enter(&seg->s_as->a_contents); 1524 if (async) 1525 AS_SETNOUNMAPWAIT(seg->s_as); 1526 if (AS_ISUNMAPWAIT(seg->s_as)) { 1527 AS_CLRUNMAPWAIT(seg->s_as); 1528 cv_broadcast(&seg->s_as->a_cv); 1529 } 1530 mutex_exit(&seg->s_as->a_contents); 1531 } 1532 } 1533 1534 if (async) 1535 mutex_exit(&shmd->shm_segfree_syncmtx); 1536 1537 return (done); 1538 } 1539 1540 /* 1541 * Do a F_SOFTUNLOCK call over the range requested. 1542 * The range must have already been F_SOFTLOCK'ed. 1543 * 1544 * The calls to acquire and release the anon map lock mutex were 1545 * removed in order to avoid a deadly embrace during a DR 1546 * memory delete operation. (Eg. DR blocks while waiting for a 1547 * exclusive lock on a page that is being used for kaio; the 1548 * thread that will complete the kaio and call segspt_softunlock 1549 * blocks on the anon map lock; another thread holding the anon 1550 * map lock blocks on another page lock via the segspt_shmfault 1551 * -> page_lookup -> page_lookup_create -> page_lock_es code flow.) 1552 * 1553 * The appropriateness of the removal is based upon the following: 1554 * 1. If we are holding a segment's reader lock and the page is held 1555 * shared, then the corresponding element in anonmap which points to 1556 * anon struct cannot change and there is no need to acquire the 1557 * anonymous map lock. 1558 * 2. Threads in segspt_softunlock have a reader lock on the segment 1559 * and already have the shared page lock, so we are guaranteed that 1560 * the anon map slot cannot change and therefore can call anon_get_ptr() 1561 * without grabbing the anonymous map lock. 1562 * 3. Threads that softlock a shared page break copy-on-write, even if 1563 * its a read. Thus cow faults can be ignored with respect to soft 1564 * unlocking, since the breaking of cow means that the anon slot(s) will 1565 * not be shared. 1566 */ 1567 static void 1568 segspt_softunlock(struct seg *seg, caddr_t sptseg_addr, 1569 size_t len, enum seg_rw rw) 1570 { 1571 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1572 struct seg *sptseg; 1573 struct spt_data *sptd; 1574 page_t *pp; 1575 caddr_t adr; 1576 struct vnode *vp; 1577 u_offset_t offset; 1578 ulong_t anon_index; 1579 struct anon_map *amp; /* XXX - for locknest */ 1580 struct anon *ap = NULL; 1581 pgcnt_t npages; 1582 1583 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1584 1585 sptseg = shmd->shm_sptseg; 1586 sptd = sptseg->s_data; 1587 1588 /* 1589 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 1590 * and therefore their pages are SE_SHARED locked 1591 * for the entire life of the segment. 1592 */ 1593 if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) && 1594 ((sptd->spt_flags & SHM_PAGEABLE) == 0)) { 1595 goto softlock_decrement; 1596 } 1597 1598 /* 1599 * Any thread is free to do a page_find and 1600 * page_unlock() on the pages within this seg. 1601 * 1602 * We are already holding the as->a_lock on the user's 1603 * real segment, but we need to hold the a_lock on the 1604 * underlying dummy as. This is mostly to satisfy the 1605 * underlying HAT layer. 1606 */ 1607 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 1608 hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len); 1609 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 1610 1611 amp = sptd->spt_amp; 1612 ASSERT(amp != NULL); 1613 anon_index = seg_page(sptseg, sptseg_addr); 1614 1615 for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) { 1616 ap = anon_get_ptr(amp->ahp, anon_index++); 1617 ASSERT(ap != NULL); 1618 swap_xlate(ap, &vp, &offset); 1619 1620 /* 1621 * Use page_find() instead of page_lookup() to 1622 * find the page since we know that it has a 1623 * "shared" lock. 1624 */ 1625 pp = page_find(vp, offset); 1626 ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1)); 1627 if (pp == NULL) { 1628 panic("segspt_softunlock: " 1629 "addr %p, ap %p, vp %p, off %llx", 1630 (void *)adr, (void *)ap, (void *)vp, offset); 1631 /*NOTREACHED*/ 1632 } 1633 1634 if (rw == S_WRITE) { 1635 hat_setrefmod(pp); 1636 } else if (rw != S_OTHER) { 1637 hat_setref(pp); 1638 } 1639 page_unlock(pp); 1640 } 1641 1642 softlock_decrement: 1643 npages = btopr(len); 1644 ASSERT(shmd->shm_softlockcnt >= npages); 1645 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages); 1646 if (shmd->shm_softlockcnt == 0) { 1647 /* 1648 * All SOFTLOCKS are gone. Wakeup any waiting 1649 * unmappers so they can try again to unmap. 1650 * Check for waiters first without the mutex 1651 * held so we don't always grab the mutex on 1652 * softunlocks. 1653 */ 1654 if (AS_ISUNMAPWAIT(seg->s_as)) { 1655 mutex_enter(&seg->s_as->a_contents); 1656 if (AS_ISUNMAPWAIT(seg->s_as)) { 1657 AS_CLRUNMAPWAIT(seg->s_as); 1658 cv_broadcast(&seg->s_as->a_cv); 1659 } 1660 mutex_exit(&seg->s_as->a_contents); 1661 } 1662 } 1663 } 1664 1665 int 1666 segspt_shmattach(struct seg *seg, caddr_t *argsp) 1667 { 1668 struct shm_data *shmd_arg = (struct shm_data *)argsp; 1669 struct shm_data *shmd; 1670 struct anon_map *shm_amp = shmd_arg->shm_amp; 1671 struct spt_data *sptd; 1672 int error = 0; 1673 1674 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 1675 1676 shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP); 1677 if (shmd == NULL) 1678 return (ENOMEM); 1679 1680 shmd->shm_sptas = shmd_arg->shm_sptas; 1681 shmd->shm_amp = shm_amp; 1682 shmd->shm_sptseg = shmd_arg->shm_sptseg; 1683 1684 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0, 1685 NULL, 0, seg->s_size); 1686 1687 mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL); 1688 1689 seg->s_data = (void *)shmd; 1690 seg->s_ops = &segspt_shmops; 1691 seg->s_szc = shmd->shm_sptseg->s_szc; 1692 sptd = shmd->shm_sptseg->s_data; 1693 1694 if (sptd->spt_flags & SHM_PAGEABLE) { 1695 if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size), 1696 KM_NOSLEEP)) == NULL) { 1697 seg->s_data = (void *)NULL; 1698 kmem_free(shmd, (sizeof (*shmd))); 1699 return (ENOMEM); 1700 } 1701 shmd->shm_lckpgs = 0; 1702 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 1703 if ((error = hat_share(seg->s_as->a_hat, seg->s_base, 1704 shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 1705 seg->s_size, seg->s_szc)) != 0) { 1706 kmem_free(shmd->shm_vpage, 1707 btopr(shm_amp->size)); 1708 } 1709 } 1710 } else { 1711 error = hat_share(seg->s_as->a_hat, seg->s_base, 1712 shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 1713 seg->s_size, seg->s_szc); 1714 } 1715 if (error) { 1716 seg->s_szc = 0; 1717 seg->s_data = (void *)NULL; 1718 kmem_free(shmd, (sizeof (*shmd))); 1719 } else { 1720 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 1721 shm_amp->refcnt++; 1722 ANON_LOCK_EXIT(&shm_amp->a_rwlock); 1723 } 1724 return (error); 1725 } 1726 1727 int 1728 segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize) 1729 { 1730 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1731 int reclaim = 1; 1732 1733 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 1734 retry: 1735 if (shmd->shm_softlockcnt > 0) { 1736 if (reclaim == 1) { 1737 segspt_purge(seg); 1738 reclaim = 0; 1739 goto retry; 1740 } 1741 return (EAGAIN); 1742 } 1743 1744 if (ssize != seg->s_size) { 1745 #ifdef DEBUG 1746 cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n", 1747 ssize, seg->s_size); 1748 #endif 1749 return (EINVAL); 1750 } 1751 1752 (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK, 1753 NULL, 0); 1754 hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc); 1755 1756 seg_free(seg); 1757 1758 return (0); 1759 } 1760 1761 void 1762 segspt_shmfree(struct seg *seg) 1763 { 1764 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1765 struct anon_map *shm_amp = shmd->shm_amp; 1766 1767 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 1768 1769 (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0, 1770 MC_UNLOCK, NULL, 0); 1771 1772 /* 1773 * Need to increment refcnt when attaching 1774 * and decrement when detaching because of dup(). 1775 */ 1776 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 1777 shm_amp->refcnt--; 1778 ANON_LOCK_EXIT(&shm_amp->a_rwlock); 1779 1780 if (shmd->shm_vpage) { /* only for DISM */ 1781 kmem_free(shmd->shm_vpage, btopr(shm_amp->size)); 1782 shmd->shm_vpage = NULL; 1783 } 1784 1785 /* 1786 * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's 1787 * still working with this segment without holding as lock. 1788 */ 1789 ASSERT(shmd->shm_softlockcnt == 0); 1790 mutex_enter(&shmd->shm_segfree_syncmtx); 1791 mutex_destroy(&shmd->shm_segfree_syncmtx); 1792 1793 kmem_free(shmd, sizeof (*shmd)); 1794 } 1795 1796 /*ARGSUSED*/ 1797 int 1798 segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 1799 { 1800 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1801 1802 /* 1803 * Shared page table is more than shared mapping. 1804 * Individual process sharing page tables can't change prot 1805 * because there is only one set of page tables. 1806 * This will be allowed after private page table is 1807 * supported. 1808 */ 1809 /* need to return correct status error? */ 1810 return (0); 1811 } 1812 1813 1814 faultcode_t 1815 segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr, 1816 size_t len, enum fault_type type, enum seg_rw rw) 1817 { 1818 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1819 struct seg *sptseg = shmd->shm_sptseg; 1820 struct as *curspt = shmd->shm_sptas; 1821 struct spt_data *sptd = sptseg->s_data; 1822 pgcnt_t npages; 1823 size_t size; 1824 caddr_t segspt_addr, shm_addr; 1825 page_t **ppa; 1826 int i; 1827 ulong_t an_idx = 0; 1828 int err = 0; 1829 int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0); 1830 size_t pgsz; 1831 pgcnt_t pgcnt; 1832 caddr_t a; 1833 pgcnt_t pidx; 1834 1835 #ifdef lint 1836 hat = hat; 1837 #endif 1838 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 1839 1840 /* 1841 * Because of the way spt is implemented 1842 * the realsize of the segment does not have to be 1843 * equal to the segment size itself. The segment size is 1844 * often in multiples of a page size larger than PAGESIZE. 1845 * The realsize is rounded up to the nearest PAGESIZE 1846 * based on what the user requested. This is a bit of 1847 * ungliness that is historical but not easily fixed 1848 * without re-designing the higher levels of ISM. 1849 */ 1850 ASSERT(addr >= seg->s_base); 1851 if (((addr + len) - seg->s_base) > sptd->spt_realsize) 1852 return (FC_NOMAP); 1853 /* 1854 * For all of the following cases except F_PROT, we need to 1855 * make any necessary adjustments to addr and len 1856 * and get all of the necessary page_t's into an array called ppa[]. 1857 * 1858 * The code in shmat() forces base addr and len of ISM segment 1859 * to be aligned to largest page size supported. Therefore, 1860 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 1861 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 1862 * in large pagesize chunks, or else we will screw up the HAT 1863 * layer by calling hat_memload_array() with differing page sizes 1864 * over a given virtual range. 1865 */ 1866 pgsz = page_get_pagesize(sptseg->s_szc); 1867 pgcnt = page_get_pagecnt(sptseg->s_szc); 1868 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 1869 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 1870 npages = btopr(size); 1871 1872 /* 1873 * Now we need to convert from addr in segshm to addr in segspt. 1874 */ 1875 an_idx = seg_page(seg, shm_addr); 1876 segspt_addr = sptseg->s_base + ptob(an_idx); 1877 1878 ASSERT((segspt_addr + ptob(npages)) <= 1879 (sptseg->s_base + sptd->spt_realsize)); 1880 ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size)); 1881 1882 switch (type) { 1883 1884 case F_SOFTLOCK: 1885 1886 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 1887 /* 1888 * Fall through to the F_INVAL case to load up the hat layer 1889 * entries with the HAT_LOAD_LOCK flag. 1890 */ 1891 /* FALLTHRU */ 1892 case F_INVAL: 1893 1894 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 1895 return (FC_NOMAP); 1896 1897 ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP); 1898 1899 err = spt_anon_getpages(sptseg, segspt_addr, size, ppa); 1900 if (err != 0) { 1901 if (type == F_SOFTLOCK) { 1902 atomic_add_long((ulong_t *)( 1903 &(shmd->shm_softlockcnt)), -npages); 1904 } 1905 goto dism_err; 1906 } 1907 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 1908 a = segspt_addr; 1909 pidx = 0; 1910 if (type == F_SOFTLOCK) { 1911 1912 /* 1913 * Load up the translation keeping it 1914 * locked and don't unlock the page. 1915 */ 1916 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 1917 hat_memload_array(sptseg->s_as->a_hat, 1918 a, pgsz, &ppa[pidx], sptd->spt_prot, 1919 HAT_LOAD_LOCK | HAT_LOAD_SHARE); 1920 } 1921 } else { 1922 if (hat == seg->s_as->a_hat) { 1923 1924 /* 1925 * Migrate pages marked for migration 1926 */ 1927 if (lgrp_optimizations()) 1928 page_migrate(seg, shm_addr, ppa, 1929 npages); 1930 1931 /* CPU HAT */ 1932 for (; pidx < npages; 1933 a += pgsz, pidx += pgcnt) { 1934 hat_memload_array(sptseg->s_as->a_hat, 1935 a, pgsz, &ppa[pidx], 1936 sptd->spt_prot, 1937 HAT_LOAD_SHARE); 1938 } 1939 } else { 1940 /* XHAT. Pass real address */ 1941 hat_memload_array(hat, shm_addr, 1942 size, ppa, sptd->spt_prot, HAT_LOAD_SHARE); 1943 } 1944 1945 /* 1946 * And now drop the SE_SHARED lock(s). 1947 */ 1948 if (dyn_ism_unmap) { 1949 for (i = 0; i < npages; i++) { 1950 page_unlock(ppa[i]); 1951 } 1952 } 1953 } 1954 1955 if (!dyn_ism_unmap) { 1956 if (hat_share(seg->s_as->a_hat, shm_addr, 1957 curspt->a_hat, segspt_addr, ptob(npages), 1958 seg->s_szc) != 0) { 1959 panic("hat_share err in DISM fault"); 1960 /* NOTREACHED */ 1961 } 1962 if (type == F_INVAL) { 1963 for (i = 0; i < npages; i++) { 1964 page_unlock(ppa[i]); 1965 } 1966 } 1967 } 1968 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 1969 dism_err: 1970 kmem_free(ppa, npages * sizeof (page_t *)); 1971 return (err); 1972 1973 case F_SOFTUNLOCK: 1974 1975 /* 1976 * This is a bit ugly, we pass in the real seg pointer, 1977 * but the segspt_addr is the virtual address within the 1978 * dummy seg. 1979 */ 1980 segspt_softunlock(seg, segspt_addr, size, rw); 1981 return (0); 1982 1983 case F_PROT: 1984 1985 /* 1986 * This takes care of the unusual case where a user 1987 * allocates a stack in shared memory and a register 1988 * window overflow is written to that stack page before 1989 * it is otherwise modified. 1990 * 1991 * We can get away with this because ISM segments are 1992 * always rw. Other than this unusual case, there 1993 * should be no instances of protection violations. 1994 */ 1995 return (0); 1996 1997 default: 1998 #ifdef DEBUG 1999 panic("segspt_dismfault default type?"); 2000 #else 2001 return (FC_NOMAP); 2002 #endif 2003 } 2004 } 2005 2006 2007 faultcode_t 2008 segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr, 2009 size_t len, enum fault_type type, enum seg_rw rw) 2010 { 2011 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2012 struct seg *sptseg = shmd->shm_sptseg; 2013 struct as *curspt = shmd->shm_sptas; 2014 struct spt_data *sptd = sptseg->s_data; 2015 pgcnt_t npages; 2016 size_t size; 2017 caddr_t sptseg_addr, shm_addr; 2018 page_t *pp, **ppa; 2019 int i; 2020 u_offset_t offset; 2021 ulong_t anon_index = 0; 2022 struct vnode *vp; 2023 struct anon_map *amp; /* XXX - for locknest */ 2024 struct anon *ap = NULL; 2025 size_t pgsz; 2026 pgcnt_t pgcnt; 2027 caddr_t a; 2028 pgcnt_t pidx; 2029 size_t sz; 2030 2031 #ifdef lint 2032 hat = hat; 2033 #endif 2034 2035 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2036 2037 if (sptd->spt_flags & SHM_PAGEABLE) { 2038 return (segspt_dismfault(hat, seg, addr, len, type, rw)); 2039 } 2040 2041 /* 2042 * Because of the way spt is implemented 2043 * the realsize of the segment does not have to be 2044 * equal to the segment size itself. The segment size is 2045 * often in multiples of a page size larger than PAGESIZE. 2046 * The realsize is rounded up to the nearest PAGESIZE 2047 * based on what the user requested. This is a bit of 2048 * ungliness that is historical but not easily fixed 2049 * without re-designing the higher levels of ISM. 2050 */ 2051 ASSERT(addr >= seg->s_base); 2052 if (((addr + len) - seg->s_base) > sptd->spt_realsize) 2053 return (FC_NOMAP); 2054 /* 2055 * For all of the following cases except F_PROT, we need to 2056 * make any necessary adjustments to addr and len 2057 * and get all of the necessary page_t's into an array called ppa[]. 2058 * 2059 * The code in shmat() forces base addr and len of ISM segment 2060 * to be aligned to largest page size supported. Therefore, 2061 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 2062 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 2063 * in large pagesize chunks, or else we will screw up the HAT 2064 * layer by calling hat_memload_array() with differing page sizes 2065 * over a given virtual range. 2066 */ 2067 pgsz = page_get_pagesize(sptseg->s_szc); 2068 pgcnt = page_get_pagecnt(sptseg->s_szc); 2069 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 2070 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 2071 npages = btopr(size); 2072 2073 /* 2074 * Now we need to convert from addr in segshm to addr in segspt. 2075 */ 2076 anon_index = seg_page(seg, shm_addr); 2077 sptseg_addr = sptseg->s_base + ptob(anon_index); 2078 2079 /* 2080 * And now we may have to adjust npages downward if we have 2081 * exceeded the realsize of the segment or initial anon 2082 * allocations. 2083 */ 2084 if ((sptseg_addr + ptob(npages)) > 2085 (sptseg->s_base + sptd->spt_realsize)) 2086 size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr; 2087 2088 npages = btopr(size); 2089 2090 ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size)); 2091 ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0); 2092 2093 switch (type) { 2094 2095 case F_SOFTLOCK: 2096 2097 /* 2098 * availrmem is decremented once during anon_swap_adjust() 2099 * and is incremented during the anon_unresv(), which is 2100 * called from shm_rm_amp() when the segment is destroyed. 2101 */ 2102 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 2103 /* 2104 * Some platforms assume that ISM pages are SE_SHARED 2105 * locked for the entire life of the segment. 2106 */ 2107 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) 2108 return (0); 2109 /* 2110 * Fall through to the F_INVAL case to load up the hat layer 2111 * entries with the HAT_LOAD_LOCK flag. 2112 */ 2113 2114 /* FALLTHRU */ 2115 case F_INVAL: 2116 2117 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 2118 return (FC_NOMAP); 2119 2120 /* 2121 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP 2122 * may still rely on this call to hat_share(). That 2123 * would imply that those hat's can fault on a 2124 * HAT_LOAD_LOCK translation, which would seem 2125 * contradictory. 2126 */ 2127 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2128 if (hat_share(seg->s_as->a_hat, seg->s_base, 2129 curspt->a_hat, sptseg->s_base, 2130 sptseg->s_size, sptseg->s_szc) != 0) { 2131 panic("hat_share error in ISM fault"); 2132 /*NOTREACHED*/ 2133 } 2134 return (0); 2135 } 2136 ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP); 2137 2138 /* 2139 * I see no need to lock the real seg, 2140 * here, because all of our work will be on the underlying 2141 * dummy seg. 2142 * 2143 * sptseg_addr and npages now account for large pages. 2144 */ 2145 amp = sptd->spt_amp; 2146 ASSERT(amp != NULL); 2147 anon_index = seg_page(sptseg, sptseg_addr); 2148 2149 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2150 for (i = 0; i < npages; i++) { 2151 ap = anon_get_ptr(amp->ahp, anon_index++); 2152 ASSERT(ap != NULL); 2153 swap_xlate(ap, &vp, &offset); 2154 pp = page_lookup(vp, offset, SE_SHARED); 2155 ASSERT(pp != NULL); 2156 ppa[i] = pp; 2157 } 2158 ANON_LOCK_EXIT(&->a_rwlock); 2159 ASSERT(i == npages); 2160 2161 /* 2162 * We are already holding the as->a_lock on the user's 2163 * real segment, but we need to hold the a_lock on the 2164 * underlying dummy as. This is mostly to satisfy the 2165 * underlying HAT layer. 2166 */ 2167 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER); 2168 a = sptseg_addr; 2169 pidx = 0; 2170 if (type == F_SOFTLOCK) { 2171 /* 2172 * Load up the translation keeping it 2173 * locked and don't unlock the page. 2174 */ 2175 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 2176 sz = MIN(pgsz, ptob(npages - pidx)); 2177 hat_memload_array(sptseg->s_as->a_hat, a, 2178 sz, &ppa[pidx], sptd->spt_prot, 2179 HAT_LOAD_LOCK | HAT_LOAD_SHARE); 2180 } 2181 } else { 2182 if (hat == seg->s_as->a_hat) { 2183 2184 /* 2185 * Migrate pages marked for migration. 2186 */ 2187 if (lgrp_optimizations()) 2188 page_migrate(seg, shm_addr, ppa, 2189 npages); 2190 2191 /* CPU HAT */ 2192 for (; pidx < npages; 2193 a += pgsz, pidx += pgcnt) { 2194 sz = MIN(pgsz, ptob(npages - pidx)); 2195 hat_memload_array(sptseg->s_as->a_hat, 2196 a, sz, &ppa[pidx], 2197 sptd->spt_prot, HAT_LOAD_SHARE); 2198 } 2199 } else { 2200 /* XHAT. Pass real address */ 2201 hat_memload_array(hat, shm_addr, 2202 ptob(npages), ppa, sptd->spt_prot, 2203 HAT_LOAD_SHARE); 2204 } 2205 2206 /* 2207 * And now drop the SE_SHARED lock(s). 2208 */ 2209 for (i = 0; i < npages; i++) 2210 page_unlock(ppa[i]); 2211 } 2212 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock); 2213 2214 kmem_free(ppa, sizeof (page_t *) * npages); 2215 return (0); 2216 case F_SOFTUNLOCK: 2217 2218 /* 2219 * This is a bit ugly, we pass in the real seg pointer, 2220 * but the sptseg_addr is the virtual address within the 2221 * dummy seg. 2222 */ 2223 segspt_softunlock(seg, sptseg_addr, ptob(npages), rw); 2224 return (0); 2225 2226 case F_PROT: 2227 2228 /* 2229 * This takes care of the unusual case where a user 2230 * allocates a stack in shared memory and a register 2231 * window overflow is written to that stack page before 2232 * it is otherwise modified. 2233 * 2234 * We can get away with this because ISM segments are 2235 * always rw. Other than this unusual case, there 2236 * should be no instances of protection violations. 2237 */ 2238 return (0); 2239 2240 default: 2241 #ifdef DEBUG 2242 cmn_err(CE_WARN, "segspt_shmfault default type?"); 2243 #endif 2244 return (FC_NOMAP); 2245 } 2246 } 2247 2248 /*ARGSUSED*/ 2249 static faultcode_t 2250 segspt_shmfaulta(struct seg *seg, caddr_t addr) 2251 { 2252 return (0); 2253 } 2254 2255 /*ARGSUSED*/ 2256 static int 2257 segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta) 2258 { 2259 return (0); 2260 } 2261 2262 /* 2263 * duplicate the shared page tables 2264 */ 2265 int 2266 segspt_shmdup(struct seg *seg, struct seg *newseg) 2267 { 2268 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2269 struct anon_map *amp = shmd->shm_amp; 2270 struct shm_data *shmd_new; 2271 struct seg *spt_seg = shmd->shm_sptseg; 2272 struct spt_data *sptd = spt_seg->s_data; 2273 int error = 0; 2274 2275 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 2276 2277 shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP); 2278 newseg->s_data = (void *)shmd_new; 2279 shmd_new->shm_sptas = shmd->shm_sptas; 2280 shmd_new->shm_amp = amp; 2281 shmd_new->shm_sptseg = shmd->shm_sptseg; 2282 newseg->s_ops = &segspt_shmops; 2283 newseg->s_szc = seg->s_szc; 2284 ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc); 2285 2286 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 2287 amp->refcnt++; 2288 ANON_LOCK_EXIT(&->a_rwlock); 2289 2290 if (sptd->spt_flags & SHM_PAGEABLE) { 2291 shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP); 2292 shmd_new->shm_lckpgs = 0; 2293 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2294 if ((error = hat_share(newseg->s_as->a_hat, 2295 newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR, 2296 seg->s_size, seg->s_szc)) != 0) { 2297 kmem_free(shmd_new->shm_vpage, 2298 btopr(amp->size)); 2299 } 2300 } 2301 return (error); 2302 } else { 2303 return (hat_share(newseg->s_as->a_hat, newseg->s_base, 2304 shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size, 2305 seg->s_szc)); 2306 2307 } 2308 } 2309 2310 /*ARGSUSED*/ 2311 int 2312 segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 2313 { 2314 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2315 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2316 2317 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2318 2319 /* 2320 * ISM segment is always rw. 2321 */ 2322 return (((sptd->spt_prot & prot) != prot) ? EACCES : 0); 2323 } 2324 2325 /* 2326 * Return an array of locked large pages, for empty slots allocate 2327 * private zero-filled anon pages. 2328 */ 2329 static int 2330 spt_anon_getpages( 2331 struct seg *sptseg, 2332 caddr_t sptaddr, 2333 size_t len, 2334 page_t *ppa[]) 2335 { 2336 struct spt_data *sptd = sptseg->s_data; 2337 struct anon_map *amp = sptd->spt_amp; 2338 enum seg_rw rw = sptd->spt_prot; 2339 uint_t szc = sptseg->s_szc; 2340 size_t pg_sz, share_sz = page_get_pagesize(szc); 2341 pgcnt_t lp_npgs; 2342 caddr_t lp_addr, e_sptaddr; 2343 uint_t vpprot, ppa_szc = 0; 2344 struct vpage *vpage = NULL; 2345 ulong_t j, ppa_idx; 2346 int err, ierr = 0; 2347 pgcnt_t an_idx; 2348 anon_sync_obj_t cookie; 2349 int anon_locked = 0; 2350 pgcnt_t amp_pgs; 2351 2352 2353 ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz)); 2354 ASSERT(len != 0); 2355 2356 pg_sz = share_sz; 2357 lp_npgs = btop(pg_sz); 2358 lp_addr = sptaddr; 2359 e_sptaddr = sptaddr + len; 2360 an_idx = seg_page(sptseg, sptaddr); 2361 ppa_idx = 0; 2362 2363 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2364 2365 amp_pgs = page_get_pagecnt(amp->a_szc); 2366 2367 /*CONSTCOND*/ 2368 while (1) { 2369 for (; lp_addr < e_sptaddr; 2370 an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) { 2371 2372 /* 2373 * If we're currently locked, and we get to a new 2374 * page, unlock our current anon chunk. 2375 */ 2376 if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) { 2377 anon_array_exit(&cookie); 2378 anon_locked = 0; 2379 } 2380 if (!anon_locked) { 2381 anon_array_enter(amp, an_idx, &cookie); 2382 anon_locked = 1; 2383 } 2384 ppa_szc = (uint_t)-1; 2385 ierr = anon_map_getpages(amp, an_idx, szc, sptseg, 2386 lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx], 2387 &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred); 2388 2389 if (ierr != 0) { 2390 if (ierr > 0) { 2391 err = FC_MAKE_ERR(ierr); 2392 goto lpgs_err; 2393 } 2394 break; 2395 } 2396 } 2397 if (lp_addr == e_sptaddr) { 2398 break; 2399 } 2400 ASSERT(lp_addr < e_sptaddr); 2401 2402 /* 2403 * ierr == -1 means we failed to allocate a large page. 2404 * so do a size down operation. 2405 * 2406 * ierr == -2 means some other process that privately shares 2407 * pages with this process has allocated a larger page and we 2408 * need to retry with larger pages. So do a size up 2409 * operation. This relies on the fact that large pages are 2410 * never partially shared i.e. if we share any constituent 2411 * page of a large page with another process we must share the 2412 * entire large page. Note this cannot happen for SOFTLOCK 2413 * case, unless current address (lpaddr) is at the beginning 2414 * of the next page size boundary because the other process 2415 * couldn't have relocated locked pages. 2416 */ 2417 ASSERT(ierr == -1 || ierr == -2); 2418 if (segvn_anypgsz) { 2419 ASSERT(ierr == -2 || szc != 0); 2420 ASSERT(ierr == -1 || szc < sptseg->s_szc); 2421 szc = (ierr == -1) ? szc - 1 : szc + 1; 2422 } else { 2423 /* 2424 * For faults and segvn_anypgsz == 0 2425 * we need to be careful not to loop forever 2426 * if existing page is found with szc other 2427 * than 0 or seg->s_szc. This could be due 2428 * to page relocations on behalf of DR or 2429 * more likely large page creation. For this 2430 * case simply re-size to existing page's szc 2431 * if returned by anon_map_getpages(). 2432 */ 2433 if (ppa_szc == (uint_t)-1) { 2434 szc = (ierr == -1) ? 0 : sptseg->s_szc; 2435 } else { 2436 ASSERT(ppa_szc <= sptseg->s_szc); 2437 ASSERT(ierr == -2 || ppa_szc < szc); 2438 ASSERT(ierr == -1 || ppa_szc > szc); 2439 szc = ppa_szc; 2440 } 2441 } 2442 pg_sz = page_get_pagesize(szc); 2443 lp_npgs = btop(pg_sz); 2444 ASSERT(IS_P2ALIGNED(lp_addr, pg_sz)); 2445 } 2446 if (anon_locked) { 2447 anon_array_exit(&cookie); 2448 } 2449 ANON_LOCK_EXIT(&->a_rwlock); 2450 return (0); 2451 2452 lpgs_err: 2453 if (anon_locked) { 2454 anon_array_exit(&cookie); 2455 } 2456 ANON_LOCK_EXIT(&->a_rwlock); 2457 for (j = 0; j < ppa_idx; j++) 2458 page_unlock(ppa[j]); 2459 return (err); 2460 } 2461 2462 /* 2463 * count the number of bytes in a set of spt pages that are currently not 2464 * locked 2465 */ 2466 static rctl_qty_t 2467 spt_unlockedbytes(pgcnt_t npages, page_t **ppa) 2468 { 2469 ulong_t i; 2470 rctl_qty_t unlocked = 0; 2471 2472 for (i = 0; i < npages; i++) { 2473 if (ppa[i]->p_lckcnt == 0) 2474 unlocked += PAGESIZE; 2475 } 2476 return (unlocked); 2477 } 2478 2479 extern u_longlong_t randtick(void); 2480 /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */ 2481 #define NLCK (NCPU_P2) 2482 /* Random number with a range [0, n-1], n must be power of two */ 2483 #define RAND_P2(n) \ 2484 ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1)) 2485 2486 int 2487 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 2488 page_t **ppa, ulong_t *lockmap, size_t pos, 2489 rctl_qty_t *locked) 2490 { 2491 struct shm_data *shmd = seg->s_data; 2492 struct spt_data *sptd = shmd->shm_sptseg->s_data; 2493 ulong_t i; 2494 int kernel; 2495 pgcnt_t nlck = 0; 2496 int rv = 0; 2497 int use_reserved = 1; 2498 2499 /* return the number of bytes actually locked */ 2500 *locked = 0; 2501 2502 /* 2503 * To avoid contention on freemem_lock, availrmem and pages_locked 2504 * global counters are updated only every nlck locked pages instead of 2505 * every time. Reserve nlck locks up front and deduct from this 2506 * reservation for each page that requires a lock. When the reservation 2507 * is consumed, reserve again. nlck is randomized, so the competing 2508 * threads do not fall into a cyclic lock contention pattern. When 2509 * memory is low, the lock ahead is disabled, and instead page_pp_lock() 2510 * is used to lock pages. 2511 */ 2512 for (i = 0; i < npages; anon_index++, pos++, i++) { 2513 if (nlck == 0 && use_reserved == 1) { 2514 nlck = NLCK + RAND_P2(NLCK); 2515 /* if fewer loops left, decrease nlck */ 2516 nlck = MIN(nlck, npages - i); 2517 /* 2518 * Reserve nlck locks up front and deduct from this 2519 * reservation for each page that requires a lock. When 2520 * the reservation is consumed, reserve again. 2521 */ 2522 mutex_enter(&freemem_lock); 2523 if ((availrmem - nlck) < pages_pp_maximum) { 2524 /* Do not do advance memory reserves */ 2525 use_reserved = 0; 2526 } else { 2527 availrmem -= nlck; 2528 pages_locked += nlck; 2529 } 2530 mutex_exit(&freemem_lock); 2531 } 2532 if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) { 2533 if (sptd->spt_ppa_lckcnt[anon_index] < 2534 (ushort_t)DISM_LOCK_MAX) { 2535 if (++sptd->spt_ppa_lckcnt[anon_index] == 2536 (ushort_t)DISM_LOCK_MAX) { 2537 cmn_err(CE_WARN, 2538 "DISM page lock limit " 2539 "reached on DISM offset 0x%lx\n", 2540 anon_index << PAGESHIFT); 2541 } 2542 kernel = (sptd->spt_ppa && 2543 sptd->spt_ppa[anon_index]); 2544 if (!page_pp_lock(ppa[i], 0, kernel || 2545 use_reserved)) { 2546 sptd->spt_ppa_lckcnt[anon_index]--; 2547 rv = EAGAIN; 2548 break; 2549 } 2550 /* if this is a newly locked page, count it */ 2551 if (ppa[i]->p_lckcnt == 1) { 2552 if (kernel == 0 && use_reserved == 1) 2553 nlck--; 2554 *locked += PAGESIZE; 2555 } 2556 shmd->shm_lckpgs++; 2557 shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED; 2558 if (lockmap != NULL) 2559 BT_SET(lockmap, pos); 2560 } 2561 } 2562 } 2563 /* Return unused lock reservation */ 2564 if (nlck != 0 && use_reserved == 1) { 2565 mutex_enter(&freemem_lock); 2566 availrmem += nlck; 2567 pages_locked -= nlck; 2568 mutex_exit(&freemem_lock); 2569 } 2570 2571 return (rv); 2572 } 2573 2574 int 2575 spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 2576 rctl_qty_t *unlocked) 2577 { 2578 struct shm_data *shmd = seg->s_data; 2579 struct spt_data *sptd = shmd->shm_sptseg->s_data; 2580 struct anon_map *amp = sptd->spt_amp; 2581 struct anon *ap; 2582 struct vnode *vp; 2583 u_offset_t off; 2584 struct page *pp; 2585 int kernel; 2586 anon_sync_obj_t cookie; 2587 ulong_t i; 2588 pgcnt_t nlck = 0; 2589 pgcnt_t nlck_limit = NLCK; 2590 2591 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2592 for (i = 0; i < npages; i++, anon_index++) { 2593 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 2594 anon_array_enter(amp, anon_index, &cookie); 2595 ap = anon_get_ptr(amp->ahp, anon_index); 2596 ASSERT(ap); 2597 2598 swap_xlate(ap, &vp, &off); 2599 anon_array_exit(&cookie); 2600 pp = page_lookup(vp, off, SE_SHARED); 2601 ASSERT(pp); 2602 /* 2603 * availrmem is decremented only for pages which are not 2604 * in seg pcache, for pages in seg pcache availrmem was 2605 * decremented in _dismpagelock() 2606 */ 2607 kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]); 2608 ASSERT(pp->p_lckcnt > 0); 2609 2610 /* 2611 * lock page but do not change availrmem, we do it 2612 * ourselves every nlck loops. 2613 */ 2614 page_pp_unlock(pp, 0, 1); 2615 if (pp->p_lckcnt == 0) { 2616 if (kernel == 0) 2617 nlck++; 2618 *unlocked += PAGESIZE; 2619 } 2620 page_unlock(pp); 2621 shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED; 2622 sptd->spt_ppa_lckcnt[anon_index]--; 2623 shmd->shm_lckpgs--; 2624 } 2625 2626 /* 2627 * To reduce freemem_lock contention, do not update availrmem 2628 * until at least NLCK pages have been unlocked. 2629 * 1. No need to update if nlck is zero 2630 * 2. Always update if the last iteration 2631 */ 2632 if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) { 2633 mutex_enter(&freemem_lock); 2634 availrmem += nlck; 2635 pages_locked -= nlck; 2636 mutex_exit(&freemem_lock); 2637 nlck = 0; 2638 nlck_limit = NLCK + RAND_P2(NLCK); 2639 } 2640 } 2641 ANON_LOCK_EXIT(&->a_rwlock); 2642 2643 return (0); 2644 } 2645 2646 /*ARGSUSED*/ 2647 static int 2648 segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 2649 int attr, int op, ulong_t *lockmap, size_t pos) 2650 { 2651 struct shm_data *shmd = seg->s_data; 2652 struct seg *sptseg = shmd->shm_sptseg; 2653 struct spt_data *sptd = sptseg->s_data; 2654 struct kshmid *sp = sptd->spt_amp->a_sp; 2655 pgcnt_t npages, a_npages; 2656 page_t **ppa; 2657 pgcnt_t an_idx, a_an_idx, ppa_idx; 2658 caddr_t spt_addr, a_addr; /* spt and aligned address */ 2659 size_t a_len; /* aligned len */ 2660 size_t share_sz; 2661 ulong_t i; 2662 int sts = 0; 2663 rctl_qty_t unlocked = 0; 2664 rctl_qty_t locked = 0; 2665 struct proc *p = curproc; 2666 kproject_t *proj; 2667 2668 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2669 ASSERT(sp != NULL); 2670 2671 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 2672 return (0); 2673 } 2674 2675 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2676 an_idx = seg_page(seg, addr); 2677 npages = btopr(len); 2678 2679 if (an_idx + npages > btopr(shmd->shm_amp->size)) { 2680 return (ENOMEM); 2681 } 2682 2683 /* 2684 * A shm's project never changes, so no lock needed. 2685 * The shm has a hold on the project, so it will not go away. 2686 * Since we have a mapping to shm within this zone, we know 2687 * that the zone will not go away. 2688 */ 2689 proj = sp->shm_perm.ipc_proj; 2690 2691 if (op == MC_LOCK) { 2692 2693 /* 2694 * Need to align addr and size request if they are not 2695 * aligned so we can always allocate large page(s) however 2696 * we only lock what was requested in initial request. 2697 */ 2698 share_sz = page_get_pagesize(sptseg->s_szc); 2699 a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 2700 a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)), 2701 share_sz); 2702 a_npages = btop(a_len); 2703 a_an_idx = seg_page(seg, a_addr); 2704 spt_addr = sptseg->s_base + ptob(a_an_idx); 2705 ppa_idx = an_idx - a_an_idx; 2706 2707 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages), 2708 KM_NOSLEEP)) == NULL) { 2709 return (ENOMEM); 2710 } 2711 2712 /* 2713 * Don't cache any new pages for IO and 2714 * flush any cached pages. 2715 */ 2716 mutex_enter(&sptd->spt_lock); 2717 if (sptd->spt_ppa != NULL) 2718 sptd->spt_flags |= DISM_PPA_CHANGED; 2719 2720 sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa); 2721 if (sts != 0) { 2722 mutex_exit(&sptd->spt_lock); 2723 kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 2724 return (sts); 2725 } 2726 2727 mutex_enter(&sp->shm_mlock); 2728 /* enforce locked memory rctl */ 2729 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]); 2730 2731 mutex_enter(&p->p_lock); 2732 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) { 2733 mutex_exit(&p->p_lock); 2734 sts = EAGAIN; 2735 } else { 2736 mutex_exit(&p->p_lock); 2737 sts = spt_lockpages(seg, an_idx, npages, 2738 &ppa[ppa_idx], lockmap, pos, &locked); 2739 2740 /* 2741 * correct locked count if not all pages could be 2742 * locked 2743 */ 2744 if ((unlocked - locked) > 0) { 2745 rctl_decr_locked_mem(NULL, proj, 2746 (unlocked - locked), 0); 2747 } 2748 } 2749 /* 2750 * unlock pages 2751 */ 2752 for (i = 0; i < a_npages; i++) 2753 page_unlock(ppa[i]); 2754 if (sptd->spt_ppa != NULL) 2755 sptd->spt_flags |= DISM_PPA_CHANGED; 2756 mutex_exit(&sp->shm_mlock); 2757 mutex_exit(&sptd->spt_lock); 2758 2759 kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 2760 2761 } else if (op == MC_UNLOCK) { /* unlock */ 2762 page_t **ppa; 2763 2764 mutex_enter(&sptd->spt_lock); 2765 if (shmd->shm_lckpgs == 0) { 2766 mutex_exit(&sptd->spt_lock); 2767 return (0); 2768 } 2769 /* 2770 * Don't cache new IO pages. 2771 */ 2772 if (sptd->spt_ppa != NULL) 2773 sptd->spt_flags |= DISM_PPA_CHANGED; 2774 2775 mutex_enter(&sp->shm_mlock); 2776 sts = spt_unlockpages(seg, an_idx, npages, &unlocked); 2777 if ((ppa = sptd->spt_ppa) != NULL) 2778 sptd->spt_flags |= DISM_PPA_CHANGED; 2779 mutex_exit(&sptd->spt_lock); 2780 2781 rctl_decr_locked_mem(NULL, proj, unlocked, 0); 2782 mutex_exit(&sp->shm_mlock); 2783 2784 if (ppa != NULL) 2785 seg_ppurge_wiredpp(ppa); 2786 } 2787 return (sts); 2788 } 2789 2790 /*ARGSUSED*/ 2791 int 2792 segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 2793 { 2794 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2795 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2796 spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1; 2797 2798 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2799 2800 /* 2801 * ISM segment is always rw. 2802 */ 2803 while (--pgno >= 0) 2804 *protv++ = sptd->spt_prot; 2805 return (0); 2806 } 2807 2808 /*ARGSUSED*/ 2809 u_offset_t 2810 segspt_shmgetoffset(struct seg *seg, caddr_t addr) 2811 { 2812 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2813 2814 /* Offset does not matter in ISM memory */ 2815 2816 return ((u_offset_t)0); 2817 } 2818 2819 /* ARGSUSED */ 2820 int 2821 segspt_shmgettype(struct seg *seg, caddr_t addr) 2822 { 2823 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2824 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2825 2826 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2827 2828 /* 2829 * The shared memory mapping is always MAP_SHARED, SWAP is only 2830 * reserved for DISM 2831 */ 2832 return (MAP_SHARED | 2833 ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE)); 2834 } 2835 2836 /*ARGSUSED*/ 2837 int 2838 segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 2839 { 2840 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2841 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2842 2843 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2844 2845 *vpp = sptd->spt_vp; 2846 return (0); 2847 } 2848 2849 /* 2850 * We need to wait for pending IO to complete to a DISM segment in order for 2851 * pages to get kicked out of the seg_pcache. 120 seconds should be more 2852 * than enough time to wait. 2853 */ 2854 static clock_t spt_pcache_wait = 120; 2855 2856 /*ARGSUSED*/ 2857 static int 2858 segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 2859 { 2860 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2861 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2862 struct anon_map *amp; 2863 pgcnt_t pg_idx; 2864 ushort_t gen; 2865 clock_t end_lbolt; 2866 int writer; 2867 page_t **ppa; 2868 2869 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 2870 2871 if (behav == MADV_FREE) { 2872 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) 2873 return (0); 2874 2875 amp = sptd->spt_amp; 2876 pg_idx = seg_page(seg, addr); 2877 2878 mutex_enter(&sptd->spt_lock); 2879 if ((ppa = sptd->spt_ppa) == NULL) { 2880 mutex_exit(&sptd->spt_lock); 2881 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2882 anon_disclaim(amp, pg_idx, len); 2883 ANON_LOCK_EXIT(&->a_rwlock); 2884 return (0); 2885 } 2886 2887 sptd->spt_flags |= DISM_PPA_CHANGED; 2888 gen = sptd->spt_gen; 2889 2890 mutex_exit(&sptd->spt_lock); 2891 2892 /* 2893 * Purge all DISM cached pages 2894 */ 2895 seg_ppurge_wiredpp(ppa); 2896 2897 /* 2898 * Drop the AS_LOCK so that other threads can grab it 2899 * in the as_pageunlock path and hopefully get the segment 2900 * kicked out of the seg_pcache. We bump the shm_softlockcnt 2901 * to keep this segment resident. 2902 */ 2903 writer = AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock); 2904 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), 1); 2905 AS_LOCK_EXIT(seg->s_as, &seg->s_as->a_lock); 2906 2907 mutex_enter(&sptd->spt_lock); 2908 2909 end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait); 2910 2911 /* 2912 * Try to wait for pages to get kicked out of the seg_pcache. 2913 */ 2914 while (sptd->spt_gen == gen && 2915 (sptd->spt_flags & DISM_PPA_CHANGED) && 2916 ddi_get_lbolt() < end_lbolt) { 2917 if (!cv_timedwait_sig(&sptd->spt_cv, 2918 &sptd->spt_lock, end_lbolt)) { 2919 break; 2920 } 2921 } 2922 2923 mutex_exit(&sptd->spt_lock); 2924 2925 /* Regrab the AS_LOCK and release our hold on the segment */ 2926 AS_LOCK_ENTER(seg->s_as, &seg->s_as->a_lock, 2927 writer ? RW_WRITER : RW_READER); 2928 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -1); 2929 if (shmd->shm_softlockcnt <= 0) { 2930 if (AS_ISUNMAPWAIT(seg->s_as)) { 2931 mutex_enter(&seg->s_as->a_contents); 2932 if (AS_ISUNMAPWAIT(seg->s_as)) { 2933 AS_CLRUNMAPWAIT(seg->s_as); 2934 cv_broadcast(&seg->s_as->a_cv); 2935 } 2936 mutex_exit(&seg->s_as->a_contents); 2937 } 2938 } 2939 2940 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2941 anon_disclaim(amp, pg_idx, len); 2942 ANON_LOCK_EXIT(&->a_rwlock); 2943 } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP || 2944 behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) { 2945 int already_set; 2946 ulong_t anon_index; 2947 lgrp_mem_policy_t policy; 2948 caddr_t shm_addr; 2949 size_t share_size; 2950 size_t size; 2951 struct seg *sptseg = shmd->shm_sptseg; 2952 caddr_t sptseg_addr; 2953 2954 /* 2955 * Align address and length to page size of underlying segment 2956 */ 2957 share_size = page_get_pagesize(shmd->shm_sptseg->s_szc); 2958 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 2959 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), 2960 share_size); 2961 2962 amp = shmd->shm_amp; 2963 anon_index = seg_page(seg, shm_addr); 2964 2965 /* 2966 * And now we may have to adjust size downward if we have 2967 * exceeded the realsize of the segment or initial anon 2968 * allocations. 2969 */ 2970 sptseg_addr = sptseg->s_base + ptob(anon_index); 2971 if ((sptseg_addr + size) > 2972 (sptseg->s_base + sptd->spt_realsize)) 2973 size = (sptseg->s_base + sptd->spt_realsize) - 2974 sptseg_addr; 2975 2976 /* 2977 * Set memory allocation policy for this segment 2978 */ 2979 policy = lgrp_madv_to_policy(behav, len, MAP_SHARED); 2980 already_set = lgrp_shm_policy_set(policy, amp, anon_index, 2981 NULL, 0, len); 2982 2983 /* 2984 * If random memory allocation policy set already, 2985 * don't bother reapplying it. 2986 */ 2987 if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy)) 2988 return (0); 2989 2990 /* 2991 * Mark any existing pages in the given range for 2992 * migration, flushing the I/O page cache, and using 2993 * underlying segment to calculate anon index and get 2994 * anonmap and vnode pointer from 2995 */ 2996 if (shmd->shm_softlockcnt > 0) 2997 segspt_purge(seg); 2998 2999 page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0); 3000 } 3001 3002 return (0); 3003 } 3004 3005 /*ARGSUSED*/ 3006 void 3007 segspt_shmdump(struct seg *seg) 3008 { 3009 /* no-op for ISM segment */ 3010 } 3011 3012 /*ARGSUSED*/ 3013 static faultcode_t 3014 segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc) 3015 { 3016 return (ENOTSUP); 3017 } 3018 3019 /* 3020 * get a memory ID for an addr in a given segment 3021 */ 3022 static int 3023 segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 3024 { 3025 struct shm_data *shmd = (struct shm_data *)seg->s_data; 3026 struct anon *ap; 3027 size_t anon_index; 3028 struct anon_map *amp = shmd->shm_amp; 3029 struct spt_data *sptd = shmd->shm_sptseg->s_data; 3030 struct seg *sptseg = shmd->shm_sptseg; 3031 anon_sync_obj_t cookie; 3032 3033 anon_index = seg_page(seg, addr); 3034 3035 if (addr > (seg->s_base + sptd->spt_realsize)) { 3036 return (EFAULT); 3037 } 3038 3039 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 3040 anon_array_enter(amp, anon_index, &cookie); 3041 ap = anon_get_ptr(amp->ahp, anon_index); 3042 if (ap == NULL) { 3043 struct page *pp; 3044 caddr_t spt_addr = sptseg->s_base + ptob(anon_index); 3045 3046 pp = anon_zero(sptseg, spt_addr, &ap, kcred); 3047 if (pp == NULL) { 3048 anon_array_exit(&cookie); 3049 ANON_LOCK_EXIT(&->a_rwlock); 3050 return (ENOMEM); 3051 } 3052 (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP); 3053 page_unlock(pp); 3054 } 3055 anon_array_exit(&cookie); 3056 ANON_LOCK_EXIT(&->a_rwlock); 3057 memidp->val[0] = (uintptr_t)ap; 3058 memidp->val[1] = (uintptr_t)addr & PAGEOFFSET; 3059 return (0); 3060 } 3061 3062 /* 3063 * Get memory allocation policy info for specified address in given segment 3064 */ 3065 static lgrp_mem_policy_info_t * 3066 segspt_shmgetpolicy(struct seg *seg, caddr_t addr) 3067 { 3068 struct anon_map *amp; 3069 ulong_t anon_index; 3070 lgrp_mem_policy_info_t *policy_info; 3071 struct shm_data *shm_data; 3072 3073 ASSERT(seg != NULL); 3074 3075 /* 3076 * Get anon_map from segshm 3077 * 3078 * Assume that no lock needs to be held on anon_map, since 3079 * it should be protected by its reference count which must be 3080 * nonzero for an existing segment 3081 * Need to grab readers lock on policy tree though 3082 */ 3083 shm_data = (struct shm_data *)seg->s_data; 3084 if (shm_data == NULL) 3085 return (NULL); 3086 amp = shm_data->shm_amp; 3087 ASSERT(amp->refcnt != 0); 3088 3089 /* 3090 * Get policy info 3091 * 3092 * Assume starting anon index of 0 3093 */ 3094 anon_index = seg_page(seg, addr); 3095 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 3096 3097 return (policy_info); 3098 } 3099 3100 /*ARGSUSED*/ 3101 static int 3102 segspt_shmcapable(struct seg *seg, segcapability_t capability) 3103 { 3104 return (0); 3105 }