1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * Portions of this source code were derived from Berkeley 4.3 BSD 31 * under license from the Regents of the University of California. 32 */ 33 34 /* 35 * VM - segment for non-faulting loads. 36 */ 37 38 #include <sys/types.h> 39 #include <sys/t_lock.h> 40 #include <sys/param.h> 41 #include <sys/mman.h> 42 #include <sys/errno.h> 43 #include <sys/kmem.h> 44 #include <sys/cmn_err.h> 45 #include <sys/vnode.h> 46 #include <sys/proc.h> 47 #include <sys/conf.h> 48 #include <sys/debug.h> 49 #include <sys/archsystm.h> 50 #include <sys/lgrp.h> 51 52 #include <vm/page.h> 53 #include <vm/hat.h> 54 #include <vm/as.h> 55 #include <vm/seg.h> 56 #include <vm/vpage.h> 57 58 /* 59 * Private seg op routines. 60 */ 61 static int segnf_dup(struct seg *seg, struct seg *newseg); 62 static int segnf_unmap(struct seg *seg, caddr_t addr, size_t len); 63 static void segnf_free(struct seg *seg); 64 static faultcode_t segnf_nomap(void); 65 static int segnf_setprot(struct seg *seg, caddr_t addr, 66 size_t len, uint_t prot); 67 static int segnf_checkprot(struct seg *seg, caddr_t addr, 68 size_t len, uint_t prot); 69 static int segnf_nop(void); 70 static int segnf_getprot(struct seg *seg, caddr_t addr, 71 size_t len, uint_t *protv); 72 static u_offset_t segnf_getoffset(struct seg *seg, caddr_t addr); 73 static int segnf_gettype(struct seg *seg, caddr_t addr); 74 static int segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 75 static void segnf_dump(struct seg *seg); 76 static int segnf_pagelock(struct seg *seg, caddr_t addr, size_t len, 77 struct page ***ppp, enum lock_type type, enum seg_rw rw); 78 static int segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len, 79 uint_t szc); 80 81 82 struct seg_ops segnf_ops = { 83 .dup = segnf_dup, 84 .unmap = segnf_unmap, 85 .free = segnf_free, 86 .fault = (faultcode_t (*)(struct hat *, struct seg *, caddr_t, 87 size_t, enum fault_type, enum seg_rw))segnf_nomap, 88 .faulta = (faultcode_t (*)(struct seg *, caddr_t)) segnf_nomap, 89 .setprot = segnf_setprot, 90 .checkprot = segnf_checkprot, 91 .sync = (int (*)(struct seg *, caddr_t, size_t, int, uint_t)) 92 segnf_nop, 93 .incore = (size_t (*)(struct seg *, caddr_t, size_t, char *)) 94 segnf_nop, 95 .lockop = (int (*)(struct seg *, caddr_t, size_t, int, int, 96 ulong_t *, size_t))segnf_nop, 97 .getprot = segnf_getprot, 98 .getoffset = segnf_getoffset, 99 .gettype = segnf_gettype, 100 .getvp = segnf_getvp, 101 .advise = (int (*)(struct seg *, caddr_t, size_t, uint_t)) 102 segnf_nop, 103 .dump = segnf_dump, 104 .pagelock = segnf_pagelock, 105 .setpagesize = segnf_setpagesize, 106 }; 107 108 /* 109 * vnode and page for the page of zeros we use for the nf mappings. 110 */ 111 static kmutex_t segnf_lock; 112 static struct vnode nfvp; 113 static struct page **nfpp; 114 115 #define addr_to_vcolor(addr) \ 116 (shm_alignment) ? \ 117 ((int)(((uintptr_t)(addr) & (shm_alignment - 1)) >> PAGESHIFT)) : 0 118 119 /* 120 * We try to limit the number of Non-fault segments created. 121 * Non fault segments are created to optimize sparc V9 code which uses 122 * the sparc nonfaulting load ASI (ASI_PRIMARY_NOFAULT). 123 * 124 * There are several reasons why creating too many non-fault segments 125 * could cause problems. 126 * 127 * First, excessive allocation of kernel resources for the seg 128 * structures and the HAT data to map the zero pages. 129 * 130 * Secondly, creating nofault segments actually uses up user virtual 131 * address space. This makes it unavailable for subsequent mmap(0, ...) 132 * calls which use as_gap() to find empty va regions. Creation of too 133 * many nofault segments could thus interfere with the ability of the 134 * runtime linker to load a shared object. 135 */ 136 #define MAXSEGFORNF (10000) 137 #define MAXNFSEARCH (5) 138 139 140 /* 141 * Must be called from startup() 142 */ 143 void 144 segnf_init() 145 { 146 mutex_init(&segnf_lock, NULL, MUTEX_DEFAULT, NULL); 147 } 148 149 150 /* 151 * Create a no-fault segment. 152 * 153 * The no-fault segment is not technically necessary, as the code in 154 * nfload() in trap.c will emulate the SPARC instruction and load 155 * a value of zero in the destination register. 156 * 157 * However, this code tries to put a page of zero's at the nofault address 158 * so that subsequent non-faulting loads to the same page will not 159 * trap with a tlb miss. 160 * 161 * In order to help limit the number of segments we merge adjacent nofault 162 * segments into a single segment. If we get a large number of segments 163 * we'll also try to delete a random other nf segment. 164 */ 165 /* ARGSUSED */ 166 int 167 segnf_create(struct seg *seg, void *argsp) 168 { 169 uint_t prot; 170 pgcnt_t vacpgs; 171 u_offset_t off = 0; 172 caddr_t vaddr = NULL; 173 int i, color; 174 struct seg *s1; 175 struct seg *s2; 176 size_t size; 177 struct as *as = seg->s_as; 178 179 ASSERT(as && AS_WRITE_HELD(as, &as->a_lock)); 180 181 /* 182 * Need a page per virtual color or just 1 if no vac. 183 */ 184 mutex_enter(&segnf_lock); 185 if (nfpp == NULL) { 186 struct seg kseg; 187 188 vacpgs = 1; 189 if (shm_alignment > PAGESIZE) { 190 vacpgs = shm_alignment >> PAGESHIFT; 191 } 192 193 nfpp = kmem_alloc(sizeof (*nfpp) * vacpgs, KM_SLEEP); 194 195 kseg.s_as = &kas; 196 for (i = 0; i < vacpgs; i++, off += PAGESIZE, 197 vaddr += PAGESIZE) { 198 nfpp[i] = page_create_va(&nfvp, off, PAGESIZE, 199 PG_WAIT | PG_NORELOC, &kseg, vaddr); 200 page_io_unlock(nfpp[i]); 201 page_downgrade(nfpp[i]); 202 pagezero(nfpp[i], 0, PAGESIZE); 203 } 204 } 205 mutex_exit(&segnf_lock); 206 207 hat_map(as->a_hat, seg->s_base, seg->s_size, HAT_MAP); 208 209 /* 210 * s_data can't be NULL because of ASSERTS in the common vm code. 211 */ 212 seg->s_ops = &segnf_ops; 213 seg->s_data = seg; 214 seg->s_flags |= S_PURGE; 215 216 mutex_enter(&as->a_contents); 217 as->a_flags |= AS_NEEDSPURGE; 218 mutex_exit(&as->a_contents); 219 220 prot = PROT_READ; 221 color = addr_to_vcolor(seg->s_base); 222 if (as != &kas) 223 prot |= PROT_USER; 224 hat_memload(as->a_hat, seg->s_base, nfpp[color], 225 prot | HAT_NOFAULT, HAT_LOAD); 226 227 /* 228 * At this point see if we can concatenate a segment to 229 * a non-fault segment immediately before and/or after it. 230 */ 231 if ((s1 = AS_SEGPREV(as, seg)) != NULL && 232 s1->s_ops == &segnf_ops && 233 s1->s_base + s1->s_size == seg->s_base) { 234 size = s1->s_size; 235 seg_free(s1); 236 seg->s_base -= size; 237 seg->s_size += size; 238 } 239 240 if ((s2 = AS_SEGNEXT(as, seg)) != NULL && 241 s2->s_ops == &segnf_ops && 242 seg->s_base + seg->s_size == s2->s_base) { 243 size = s2->s_size; 244 seg_free(s2); 245 seg->s_size += size; 246 } 247 248 /* 249 * if we already have a lot of segments, try to delete some other 250 * nofault segment to reduce the probability of uncontrolled segment 251 * creation. 252 * 253 * the code looks around quickly (no more than MAXNFSEARCH segments 254 * each way) for another NF segment and then deletes it. 255 */ 256 if (avl_numnodes(&as->a_segtree) > MAXSEGFORNF) { 257 size = 0; 258 s2 = NULL; 259 s1 = AS_SEGPREV(as, seg); 260 while (size++ < MAXNFSEARCH && s1 != NULL) { 261 if (s1->s_ops == &segnf_ops) 262 s2 = s1; 263 s1 = AS_SEGPREV(s1->s_as, seg); 264 } 265 if (s2 == NULL) { 266 s1 = AS_SEGNEXT(as, seg); 267 while (size-- > 0 && s1 != NULL) { 268 if (s1->s_ops == &segnf_ops) 269 s2 = s1; 270 s1 = AS_SEGNEXT(as, seg); 271 } 272 } 273 if (s2 != NULL) 274 seg_unmap(s2); 275 } 276 277 return (0); 278 } 279 280 /* 281 * Never really need "No fault" segments, so they aren't dup'd. 282 */ 283 /* ARGSUSED */ 284 static int 285 segnf_dup(struct seg *seg, struct seg *newseg) 286 { 287 panic("segnf_dup"); 288 return (0); 289 } 290 291 /* 292 * Split a segment at addr for length len. 293 */ 294 static int 295 segnf_unmap(struct seg *seg, caddr_t addr, size_t len) 296 { 297 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 298 299 /* 300 * Check for bad sizes. 301 */ 302 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size || 303 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) { 304 cmn_err(CE_PANIC, "segnf_unmap: bad unmap size"); 305 } 306 307 /* 308 * Unload any hardware translations in the range to be taken out. 309 */ 310 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP); 311 312 if (addr == seg->s_base && len == seg->s_size) { 313 /* 314 * Freeing entire segment. 315 */ 316 seg_free(seg); 317 } else if (addr == seg->s_base) { 318 /* 319 * Freeing the beginning of the segment. 320 */ 321 seg->s_base += len; 322 seg->s_size -= len; 323 } else if (addr + len == seg->s_base + seg->s_size) { 324 /* 325 * Freeing the end of the segment. 326 */ 327 seg->s_size -= len; 328 } else { 329 /* 330 * The section to go is in the middle of the segment, so we 331 * have to cut it into two segments. We shrink the existing 332 * "seg" at the low end, and create "nseg" for the high end. 333 */ 334 caddr_t nbase = addr + len; 335 size_t nsize = (seg->s_base + seg->s_size) - nbase; 336 struct seg *nseg; 337 338 /* 339 * Trim down "seg" before trying to stick "nseg" into the as. 340 */ 341 seg->s_size = addr - seg->s_base; 342 nseg = seg_alloc(seg->s_as, nbase, nsize); 343 if (nseg == NULL) 344 cmn_err(CE_PANIC, "segnf_unmap: seg_alloc failed"); 345 346 /* 347 * s_data can't be NULL because of ASSERTs in common VM code. 348 */ 349 nseg->s_ops = seg->s_ops; 350 nseg->s_data = nseg; 351 nseg->s_flags |= S_PURGE; 352 mutex_enter(&seg->s_as->a_contents); 353 seg->s_as->a_flags |= AS_NEEDSPURGE; 354 mutex_exit(&seg->s_as->a_contents); 355 } 356 357 return (0); 358 } 359 360 /* 361 * Free a segment. 362 */ 363 static void 364 segnf_free(struct seg *seg) 365 { 366 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 367 } 368 369 /* 370 * No faults allowed on segnf. 371 */ 372 static faultcode_t 373 segnf_nomap(void) 374 { 375 return (FC_NOMAP); 376 } 377 378 /* ARGSUSED */ 379 static int 380 segnf_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 381 { 382 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 383 return (EACCES); 384 } 385 386 /* ARGSUSED */ 387 static int 388 segnf_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 389 { 390 uint_t sprot; 391 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 392 393 sprot = seg->s_as == &kas ? PROT_READ : PROT_READ|PROT_USER; 394 return ((prot & sprot) == prot ? 0 : EACCES); 395 } 396 397 static int 398 segnf_nop(void) 399 { 400 return (0); 401 } 402 403 static int 404 segnf_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 405 { 406 size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1; 407 size_t p; 408 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 409 410 for (p = 0; p < pgno; ++p) 411 protv[p] = PROT_READ; 412 return (0); 413 } 414 415 /* ARGSUSED */ 416 static u_offset_t 417 segnf_getoffset(struct seg *seg, caddr_t addr) 418 { 419 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 420 421 return ((u_offset_t)0); 422 } 423 424 /* ARGSUSED */ 425 static int 426 segnf_gettype(struct seg *seg, caddr_t addr) 427 { 428 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 429 430 return (MAP_SHARED); 431 } 432 433 /* ARGSUSED */ 434 static int 435 segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 436 { 437 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 438 439 *vpp = &nfvp; 440 return (0); 441 } 442 443 /* 444 * segnf pages are not dumped, so we just return 445 */ 446 /* ARGSUSED */ 447 static void 448 segnf_dump(struct seg *seg) 449 {} 450 451 /*ARGSUSED*/ 452 static int 453 segnf_pagelock(struct seg *seg, caddr_t addr, size_t len, 454 struct page ***ppp, enum lock_type type, enum seg_rw rw) 455 { 456 return (ENOTSUP); 457 } 458 459 /*ARGSUSED*/ 460 static int 461 segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len, 462 uint_t szc) 463 { 464 return (ENOTSUP); 465 }