1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
  27 /* All Rights Reserved */
  28 
  29 /*
  30  * Portions of this source code were derived from Berkeley 4.3 BSD
  31  * under license from the Regents of the University of California.
  32  */
  33 
  34 /*
  35  * VM - segment for non-faulting loads.
  36  */
  37 
  38 #include <sys/types.h>
  39 #include <sys/t_lock.h>
  40 #include <sys/param.h>
  41 #include <sys/mman.h>
  42 #include <sys/errno.h>
  43 #include <sys/kmem.h>
  44 #include <sys/cmn_err.h>
  45 #include <sys/vnode.h>
  46 #include <sys/proc.h>
  47 #include <sys/conf.h>
  48 #include <sys/debug.h>
  49 #include <sys/archsystm.h>
  50 #include <sys/lgrp.h>
  51 
  52 #include <vm/page.h>
  53 #include <vm/hat.h>
  54 #include <vm/as.h>
  55 #include <vm/seg.h>
  56 #include <vm/vpage.h>
  57 
  58 /*
  59  * Private seg op routines.
  60  */
  61 static int      segnf_dup(struct seg *seg, struct seg *newseg);
  62 static int      segnf_unmap(struct seg *seg, caddr_t addr, size_t len);
  63 static void     segnf_free(struct seg *seg);
  64 static faultcode_t segnf_nomap(void);
  65 static int      segnf_setprot(struct seg *seg, caddr_t addr,
  66                     size_t len, uint_t prot);
  67 static int      segnf_checkprot(struct seg *seg, caddr_t addr,
  68                     size_t len, uint_t prot);
  69 static void     segnf_badop(void);
  70 static int      segnf_nop(void);
  71 static int      segnf_getprot(struct seg *seg, caddr_t addr,
  72                     size_t len, uint_t *protv);
  73 static u_offset_t segnf_getoffset(struct seg *seg, caddr_t addr);
  74 static int      segnf_gettype(struct seg *seg, caddr_t addr);
  75 static int      segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
  76 static void     segnf_dump(struct seg *seg);
  77 static int      segnf_pagelock(struct seg *seg, caddr_t addr, size_t len,
  78                     struct page ***ppp, enum lock_type type, enum seg_rw rw);
  79 static int      segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len,
  80                     uint_t szc);
  81 static int      segnf_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
  82 
  83 
  84 struct seg_ops segnf_ops = {
  85         .dup            = segnf_dup,
  86         .unmap          = segnf_unmap,
  87         .free           = segnf_free,
  88         .fault          = (faultcode_t (*)(struct hat *, struct seg *, caddr_t,
  89             size_t, enum fault_type, enum seg_rw))segnf_nomap,
  90         .faulta         = (faultcode_t (*)(struct seg *, caddr_t)) segnf_nomap,
  91         .setprot        = segnf_setprot,
  92         .checkprot      = segnf_checkprot,
  93         .kluster        = (int (*)())segnf_badop,
  94         .sync           = (int (*)(struct seg *, caddr_t, size_t, int, uint_t))
  95                 segnf_nop,
  96         .incore         = (size_t (*)(struct seg *, caddr_t, size_t, char *))
  97                 segnf_nop,
  98         .lockop         = (int (*)(struct seg *, caddr_t, size_t, int, int,
  99             ulong_t *, size_t))segnf_nop,
 100         .getprot        = segnf_getprot,
 101         .getoffset      = segnf_getoffset,
 102         .gettype        = segnf_gettype,
 103         .getvp          = segnf_getvp,
 104         .advise         = (int (*)(struct seg *, caddr_t, size_t, uint_t))
 105                 segnf_nop,
 106         .dump           = segnf_dump,
 107         .pagelock       = segnf_pagelock,
 108         .setpagesize    = segnf_setpagesize,
 109         .getmemid       = segnf_getmemid,
 110 };
 111 
 112 /*
 113  * vnode and page for the page of zeros we use for the nf mappings.
 114  */
 115 static kmutex_t segnf_lock;
 116 static struct vnode nfvp;
 117 static struct page **nfpp;
 118 
 119 #define addr_to_vcolor(addr)                                            \
 120         (shm_alignment) ?                                               \
 121         ((int)(((uintptr_t)(addr) & (shm_alignment - 1)) >> PAGESHIFT)) : 0
 122 
 123 /*
 124  * We try to limit the number of Non-fault segments created.
 125  * Non fault segments are created to optimize sparc V9 code which uses
 126  * the sparc nonfaulting load ASI (ASI_PRIMARY_NOFAULT).
 127  *
 128  * There are several reasons why creating too many non-fault segments
 129  * could cause problems.
 130  *
 131  *      First, excessive allocation of kernel resources for the seg
 132  *      structures and the HAT data to map the zero pages.
 133  *
 134  *      Secondly, creating nofault segments actually uses up user virtual
 135  *      address space. This makes it unavailable for subsequent mmap(0, ...)
 136  *      calls which use as_gap() to find empty va regions.  Creation of too
 137  *      many nofault segments could thus interfere with the ability of the
 138  *      runtime linker to load a shared object.
 139  */
 140 #define MAXSEGFORNF     (10000)
 141 #define MAXNFSEARCH     (5)
 142 
 143 
 144 /*
 145  * Must be called from startup()
 146  */
 147 void
 148 segnf_init()
 149 {
 150         mutex_init(&segnf_lock, NULL, MUTEX_DEFAULT, NULL);
 151 }
 152 
 153 
 154 /*
 155  * Create a no-fault segment.
 156  *
 157  * The no-fault segment is not technically necessary, as the code in
 158  * nfload() in trap.c will emulate the SPARC instruction and load
 159  * a value of zero in the destination register.
 160  *
 161  * However, this code tries to put a page of zero's at the nofault address
 162  * so that subsequent non-faulting loads to the same page will not
 163  * trap with a tlb miss.
 164  *
 165  * In order to help limit the number of segments we merge adjacent nofault
 166  * segments into a single segment.  If we get a large number of segments
 167  * we'll also try to delete a random other nf segment.
 168  */
 169 /* ARGSUSED */
 170 int
 171 segnf_create(struct seg *seg, void *argsp)
 172 {
 173         uint_t prot;
 174         pgcnt_t vacpgs;
 175         u_offset_t off = 0;
 176         caddr_t vaddr = NULL;
 177         int i, color;
 178         struct seg *s1;
 179         struct seg *s2;
 180         size_t size;
 181         struct as *as = seg->s_as;
 182 
 183         ASSERT(as && AS_WRITE_HELD(as, &as->a_lock));
 184 
 185         /*
 186          * Need a page per virtual color or just 1 if no vac.
 187          */
 188         mutex_enter(&segnf_lock);
 189         if (nfpp == NULL) {
 190                 struct seg kseg;
 191 
 192                 vacpgs = 1;
 193                 if (shm_alignment > PAGESIZE) {
 194                         vacpgs = shm_alignment >> PAGESHIFT;
 195                 }
 196 
 197                 nfpp = kmem_alloc(sizeof (*nfpp) * vacpgs, KM_SLEEP);
 198 
 199                 kseg.s_as = &kas;
 200                 for (i = 0; i < vacpgs; i++, off += PAGESIZE,
 201                     vaddr += PAGESIZE) {
 202                         nfpp[i] = page_create_va(&nfvp, off, PAGESIZE,
 203                             PG_WAIT | PG_NORELOC, &kseg, vaddr);
 204                         page_io_unlock(nfpp[i]);
 205                         page_downgrade(nfpp[i]);
 206                         pagezero(nfpp[i], 0, PAGESIZE);
 207                 }
 208         }
 209         mutex_exit(&segnf_lock);
 210 
 211         hat_map(as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
 212 
 213         /*
 214          * s_data can't be NULL because of ASSERTS in the common vm code.
 215          */
 216         seg->s_ops = &segnf_ops;
 217         seg->s_data = seg;
 218         seg->s_flags |= S_PURGE;
 219 
 220         mutex_enter(&as->a_contents);
 221         as->a_flags |= AS_NEEDSPURGE;
 222         mutex_exit(&as->a_contents);
 223 
 224         prot = PROT_READ;
 225         color = addr_to_vcolor(seg->s_base);
 226         if (as != &kas)
 227                 prot |= PROT_USER;
 228         hat_memload(as->a_hat, seg->s_base, nfpp[color],
 229             prot | HAT_NOFAULT, HAT_LOAD);
 230 
 231         /*
 232          * At this point see if we can concatenate a segment to
 233          * a non-fault segment immediately before and/or after it.
 234          */
 235         if ((s1 = AS_SEGPREV(as, seg)) != NULL &&
 236             s1->s_ops == &segnf_ops &&
 237             s1->s_base + s1->s_size == seg->s_base) {
 238                 size = s1->s_size;
 239                 seg_free(s1);
 240                 seg->s_base -= size;
 241                 seg->s_size += size;
 242         }
 243 
 244         if ((s2 = AS_SEGNEXT(as, seg)) != NULL &&
 245             s2->s_ops == &segnf_ops &&
 246             seg->s_base + seg->s_size == s2->s_base) {
 247                 size = s2->s_size;
 248                 seg_free(s2);
 249                 seg->s_size += size;
 250         }
 251 
 252         /*
 253          * if we already have a lot of segments, try to delete some other
 254          * nofault segment to reduce the probability of uncontrolled segment
 255          * creation.
 256          *
 257          * the code looks around quickly (no more than MAXNFSEARCH segments
 258          * each way) for another NF segment and then deletes it.
 259          */
 260         if (avl_numnodes(&as->a_segtree) > MAXSEGFORNF) {
 261                 size = 0;
 262                 s2 = NULL;
 263                 s1 = AS_SEGPREV(as, seg);
 264                 while (size++ < MAXNFSEARCH && s1 != NULL) {
 265                         if (s1->s_ops == &segnf_ops)
 266                                 s2 = s1;
 267                         s1 = AS_SEGPREV(s1->s_as, seg);
 268                 }
 269                 if (s2 == NULL) {
 270                         s1 = AS_SEGNEXT(as, seg);
 271                         while (size-- > 0 && s1 != NULL) {
 272                                 if (s1->s_ops == &segnf_ops)
 273                                         s2 = s1;
 274                                 s1 = AS_SEGNEXT(as, seg);
 275                         }
 276                 }
 277                 if (s2 != NULL)
 278                         seg_unmap(s2);
 279         }
 280 
 281         return (0);
 282 }
 283 
 284 /*
 285  * Never really need "No fault" segments, so they aren't dup'd.
 286  */
 287 /* ARGSUSED */
 288 static int
 289 segnf_dup(struct seg *seg, struct seg *newseg)
 290 {
 291         panic("segnf_dup");
 292         return (0);
 293 }
 294 
 295 /*
 296  * Split a segment at addr for length len.
 297  */
 298 static int
 299 segnf_unmap(struct seg *seg, caddr_t addr, size_t len)
 300 {
 301         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 302 
 303         /*
 304          * Check for bad sizes.
 305          */
 306         if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
 307             (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) {
 308                 cmn_err(CE_PANIC, "segnf_unmap: bad unmap size");
 309         }
 310 
 311         /*
 312          * Unload any hardware translations in the range to be taken out.
 313          */
 314         hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
 315 
 316         if (addr == seg->s_base && len == seg->s_size) {
 317                 /*
 318                  * Freeing entire segment.
 319                  */
 320                 seg_free(seg);
 321         } else if (addr == seg->s_base) {
 322                 /*
 323                  * Freeing the beginning of the segment.
 324                  */
 325                 seg->s_base += len;
 326                 seg->s_size -= len;
 327         } else if (addr + len == seg->s_base + seg->s_size) {
 328                 /*
 329                  * Freeing the end of the segment.
 330                  */
 331                 seg->s_size -= len;
 332         } else {
 333                 /*
 334                  * The section to go is in the middle of the segment, so we
 335                  * have to cut it into two segments.  We shrink the existing
 336                  * "seg" at the low end, and create "nseg" for the high end.
 337                  */
 338                 caddr_t nbase = addr + len;
 339                 size_t nsize = (seg->s_base + seg->s_size) - nbase;
 340                 struct seg *nseg;
 341 
 342                 /*
 343                  * Trim down "seg" before trying to stick "nseg" into the as.
 344                  */
 345                 seg->s_size = addr - seg->s_base;
 346                 nseg = seg_alloc(seg->s_as, nbase, nsize);
 347                 if (nseg == NULL)
 348                         cmn_err(CE_PANIC, "segnf_unmap: seg_alloc failed");
 349 
 350                 /*
 351                  * s_data can't be NULL because of ASSERTs in common VM code.
 352                  */
 353                 nseg->s_ops = seg->s_ops;
 354                 nseg->s_data = nseg;
 355                 nseg->s_flags |= S_PURGE;
 356                 mutex_enter(&seg->s_as->a_contents);
 357                 seg->s_as->a_flags |= AS_NEEDSPURGE;
 358                 mutex_exit(&seg->s_as->a_contents);
 359         }
 360 
 361         return (0);
 362 }
 363 
 364 /*
 365  * Free a segment.
 366  */
 367 static void
 368 segnf_free(struct seg *seg)
 369 {
 370         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 371 }
 372 
 373 /*
 374  * No faults allowed on segnf.
 375  */
 376 static faultcode_t
 377 segnf_nomap(void)
 378 {
 379         return (FC_NOMAP);
 380 }
 381 
 382 /* ARGSUSED */
 383 static int
 384 segnf_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
 385 {
 386         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 387         return (EACCES);
 388 }
 389 
 390 /* ARGSUSED */
 391 static int
 392 segnf_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
 393 {
 394         uint_t sprot;
 395         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 396 
 397         sprot = seg->s_as == &kas ?  PROT_READ : PROT_READ|PROT_USER;
 398         return ((prot & sprot) == prot ? 0 : EACCES);
 399 }
 400 
 401 static void
 402 segnf_badop(void)
 403 {
 404         panic("segnf_badop");
 405         /*NOTREACHED*/
 406 }
 407 
 408 static int
 409 segnf_nop(void)
 410 {
 411         return (0);
 412 }
 413 
 414 static int
 415 segnf_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
 416 {
 417         size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
 418         size_t p;
 419         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 420 
 421         for (p = 0; p < pgno; ++p)
 422                 protv[p] = PROT_READ;
 423         return (0);
 424 }
 425 
 426 /* ARGSUSED */
 427 static u_offset_t
 428 segnf_getoffset(struct seg *seg, caddr_t addr)
 429 {
 430         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 431 
 432         return ((u_offset_t)0);
 433 }
 434 
 435 /* ARGSUSED */
 436 static int
 437 segnf_gettype(struct seg *seg, caddr_t addr)
 438 {
 439         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 440 
 441         return (MAP_SHARED);
 442 }
 443 
 444 /* ARGSUSED */
 445 static int
 446 segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
 447 {
 448         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 449 
 450         *vpp = &nfvp;
 451         return (0);
 452 }
 453 
 454 /*
 455  * segnf pages are not dumped, so we just return
 456  */
 457 /* ARGSUSED */
 458 static void
 459 segnf_dump(struct seg *seg)
 460 {}
 461 
 462 /*ARGSUSED*/
 463 static int
 464 segnf_pagelock(struct seg *seg, caddr_t addr, size_t len,
 465     struct page ***ppp, enum lock_type type, enum seg_rw rw)
 466 {
 467         return (ENOTSUP);
 468 }
 469 
 470 /*ARGSUSED*/
 471 static int
 472 segnf_setpagesize(struct seg *seg, caddr_t addr, size_t len,
 473     uint_t szc)
 474 {
 475         return (ENOTSUP);
 476 }
 477 
 478 /*ARGSUSED*/
 479 static int
 480 segnf_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
 481 {
 482         return (ENODEV);
 483 }