1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 /*
  31  * University Copyright- Copyright (c) 1982, 1986, 1988
  32  * The Regents of the University of California
  33  * All Rights Reserved
  34  *
  35  * University Acknowledgment- Portions of this document are derived from
  36  * software developed by the University of California, Berkeley, and its
  37  * contributors.
  38  */
  39 
  40 /*
  41  * VM - segment of a mapped device.
  42  *
  43  * This segment driver is used when mapping character special devices.
  44  */
  45 
  46 #include <sys/types.h>
  47 #include <sys/t_lock.h>
  48 #include <sys/sysmacros.h>
  49 #include <sys/vtrace.h>
  50 #include <sys/systm.h>
  51 #include <sys/vmsystm.h>
  52 #include <sys/mman.h>
  53 #include <sys/errno.h>
  54 #include <sys/kmem.h>
  55 #include <sys/cmn_err.h>
  56 #include <sys/vnode.h>
  57 #include <sys/proc.h>
  58 #include <sys/conf.h>
  59 #include <sys/debug.h>
  60 #include <sys/ddidevmap.h>
  61 #include <sys/ddi_implfuncs.h>
  62 #include <sys/lgrp.h>
  63 
  64 #include <vm/page.h>
  65 #include <vm/hat.h>
  66 #include <vm/as.h>
  67 #include <vm/seg.h>
  68 #include <vm/seg_dev.h>
  69 #include <vm/seg_kp.h>
  70 #include <vm/seg_kmem.h>
  71 #include <vm/vpage.h>
  72 
  73 #include <sys/sunddi.h>
  74 #include <sys/esunddi.h>
  75 #include <sys/fs/snode.h>
  76 
  77 
  78 #if DEBUG
  79 int segdev_debug;
  80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
  81 #else
  82 #define DEBUGF(level, args)
  83 #endif
  84 
  85 /* Default timeout for devmap context management */
  86 #define CTX_TIMEOUT_VALUE 0
  87 
  88 #define HOLD_DHP_LOCK(dhp)  if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
  89                         { mutex_enter(&dhp->dh_lock); }
  90 
  91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
  92                         { mutex_exit(&dhp->dh_lock); }
  93 
  94 #define round_down_p2(a, s)     ((a) & ~((s) - 1))
  95 #define round_up_p2(a, s)       (((a) + (s) - 1) & ~((s) - 1))
  96 
  97 /*
  98  * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
  99  * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
 100  */
 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize)            \
 102         (((uvaddr | paddr) & (pgsize - 1)) == 0)
 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize)     \
 104         (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
 105 
 106 #define vpgtob(n)       ((n) * sizeof (struct vpage))   /* For brevity */
 107 
 108 #define VTOCVP(vp)      (VTOS(vp)->s_commonvp)       /* we "know" it's an snode */
 109 
 110 static struct devmap_ctx *devmapctx_list = NULL;
 111 static struct devmap_softlock *devmap_slist = NULL;
 112 
 113 /*
 114  * mutex, vnode and page for the page of zeros we use for the trash mappings.
 115  * One trash page is allocated on the first ddi_umem_setup call that uses it
 116  * XXX Eventually, we may want to combine this with what segnf does when all
 117  * hat layers implement HAT_NOFAULT.
 118  *
 119  * The trash page is used when the backing store for a userland mapping is
 120  * removed but the application semantics do not take kindly to a SIGBUS.
 121  * In that scenario, the applications pages are mapped to some dummy page
 122  * which returns garbage on read and writes go into a common place.
 123  * (Perfect for NO_FAULT semantics)
 124  * The device driver is responsible to communicating to the app with some
 125  * other mechanism that such remapping has happened and the app should take
 126  * corrective action.
 127  * We can also use an anonymous memory page as there is no requirement to
 128  * keep the page locked, however this complicates the fault code. RFE.
 129  */
 130 static struct vnode trashvp;
 131 static struct page *trashpp;
 132 
 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
 134 static vmem_t *umem_np_arena;
 135 
 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
 137 #define DEVMAP_DEVMEM_COOKIE    ((ddi_umem_cookie_t)0x1)
 138 
 139 /*
 140  * Macros to check if type of devmap handle
 141  */
 142 #define cookie_is_devmem(c)     \
 143         ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
 144 
 145 #define cookie_is_pmem(c)       \
 146         ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
 147 
 148 #define cookie_is_kpmem(c)      (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
 149         ((c)->type == KMEM_PAGEABLE))
 150 
 151 #define dhp_is_devmem(dhp)      \
 152         (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 153 
 154 #define dhp_is_pmem(dhp)        \
 155         (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 156 
 157 #define dhp_is_kpmem(dhp)       \
 158         (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 159 
 160 /*
 161  * Private seg op routines.
 162  */
 163 static int      segdev_dup(struct seg *, struct seg *);
 164 static int      segdev_unmap(struct seg *, caddr_t, size_t);
 165 static void     segdev_free(struct seg *);
 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
 167                     enum fault_type, enum seg_rw);
 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
 169 static int      segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
 170 static int      segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
 171 static void     segdev_badop(void);
 172 static int      segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
 173 static size_t   segdev_incore(struct seg *, caddr_t, size_t, char *);
 174 static int      segdev_lockop(struct seg *, caddr_t, size_t, int, int,
 175                     ulong_t *, size_t);
 176 static int      segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
 177 static u_offset_t       segdev_getoffset(struct seg *, caddr_t);
 178 static int      segdev_gettype(struct seg *, caddr_t);
 179 static int      segdev_getvp(struct seg *, caddr_t, struct vnode **);
 180 static int      segdev_advise(struct seg *, caddr_t, size_t, uint_t);
 181 static void     segdev_dump(struct seg *);
 182 static int      segdev_pagelock(struct seg *, caddr_t, size_t,
 183                     struct page ***, enum lock_type, enum seg_rw);
 184 static int      segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
 185 static int      segdev_getmemid(struct seg *, caddr_t, memid_t *);
 186 static lgrp_mem_policy_info_t   *segdev_getpolicy(struct seg *, caddr_t);
 187 static int      segdev_capable(struct seg *, segcapability_t);
 188 
 189 /*
 190  * XXX  this struct is used by rootnex_map_fault to identify
 191  *      the segment it has been passed. So if you make it
 192  *      "static" you'll need to fix rootnex_map_fault.
 193  */
 194 struct seg_ops segdev_ops = {
 195         segdev_dup,
 196         segdev_unmap,
 197         segdev_free,
 198         segdev_fault,
 199         segdev_faulta,
 200         segdev_setprot,
 201         segdev_checkprot,
 202         (int (*)())segdev_badop,        /* kluster */
 203         (size_t (*)(struct seg *))NULL, /* swapout */
 204         segdev_sync,                    /* sync */
 205         segdev_incore,
 206         segdev_lockop,                  /* lockop */
 207         segdev_getprot,
 208         segdev_getoffset,
 209         segdev_gettype,
 210         segdev_getvp,
 211         segdev_advise,
 212         segdev_dump,
 213         segdev_pagelock,
 214         segdev_setpagesize,
 215         segdev_getmemid,
 216         segdev_getpolicy,
 217         segdev_capable,
 218 };
 219 
 220 /*
 221  * Private segdev support routines
 222  */
 223 static struct segdev_data *sdp_alloc(void);
 224 
 225 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
 226     size_t, enum seg_rw);
 227 
 228 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
 229     struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
 230 
 231 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
 232     size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
 233 
 234 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
 235 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
 236 static void devmap_softlock_rele(devmap_handle_t *);
 237 static void devmap_ctx_rele(devmap_handle_t *);
 238 
 239 static void devmap_ctxto(void *);
 240 
 241 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
 242     caddr_t addr);
 243 
 244 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
 245     ulong_t *opfn, ulong_t *pagesize);
 246 
 247 static void free_devmap_handle(devmap_handle_t *dhp);
 248 
 249 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
 250     struct seg *newseg);
 251 
 252 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
 253 
 254 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
 255 
 256 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
 257 
 258 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
 259     offset_t off, size_t len, uint_t flags);
 260 
 261 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
 262     caddr_t addr, size_t *llen, caddr_t *laddr);
 263 
 264 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
 265 
 266 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
 267 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
 268 
 269 static void *devmap_umem_alloc_np(size_t size, size_t flags);
 270 static void devmap_umem_free_np(void *addr, size_t size);
 271 
 272 /*
 273  * routines to lock and unlock underlying segkp segment for
 274  * KMEM_PAGEABLE type cookies.
 275  */
 276 static faultcode_t  acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
 277 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
 278 
 279 /*
 280  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
 281  * drivers with devmap_access callbacks
 282  */
 283 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
 284         enum fault_type);
 285 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
 286         enum fault_type);
 287 
 288 static kmutex_t devmapctx_lock;
 289 
 290 static kmutex_t devmap_slock;
 291 
 292 /*
 293  * Initialize the thread callbacks and thread private data.
 294  */
 295 static struct devmap_ctx *
 296 devmap_ctxinit(dev_t dev, ulong_t id)
 297 {
 298         struct devmap_ctx       *devctx;
 299         struct devmap_ctx       *tmp;
 300         dev_info_t              *dip;
 301 
 302         tmp =  kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
 303 
 304         mutex_enter(&devmapctx_lock);
 305 
 306         dip = e_ddi_hold_devi_by_dev(dev, 0);
 307         ASSERT(dip != NULL);
 308         ddi_release_devi(dip);
 309 
 310         for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
 311                 if ((devctx->dip == dip) && (devctx->id == id))
 312                         break;
 313 
 314         if (devctx == NULL) {
 315                 devctx = tmp;
 316                 devctx->dip = dip;
 317                 devctx->id = id;
 318                 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
 319                 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
 320                 devctx->next = devmapctx_list;
 321                 devmapctx_list = devctx;
 322         } else
 323                 kmem_free(tmp, sizeof (struct devmap_ctx));
 324 
 325         mutex_enter(&devctx->lock);
 326         devctx->refcnt++;
 327         mutex_exit(&devctx->lock);
 328         mutex_exit(&devmapctx_lock);
 329 
 330         return (devctx);
 331 }
 332 
 333 /*
 334  * Timeout callback called if a CPU has not given up the device context
 335  * within dhp->dh_timeout_length ticks
 336  */
 337 static void
 338 devmap_ctxto(void *data)
 339 {
 340         struct devmap_ctx *devctx = data;
 341 
 342         TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
 343             "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
 344         mutex_enter(&devctx->lock);
 345         /*
 346          * Set oncpu = 0 so the next mapping trying to get the device context
 347          * can.
 348          */
 349         devctx->oncpu = 0;
 350         devctx->timeout = 0;
 351         cv_signal(&devctx->cv);
 352         mutex_exit(&devctx->lock);
 353 }
 354 
 355 /*
 356  * Create a device segment.
 357  */
 358 int
 359 segdev_create(struct seg *seg, void *argsp)
 360 {
 361         struct segdev_data *sdp;
 362         struct segdev_crargs *a = (struct segdev_crargs *)argsp;
 363         devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
 364         int error;
 365 
 366         /*
 367          * Since the address space is "write" locked, we
 368          * don't need the segment lock to protect "segdev" data.
 369          */
 370         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 371 
 372         hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
 373 
 374         sdp = sdp_alloc();
 375 
 376         sdp->mapfunc = a->mapfunc;
 377         sdp->offset = a->offset;
 378         sdp->prot = a->prot;
 379         sdp->maxprot = a->maxprot;
 380         sdp->type = a->type;
 381         sdp->pageprot = 0;
 382         sdp->softlockcnt = 0;
 383         sdp->vpage = NULL;
 384 
 385         if (sdp->mapfunc == NULL)
 386                 sdp->devmap_data = dhp;
 387         else
 388                 sdp->devmap_data = dhp = NULL;
 389 
 390         sdp->hat_flags = a->hat_flags;
 391         sdp->hat_attr = a->hat_attr;
 392 
 393         /*
 394          * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
 395          */
 396         ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
 397 
 398         /*
 399          * Hold shadow vnode -- segdev only deals with
 400          * character (VCHR) devices. We use the common
 401          * vp to hang pages on.
 402          */
 403         sdp->vp = specfind(a->dev, VCHR);
 404         ASSERT(sdp->vp != NULL);
 405 
 406         seg->s_ops = &segdev_ops;
 407         seg->s_data = sdp;
 408 
 409         while (dhp != NULL) {
 410                 dhp->dh_seg = seg;
 411                 dhp = dhp->dh_next;
 412         }
 413 
 414         /*
 415          * Inform the vnode of the new mapping.
 416          */
 417         /*
 418          * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
 419          * dhp specific maxprot because spec_addmap does not use maxprot.
 420          */
 421         error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
 422             seg->s_as, seg->s_base, seg->s_size,
 423             sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
 424 
 425         if (error != 0) {
 426                 sdp->devmap_data = NULL;
 427                 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
 428                     HAT_UNLOAD_UNMAP);
 429         } else {
 430                 /*
 431                  * Mappings of /dev/null don't count towards the VSZ of a
 432                  * process.  Mappings of /dev/null have no mapping type.
 433                  */
 434                 if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
 435                     MAP_PRIVATE)) == 0) {
 436                         seg->s_as->a_resvsize -= seg->s_size;
 437                 }
 438         }
 439 
 440         return (error);
 441 }
 442 
 443 static struct segdev_data *
 444 sdp_alloc(void)
 445 {
 446         struct segdev_data *sdp;
 447 
 448         sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
 449         rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
 450 
 451         return (sdp);
 452 }
 453 
 454 /*
 455  * Duplicate seg and return new segment in newseg.
 456  */
 457 static int
 458 segdev_dup(struct seg *seg, struct seg *newseg)
 459 {
 460         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
 461         struct segdev_data *newsdp;
 462         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
 463         size_t npages;
 464         int ret;
 465 
 466         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
 467             "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
 468 
 469         DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
 470             (void *)dhp, (void *)seg));
 471 
 472         /*
 473          * Since the address space is "write" locked, we
 474          * don't need the segment lock to protect "segdev" data.
 475          */
 476         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 477 
 478         newsdp = sdp_alloc();
 479 
 480         newseg->s_ops = seg->s_ops;
 481         newseg->s_data = (void *)newsdp;
 482 
 483         VN_HOLD(sdp->vp);
 484         newsdp->vp   = sdp->vp;
 485         newsdp->mapfunc = sdp->mapfunc;
 486         newsdp->offset       = sdp->offset;
 487         newsdp->pageprot = sdp->pageprot;
 488         newsdp->prot = sdp->prot;
 489         newsdp->maxprot = sdp->maxprot;
 490         newsdp->type = sdp->type;
 491         newsdp->hat_attr = sdp->hat_attr;
 492         newsdp->hat_flags = sdp->hat_flags;
 493         newsdp->softlockcnt = 0;
 494 
 495         /*
 496          * Initialize per page data if the segment we are
 497          * dup'ing has per page information.
 498          */
 499         npages = seg_pages(newseg);
 500 
 501         if (sdp->vpage != NULL) {
 502                 size_t nbytes = vpgtob(npages);
 503 
 504                 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
 505                 bcopy(sdp->vpage, newsdp->vpage, nbytes);
 506         } else
 507                 newsdp->vpage = NULL;
 508 
 509         /*
 510          * duplicate devmap handles
 511          */
 512         if (dhp != NULL) {
 513                 ret = devmap_handle_dup(dhp,
 514                     (devmap_handle_t **)&newsdp->devmap_data, newseg);
 515                 if (ret != 0) {
 516                         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
 517                             "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
 518                             ret, (void *)dhp, (void *)seg);
 519                         DEBUGF(1, (CE_CONT,
 520                             "segdev_dup: ret %x dhp %p seg %p\n",
 521                             ret, (void *)dhp, (void *)seg));
 522                         return (ret);
 523                 }
 524         }
 525 
 526         /*
 527          * Inform the common vnode of the new mapping.
 528          */
 529         return (VOP_ADDMAP(VTOCVP(newsdp->vp),
 530             newsdp->offset, newseg->s_as,
 531             newseg->s_base, newseg->s_size, newsdp->prot,
 532             newsdp->maxprot, sdp->type, CRED(), NULL));
 533 }
 534 
 535 /*
 536  * duplicate devmap handles
 537  */
 538 static int
 539 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
 540     struct seg *newseg)
 541 {
 542         devmap_handle_t *newdhp_save = NULL;
 543         devmap_handle_t *newdhp = NULL;
 544         struct devmap_callback_ctl *callbackops;
 545 
 546         while (dhp != NULL) {
 547                 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
 548 
 549                 /* Need to lock the original dhp while copying if REMAP */
 550                 HOLD_DHP_LOCK(dhp);
 551                 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
 552                 RELE_DHP_LOCK(dhp);
 553                 newdhp->dh_seg = newseg;
 554                 newdhp->dh_next = NULL;
 555                 if (newdhp_save != NULL)
 556                         newdhp_save->dh_next = newdhp;
 557                 else
 558                         *new_dhp = newdhp;
 559                 newdhp_save = newdhp;
 560 
 561                 callbackops = &newdhp->dh_callbackops;
 562 
 563                 if (dhp->dh_softlock != NULL)
 564                         newdhp->dh_softlock = devmap_softlock_init(
 565                             newdhp->dh_dev,
 566                             (ulong_t)callbackops->devmap_access);
 567                 if (dhp->dh_ctx != NULL)
 568                         newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
 569                             (ulong_t)callbackops->devmap_access);
 570 
 571                 /*
 572                  * Initialize dh_lock if we want to do remap.
 573                  */
 574                 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
 575                         mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
 576                         newdhp->dh_flags |= DEVMAP_LOCK_INITED;
 577                 }
 578 
 579                 if (callbackops->devmap_dup != NULL) {
 580                         int ret;
 581 
 582                         /*
 583                          * Call the dup callback so that the driver can
 584                          * duplicate its private data.
 585                          */
 586                         ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
 587                             (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
 588 
 589                         if (ret != 0) {
 590                                 /*
 591                                  * We want to free up this segment as the driver
 592                                  * has indicated that we can't dup it.  But we
 593                                  * don't want to call the drivers, devmap_unmap,
 594                                  * callback function as the driver does not
 595                                  * think this segment exists. The caller of
 596                                  * devmap_dup will call seg_free on newseg
 597                                  * as it was the caller that allocated the
 598                                  * segment.
 599                                  */
 600                                 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
 601                                     "newdhp %p dhp %p\n", (void *)newdhp,
 602                                     (void *)dhp));
 603                                 callbackops->devmap_unmap = NULL;
 604                                 return (ret);
 605                         }
 606                 }
 607 
 608                 dhp = dhp->dh_next;
 609         }
 610 
 611         return (0);
 612 }
 613 
 614 /*
 615  * Split a segment at addr for length len.
 616  */
 617 /*ARGSUSED*/
 618 static int
 619 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
 620 {
 621         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
 622         register struct segdev_data *nsdp;
 623         register struct seg *nseg;
 624         register size_t opages;         /* old segment size in pages */
 625         register size_t npages;         /* new segment size in pages */
 626         register size_t dpages;         /* pages being deleted (unmapped) */
 627         register size_t nbytes;
 628         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
 629         devmap_handle_t *dhpp;
 630         devmap_handle_t *newdhp;
 631         struct devmap_callback_ctl *callbackops;
 632         caddr_t nbase;
 633         offset_t off;
 634         ulong_t nsize;
 635         size_t mlen, sz;
 636 
 637         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
 638             "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
 639             (void *)dhp, (void *)seg, (void *)addr, len);
 640 
 641         DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
 642             (void *)dhp, (void *)seg, (void *)addr, len));
 643 
 644         /*
 645          * Since the address space is "write" locked, we
 646          * don't need the segment lock to protect "segdev" data.
 647          */
 648         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 649 
 650         if ((sz = sdp->softlockcnt) > 0) {
 651                 /*
 652                  * Fail the unmap if pages are SOFTLOCKed through this mapping.
 653                  * softlockcnt is protected from change by the as write lock.
 654                  */
 655                 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
 656                     "segdev_unmap:error softlockcnt = %ld", sz);
 657                 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
 658                 return (EAGAIN);
 659         }
 660 
 661         /*
 662          * Check for bad sizes
 663          */
 664         if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
 665             (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
 666                 panic("segdev_unmap");
 667 
 668         if (dhp != NULL) {
 669                 devmap_handle_t *tdhp;
 670                 /*
 671                  * If large page size was used in hat_devload(),
 672                  * the same page size must be used in hat_unload().
 673                  */
 674                 dhpp = tdhp = devmap_find_handle(dhp, addr);
 675                 while (tdhp != NULL) {
 676                         if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
 677                                 break;
 678                         }
 679                         tdhp = tdhp->dh_next;
 680                 }
 681                 if (tdhp != NULL) {     /* found a dhp using large pages */
 682                         size_t slen = len;
 683                         size_t mlen;
 684                         size_t soff;
 685 
 686                         soff = (ulong_t)(addr - dhpp->dh_uvaddr);
 687                         while (slen != 0) {
 688                                 mlen = MIN(slen, (dhpp->dh_len - soff));
 689                                 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
 690                                     dhpp->dh_len, HAT_UNLOAD_UNMAP);
 691                                 dhpp = dhpp->dh_next;
 692                                 ASSERT(slen >= mlen);
 693                                 slen -= mlen;
 694                                 soff = 0;
 695                         }
 696                 } else
 697                         hat_unload(seg->s_as->a_hat, addr, len,
 698                             HAT_UNLOAD_UNMAP);
 699         } else {
 700                 /*
 701                  * Unload any hardware translations in the range
 702                  * to be taken out.
 703                  */
 704                 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
 705         }
 706 
 707         /*
 708          * get the user offset which will used in the driver callbacks
 709          */
 710         off = sdp->offset + (offset_t)(addr - seg->s_base);
 711 
 712         /*
 713          * Inform the vnode of the unmapping.
 714          */
 715         ASSERT(sdp->vp != NULL);
 716         (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
 717             sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
 718 
 719         /*
 720          * Check for entire segment
 721          */
 722         if (addr == seg->s_base && len == seg->s_size) {
 723                 seg_free(seg);
 724                 return (0);
 725         }
 726 
 727         opages = seg_pages(seg);
 728         dpages = btop(len);
 729         npages = opages - dpages;
 730 
 731         /*
 732          * Check for beginning of segment
 733          */
 734         if (addr == seg->s_base) {
 735                 if (sdp->vpage != NULL) {
 736                         register struct vpage *ovpage;
 737 
 738                         ovpage = sdp->vpage; /* keep pointer to vpage */
 739 
 740                         nbytes = vpgtob(npages);
 741                         sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 742                         bcopy(&ovpage[dpages], sdp->vpage, nbytes);
 743 
 744                         /* free up old vpage */
 745                         kmem_free(ovpage, vpgtob(opages));
 746                 }
 747 
 748                 /*
 749                  * free devmap handles from the beginning of the mapping.
 750                  */
 751                 if (dhp != NULL)
 752                         devmap_handle_unmap_head(dhp, len);
 753 
 754                 sdp->offset += (offset_t)len;
 755 
 756                 seg->s_base += len;
 757                 seg->s_size -= len;
 758 
 759                 return (0);
 760         }
 761 
 762         /*
 763          * Check for end of segment
 764          */
 765         if (addr + len == seg->s_base + seg->s_size) {
 766                 if (sdp->vpage != NULL) {
 767                         register struct vpage *ovpage;
 768 
 769                         ovpage = sdp->vpage; /* keep pointer to vpage */
 770 
 771                         nbytes = vpgtob(npages);
 772                         sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 773                         bcopy(ovpage, sdp->vpage, nbytes);
 774 
 775                         /* free up old vpage */
 776                         kmem_free(ovpage, vpgtob(opages));
 777                 }
 778                 seg->s_size -= len;
 779 
 780                 /*
 781                  * free devmap handles from addr to the end of the mapping.
 782                  */
 783                 if (dhp != NULL)
 784                         devmap_handle_unmap_tail(dhp, addr);
 785 
 786                 return (0);
 787         }
 788 
 789         /*
 790          * The section to go is in the middle of the segment,
 791          * have to make it into two segments.  nseg is made for
 792          * the high end while seg is cut down at the low end.
 793          */
 794         nbase = addr + len;                             /* new seg base */
 795         nsize = (seg->s_base + seg->s_size) - nbase;      /* new seg size */
 796         seg->s_size = addr - seg->s_base;         /* shrink old seg */
 797         nseg = seg_alloc(seg->s_as, nbase, nsize);
 798         if (nseg == NULL)
 799                 panic("segdev_unmap seg_alloc");
 800 
 801         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
 802             "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
 803         DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
 804             (void *)seg, (void *)nseg));
 805         nsdp = sdp_alloc();
 806 
 807         nseg->s_ops = seg->s_ops;
 808         nseg->s_data = (void *)nsdp;
 809 
 810         VN_HOLD(sdp->vp);
 811         nsdp->mapfunc = sdp->mapfunc;
 812         nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
 813         nsdp->vp     = sdp->vp;
 814         nsdp->pageprot = sdp->pageprot;
 815         nsdp->prot   = sdp->prot;
 816         nsdp->maxprot = sdp->maxprot;
 817         nsdp->type = sdp->type;
 818         nsdp->hat_attr = sdp->hat_attr;
 819         nsdp->hat_flags = sdp->hat_flags;
 820         nsdp->softlockcnt = 0;
 821 
 822         /*
 823          * Initialize per page data if the segment we are
 824          * dup'ing has per page information.
 825          */
 826         if (sdp->vpage != NULL) {
 827                 /* need to split vpage into two arrays */
 828                 register size_t nnbytes;
 829                 register size_t nnpages;
 830                 register struct vpage *ovpage;
 831 
 832                 ovpage = sdp->vpage;         /* keep pointer to vpage */
 833 
 834                 npages = seg_pages(seg);        /* seg has shrunk */
 835                 nbytes = vpgtob(npages);
 836                 nnpages = seg_pages(nseg);
 837                 nnbytes = vpgtob(nnpages);
 838 
 839                 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 840                 bcopy(ovpage, sdp->vpage, nbytes);
 841 
 842                 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
 843                 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
 844 
 845                 /* free up old vpage */
 846                 kmem_free(ovpage, vpgtob(opages));
 847         } else
 848                 nsdp->vpage = NULL;
 849 
 850         /*
 851          * unmap dhps.
 852          */
 853         if (dhp == NULL) {
 854                 nsdp->devmap_data = NULL;
 855                 return (0);
 856         }
 857         while (dhp != NULL) {
 858                 callbackops = &dhp->dh_callbackops;
 859                 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
 860                     "segdev_unmap: dhp=%p addr=%p", dhp, addr);
 861                 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
 862                     (void *)dhp, (void *)addr,
 863                     (void *)dhp->dh_uvaddr, dhp->dh_len));
 864 
 865                 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
 866                         dhpp = dhp->dh_next;
 867                         dhp->dh_next = NULL;
 868                         dhp = dhpp;
 869                 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
 870                         dhp = dhp->dh_next;
 871                 } else if (addr > dhp->dh_uvaddr &&
 872                     (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
 873                         /*
 874                          * <addr, addr+len> is enclosed by dhp.
 875                          * create a newdhp that begins at addr+len and
 876                          * ends at dhp->dh_uvaddr+dhp->dh_len.
 877                          */
 878                         newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
 879                         HOLD_DHP_LOCK(dhp);
 880                         bcopy(dhp, newdhp, sizeof (devmap_handle_t));
 881                         RELE_DHP_LOCK(dhp);
 882                         newdhp->dh_seg = nseg;
 883                         newdhp->dh_next = dhp->dh_next;
 884                         if (dhp->dh_softlock != NULL)
 885                                 newdhp->dh_softlock = devmap_softlock_init(
 886                                     newdhp->dh_dev,
 887                                     (ulong_t)callbackops->devmap_access);
 888                         if (dhp->dh_ctx != NULL)
 889                                 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
 890                                     (ulong_t)callbackops->devmap_access);
 891                         if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
 892                                 mutex_init(&newdhp->dh_lock,
 893                                     NULL, MUTEX_DEFAULT, NULL);
 894                         }
 895                         if (callbackops->devmap_unmap != NULL)
 896                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 897                                     off, len, dhp, &dhp->dh_pvtp,
 898                                     newdhp, &newdhp->dh_pvtp);
 899                         mlen = len + (addr - dhp->dh_uvaddr);
 900                         devmap_handle_reduce_len(newdhp, mlen);
 901                         nsdp->devmap_data = newdhp;
 902                         /* XX Changing len should recalculate LARGE flag */
 903                         dhp->dh_len = addr - dhp->dh_uvaddr;
 904                         dhpp = dhp->dh_next;
 905                         dhp->dh_next = NULL;
 906                         dhp = dhpp;
 907                 } else if ((addr > dhp->dh_uvaddr) &&
 908                     ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
 909                         mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
 910                         /*
 911                          * <addr, addr+len> spans over dhps.
 912                          */
 913                         if (callbackops->devmap_unmap != NULL)
 914                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 915                                     off, mlen, (devmap_cookie_t *)dhp,
 916                                     &dhp->dh_pvtp, NULL, NULL);
 917                         /* XX Changing len should recalculate LARGE flag */
 918                         dhp->dh_len = addr - dhp->dh_uvaddr;
 919                         dhpp = dhp->dh_next;
 920                         dhp->dh_next = NULL;
 921                         dhp = dhpp;
 922                         nsdp->devmap_data = dhp;
 923                 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
 924                         /*
 925                          * dhp is enclosed by <addr, addr+len>.
 926                          */
 927                         dhp->dh_seg = nseg;
 928                         nsdp->devmap_data = dhp;
 929                         dhp = devmap_handle_unmap(dhp);
 930                         nsdp->devmap_data = dhp; /* XX redundant? */
 931                 } else if (((addr + len) > dhp->dh_uvaddr) &&
 932                     ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
 933                         mlen = addr + len - dhp->dh_uvaddr;
 934                         if (callbackops->devmap_unmap != NULL)
 935                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 936                                     dhp->dh_uoff, mlen, NULL,
 937                                     NULL, dhp, &dhp->dh_pvtp);
 938                         devmap_handle_reduce_len(dhp, mlen);
 939                         nsdp->devmap_data = dhp;
 940                         dhp->dh_seg = nseg;
 941                         dhp = dhp->dh_next;
 942                 } else {
 943                         dhp->dh_seg = nseg;
 944                         dhp = dhp->dh_next;
 945                 }
 946         }
 947         return (0);
 948 }
 949 
 950 /*
 951  * Utility function handles reducing the length of a devmap handle during unmap
 952  * Note that is only used for unmapping the front portion of the handler,
 953  * i.e., we are bumping up the offset/pfn etc up by len
 954  * Do not use if reducing length at the tail.
 955  */
 956 static void
 957 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
 958 {
 959         struct ddi_umem_cookie *cp;
 960         struct devmap_pmem_cookie *pcp;
 961         /*
 962          * adjust devmap handle fields
 963          */
 964         ASSERT(len < dhp->dh_len);
 965 
 966         /* Make sure only page-aligned changes are done */
 967         ASSERT((len & PAGEOFFSET) == 0);
 968 
 969         dhp->dh_len -= len;
 970         dhp->dh_uoff += (offset_t)len;
 971         dhp->dh_roff += (offset_t)len;
 972         dhp->dh_uvaddr += len;
 973         /* Need to grab dhp lock if REMAP */
 974         HOLD_DHP_LOCK(dhp);
 975         cp = dhp->dh_cookie;
 976         if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
 977                 if (cookie_is_devmem(cp)) {
 978                         dhp->dh_pfn += btop(len);
 979                 } else if (cookie_is_pmem(cp)) {
 980                         pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
 981                         ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
 982                             dhp->dh_roff < ptob(pcp->dp_npages));
 983                 } else {
 984                         ASSERT(dhp->dh_roff < cp->size);
 985                         ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
 986                             dhp->dh_cvaddr < (cp->cvaddr + cp->size));
 987                         ASSERT((dhp->dh_cvaddr + len) <=
 988                             (cp->cvaddr + cp->size));
 989 
 990                         dhp->dh_cvaddr += len;
 991                 }
 992         }
 993         /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
 994         RELE_DHP_LOCK(dhp);
 995 }
 996 
 997 /*
 998  * Free devmap handle, dhp.
 999  * Return the next devmap handle on the linked list.
1000  */
1001 static devmap_handle_t *
1002 devmap_handle_unmap(devmap_handle_t *dhp)
1003 {
1004         struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1005         struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1006         devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1007 
1008         ASSERT(dhp != NULL);
1009 
1010         /*
1011          * before we free up dhp, call the driver's devmap_unmap entry point
1012          * to free resources allocated for this dhp.
1013          */
1014         if (callbackops->devmap_unmap != NULL) {
1015                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1016                     dhp->dh_len, NULL, NULL, NULL, NULL);
1017         }
1018 
1019         if (dhpp == dhp) {      /* releasing first dhp, change sdp data */
1020                 sdp->devmap_data = dhp->dh_next;
1021         } else {
1022                 while (dhpp->dh_next != dhp) {
1023                         dhpp = dhpp->dh_next;
1024                 }
1025                 dhpp->dh_next = dhp->dh_next;
1026         }
1027         dhpp = dhp->dh_next; /* return value is next dhp in chain */
1028 
1029         if (dhp->dh_softlock != NULL)
1030                 devmap_softlock_rele(dhp);
1031 
1032         if (dhp->dh_ctx != NULL)
1033                 devmap_ctx_rele(dhp);
1034 
1035         if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1036                 mutex_destroy(&dhp->dh_lock);
1037         }
1038         kmem_free(dhp, sizeof (devmap_handle_t));
1039 
1040         return (dhpp);
1041 }
1042 
1043 /*
1044  * Free complete devmap handles from dhp for len bytes
1045  * dhp can be either the first handle or a subsequent handle
1046  */
1047 static void
1048 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1049 {
1050         struct devmap_callback_ctl *callbackops;
1051 
1052         /*
1053          * free the devmap handles covered by len.
1054          */
1055         while (len >= dhp->dh_len) {
1056                 len -= dhp->dh_len;
1057                 dhp = devmap_handle_unmap(dhp);
1058         }
1059         if (len != 0) { /* partial unmap at head of first remaining dhp */
1060                 callbackops = &dhp->dh_callbackops;
1061 
1062                 /*
1063                  * Call the unmap callback so the drivers can make
1064                  * adjustment on its private data.
1065                  */
1066                 if (callbackops->devmap_unmap != NULL)
1067                         (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1068                             dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1069                 devmap_handle_reduce_len(dhp, len);
1070         }
1071 }
1072 
1073 /*
1074  * Free devmap handles to truncate  the mapping after addr
1075  * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1076  *      Also could then use the routine in middle unmap case too
1077  */
1078 static void
1079 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1080 {
1081         register struct seg *seg = dhp->dh_seg;
1082         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1083         register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1084         struct devmap_callback_ctl *callbackops;
1085         register devmap_handle_t *dhpp;
1086         size_t maplen;
1087         ulong_t off;
1088         size_t len;
1089 
1090         maplen = (size_t)(addr - dhp->dh_uvaddr);
1091         dhph = devmap_find_handle(dhph, addr);
1092 
1093         while (dhph != NULL) {
1094                 if (maplen == 0) {
1095                         dhph =  devmap_handle_unmap(dhph);
1096                 } else {
1097                         callbackops = &dhph->dh_callbackops;
1098                         len = dhph->dh_len - maplen;
1099                         off = (ulong_t)sdp->offset + (addr - seg->s_base);
1100                         /*
1101                          * Call the unmap callback so the driver
1102                          * can make adjustments on its private data.
1103                          */
1104                         if (callbackops->devmap_unmap != NULL)
1105                                 (*callbackops->devmap_unmap)(dhph,
1106                                     dhph->dh_pvtp, off, len,
1107                                     (devmap_cookie_t *)dhph,
1108                                     &dhph->dh_pvtp, NULL, NULL);
1109                         /* XXX Reducing len needs to recalculate LARGE flag */
1110                         dhph->dh_len = maplen;
1111                         maplen = 0;
1112                         dhpp = dhph->dh_next;
1113                         dhph->dh_next = NULL;
1114                         dhph = dhpp;
1115                 }
1116         } /* end while */
1117 }
1118 
1119 /*
1120  * Free a segment.
1121  */
1122 static void
1123 segdev_free(struct seg *seg)
1124 {
1125         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1126         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1127 
1128         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1129             "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1130         DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1131             (void *)dhp, (void *)seg));
1132 
1133         /*
1134          * Since the address space is "write" locked, we
1135          * don't need the segment lock to protect "segdev" data.
1136          */
1137         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1138 
1139         while (dhp != NULL)
1140                 dhp = devmap_handle_unmap(dhp);
1141 
1142         VN_RELE(sdp->vp);
1143         if (sdp->vpage != NULL)
1144                 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1145 
1146         rw_destroy(&sdp->lock);
1147         kmem_free(sdp, sizeof (*sdp));
1148 }
1149 
1150 static void
1151 free_devmap_handle(devmap_handle_t *dhp)
1152 {
1153         register devmap_handle_t *dhpp;
1154 
1155         /*
1156          * free up devmap handle
1157          */
1158         while (dhp != NULL) {
1159                 dhpp = dhp->dh_next;
1160                 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1161                         mutex_destroy(&dhp->dh_lock);
1162                 }
1163 
1164                 if (dhp->dh_softlock != NULL)
1165                         devmap_softlock_rele(dhp);
1166 
1167                 if (dhp->dh_ctx != NULL)
1168                         devmap_ctx_rele(dhp);
1169 
1170                 kmem_free(dhp, sizeof (devmap_handle_t));
1171                 dhp = dhpp;
1172         }
1173 }
1174 
1175 /*
1176  * routines to lock and unlock underlying segkp segment for
1177  * KMEM_PAGEABLE type cookies.
1178  * segkp only allows a single pending F_SOFTLOCK
1179  * we keep track of number of locks in the cookie so we can
1180  * have multiple pending faults and manage the calls to segkp.
1181  * RFE: if segkp supports either pagelock or can support multiple
1182  * calls to F_SOFTLOCK, then these routines can go away.
1183  *      If pagelock, segdev_faultpage can fault on a page by page basis
1184  *              and simplifies the code quite a bit.
1185  *      if multiple calls allowed but not partial ranges, then need for
1186  *      cookie->lock and locked count goes away, code can call as_fault directly
1187  */
1188 static faultcode_t
1189 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1190 {
1191         int err = 0;
1192         ASSERT(cookie_is_kpmem(cookie));
1193         /*
1194          * Fault in pages in segkp with F_SOFTLOCK.
1195          * We want to hold the lock until all pages have been loaded.
1196          * segkp only allows single caller to hold SOFTLOCK, so cookie
1197          * holds a count so we dont call into segkp multiple times
1198          */
1199         mutex_enter(&cookie->lock);
1200 
1201         /*
1202          * Check for overflow in locked field
1203          */
1204         if ((UINT32_MAX - cookie->locked) < npages) {
1205                 err = FC_MAKE_ERR(ENOMEM);
1206         } else if (cookie->locked == 0) {
1207                 /* First time locking */
1208                 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1209                     cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1210         }
1211         if (!err) {
1212                 cookie->locked += npages;
1213         }
1214         mutex_exit(&cookie->lock);
1215         return (err);
1216 }
1217 
1218 static void
1219 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1220 {
1221         mutex_enter(&cookie->lock);
1222         ASSERT(cookie_is_kpmem(cookie));
1223         ASSERT(cookie->locked >= npages);
1224         cookie->locked -= (uint_t)npages;
1225         if (cookie->locked == 0) {
1226                 /* Last unlock */
1227                 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1228                     cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1229                         panic("segdev releasing kpmem lock %p", (void *)cookie);
1230         }
1231         mutex_exit(&cookie->lock);
1232 }
1233 
1234 /*
1235  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1236  * drivers with devmap_access callbacks
1237  * slock->softlocked basically works like a rw lock
1238  *      -ve counts => F_SOFTLOCK in progress
1239  *      +ve counts => F_INVAL/F_PROT in progress
1240  * We allow only one F_SOFTLOCK at a time
1241  * but can have multiple pending F_INVAL/F_PROT calls
1242  *
1243  * This routine waits using cv_wait_sig so killing processes is more graceful
1244  * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1245  */
1246 static int devmap_softlock_enter(
1247         struct devmap_softlock *slock,
1248         size_t npages,
1249         enum fault_type type)
1250 {
1251         if (npages == 0)
1252                 return (0);
1253         mutex_enter(&(slock->lock));
1254         switch (type) {
1255         case F_SOFTLOCK :
1256                 while (slock->softlocked) {
1257                         if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1258                                 /* signalled */
1259                                 mutex_exit(&(slock->lock));
1260                                 return (EINTR);
1261                         }
1262                 }
1263                 slock->softlocked -= npages; /* -ve count => locked */
1264                 break;
1265         case F_INVAL :
1266         case F_PROT :
1267                 while (slock->softlocked < 0)
1268                         if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1269                                 /* signalled */
1270                                 mutex_exit(&(slock->lock));
1271                                 return (EINTR);
1272                         }
1273                 slock->softlocked += npages; /* +ve count => f_invals */
1274                 break;
1275         default:
1276                 ASSERT(0);
1277         }
1278         mutex_exit(&(slock->lock));
1279         return (0);
1280 }
1281 
1282 static void devmap_softlock_exit(
1283         struct devmap_softlock *slock,
1284         size_t npages,
1285         enum fault_type type)
1286 {
1287         if (slock == NULL)
1288                 return;
1289         mutex_enter(&(slock->lock));
1290         switch (type) {
1291         case F_SOFTLOCK :
1292                 ASSERT(-slock->softlocked >= npages);
1293                 slock->softlocked += npages; /* -ve count is softlocked */
1294                 if (slock->softlocked == 0)
1295                         cv_signal(&slock->cv);
1296                 break;
1297         case F_INVAL :
1298         case F_PROT:
1299                 ASSERT(slock->softlocked >= npages);
1300                 slock->softlocked -= npages;
1301                 if (slock->softlocked == 0)
1302                         cv_signal(&slock->cv);
1303                 break;
1304         default:
1305                 ASSERT(0);
1306         }
1307         mutex_exit(&(slock->lock));
1308 }
1309 
1310 /*
1311  * Do a F_SOFTUNLOCK call over the range requested.
1312  * The range must have already been F_SOFTLOCK'ed.
1313  * The segment lock should be held, (but not the segment private lock?)
1314  *  The softunlock code below does not adjust for large page sizes
1315  *      assumes the caller already did any addr/len adjustments for
1316  *      pagesize mappings before calling.
1317  */
1318 /*ARGSUSED*/
1319 static void
1320 segdev_softunlock(
1321         struct hat *hat,                /* the hat */
1322         struct seg *seg,                /* seg_dev of interest */
1323         caddr_t addr,                   /* base address of range */
1324         size_t len,                     /* number of bytes */
1325         enum seg_rw rw)                 /* type of access at fault */
1326 {
1327         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1328         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1329 
1330         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1331             "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1332             dhp_head, sdp, addr, len);
1333         DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1334             "addr %p len %lx\n",
1335             (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1336 
1337         hat_unlock(hat, addr, len);
1338 
1339         if (dhp_head != NULL) {
1340                 devmap_handle_t *dhp;
1341                 size_t mlen;
1342                 size_t tlen = len;
1343                 ulong_t off;
1344 
1345                 dhp = devmap_find_handle(dhp_head, addr);
1346                 ASSERT(dhp != NULL);
1347 
1348                 off = (ulong_t)(addr - dhp->dh_uvaddr);
1349                 while (tlen != 0) {
1350                         mlen = MIN(tlen, (dhp->dh_len - off));
1351 
1352                         /*
1353                          * unlock segkp memory, locked during F_SOFTLOCK
1354                          */
1355                         if (dhp_is_kpmem(dhp)) {
1356                                 release_kpmem_lock(
1357                                     (struct ddi_umem_cookie *)dhp->dh_cookie,
1358                                     btopr(mlen));
1359                         }
1360 
1361                         /*
1362                          * Do the softlock accounting for devmap_access
1363                          */
1364                         if (dhp->dh_callbackops.devmap_access != NULL) {
1365                                 devmap_softlock_exit(dhp->dh_softlock,
1366                                     btopr(mlen), F_SOFTLOCK);
1367                         }
1368 
1369                         tlen -= mlen;
1370                         dhp = dhp->dh_next;
1371                         off = 0;
1372                 }
1373         }
1374 
1375         mutex_enter(&freemem_lock);
1376         ASSERT(sdp->softlockcnt >= btopr(len));
1377         sdp->softlockcnt -= btopr(len);
1378         mutex_exit(&freemem_lock);
1379         if (sdp->softlockcnt == 0) {
1380                 /*
1381                  * All SOFTLOCKS are gone. Wakeup any waiting
1382                  * unmappers so they can try again to unmap.
1383                  * Check for waiters first without the mutex
1384                  * held so we don't always grab the mutex on
1385                  * softunlocks.
1386                  */
1387                 if (AS_ISUNMAPWAIT(seg->s_as)) {
1388                         mutex_enter(&seg->s_as->a_contents);
1389                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1390                                 AS_CLRUNMAPWAIT(seg->s_as);
1391                                 cv_broadcast(&seg->s_as->a_cv);
1392                         }
1393                         mutex_exit(&seg->s_as->a_contents);
1394                 }
1395         }
1396 
1397 }
1398 
1399 /*
1400  * Handle fault for a single page.
1401  * Done in a separate routine so we can handle errors more easily.
1402  * This routine is called only from segdev_faultpages()
1403  * when looping over the range of addresses requested. The segment lock is held.
1404  */
1405 static faultcode_t
1406 segdev_faultpage(
1407         struct hat *hat,                /* the hat */
1408         struct seg *seg,                /* seg_dev of interest */
1409         caddr_t addr,                   /* address in as */
1410         struct vpage *vpage,            /* pointer to vpage for seg, addr */
1411         enum fault_type type,           /* type of fault */
1412         enum seg_rw rw,                 /* type of access at fault */
1413         devmap_handle_t *dhp)           /* devmap handle if any for this page */
1414 {
1415         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1416         uint_t prot;
1417         pfn_t pfnum = PFN_INVALID;
1418         u_offset_t offset;
1419         uint_t hat_flags;
1420         dev_info_t *dip;
1421 
1422         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1423             "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1424         DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1425             (void *)dhp, (void *)seg, (void *)addr));
1426 
1427         /*
1428          * Initialize protection value for this page.
1429          * If we have per page protection values check it now.
1430          */
1431         if (sdp->pageprot) {
1432                 uint_t protchk;
1433 
1434                 switch (rw) {
1435                 case S_READ:
1436                         protchk = PROT_READ;
1437                         break;
1438                 case S_WRITE:
1439                         protchk = PROT_WRITE;
1440                         break;
1441                 case S_EXEC:
1442                         protchk = PROT_EXEC;
1443                         break;
1444                 case S_OTHER:
1445                 default:
1446                         protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1447                         break;
1448                 }
1449 
1450                 prot = VPP_PROT(vpage);
1451                 if ((prot & protchk) == 0)
1452                         return (FC_PROT);       /* illegal access type */
1453         } else {
1454                 prot = sdp->prot;
1455                 /* caller has already done segment level protection check */
1456         }
1457 
1458         if (type == F_SOFTLOCK) {
1459                 mutex_enter(&freemem_lock);
1460                 sdp->softlockcnt++;
1461                 mutex_exit(&freemem_lock);
1462         }
1463 
1464         hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1465         offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1466         /*
1467          * In the devmap framework, sdp->mapfunc is set to NULL.  we can get
1468          * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1469          * seg->s_base.
1470          */
1471         if (dhp == NULL) {
1472                 /* If segment has devmap_data, then dhp should be non-NULL */
1473                 ASSERT(sdp->devmap_data == NULL);
1474                 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1475                     (off_t)offset, prot);
1476                 prot |= sdp->hat_attr;
1477         } else {
1478                 ulong_t off;
1479                 struct ddi_umem_cookie *cp;
1480                 struct devmap_pmem_cookie *pcp;
1481 
1482                 /* ensure the dhp passed in contains addr. */
1483                 ASSERT(dhp == devmap_find_handle(
1484                     (devmap_handle_t *)sdp->devmap_data, addr));
1485 
1486                 off = addr - dhp->dh_uvaddr;
1487 
1488                 /*
1489                  * This routine assumes that the caller makes sure that the
1490                  * fields in dhp used below are unchanged due to remap during
1491                  * this call. Caller does HOLD_DHP_LOCK if neeed
1492                  */
1493                 cp = dhp->dh_cookie;
1494                 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1495                         pfnum = PFN_INVALID;
1496                 } else if (cookie_is_devmem(cp)) {
1497                         pfnum = dhp->dh_pfn + btop(off);
1498                 } else if (cookie_is_pmem(cp)) {
1499                         pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1500                         ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1501                             dhp->dh_roff < ptob(pcp->dp_npages));
1502                         pfnum = page_pptonum(
1503                             pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1504                 } else {
1505                         ASSERT(dhp->dh_roff < cp->size);
1506                         ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1507                             dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1508                         ASSERT((dhp->dh_cvaddr + off) <=
1509                             (cp->cvaddr + cp->size));
1510                         ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1511                             (cp->cvaddr + cp->size));
1512 
1513                         switch (cp->type) {
1514                         case UMEM_LOCKED :
1515                                 if (cp->pparray != NULL) {
1516                                         ASSERT((dhp->dh_roff &
1517                                             PAGEOFFSET) == 0);
1518                                         pfnum = page_pptonum(
1519                                             cp->pparray[btop(off +
1520                                             dhp->dh_roff)]);
1521                                 } else {
1522                                         pfnum = hat_getpfnum(
1523                                             ((proc_t *)cp->procp)->p_as->a_hat,
1524                                             cp->cvaddr + off);
1525                                 }
1526                         break;
1527                         case UMEM_TRASH :
1528                                 pfnum = page_pptonum(trashpp);
1529                                 /*
1530                                  * We should set hat_flags to HAT_NOFAULT also
1531                                  * However, not all hat layers implement this
1532                                  */
1533                                 break;
1534                         case KMEM_PAGEABLE:
1535                         case KMEM_NON_PAGEABLE:
1536                                 pfnum = hat_getpfnum(kas.a_hat,
1537                                     dhp->dh_cvaddr + off);
1538                                 break;
1539                         default :
1540                                 pfnum = PFN_INVALID;
1541                                 break;
1542                         }
1543                 }
1544                 prot |= dhp->dh_hat_attr;
1545         }
1546         if (pfnum == PFN_INVALID) {
1547                 return (FC_MAKE_ERR(EFAULT));
1548         }
1549         /* prot should already be OR'ed in with hat_attributes if needed */
1550 
1551         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1552             "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1553             pfnum, pf_is_memory(pfnum), prot, hat_flags);
1554         DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1555             "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1556 
1557         if (pf_is_memory(pfnum) || (dhp != NULL)) {
1558                 /*
1559                  * It's not _really_ required here to pass sdp->hat_flags
1560                  * to hat_devload even though we do it.
1561                  * This is because hat figures it out DEVMEM mappings
1562                  * are non-consistent, anyway.
1563                  */
1564                 hat_devload(hat, addr, PAGESIZE, pfnum,
1565                     prot, hat_flags | sdp->hat_flags);
1566                 return (0);
1567         }
1568 
1569         /*
1570          * Fall through to the case where devmap is not used and need to call
1571          * up the device tree to set up the mapping
1572          */
1573 
1574         dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1575         ASSERT(dip);
1576 
1577         /*
1578          * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1579          * This is because this calls drivers which may not expect
1580          * prot to have any other values than PROT_ALL
1581          * The root nexus driver has a hack to peek into the segment
1582          * structure and then OR in sdp->hat_attr.
1583          * XX In case the bus_ops interfaces are ever revisited
1584          * we need to fix this. prot should include other hat attributes
1585          */
1586         if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1587             (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1588                 return (FC_MAKE_ERR(EFAULT));
1589         }
1590         return (0);
1591 }
1592 
1593 static faultcode_t
1594 segdev_fault(
1595         struct hat *hat,                /* the hat */
1596         struct seg *seg,                /* the seg_dev of interest */
1597         caddr_t addr,                   /* the address of the fault */
1598         size_t len,                     /* the length of the range */
1599         enum fault_type type,           /* type of fault */
1600         enum seg_rw rw)                 /* type of access at fault */
1601 {
1602         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1603         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1604         devmap_handle_t *dhp;
1605         struct devmap_softlock *slock = NULL;
1606         ulong_t slpage = 0;
1607         ulong_t off;
1608         caddr_t maddr = addr;
1609         int err;
1610         int err_is_faultcode = 0;
1611 
1612         TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1613             "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1614             (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1615         DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1616             "addr %p len %lx type %x\n",
1617             (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1618 
1619         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1620 
1621         /* Handle non-devmap case */
1622         if (dhp_head == NULL)
1623                 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1624 
1625         /* Find devmap handle */
1626         if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1627                 return (FC_NOMAP);
1628 
1629         /*
1630          * The seg_dev driver does not implement copy-on-write,
1631          * and always loads translations with maximal allowed permissions
1632          * but we got an fault trying to access the device.
1633          * Servicing the fault is not going to result in any better result
1634          * RFE: If we want devmap_access callbacks to be involved in F_PROT
1635          *      faults, then the code below is written for that
1636          *      Pending resolution of the following:
1637          *      - determine if the F_INVAL/F_SOFTLOCK syncing
1638          *      is needed for F_PROT also or not. The code below assumes it does
1639          *      - If driver sees F_PROT and calls devmap_load with same type,
1640          *      then segdev_faultpages will fail with FC_PROT anyway, need to
1641          *      change that so calls from devmap_load to segdev_faultpages for
1642          *      F_PROT type are retagged to F_INVAL.
1643          * RFE: Today we dont have drivers that use devmap and want to handle
1644          *      F_PROT calls. The code in segdev_fault* is written to allow
1645          *      this case but is not tested. A driver that needs this capability
1646          *      should be able to remove the short-circuit case; resolve the
1647          *      above issues and "should" work.
1648          */
1649         if (type == F_PROT) {
1650                 return (FC_PROT);
1651         }
1652 
1653         /*
1654          * Loop through dhp list calling devmap_access or segdev_faultpages for
1655          * each devmap handle.
1656          * drivers which implement devmap_access can interpose on faults and do
1657          * device-appropriate special actions before calling devmap_load.
1658          */
1659 
1660         /*
1661          * Unfortunately, this simple loop has turned out to expose a variety
1662          * of complex problems which results in the following convoluted code.
1663          *
1664          * First, a desire to handle a serialization of F_SOFTLOCK calls
1665          * to the driver within the framework.
1666          *      This results in a dh_softlock structure that is on a per device
1667          *      (or device instance) basis and serializes devmap_access calls.
1668          *      Ideally we would need to do this for underlying
1669          *      memory/device regions that are being faulted on
1670          *      but that is hard to identify and with REMAP, harder
1671          * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1672          *      to F_SOFTLOCK calls to the driver.
1673          * These serializations are to simplify the driver programmer model.
1674          * To support these two features, the code first goes through the
1675          *      devmap handles and counts the pages (slpage) that are covered
1676          *      by devmap_access callbacks.
1677          * This part ends with a devmap_softlock_enter call
1678          *      which allows only one F_SOFTLOCK active on a device instance,
1679          *      but multiple F_INVAL/F_PROTs can be active except when a
1680          *      F_SOFTLOCK is active
1681          *
1682          * Next, we dont short-circuit the fault code upfront to call
1683          *      segdev_softunlock for F_SOFTUNLOCK, because we must use
1684          *      the same length when we softlock and softunlock.
1685          *
1686          *      -Hat layers may not support softunlocking lengths less than the
1687          *      original length when there is large page support.
1688          *      -kpmem locking is dependent on keeping the lengths same.
1689          *      -if drivers handled F_SOFTLOCK, they probably also expect to
1690          *              see an F_SOFTUNLOCK of the same length
1691          *      Hence, if extending lengths during softlock,
1692          *      softunlock has to make the same adjustments and goes through
1693          *      the same loop calling segdev_faultpages/segdev_softunlock
1694          *      But some of the synchronization and error handling is different
1695          */
1696 
1697         if (type != F_SOFTUNLOCK) {
1698                 devmap_handle_t *dhpp = dhp;
1699                 size_t slen = len;
1700 
1701                 /*
1702                  * Calculate count of pages that are :
1703                  * a) within the (potentially extended) fault region
1704                  * b) AND covered by devmap handle with devmap_access
1705                  */
1706                 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1707                 while (slen != 0) {
1708                         size_t mlen;
1709 
1710                         /*
1711                          * Softlocking on a region that allows remap is
1712                          * unsupported due to unresolved locking issues
1713                          * XXX: unclear what these are?
1714                          *      One potential is that if there is a pending
1715                          *      softlock, then a remap should not be allowed
1716                          *      until the unlock is done. This is easily
1717                          *      fixed by returning error in devmap*remap on
1718                          *      checking the dh->dh_softlock->softlocked value
1719                          */
1720                         if ((type == F_SOFTLOCK) &&
1721                             (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1722                                 return (FC_NOSUPPORT);
1723                         }
1724 
1725                         mlen = MIN(slen, (dhpp->dh_len - off));
1726                         if (dhpp->dh_callbackops.devmap_access) {
1727                                 size_t llen;
1728                                 caddr_t laddr;
1729                                 /*
1730                                  * use extended length for large page mappings
1731                                  */
1732                                 HOLD_DHP_LOCK(dhpp);
1733                                 if ((sdp->pageprot == 0) &&
1734                                     (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1735                                         devmap_get_large_pgsize(dhpp,
1736                                             mlen, maddr, &llen, &laddr);
1737                                 } else {
1738                                         llen = mlen;
1739                                 }
1740                                 RELE_DHP_LOCK(dhpp);
1741 
1742                                 slpage += btopr(llen);
1743                                 slock = dhpp->dh_softlock;
1744                         }
1745                         maddr += mlen;
1746                         ASSERT(slen >= mlen);
1747                         slen -= mlen;
1748                         dhpp = dhpp->dh_next;
1749                         off = 0;
1750                 }
1751                 /*
1752                  * synchonize with other faulting threads and wait till safe
1753                  * devmap_softlock_enter might return due to signal in cv_wait
1754                  *
1755                  * devmap_softlock_enter has to be called outside of while loop
1756                  * to prevent a deadlock if len spans over multiple dhps.
1757                  * dh_softlock is based on device instance and if multiple dhps
1758                  * use the same device instance, the second dhp's LOCK call
1759                  * will hang waiting on the first to complete.
1760                  * devmap_setup verifies that slocks in a dhp_chain are same.
1761                  * RFE: this deadlock only hold true for F_SOFTLOCK. For
1762                  *      F_INVAL/F_PROT, since we now allow multiple in parallel,
1763                  *      we could have done the softlock_enter inside the loop
1764                  *      and supported multi-dhp mappings with dissimilar devices
1765                  */
1766                 if (err = devmap_softlock_enter(slock, slpage, type))
1767                         return (FC_MAKE_ERR(err));
1768         }
1769 
1770         /* reset 'maddr' to the start addr of the range of fault. */
1771         maddr = addr;
1772 
1773         /* calculate the offset corresponds to 'addr' in the first dhp. */
1774         off = (ulong_t)(addr - dhp->dh_uvaddr);
1775 
1776         /*
1777          * The fault length may span over multiple dhps.
1778          * Loop until the total length is satisfied.
1779          */
1780         while (len != 0) {
1781                 size_t llen;
1782                 size_t mlen;
1783                 caddr_t laddr;
1784 
1785                 /*
1786                  * mlen is the smaller of 'len' and the length
1787                  * from addr to the end of mapping defined by dhp.
1788                  */
1789                 mlen = MIN(len, (dhp->dh_len - off));
1790 
1791                 HOLD_DHP_LOCK(dhp);
1792                 /*
1793                  * Pass the extended length and address to devmap_access
1794                  * if large pagesize is used for loading address translations.
1795                  */
1796                 if ((sdp->pageprot == 0) &&
1797                     (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1798                         devmap_get_large_pgsize(dhp, mlen, maddr,
1799                             &llen, &laddr);
1800                         ASSERT(maddr == addr || laddr == maddr);
1801                 } else {
1802                         llen = mlen;
1803                         laddr = maddr;
1804                 }
1805 
1806                 if (dhp->dh_callbackops.devmap_access != NULL) {
1807                         offset_t aoff;
1808 
1809                         aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1810 
1811                         /*
1812                          * call driver's devmap_access entry point which will
1813                          * call devmap_load/contextmgmt to load the translations
1814                          *
1815                          * We drop the dhp_lock before calling access so
1816                          * drivers can call devmap_*_remap within access
1817                          */
1818                         RELE_DHP_LOCK(dhp);
1819 
1820                         err = (*dhp->dh_callbackops.devmap_access)(
1821                             dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1822                 } else {
1823                         /*
1824                          * If no devmap_access entry point, then load mappings
1825                          * hold dhp_lock across faultpages if REMAP
1826                          */
1827                         err = segdev_faultpages(hat, seg, laddr, llen,
1828                             type, rw, dhp);
1829                         err_is_faultcode = 1;
1830                         RELE_DHP_LOCK(dhp);
1831                 }
1832 
1833                 if (err) {
1834                         if ((type == F_SOFTLOCK) && (maddr > addr)) {
1835                                 /*
1836                                  * If not first dhp, use
1837                                  * segdev_fault(F_SOFTUNLOCK) for prior dhps
1838                                  * While this is recursion, it is incorrect to
1839                                  * call just segdev_softunlock
1840                                  * if we are using either large pages
1841                                  * or devmap_access. It will be more right
1842                                  * to go through the same loop as above
1843                                  * rather than call segdev_softunlock directly
1844                                  * It will use the right lenghths as well as
1845                                  * call into the driver devmap_access routines.
1846                                  */
1847                                 size_t done = (size_t)(maddr - addr);
1848                                 (void) segdev_fault(hat, seg, addr, done,
1849                                     F_SOFTUNLOCK, S_OTHER);
1850                                 /*
1851                                  * reduce slpage by number of pages
1852                                  * released by segdev_softunlock
1853                                  */
1854                                 ASSERT(slpage >= btopr(done));
1855                                 devmap_softlock_exit(slock,
1856                                     slpage - btopr(done), type);
1857                         } else {
1858                                 devmap_softlock_exit(slock, slpage, type);
1859                         }
1860 
1861 
1862                         /*
1863                          * Segdev_faultpages() already returns a faultcode,
1864                          * hence, result from segdev_faultpages() should be
1865                          * returned directly.
1866                          */
1867                         if (err_is_faultcode)
1868                                 return (err);
1869                         return (FC_MAKE_ERR(err));
1870                 }
1871 
1872                 maddr += mlen;
1873                 ASSERT(len >= mlen);
1874                 len -= mlen;
1875                 dhp = dhp->dh_next;
1876                 off = 0;
1877 
1878                 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1879         }
1880         /*
1881          * release the softlock count at end of fault
1882          * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1883          */
1884         if ((type == F_INVAL) || (type == F_PROT))
1885                 devmap_softlock_exit(slock, slpage, type);
1886         return (0);
1887 }
1888 
1889 /*
1890  * segdev_faultpages
1891  *
1892  * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1893  * This routine assumes that the callers makes sure that the fields
1894  * in dhp used below are not changed due to remap during this call.
1895  * Caller does HOLD_DHP_LOCK if neeed
1896  * This routine returns a faultcode_t as a return value for segdev_fault.
1897  */
1898 static faultcode_t
1899 segdev_faultpages(
1900         struct hat *hat,                /* the hat */
1901         struct seg *seg,                /* the seg_dev of interest */
1902         caddr_t addr,                   /* the address of the fault */
1903         size_t len,                     /* the length of the range */
1904         enum fault_type type,           /* type of fault */
1905         enum seg_rw rw,                 /* type of access at fault */
1906         devmap_handle_t *dhp)           /* devmap handle */
1907 {
1908         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1909         register caddr_t a;
1910         struct vpage *vpage;
1911         struct ddi_umem_cookie *kpmem_cookie = NULL;
1912         int err;
1913 
1914         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1915             "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1916             (void *)dhp, (void *)seg, (void *)addr, len);
1917         DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1918             "dhp %p seg %p addr %p len %lx\n",
1919             (void *)dhp, (void *)seg, (void *)addr, len));
1920 
1921         /*
1922          * The seg_dev driver does not implement copy-on-write,
1923          * and always loads translations with maximal allowed permissions
1924          * but we got an fault trying to access the device.
1925          * Servicing the fault is not going to result in any better result
1926          * XXX: If we want to allow devmap_access to handle F_PROT calls,
1927          * This code should be removed and let the normal fault handling
1928          * take care of finding the error
1929          */
1930         if (type == F_PROT) {
1931                 return (FC_PROT);
1932         }
1933 
1934         if (type == F_SOFTUNLOCK) {
1935                 segdev_softunlock(hat, seg, addr, len, rw);
1936                 return (0);
1937         }
1938 
1939         /*
1940          * For kernel pageable memory, fault/lock segkp pages
1941          * We hold this until the completion of this
1942          * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1943          */
1944         if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1945                 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1946                 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1947                         return (err);
1948         }
1949 
1950         /*
1951          * If we have the same protections for the entire segment,
1952          * insure that the access being attempted is legitimate.
1953          */
1954         rw_enter(&sdp->lock, RW_READER);
1955         if (sdp->pageprot == 0) {
1956                 uint_t protchk;
1957 
1958                 switch (rw) {
1959                 case S_READ:
1960                         protchk = PROT_READ;
1961                         break;
1962                 case S_WRITE:
1963                         protchk = PROT_WRITE;
1964                         break;
1965                 case S_EXEC:
1966                         protchk = PROT_EXEC;
1967                         break;
1968                 case S_OTHER:
1969                 default:
1970                         protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1971                         break;
1972                 }
1973 
1974                 if ((sdp->prot & protchk) == 0) {
1975                         rw_exit(&sdp->lock);
1976                         /* undo kpmem locking */
1977                         if (kpmem_cookie != NULL) {
1978                                 release_kpmem_lock(kpmem_cookie, btopr(len));
1979                         }
1980                         return (FC_PROT);       /* illegal access type */
1981                 }
1982         }
1983 
1984         /*
1985          * we do a single hat_devload for the range if
1986          *   - devmap framework (dhp is not NULL),
1987          *   - pageprot == 0, i.e., no per-page protection set and
1988          *   - is device pages, irrespective of whether we are using large pages
1989          */
1990         if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1991                 pfn_t pfnum;
1992                 uint_t hat_flags;
1993 
1994                 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1995                         rw_exit(&sdp->lock);
1996                         return (FC_NOMAP);
1997                 }
1998 
1999                 if (type == F_SOFTLOCK) {
2000                         mutex_enter(&freemem_lock);
2001                         sdp->softlockcnt += btopr(len);
2002                         mutex_exit(&freemem_lock);
2003                 }
2004 
2005                 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2006                 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2007                 ASSERT(!pf_is_memory(pfnum));
2008 
2009                 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2010                     hat_flags | sdp->hat_flags);
2011                 rw_exit(&sdp->lock);
2012                 return (0);
2013         }
2014 
2015         /* Handle cases where we have to loop through fault handling per-page */
2016 
2017         if (sdp->vpage == NULL)
2018                 vpage = NULL;
2019         else
2020                 vpage = &sdp->vpage[seg_page(seg, addr)];
2021 
2022         /* loop over the address range handling each fault */
2023         for (a = addr; a < addr + len; a += PAGESIZE) {
2024                 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2025                         break;
2026                 }
2027                 if (vpage != NULL)
2028                         vpage++;
2029         }
2030         rw_exit(&sdp->lock);
2031         if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2032                 size_t done = (size_t)(a - addr); /* pages fault successfully */
2033                 if (done > 0) {
2034                         /* use softunlock for those pages */
2035                         segdev_softunlock(hat, seg, addr, done, S_OTHER);
2036                 }
2037                 if (kpmem_cookie != NULL) {
2038                         /* release kpmem lock for rest of pages */
2039                         ASSERT(len >= done);
2040                         release_kpmem_lock(kpmem_cookie, btopr(len - done));
2041                 }
2042         } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2043                 /* for non-SOFTLOCK cases, release kpmem */
2044                 release_kpmem_lock(kpmem_cookie, btopr(len));
2045         }
2046         return (err);
2047 }
2048 
2049 /*
2050  * Asynchronous page fault.  We simply do nothing since this
2051  * entry point is not supposed to load up the translation.
2052  */
2053 /*ARGSUSED*/
2054 static faultcode_t
2055 segdev_faulta(struct seg *seg, caddr_t addr)
2056 {
2057         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2058             "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2059         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2060 
2061         return (0);
2062 }
2063 
2064 static int
2065 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2066 {
2067         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2068         register devmap_handle_t *dhp;
2069         register struct vpage *vp, *evp;
2070         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2071         ulong_t off;
2072         size_t mlen, sz;
2073 
2074         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2075             "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2076             (void *)seg, (void *)addr, len, prot);
2077         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2078 
2079         if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2080                 /*
2081                  * Fail the setprot if pages are SOFTLOCKed through this
2082                  * mapping.
2083                  * Softlockcnt is protected from change by the as read lock.
2084                  */
2085                 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2086                     "segdev_setprot:error softlockcnt=%lx", sz);
2087                 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2088                 return (EAGAIN);
2089         }
2090 
2091         if (dhp_head != NULL) {
2092                 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2093                         return (EINVAL);
2094 
2095                 /*
2096                  * check if violate maxprot.
2097                  */
2098                 off = (ulong_t)(addr - dhp->dh_uvaddr);
2099                 mlen  = len;
2100                 while (dhp) {
2101                         if ((dhp->dh_maxprot & prot) != prot)
2102                                 return (EACCES);        /* violated maxprot */
2103 
2104                         if (mlen > (dhp->dh_len - off)) {
2105                                 mlen -= dhp->dh_len - off;
2106                                 dhp = dhp->dh_next;
2107                                 off = 0;
2108                         } else
2109                                 break;
2110                 }
2111         } else {
2112                 if ((sdp->maxprot & prot) != prot)
2113                         return (EACCES);
2114         }
2115 
2116         rw_enter(&sdp->lock, RW_WRITER);
2117         if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2118                 if (sdp->prot == prot) {
2119                         rw_exit(&sdp->lock);
2120                         return (0);                     /* all done */
2121                 }
2122                 sdp->prot = (uchar_t)prot;
2123         } else {
2124                 sdp->pageprot = 1;
2125                 if (sdp->vpage == NULL) {
2126                         /*
2127                          * First time through setting per page permissions,
2128                          * initialize all the vpage structures to prot
2129                          */
2130                         sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2131                             KM_SLEEP);
2132                         evp = &sdp->vpage[seg_pages(seg)];
2133                         for (vp = sdp->vpage; vp < evp; vp++)
2134                                 VPP_SETPROT(vp, sdp->prot);
2135                 }
2136                 /*
2137                  * Now go change the needed vpages protections.
2138                  */
2139                 evp = &sdp->vpage[seg_page(seg, addr + len)];
2140                 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2141                         VPP_SETPROT(vp, prot);
2142         }
2143         rw_exit(&sdp->lock);
2144 
2145         if (dhp_head != NULL) {
2146                 devmap_handle_t *tdhp;
2147                 /*
2148                  * If large page size was used in hat_devload(),
2149                  * the same page size must be used in hat_unload().
2150                  */
2151                 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2152                 while (tdhp != NULL) {
2153                         if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2154                                 break;
2155                         }
2156                         tdhp = tdhp->dh_next;
2157                 }
2158                 if (tdhp) {
2159                         size_t slen = len;
2160                         size_t mlen;
2161                         size_t soff;
2162 
2163                         soff = (ulong_t)(addr - dhp->dh_uvaddr);
2164                         while (slen != 0) {
2165                                 mlen = MIN(slen, (dhp->dh_len - soff));
2166                                 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2167                                     dhp->dh_len, HAT_UNLOAD);
2168                                 dhp = dhp->dh_next;
2169                                 ASSERT(slen >= mlen);
2170                                 slen -= mlen;
2171                                 soff = 0;
2172                         }
2173                         return (0);
2174                 }
2175         }
2176 
2177         if ((prot & ~PROT_USER) == PROT_NONE) {
2178                 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2179         } else {
2180                 /*
2181                  * RFE: the segment should keep track of all attributes
2182                  * allowing us to remove the deprecated hat_chgprot
2183                  * and use hat_chgattr.
2184                  */
2185                 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2186         }
2187 
2188         return (0);
2189 }
2190 
2191 static int
2192 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2193 {
2194         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2195         struct vpage *vp, *evp;
2196 
2197         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2198             "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2199             (void *)seg, (void *)addr, len, prot);
2200         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2201 
2202         /*
2203          * If segment protection can be used, simply check against them
2204          */
2205         rw_enter(&sdp->lock, RW_READER);
2206         if (sdp->pageprot == 0) {
2207                 register int err;
2208 
2209                 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2210                 rw_exit(&sdp->lock);
2211                 return (err);
2212         }
2213 
2214         /*
2215          * Have to check down to the vpage level
2216          */
2217         evp = &sdp->vpage[seg_page(seg, addr + len)];
2218         for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2219                 if ((VPP_PROT(vp) & prot) != prot) {
2220                         rw_exit(&sdp->lock);
2221                         return (EACCES);
2222                 }
2223         }
2224         rw_exit(&sdp->lock);
2225         return (0);
2226 }
2227 
2228 static int
2229 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2230 {
2231         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2232         size_t pgno;
2233 
2234         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2235             "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2236             (void *)seg, (void *)addr, len, (void *)protv);
2237         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2238 
2239         pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2240         if (pgno != 0) {
2241                 rw_enter(&sdp->lock, RW_READER);
2242                 if (sdp->pageprot == 0) {
2243                         do {
2244                                 protv[--pgno] = sdp->prot;
2245                         } while (pgno != 0);
2246                 } else {
2247                         size_t pgoff = seg_page(seg, addr);
2248 
2249                         do {
2250                                 pgno--;
2251                                 protv[pgno] =
2252                                     VPP_PROT(&sdp->vpage[pgno + pgoff]);
2253                         } while (pgno != 0);
2254                 }
2255                 rw_exit(&sdp->lock);
2256         }
2257         return (0);
2258 }
2259 
2260 static u_offset_t
2261 segdev_getoffset(register struct seg *seg, caddr_t addr)
2262 {
2263         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2264 
2265         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2266             "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2267 
2268         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2269 
2270         return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2271 }
2272 
2273 /*ARGSUSED*/
2274 static int
2275 segdev_gettype(register struct seg *seg, caddr_t addr)
2276 {
2277         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2278 
2279         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2280             "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2281 
2282         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2283 
2284         return (sdp->type);
2285 }
2286 
2287 
2288 /*ARGSUSED*/
2289 static int
2290 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2291 {
2292         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2293 
2294         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2295             "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2296 
2297         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2298 
2299         /*
2300          * Note that this vp is the common_vp of the device, where the
2301          * pages are hung ..
2302          */
2303         *vpp = VTOCVP(sdp->vp);
2304 
2305         return (0);
2306 }
2307 
2308 static void
2309 segdev_badop(void)
2310 {
2311         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2312             "segdev_badop:start");
2313         panic("segdev_badop");
2314         /*NOTREACHED*/
2315 }
2316 
2317 /*
2318  * segdev pages are not in the cache, and thus can't really be controlled.
2319  * Hence, syncs are simply always successful.
2320  */
2321 /*ARGSUSED*/
2322 static int
2323 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2324 {
2325         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2326 
2327         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2328 
2329         return (0);
2330 }
2331 
2332 /*
2333  * segdev pages are always "in core".
2334  */
2335 /*ARGSUSED*/
2336 static size_t
2337 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2338 {
2339         size_t v = 0;
2340 
2341         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2342 
2343         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2344 
2345         for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2346             v += PAGESIZE)
2347                 *vec++ = 1;
2348         return (v);
2349 }
2350 
2351 /*
2352  * segdev pages are not in the cache, and thus can't really be controlled.
2353  * Hence, locks are simply always successful.
2354  */
2355 /*ARGSUSED*/
2356 static int
2357 segdev_lockop(struct seg *seg, caddr_t addr,
2358     size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2359 {
2360         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2361 
2362         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2363 
2364         return (0);
2365 }
2366 
2367 /*
2368  * segdev pages are not in the cache, and thus can't really be controlled.
2369  * Hence, advise is simply always successful.
2370  */
2371 /*ARGSUSED*/
2372 static int
2373 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2374 {
2375         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2376 
2377         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2378 
2379         return (0);
2380 }
2381 
2382 /*
2383  * segdev pages are not dumped, so we just return
2384  */
2385 /*ARGSUSED*/
2386 static void
2387 segdev_dump(struct seg *seg)
2388 {}
2389 
2390 /*
2391  * ddi_segmap_setup:    Used by drivers who wish specify mapping attributes
2392  *                      for a segment.  Called from a drivers segmap(9E)
2393  *                      routine.
2394  */
2395 /*ARGSUSED*/
2396 int
2397 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2398     off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2399     ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2400 {
2401         struct segdev_crargs dev_a;
2402         int (*mapfunc)(dev_t dev, off_t off, int prot);
2403         uint_t hat_attr;
2404         pfn_t pfn;
2405         int     error, i;
2406 
2407         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2408             "ddi_segmap_setup:start");
2409 
2410         if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2411                 return (ENODEV);
2412 
2413         /*
2414          * Character devices that support the d_mmap
2415          * interface can only be mmap'ed shared.
2416          */
2417         if ((flags & MAP_TYPE) != MAP_SHARED)
2418                 return (EINVAL);
2419 
2420         /*
2421          * Check that this region is indeed mappable on this platform.
2422          * Use the mapping function.
2423          */
2424         if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2425                 return (ENXIO);
2426 
2427         /*
2428          * Check to ensure that the entire range is
2429          * legal and we are not trying to map in
2430          * more than the device will let us.
2431          */
2432         for (i = 0; i < len; i += PAGESIZE) {
2433                 if (i == 0) {
2434                         /*
2435                          * Save the pfn at offset here. This pfn will be
2436                          * used later to get user address.
2437                          */
2438                         if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2439                             maxprot)) == PFN_INVALID)
2440                                 return (ENXIO);
2441                 } else {
2442                         if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2443                             PFN_INVALID)
2444                                 return (ENXIO);
2445                 }
2446         }
2447 
2448         as_rangelock(as);
2449         /* Pick an address w/o worrying about any vac alignment constraints. */
2450         error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2451         if (error != 0) {
2452                 as_rangeunlock(as);
2453                 return (error);
2454         }
2455 
2456         dev_a.mapfunc = mapfunc;
2457         dev_a.dev = dev;
2458         dev_a.offset = (offset_t)offset;
2459         dev_a.type = flags & MAP_TYPE;
2460         dev_a.prot = (uchar_t)prot;
2461         dev_a.maxprot = (uchar_t)maxprot;
2462         dev_a.hat_attr = hat_attr;
2463         dev_a.hat_flags = 0;
2464         dev_a.devmap_data = NULL;
2465 
2466         error = as_map(as, *addrp, len, segdev_create, &dev_a);
2467         as_rangeunlock(as);
2468         return (error);
2469 
2470 }
2471 
2472 /*ARGSUSED*/
2473 static int
2474 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2475     struct page ***ppp, enum lock_type type, enum seg_rw rw)
2476 {
2477         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2478             "segdev_pagelock:start");
2479         return (ENOTSUP);
2480 }
2481 
2482 /*ARGSUSED*/
2483 static int
2484 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2485     uint_t szc)
2486 {
2487         return (ENOTSUP);
2488 }
2489 
2490 /*
2491  * devmap_device: Used by devmap framework to establish mapping
2492  *                called by devmap_seup(9F) during map setup time.
2493  */
2494 /*ARGSUSED*/
2495 static int
2496 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2497     offset_t off, size_t len, uint_t flags)
2498 {
2499         devmap_handle_t *rdhp, *maxdhp;
2500         struct segdev_crargs dev_a;
2501         int     err;
2502         uint_t maxprot = PROT_ALL;
2503         offset_t offset = 0;
2504         pfn_t pfn;
2505         struct devmap_pmem_cookie *pcp;
2506 
2507         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2508             "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2509             (void *)dhp, (void *)addr, off, len);
2510 
2511         DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2512             (void *)dhp, (void *)addr, off, len));
2513 
2514         as_rangelock(as);
2515         if ((flags & MAP_FIXED) == 0) {
2516                 offset_t aligned_off;
2517 
2518                 rdhp = maxdhp = dhp;
2519                 while (rdhp != NULL) {
2520                         maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2521                             maxdhp : rdhp;
2522                         rdhp = rdhp->dh_next;
2523                         maxprot |= dhp->dh_maxprot;
2524                 }
2525                 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2526 
2527                 /*
2528                  * Use the dhp that has the
2529                  * largest len to get user address.
2530                  */
2531                 /*
2532                  * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2533                  * use 0 which is as good as any other.
2534                  */
2535                 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2536                         aligned_off = (offset_t)0;
2537                 } else if (dhp_is_devmem(maxdhp)) {
2538                         aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2539                 } else if (dhp_is_pmem(maxdhp)) {
2540                         pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2541                         pfn = page_pptonum(
2542                             pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2543                         aligned_off = (offset_t)ptob(pfn) - offset;
2544                 } else {
2545                         aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2546                             offset;
2547                 }
2548 
2549                 /*
2550                  * Pick an address aligned to dh_cookie.
2551                  * for kernel memory/user memory, cookie is cvaddr.
2552                  * for device memory, cookie is physical address.
2553                  */
2554                 map_addr(addr, len, aligned_off, 1, flags);
2555                 if (*addr == NULL) {
2556                         as_rangeunlock(as);
2557                         return (ENOMEM);
2558                 }
2559         } else {
2560                 /*
2561                  * User-specified address; blow away any previous mappings.
2562                  */
2563                 (void) as_unmap(as, *addr, len);
2564         }
2565 
2566         dev_a.mapfunc = NULL;
2567         dev_a.dev = dhp->dh_dev;
2568         dev_a.type = flags & MAP_TYPE;
2569         dev_a.offset = off;
2570         /*
2571          * sdp->maxprot has the least restrict protection of all dhps.
2572          */
2573         dev_a.maxprot = maxprot;
2574         dev_a.prot = dhp->dh_prot;
2575         /*
2576          * devmap uses dhp->dh_hat_attr for hat.
2577          */
2578         dev_a.hat_flags = 0;
2579         dev_a.hat_attr = 0;
2580         dev_a.devmap_data = (void *)dhp;
2581 
2582         err = as_map(as, *addr, len, segdev_create, &dev_a);
2583         as_rangeunlock(as);
2584         return (err);
2585 }
2586 
2587 int
2588 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2589     uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2590     size_t, uint_t, uint_t))
2591 {
2592         register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2593         struct devmap_ctx *devctx;
2594         int do_timeout = 0;
2595         int ret;
2596 
2597 #ifdef lint
2598         pvtp = pvtp;
2599 #endif
2600 
2601         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2602             "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2603             (void *)dhp, off, len);
2604         DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2605             (void *)dhp, off, len));
2606 
2607         if (ctxmgt == NULL)
2608                 return (FC_HWERR);
2609 
2610         devctx = dhp->dh_ctx;
2611 
2612         /*
2613          * If we are on an MP system with more than one cpu running
2614          * and if a thread on some CPU already has the context, wait
2615          * for it to finish if there is a hysteresis timeout.
2616          *
2617          * We call cv_wait() instead of cv_wait_sig() because
2618          * it does not matter much if it returned due to a signal
2619          * or due to a cv_signal() or cv_broadcast().  In either event
2620          * we need to complete the mapping otherwise the processes
2621          * will die with a SEGV.
2622          */
2623         if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2624                 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2625                     "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2626                     devctx, dhp);
2627                 do_timeout = 1;
2628                 mutex_enter(&devctx->lock);
2629                 while (devctx->oncpu)
2630                         cv_wait(&devctx->cv, &devctx->lock);
2631                 devctx->oncpu = 1;
2632                 mutex_exit(&devctx->lock);
2633         }
2634 
2635         /*
2636          * Call the contextmgt callback so that the driver can handle
2637          * the fault.
2638          */
2639         ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2640 
2641         /*
2642          * If devmap_access() returned -1, then there was a hardware
2643          * error so we need to convert the return value to something
2644          * that trap() will understand.  Otherwise, the return value
2645          * is already a fault code generated by devmap_unload()
2646          * or devmap_load().
2647          */
2648         if (ret) {
2649                 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2650                     "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2651                     ret, dhp, devctx);
2652                 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2653                     ret, (void *)dhp));
2654                 if (devctx->oncpu) {
2655                         mutex_enter(&devctx->lock);
2656                         devctx->oncpu = 0;
2657                         cv_signal(&devctx->cv);
2658                         mutex_exit(&devctx->lock);
2659                 }
2660                 return (FC_HWERR);
2661         }
2662 
2663         /*
2664          * Setup the timeout if we need to
2665          */
2666         if (do_timeout) {
2667                 mutex_enter(&devctx->lock);
2668                 if (dhp->dh_timeout_length > 0) {
2669                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2670                             "devmap_do_ctxmgt:timeout set");
2671                         devctx->timeout = timeout(devmap_ctxto,
2672                             devctx, dhp->dh_timeout_length);
2673                 } else {
2674                         /*
2675                          * We don't want to wait so set oncpu to
2676                          * 0 and wake up anyone waiting.
2677                          */
2678                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2679                             "devmap_do_ctxmgt:timeout not set");
2680                         devctx->oncpu = 0;
2681                         cv_signal(&devctx->cv);
2682                 }
2683                 mutex_exit(&devctx->lock);
2684         }
2685 
2686         return (DDI_SUCCESS);
2687 }
2688 
2689 /*
2690  *                                       end of mapping
2691  *                    poff   fault_offset         |
2692  *            base     |        |                 |
2693  *              |      |        |                 |
2694  *              V      V        V                 V
2695  *  +-----------+---------------+-------+---------+-------+
2696  *              ^               ^       ^         ^
2697  *              |<--- offset--->|<-len->|         |
2698  *              |<--- dh_len(size of mapping) --->|
2699  *                     |<--  pg -->|
2700  *                              -->|rlen|<--
2701  */
2702 static ulong_t
2703 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2704     ulong_t *opfn, ulong_t *pagesize)
2705 {
2706         register int level;
2707         ulong_t pg;
2708         ulong_t poff;
2709         ulong_t base;
2710         caddr_t uvaddr;
2711         long rlen;
2712 
2713         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2714             "devmap_roundup:start dhp=%p off=%lx len=%lx",
2715             (void *)dhp, offset, len);
2716         DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2717             (void *)dhp, offset, len));
2718 
2719         /*
2720          * get the max. pagesize that is aligned within the range
2721          * <dh_pfn, dh_pfn+offset>.
2722          *
2723          * The calculations below use physical address to ddetermine
2724          * the page size to use. The same calculations can use the
2725          * virtual address to determine the page size.
2726          */
2727         base = (ulong_t)ptob(dhp->dh_pfn);
2728         for (level = dhp->dh_mmulevel; level >= 0; level--) {
2729                 pg = page_get_pagesize(level);
2730                 poff = ((base + offset) & ~(pg - 1));
2731                 uvaddr = dhp->dh_uvaddr + (poff - base);
2732                 if ((poff >= base) &&
2733                     ((poff + pg) <= (base + dhp->dh_len)) &&
2734                     VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2735                         break;
2736         }
2737 
2738         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2739             "devmap_roundup: base=%lx poff=%lx dhp=%p",
2740             base, poff, dhp);
2741         DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2742             base, poff, dhp->dh_pfn));
2743 
2744         ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2745         ASSERT(level >= 0);
2746 
2747         *pagesize = pg;
2748         *opfn = dhp->dh_pfn + btop(poff - base);
2749 
2750         rlen = len + offset - (poff - base + pg);
2751 
2752         ASSERT(rlen < (long)len);
2753 
2754         TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2755             "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2756             (void *)dhp, level, rlen, pagesize, opfn);
2757         DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2758             "level %x rlen %lx psize %lx opfn %lx\n",
2759             (void *)dhp, level, rlen, *pagesize, *opfn));
2760 
2761         return ((ulong_t)((rlen > 0) ? rlen : 0));
2762 }
2763 
2764 /*
2765  * find the dhp that contains addr.
2766  */
2767 static devmap_handle_t *
2768 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2769 {
2770         devmap_handle_t *dhp;
2771 
2772         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2773             "devmap_find_handle:start");
2774 
2775         dhp = dhp_head;
2776         while (dhp) {
2777                 if (addr >= dhp->dh_uvaddr &&
2778                     addr < (dhp->dh_uvaddr + dhp->dh_len))
2779                         return (dhp);
2780                 dhp = dhp->dh_next;
2781         }
2782 
2783         return ((devmap_handle_t *)NULL);
2784 }
2785 
2786 /*
2787  * devmap_unload:
2788  *                      Marks a segdev segment or pages if offset->offset+len
2789  *                      is not the entire segment as intercept and unloads the
2790  *                      pages in the range offset -> offset+len.
2791  */
2792 int
2793 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2794 {
2795         register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2796         caddr_t addr;
2797         ulong_t size;
2798         ssize_t soff;
2799 
2800         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2801             "devmap_unload:start dhp=%p offset=%llx len=%lx",
2802             (void *)dhp, offset, len);
2803         DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2804             (void *)dhp, offset, len));
2805 
2806         soff = (ssize_t)(offset - dhp->dh_uoff);
2807         soff = round_down_p2(soff, PAGESIZE);
2808         if (soff < 0 || soff >= dhp->dh_len)
2809                 return (FC_MAKE_ERR(EINVAL));
2810 
2811         /*
2812          * Address and size must be page aligned.  Len is set to the
2813          * number of bytes in the number of pages that are required to
2814          * support len.  Offset is set to the byte offset of the first byte
2815          * of the page that contains offset.
2816          */
2817         len = round_up_p2(len, PAGESIZE);
2818 
2819         /*
2820          * If len is == 0, then calculate the size by getting
2821          * the number of bytes from offset to the end of the segment.
2822          */
2823         if (len == 0)
2824                 size = dhp->dh_len - soff;
2825         else {
2826                 size = len;
2827                 if ((soff + size) > dhp->dh_len)
2828                         return (FC_MAKE_ERR(EINVAL));
2829         }
2830 
2831         /*
2832          * The address is offset bytes from the base address of
2833          * the dhp.
2834          */
2835         addr = (caddr_t)(soff + dhp->dh_uvaddr);
2836 
2837         /*
2838          * If large page size was used in hat_devload(),
2839          * the same page size must be used in hat_unload().
2840          */
2841         if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2842                 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2843                     dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2844         } else {
2845                 hat_unload(dhp->dh_seg->s_as->a_hat,  addr, size,
2846                     HAT_UNLOAD|HAT_UNLOAD_OTHER);
2847         }
2848 
2849         return (0);
2850 }
2851 
2852 /*
2853  * calculates the optimal page size that will be used for hat_devload().
2854  */
2855 static void
2856 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2857     size_t *llen, caddr_t *laddr)
2858 {
2859         ulong_t off;
2860         ulong_t pfn;
2861         ulong_t pgsize;
2862         uint_t first = 1;
2863 
2864         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2865             "devmap_get_large_pgsize:start");
2866 
2867         /*
2868          * RFE - Code only supports large page mappings for devmem
2869          * This code could be changed in future if we want to support
2870          * large page mappings for kernel exported memory.
2871          */
2872         ASSERT(dhp_is_devmem(dhp));
2873         ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2874 
2875         *llen = 0;
2876         off = (ulong_t)(addr - dhp->dh_uvaddr);
2877         while ((long)len > 0) {
2878                 /*
2879                  * get the optimal pfn to minimize address translations.
2880                  * devmap_roundup() returns residue bytes for next round
2881                  * calculations.
2882                  */
2883                 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2884 
2885                 if (first) {
2886                         *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2887                         first = 0;
2888                 }
2889 
2890                 *llen += pgsize;
2891                 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2892         }
2893         /* Large page mapping len/addr cover more range than original fault */
2894         ASSERT(*llen >= len && *laddr <= addr);
2895         ASSERT((*laddr + *llen) >= (addr + len));
2896 }
2897 
2898 /*
2899  * Initialize the devmap_softlock structure.
2900  */
2901 static struct devmap_softlock *
2902 devmap_softlock_init(dev_t dev, ulong_t id)
2903 {
2904         struct devmap_softlock *slock;
2905         struct devmap_softlock *tmp;
2906 
2907         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2908             "devmap_softlock_init:start");
2909 
2910         tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2911         mutex_enter(&devmap_slock);
2912 
2913         for (slock = devmap_slist; slock != NULL; slock = slock->next)
2914                 if ((slock->dev == dev) && (slock->id == id))
2915                         break;
2916 
2917         if (slock == NULL) {
2918                 slock = tmp;
2919                 slock->dev = dev;
2920                 slock->id = id;
2921                 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2922                 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2923                 slock->next = devmap_slist;
2924                 devmap_slist = slock;
2925         } else
2926                 kmem_free(tmp, sizeof (struct devmap_softlock));
2927 
2928         mutex_enter(&slock->lock);
2929         slock->refcnt++;
2930         mutex_exit(&slock->lock);
2931         mutex_exit(&devmap_slock);
2932 
2933         return (slock);
2934 }
2935 
2936 /*
2937  * Wake up processes that sleep on softlocked.
2938  * Free dh_softlock if refcnt is 0.
2939  */
2940 static void
2941 devmap_softlock_rele(devmap_handle_t *dhp)
2942 {
2943         struct devmap_softlock *slock = dhp->dh_softlock;
2944         struct devmap_softlock *tmp;
2945         struct devmap_softlock *parent;
2946 
2947         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2948             "devmap_softlock_rele:start");
2949 
2950         mutex_enter(&devmap_slock);
2951         mutex_enter(&slock->lock);
2952 
2953         ASSERT(slock->refcnt > 0);
2954 
2955         slock->refcnt--;
2956 
2957         /*
2958          * If no one is using the device, free up the slock data.
2959          */
2960         if (slock->refcnt == 0) {
2961                 slock->softlocked = 0;
2962                 cv_signal(&slock->cv);
2963 
2964                 if (devmap_slist == slock)
2965                         devmap_slist = slock->next;
2966                 else {
2967                         parent = devmap_slist;
2968                         for (tmp = devmap_slist->next; tmp != NULL;
2969                             tmp = tmp->next) {
2970                                 if (tmp == slock) {
2971                                         parent->next = tmp->next;
2972                                         break;
2973                                 }
2974                                 parent = tmp;
2975                         }
2976                 }
2977                 mutex_exit(&slock->lock);
2978                 mutex_destroy(&slock->lock);
2979                 cv_destroy(&slock->cv);
2980                 kmem_free(slock, sizeof (struct devmap_softlock));
2981         } else
2982                 mutex_exit(&slock->lock);
2983 
2984         mutex_exit(&devmap_slock);
2985 }
2986 
2987 /*
2988  * Wake up processes that sleep on dh_ctx->locked.
2989  * Free dh_ctx if refcnt is 0.
2990  */
2991 static void
2992 devmap_ctx_rele(devmap_handle_t *dhp)
2993 {
2994         struct devmap_ctx *devctx = dhp->dh_ctx;
2995         struct devmap_ctx *tmp;
2996         struct devmap_ctx *parent;
2997         timeout_id_t tid;
2998 
2999         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3000             "devmap_ctx_rele:start");
3001 
3002         mutex_enter(&devmapctx_lock);
3003         mutex_enter(&devctx->lock);
3004 
3005         ASSERT(devctx->refcnt > 0);
3006 
3007         devctx->refcnt--;
3008 
3009         /*
3010          * If no one is using the device, free up the devctx data.
3011          */
3012         if (devctx->refcnt == 0) {
3013                 /*
3014                  * Untimeout any threads using this mapping as they are about
3015                  * to go away.
3016                  */
3017                 if (devctx->timeout != 0) {
3018                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3019                             "devmap_ctx_rele:untimeout ctx->timeout");
3020 
3021                         tid = devctx->timeout;
3022                         mutex_exit(&devctx->lock);
3023                         (void) untimeout(tid);
3024                         mutex_enter(&devctx->lock);
3025                 }
3026 
3027                 devctx->oncpu = 0;
3028                 cv_signal(&devctx->cv);
3029 
3030                 if (devmapctx_list == devctx)
3031                         devmapctx_list = devctx->next;
3032                 else {
3033                         parent = devmapctx_list;
3034                         for (tmp = devmapctx_list->next; tmp != NULL;
3035                             tmp = tmp->next) {
3036                                 if (tmp == devctx) {
3037                                         parent->next = tmp->next;
3038                                         break;
3039                                 }
3040                                 parent = tmp;
3041                         }
3042                 }
3043                 mutex_exit(&devctx->lock);
3044                 mutex_destroy(&devctx->lock);
3045                 cv_destroy(&devctx->cv);
3046                 kmem_free(devctx, sizeof (struct devmap_ctx));
3047         } else
3048                 mutex_exit(&devctx->lock);
3049 
3050         mutex_exit(&devmapctx_lock);
3051 }
3052 
3053 /*
3054  * devmap_load:
3055  *                      Marks a segdev segment or pages if offset->offset+len
3056  *                      is not the entire segment as nointercept and faults in
3057  *                      the pages in the range offset -> offset+len.
3058  */
3059 int
3060 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3061     uint_t rw)
3062 {
3063         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3064         struct as *asp = dhp->dh_seg->s_as;
3065         caddr_t addr;
3066         ulong_t size;
3067         ssize_t soff;   /* offset from the beginning of the segment */
3068         int rc;
3069 
3070         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3071             "devmap_load:start dhp=%p offset=%llx len=%lx",
3072             (void *)dhp, offset, len);
3073 
3074         DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3075             (void *)dhp, offset, len));
3076 
3077         /*
3078          *      Hat layer only supports devload to process' context for which
3079          *      the as lock is held. Verify here and return error if drivers
3080          *      inadvertently call devmap_load on a wrong devmap handle.
3081          */
3082         if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3083                 return (FC_MAKE_ERR(EINVAL));
3084 
3085         soff = (ssize_t)(offset - dhp->dh_uoff);
3086         soff = round_down_p2(soff, PAGESIZE);
3087         if (soff < 0 || soff >= dhp->dh_len)
3088                 return (FC_MAKE_ERR(EINVAL));
3089 
3090         /*
3091          * Address and size must be page aligned.  Len is set to the
3092          * number of bytes in the number of pages that are required to
3093          * support len.  Offset is set to the byte offset of the first byte
3094          * of the page that contains offset.
3095          */
3096         len = round_up_p2(len, PAGESIZE);
3097 
3098         /*
3099          * If len == 0, then calculate the size by getting
3100          * the number of bytes from offset to the end of the segment.
3101          */
3102         if (len == 0)
3103                 size = dhp->dh_len - soff;
3104         else {
3105                 size = len;
3106                 if ((soff + size) > dhp->dh_len)
3107                         return (FC_MAKE_ERR(EINVAL));
3108         }
3109 
3110         /*
3111          * The address is offset bytes from the base address of
3112          * the segment.
3113          */
3114         addr = (caddr_t)(soff + dhp->dh_uvaddr);
3115 
3116         HOLD_DHP_LOCK(dhp);
3117         rc = segdev_faultpages(asp->a_hat,
3118             dhp->dh_seg, addr, size, type, rw, dhp);
3119         RELE_DHP_LOCK(dhp);
3120         return (rc);
3121 }
3122 
3123 int
3124 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3125     size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3126 {
3127         register devmap_handle_t *dhp;
3128         int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3129             size_t *, uint_t);
3130         int (*mmap)(dev_t, off_t, int);
3131         struct devmap_callback_ctl *callbackops;
3132         devmap_handle_t *dhp_head = NULL;
3133         devmap_handle_t *dhp_prev = NULL;
3134         devmap_handle_t *dhp_curr;
3135         caddr_t addr;
3136         int map_flag;
3137         int ret;
3138         ulong_t total_len;
3139         size_t map_len;
3140         size_t resid_len = len;
3141         offset_t map_off = off;
3142         struct devmap_softlock *slock = NULL;
3143 
3144 #ifdef lint
3145         cred = cred;
3146 #endif
3147 
3148         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3149             "devmap_setup:start off=%llx len=%lx", off, len);
3150         DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3151             off, len));
3152 
3153         devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3154         mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3155 
3156         /*
3157          * driver must provide devmap(9E) entry point in cb_ops to use the
3158          * devmap framework.
3159          */
3160         if (devmap == NULL || devmap == nulldev || devmap == nodev)
3161                 return (EINVAL);
3162 
3163         /*
3164          * To protect from an inadvertent entry because the devmap entry point
3165          * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3166          * mmap is NULL.
3167          */
3168         map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3169         if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3170                 return (EINVAL);
3171 
3172         /*
3173          * devmap allows mmap(2) to map multiple registers.
3174          * one devmap_handle is created for each register mapped.
3175          */
3176         for (total_len = 0; total_len < len; total_len += map_len) {
3177                 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3178 
3179                 if (dhp_prev != NULL)
3180                         dhp_prev->dh_next = dhp;
3181                 else
3182                         dhp_head = dhp;
3183                 dhp_prev = dhp;
3184 
3185                 dhp->dh_prot = prot;
3186                 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3187                 dhp->dh_dev = dev;
3188                 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3189                 dhp->dh_uoff = map_off;
3190 
3191                 /*
3192                  * Get mapping specific info from
3193                  * the driver, such as rnumber, roff, len, callbackops,
3194                  * accattrp and, if the mapping is for kernel memory,
3195                  * ddi_umem_cookie.
3196                  */
3197                 if ((ret = cdev_devmap(dev, dhp, map_off,
3198                     resid_len, &map_len, get_udatamodel())) != 0) {
3199                         free_devmap_handle(dhp_head);
3200                         return (ENXIO);
3201                 }
3202 
3203                 if (map_len & PAGEOFFSET) {
3204                         free_devmap_handle(dhp_head);
3205                         return (EINVAL);
3206                 }
3207 
3208                 callbackops = &dhp->dh_callbackops;
3209 
3210                 if ((callbackops->devmap_access == NULL) ||
3211                     (callbackops->devmap_access == nulldev) ||
3212                     (callbackops->devmap_access == nodev)) {
3213                         /*
3214                          * Normally devmap does not support MAP_PRIVATE unless
3215                          * the drivers provide a valid devmap_access routine.
3216                          */
3217                         if ((flags & MAP_PRIVATE) != 0) {
3218                                 free_devmap_handle(dhp_head);
3219                                 return (EINVAL);
3220                         }
3221                 } else {
3222                         /*
3223                          * Initialize dhp_softlock and dh_ctx if the drivers
3224                          * provide devmap_access.
3225                          */
3226                         dhp->dh_softlock = devmap_softlock_init(dev,
3227                             (ulong_t)callbackops->devmap_access);
3228                         dhp->dh_ctx = devmap_ctxinit(dev,
3229                             (ulong_t)callbackops->devmap_access);
3230 
3231                         /*
3232                          * segdev_fault can only work when all
3233                          * dh_softlock in a multi-dhp mapping
3234                          * are same. see comments in segdev_fault
3235                          * This code keeps track of the first
3236                          * dh_softlock allocated in slock and
3237                          * compares all later allocations and if
3238                          * not similar, returns an error.
3239                          */
3240                         if (slock == NULL)
3241                                 slock = dhp->dh_softlock;
3242                         if (slock != dhp->dh_softlock) {
3243                                 free_devmap_handle(dhp_head);
3244                                 return (ENOTSUP);
3245                         }
3246                 }
3247 
3248                 map_off += map_len;
3249                 resid_len -= map_len;
3250         }
3251 
3252         /*
3253          * get the user virtual address and establish the mapping between
3254          * uvaddr and device physical address.
3255          */
3256         if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3257             != 0) {
3258                 /*
3259                  * free devmap handles if error during the mapping.
3260                  */
3261                 free_devmap_handle(dhp_head);
3262 
3263                 return (ret);
3264         }
3265 
3266         /*
3267          * call the driver's devmap_map callback to do more after the mapping,
3268          * such as to allocate driver private data for context management.
3269          */
3270         dhp = dhp_head;
3271         map_off = off;
3272         addr = *addrp;
3273         while (dhp != NULL) {
3274                 callbackops = &dhp->dh_callbackops;
3275                 dhp->dh_uvaddr = addr;
3276                 dhp_curr = dhp;
3277                 if (callbackops->devmap_map != NULL) {
3278                         ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3279                             dev, flags, map_off,
3280                             dhp->dh_len, &dhp->dh_pvtp);
3281                         if (ret != 0) {
3282                                 struct segdev_data *sdp;
3283 
3284                                 /*
3285                                  * call driver's devmap_unmap entry point
3286                                  * to free driver resources.
3287                                  */
3288                                 dhp = dhp_head;
3289                                 map_off = off;
3290                                 while (dhp != dhp_curr) {
3291                                         callbackops = &dhp->dh_callbackops;
3292                                         if (callbackops->devmap_unmap != NULL) {
3293                                                 (*callbackops->devmap_unmap)(
3294                                                     dhp, dhp->dh_pvtp,
3295                                                     map_off, dhp->dh_len,
3296                                                     NULL, NULL, NULL, NULL);
3297                                         }
3298                                         map_off += dhp->dh_len;
3299                                         dhp = dhp->dh_next;
3300                                 }
3301                                 sdp = dhp_head->dh_seg->s_data;
3302                                 sdp->devmap_data = NULL;
3303                                 free_devmap_handle(dhp_head);
3304                                 return (ENXIO);
3305                         }
3306                 }
3307                 map_off += dhp->dh_len;
3308                 addr += dhp->dh_len;
3309                 dhp = dhp->dh_next;
3310         }
3311 
3312         return (0);
3313 }
3314 
3315 int
3316 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3317     off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3318 {
3319         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3320             "devmap_segmap:start");
3321         return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3322             (size_t)len, prot, maxprot, flags, cred));
3323 }
3324 
3325 /*
3326  * Called from devmap_devmem_setup/remap to see if can use large pages for
3327  * this device mapping.
3328  * Also calculate the max. page size for this mapping.
3329  * this page size will be used in fault routine for
3330  * optimal page size calculations.
3331  */
3332 static void
3333 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3334 {
3335         ASSERT(dhp_is_devmem(dhp));
3336         dhp->dh_mmulevel = 0;
3337 
3338         /*
3339          * use large page size only if:
3340          *  1. device memory.
3341          *  2. mmu supports multiple page sizes,
3342          *  3. Driver did not disallow it
3343          *  4. dhp length is at least as big as the large pagesize
3344          *  5. the uvaddr and pfn are large pagesize aligned
3345          */
3346         if (page_num_pagesizes() > 1 &&
3347             !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3348                 ulong_t base;
3349                 int level;
3350 
3351                 base = (ulong_t)ptob(dhp->dh_pfn);
3352                 for (level = 1; level < page_num_pagesizes(); level++) {
3353                         size_t pgsize = page_get_pagesize(level);
3354                         if ((dhp->dh_len < pgsize) ||
3355                             (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3356                             base, pgsize))) {
3357                                 break;
3358                         }
3359                 }
3360                 dhp->dh_mmulevel = level - 1;
3361         }
3362         if (dhp->dh_mmulevel > 0) {
3363                 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3364         } else {
3365                 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3366         }
3367 }
3368 
3369 /*
3370  * Called by driver devmap routine to pass device specific info to
3371  * the framework.    used for device memory mapping only.
3372  */
3373 int
3374 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3375     struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3376     size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3377 {
3378         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3379         ddi_acc_handle_t handle;
3380         ddi_map_req_t mr;
3381         ddi_acc_hdl_t *hp;
3382         int err;
3383 
3384         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3385             "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3386             (void *)dhp, roff, rnumber, (uint_t)len);
3387         DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3388             "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3389 
3390         /*
3391          * First to check if this function has been called for this dhp.
3392          */
3393         if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3394                 return (DDI_FAILURE);
3395 
3396         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3397                 return (DDI_FAILURE);
3398 
3399         if (flags & DEVMAP_MAPPING_INVALID) {
3400                 /*
3401                  * Don't go up the tree to get pfn if the driver specifies
3402                  * DEVMAP_MAPPING_INVALID in flags.
3403                  *
3404                  * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3405                  * remap permission.
3406                  */
3407                 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3408                         return (DDI_FAILURE);
3409                 }
3410                 dhp->dh_pfn = PFN_INVALID;
3411         } else {
3412                 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3413                 if (handle == NULL)
3414                         return (DDI_FAILURE);
3415 
3416                 hp = impl_acc_hdl_get(handle);
3417                 hp->ah_vers = VERS_ACCHDL;
3418                 hp->ah_dip = dip;
3419                 hp->ah_rnumber = rnumber;
3420                 hp->ah_offset = roff;
3421                 hp->ah_len = len;
3422                 if (accattrp != NULL)
3423                         hp->ah_acc = *accattrp;
3424 
3425                 mr.map_op = DDI_MO_MAP_LOCKED;
3426                 mr.map_type = DDI_MT_RNUMBER;
3427                 mr.map_obj.rnumber = rnumber;
3428                 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3429                 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3430                 mr.map_handlep = hp;
3431                 mr.map_vers = DDI_MAP_VERSION;
3432 
3433                 /*
3434                  * up the device tree to get pfn.
3435                  * The rootnex_map_regspec() routine in nexus drivers has been
3436                  * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3437                  */
3438                 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3439                 dhp->dh_hat_attr = hp->ah_hat_flags;
3440                 impl_acc_hdl_free(handle);
3441 
3442                 if (err)
3443                         return (DDI_FAILURE);
3444         }
3445         /* Should not be using devmem setup for memory pages */
3446         ASSERT(!pf_is_memory(dhp->dh_pfn));
3447 
3448         /* Only some of the flags bits are settable by the driver */
3449         dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3450         dhp->dh_len = ptob(btopr(len));
3451 
3452         dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3453         dhp->dh_roff = ptob(btop(roff));
3454 
3455         /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3456         devmap_devmem_large_page_setup(dhp);
3457         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3458         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3459 
3460 
3461         if (callbackops != NULL) {
3462                 bcopy(callbackops, &dhp->dh_callbackops,
3463                     sizeof (struct devmap_callback_ctl));
3464         }
3465 
3466         /*
3467          * Initialize dh_lock if we want to do remap.
3468          */
3469         if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3470                 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3471                 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3472         }
3473 
3474         dhp->dh_flags |= DEVMAP_SETUP_DONE;
3475 
3476         return (DDI_SUCCESS);
3477 }
3478 
3479 int
3480 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3481     uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3482     uint_t flags, ddi_device_acc_attr_t *accattrp)
3483 {
3484         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3485         ddi_acc_handle_t handle;
3486         ddi_map_req_t mr;
3487         ddi_acc_hdl_t *hp;
3488         pfn_t   pfn;
3489         uint_t  hat_flags;
3490         int     err;
3491 
3492         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3493             "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3494             (void *)dhp, roff, rnumber, (uint_t)len);
3495         DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3496             "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3497 
3498         /*
3499          * Return failure if setup has not been done or no remap permission
3500          * has been granted during the setup.
3501          */
3502         if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3503             (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3504                 return (DDI_FAILURE);
3505 
3506         /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3507         if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3508                 return (DDI_FAILURE);
3509 
3510         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3511                 return (DDI_FAILURE);
3512 
3513         if (!(flags & DEVMAP_MAPPING_INVALID)) {
3514                 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3515                 if (handle == NULL)
3516                         return (DDI_FAILURE);
3517         }
3518 
3519         HOLD_DHP_LOCK(dhp);
3520 
3521         /*
3522          * Unload the old mapping, so next fault will setup the new mappings
3523          * Do this while holding the dhp lock so other faults dont reestablish
3524          * the mappings
3525          */
3526         hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3527             dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3528 
3529         if (flags & DEVMAP_MAPPING_INVALID) {
3530                 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3531                 dhp->dh_pfn = PFN_INVALID;
3532         } else {
3533                 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3534                 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3535                 hp = impl_acc_hdl_get(handle);
3536                 hp->ah_vers = VERS_ACCHDL;
3537                 hp->ah_dip = dip;
3538                 hp->ah_rnumber = rnumber;
3539                 hp->ah_offset = roff;
3540                 hp->ah_len = len;
3541                 if (accattrp != NULL)
3542                         hp->ah_acc = *accattrp;
3543 
3544                 mr.map_op = DDI_MO_MAP_LOCKED;
3545                 mr.map_type = DDI_MT_RNUMBER;
3546                 mr.map_obj.rnumber = rnumber;
3547                 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3548                 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3549                 mr.map_handlep = hp;
3550                 mr.map_vers = DDI_MAP_VERSION;
3551 
3552                 /*
3553                  * up the device tree to get pfn.
3554                  * The rootnex_map_regspec() routine in nexus drivers has been
3555                  * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3556                  */
3557                 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3558                 hat_flags = hp->ah_hat_flags;
3559                 impl_acc_hdl_free(handle);
3560                 if (err) {
3561                         RELE_DHP_LOCK(dhp);
3562                         return (DDI_FAILURE);
3563                 }
3564                 /*
3565                  * Store result of ddi_map first in local variables, as we do
3566                  * not want to overwrite the existing dhp with wrong data.
3567                  */
3568                 dhp->dh_pfn = pfn;
3569                 dhp->dh_hat_attr = hat_flags;
3570         }
3571 
3572         /* clear the large page size flag */
3573         dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3574 
3575         dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3576         dhp->dh_roff = ptob(btop(roff));
3577 
3578         /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3579         devmap_devmem_large_page_setup(dhp);
3580         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3581         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3582 
3583         RELE_DHP_LOCK(dhp);
3584         return (DDI_SUCCESS);
3585 }
3586 
3587 /*
3588  * called by driver devmap routine to pass kernel virtual address  mapping
3589  * info to the framework.    used only for kernel memory
3590  * allocated from ddi_umem_alloc().
3591  */
3592 int
3593 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3594     struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3595     offset_t off, size_t len, uint_t maxprot, uint_t flags,
3596     ddi_device_acc_attr_t *accattrp)
3597 {
3598         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3599         struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3600 
3601 #ifdef lint
3602         dip = dip;
3603 #endif
3604 
3605         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3606             "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3607             (void *)dhp, off, cookie, len);
3608         DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3609             "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3610 
3611         if (cookie == NULL)
3612                 return (DDI_FAILURE);
3613 
3614         /* For UMEM_TRASH, this restriction is not needed */
3615         if ((off + len) > cp->size)
3616                 return (DDI_FAILURE);
3617 
3618         /* check if the cache attributes are supported */
3619         if (i_ddi_check_cache_attr(flags) == B_FALSE)
3620                 return (DDI_FAILURE);
3621 
3622         /*
3623          * First to check if this function has been called for this dhp.
3624          */
3625         if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3626                 return (DDI_FAILURE);
3627 
3628         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3629                 return (DDI_FAILURE);
3630 
3631         if (flags & DEVMAP_MAPPING_INVALID) {
3632                 /*
3633                  * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3634                  * remap permission.
3635                  */
3636                 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3637                         return (DDI_FAILURE);
3638                 }
3639         } else {
3640                 dhp->dh_cookie = cookie;
3641                 dhp->dh_roff = ptob(btop(off));
3642                 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3643                 /* set HAT cache attributes */
3644                 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3645                 /* set HAT endianess attributes */
3646                 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3647         }
3648 
3649         /*
3650          * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3651          * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3652          * create consistent mappings but our intention was to create
3653          * non-consistent mappings.
3654          *
3655          * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3656          * mappings.
3657          *
3658          * kernel exported memory: hat figures it out it's memory and always
3659          * creates consistent mappings.
3660          *
3661          * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3662          *
3663          * /dev/kmem: consistent mappings are created unless they are
3664          * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3665          * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3666          * mappings of /dev/kmem. See common/io/mem.c
3667          */
3668 
3669         /* Only some of the flags bits are settable by the driver */
3670         dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3671 
3672         dhp->dh_len = ptob(btopr(len));
3673         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3674         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3675 
3676         if (callbackops != NULL) {
3677                 bcopy(callbackops, &dhp->dh_callbackops,
3678                     sizeof (struct devmap_callback_ctl));
3679         }
3680         /*
3681          * Initialize dh_lock if we want to do remap.
3682          */
3683         if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3684                 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3685                 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3686         }
3687 
3688         dhp->dh_flags |= DEVMAP_SETUP_DONE;
3689 
3690         return (DDI_SUCCESS);
3691 }
3692 
3693 int
3694 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3695     ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3696     uint_t flags, ddi_device_acc_attr_t *accattrp)
3697 {
3698         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3699         struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3700 
3701         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3702             "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3703             (void *)dhp, off, cookie, len);
3704         DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3705             "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3706 
3707 #ifdef lint
3708         dip = dip;
3709         accattrp = accattrp;
3710 #endif
3711         /*
3712          * Reture failure if setup has not been done or no remap permission
3713          * has been granted during the setup.
3714          */
3715         if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3716             (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3717                 return (DDI_FAILURE);
3718 
3719         /* No flags supported for remap yet */
3720         if (flags != 0)
3721                 return (DDI_FAILURE);
3722 
3723         /* check if the cache attributes are supported */
3724         if (i_ddi_check_cache_attr(flags) == B_FALSE)
3725                 return (DDI_FAILURE);
3726 
3727         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3728                 return (DDI_FAILURE);
3729 
3730         /* For UMEM_TRASH, this restriction is not needed */
3731         if ((off + len) > cp->size)
3732                 return (DDI_FAILURE);
3733 
3734         HOLD_DHP_LOCK(dhp);
3735         /*
3736          * Unload the old mapping, so next fault will setup the new mappings
3737          * Do this while holding the dhp lock so other faults dont reestablish
3738          * the mappings
3739          */
3740         hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3741             dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3742 
3743         dhp->dh_cookie = cookie;
3744         dhp->dh_roff = ptob(btop(off));
3745         dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3746         /* set HAT cache attributes */
3747         i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3748         /* set HAT endianess attributes */
3749         i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3750 
3751         /* clear the large page size flag */
3752         dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3753 
3754         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3755         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3756         RELE_DHP_LOCK(dhp);
3757         return (DDI_SUCCESS);
3758 }
3759 
3760 /*
3761  * to set timeout value for the driver's context management callback, e.g.
3762  * devmap_access().
3763  */
3764 void
3765 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3766 {
3767         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3768 
3769         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3770             "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3771             (void *)dhp, ticks);
3772         dhp->dh_timeout_length = ticks;
3773 }
3774 
3775 int
3776 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3777     size_t len, uint_t type, uint_t rw)
3778 {
3779 #ifdef lint
3780         pvtp = pvtp;
3781 #endif
3782 
3783         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3784             "devmap_default_access:start");
3785         return (devmap_load(dhp, off, len, type, rw));
3786 }
3787 
3788 /*
3789  * segkmem_alloc() wrapper to allocate memory which is both
3790  * non-relocatable (for DR) and sharelocked, since the rest
3791  * of this segment driver requires it.
3792  */
3793 static void *
3794 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3795 {
3796         ASSERT(vmp != NULL);
3797         ASSERT(kvseg.s_base != NULL);
3798         vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3799         return (segkmem_alloc(vmp, size, vmflag));
3800 }
3801 
3802 /*
3803  * This is where things are a bit incestuous with seg_kmem: unlike
3804  * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3805  * we need to do a bit of a dance around that to prevent duplication of
3806  * code until we decide to bite the bullet and implement a new kernel
3807  * segment for driver-allocated memory that is exported to user space.
3808  */
3809 static void
3810 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3811 {
3812         page_t *pp;
3813         caddr_t addr = inaddr;
3814         caddr_t eaddr;
3815         pgcnt_t npages = btopr(size);
3816 
3817         ASSERT(vmp != NULL);
3818         ASSERT(kvseg.s_base != NULL);
3819         ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3820 
3821         hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3822 
3823         for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3824                 /*
3825                  * Use page_find() instead of page_lookup() to find the page
3826                  * since we know that it is hashed and has a shared lock.
3827                  */
3828                 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3829 
3830                 if (pp == NULL)
3831                         panic("devmap_free_pages: page not found");
3832                 if (!page_tryupgrade(pp)) {
3833                         page_unlock(pp);
3834                         pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3835                             SE_EXCL);
3836                         if (pp == NULL)
3837                                 panic("devmap_free_pages: page already freed");
3838                 }
3839                 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3840                 pp->p_lckcnt = 0;
3841                 page_destroy(pp, 0);
3842         }
3843         page_unresv(npages);
3844 
3845         if (vmp != NULL)
3846                 vmem_free(vmp, inaddr, size);
3847 }
3848 
3849 /*
3850  * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3851  * allocating non-pageable kmem in response to a ddi_umem_alloc()
3852  * default request. For now we allocate our own pages and we keep
3853  * them long-term sharelocked, since: A) the fault routines expect the
3854  * memory to already be locked; B) pageable umem is already long-term
3855  * locked; C) it's a lot of work to make it otherwise, particularly
3856  * since the nexus layer expects the pages to never fault. An RFE is to
3857  * not keep the pages long-term locked, but instead to be able to
3858  * take faults on them and simply look them up in kvp in case we
3859  * fault on them. Even then, we must take care not to let pageout
3860  * steal them from us since the data must remain resident; if we
3861  * do this we must come up with some way to pin the pages to prevent
3862  * faults while a driver is doing DMA to/from them.
3863  */
3864 static void *
3865 devmap_umem_alloc_np(size_t size, size_t flags)
3866 {
3867         void *buf;
3868         int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3869 
3870         buf = vmem_alloc(umem_np_arena, size, vmflags);
3871         if (buf != NULL)
3872                 bzero(buf, size);
3873         return (buf);
3874 }
3875 
3876 static void
3877 devmap_umem_free_np(void *addr, size_t size)
3878 {
3879         vmem_free(umem_np_arena, addr, size);
3880 }
3881 
3882 /*
3883  * allocate page aligned kernel memory for exporting to user land.
3884  * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3885  * to find a user virtual address that is in same color as the address
3886  * allocated here.
3887  */
3888 void *
3889 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3890 {
3891         register size_t len = ptob(btopr(size));
3892         void *buf = NULL;
3893         struct ddi_umem_cookie *cp;
3894         int iflags = 0;
3895 
3896         *cookie = NULL;
3897 
3898         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3899             "devmap_umem_alloc:start");
3900         if (len == 0)
3901                 return ((void *)NULL);
3902 
3903         /*
3904          * allocate cookie
3905          */
3906         if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3907             flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3908                 ASSERT(flags & DDI_UMEM_NOSLEEP);
3909                 return ((void *)NULL);
3910         }
3911 
3912         if (flags & DDI_UMEM_PAGEABLE) {
3913                 /* Only one of the flags is allowed */
3914                 ASSERT(!(flags & DDI_UMEM_TRASH));
3915                 /* initialize resource with 0 */
3916                 iflags = KPD_ZERO;
3917 
3918                 /*
3919                  * to allocate unlocked pageable memory, use segkp_get() to
3920                  * create a segkp segment.  Since segkp can only service kas,
3921                  * other segment drivers such as segdev have to do
3922                  * as_fault(segkp, SOFTLOCK) in its fault routine,
3923                  */
3924                 if (flags & DDI_UMEM_NOSLEEP)
3925                         iflags |= KPD_NOWAIT;
3926 
3927                 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3928                         kmem_free(cp, sizeof (struct ddi_umem_cookie));
3929                         return ((void *)NULL);
3930                 }
3931                 cp->type = KMEM_PAGEABLE;
3932                 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3933                 cp->locked = 0;
3934         } else if (flags & DDI_UMEM_TRASH) {
3935                 /* Only one of the flags is allowed */
3936                 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3937                 cp->type = UMEM_TRASH;
3938                 buf = NULL;
3939         } else {
3940                 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3941                         kmem_free(cp, sizeof (struct ddi_umem_cookie));
3942                         return ((void *)NULL);
3943                 }
3944 
3945                 cp->type = KMEM_NON_PAGEABLE;
3946         }
3947 
3948         /*
3949          * need to save size here.  size will be used when
3950          * we do kmem_free.
3951          */
3952         cp->size = len;
3953         cp->cvaddr = (caddr_t)buf;
3954 
3955         *cookie =  (void *)cp;
3956         return (buf);
3957 }
3958 
3959 void
3960 ddi_umem_free(ddi_umem_cookie_t cookie)
3961 {
3962         struct ddi_umem_cookie *cp;
3963 
3964         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3965             "devmap_umem_free:start");
3966 
3967         /*
3968          * if cookie is NULL, no effects on the system
3969          */
3970         if (cookie == NULL)
3971                 return;
3972 
3973         cp = (struct ddi_umem_cookie *)cookie;
3974 
3975         switch (cp->type) {
3976         case KMEM_PAGEABLE :
3977                 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3978                 /*
3979                  * Check if there are still any pending faults on the cookie
3980                  * while the driver is deleting it,
3981                  * XXX - could change to an ASSERT but wont catch errant drivers
3982                  */
3983                 mutex_enter(&cp->lock);
3984                 if (cp->locked) {
3985                         mutex_exit(&cp->lock);
3986                         panic("ddi_umem_free for cookie with pending faults %p",
3987                             (void *)cp);
3988                         return;
3989                 }
3990 
3991                 segkp_release(segkp, cp->cvaddr);
3992 
3993                 /*
3994                  * release mutex associated with this cookie.
3995                  */
3996                 mutex_destroy(&cp->lock);
3997                 break;
3998         case KMEM_NON_PAGEABLE :
3999                 ASSERT(cp->cvaddr != NULL && cp->size != 0);
4000                 devmap_umem_free_np(cp->cvaddr, cp->size);
4001                 break;
4002         case UMEM_TRASH :
4003                 break;
4004         case UMEM_LOCKED :
4005                 /* Callers should use ddi_umem_unlock for this type */
4006                 ddi_umem_unlock(cookie);
4007                 /* Frees the cookie too */
4008                 return;
4009         default:
4010                 /* panic so we can diagnose the underlying cause */
4011                 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4012                     cp->type);
4013         }
4014 
4015         kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4016 }
4017 
4018 
4019 static int
4020 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4021 {
4022         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4023 
4024         /*
4025          * It looks as if it is always mapped shared
4026          */
4027         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4028             "segdev_getmemid:start");
4029         memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4030         memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4031         return (0);
4032 }
4033 
4034 /*ARGSUSED*/
4035 static lgrp_mem_policy_info_t *
4036 segdev_getpolicy(struct seg *seg, caddr_t addr)
4037 {
4038         return (NULL);
4039 }
4040 
4041 /*ARGSUSED*/
4042 static int
4043 segdev_capable(struct seg *seg, segcapability_t capability)
4044 {
4045         return (0);
4046 }
4047 
4048 /*
4049  * ddi_umem_alloc() non-pageable quantum cache max size.
4050  * This is just a SWAG.
4051  */
4052 #define DEVMAP_UMEM_QUANTUM     (8*PAGESIZE)
4053 
4054 /*
4055  * Initialize seg_dev from boot. This routine sets up the trash page
4056  * and creates the umem_np_arena used to back non-pageable memory
4057  * requests.
4058  */
4059 void
4060 segdev_init(void)
4061 {
4062         struct seg kseg;
4063 
4064         umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4065             devmap_alloc_pages, devmap_free_pages, heap_arena,
4066             DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4067 
4068         kseg.s_as = &kas;
4069         trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4070             PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4071         if (trashpp == NULL)
4072                 panic("segdev_init: failed to create trash page");
4073         pagezero(trashpp, 0, PAGESIZE);
4074         page_downgrade(trashpp);
4075 }
4076 
4077 /*
4078  * Invoke platform-dependent support routines so that /proc can have
4079  * the platform code deal with curious hardware.
4080  */
4081 int
4082 segdev_copyfrom(struct seg *seg,
4083     caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4084 {
4085         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4086         struct snode *sp = VTOS(VTOCVP(sdp->vp));
4087 
4088         return (e_ddi_copyfromdev(sp->s_dip,
4089             (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4090 }
4091 
4092 int
4093 segdev_copyto(struct seg *seg,
4094     caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4095 {
4096         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4097         struct snode *sp = VTOS(VTOCVP(sdp->vp));
4098 
4099         return (e_ddi_copytodev(sp->s_dip,
4100             (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4101 }