1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 /*
  31  * University Copyright- Copyright (c) 1982, 1986, 1988
  32  * The Regents of the University of California
  33  * All Rights Reserved
  34  *
  35  * University Acknowledgment- Portions of this document are derived from
  36  * software developed by the University of California, Berkeley, and its
  37  * contributors.
  38  */
  39 
  40 /*
  41  * VM - segment of a mapped device.
  42  *
  43  * This segment driver is used when mapping character special devices.
  44  */
  45 
  46 #include <sys/types.h>
  47 #include <sys/t_lock.h>
  48 #include <sys/sysmacros.h>
  49 #include <sys/vtrace.h>
  50 #include <sys/systm.h>
  51 #include <sys/vmsystm.h>
  52 #include <sys/mman.h>
  53 #include <sys/errno.h>
  54 #include <sys/kmem.h>
  55 #include <sys/cmn_err.h>
  56 #include <sys/vnode.h>
  57 #include <sys/proc.h>
  58 #include <sys/conf.h>
  59 #include <sys/debug.h>
  60 #include <sys/ddidevmap.h>
  61 #include <sys/ddi_implfuncs.h>
  62 #include <sys/lgrp.h>
  63 
  64 #include <vm/page.h>
  65 #include <vm/hat.h>
  66 #include <vm/as.h>
  67 #include <vm/seg.h>
  68 #include <vm/seg_dev.h>
  69 #include <vm/seg_kp.h>
  70 #include <vm/seg_kmem.h>
  71 #include <vm/vpage.h>
  72 
  73 #include <sys/sunddi.h>
  74 #include <sys/esunddi.h>
  75 #include <sys/fs/snode.h>
  76 
  77 
  78 #if DEBUG
  79 int segdev_debug;
  80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
  81 #else
  82 #define DEBUGF(level, args)
  83 #endif
  84 
  85 /* Default timeout for devmap context management */
  86 #define CTX_TIMEOUT_VALUE 0
  87 
  88 #define HOLD_DHP_LOCK(dhp)  if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
  89                         { mutex_enter(&dhp->dh_lock); }
  90 
  91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
  92                         { mutex_exit(&dhp->dh_lock); }
  93 
  94 #define round_down_p2(a, s)     ((a) & ~((s) - 1))
  95 #define round_up_p2(a, s)       (((a) + (s) - 1) & ~((s) - 1))
  96 
  97 /*
  98  * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
  99  * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
 100  */
 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize)            \
 102         (((uvaddr | paddr) & (pgsize - 1)) == 0)
 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize)     \
 104         (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
 105 
 106 #define vpgtob(n)       ((n) * sizeof (struct vpage))   /* For brevity */
 107 
 108 #define VTOCVP(vp)      (VTOS(vp)->s_commonvp)       /* we "know" it's an snode */
 109 
 110 static struct devmap_ctx *devmapctx_list = NULL;
 111 static struct devmap_softlock *devmap_slist = NULL;
 112 
 113 /*
 114  * mutex, vnode and page for the page of zeros we use for the trash mappings.
 115  * One trash page is allocated on the first ddi_umem_setup call that uses it
 116  * XXX Eventually, we may want to combine this with what segnf does when all
 117  * hat layers implement HAT_NOFAULT.
 118  *
 119  * The trash page is used when the backing store for a userland mapping is
 120  * removed but the application semantics do not take kindly to a SIGBUS.
 121  * In that scenario, the applications pages are mapped to some dummy page
 122  * which returns garbage on read and writes go into a common place.
 123  * (Perfect for NO_FAULT semantics)
 124  * The device driver is responsible to communicating to the app with some
 125  * other mechanism that such remapping has happened and the app should take
 126  * corrective action.
 127  * We can also use an anonymous memory page as there is no requirement to
 128  * keep the page locked, however this complicates the fault code. RFE.
 129  */
 130 static struct vnode trashvp;
 131 static struct page *trashpp;
 132 
 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
 134 static vmem_t *umem_np_arena;
 135 
 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
 137 #define DEVMAP_DEVMEM_COOKIE    ((ddi_umem_cookie_t)0x1)
 138 
 139 /*
 140  * Macros to check if type of devmap handle
 141  */
 142 #define cookie_is_devmem(c)     \
 143         ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
 144 
 145 #define cookie_is_pmem(c)       \
 146         ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
 147 
 148 #define cookie_is_kpmem(c)      (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
 149         ((c)->type == KMEM_PAGEABLE))
 150 
 151 #define dhp_is_devmem(dhp)      \
 152         (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 153 
 154 #define dhp_is_pmem(dhp)        \
 155         (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 156 
 157 #define dhp_is_kpmem(dhp)       \
 158         (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 159 
 160 /*
 161  * Private seg op routines.
 162  */
 163 static int      segdev_dup(struct seg *, struct seg *);
 164 static int      segdev_unmap(struct seg *, caddr_t, size_t);
 165 static void     segdev_free(struct seg *);
 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
 167                     enum fault_type, enum seg_rw);
 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
 169 static int      segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
 170 static int      segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
 171 static void     segdev_badop(void);
 172 static int      segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
 173 static size_t   segdev_incore(struct seg *, caddr_t, size_t, char *);
 174 static int      segdev_lockop(struct seg *, caddr_t, size_t, int, int,
 175                     ulong_t *, size_t);
 176 static int      segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
 177 static u_offset_t       segdev_getoffset(struct seg *, caddr_t);
 178 static int      segdev_gettype(struct seg *, caddr_t);
 179 static int      segdev_getvp(struct seg *, caddr_t, struct vnode **);
 180 static int      segdev_advise(struct seg *, caddr_t, size_t, uint_t);
 181 static void     segdev_dump(struct seg *);
 182 static int      segdev_pagelock(struct seg *, caddr_t, size_t,
 183                     struct page ***, enum lock_type, enum seg_rw);
 184 static int      segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
 185 static int      segdev_getmemid(struct seg *, caddr_t, memid_t *);
 186 static lgrp_mem_policy_info_t   *segdev_getpolicy(struct seg *, caddr_t);
 187 static int      segdev_capable(struct seg *, segcapability_t);
 188 
 189 /*
 190  * XXX  this struct is used by rootnex_map_fault to identify
 191  *      the segment it has been passed. So if you make it
 192  *      "static" you'll need to fix rootnex_map_fault.
 193  */
 194 struct seg_ops segdev_ops = {
 195         segdev_dup,
 196         segdev_unmap,
 197         segdev_free,
 198         segdev_fault,
 199         segdev_faulta,
 200         segdev_setprot,
 201         segdev_checkprot,
 202         (int (*)())segdev_badop,        /* kluster */
 203         segdev_sync,                    /* sync */
 204         segdev_incore,
 205         segdev_lockop,                  /* lockop */
 206         segdev_getprot,
 207         segdev_getoffset,
 208         segdev_gettype,
 209         segdev_getvp,
 210         segdev_advise,
 211         segdev_dump,
 212         segdev_pagelock,
 213         segdev_setpagesize,
 214         segdev_getmemid,
 215         segdev_getpolicy,
 216         segdev_capable,
 217 };
 218 
 219 /*
 220  * Private segdev support routines
 221  */
 222 static struct segdev_data *sdp_alloc(void);
 223 
 224 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
 225     size_t, enum seg_rw);
 226 
 227 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
 228     struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
 229 
 230 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
 231     size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
 232 
 233 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
 234 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
 235 static void devmap_softlock_rele(devmap_handle_t *);
 236 static void devmap_ctx_rele(devmap_handle_t *);
 237 
 238 static void devmap_ctxto(void *);
 239 
 240 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
 241     caddr_t addr);
 242 
 243 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
 244     ulong_t *opfn, ulong_t *pagesize);
 245 
 246 static void free_devmap_handle(devmap_handle_t *dhp);
 247 
 248 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
 249     struct seg *newseg);
 250 
 251 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
 252 
 253 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
 254 
 255 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
 256 
 257 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
 258     offset_t off, size_t len, uint_t flags);
 259 
 260 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
 261     caddr_t addr, size_t *llen, caddr_t *laddr);
 262 
 263 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
 264 
 265 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
 266 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
 267 
 268 static void *devmap_umem_alloc_np(size_t size, size_t flags);
 269 static void devmap_umem_free_np(void *addr, size_t size);
 270 
 271 /*
 272  * routines to lock and unlock underlying segkp segment for
 273  * KMEM_PAGEABLE type cookies.
 274  */
 275 static faultcode_t  acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
 276 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
 277 
 278 /*
 279  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
 280  * drivers with devmap_access callbacks
 281  */
 282 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
 283         enum fault_type);
 284 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
 285         enum fault_type);
 286 
 287 static kmutex_t devmapctx_lock;
 288 
 289 static kmutex_t devmap_slock;
 290 
 291 /*
 292  * Initialize the thread callbacks and thread private data.
 293  */
 294 static struct devmap_ctx *
 295 devmap_ctxinit(dev_t dev, ulong_t id)
 296 {
 297         struct devmap_ctx       *devctx;
 298         struct devmap_ctx       *tmp;
 299         dev_info_t              *dip;
 300 
 301         tmp =  kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
 302 
 303         mutex_enter(&devmapctx_lock);
 304 
 305         dip = e_ddi_hold_devi_by_dev(dev, 0);
 306         ASSERT(dip != NULL);
 307         ddi_release_devi(dip);
 308 
 309         for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
 310                 if ((devctx->dip == dip) && (devctx->id == id))
 311                         break;
 312 
 313         if (devctx == NULL) {
 314                 devctx = tmp;
 315                 devctx->dip = dip;
 316                 devctx->id = id;
 317                 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
 318                 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
 319                 devctx->next = devmapctx_list;
 320                 devmapctx_list = devctx;
 321         } else
 322                 kmem_free(tmp, sizeof (struct devmap_ctx));
 323 
 324         mutex_enter(&devctx->lock);
 325         devctx->refcnt++;
 326         mutex_exit(&devctx->lock);
 327         mutex_exit(&devmapctx_lock);
 328 
 329         return (devctx);
 330 }
 331 
 332 /*
 333  * Timeout callback called if a CPU has not given up the device context
 334  * within dhp->dh_timeout_length ticks
 335  */
 336 static void
 337 devmap_ctxto(void *data)
 338 {
 339         struct devmap_ctx *devctx = data;
 340 
 341         TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
 342             "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
 343         mutex_enter(&devctx->lock);
 344         /*
 345          * Set oncpu = 0 so the next mapping trying to get the device context
 346          * can.
 347          */
 348         devctx->oncpu = 0;
 349         devctx->timeout = 0;
 350         cv_signal(&devctx->cv);
 351         mutex_exit(&devctx->lock);
 352 }
 353 
 354 /*
 355  * Create a device segment.
 356  */
 357 int
 358 segdev_create(struct seg *seg, void *argsp)
 359 {
 360         struct segdev_data *sdp;
 361         struct segdev_crargs *a = (struct segdev_crargs *)argsp;
 362         devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
 363         int error;
 364 
 365         /*
 366          * Since the address space is "write" locked, we
 367          * don't need the segment lock to protect "segdev" data.
 368          */
 369         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 370 
 371         hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
 372 
 373         sdp = sdp_alloc();
 374 
 375         sdp->mapfunc = a->mapfunc;
 376         sdp->offset = a->offset;
 377         sdp->prot = a->prot;
 378         sdp->maxprot = a->maxprot;
 379         sdp->type = a->type;
 380         sdp->pageprot = 0;
 381         sdp->softlockcnt = 0;
 382         sdp->vpage = NULL;
 383 
 384         if (sdp->mapfunc == NULL)
 385                 sdp->devmap_data = dhp;
 386         else
 387                 sdp->devmap_data = dhp = NULL;
 388 
 389         sdp->hat_flags = a->hat_flags;
 390         sdp->hat_attr = a->hat_attr;
 391 
 392         /*
 393          * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
 394          */
 395         ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
 396 
 397         /*
 398          * Hold shadow vnode -- segdev only deals with
 399          * character (VCHR) devices. We use the common
 400          * vp to hang pages on.
 401          */
 402         sdp->vp = specfind(a->dev, VCHR);
 403         ASSERT(sdp->vp != NULL);
 404 
 405         seg->s_ops = &segdev_ops;
 406         seg->s_data = sdp;
 407 
 408         while (dhp != NULL) {
 409                 dhp->dh_seg = seg;
 410                 dhp = dhp->dh_next;
 411         }
 412 
 413         /*
 414          * Inform the vnode of the new mapping.
 415          */
 416         /*
 417          * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
 418          * dhp specific maxprot because spec_addmap does not use maxprot.
 419          */
 420         error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
 421             seg->s_as, seg->s_base, seg->s_size,
 422             sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
 423 
 424         if (error != 0) {
 425                 sdp->devmap_data = NULL;
 426                 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
 427                     HAT_UNLOAD_UNMAP);
 428         } else {
 429                 /*
 430                  * Mappings of /dev/null don't count towards the VSZ of a
 431                  * process.  Mappings of /dev/null have no mapping type.
 432                  */
 433                 if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
 434                     MAP_PRIVATE)) == 0) {
 435                         seg->s_as->a_resvsize -= seg->s_size;
 436                 }
 437         }
 438 
 439         return (error);
 440 }
 441 
 442 static struct segdev_data *
 443 sdp_alloc(void)
 444 {
 445         struct segdev_data *sdp;
 446 
 447         sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
 448         rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
 449 
 450         return (sdp);
 451 }
 452 
 453 /*
 454  * Duplicate seg and return new segment in newseg.
 455  */
 456 static int
 457 segdev_dup(struct seg *seg, struct seg *newseg)
 458 {
 459         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
 460         struct segdev_data *newsdp;
 461         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
 462         size_t npages;
 463         int ret;
 464 
 465         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
 466             "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
 467 
 468         DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
 469             (void *)dhp, (void *)seg));
 470 
 471         /*
 472          * Since the address space is "write" locked, we
 473          * don't need the segment lock to protect "segdev" data.
 474          */
 475         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 476 
 477         newsdp = sdp_alloc();
 478 
 479         newseg->s_ops = seg->s_ops;
 480         newseg->s_data = (void *)newsdp;
 481 
 482         VN_HOLD(sdp->vp);
 483         newsdp->vp   = sdp->vp;
 484         newsdp->mapfunc = sdp->mapfunc;
 485         newsdp->offset       = sdp->offset;
 486         newsdp->pageprot = sdp->pageprot;
 487         newsdp->prot = sdp->prot;
 488         newsdp->maxprot = sdp->maxprot;
 489         newsdp->type = sdp->type;
 490         newsdp->hat_attr = sdp->hat_attr;
 491         newsdp->hat_flags = sdp->hat_flags;
 492         newsdp->softlockcnt = 0;
 493 
 494         /*
 495          * Initialize per page data if the segment we are
 496          * dup'ing has per page information.
 497          */
 498         npages = seg_pages(newseg);
 499 
 500         if (sdp->vpage != NULL) {
 501                 size_t nbytes = vpgtob(npages);
 502 
 503                 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
 504                 bcopy(sdp->vpage, newsdp->vpage, nbytes);
 505         } else
 506                 newsdp->vpage = NULL;
 507 
 508         /*
 509          * duplicate devmap handles
 510          */
 511         if (dhp != NULL) {
 512                 ret = devmap_handle_dup(dhp,
 513                     (devmap_handle_t **)&newsdp->devmap_data, newseg);
 514                 if (ret != 0) {
 515                         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
 516                             "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
 517                             ret, (void *)dhp, (void *)seg);
 518                         DEBUGF(1, (CE_CONT,
 519                             "segdev_dup: ret %x dhp %p seg %p\n",
 520                             ret, (void *)dhp, (void *)seg));
 521                         return (ret);
 522                 }
 523         }
 524 
 525         /*
 526          * Inform the common vnode of the new mapping.
 527          */
 528         return (VOP_ADDMAP(VTOCVP(newsdp->vp),
 529             newsdp->offset, newseg->s_as,
 530             newseg->s_base, newseg->s_size, newsdp->prot,
 531             newsdp->maxprot, sdp->type, CRED(), NULL));
 532 }
 533 
 534 /*
 535  * duplicate devmap handles
 536  */
 537 static int
 538 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
 539     struct seg *newseg)
 540 {
 541         devmap_handle_t *newdhp_save = NULL;
 542         devmap_handle_t *newdhp = NULL;
 543         struct devmap_callback_ctl *callbackops;
 544 
 545         while (dhp != NULL) {
 546                 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
 547 
 548                 /* Need to lock the original dhp while copying if REMAP */
 549                 HOLD_DHP_LOCK(dhp);
 550                 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
 551                 RELE_DHP_LOCK(dhp);
 552                 newdhp->dh_seg = newseg;
 553                 newdhp->dh_next = NULL;
 554                 if (newdhp_save != NULL)
 555                         newdhp_save->dh_next = newdhp;
 556                 else
 557                         *new_dhp = newdhp;
 558                 newdhp_save = newdhp;
 559 
 560                 callbackops = &newdhp->dh_callbackops;
 561 
 562                 if (dhp->dh_softlock != NULL)
 563                         newdhp->dh_softlock = devmap_softlock_init(
 564                             newdhp->dh_dev,
 565                             (ulong_t)callbackops->devmap_access);
 566                 if (dhp->dh_ctx != NULL)
 567                         newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
 568                             (ulong_t)callbackops->devmap_access);
 569 
 570                 /*
 571                  * Initialize dh_lock if we want to do remap.
 572                  */
 573                 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
 574                         mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
 575                         newdhp->dh_flags |= DEVMAP_LOCK_INITED;
 576                 }
 577 
 578                 if (callbackops->devmap_dup != NULL) {
 579                         int ret;
 580 
 581                         /*
 582                          * Call the dup callback so that the driver can
 583                          * duplicate its private data.
 584                          */
 585                         ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
 586                             (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
 587 
 588                         if (ret != 0) {
 589                                 /*
 590                                  * We want to free up this segment as the driver
 591                                  * has indicated that we can't dup it.  But we
 592                                  * don't want to call the drivers, devmap_unmap,
 593                                  * callback function as the driver does not
 594                                  * think this segment exists. The caller of
 595                                  * devmap_dup will call seg_free on newseg
 596                                  * as it was the caller that allocated the
 597                                  * segment.
 598                                  */
 599                                 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
 600                                     "newdhp %p dhp %p\n", (void *)newdhp,
 601                                     (void *)dhp));
 602                                 callbackops->devmap_unmap = NULL;
 603                                 return (ret);
 604                         }
 605                 }
 606 
 607                 dhp = dhp->dh_next;
 608         }
 609 
 610         return (0);
 611 }
 612 
 613 /*
 614  * Split a segment at addr for length len.
 615  */
 616 /*ARGSUSED*/
 617 static int
 618 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
 619 {
 620         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
 621         register struct segdev_data *nsdp;
 622         register struct seg *nseg;
 623         register size_t opages;         /* old segment size in pages */
 624         register size_t npages;         /* new segment size in pages */
 625         register size_t dpages;         /* pages being deleted (unmapped) */
 626         register size_t nbytes;
 627         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
 628         devmap_handle_t *dhpp;
 629         devmap_handle_t *newdhp;
 630         struct devmap_callback_ctl *callbackops;
 631         caddr_t nbase;
 632         offset_t off;
 633         ulong_t nsize;
 634         size_t mlen, sz;
 635 
 636         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
 637             "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
 638             (void *)dhp, (void *)seg, (void *)addr, len);
 639 
 640         DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
 641             (void *)dhp, (void *)seg, (void *)addr, len));
 642 
 643         /*
 644          * Since the address space is "write" locked, we
 645          * don't need the segment lock to protect "segdev" data.
 646          */
 647         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 648 
 649         if ((sz = sdp->softlockcnt) > 0) {
 650                 /*
 651                  * Fail the unmap if pages are SOFTLOCKed through this mapping.
 652                  * softlockcnt is protected from change by the as write lock.
 653                  */
 654                 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
 655                     "segdev_unmap:error softlockcnt = %ld", sz);
 656                 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
 657                 return (EAGAIN);
 658         }
 659 
 660         /*
 661          * Check for bad sizes
 662          */
 663         if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
 664             (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
 665                 panic("segdev_unmap");
 666 
 667         if (dhp != NULL) {
 668                 devmap_handle_t *tdhp;
 669                 /*
 670                  * If large page size was used in hat_devload(),
 671                  * the same page size must be used in hat_unload().
 672                  */
 673                 dhpp = tdhp = devmap_find_handle(dhp, addr);
 674                 while (tdhp != NULL) {
 675                         if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
 676                                 break;
 677                         }
 678                         tdhp = tdhp->dh_next;
 679                 }
 680                 if (tdhp != NULL) {     /* found a dhp using large pages */
 681                         size_t slen = len;
 682                         size_t mlen;
 683                         size_t soff;
 684 
 685                         soff = (ulong_t)(addr - dhpp->dh_uvaddr);
 686                         while (slen != 0) {
 687                                 mlen = MIN(slen, (dhpp->dh_len - soff));
 688                                 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
 689                                     dhpp->dh_len, HAT_UNLOAD_UNMAP);
 690                                 dhpp = dhpp->dh_next;
 691                                 ASSERT(slen >= mlen);
 692                                 slen -= mlen;
 693                                 soff = 0;
 694                         }
 695                 } else
 696                         hat_unload(seg->s_as->a_hat, addr, len,
 697                             HAT_UNLOAD_UNMAP);
 698         } else {
 699                 /*
 700                  * Unload any hardware translations in the range
 701                  * to be taken out.
 702                  */
 703                 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
 704         }
 705 
 706         /*
 707          * get the user offset which will used in the driver callbacks
 708          */
 709         off = sdp->offset + (offset_t)(addr - seg->s_base);
 710 
 711         /*
 712          * Inform the vnode of the unmapping.
 713          */
 714         ASSERT(sdp->vp != NULL);
 715         (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
 716             sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
 717 
 718         /*
 719          * Check for entire segment
 720          */
 721         if (addr == seg->s_base && len == seg->s_size) {
 722                 seg_free(seg);
 723                 return (0);
 724         }
 725 
 726         opages = seg_pages(seg);
 727         dpages = btop(len);
 728         npages = opages - dpages;
 729 
 730         /*
 731          * Check for beginning of segment
 732          */
 733         if (addr == seg->s_base) {
 734                 if (sdp->vpage != NULL) {
 735                         register struct vpage *ovpage;
 736 
 737                         ovpage = sdp->vpage; /* keep pointer to vpage */
 738 
 739                         nbytes = vpgtob(npages);
 740                         sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 741                         bcopy(&ovpage[dpages], sdp->vpage, nbytes);
 742 
 743                         /* free up old vpage */
 744                         kmem_free(ovpage, vpgtob(opages));
 745                 }
 746 
 747                 /*
 748                  * free devmap handles from the beginning of the mapping.
 749                  */
 750                 if (dhp != NULL)
 751                         devmap_handle_unmap_head(dhp, len);
 752 
 753                 sdp->offset += (offset_t)len;
 754 
 755                 seg->s_base += len;
 756                 seg->s_size -= len;
 757 
 758                 return (0);
 759         }
 760 
 761         /*
 762          * Check for end of segment
 763          */
 764         if (addr + len == seg->s_base + seg->s_size) {
 765                 if (sdp->vpage != NULL) {
 766                         register struct vpage *ovpage;
 767 
 768                         ovpage = sdp->vpage; /* keep pointer to vpage */
 769 
 770                         nbytes = vpgtob(npages);
 771                         sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 772                         bcopy(ovpage, sdp->vpage, nbytes);
 773 
 774                         /* free up old vpage */
 775                         kmem_free(ovpage, vpgtob(opages));
 776                 }
 777                 seg->s_size -= len;
 778 
 779                 /*
 780                  * free devmap handles from addr to the end of the mapping.
 781                  */
 782                 if (dhp != NULL)
 783                         devmap_handle_unmap_tail(dhp, addr);
 784 
 785                 return (0);
 786         }
 787 
 788         /*
 789          * The section to go is in the middle of the segment,
 790          * have to make it into two segments.  nseg is made for
 791          * the high end while seg is cut down at the low end.
 792          */
 793         nbase = addr + len;                             /* new seg base */
 794         nsize = (seg->s_base + seg->s_size) - nbase;      /* new seg size */
 795         seg->s_size = addr - seg->s_base;         /* shrink old seg */
 796         nseg = seg_alloc(seg->s_as, nbase, nsize);
 797         if (nseg == NULL)
 798                 panic("segdev_unmap seg_alloc");
 799 
 800         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
 801             "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
 802         DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
 803             (void *)seg, (void *)nseg));
 804         nsdp = sdp_alloc();
 805 
 806         nseg->s_ops = seg->s_ops;
 807         nseg->s_data = (void *)nsdp;
 808 
 809         VN_HOLD(sdp->vp);
 810         nsdp->mapfunc = sdp->mapfunc;
 811         nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
 812         nsdp->vp     = sdp->vp;
 813         nsdp->pageprot = sdp->pageprot;
 814         nsdp->prot   = sdp->prot;
 815         nsdp->maxprot = sdp->maxprot;
 816         nsdp->type = sdp->type;
 817         nsdp->hat_attr = sdp->hat_attr;
 818         nsdp->hat_flags = sdp->hat_flags;
 819         nsdp->softlockcnt = 0;
 820 
 821         /*
 822          * Initialize per page data if the segment we are
 823          * dup'ing has per page information.
 824          */
 825         if (sdp->vpage != NULL) {
 826                 /* need to split vpage into two arrays */
 827                 register size_t nnbytes;
 828                 register size_t nnpages;
 829                 register struct vpage *ovpage;
 830 
 831                 ovpage = sdp->vpage;         /* keep pointer to vpage */
 832 
 833                 npages = seg_pages(seg);        /* seg has shrunk */
 834                 nbytes = vpgtob(npages);
 835                 nnpages = seg_pages(nseg);
 836                 nnbytes = vpgtob(nnpages);
 837 
 838                 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 839                 bcopy(ovpage, sdp->vpage, nbytes);
 840 
 841                 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
 842                 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
 843 
 844                 /* free up old vpage */
 845                 kmem_free(ovpage, vpgtob(opages));
 846         } else
 847                 nsdp->vpage = NULL;
 848 
 849         /*
 850          * unmap dhps.
 851          */
 852         if (dhp == NULL) {
 853                 nsdp->devmap_data = NULL;
 854                 return (0);
 855         }
 856         while (dhp != NULL) {
 857                 callbackops = &dhp->dh_callbackops;
 858                 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
 859                     "segdev_unmap: dhp=%p addr=%p", dhp, addr);
 860                 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
 861                     (void *)dhp, (void *)addr,
 862                     (void *)dhp->dh_uvaddr, dhp->dh_len));
 863 
 864                 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
 865                         dhpp = dhp->dh_next;
 866                         dhp->dh_next = NULL;
 867                         dhp = dhpp;
 868                 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
 869                         dhp = dhp->dh_next;
 870                 } else if (addr > dhp->dh_uvaddr &&
 871                     (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
 872                         /*
 873                          * <addr, addr+len> is enclosed by dhp.
 874                          * create a newdhp that begins at addr+len and
 875                          * ends at dhp->dh_uvaddr+dhp->dh_len.
 876                          */
 877                         newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
 878                         HOLD_DHP_LOCK(dhp);
 879                         bcopy(dhp, newdhp, sizeof (devmap_handle_t));
 880                         RELE_DHP_LOCK(dhp);
 881                         newdhp->dh_seg = nseg;
 882                         newdhp->dh_next = dhp->dh_next;
 883                         if (dhp->dh_softlock != NULL)
 884                                 newdhp->dh_softlock = devmap_softlock_init(
 885                                     newdhp->dh_dev,
 886                                     (ulong_t)callbackops->devmap_access);
 887                         if (dhp->dh_ctx != NULL)
 888                                 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
 889                                     (ulong_t)callbackops->devmap_access);
 890                         if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
 891                                 mutex_init(&newdhp->dh_lock,
 892                                     NULL, MUTEX_DEFAULT, NULL);
 893                         }
 894                         if (callbackops->devmap_unmap != NULL)
 895                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 896                                     off, len, dhp, &dhp->dh_pvtp,
 897                                     newdhp, &newdhp->dh_pvtp);
 898                         mlen = len + (addr - dhp->dh_uvaddr);
 899                         devmap_handle_reduce_len(newdhp, mlen);
 900                         nsdp->devmap_data = newdhp;
 901                         /* XX Changing len should recalculate LARGE flag */
 902                         dhp->dh_len = addr - dhp->dh_uvaddr;
 903                         dhpp = dhp->dh_next;
 904                         dhp->dh_next = NULL;
 905                         dhp = dhpp;
 906                 } else if ((addr > dhp->dh_uvaddr) &&
 907                     ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
 908                         mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
 909                         /*
 910                          * <addr, addr+len> spans over dhps.
 911                          */
 912                         if (callbackops->devmap_unmap != NULL)
 913                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 914                                     off, mlen, (devmap_cookie_t *)dhp,
 915                                     &dhp->dh_pvtp, NULL, NULL);
 916                         /* XX Changing len should recalculate LARGE flag */
 917                         dhp->dh_len = addr - dhp->dh_uvaddr;
 918                         dhpp = dhp->dh_next;
 919                         dhp->dh_next = NULL;
 920                         dhp = dhpp;
 921                         nsdp->devmap_data = dhp;
 922                 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
 923                         /*
 924                          * dhp is enclosed by <addr, addr+len>.
 925                          */
 926                         dhp->dh_seg = nseg;
 927                         nsdp->devmap_data = dhp;
 928                         dhp = devmap_handle_unmap(dhp);
 929                         nsdp->devmap_data = dhp; /* XX redundant? */
 930                 } else if (((addr + len) > dhp->dh_uvaddr) &&
 931                     ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
 932                         mlen = addr + len - dhp->dh_uvaddr;
 933                         if (callbackops->devmap_unmap != NULL)
 934                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 935                                     dhp->dh_uoff, mlen, NULL,
 936                                     NULL, dhp, &dhp->dh_pvtp);
 937                         devmap_handle_reduce_len(dhp, mlen);
 938                         nsdp->devmap_data = dhp;
 939                         dhp->dh_seg = nseg;
 940                         dhp = dhp->dh_next;
 941                 } else {
 942                         dhp->dh_seg = nseg;
 943                         dhp = dhp->dh_next;
 944                 }
 945         }
 946         return (0);
 947 }
 948 
 949 /*
 950  * Utility function handles reducing the length of a devmap handle during unmap
 951  * Note that is only used for unmapping the front portion of the handler,
 952  * i.e., we are bumping up the offset/pfn etc up by len
 953  * Do not use if reducing length at the tail.
 954  */
 955 static void
 956 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
 957 {
 958         struct ddi_umem_cookie *cp;
 959         struct devmap_pmem_cookie *pcp;
 960         /*
 961          * adjust devmap handle fields
 962          */
 963         ASSERT(len < dhp->dh_len);
 964 
 965         /* Make sure only page-aligned changes are done */
 966         ASSERT((len & PAGEOFFSET) == 0);
 967 
 968         dhp->dh_len -= len;
 969         dhp->dh_uoff += (offset_t)len;
 970         dhp->dh_roff += (offset_t)len;
 971         dhp->dh_uvaddr += len;
 972         /* Need to grab dhp lock if REMAP */
 973         HOLD_DHP_LOCK(dhp);
 974         cp = dhp->dh_cookie;
 975         if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
 976                 if (cookie_is_devmem(cp)) {
 977                         dhp->dh_pfn += btop(len);
 978                 } else if (cookie_is_pmem(cp)) {
 979                         pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
 980                         ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
 981                             dhp->dh_roff < ptob(pcp->dp_npages));
 982                 } else {
 983                         ASSERT(dhp->dh_roff < cp->size);
 984                         ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
 985                             dhp->dh_cvaddr < (cp->cvaddr + cp->size));
 986                         ASSERT((dhp->dh_cvaddr + len) <=
 987                             (cp->cvaddr + cp->size));
 988 
 989                         dhp->dh_cvaddr += len;
 990                 }
 991         }
 992         /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
 993         RELE_DHP_LOCK(dhp);
 994 }
 995 
 996 /*
 997  * Free devmap handle, dhp.
 998  * Return the next devmap handle on the linked list.
 999  */
1000 static devmap_handle_t *
1001 devmap_handle_unmap(devmap_handle_t *dhp)
1002 {
1003         struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1004         struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1005         devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1006 
1007         ASSERT(dhp != NULL);
1008 
1009         /*
1010          * before we free up dhp, call the driver's devmap_unmap entry point
1011          * to free resources allocated for this dhp.
1012          */
1013         if (callbackops->devmap_unmap != NULL) {
1014                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1015                     dhp->dh_len, NULL, NULL, NULL, NULL);
1016         }
1017 
1018         if (dhpp == dhp) {      /* releasing first dhp, change sdp data */
1019                 sdp->devmap_data = dhp->dh_next;
1020         } else {
1021                 while (dhpp->dh_next != dhp) {
1022                         dhpp = dhpp->dh_next;
1023                 }
1024                 dhpp->dh_next = dhp->dh_next;
1025         }
1026         dhpp = dhp->dh_next; /* return value is next dhp in chain */
1027 
1028         if (dhp->dh_softlock != NULL)
1029                 devmap_softlock_rele(dhp);
1030 
1031         if (dhp->dh_ctx != NULL)
1032                 devmap_ctx_rele(dhp);
1033 
1034         if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1035                 mutex_destroy(&dhp->dh_lock);
1036         }
1037         kmem_free(dhp, sizeof (devmap_handle_t));
1038 
1039         return (dhpp);
1040 }
1041 
1042 /*
1043  * Free complete devmap handles from dhp for len bytes
1044  * dhp can be either the first handle or a subsequent handle
1045  */
1046 static void
1047 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1048 {
1049         struct devmap_callback_ctl *callbackops;
1050 
1051         /*
1052          * free the devmap handles covered by len.
1053          */
1054         while (len >= dhp->dh_len) {
1055                 len -= dhp->dh_len;
1056                 dhp = devmap_handle_unmap(dhp);
1057         }
1058         if (len != 0) { /* partial unmap at head of first remaining dhp */
1059                 callbackops = &dhp->dh_callbackops;
1060 
1061                 /*
1062                  * Call the unmap callback so the drivers can make
1063                  * adjustment on its private data.
1064                  */
1065                 if (callbackops->devmap_unmap != NULL)
1066                         (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1067                             dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1068                 devmap_handle_reduce_len(dhp, len);
1069         }
1070 }
1071 
1072 /*
1073  * Free devmap handles to truncate  the mapping after addr
1074  * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1075  *      Also could then use the routine in middle unmap case too
1076  */
1077 static void
1078 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1079 {
1080         register struct seg *seg = dhp->dh_seg;
1081         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1082         register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1083         struct devmap_callback_ctl *callbackops;
1084         register devmap_handle_t *dhpp;
1085         size_t maplen;
1086         ulong_t off;
1087         size_t len;
1088 
1089         maplen = (size_t)(addr - dhp->dh_uvaddr);
1090         dhph = devmap_find_handle(dhph, addr);
1091 
1092         while (dhph != NULL) {
1093                 if (maplen == 0) {
1094                         dhph =  devmap_handle_unmap(dhph);
1095                 } else {
1096                         callbackops = &dhph->dh_callbackops;
1097                         len = dhph->dh_len - maplen;
1098                         off = (ulong_t)sdp->offset + (addr - seg->s_base);
1099                         /*
1100                          * Call the unmap callback so the driver
1101                          * can make adjustments on its private data.
1102                          */
1103                         if (callbackops->devmap_unmap != NULL)
1104                                 (*callbackops->devmap_unmap)(dhph,
1105                                     dhph->dh_pvtp, off, len,
1106                                     (devmap_cookie_t *)dhph,
1107                                     &dhph->dh_pvtp, NULL, NULL);
1108                         /* XXX Reducing len needs to recalculate LARGE flag */
1109                         dhph->dh_len = maplen;
1110                         maplen = 0;
1111                         dhpp = dhph->dh_next;
1112                         dhph->dh_next = NULL;
1113                         dhph = dhpp;
1114                 }
1115         } /* end while */
1116 }
1117 
1118 /*
1119  * Free a segment.
1120  */
1121 static void
1122 segdev_free(struct seg *seg)
1123 {
1124         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1125         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1126 
1127         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1128             "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1129         DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1130             (void *)dhp, (void *)seg));
1131 
1132         /*
1133          * Since the address space is "write" locked, we
1134          * don't need the segment lock to protect "segdev" data.
1135          */
1136         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1137 
1138         while (dhp != NULL)
1139                 dhp = devmap_handle_unmap(dhp);
1140 
1141         VN_RELE(sdp->vp);
1142         if (sdp->vpage != NULL)
1143                 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1144 
1145         rw_destroy(&sdp->lock);
1146         kmem_free(sdp, sizeof (*sdp));
1147 }
1148 
1149 static void
1150 free_devmap_handle(devmap_handle_t *dhp)
1151 {
1152         register devmap_handle_t *dhpp;
1153 
1154         /*
1155          * free up devmap handle
1156          */
1157         while (dhp != NULL) {
1158                 dhpp = dhp->dh_next;
1159                 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1160                         mutex_destroy(&dhp->dh_lock);
1161                 }
1162 
1163                 if (dhp->dh_softlock != NULL)
1164                         devmap_softlock_rele(dhp);
1165 
1166                 if (dhp->dh_ctx != NULL)
1167                         devmap_ctx_rele(dhp);
1168 
1169                 kmem_free(dhp, sizeof (devmap_handle_t));
1170                 dhp = dhpp;
1171         }
1172 }
1173 
1174 /*
1175  * routines to lock and unlock underlying segkp segment for
1176  * KMEM_PAGEABLE type cookies.
1177  * segkp only allows a single pending F_SOFTLOCK
1178  * we keep track of number of locks in the cookie so we can
1179  * have multiple pending faults and manage the calls to segkp.
1180  * RFE: if segkp supports either pagelock or can support multiple
1181  * calls to F_SOFTLOCK, then these routines can go away.
1182  *      If pagelock, segdev_faultpage can fault on a page by page basis
1183  *              and simplifies the code quite a bit.
1184  *      if multiple calls allowed but not partial ranges, then need for
1185  *      cookie->lock and locked count goes away, code can call as_fault directly
1186  */
1187 static faultcode_t
1188 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1189 {
1190         int err = 0;
1191         ASSERT(cookie_is_kpmem(cookie));
1192         /*
1193          * Fault in pages in segkp with F_SOFTLOCK.
1194          * We want to hold the lock until all pages have been loaded.
1195          * segkp only allows single caller to hold SOFTLOCK, so cookie
1196          * holds a count so we dont call into segkp multiple times
1197          */
1198         mutex_enter(&cookie->lock);
1199 
1200         /*
1201          * Check for overflow in locked field
1202          */
1203         if ((UINT32_MAX - cookie->locked) < npages) {
1204                 err = FC_MAKE_ERR(ENOMEM);
1205         } else if (cookie->locked == 0) {
1206                 /* First time locking */
1207                 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1208                     cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1209         }
1210         if (!err) {
1211                 cookie->locked += npages;
1212         }
1213         mutex_exit(&cookie->lock);
1214         return (err);
1215 }
1216 
1217 static void
1218 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1219 {
1220         mutex_enter(&cookie->lock);
1221         ASSERT(cookie_is_kpmem(cookie));
1222         ASSERT(cookie->locked >= npages);
1223         cookie->locked -= (uint_t)npages;
1224         if (cookie->locked == 0) {
1225                 /* Last unlock */
1226                 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1227                     cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1228                         panic("segdev releasing kpmem lock %p", (void *)cookie);
1229         }
1230         mutex_exit(&cookie->lock);
1231 }
1232 
1233 /*
1234  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1235  * drivers with devmap_access callbacks
1236  * slock->softlocked basically works like a rw lock
1237  *      -ve counts => F_SOFTLOCK in progress
1238  *      +ve counts => F_INVAL/F_PROT in progress
1239  * We allow only one F_SOFTLOCK at a time
1240  * but can have multiple pending F_INVAL/F_PROT calls
1241  *
1242  * This routine waits using cv_wait_sig so killing processes is more graceful
1243  * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1244  */
1245 static int devmap_softlock_enter(
1246         struct devmap_softlock *slock,
1247         size_t npages,
1248         enum fault_type type)
1249 {
1250         if (npages == 0)
1251                 return (0);
1252         mutex_enter(&(slock->lock));
1253         switch (type) {
1254         case F_SOFTLOCK :
1255                 while (slock->softlocked) {
1256                         if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1257                                 /* signalled */
1258                                 mutex_exit(&(slock->lock));
1259                                 return (EINTR);
1260                         }
1261                 }
1262                 slock->softlocked -= npages; /* -ve count => locked */
1263                 break;
1264         case F_INVAL :
1265         case F_PROT :
1266                 while (slock->softlocked < 0)
1267                         if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1268                                 /* signalled */
1269                                 mutex_exit(&(slock->lock));
1270                                 return (EINTR);
1271                         }
1272                 slock->softlocked += npages; /* +ve count => f_invals */
1273                 break;
1274         default:
1275                 ASSERT(0);
1276         }
1277         mutex_exit(&(slock->lock));
1278         return (0);
1279 }
1280 
1281 static void devmap_softlock_exit(
1282         struct devmap_softlock *slock,
1283         size_t npages,
1284         enum fault_type type)
1285 {
1286         if (slock == NULL)
1287                 return;
1288         mutex_enter(&(slock->lock));
1289         switch (type) {
1290         case F_SOFTLOCK :
1291                 ASSERT(-slock->softlocked >= npages);
1292                 slock->softlocked += npages; /* -ve count is softlocked */
1293                 if (slock->softlocked == 0)
1294                         cv_signal(&slock->cv);
1295                 break;
1296         case F_INVAL :
1297         case F_PROT:
1298                 ASSERT(slock->softlocked >= npages);
1299                 slock->softlocked -= npages;
1300                 if (slock->softlocked == 0)
1301                         cv_signal(&slock->cv);
1302                 break;
1303         default:
1304                 ASSERT(0);
1305         }
1306         mutex_exit(&(slock->lock));
1307 }
1308 
1309 /*
1310  * Do a F_SOFTUNLOCK call over the range requested.
1311  * The range must have already been F_SOFTLOCK'ed.
1312  * The segment lock should be held, (but not the segment private lock?)
1313  *  The softunlock code below does not adjust for large page sizes
1314  *      assumes the caller already did any addr/len adjustments for
1315  *      pagesize mappings before calling.
1316  */
1317 /*ARGSUSED*/
1318 static void
1319 segdev_softunlock(
1320         struct hat *hat,                /* the hat */
1321         struct seg *seg,                /* seg_dev of interest */
1322         caddr_t addr,                   /* base address of range */
1323         size_t len,                     /* number of bytes */
1324         enum seg_rw rw)                 /* type of access at fault */
1325 {
1326         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1327         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1328 
1329         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1330             "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1331             dhp_head, sdp, addr, len);
1332         DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1333             "addr %p len %lx\n",
1334             (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1335 
1336         hat_unlock(hat, addr, len);
1337 
1338         if (dhp_head != NULL) {
1339                 devmap_handle_t *dhp;
1340                 size_t mlen;
1341                 size_t tlen = len;
1342                 ulong_t off;
1343 
1344                 dhp = devmap_find_handle(dhp_head, addr);
1345                 ASSERT(dhp != NULL);
1346 
1347                 off = (ulong_t)(addr - dhp->dh_uvaddr);
1348                 while (tlen != 0) {
1349                         mlen = MIN(tlen, (dhp->dh_len - off));
1350 
1351                         /*
1352                          * unlock segkp memory, locked during F_SOFTLOCK
1353                          */
1354                         if (dhp_is_kpmem(dhp)) {
1355                                 release_kpmem_lock(
1356                                     (struct ddi_umem_cookie *)dhp->dh_cookie,
1357                                     btopr(mlen));
1358                         }
1359 
1360                         /*
1361                          * Do the softlock accounting for devmap_access
1362                          */
1363                         if (dhp->dh_callbackops.devmap_access != NULL) {
1364                                 devmap_softlock_exit(dhp->dh_softlock,
1365                                     btopr(mlen), F_SOFTLOCK);
1366                         }
1367 
1368                         tlen -= mlen;
1369                         dhp = dhp->dh_next;
1370                         off = 0;
1371                 }
1372         }
1373 
1374         mutex_enter(&freemem_lock);
1375         ASSERT(sdp->softlockcnt >= btopr(len));
1376         sdp->softlockcnt -= btopr(len);
1377         mutex_exit(&freemem_lock);
1378         if (sdp->softlockcnt == 0) {
1379                 /*
1380                  * All SOFTLOCKS are gone. Wakeup any waiting
1381                  * unmappers so they can try again to unmap.
1382                  * Check for waiters first without the mutex
1383                  * held so we don't always grab the mutex on
1384                  * softunlocks.
1385                  */
1386                 if (AS_ISUNMAPWAIT(seg->s_as)) {
1387                         mutex_enter(&seg->s_as->a_contents);
1388                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1389                                 AS_CLRUNMAPWAIT(seg->s_as);
1390                                 cv_broadcast(&seg->s_as->a_cv);
1391                         }
1392                         mutex_exit(&seg->s_as->a_contents);
1393                 }
1394         }
1395 
1396 }
1397 
1398 /*
1399  * Handle fault for a single page.
1400  * Done in a separate routine so we can handle errors more easily.
1401  * This routine is called only from segdev_faultpages()
1402  * when looping over the range of addresses requested. The segment lock is held.
1403  */
1404 static faultcode_t
1405 segdev_faultpage(
1406         struct hat *hat,                /* the hat */
1407         struct seg *seg,                /* seg_dev of interest */
1408         caddr_t addr,                   /* address in as */
1409         struct vpage *vpage,            /* pointer to vpage for seg, addr */
1410         enum fault_type type,           /* type of fault */
1411         enum seg_rw rw,                 /* type of access at fault */
1412         devmap_handle_t *dhp)           /* devmap handle if any for this page */
1413 {
1414         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1415         uint_t prot;
1416         pfn_t pfnum = PFN_INVALID;
1417         u_offset_t offset;
1418         uint_t hat_flags;
1419         dev_info_t *dip;
1420 
1421         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1422             "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1423         DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1424             (void *)dhp, (void *)seg, (void *)addr));
1425 
1426         /*
1427          * Initialize protection value for this page.
1428          * If we have per page protection values check it now.
1429          */
1430         if (sdp->pageprot) {
1431                 uint_t protchk;
1432 
1433                 switch (rw) {
1434                 case S_READ:
1435                         protchk = PROT_READ;
1436                         break;
1437                 case S_WRITE:
1438                         protchk = PROT_WRITE;
1439                         break;
1440                 case S_EXEC:
1441                         protchk = PROT_EXEC;
1442                         break;
1443                 case S_OTHER:
1444                 default:
1445                         protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1446                         break;
1447                 }
1448 
1449                 prot = VPP_PROT(vpage);
1450                 if ((prot & protchk) == 0)
1451                         return (FC_PROT);       /* illegal access type */
1452         } else {
1453                 prot = sdp->prot;
1454                 /* caller has already done segment level protection check */
1455         }
1456 
1457         if (type == F_SOFTLOCK) {
1458                 mutex_enter(&freemem_lock);
1459                 sdp->softlockcnt++;
1460                 mutex_exit(&freemem_lock);
1461         }
1462 
1463         hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1464         offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1465         /*
1466          * In the devmap framework, sdp->mapfunc is set to NULL.  we can get
1467          * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1468          * seg->s_base.
1469          */
1470         if (dhp == NULL) {
1471                 /* If segment has devmap_data, then dhp should be non-NULL */
1472                 ASSERT(sdp->devmap_data == NULL);
1473                 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1474                     (off_t)offset, prot);
1475                 prot |= sdp->hat_attr;
1476         } else {
1477                 ulong_t off;
1478                 struct ddi_umem_cookie *cp;
1479                 struct devmap_pmem_cookie *pcp;
1480 
1481                 /* ensure the dhp passed in contains addr. */
1482                 ASSERT(dhp == devmap_find_handle(
1483                     (devmap_handle_t *)sdp->devmap_data, addr));
1484 
1485                 off = addr - dhp->dh_uvaddr;
1486 
1487                 /*
1488                  * This routine assumes that the caller makes sure that the
1489                  * fields in dhp used below are unchanged due to remap during
1490                  * this call. Caller does HOLD_DHP_LOCK if neeed
1491                  */
1492                 cp = dhp->dh_cookie;
1493                 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1494                         pfnum = PFN_INVALID;
1495                 } else if (cookie_is_devmem(cp)) {
1496                         pfnum = dhp->dh_pfn + btop(off);
1497                 } else if (cookie_is_pmem(cp)) {
1498                         pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1499                         ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1500                             dhp->dh_roff < ptob(pcp->dp_npages));
1501                         pfnum = page_pptonum(
1502                             pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1503                 } else {
1504                         ASSERT(dhp->dh_roff < cp->size);
1505                         ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1506                             dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1507                         ASSERT((dhp->dh_cvaddr + off) <=
1508                             (cp->cvaddr + cp->size));
1509                         ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1510                             (cp->cvaddr + cp->size));
1511 
1512                         switch (cp->type) {
1513                         case UMEM_LOCKED :
1514                                 if (cp->pparray != NULL) {
1515                                         ASSERT((dhp->dh_roff &
1516                                             PAGEOFFSET) == 0);
1517                                         pfnum = page_pptonum(
1518                                             cp->pparray[btop(off +
1519                                             dhp->dh_roff)]);
1520                                 } else {
1521                                         pfnum = hat_getpfnum(
1522                                             ((proc_t *)cp->procp)->p_as->a_hat,
1523                                             cp->cvaddr + off);
1524                                 }
1525                         break;
1526                         case UMEM_TRASH :
1527                                 pfnum = page_pptonum(trashpp);
1528                                 /*
1529                                  * We should set hat_flags to HAT_NOFAULT also
1530                                  * However, not all hat layers implement this
1531                                  */
1532                                 break;
1533                         case KMEM_PAGEABLE:
1534                         case KMEM_NON_PAGEABLE:
1535                                 pfnum = hat_getpfnum(kas.a_hat,
1536                                     dhp->dh_cvaddr + off);
1537                                 break;
1538                         default :
1539                                 pfnum = PFN_INVALID;
1540                                 break;
1541                         }
1542                 }
1543                 prot |= dhp->dh_hat_attr;
1544         }
1545         if (pfnum == PFN_INVALID) {
1546                 return (FC_MAKE_ERR(EFAULT));
1547         }
1548         /* prot should already be OR'ed in with hat_attributes if needed */
1549 
1550         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1551             "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1552             pfnum, pf_is_memory(pfnum), prot, hat_flags);
1553         DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1554             "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1555 
1556         if (pf_is_memory(pfnum) || (dhp != NULL)) {
1557                 /*
1558                  * It's not _really_ required here to pass sdp->hat_flags
1559                  * to hat_devload even though we do it.
1560                  * This is because hat figures it out DEVMEM mappings
1561                  * are non-consistent, anyway.
1562                  */
1563                 hat_devload(hat, addr, PAGESIZE, pfnum,
1564                     prot, hat_flags | sdp->hat_flags);
1565                 return (0);
1566         }
1567 
1568         /*
1569          * Fall through to the case where devmap is not used and need to call
1570          * up the device tree to set up the mapping
1571          */
1572 
1573         dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1574         ASSERT(dip);
1575 
1576         /*
1577          * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1578          * This is because this calls drivers which may not expect
1579          * prot to have any other values than PROT_ALL
1580          * The root nexus driver has a hack to peek into the segment
1581          * structure and then OR in sdp->hat_attr.
1582          * XX In case the bus_ops interfaces are ever revisited
1583          * we need to fix this. prot should include other hat attributes
1584          */
1585         if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1586             (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1587                 return (FC_MAKE_ERR(EFAULT));
1588         }
1589         return (0);
1590 }
1591 
1592 static faultcode_t
1593 segdev_fault(
1594         struct hat *hat,                /* the hat */
1595         struct seg *seg,                /* the seg_dev of interest */
1596         caddr_t addr,                   /* the address of the fault */
1597         size_t len,                     /* the length of the range */
1598         enum fault_type type,           /* type of fault */
1599         enum seg_rw rw)                 /* type of access at fault */
1600 {
1601         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1602         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1603         devmap_handle_t *dhp;
1604         struct devmap_softlock *slock = NULL;
1605         ulong_t slpage = 0;
1606         ulong_t off;
1607         caddr_t maddr = addr;
1608         int err;
1609         int err_is_faultcode = 0;
1610 
1611         TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1612             "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1613             (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1614         DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1615             "addr %p len %lx type %x\n",
1616             (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1617 
1618         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1619 
1620         /* Handle non-devmap case */
1621         if (dhp_head == NULL)
1622                 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1623 
1624         /* Find devmap handle */
1625         if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1626                 return (FC_NOMAP);
1627 
1628         /*
1629          * The seg_dev driver does not implement copy-on-write,
1630          * and always loads translations with maximal allowed permissions
1631          * but we got an fault trying to access the device.
1632          * Servicing the fault is not going to result in any better result
1633          * RFE: If we want devmap_access callbacks to be involved in F_PROT
1634          *      faults, then the code below is written for that
1635          *      Pending resolution of the following:
1636          *      - determine if the F_INVAL/F_SOFTLOCK syncing
1637          *      is needed for F_PROT also or not. The code below assumes it does
1638          *      - If driver sees F_PROT and calls devmap_load with same type,
1639          *      then segdev_faultpages will fail with FC_PROT anyway, need to
1640          *      change that so calls from devmap_load to segdev_faultpages for
1641          *      F_PROT type are retagged to F_INVAL.
1642          * RFE: Today we dont have drivers that use devmap and want to handle
1643          *      F_PROT calls. The code in segdev_fault* is written to allow
1644          *      this case but is not tested. A driver that needs this capability
1645          *      should be able to remove the short-circuit case; resolve the
1646          *      above issues and "should" work.
1647          */
1648         if (type == F_PROT) {
1649                 return (FC_PROT);
1650         }
1651 
1652         /*
1653          * Loop through dhp list calling devmap_access or segdev_faultpages for
1654          * each devmap handle.
1655          * drivers which implement devmap_access can interpose on faults and do
1656          * device-appropriate special actions before calling devmap_load.
1657          */
1658 
1659         /*
1660          * Unfortunately, this simple loop has turned out to expose a variety
1661          * of complex problems which results in the following convoluted code.
1662          *
1663          * First, a desire to handle a serialization of F_SOFTLOCK calls
1664          * to the driver within the framework.
1665          *      This results in a dh_softlock structure that is on a per device
1666          *      (or device instance) basis and serializes devmap_access calls.
1667          *      Ideally we would need to do this for underlying
1668          *      memory/device regions that are being faulted on
1669          *      but that is hard to identify and with REMAP, harder
1670          * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1671          *      to F_SOFTLOCK calls to the driver.
1672          * These serializations are to simplify the driver programmer model.
1673          * To support these two features, the code first goes through the
1674          *      devmap handles and counts the pages (slpage) that are covered
1675          *      by devmap_access callbacks.
1676          * This part ends with a devmap_softlock_enter call
1677          *      which allows only one F_SOFTLOCK active on a device instance,
1678          *      but multiple F_INVAL/F_PROTs can be active except when a
1679          *      F_SOFTLOCK is active
1680          *
1681          * Next, we dont short-circuit the fault code upfront to call
1682          *      segdev_softunlock for F_SOFTUNLOCK, because we must use
1683          *      the same length when we softlock and softunlock.
1684          *
1685          *      -Hat layers may not support softunlocking lengths less than the
1686          *      original length when there is large page support.
1687          *      -kpmem locking is dependent on keeping the lengths same.
1688          *      -if drivers handled F_SOFTLOCK, they probably also expect to
1689          *              see an F_SOFTUNLOCK of the same length
1690          *      Hence, if extending lengths during softlock,
1691          *      softunlock has to make the same adjustments and goes through
1692          *      the same loop calling segdev_faultpages/segdev_softunlock
1693          *      But some of the synchronization and error handling is different
1694          */
1695 
1696         if (type != F_SOFTUNLOCK) {
1697                 devmap_handle_t *dhpp = dhp;
1698                 size_t slen = len;
1699 
1700                 /*
1701                  * Calculate count of pages that are :
1702                  * a) within the (potentially extended) fault region
1703                  * b) AND covered by devmap handle with devmap_access
1704                  */
1705                 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1706                 while (slen != 0) {
1707                         size_t mlen;
1708 
1709                         /*
1710                          * Softlocking on a region that allows remap is
1711                          * unsupported due to unresolved locking issues
1712                          * XXX: unclear what these are?
1713                          *      One potential is that if there is a pending
1714                          *      softlock, then a remap should not be allowed
1715                          *      until the unlock is done. This is easily
1716                          *      fixed by returning error in devmap*remap on
1717                          *      checking the dh->dh_softlock->softlocked value
1718                          */
1719                         if ((type == F_SOFTLOCK) &&
1720                             (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1721                                 return (FC_NOSUPPORT);
1722                         }
1723 
1724                         mlen = MIN(slen, (dhpp->dh_len - off));
1725                         if (dhpp->dh_callbackops.devmap_access) {
1726                                 size_t llen;
1727                                 caddr_t laddr;
1728                                 /*
1729                                  * use extended length for large page mappings
1730                                  */
1731                                 HOLD_DHP_LOCK(dhpp);
1732                                 if ((sdp->pageprot == 0) &&
1733                                     (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1734                                         devmap_get_large_pgsize(dhpp,
1735                                             mlen, maddr, &llen, &laddr);
1736                                 } else {
1737                                         llen = mlen;
1738                                 }
1739                                 RELE_DHP_LOCK(dhpp);
1740 
1741                                 slpage += btopr(llen);
1742                                 slock = dhpp->dh_softlock;
1743                         }
1744                         maddr += mlen;
1745                         ASSERT(slen >= mlen);
1746                         slen -= mlen;
1747                         dhpp = dhpp->dh_next;
1748                         off = 0;
1749                 }
1750                 /*
1751                  * synchonize with other faulting threads and wait till safe
1752                  * devmap_softlock_enter might return due to signal in cv_wait
1753                  *
1754                  * devmap_softlock_enter has to be called outside of while loop
1755                  * to prevent a deadlock if len spans over multiple dhps.
1756                  * dh_softlock is based on device instance and if multiple dhps
1757                  * use the same device instance, the second dhp's LOCK call
1758                  * will hang waiting on the first to complete.
1759                  * devmap_setup verifies that slocks in a dhp_chain are same.
1760                  * RFE: this deadlock only hold true for F_SOFTLOCK. For
1761                  *      F_INVAL/F_PROT, since we now allow multiple in parallel,
1762                  *      we could have done the softlock_enter inside the loop
1763                  *      and supported multi-dhp mappings with dissimilar devices
1764                  */
1765                 if (err = devmap_softlock_enter(slock, slpage, type))
1766                         return (FC_MAKE_ERR(err));
1767         }
1768 
1769         /* reset 'maddr' to the start addr of the range of fault. */
1770         maddr = addr;
1771 
1772         /* calculate the offset corresponds to 'addr' in the first dhp. */
1773         off = (ulong_t)(addr - dhp->dh_uvaddr);
1774 
1775         /*
1776          * The fault length may span over multiple dhps.
1777          * Loop until the total length is satisfied.
1778          */
1779         while (len != 0) {
1780                 size_t llen;
1781                 size_t mlen;
1782                 caddr_t laddr;
1783 
1784                 /*
1785                  * mlen is the smaller of 'len' and the length
1786                  * from addr to the end of mapping defined by dhp.
1787                  */
1788                 mlen = MIN(len, (dhp->dh_len - off));
1789 
1790                 HOLD_DHP_LOCK(dhp);
1791                 /*
1792                  * Pass the extended length and address to devmap_access
1793                  * if large pagesize is used for loading address translations.
1794                  */
1795                 if ((sdp->pageprot == 0) &&
1796                     (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1797                         devmap_get_large_pgsize(dhp, mlen, maddr,
1798                             &llen, &laddr);
1799                         ASSERT(maddr == addr || laddr == maddr);
1800                 } else {
1801                         llen = mlen;
1802                         laddr = maddr;
1803                 }
1804 
1805                 if (dhp->dh_callbackops.devmap_access != NULL) {
1806                         offset_t aoff;
1807 
1808                         aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1809 
1810                         /*
1811                          * call driver's devmap_access entry point which will
1812                          * call devmap_load/contextmgmt to load the translations
1813                          *
1814                          * We drop the dhp_lock before calling access so
1815                          * drivers can call devmap_*_remap within access
1816                          */
1817                         RELE_DHP_LOCK(dhp);
1818 
1819                         err = (*dhp->dh_callbackops.devmap_access)(
1820                             dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1821                 } else {
1822                         /*
1823                          * If no devmap_access entry point, then load mappings
1824                          * hold dhp_lock across faultpages if REMAP
1825                          */
1826                         err = segdev_faultpages(hat, seg, laddr, llen,
1827                             type, rw, dhp);
1828                         err_is_faultcode = 1;
1829                         RELE_DHP_LOCK(dhp);
1830                 }
1831 
1832                 if (err) {
1833                         if ((type == F_SOFTLOCK) && (maddr > addr)) {
1834                                 /*
1835                                  * If not first dhp, use
1836                                  * segdev_fault(F_SOFTUNLOCK) for prior dhps
1837                                  * While this is recursion, it is incorrect to
1838                                  * call just segdev_softunlock
1839                                  * if we are using either large pages
1840                                  * or devmap_access. It will be more right
1841                                  * to go through the same loop as above
1842                                  * rather than call segdev_softunlock directly
1843                                  * It will use the right lenghths as well as
1844                                  * call into the driver devmap_access routines.
1845                                  */
1846                                 size_t done = (size_t)(maddr - addr);
1847                                 (void) segdev_fault(hat, seg, addr, done,
1848                                     F_SOFTUNLOCK, S_OTHER);
1849                                 /*
1850                                  * reduce slpage by number of pages
1851                                  * released by segdev_softunlock
1852                                  */
1853                                 ASSERT(slpage >= btopr(done));
1854                                 devmap_softlock_exit(slock,
1855                                     slpage - btopr(done), type);
1856                         } else {
1857                                 devmap_softlock_exit(slock, slpage, type);
1858                         }
1859 
1860 
1861                         /*
1862                          * Segdev_faultpages() already returns a faultcode,
1863                          * hence, result from segdev_faultpages() should be
1864                          * returned directly.
1865                          */
1866                         if (err_is_faultcode)
1867                                 return (err);
1868                         return (FC_MAKE_ERR(err));
1869                 }
1870 
1871                 maddr += mlen;
1872                 ASSERT(len >= mlen);
1873                 len -= mlen;
1874                 dhp = dhp->dh_next;
1875                 off = 0;
1876 
1877                 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1878         }
1879         /*
1880          * release the softlock count at end of fault
1881          * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1882          */
1883         if ((type == F_INVAL) || (type == F_PROT))
1884                 devmap_softlock_exit(slock, slpage, type);
1885         return (0);
1886 }
1887 
1888 /*
1889  * segdev_faultpages
1890  *
1891  * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1892  * This routine assumes that the callers makes sure that the fields
1893  * in dhp used below are not changed due to remap during this call.
1894  * Caller does HOLD_DHP_LOCK if neeed
1895  * This routine returns a faultcode_t as a return value for segdev_fault.
1896  */
1897 static faultcode_t
1898 segdev_faultpages(
1899         struct hat *hat,                /* the hat */
1900         struct seg *seg,                /* the seg_dev of interest */
1901         caddr_t addr,                   /* the address of the fault */
1902         size_t len,                     /* the length of the range */
1903         enum fault_type type,           /* type of fault */
1904         enum seg_rw rw,                 /* type of access at fault */
1905         devmap_handle_t *dhp)           /* devmap handle */
1906 {
1907         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1908         register caddr_t a;
1909         struct vpage *vpage;
1910         struct ddi_umem_cookie *kpmem_cookie = NULL;
1911         int err;
1912 
1913         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1914             "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1915             (void *)dhp, (void *)seg, (void *)addr, len);
1916         DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1917             "dhp %p seg %p addr %p len %lx\n",
1918             (void *)dhp, (void *)seg, (void *)addr, len));
1919 
1920         /*
1921          * The seg_dev driver does not implement copy-on-write,
1922          * and always loads translations with maximal allowed permissions
1923          * but we got an fault trying to access the device.
1924          * Servicing the fault is not going to result in any better result
1925          * XXX: If we want to allow devmap_access to handle F_PROT calls,
1926          * This code should be removed and let the normal fault handling
1927          * take care of finding the error
1928          */
1929         if (type == F_PROT) {
1930                 return (FC_PROT);
1931         }
1932 
1933         if (type == F_SOFTUNLOCK) {
1934                 segdev_softunlock(hat, seg, addr, len, rw);
1935                 return (0);
1936         }
1937 
1938         /*
1939          * For kernel pageable memory, fault/lock segkp pages
1940          * We hold this until the completion of this
1941          * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1942          */
1943         if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1944                 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1945                 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1946                         return (err);
1947         }
1948 
1949         /*
1950          * If we have the same protections for the entire segment,
1951          * insure that the access being attempted is legitimate.
1952          */
1953         rw_enter(&sdp->lock, RW_READER);
1954         if (sdp->pageprot == 0) {
1955                 uint_t protchk;
1956 
1957                 switch (rw) {
1958                 case S_READ:
1959                         protchk = PROT_READ;
1960                         break;
1961                 case S_WRITE:
1962                         protchk = PROT_WRITE;
1963                         break;
1964                 case S_EXEC:
1965                         protchk = PROT_EXEC;
1966                         break;
1967                 case S_OTHER:
1968                 default:
1969                         protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1970                         break;
1971                 }
1972 
1973                 if ((sdp->prot & protchk) == 0) {
1974                         rw_exit(&sdp->lock);
1975                         /* undo kpmem locking */
1976                         if (kpmem_cookie != NULL) {
1977                                 release_kpmem_lock(kpmem_cookie, btopr(len));
1978                         }
1979                         return (FC_PROT);       /* illegal access type */
1980                 }
1981         }
1982 
1983         /*
1984          * we do a single hat_devload for the range if
1985          *   - devmap framework (dhp is not NULL),
1986          *   - pageprot == 0, i.e., no per-page protection set and
1987          *   - is device pages, irrespective of whether we are using large pages
1988          */
1989         if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1990                 pfn_t pfnum;
1991                 uint_t hat_flags;
1992 
1993                 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1994                         rw_exit(&sdp->lock);
1995                         return (FC_NOMAP);
1996                 }
1997 
1998                 if (type == F_SOFTLOCK) {
1999                         mutex_enter(&freemem_lock);
2000                         sdp->softlockcnt += btopr(len);
2001                         mutex_exit(&freemem_lock);
2002                 }
2003 
2004                 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2005                 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2006                 ASSERT(!pf_is_memory(pfnum));
2007 
2008                 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2009                     hat_flags | sdp->hat_flags);
2010                 rw_exit(&sdp->lock);
2011                 return (0);
2012         }
2013 
2014         /* Handle cases where we have to loop through fault handling per-page */
2015 
2016         if (sdp->vpage == NULL)
2017                 vpage = NULL;
2018         else
2019                 vpage = &sdp->vpage[seg_page(seg, addr)];
2020 
2021         /* loop over the address range handling each fault */
2022         for (a = addr; a < addr + len; a += PAGESIZE) {
2023                 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2024                         break;
2025                 }
2026                 if (vpage != NULL)
2027                         vpage++;
2028         }
2029         rw_exit(&sdp->lock);
2030         if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2031                 size_t done = (size_t)(a - addr); /* pages fault successfully */
2032                 if (done > 0) {
2033                         /* use softunlock for those pages */
2034                         segdev_softunlock(hat, seg, addr, done, S_OTHER);
2035                 }
2036                 if (kpmem_cookie != NULL) {
2037                         /* release kpmem lock for rest of pages */
2038                         ASSERT(len >= done);
2039                         release_kpmem_lock(kpmem_cookie, btopr(len - done));
2040                 }
2041         } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2042                 /* for non-SOFTLOCK cases, release kpmem */
2043                 release_kpmem_lock(kpmem_cookie, btopr(len));
2044         }
2045         return (err);
2046 }
2047 
2048 /*
2049  * Asynchronous page fault.  We simply do nothing since this
2050  * entry point is not supposed to load up the translation.
2051  */
2052 /*ARGSUSED*/
2053 static faultcode_t
2054 segdev_faulta(struct seg *seg, caddr_t addr)
2055 {
2056         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2057             "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2058         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2059 
2060         return (0);
2061 }
2062 
2063 static int
2064 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2065 {
2066         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2067         register devmap_handle_t *dhp;
2068         register struct vpage *vp, *evp;
2069         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2070         ulong_t off;
2071         size_t mlen, sz;
2072 
2073         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2074             "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2075             (void *)seg, (void *)addr, len, prot);
2076         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2077 
2078         if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2079                 /*
2080                  * Fail the setprot if pages are SOFTLOCKed through this
2081                  * mapping.
2082                  * Softlockcnt is protected from change by the as read lock.
2083                  */
2084                 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2085                     "segdev_setprot:error softlockcnt=%lx", sz);
2086                 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2087                 return (EAGAIN);
2088         }
2089 
2090         if (dhp_head != NULL) {
2091                 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2092                         return (EINVAL);
2093 
2094                 /*
2095                  * check if violate maxprot.
2096                  */
2097                 off = (ulong_t)(addr - dhp->dh_uvaddr);
2098                 mlen  = len;
2099                 while (dhp) {
2100                         if ((dhp->dh_maxprot & prot) != prot)
2101                                 return (EACCES);        /* violated maxprot */
2102 
2103                         if (mlen > (dhp->dh_len - off)) {
2104                                 mlen -= dhp->dh_len - off;
2105                                 dhp = dhp->dh_next;
2106                                 off = 0;
2107                         } else
2108                                 break;
2109                 }
2110         } else {
2111                 if ((sdp->maxprot & prot) != prot)
2112                         return (EACCES);
2113         }
2114 
2115         rw_enter(&sdp->lock, RW_WRITER);
2116         if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2117                 if (sdp->prot == prot) {
2118                         rw_exit(&sdp->lock);
2119                         return (0);                     /* all done */
2120                 }
2121                 sdp->prot = (uchar_t)prot;
2122         } else {
2123                 sdp->pageprot = 1;
2124                 if (sdp->vpage == NULL) {
2125                         /*
2126                          * First time through setting per page permissions,
2127                          * initialize all the vpage structures to prot
2128                          */
2129                         sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2130                             KM_SLEEP);
2131                         evp = &sdp->vpage[seg_pages(seg)];
2132                         for (vp = sdp->vpage; vp < evp; vp++)
2133                                 VPP_SETPROT(vp, sdp->prot);
2134                 }
2135                 /*
2136                  * Now go change the needed vpages protections.
2137                  */
2138                 evp = &sdp->vpage[seg_page(seg, addr + len)];
2139                 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2140                         VPP_SETPROT(vp, prot);
2141         }
2142         rw_exit(&sdp->lock);
2143 
2144         if (dhp_head != NULL) {
2145                 devmap_handle_t *tdhp;
2146                 /*
2147                  * If large page size was used in hat_devload(),
2148                  * the same page size must be used in hat_unload().
2149                  */
2150                 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2151                 while (tdhp != NULL) {
2152                         if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2153                                 break;
2154                         }
2155                         tdhp = tdhp->dh_next;
2156                 }
2157                 if (tdhp) {
2158                         size_t slen = len;
2159                         size_t mlen;
2160                         size_t soff;
2161 
2162                         soff = (ulong_t)(addr - dhp->dh_uvaddr);
2163                         while (slen != 0) {
2164                                 mlen = MIN(slen, (dhp->dh_len - soff));
2165                                 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2166                                     dhp->dh_len, HAT_UNLOAD);
2167                                 dhp = dhp->dh_next;
2168                                 ASSERT(slen >= mlen);
2169                                 slen -= mlen;
2170                                 soff = 0;
2171                         }
2172                         return (0);
2173                 }
2174         }
2175 
2176         if ((prot & ~PROT_USER) == PROT_NONE) {
2177                 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2178         } else {
2179                 /*
2180                  * RFE: the segment should keep track of all attributes
2181                  * allowing us to remove the deprecated hat_chgprot
2182                  * and use hat_chgattr.
2183                  */
2184                 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2185         }
2186 
2187         return (0);
2188 }
2189 
2190 static int
2191 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2192 {
2193         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2194         struct vpage *vp, *evp;
2195 
2196         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2197             "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2198             (void *)seg, (void *)addr, len, prot);
2199         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2200 
2201         /*
2202          * If segment protection can be used, simply check against them
2203          */
2204         rw_enter(&sdp->lock, RW_READER);
2205         if (sdp->pageprot == 0) {
2206                 register int err;
2207 
2208                 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2209                 rw_exit(&sdp->lock);
2210                 return (err);
2211         }
2212 
2213         /*
2214          * Have to check down to the vpage level
2215          */
2216         evp = &sdp->vpage[seg_page(seg, addr + len)];
2217         for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2218                 if ((VPP_PROT(vp) & prot) != prot) {
2219                         rw_exit(&sdp->lock);
2220                         return (EACCES);
2221                 }
2222         }
2223         rw_exit(&sdp->lock);
2224         return (0);
2225 }
2226 
2227 static int
2228 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2229 {
2230         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2231         size_t pgno;
2232 
2233         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2234             "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2235             (void *)seg, (void *)addr, len, (void *)protv);
2236         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2237 
2238         pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2239         if (pgno != 0) {
2240                 rw_enter(&sdp->lock, RW_READER);
2241                 if (sdp->pageprot == 0) {
2242                         do {
2243                                 protv[--pgno] = sdp->prot;
2244                         } while (pgno != 0);
2245                 } else {
2246                         size_t pgoff = seg_page(seg, addr);
2247 
2248                         do {
2249                                 pgno--;
2250                                 protv[pgno] =
2251                                     VPP_PROT(&sdp->vpage[pgno + pgoff]);
2252                         } while (pgno != 0);
2253                 }
2254                 rw_exit(&sdp->lock);
2255         }
2256         return (0);
2257 }
2258 
2259 static u_offset_t
2260 segdev_getoffset(register struct seg *seg, caddr_t addr)
2261 {
2262         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2263 
2264         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2265             "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2266 
2267         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2268 
2269         return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2270 }
2271 
2272 /*ARGSUSED*/
2273 static int
2274 segdev_gettype(register struct seg *seg, caddr_t addr)
2275 {
2276         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2277 
2278         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2279             "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2280 
2281         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2282 
2283         return (sdp->type);
2284 }
2285 
2286 
2287 /*ARGSUSED*/
2288 static int
2289 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2290 {
2291         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2292 
2293         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2294             "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2295 
2296         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2297 
2298         /*
2299          * Note that this vp is the common_vp of the device, where the
2300          * pages are hung ..
2301          */
2302         *vpp = VTOCVP(sdp->vp);
2303 
2304         return (0);
2305 }
2306 
2307 static void
2308 segdev_badop(void)
2309 {
2310         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2311             "segdev_badop:start");
2312         panic("segdev_badop");
2313         /*NOTREACHED*/
2314 }
2315 
2316 /*
2317  * segdev pages are not in the cache, and thus can't really be controlled.
2318  * Hence, syncs are simply always successful.
2319  */
2320 /*ARGSUSED*/
2321 static int
2322 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2323 {
2324         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2325 
2326         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2327 
2328         return (0);
2329 }
2330 
2331 /*
2332  * segdev pages are always "in core".
2333  */
2334 /*ARGSUSED*/
2335 static size_t
2336 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2337 {
2338         size_t v = 0;
2339 
2340         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2341 
2342         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2343 
2344         for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2345             v += PAGESIZE)
2346                 *vec++ = 1;
2347         return (v);
2348 }
2349 
2350 /*
2351  * segdev pages are not in the cache, and thus can't really be controlled.
2352  * Hence, locks are simply always successful.
2353  */
2354 /*ARGSUSED*/
2355 static int
2356 segdev_lockop(struct seg *seg, caddr_t addr,
2357     size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2358 {
2359         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2360 
2361         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2362 
2363         return (0);
2364 }
2365 
2366 /*
2367  * segdev pages are not in the cache, and thus can't really be controlled.
2368  * Hence, advise is simply always successful.
2369  */
2370 /*ARGSUSED*/
2371 static int
2372 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2373 {
2374         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2375 
2376         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2377 
2378         return (0);
2379 }
2380 
2381 /*
2382  * segdev pages are not dumped, so we just return
2383  */
2384 /*ARGSUSED*/
2385 static void
2386 segdev_dump(struct seg *seg)
2387 {}
2388 
2389 /*
2390  * ddi_segmap_setup:    Used by drivers who wish specify mapping attributes
2391  *                      for a segment.  Called from a drivers segmap(9E)
2392  *                      routine.
2393  */
2394 /*ARGSUSED*/
2395 int
2396 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2397     off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2398     ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2399 {
2400         struct segdev_crargs dev_a;
2401         int (*mapfunc)(dev_t dev, off_t off, int prot);
2402         uint_t hat_attr;
2403         pfn_t pfn;
2404         int     error, i;
2405 
2406         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2407             "ddi_segmap_setup:start");
2408 
2409         if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2410                 return (ENODEV);
2411 
2412         /*
2413          * Character devices that support the d_mmap
2414          * interface can only be mmap'ed shared.
2415          */
2416         if ((flags & MAP_TYPE) != MAP_SHARED)
2417                 return (EINVAL);
2418 
2419         /*
2420          * Check that this region is indeed mappable on this platform.
2421          * Use the mapping function.
2422          */
2423         if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2424                 return (ENXIO);
2425 
2426         /*
2427          * Check to ensure that the entire range is
2428          * legal and we are not trying to map in
2429          * more than the device will let us.
2430          */
2431         for (i = 0; i < len; i += PAGESIZE) {
2432                 if (i == 0) {
2433                         /*
2434                          * Save the pfn at offset here. This pfn will be
2435                          * used later to get user address.
2436                          */
2437                         if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2438                             maxprot)) == PFN_INVALID)
2439                                 return (ENXIO);
2440                 } else {
2441                         if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2442                             PFN_INVALID)
2443                                 return (ENXIO);
2444                 }
2445         }
2446 
2447         as_rangelock(as);
2448         /* Pick an address w/o worrying about any vac alignment constraints. */
2449         error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2450         if (error != 0) {
2451                 as_rangeunlock(as);
2452                 return (error);
2453         }
2454 
2455         dev_a.mapfunc = mapfunc;
2456         dev_a.dev = dev;
2457         dev_a.offset = (offset_t)offset;
2458         dev_a.type = flags & MAP_TYPE;
2459         dev_a.prot = (uchar_t)prot;
2460         dev_a.maxprot = (uchar_t)maxprot;
2461         dev_a.hat_attr = hat_attr;
2462         dev_a.hat_flags = 0;
2463         dev_a.devmap_data = NULL;
2464 
2465         error = as_map(as, *addrp, len, segdev_create, &dev_a);
2466         as_rangeunlock(as);
2467         return (error);
2468 
2469 }
2470 
2471 /*ARGSUSED*/
2472 static int
2473 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2474     struct page ***ppp, enum lock_type type, enum seg_rw rw)
2475 {
2476         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2477             "segdev_pagelock:start");
2478         return (ENOTSUP);
2479 }
2480 
2481 /*ARGSUSED*/
2482 static int
2483 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2484     uint_t szc)
2485 {
2486         return (ENOTSUP);
2487 }
2488 
2489 /*
2490  * devmap_device: Used by devmap framework to establish mapping
2491  *                called by devmap_seup(9F) during map setup time.
2492  */
2493 /*ARGSUSED*/
2494 static int
2495 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2496     offset_t off, size_t len, uint_t flags)
2497 {
2498         devmap_handle_t *rdhp, *maxdhp;
2499         struct segdev_crargs dev_a;
2500         int     err;
2501         uint_t maxprot = PROT_ALL;
2502         offset_t offset = 0;
2503         pfn_t pfn;
2504         struct devmap_pmem_cookie *pcp;
2505 
2506         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2507             "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2508             (void *)dhp, (void *)addr, off, len);
2509 
2510         DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2511             (void *)dhp, (void *)addr, off, len));
2512 
2513         as_rangelock(as);
2514         if ((flags & MAP_FIXED) == 0) {
2515                 offset_t aligned_off;
2516 
2517                 rdhp = maxdhp = dhp;
2518                 while (rdhp != NULL) {
2519                         maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2520                             maxdhp : rdhp;
2521                         rdhp = rdhp->dh_next;
2522                         maxprot |= dhp->dh_maxprot;
2523                 }
2524                 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2525 
2526                 /*
2527                  * Use the dhp that has the
2528                  * largest len to get user address.
2529                  */
2530                 /*
2531                  * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2532                  * use 0 which is as good as any other.
2533                  */
2534                 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2535                         aligned_off = (offset_t)0;
2536                 } else if (dhp_is_devmem(maxdhp)) {
2537                         aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2538                 } else if (dhp_is_pmem(maxdhp)) {
2539                         pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2540                         pfn = page_pptonum(
2541                             pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2542                         aligned_off = (offset_t)ptob(pfn) - offset;
2543                 } else {
2544                         aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2545                             offset;
2546                 }
2547 
2548                 /*
2549                  * Pick an address aligned to dh_cookie.
2550                  * for kernel memory/user memory, cookie is cvaddr.
2551                  * for device memory, cookie is physical address.
2552                  */
2553                 map_addr(addr, len, aligned_off, 1, flags);
2554                 if (*addr == NULL) {
2555                         as_rangeunlock(as);
2556                         return (ENOMEM);
2557                 }
2558         } else {
2559                 /*
2560                  * User-specified address; blow away any previous mappings.
2561                  */
2562                 (void) as_unmap(as, *addr, len);
2563         }
2564 
2565         dev_a.mapfunc = NULL;
2566         dev_a.dev = dhp->dh_dev;
2567         dev_a.type = flags & MAP_TYPE;
2568         dev_a.offset = off;
2569         /*
2570          * sdp->maxprot has the least restrict protection of all dhps.
2571          */
2572         dev_a.maxprot = maxprot;
2573         dev_a.prot = dhp->dh_prot;
2574         /*
2575          * devmap uses dhp->dh_hat_attr for hat.
2576          */
2577         dev_a.hat_flags = 0;
2578         dev_a.hat_attr = 0;
2579         dev_a.devmap_data = (void *)dhp;
2580 
2581         err = as_map(as, *addr, len, segdev_create, &dev_a);
2582         as_rangeunlock(as);
2583         return (err);
2584 }
2585 
2586 int
2587 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2588     uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2589     size_t, uint_t, uint_t))
2590 {
2591         register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2592         struct devmap_ctx *devctx;
2593         int do_timeout = 0;
2594         int ret;
2595 
2596 #ifdef lint
2597         pvtp = pvtp;
2598 #endif
2599 
2600         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2601             "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2602             (void *)dhp, off, len);
2603         DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2604             (void *)dhp, off, len));
2605 
2606         if (ctxmgt == NULL)
2607                 return (FC_HWERR);
2608 
2609         devctx = dhp->dh_ctx;
2610 
2611         /*
2612          * If we are on an MP system with more than one cpu running
2613          * and if a thread on some CPU already has the context, wait
2614          * for it to finish if there is a hysteresis timeout.
2615          *
2616          * We call cv_wait() instead of cv_wait_sig() because
2617          * it does not matter much if it returned due to a signal
2618          * or due to a cv_signal() or cv_broadcast().  In either event
2619          * we need to complete the mapping otherwise the processes
2620          * will die with a SEGV.
2621          */
2622         if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2623                 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2624                     "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2625                     devctx, dhp);
2626                 do_timeout = 1;
2627                 mutex_enter(&devctx->lock);
2628                 while (devctx->oncpu)
2629                         cv_wait(&devctx->cv, &devctx->lock);
2630                 devctx->oncpu = 1;
2631                 mutex_exit(&devctx->lock);
2632         }
2633 
2634         /*
2635          * Call the contextmgt callback so that the driver can handle
2636          * the fault.
2637          */
2638         ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2639 
2640         /*
2641          * If devmap_access() returned -1, then there was a hardware
2642          * error so we need to convert the return value to something
2643          * that trap() will understand.  Otherwise, the return value
2644          * is already a fault code generated by devmap_unload()
2645          * or devmap_load().
2646          */
2647         if (ret) {
2648                 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2649                     "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2650                     ret, dhp, devctx);
2651                 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2652                     ret, (void *)dhp));
2653                 if (devctx->oncpu) {
2654                         mutex_enter(&devctx->lock);
2655                         devctx->oncpu = 0;
2656                         cv_signal(&devctx->cv);
2657                         mutex_exit(&devctx->lock);
2658                 }
2659                 return (FC_HWERR);
2660         }
2661 
2662         /*
2663          * Setup the timeout if we need to
2664          */
2665         if (do_timeout) {
2666                 mutex_enter(&devctx->lock);
2667                 if (dhp->dh_timeout_length > 0) {
2668                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2669                             "devmap_do_ctxmgt:timeout set");
2670                         devctx->timeout = timeout(devmap_ctxto,
2671                             devctx, dhp->dh_timeout_length);
2672                 } else {
2673                         /*
2674                          * We don't want to wait so set oncpu to
2675                          * 0 and wake up anyone waiting.
2676                          */
2677                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2678                             "devmap_do_ctxmgt:timeout not set");
2679                         devctx->oncpu = 0;
2680                         cv_signal(&devctx->cv);
2681                 }
2682                 mutex_exit(&devctx->lock);
2683         }
2684 
2685         return (DDI_SUCCESS);
2686 }
2687 
2688 /*
2689  *                                       end of mapping
2690  *                    poff   fault_offset         |
2691  *            base     |        |                 |
2692  *              |      |        |                 |
2693  *              V      V        V                 V
2694  *  +-----------+---------------+-------+---------+-------+
2695  *              ^               ^       ^         ^
2696  *              |<--- offset--->|<-len->|         |
2697  *              |<--- dh_len(size of mapping) --->|
2698  *                     |<--  pg -->|
2699  *                              -->|rlen|<--
2700  */
2701 static ulong_t
2702 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2703     ulong_t *opfn, ulong_t *pagesize)
2704 {
2705         register int level;
2706         ulong_t pg;
2707         ulong_t poff;
2708         ulong_t base;
2709         caddr_t uvaddr;
2710         long rlen;
2711 
2712         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2713             "devmap_roundup:start dhp=%p off=%lx len=%lx",
2714             (void *)dhp, offset, len);
2715         DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2716             (void *)dhp, offset, len));
2717 
2718         /*
2719          * get the max. pagesize that is aligned within the range
2720          * <dh_pfn, dh_pfn+offset>.
2721          *
2722          * The calculations below use physical address to ddetermine
2723          * the page size to use. The same calculations can use the
2724          * virtual address to determine the page size.
2725          */
2726         base = (ulong_t)ptob(dhp->dh_pfn);
2727         for (level = dhp->dh_mmulevel; level >= 0; level--) {
2728                 pg = page_get_pagesize(level);
2729                 poff = ((base + offset) & ~(pg - 1));
2730                 uvaddr = dhp->dh_uvaddr + (poff - base);
2731                 if ((poff >= base) &&
2732                     ((poff + pg) <= (base + dhp->dh_len)) &&
2733                     VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2734                         break;
2735         }
2736 
2737         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2738             "devmap_roundup: base=%lx poff=%lx dhp=%p",
2739             base, poff, dhp);
2740         DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2741             base, poff, dhp->dh_pfn));
2742 
2743         ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2744         ASSERT(level >= 0);
2745 
2746         *pagesize = pg;
2747         *opfn = dhp->dh_pfn + btop(poff - base);
2748 
2749         rlen = len + offset - (poff - base + pg);
2750 
2751         ASSERT(rlen < (long)len);
2752 
2753         TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2754             "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2755             (void *)dhp, level, rlen, pagesize, opfn);
2756         DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2757             "level %x rlen %lx psize %lx opfn %lx\n",
2758             (void *)dhp, level, rlen, *pagesize, *opfn));
2759 
2760         return ((ulong_t)((rlen > 0) ? rlen : 0));
2761 }
2762 
2763 /*
2764  * find the dhp that contains addr.
2765  */
2766 static devmap_handle_t *
2767 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2768 {
2769         devmap_handle_t *dhp;
2770 
2771         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2772             "devmap_find_handle:start");
2773 
2774         dhp = dhp_head;
2775         while (dhp) {
2776                 if (addr >= dhp->dh_uvaddr &&
2777                     addr < (dhp->dh_uvaddr + dhp->dh_len))
2778                         return (dhp);
2779                 dhp = dhp->dh_next;
2780         }
2781 
2782         return ((devmap_handle_t *)NULL);
2783 }
2784 
2785 /*
2786  * devmap_unload:
2787  *                      Marks a segdev segment or pages if offset->offset+len
2788  *                      is not the entire segment as intercept and unloads the
2789  *                      pages in the range offset -> offset+len.
2790  */
2791 int
2792 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2793 {
2794         register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2795         caddr_t addr;
2796         ulong_t size;
2797         ssize_t soff;
2798 
2799         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2800             "devmap_unload:start dhp=%p offset=%llx len=%lx",
2801             (void *)dhp, offset, len);
2802         DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2803             (void *)dhp, offset, len));
2804 
2805         soff = (ssize_t)(offset - dhp->dh_uoff);
2806         soff = round_down_p2(soff, PAGESIZE);
2807         if (soff < 0 || soff >= dhp->dh_len)
2808                 return (FC_MAKE_ERR(EINVAL));
2809 
2810         /*
2811          * Address and size must be page aligned.  Len is set to the
2812          * number of bytes in the number of pages that are required to
2813          * support len.  Offset is set to the byte offset of the first byte
2814          * of the page that contains offset.
2815          */
2816         len = round_up_p2(len, PAGESIZE);
2817 
2818         /*
2819          * If len is == 0, then calculate the size by getting
2820          * the number of bytes from offset to the end of the segment.
2821          */
2822         if (len == 0)
2823                 size = dhp->dh_len - soff;
2824         else {
2825                 size = len;
2826                 if ((soff + size) > dhp->dh_len)
2827                         return (FC_MAKE_ERR(EINVAL));
2828         }
2829 
2830         /*
2831          * The address is offset bytes from the base address of
2832          * the dhp.
2833          */
2834         addr = (caddr_t)(soff + dhp->dh_uvaddr);
2835 
2836         /*
2837          * If large page size was used in hat_devload(),
2838          * the same page size must be used in hat_unload().
2839          */
2840         if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2841                 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2842                     dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2843         } else {
2844                 hat_unload(dhp->dh_seg->s_as->a_hat,  addr, size,
2845                     HAT_UNLOAD|HAT_UNLOAD_OTHER);
2846         }
2847 
2848         return (0);
2849 }
2850 
2851 /*
2852  * calculates the optimal page size that will be used for hat_devload().
2853  */
2854 static void
2855 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2856     size_t *llen, caddr_t *laddr)
2857 {
2858         ulong_t off;
2859         ulong_t pfn;
2860         ulong_t pgsize;
2861         uint_t first = 1;
2862 
2863         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2864             "devmap_get_large_pgsize:start");
2865 
2866         /*
2867          * RFE - Code only supports large page mappings for devmem
2868          * This code could be changed in future if we want to support
2869          * large page mappings for kernel exported memory.
2870          */
2871         ASSERT(dhp_is_devmem(dhp));
2872         ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2873 
2874         *llen = 0;
2875         off = (ulong_t)(addr - dhp->dh_uvaddr);
2876         while ((long)len > 0) {
2877                 /*
2878                  * get the optimal pfn to minimize address translations.
2879                  * devmap_roundup() returns residue bytes for next round
2880                  * calculations.
2881                  */
2882                 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2883 
2884                 if (first) {
2885                         *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2886                         first = 0;
2887                 }
2888 
2889                 *llen += pgsize;
2890                 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2891         }
2892         /* Large page mapping len/addr cover more range than original fault */
2893         ASSERT(*llen >= len && *laddr <= addr);
2894         ASSERT((*laddr + *llen) >= (addr + len));
2895 }
2896 
2897 /*
2898  * Initialize the devmap_softlock structure.
2899  */
2900 static struct devmap_softlock *
2901 devmap_softlock_init(dev_t dev, ulong_t id)
2902 {
2903         struct devmap_softlock *slock;
2904         struct devmap_softlock *tmp;
2905 
2906         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2907             "devmap_softlock_init:start");
2908 
2909         tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2910         mutex_enter(&devmap_slock);
2911 
2912         for (slock = devmap_slist; slock != NULL; slock = slock->next)
2913                 if ((slock->dev == dev) && (slock->id == id))
2914                         break;
2915 
2916         if (slock == NULL) {
2917                 slock = tmp;
2918                 slock->dev = dev;
2919                 slock->id = id;
2920                 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2921                 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2922                 slock->next = devmap_slist;
2923                 devmap_slist = slock;
2924         } else
2925                 kmem_free(tmp, sizeof (struct devmap_softlock));
2926 
2927         mutex_enter(&slock->lock);
2928         slock->refcnt++;
2929         mutex_exit(&slock->lock);
2930         mutex_exit(&devmap_slock);
2931 
2932         return (slock);
2933 }
2934 
2935 /*
2936  * Wake up processes that sleep on softlocked.
2937  * Free dh_softlock if refcnt is 0.
2938  */
2939 static void
2940 devmap_softlock_rele(devmap_handle_t *dhp)
2941 {
2942         struct devmap_softlock *slock = dhp->dh_softlock;
2943         struct devmap_softlock *tmp;
2944         struct devmap_softlock *parent;
2945 
2946         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2947             "devmap_softlock_rele:start");
2948 
2949         mutex_enter(&devmap_slock);
2950         mutex_enter(&slock->lock);
2951 
2952         ASSERT(slock->refcnt > 0);
2953 
2954         slock->refcnt--;
2955 
2956         /*
2957          * If no one is using the device, free up the slock data.
2958          */
2959         if (slock->refcnt == 0) {
2960                 slock->softlocked = 0;
2961                 cv_signal(&slock->cv);
2962 
2963                 if (devmap_slist == slock)
2964                         devmap_slist = slock->next;
2965                 else {
2966                         parent = devmap_slist;
2967                         for (tmp = devmap_slist->next; tmp != NULL;
2968                             tmp = tmp->next) {
2969                                 if (tmp == slock) {
2970                                         parent->next = tmp->next;
2971                                         break;
2972                                 }
2973                                 parent = tmp;
2974                         }
2975                 }
2976                 mutex_exit(&slock->lock);
2977                 mutex_destroy(&slock->lock);
2978                 cv_destroy(&slock->cv);
2979                 kmem_free(slock, sizeof (struct devmap_softlock));
2980         } else
2981                 mutex_exit(&slock->lock);
2982 
2983         mutex_exit(&devmap_slock);
2984 }
2985 
2986 /*
2987  * Wake up processes that sleep on dh_ctx->locked.
2988  * Free dh_ctx if refcnt is 0.
2989  */
2990 static void
2991 devmap_ctx_rele(devmap_handle_t *dhp)
2992 {
2993         struct devmap_ctx *devctx = dhp->dh_ctx;
2994         struct devmap_ctx *tmp;
2995         struct devmap_ctx *parent;
2996         timeout_id_t tid;
2997 
2998         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
2999             "devmap_ctx_rele:start");
3000 
3001         mutex_enter(&devmapctx_lock);
3002         mutex_enter(&devctx->lock);
3003 
3004         ASSERT(devctx->refcnt > 0);
3005 
3006         devctx->refcnt--;
3007 
3008         /*
3009          * If no one is using the device, free up the devctx data.
3010          */
3011         if (devctx->refcnt == 0) {
3012                 /*
3013                  * Untimeout any threads using this mapping as they are about
3014                  * to go away.
3015                  */
3016                 if (devctx->timeout != 0) {
3017                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3018                             "devmap_ctx_rele:untimeout ctx->timeout");
3019 
3020                         tid = devctx->timeout;
3021                         mutex_exit(&devctx->lock);
3022                         (void) untimeout(tid);
3023                         mutex_enter(&devctx->lock);
3024                 }
3025 
3026                 devctx->oncpu = 0;
3027                 cv_signal(&devctx->cv);
3028 
3029                 if (devmapctx_list == devctx)
3030                         devmapctx_list = devctx->next;
3031                 else {
3032                         parent = devmapctx_list;
3033                         for (tmp = devmapctx_list->next; tmp != NULL;
3034                             tmp = tmp->next) {
3035                                 if (tmp == devctx) {
3036                                         parent->next = tmp->next;
3037                                         break;
3038                                 }
3039                                 parent = tmp;
3040                         }
3041                 }
3042                 mutex_exit(&devctx->lock);
3043                 mutex_destroy(&devctx->lock);
3044                 cv_destroy(&devctx->cv);
3045                 kmem_free(devctx, sizeof (struct devmap_ctx));
3046         } else
3047                 mutex_exit(&devctx->lock);
3048 
3049         mutex_exit(&devmapctx_lock);
3050 }
3051 
3052 /*
3053  * devmap_load:
3054  *                      Marks a segdev segment or pages if offset->offset+len
3055  *                      is not the entire segment as nointercept and faults in
3056  *                      the pages in the range offset -> offset+len.
3057  */
3058 int
3059 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3060     uint_t rw)
3061 {
3062         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3063         struct as *asp = dhp->dh_seg->s_as;
3064         caddr_t addr;
3065         ulong_t size;
3066         ssize_t soff;   /* offset from the beginning of the segment */
3067         int rc;
3068 
3069         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3070             "devmap_load:start dhp=%p offset=%llx len=%lx",
3071             (void *)dhp, offset, len);
3072 
3073         DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3074             (void *)dhp, offset, len));
3075 
3076         /*
3077          *      Hat layer only supports devload to process' context for which
3078          *      the as lock is held. Verify here and return error if drivers
3079          *      inadvertently call devmap_load on a wrong devmap handle.
3080          */
3081         if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3082                 return (FC_MAKE_ERR(EINVAL));
3083 
3084         soff = (ssize_t)(offset - dhp->dh_uoff);
3085         soff = round_down_p2(soff, PAGESIZE);
3086         if (soff < 0 || soff >= dhp->dh_len)
3087                 return (FC_MAKE_ERR(EINVAL));
3088 
3089         /*
3090          * Address and size must be page aligned.  Len is set to the
3091          * number of bytes in the number of pages that are required to
3092          * support len.  Offset is set to the byte offset of the first byte
3093          * of the page that contains offset.
3094          */
3095         len = round_up_p2(len, PAGESIZE);
3096 
3097         /*
3098          * If len == 0, then calculate the size by getting
3099          * the number of bytes from offset to the end of the segment.
3100          */
3101         if (len == 0)
3102                 size = dhp->dh_len - soff;
3103         else {
3104                 size = len;
3105                 if ((soff + size) > dhp->dh_len)
3106                         return (FC_MAKE_ERR(EINVAL));
3107         }
3108 
3109         /*
3110          * The address is offset bytes from the base address of
3111          * the segment.
3112          */
3113         addr = (caddr_t)(soff + dhp->dh_uvaddr);
3114 
3115         HOLD_DHP_LOCK(dhp);
3116         rc = segdev_faultpages(asp->a_hat,
3117             dhp->dh_seg, addr, size, type, rw, dhp);
3118         RELE_DHP_LOCK(dhp);
3119         return (rc);
3120 }
3121 
3122 int
3123 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3124     size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3125 {
3126         register devmap_handle_t *dhp;
3127         int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3128             size_t *, uint_t);
3129         int (*mmap)(dev_t, off_t, int);
3130         struct devmap_callback_ctl *callbackops;
3131         devmap_handle_t *dhp_head = NULL;
3132         devmap_handle_t *dhp_prev = NULL;
3133         devmap_handle_t *dhp_curr;
3134         caddr_t addr;
3135         int map_flag;
3136         int ret;
3137         ulong_t total_len;
3138         size_t map_len;
3139         size_t resid_len = len;
3140         offset_t map_off = off;
3141         struct devmap_softlock *slock = NULL;
3142 
3143 #ifdef lint
3144         cred = cred;
3145 #endif
3146 
3147         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3148             "devmap_setup:start off=%llx len=%lx", off, len);
3149         DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3150             off, len));
3151 
3152         devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3153         mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3154 
3155         /*
3156          * driver must provide devmap(9E) entry point in cb_ops to use the
3157          * devmap framework.
3158          */
3159         if (devmap == NULL || devmap == nulldev || devmap == nodev)
3160                 return (EINVAL);
3161 
3162         /*
3163          * To protect from an inadvertent entry because the devmap entry point
3164          * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3165          * mmap is NULL.
3166          */
3167         map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3168         if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3169                 return (EINVAL);
3170 
3171         /*
3172          * devmap allows mmap(2) to map multiple registers.
3173          * one devmap_handle is created for each register mapped.
3174          */
3175         for (total_len = 0; total_len < len; total_len += map_len) {
3176                 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3177 
3178                 if (dhp_prev != NULL)
3179                         dhp_prev->dh_next = dhp;
3180                 else
3181                         dhp_head = dhp;
3182                 dhp_prev = dhp;
3183 
3184                 dhp->dh_prot = prot;
3185                 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3186                 dhp->dh_dev = dev;
3187                 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3188                 dhp->dh_uoff = map_off;
3189 
3190                 /*
3191                  * Get mapping specific info from
3192                  * the driver, such as rnumber, roff, len, callbackops,
3193                  * accattrp and, if the mapping is for kernel memory,
3194                  * ddi_umem_cookie.
3195                  */
3196                 if ((ret = cdev_devmap(dev, dhp, map_off,
3197                     resid_len, &map_len, get_udatamodel())) != 0) {
3198                         free_devmap_handle(dhp_head);
3199                         return (ENXIO);
3200                 }
3201 
3202                 if (map_len & PAGEOFFSET) {
3203                         free_devmap_handle(dhp_head);
3204                         return (EINVAL);
3205                 }
3206 
3207                 callbackops = &dhp->dh_callbackops;
3208 
3209                 if ((callbackops->devmap_access == NULL) ||
3210                     (callbackops->devmap_access == nulldev) ||
3211                     (callbackops->devmap_access == nodev)) {
3212                         /*
3213                          * Normally devmap does not support MAP_PRIVATE unless
3214                          * the drivers provide a valid devmap_access routine.
3215                          */
3216                         if ((flags & MAP_PRIVATE) != 0) {
3217                                 free_devmap_handle(dhp_head);
3218                                 return (EINVAL);
3219                         }
3220                 } else {
3221                         /*
3222                          * Initialize dhp_softlock and dh_ctx if the drivers
3223                          * provide devmap_access.
3224                          */
3225                         dhp->dh_softlock = devmap_softlock_init(dev,
3226                             (ulong_t)callbackops->devmap_access);
3227                         dhp->dh_ctx = devmap_ctxinit(dev,
3228                             (ulong_t)callbackops->devmap_access);
3229 
3230                         /*
3231                          * segdev_fault can only work when all
3232                          * dh_softlock in a multi-dhp mapping
3233                          * are same. see comments in segdev_fault
3234                          * This code keeps track of the first
3235                          * dh_softlock allocated in slock and
3236                          * compares all later allocations and if
3237                          * not similar, returns an error.
3238                          */
3239                         if (slock == NULL)
3240                                 slock = dhp->dh_softlock;
3241                         if (slock != dhp->dh_softlock) {
3242                                 free_devmap_handle(dhp_head);
3243                                 return (ENOTSUP);
3244                         }
3245                 }
3246 
3247                 map_off += map_len;
3248                 resid_len -= map_len;
3249         }
3250 
3251         /*
3252          * get the user virtual address and establish the mapping between
3253          * uvaddr and device physical address.
3254          */
3255         if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3256             != 0) {
3257                 /*
3258                  * free devmap handles if error during the mapping.
3259                  */
3260                 free_devmap_handle(dhp_head);
3261 
3262                 return (ret);
3263         }
3264 
3265         /*
3266          * call the driver's devmap_map callback to do more after the mapping,
3267          * such as to allocate driver private data for context management.
3268          */
3269         dhp = dhp_head;
3270         map_off = off;
3271         addr = *addrp;
3272         while (dhp != NULL) {
3273                 callbackops = &dhp->dh_callbackops;
3274                 dhp->dh_uvaddr = addr;
3275                 dhp_curr = dhp;
3276                 if (callbackops->devmap_map != NULL) {
3277                         ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3278                             dev, flags, map_off,
3279                             dhp->dh_len, &dhp->dh_pvtp);
3280                         if (ret != 0) {
3281                                 struct segdev_data *sdp;
3282 
3283                                 /*
3284                                  * call driver's devmap_unmap entry point
3285                                  * to free driver resources.
3286                                  */
3287                                 dhp = dhp_head;
3288                                 map_off = off;
3289                                 while (dhp != dhp_curr) {
3290                                         callbackops = &dhp->dh_callbackops;
3291                                         if (callbackops->devmap_unmap != NULL) {
3292                                                 (*callbackops->devmap_unmap)(
3293                                                     dhp, dhp->dh_pvtp,
3294                                                     map_off, dhp->dh_len,
3295                                                     NULL, NULL, NULL, NULL);
3296                                         }
3297                                         map_off += dhp->dh_len;
3298                                         dhp = dhp->dh_next;
3299                                 }
3300                                 sdp = dhp_head->dh_seg->s_data;
3301                                 sdp->devmap_data = NULL;
3302                                 free_devmap_handle(dhp_head);
3303                                 return (ENXIO);
3304                         }
3305                 }
3306                 map_off += dhp->dh_len;
3307                 addr += dhp->dh_len;
3308                 dhp = dhp->dh_next;
3309         }
3310 
3311         return (0);
3312 }
3313 
3314 int
3315 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3316     off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3317 {
3318         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3319             "devmap_segmap:start");
3320         return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3321             (size_t)len, prot, maxprot, flags, cred));
3322 }
3323 
3324 /*
3325  * Called from devmap_devmem_setup/remap to see if can use large pages for
3326  * this device mapping.
3327  * Also calculate the max. page size for this mapping.
3328  * this page size will be used in fault routine for
3329  * optimal page size calculations.
3330  */
3331 static void
3332 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3333 {
3334         ASSERT(dhp_is_devmem(dhp));
3335         dhp->dh_mmulevel = 0;
3336 
3337         /*
3338          * use large page size only if:
3339          *  1. device memory.
3340          *  2. mmu supports multiple page sizes,
3341          *  3. Driver did not disallow it
3342          *  4. dhp length is at least as big as the large pagesize
3343          *  5. the uvaddr and pfn are large pagesize aligned
3344          */
3345         if (page_num_pagesizes() > 1 &&
3346             !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3347                 ulong_t base;
3348                 int level;
3349 
3350                 base = (ulong_t)ptob(dhp->dh_pfn);
3351                 for (level = 1; level < page_num_pagesizes(); level++) {
3352                         size_t pgsize = page_get_pagesize(level);
3353                         if ((dhp->dh_len < pgsize) ||
3354                             (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3355                             base, pgsize))) {
3356                                 break;
3357                         }
3358                 }
3359                 dhp->dh_mmulevel = level - 1;
3360         }
3361         if (dhp->dh_mmulevel > 0) {
3362                 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3363         } else {
3364                 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3365         }
3366 }
3367 
3368 /*
3369  * Called by driver devmap routine to pass device specific info to
3370  * the framework.    used for device memory mapping only.
3371  */
3372 int
3373 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3374     struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3375     size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3376 {
3377         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3378         ddi_acc_handle_t handle;
3379         ddi_map_req_t mr;
3380         ddi_acc_hdl_t *hp;
3381         int err;
3382 
3383         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3384             "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3385             (void *)dhp, roff, rnumber, (uint_t)len);
3386         DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3387             "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3388 
3389         /*
3390          * First to check if this function has been called for this dhp.
3391          */
3392         if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3393                 return (DDI_FAILURE);
3394 
3395         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3396                 return (DDI_FAILURE);
3397 
3398         if (flags & DEVMAP_MAPPING_INVALID) {
3399                 /*
3400                  * Don't go up the tree to get pfn if the driver specifies
3401                  * DEVMAP_MAPPING_INVALID in flags.
3402                  *
3403                  * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3404                  * remap permission.
3405                  */
3406                 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3407                         return (DDI_FAILURE);
3408                 }
3409                 dhp->dh_pfn = PFN_INVALID;
3410         } else {
3411                 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3412                 if (handle == NULL)
3413                         return (DDI_FAILURE);
3414 
3415                 hp = impl_acc_hdl_get(handle);
3416                 hp->ah_vers = VERS_ACCHDL;
3417                 hp->ah_dip = dip;
3418                 hp->ah_rnumber = rnumber;
3419                 hp->ah_offset = roff;
3420                 hp->ah_len = len;
3421                 if (accattrp != NULL)
3422                         hp->ah_acc = *accattrp;
3423 
3424                 mr.map_op = DDI_MO_MAP_LOCKED;
3425                 mr.map_type = DDI_MT_RNUMBER;
3426                 mr.map_obj.rnumber = rnumber;
3427                 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3428                 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3429                 mr.map_handlep = hp;
3430                 mr.map_vers = DDI_MAP_VERSION;
3431 
3432                 /*
3433                  * up the device tree to get pfn.
3434                  * The rootnex_map_regspec() routine in nexus drivers has been
3435                  * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3436                  */
3437                 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3438                 dhp->dh_hat_attr = hp->ah_hat_flags;
3439                 impl_acc_hdl_free(handle);
3440 
3441                 if (err)
3442                         return (DDI_FAILURE);
3443         }
3444         /* Should not be using devmem setup for memory pages */
3445         ASSERT(!pf_is_memory(dhp->dh_pfn));
3446 
3447         /* Only some of the flags bits are settable by the driver */
3448         dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3449         dhp->dh_len = ptob(btopr(len));
3450 
3451         dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3452         dhp->dh_roff = ptob(btop(roff));
3453 
3454         /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3455         devmap_devmem_large_page_setup(dhp);
3456         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3457         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3458 
3459 
3460         if (callbackops != NULL) {
3461                 bcopy(callbackops, &dhp->dh_callbackops,
3462                     sizeof (struct devmap_callback_ctl));
3463         }
3464 
3465         /*
3466          * Initialize dh_lock if we want to do remap.
3467          */
3468         if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3469                 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3470                 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3471         }
3472 
3473         dhp->dh_flags |= DEVMAP_SETUP_DONE;
3474 
3475         return (DDI_SUCCESS);
3476 }
3477 
3478 int
3479 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3480     uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3481     uint_t flags, ddi_device_acc_attr_t *accattrp)
3482 {
3483         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3484         ddi_acc_handle_t handle;
3485         ddi_map_req_t mr;
3486         ddi_acc_hdl_t *hp;
3487         pfn_t   pfn;
3488         uint_t  hat_flags;
3489         int     err;
3490 
3491         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3492             "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3493             (void *)dhp, roff, rnumber, (uint_t)len);
3494         DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3495             "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3496 
3497         /*
3498          * Return failure if setup has not been done or no remap permission
3499          * has been granted during the setup.
3500          */
3501         if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3502             (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3503                 return (DDI_FAILURE);
3504 
3505         /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3506         if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3507                 return (DDI_FAILURE);
3508 
3509         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3510                 return (DDI_FAILURE);
3511 
3512         if (!(flags & DEVMAP_MAPPING_INVALID)) {
3513                 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3514                 if (handle == NULL)
3515                         return (DDI_FAILURE);
3516         }
3517 
3518         HOLD_DHP_LOCK(dhp);
3519 
3520         /*
3521          * Unload the old mapping, so next fault will setup the new mappings
3522          * Do this while holding the dhp lock so other faults dont reestablish
3523          * the mappings
3524          */
3525         hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3526             dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3527 
3528         if (flags & DEVMAP_MAPPING_INVALID) {
3529                 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3530                 dhp->dh_pfn = PFN_INVALID;
3531         } else {
3532                 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3533                 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3534                 hp = impl_acc_hdl_get(handle);
3535                 hp->ah_vers = VERS_ACCHDL;
3536                 hp->ah_dip = dip;
3537                 hp->ah_rnumber = rnumber;
3538                 hp->ah_offset = roff;
3539                 hp->ah_len = len;
3540                 if (accattrp != NULL)
3541                         hp->ah_acc = *accattrp;
3542 
3543                 mr.map_op = DDI_MO_MAP_LOCKED;
3544                 mr.map_type = DDI_MT_RNUMBER;
3545                 mr.map_obj.rnumber = rnumber;
3546                 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3547                 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3548                 mr.map_handlep = hp;
3549                 mr.map_vers = DDI_MAP_VERSION;
3550 
3551                 /*
3552                  * up the device tree to get pfn.
3553                  * The rootnex_map_regspec() routine in nexus drivers has been
3554                  * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3555                  */
3556                 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3557                 hat_flags = hp->ah_hat_flags;
3558                 impl_acc_hdl_free(handle);
3559                 if (err) {
3560                         RELE_DHP_LOCK(dhp);
3561                         return (DDI_FAILURE);
3562                 }
3563                 /*
3564                  * Store result of ddi_map first in local variables, as we do
3565                  * not want to overwrite the existing dhp with wrong data.
3566                  */
3567                 dhp->dh_pfn = pfn;
3568                 dhp->dh_hat_attr = hat_flags;
3569         }
3570 
3571         /* clear the large page size flag */
3572         dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3573 
3574         dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3575         dhp->dh_roff = ptob(btop(roff));
3576 
3577         /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3578         devmap_devmem_large_page_setup(dhp);
3579         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3580         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3581 
3582         RELE_DHP_LOCK(dhp);
3583         return (DDI_SUCCESS);
3584 }
3585 
3586 /*
3587  * called by driver devmap routine to pass kernel virtual address  mapping
3588  * info to the framework.    used only for kernel memory
3589  * allocated from ddi_umem_alloc().
3590  */
3591 int
3592 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3593     struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3594     offset_t off, size_t len, uint_t maxprot, uint_t flags,
3595     ddi_device_acc_attr_t *accattrp)
3596 {
3597         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3598         struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3599 
3600 #ifdef lint
3601         dip = dip;
3602 #endif
3603 
3604         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3605             "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3606             (void *)dhp, off, cookie, len);
3607         DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3608             "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3609 
3610         if (cookie == NULL)
3611                 return (DDI_FAILURE);
3612 
3613         /* For UMEM_TRASH, this restriction is not needed */
3614         if ((off + len) > cp->size)
3615                 return (DDI_FAILURE);
3616 
3617         /* check if the cache attributes are supported */
3618         if (i_ddi_check_cache_attr(flags) == B_FALSE)
3619                 return (DDI_FAILURE);
3620 
3621         /*
3622          * First to check if this function has been called for this dhp.
3623          */
3624         if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3625                 return (DDI_FAILURE);
3626 
3627         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3628                 return (DDI_FAILURE);
3629 
3630         if (flags & DEVMAP_MAPPING_INVALID) {
3631                 /*
3632                  * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3633                  * remap permission.
3634                  */
3635                 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3636                         return (DDI_FAILURE);
3637                 }
3638         } else {
3639                 dhp->dh_cookie = cookie;
3640                 dhp->dh_roff = ptob(btop(off));
3641                 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3642                 /* set HAT cache attributes */
3643                 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3644                 /* set HAT endianess attributes */
3645                 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3646         }
3647 
3648         /*
3649          * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3650          * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3651          * create consistent mappings but our intention was to create
3652          * non-consistent mappings.
3653          *
3654          * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3655          * mappings.
3656          *
3657          * kernel exported memory: hat figures it out it's memory and always
3658          * creates consistent mappings.
3659          *
3660          * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3661          *
3662          * /dev/kmem: consistent mappings are created unless they are
3663          * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3664          * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3665          * mappings of /dev/kmem. See common/io/mem.c
3666          */
3667 
3668         /* Only some of the flags bits are settable by the driver */
3669         dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3670 
3671         dhp->dh_len = ptob(btopr(len));
3672         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3673         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3674 
3675         if (callbackops != NULL) {
3676                 bcopy(callbackops, &dhp->dh_callbackops,
3677                     sizeof (struct devmap_callback_ctl));
3678         }
3679         /*
3680          * Initialize dh_lock if we want to do remap.
3681          */
3682         if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3683                 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3684                 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3685         }
3686 
3687         dhp->dh_flags |= DEVMAP_SETUP_DONE;
3688 
3689         return (DDI_SUCCESS);
3690 }
3691 
3692 int
3693 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3694     ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3695     uint_t flags, ddi_device_acc_attr_t *accattrp)
3696 {
3697         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3698         struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3699 
3700         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3701             "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3702             (void *)dhp, off, cookie, len);
3703         DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3704             "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3705 
3706 #ifdef lint
3707         dip = dip;
3708         accattrp = accattrp;
3709 #endif
3710         /*
3711          * Reture failure if setup has not been done or no remap permission
3712          * has been granted during the setup.
3713          */
3714         if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3715             (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3716                 return (DDI_FAILURE);
3717 
3718         /* No flags supported for remap yet */
3719         if (flags != 0)
3720                 return (DDI_FAILURE);
3721 
3722         /* check if the cache attributes are supported */
3723         if (i_ddi_check_cache_attr(flags) == B_FALSE)
3724                 return (DDI_FAILURE);
3725 
3726         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3727                 return (DDI_FAILURE);
3728 
3729         /* For UMEM_TRASH, this restriction is not needed */
3730         if ((off + len) > cp->size)
3731                 return (DDI_FAILURE);
3732 
3733         HOLD_DHP_LOCK(dhp);
3734         /*
3735          * Unload the old mapping, so next fault will setup the new mappings
3736          * Do this while holding the dhp lock so other faults dont reestablish
3737          * the mappings
3738          */
3739         hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3740             dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3741 
3742         dhp->dh_cookie = cookie;
3743         dhp->dh_roff = ptob(btop(off));
3744         dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3745         /* set HAT cache attributes */
3746         i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3747         /* set HAT endianess attributes */
3748         i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3749 
3750         /* clear the large page size flag */
3751         dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3752 
3753         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3754         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3755         RELE_DHP_LOCK(dhp);
3756         return (DDI_SUCCESS);
3757 }
3758 
3759 /*
3760  * to set timeout value for the driver's context management callback, e.g.
3761  * devmap_access().
3762  */
3763 void
3764 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3765 {
3766         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3767 
3768         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3769             "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3770             (void *)dhp, ticks);
3771         dhp->dh_timeout_length = ticks;
3772 }
3773 
3774 int
3775 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3776     size_t len, uint_t type, uint_t rw)
3777 {
3778 #ifdef lint
3779         pvtp = pvtp;
3780 #endif
3781 
3782         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3783             "devmap_default_access:start");
3784         return (devmap_load(dhp, off, len, type, rw));
3785 }
3786 
3787 /*
3788  * segkmem_alloc() wrapper to allocate memory which is both
3789  * non-relocatable (for DR) and sharelocked, since the rest
3790  * of this segment driver requires it.
3791  */
3792 static void *
3793 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3794 {
3795         ASSERT(vmp != NULL);
3796         ASSERT(kvseg.s_base != NULL);
3797         vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3798         return (segkmem_alloc(vmp, size, vmflag));
3799 }
3800 
3801 /*
3802  * This is where things are a bit incestuous with seg_kmem: unlike
3803  * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3804  * we need to do a bit of a dance around that to prevent duplication of
3805  * code until we decide to bite the bullet and implement a new kernel
3806  * segment for driver-allocated memory that is exported to user space.
3807  */
3808 static void
3809 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3810 {
3811         page_t *pp;
3812         caddr_t addr = inaddr;
3813         caddr_t eaddr;
3814         pgcnt_t npages = btopr(size);
3815 
3816         ASSERT(vmp != NULL);
3817         ASSERT(kvseg.s_base != NULL);
3818         ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3819 
3820         hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3821 
3822         for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3823                 /*
3824                  * Use page_find() instead of page_lookup() to find the page
3825                  * since we know that it is hashed and has a shared lock.
3826                  */
3827                 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3828 
3829                 if (pp == NULL)
3830                         panic("devmap_free_pages: page not found");
3831                 if (!page_tryupgrade(pp)) {
3832                         page_unlock(pp);
3833                         pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3834                             SE_EXCL);
3835                         if (pp == NULL)
3836                                 panic("devmap_free_pages: page already freed");
3837                 }
3838                 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3839                 pp->p_lckcnt = 0;
3840                 page_destroy(pp, 0);
3841         }
3842         page_unresv(npages);
3843 
3844         if (vmp != NULL)
3845                 vmem_free(vmp, inaddr, size);
3846 }
3847 
3848 /*
3849  * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3850  * allocating non-pageable kmem in response to a ddi_umem_alloc()
3851  * default request. For now we allocate our own pages and we keep
3852  * them long-term sharelocked, since: A) the fault routines expect the
3853  * memory to already be locked; B) pageable umem is already long-term
3854  * locked; C) it's a lot of work to make it otherwise, particularly
3855  * since the nexus layer expects the pages to never fault. An RFE is to
3856  * not keep the pages long-term locked, but instead to be able to
3857  * take faults on them and simply look them up in kvp in case we
3858  * fault on them. Even then, we must take care not to let pageout
3859  * steal them from us since the data must remain resident; if we
3860  * do this we must come up with some way to pin the pages to prevent
3861  * faults while a driver is doing DMA to/from them.
3862  */
3863 static void *
3864 devmap_umem_alloc_np(size_t size, size_t flags)
3865 {
3866         void *buf;
3867         int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3868 
3869         buf = vmem_alloc(umem_np_arena, size, vmflags);
3870         if (buf != NULL)
3871                 bzero(buf, size);
3872         return (buf);
3873 }
3874 
3875 static void
3876 devmap_umem_free_np(void *addr, size_t size)
3877 {
3878         vmem_free(umem_np_arena, addr, size);
3879 }
3880 
3881 /*
3882  * allocate page aligned kernel memory for exporting to user land.
3883  * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3884  * to find a user virtual address that is in same color as the address
3885  * allocated here.
3886  */
3887 void *
3888 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3889 {
3890         register size_t len = ptob(btopr(size));
3891         void *buf = NULL;
3892         struct ddi_umem_cookie *cp;
3893         int iflags = 0;
3894 
3895         *cookie = NULL;
3896 
3897         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3898             "devmap_umem_alloc:start");
3899         if (len == 0)
3900                 return ((void *)NULL);
3901 
3902         /*
3903          * allocate cookie
3904          */
3905         if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3906             flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3907                 ASSERT(flags & DDI_UMEM_NOSLEEP);
3908                 return ((void *)NULL);
3909         }
3910 
3911         if (flags & DDI_UMEM_PAGEABLE) {
3912                 /* Only one of the flags is allowed */
3913                 ASSERT(!(flags & DDI_UMEM_TRASH));
3914                 /* initialize resource with 0 */
3915                 iflags = KPD_ZERO;
3916 
3917                 /*
3918                  * to allocate unlocked pageable memory, use segkp_get() to
3919                  * create a segkp segment.  Since segkp can only service kas,
3920                  * other segment drivers such as segdev have to do
3921                  * as_fault(segkp, SOFTLOCK) in its fault routine,
3922                  */
3923                 if (flags & DDI_UMEM_NOSLEEP)
3924                         iflags |= KPD_NOWAIT;
3925 
3926                 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3927                         kmem_free(cp, sizeof (struct ddi_umem_cookie));
3928                         return ((void *)NULL);
3929                 }
3930                 cp->type = KMEM_PAGEABLE;
3931                 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3932                 cp->locked = 0;
3933         } else if (flags & DDI_UMEM_TRASH) {
3934                 /* Only one of the flags is allowed */
3935                 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3936                 cp->type = UMEM_TRASH;
3937                 buf = NULL;
3938         } else {
3939                 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3940                         kmem_free(cp, sizeof (struct ddi_umem_cookie));
3941                         return ((void *)NULL);
3942                 }
3943 
3944                 cp->type = KMEM_NON_PAGEABLE;
3945         }
3946 
3947         /*
3948          * need to save size here.  size will be used when
3949          * we do kmem_free.
3950          */
3951         cp->size = len;
3952         cp->cvaddr = (caddr_t)buf;
3953 
3954         *cookie =  (void *)cp;
3955         return (buf);
3956 }
3957 
3958 void
3959 ddi_umem_free(ddi_umem_cookie_t cookie)
3960 {
3961         struct ddi_umem_cookie *cp;
3962 
3963         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3964             "devmap_umem_free:start");
3965 
3966         /*
3967          * if cookie is NULL, no effects on the system
3968          */
3969         if (cookie == NULL)
3970                 return;
3971 
3972         cp = (struct ddi_umem_cookie *)cookie;
3973 
3974         switch (cp->type) {
3975         case KMEM_PAGEABLE :
3976                 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3977                 /*
3978                  * Check if there are still any pending faults on the cookie
3979                  * while the driver is deleting it,
3980                  * XXX - could change to an ASSERT but wont catch errant drivers
3981                  */
3982                 mutex_enter(&cp->lock);
3983                 if (cp->locked) {
3984                         mutex_exit(&cp->lock);
3985                         panic("ddi_umem_free for cookie with pending faults %p",
3986                             (void *)cp);
3987                         return;
3988                 }
3989 
3990                 segkp_release(segkp, cp->cvaddr);
3991 
3992                 /*
3993                  * release mutex associated with this cookie.
3994                  */
3995                 mutex_destroy(&cp->lock);
3996                 break;
3997         case KMEM_NON_PAGEABLE :
3998                 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3999                 devmap_umem_free_np(cp->cvaddr, cp->size);
4000                 break;
4001         case UMEM_TRASH :
4002                 break;
4003         case UMEM_LOCKED :
4004                 /* Callers should use ddi_umem_unlock for this type */
4005                 ddi_umem_unlock(cookie);
4006                 /* Frees the cookie too */
4007                 return;
4008         default:
4009                 /* panic so we can diagnose the underlying cause */
4010                 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4011                     cp->type);
4012         }
4013 
4014         kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4015 }
4016 
4017 
4018 static int
4019 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4020 {
4021         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4022 
4023         /*
4024          * It looks as if it is always mapped shared
4025          */
4026         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4027             "segdev_getmemid:start");
4028         memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4029         memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4030         return (0);
4031 }
4032 
4033 /*ARGSUSED*/
4034 static lgrp_mem_policy_info_t *
4035 segdev_getpolicy(struct seg *seg, caddr_t addr)
4036 {
4037         return (NULL);
4038 }
4039 
4040 /*ARGSUSED*/
4041 static int
4042 segdev_capable(struct seg *seg, segcapability_t capability)
4043 {
4044         return (0);
4045 }
4046 
4047 /*
4048  * ddi_umem_alloc() non-pageable quantum cache max size.
4049  * This is just a SWAG.
4050  */
4051 #define DEVMAP_UMEM_QUANTUM     (8*PAGESIZE)
4052 
4053 /*
4054  * Initialize seg_dev from boot. This routine sets up the trash page
4055  * and creates the umem_np_arena used to back non-pageable memory
4056  * requests.
4057  */
4058 void
4059 segdev_init(void)
4060 {
4061         struct seg kseg;
4062 
4063         umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4064             devmap_alloc_pages, devmap_free_pages, heap_arena,
4065             DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4066 
4067         kseg.s_as = &kas;
4068         trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4069             PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4070         if (trashpp == NULL)
4071                 panic("segdev_init: failed to create trash page");
4072         pagezero(trashpp, 0, PAGESIZE);
4073         page_downgrade(trashpp);
4074 }
4075 
4076 /*
4077  * Invoke platform-dependent support routines so that /proc can have
4078  * the platform code deal with curious hardware.
4079  */
4080 int
4081 segdev_copyfrom(struct seg *seg,
4082     caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4083 {
4084         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4085         struct snode *sp = VTOS(VTOCVP(sdp->vp));
4086 
4087         return (e_ddi_copyfromdev(sp->s_dip,
4088             (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4089 }
4090 
4091 int
4092 segdev_copyto(struct seg *seg,
4093     caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4094 {
4095         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4096         struct snode *sp = VTOS(VTOCVP(sdp->vp));
4097 
4098         return (e_ddi_copytodev(sp->s_dip,
4099             (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4100 }