1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  28 /*        All Rights Reserved   */
  29 
  30 /*
  31  * University Copyright- Copyright (c) 1982, 1986, 1988
  32  * The Regents of the University of California
  33  * All Rights Reserved
  34  *
  35  * University Acknowledgment- Portions of this document are derived from
  36  * software developed by the University of California, Berkeley, and its
  37  * contributors.
  38  */
  39 
  40 /*
  41  * VM - segment of a mapped device.
  42  *
  43  * This segment driver is used when mapping character special devices.
  44  */
  45 
  46 #include <sys/types.h>
  47 #include <sys/t_lock.h>
  48 #include <sys/sysmacros.h>
  49 #include <sys/vtrace.h>
  50 #include <sys/systm.h>
  51 #include <sys/vmsystm.h>
  52 #include <sys/mman.h>
  53 #include <sys/errno.h>
  54 #include <sys/kmem.h>
  55 #include <sys/cmn_err.h>
  56 #include <sys/vnode.h>
  57 #include <sys/proc.h>
  58 #include <sys/conf.h>
  59 #include <sys/debug.h>
  60 #include <sys/ddidevmap.h>
  61 #include <sys/ddi_implfuncs.h>
  62 #include <sys/lgrp.h>
  63 
  64 #include <vm/page.h>
  65 #include <vm/hat.h>
  66 #include <vm/as.h>
  67 #include <vm/seg.h>
  68 #include <vm/seg_dev.h>
  69 #include <vm/seg_kp.h>
  70 #include <vm/seg_kmem.h>
  71 #include <vm/vpage.h>
  72 
  73 #include <sys/sunddi.h>
  74 #include <sys/esunddi.h>
  75 #include <sys/fs/snode.h>
  76 
  77 
  78 #if DEBUG
  79 int segdev_debug;
  80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
  81 #else
  82 #define DEBUGF(level, args)
  83 #endif
  84 
  85 /* Default timeout for devmap context management */
  86 #define CTX_TIMEOUT_VALUE 0
  87 
  88 #define HOLD_DHP_LOCK(dhp)  if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
  89                         { mutex_enter(&dhp->dh_lock); }
  90 
  91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
  92                         { mutex_exit(&dhp->dh_lock); }
  93 
  94 #define round_down_p2(a, s)     ((a) & ~((s) - 1))
  95 #define round_up_p2(a, s)       (((a) + (s) - 1) & ~((s) - 1))
  96 
  97 /*
  98  * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
  99  * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
 100  */
 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize)            \
 102         (((uvaddr | paddr) & (pgsize - 1)) == 0)
 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize)     \
 104         (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
 105 
 106 #define vpgtob(n)       ((n) * sizeof (struct vpage))   /* For brevity */
 107 
 108 #define VTOCVP(vp)      (VTOS(vp)->s_commonvp)       /* we "know" it's an snode */
 109 
 110 static struct devmap_ctx *devmapctx_list = NULL;
 111 static struct devmap_softlock *devmap_slist = NULL;
 112 
 113 /*
 114  * mutex, vnode and page for the page of zeros we use for the trash mappings.
 115  * One trash page is allocated on the first ddi_umem_setup call that uses it
 116  * XXX Eventually, we may want to combine this with what segnf does when all
 117  * hat layers implement HAT_NOFAULT.
 118  *
 119  * The trash page is used when the backing store for a userland mapping is
 120  * removed but the application semantics do not take kindly to a SIGBUS.
 121  * In that scenario, the applications pages are mapped to some dummy page
 122  * which returns garbage on read and writes go into a common place.
 123  * (Perfect for NO_FAULT semantics)
 124  * The device driver is responsible to communicating to the app with some
 125  * other mechanism that such remapping has happened and the app should take
 126  * corrective action.
 127  * We can also use an anonymous memory page as there is no requirement to
 128  * keep the page locked, however this complicates the fault code. RFE.
 129  */
 130 static struct vnode trashvp;
 131 static struct page *trashpp;
 132 
 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
 134 static vmem_t *umem_np_arena;
 135 
 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
 137 #define DEVMAP_DEVMEM_COOKIE    ((ddi_umem_cookie_t)0x1)
 138 
 139 /*
 140  * Macros to check if type of devmap handle
 141  */
 142 #define cookie_is_devmem(c)     \
 143         ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
 144 
 145 #define cookie_is_pmem(c)       \
 146         ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
 147 
 148 #define cookie_is_kpmem(c)      (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
 149         ((c)->type == KMEM_PAGEABLE))
 150 
 151 #define dhp_is_devmem(dhp)      \
 152         (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 153 
 154 #define dhp_is_pmem(dhp)        \
 155         (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 156 
 157 #define dhp_is_kpmem(dhp)       \
 158         (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
 159 
 160 /*
 161  * Private seg op routines.
 162  */
 163 static int      segdev_dup(struct seg *, struct seg *);
 164 static int      segdev_unmap(struct seg *, caddr_t, size_t);
 165 static void     segdev_free(struct seg *);
 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
 167                     enum fault_type, enum seg_rw);
 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
 169 static int      segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
 170 static int      segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
 171 static void     segdev_badop(void);
 172 static int      segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
 173 static size_t   segdev_incore(struct seg *, caddr_t, size_t, char *);
 174 static int      segdev_lockop(struct seg *, caddr_t, size_t, int, int,
 175                     ulong_t *, size_t);
 176 static int      segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
 177 static u_offset_t       segdev_getoffset(struct seg *, caddr_t);
 178 static int      segdev_gettype(struct seg *, caddr_t);
 179 static int      segdev_getvp(struct seg *, caddr_t, struct vnode **);
 180 static int      segdev_advise(struct seg *, caddr_t, size_t, uint_t);
 181 static void     segdev_dump(struct seg *);
 182 static int      segdev_pagelock(struct seg *, caddr_t, size_t,
 183                     struct page ***, enum lock_type, enum seg_rw);
 184 static int      segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
 185 static int      segdev_getmemid(struct seg *, caddr_t, memid_t *);
 186 static lgrp_mem_policy_info_t   *segdev_getpolicy(struct seg *, caddr_t);
 187 static int      segdev_capable(struct seg *, segcapability_t);
 188 
 189 /*
 190  * XXX  this struct is used by rootnex_map_fault to identify
 191  *      the segment it has been passed. So if you make it
 192  *      "static" you'll need to fix rootnex_map_fault.
 193  */
 194 struct seg_ops segdev_ops = {
 195         segdev_dup,
 196         segdev_unmap,
 197         segdev_free,
 198         segdev_fault,
 199         segdev_faulta,
 200         segdev_setprot,
 201         segdev_checkprot,
 202         (int (*)())segdev_badop,        /* kluster */
 203         (size_t (*)(struct seg *))NULL, /* swapout */
 204         segdev_sync,                    /* sync */
 205         segdev_incore,
 206         segdev_lockop,                  /* lockop */
 207         segdev_getprot,
 208         segdev_getoffset,
 209         segdev_gettype,
 210         segdev_getvp,
 211         segdev_advise,
 212         segdev_dump,
 213         segdev_pagelock,
 214         segdev_setpagesize,
 215         segdev_getmemid,
 216         segdev_getpolicy,
 217         segdev_capable,
 218         seg_inherit_notsup
 219 };
 220 
 221 /*
 222  * Private segdev support routines
 223  */
 224 static struct segdev_data *sdp_alloc(void);
 225 
 226 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
 227     size_t, enum seg_rw);
 228 
 229 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
 230     struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
 231 
 232 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
 233     size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
 234 
 235 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
 236 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
 237 static void devmap_softlock_rele(devmap_handle_t *);
 238 static void devmap_ctx_rele(devmap_handle_t *);
 239 
 240 static void devmap_ctxto(void *);
 241 
 242 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
 243     caddr_t addr);
 244 
 245 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
 246     ulong_t *opfn, ulong_t *pagesize);
 247 
 248 static void free_devmap_handle(devmap_handle_t *dhp);
 249 
 250 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
 251     struct seg *newseg);
 252 
 253 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
 254 
 255 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
 256 
 257 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
 258 
 259 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
 260     offset_t off, size_t len, uint_t flags);
 261 
 262 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
 263     caddr_t addr, size_t *llen, caddr_t *laddr);
 264 
 265 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
 266 
 267 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
 268 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
 269 
 270 static void *devmap_umem_alloc_np(size_t size, size_t flags);
 271 static void devmap_umem_free_np(void *addr, size_t size);
 272 
 273 /*
 274  * routines to lock and unlock underlying segkp segment for
 275  * KMEM_PAGEABLE type cookies.
 276  */
 277 static faultcode_t  acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
 278 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
 279 
 280 /*
 281  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
 282  * drivers with devmap_access callbacks
 283  */
 284 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
 285         enum fault_type);
 286 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
 287         enum fault_type);
 288 
 289 static kmutex_t devmapctx_lock;
 290 
 291 static kmutex_t devmap_slock;
 292 
 293 /*
 294  * Initialize the thread callbacks and thread private data.
 295  */
 296 static struct devmap_ctx *
 297 devmap_ctxinit(dev_t dev, ulong_t id)
 298 {
 299         struct devmap_ctx       *devctx;
 300         struct devmap_ctx       *tmp;
 301         dev_info_t              *dip;
 302 
 303         tmp =  kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
 304 
 305         mutex_enter(&devmapctx_lock);
 306 
 307         dip = e_ddi_hold_devi_by_dev(dev, 0);
 308         ASSERT(dip != NULL);
 309         ddi_release_devi(dip);
 310 
 311         for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
 312                 if ((devctx->dip == dip) && (devctx->id == id))
 313                         break;
 314 
 315         if (devctx == NULL) {
 316                 devctx = tmp;
 317                 devctx->dip = dip;
 318                 devctx->id = id;
 319                 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
 320                 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
 321                 devctx->next = devmapctx_list;
 322                 devmapctx_list = devctx;
 323         } else
 324                 kmem_free(tmp, sizeof (struct devmap_ctx));
 325 
 326         mutex_enter(&devctx->lock);
 327         devctx->refcnt++;
 328         mutex_exit(&devctx->lock);
 329         mutex_exit(&devmapctx_lock);
 330 
 331         return (devctx);
 332 }
 333 
 334 /*
 335  * Timeout callback called if a CPU has not given up the device context
 336  * within dhp->dh_timeout_length ticks
 337  */
 338 static void
 339 devmap_ctxto(void *data)
 340 {
 341         struct devmap_ctx *devctx = data;
 342 
 343         TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
 344             "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
 345         mutex_enter(&devctx->lock);
 346         /*
 347          * Set oncpu = 0 so the next mapping trying to get the device context
 348          * can.
 349          */
 350         devctx->oncpu = 0;
 351         devctx->timeout = 0;
 352         cv_signal(&devctx->cv);
 353         mutex_exit(&devctx->lock);
 354 }
 355 
 356 /*
 357  * Create a device segment.
 358  */
 359 int
 360 segdev_create(struct seg *seg, void *argsp)
 361 {
 362         struct segdev_data *sdp;
 363         struct segdev_crargs *a = (struct segdev_crargs *)argsp;
 364         devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
 365         int error;
 366 
 367         /*
 368          * Since the address space is "write" locked, we
 369          * don't need the segment lock to protect "segdev" data.
 370          */
 371         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 372 
 373         hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
 374 
 375         sdp = sdp_alloc();
 376 
 377         sdp->mapfunc = a->mapfunc;
 378         sdp->offset = a->offset;
 379         sdp->prot = a->prot;
 380         sdp->maxprot = a->maxprot;
 381         sdp->type = a->type;
 382         sdp->pageprot = 0;
 383         sdp->softlockcnt = 0;
 384         sdp->vpage = NULL;
 385 
 386         if (sdp->mapfunc == NULL)
 387                 sdp->devmap_data = dhp;
 388         else
 389                 sdp->devmap_data = dhp = NULL;
 390 
 391         sdp->hat_flags = a->hat_flags;
 392         sdp->hat_attr = a->hat_attr;
 393 
 394         /*
 395          * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
 396          */
 397         ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
 398 
 399         /*
 400          * Hold shadow vnode -- segdev only deals with
 401          * character (VCHR) devices. We use the common
 402          * vp to hang pages on.
 403          */
 404         sdp->vp = specfind(a->dev, VCHR);
 405         ASSERT(sdp->vp != NULL);
 406 
 407         seg->s_ops = &segdev_ops;
 408         seg->s_data = sdp;
 409 
 410         while (dhp != NULL) {
 411                 dhp->dh_seg = seg;
 412                 dhp = dhp->dh_next;
 413         }
 414 
 415         /*
 416          * Inform the vnode of the new mapping.
 417          */
 418         /*
 419          * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
 420          * dhp specific maxprot because spec_addmap does not use maxprot.
 421          */
 422         error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
 423             seg->s_as, seg->s_base, seg->s_size,
 424             sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
 425 
 426         if (error != 0) {
 427                 sdp->devmap_data = NULL;
 428                 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
 429                     HAT_UNLOAD_UNMAP);
 430         } else {
 431                 /*
 432                  * Mappings of /dev/null don't count towards the VSZ of a
 433                  * process.  Mappings of /dev/null have no mapping type.
 434                  */
 435                 if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
 436                     MAP_PRIVATE)) == 0) {
 437                         seg->s_as->a_resvsize -= seg->s_size;
 438                 }
 439         }
 440 
 441         return (error);
 442 }
 443 
 444 static struct segdev_data *
 445 sdp_alloc(void)
 446 {
 447         struct segdev_data *sdp;
 448 
 449         sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
 450         rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
 451 
 452         return (sdp);
 453 }
 454 
 455 /*
 456  * Duplicate seg and return new segment in newseg.
 457  */
 458 static int
 459 segdev_dup(struct seg *seg, struct seg *newseg)
 460 {
 461         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
 462         struct segdev_data *newsdp;
 463         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
 464         size_t npages;
 465         int ret;
 466 
 467         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
 468             "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
 469 
 470         DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
 471             (void *)dhp, (void *)seg));
 472 
 473         /*
 474          * Since the address space is "write" locked, we
 475          * don't need the segment lock to protect "segdev" data.
 476          */
 477         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 478 
 479         newsdp = sdp_alloc();
 480 
 481         newseg->s_ops = seg->s_ops;
 482         newseg->s_data = (void *)newsdp;
 483 
 484         VN_HOLD(sdp->vp);
 485         newsdp->vp   = sdp->vp;
 486         newsdp->mapfunc = sdp->mapfunc;
 487         newsdp->offset       = sdp->offset;
 488         newsdp->pageprot = sdp->pageprot;
 489         newsdp->prot = sdp->prot;
 490         newsdp->maxprot = sdp->maxprot;
 491         newsdp->type = sdp->type;
 492         newsdp->hat_attr = sdp->hat_attr;
 493         newsdp->hat_flags = sdp->hat_flags;
 494         newsdp->softlockcnt = 0;
 495 
 496         /*
 497          * Initialize per page data if the segment we are
 498          * dup'ing has per page information.
 499          */
 500         npages = seg_pages(newseg);
 501 
 502         if (sdp->vpage != NULL) {
 503                 size_t nbytes = vpgtob(npages);
 504 
 505                 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
 506                 bcopy(sdp->vpage, newsdp->vpage, nbytes);
 507         } else
 508                 newsdp->vpage = NULL;
 509 
 510         /*
 511          * duplicate devmap handles
 512          */
 513         if (dhp != NULL) {
 514                 ret = devmap_handle_dup(dhp,
 515                     (devmap_handle_t **)&newsdp->devmap_data, newseg);
 516                 if (ret != 0) {
 517                         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
 518                             "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
 519                             ret, (void *)dhp, (void *)seg);
 520                         DEBUGF(1, (CE_CONT,
 521                             "segdev_dup: ret %x dhp %p seg %p\n",
 522                             ret, (void *)dhp, (void *)seg));
 523                         return (ret);
 524                 }
 525         }
 526 
 527         /*
 528          * Inform the common vnode of the new mapping.
 529          */
 530         return (VOP_ADDMAP(VTOCVP(newsdp->vp),
 531             newsdp->offset, newseg->s_as,
 532             newseg->s_base, newseg->s_size, newsdp->prot,
 533             newsdp->maxprot, sdp->type, CRED(), NULL));
 534 }
 535 
 536 /*
 537  * duplicate devmap handles
 538  */
 539 static int
 540 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
 541     struct seg *newseg)
 542 {
 543         devmap_handle_t *newdhp_save = NULL;
 544         devmap_handle_t *newdhp = NULL;
 545         struct devmap_callback_ctl *callbackops;
 546 
 547         while (dhp != NULL) {
 548                 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
 549 
 550                 /* Need to lock the original dhp while copying if REMAP */
 551                 HOLD_DHP_LOCK(dhp);
 552                 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
 553                 RELE_DHP_LOCK(dhp);
 554                 newdhp->dh_seg = newseg;
 555                 newdhp->dh_next = NULL;
 556                 if (newdhp_save != NULL)
 557                         newdhp_save->dh_next = newdhp;
 558                 else
 559                         *new_dhp = newdhp;
 560                 newdhp_save = newdhp;
 561 
 562                 callbackops = &newdhp->dh_callbackops;
 563 
 564                 if (dhp->dh_softlock != NULL)
 565                         newdhp->dh_softlock = devmap_softlock_init(
 566                             newdhp->dh_dev,
 567                             (ulong_t)callbackops->devmap_access);
 568                 if (dhp->dh_ctx != NULL)
 569                         newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
 570                             (ulong_t)callbackops->devmap_access);
 571 
 572                 /*
 573                  * Initialize dh_lock if we want to do remap.
 574                  */
 575                 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
 576                         mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
 577                         newdhp->dh_flags |= DEVMAP_LOCK_INITED;
 578                 }
 579 
 580                 if (callbackops->devmap_dup != NULL) {
 581                         int ret;
 582 
 583                         /*
 584                          * Call the dup callback so that the driver can
 585                          * duplicate its private data.
 586                          */
 587                         ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
 588                             (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
 589 
 590                         if (ret != 0) {
 591                                 /*
 592                                  * We want to free up this segment as the driver
 593                                  * has indicated that we can't dup it.  But we
 594                                  * don't want to call the drivers, devmap_unmap,
 595                                  * callback function as the driver does not
 596                                  * think this segment exists. The caller of
 597                                  * devmap_dup will call seg_free on newseg
 598                                  * as it was the caller that allocated the
 599                                  * segment.
 600                                  */
 601                                 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
 602                                     "newdhp %p dhp %p\n", (void *)newdhp,
 603                                     (void *)dhp));
 604                                 callbackops->devmap_unmap = NULL;
 605                                 return (ret);
 606                         }
 607                 }
 608 
 609                 dhp = dhp->dh_next;
 610         }
 611 
 612         return (0);
 613 }
 614 
 615 /*
 616  * Split a segment at addr for length len.
 617  */
 618 /*ARGSUSED*/
 619 static int
 620 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
 621 {
 622         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
 623         register struct segdev_data *nsdp;
 624         register struct seg *nseg;
 625         register size_t opages;         /* old segment size in pages */
 626         register size_t npages;         /* new segment size in pages */
 627         register size_t dpages;         /* pages being deleted (unmapped) */
 628         register size_t nbytes;
 629         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
 630         devmap_handle_t *dhpp;
 631         devmap_handle_t *newdhp;
 632         struct devmap_callback_ctl *callbackops;
 633         caddr_t nbase;
 634         offset_t off;
 635         ulong_t nsize;
 636         size_t mlen, sz;
 637 
 638         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
 639             "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
 640             (void *)dhp, (void *)seg, (void *)addr, len);
 641 
 642         DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
 643             (void *)dhp, (void *)seg, (void *)addr, len));
 644 
 645         /*
 646          * Since the address space is "write" locked, we
 647          * don't need the segment lock to protect "segdev" data.
 648          */
 649         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 650 
 651         if ((sz = sdp->softlockcnt) > 0) {
 652                 /*
 653                  * Fail the unmap if pages are SOFTLOCKed through this mapping.
 654                  * softlockcnt is protected from change by the as write lock.
 655                  */
 656                 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
 657                     "segdev_unmap:error softlockcnt = %ld", sz);
 658                 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
 659                 return (EAGAIN);
 660         }
 661 
 662         /*
 663          * Check for bad sizes
 664          */
 665         if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
 666             (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
 667                 panic("segdev_unmap");
 668 
 669         if (dhp != NULL) {
 670                 devmap_handle_t *tdhp;
 671                 /*
 672                  * If large page size was used in hat_devload(),
 673                  * the same page size must be used in hat_unload().
 674                  */
 675                 dhpp = tdhp = devmap_find_handle(dhp, addr);
 676                 while (tdhp != NULL) {
 677                         if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
 678                                 break;
 679                         }
 680                         tdhp = tdhp->dh_next;
 681                 }
 682                 if (tdhp != NULL) {     /* found a dhp using large pages */
 683                         size_t slen = len;
 684                         size_t mlen;
 685                         size_t soff;
 686 
 687                         soff = (ulong_t)(addr - dhpp->dh_uvaddr);
 688                         while (slen != 0) {
 689                                 mlen = MIN(slen, (dhpp->dh_len - soff));
 690                                 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
 691                                     dhpp->dh_len, HAT_UNLOAD_UNMAP);
 692                                 dhpp = dhpp->dh_next;
 693                                 ASSERT(slen >= mlen);
 694                                 slen -= mlen;
 695                                 soff = 0;
 696                         }
 697                 } else
 698                         hat_unload(seg->s_as->a_hat, addr, len,
 699                             HAT_UNLOAD_UNMAP);
 700         } else {
 701                 /*
 702                  * Unload any hardware translations in the range
 703                  * to be taken out.
 704                  */
 705                 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
 706         }
 707 
 708         /*
 709          * get the user offset which will used in the driver callbacks
 710          */
 711         off = sdp->offset + (offset_t)(addr - seg->s_base);
 712 
 713         /*
 714          * Inform the vnode of the unmapping.
 715          */
 716         ASSERT(sdp->vp != NULL);
 717         (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
 718             sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
 719 
 720         /*
 721          * Check for entire segment
 722          */
 723         if (addr == seg->s_base && len == seg->s_size) {
 724                 seg_free(seg);
 725                 return (0);
 726         }
 727 
 728         opages = seg_pages(seg);
 729         dpages = btop(len);
 730         npages = opages - dpages;
 731 
 732         /*
 733          * Check for beginning of segment
 734          */
 735         if (addr == seg->s_base) {
 736                 if (sdp->vpage != NULL) {
 737                         register struct vpage *ovpage;
 738 
 739                         ovpage = sdp->vpage; /* keep pointer to vpage */
 740 
 741                         nbytes = vpgtob(npages);
 742                         sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 743                         bcopy(&ovpage[dpages], sdp->vpage, nbytes);
 744 
 745                         /* free up old vpage */
 746                         kmem_free(ovpage, vpgtob(opages));
 747                 }
 748 
 749                 /*
 750                  * free devmap handles from the beginning of the mapping.
 751                  */
 752                 if (dhp != NULL)
 753                         devmap_handle_unmap_head(dhp, len);
 754 
 755                 sdp->offset += (offset_t)len;
 756 
 757                 seg->s_base += len;
 758                 seg->s_size -= len;
 759 
 760                 return (0);
 761         }
 762 
 763         /*
 764          * Check for end of segment
 765          */
 766         if (addr + len == seg->s_base + seg->s_size) {
 767                 if (sdp->vpage != NULL) {
 768                         register struct vpage *ovpage;
 769 
 770                         ovpage = sdp->vpage; /* keep pointer to vpage */
 771 
 772                         nbytes = vpgtob(npages);
 773                         sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 774                         bcopy(ovpage, sdp->vpage, nbytes);
 775 
 776                         /* free up old vpage */
 777                         kmem_free(ovpage, vpgtob(opages));
 778                 }
 779                 seg->s_size -= len;
 780 
 781                 /*
 782                  * free devmap handles from addr to the end of the mapping.
 783                  */
 784                 if (dhp != NULL)
 785                         devmap_handle_unmap_tail(dhp, addr);
 786 
 787                 return (0);
 788         }
 789 
 790         /*
 791          * The section to go is in the middle of the segment,
 792          * have to make it into two segments.  nseg is made for
 793          * the high end while seg is cut down at the low end.
 794          */
 795         nbase = addr + len;                             /* new seg base */
 796         nsize = (seg->s_base + seg->s_size) - nbase;      /* new seg size */
 797         seg->s_size = addr - seg->s_base;         /* shrink old seg */
 798         nseg = seg_alloc(seg->s_as, nbase, nsize);
 799         if (nseg == NULL)
 800                 panic("segdev_unmap seg_alloc");
 801 
 802         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
 803             "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
 804         DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
 805             (void *)seg, (void *)nseg));
 806         nsdp = sdp_alloc();
 807 
 808         nseg->s_ops = seg->s_ops;
 809         nseg->s_data = (void *)nsdp;
 810 
 811         VN_HOLD(sdp->vp);
 812         nsdp->mapfunc = sdp->mapfunc;
 813         nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
 814         nsdp->vp     = sdp->vp;
 815         nsdp->pageprot = sdp->pageprot;
 816         nsdp->prot   = sdp->prot;
 817         nsdp->maxprot = sdp->maxprot;
 818         nsdp->type = sdp->type;
 819         nsdp->hat_attr = sdp->hat_attr;
 820         nsdp->hat_flags = sdp->hat_flags;
 821         nsdp->softlockcnt = 0;
 822 
 823         /*
 824          * Initialize per page data if the segment we are
 825          * dup'ing has per page information.
 826          */
 827         if (sdp->vpage != NULL) {
 828                 /* need to split vpage into two arrays */
 829                 register size_t nnbytes;
 830                 register size_t nnpages;
 831                 register struct vpage *ovpage;
 832 
 833                 ovpage = sdp->vpage;         /* keep pointer to vpage */
 834 
 835                 npages = seg_pages(seg);        /* seg has shrunk */
 836                 nbytes = vpgtob(npages);
 837                 nnpages = seg_pages(nseg);
 838                 nnbytes = vpgtob(nnpages);
 839 
 840                 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
 841                 bcopy(ovpage, sdp->vpage, nbytes);
 842 
 843                 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
 844                 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
 845 
 846                 /* free up old vpage */
 847                 kmem_free(ovpage, vpgtob(opages));
 848         } else
 849                 nsdp->vpage = NULL;
 850 
 851         /*
 852          * unmap dhps.
 853          */
 854         if (dhp == NULL) {
 855                 nsdp->devmap_data = NULL;
 856                 return (0);
 857         }
 858         while (dhp != NULL) {
 859                 callbackops = &dhp->dh_callbackops;
 860                 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
 861                     "segdev_unmap: dhp=%p addr=%p", dhp, addr);
 862                 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
 863                     (void *)dhp, (void *)addr,
 864                     (void *)dhp->dh_uvaddr, dhp->dh_len));
 865 
 866                 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
 867                         dhpp = dhp->dh_next;
 868                         dhp->dh_next = NULL;
 869                         dhp = dhpp;
 870                 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
 871                         dhp = dhp->dh_next;
 872                 } else if (addr > dhp->dh_uvaddr &&
 873                     (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
 874                         /*
 875                          * <addr, addr+len> is enclosed by dhp.
 876                          * create a newdhp that begins at addr+len and
 877                          * ends at dhp->dh_uvaddr+dhp->dh_len.
 878                          */
 879                         newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
 880                         HOLD_DHP_LOCK(dhp);
 881                         bcopy(dhp, newdhp, sizeof (devmap_handle_t));
 882                         RELE_DHP_LOCK(dhp);
 883                         newdhp->dh_seg = nseg;
 884                         newdhp->dh_next = dhp->dh_next;
 885                         if (dhp->dh_softlock != NULL)
 886                                 newdhp->dh_softlock = devmap_softlock_init(
 887                                     newdhp->dh_dev,
 888                                     (ulong_t)callbackops->devmap_access);
 889                         if (dhp->dh_ctx != NULL)
 890                                 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
 891                                     (ulong_t)callbackops->devmap_access);
 892                         if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
 893                                 mutex_init(&newdhp->dh_lock,
 894                                     NULL, MUTEX_DEFAULT, NULL);
 895                         }
 896                         if (callbackops->devmap_unmap != NULL)
 897                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 898                                     off, len, dhp, &dhp->dh_pvtp,
 899                                     newdhp, &newdhp->dh_pvtp);
 900                         mlen = len + (addr - dhp->dh_uvaddr);
 901                         devmap_handle_reduce_len(newdhp, mlen);
 902                         nsdp->devmap_data = newdhp;
 903                         /* XX Changing len should recalculate LARGE flag */
 904                         dhp->dh_len = addr - dhp->dh_uvaddr;
 905                         dhpp = dhp->dh_next;
 906                         dhp->dh_next = NULL;
 907                         dhp = dhpp;
 908                 } else if ((addr > dhp->dh_uvaddr) &&
 909                     ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
 910                         mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
 911                         /*
 912                          * <addr, addr+len> spans over dhps.
 913                          */
 914                         if (callbackops->devmap_unmap != NULL)
 915                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 916                                     off, mlen, (devmap_cookie_t *)dhp,
 917                                     &dhp->dh_pvtp, NULL, NULL);
 918                         /* XX Changing len should recalculate LARGE flag */
 919                         dhp->dh_len = addr - dhp->dh_uvaddr;
 920                         dhpp = dhp->dh_next;
 921                         dhp->dh_next = NULL;
 922                         dhp = dhpp;
 923                         nsdp->devmap_data = dhp;
 924                 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
 925                         /*
 926                          * dhp is enclosed by <addr, addr+len>.
 927                          */
 928                         dhp->dh_seg = nseg;
 929                         nsdp->devmap_data = dhp;
 930                         dhp = devmap_handle_unmap(dhp);
 931                         nsdp->devmap_data = dhp; /* XX redundant? */
 932                 } else if (((addr + len) > dhp->dh_uvaddr) &&
 933                     ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
 934                         mlen = addr + len - dhp->dh_uvaddr;
 935                         if (callbackops->devmap_unmap != NULL)
 936                                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
 937                                     dhp->dh_uoff, mlen, NULL,
 938                                     NULL, dhp, &dhp->dh_pvtp);
 939                         devmap_handle_reduce_len(dhp, mlen);
 940                         nsdp->devmap_data = dhp;
 941                         dhp->dh_seg = nseg;
 942                         dhp = dhp->dh_next;
 943                 } else {
 944                         dhp->dh_seg = nseg;
 945                         dhp = dhp->dh_next;
 946                 }
 947         }
 948         return (0);
 949 }
 950 
 951 /*
 952  * Utility function handles reducing the length of a devmap handle during unmap
 953  * Note that is only used for unmapping the front portion of the handler,
 954  * i.e., we are bumping up the offset/pfn etc up by len
 955  * Do not use if reducing length at the tail.
 956  */
 957 static void
 958 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
 959 {
 960         struct ddi_umem_cookie *cp;
 961         struct devmap_pmem_cookie *pcp;
 962         /*
 963          * adjust devmap handle fields
 964          */
 965         ASSERT(len < dhp->dh_len);
 966 
 967         /* Make sure only page-aligned changes are done */
 968         ASSERT((len & PAGEOFFSET) == 0);
 969 
 970         dhp->dh_len -= len;
 971         dhp->dh_uoff += (offset_t)len;
 972         dhp->dh_roff += (offset_t)len;
 973         dhp->dh_uvaddr += len;
 974         /* Need to grab dhp lock if REMAP */
 975         HOLD_DHP_LOCK(dhp);
 976         cp = dhp->dh_cookie;
 977         if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
 978                 if (cookie_is_devmem(cp)) {
 979                         dhp->dh_pfn += btop(len);
 980                 } else if (cookie_is_pmem(cp)) {
 981                         pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
 982                         ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
 983                             dhp->dh_roff < ptob(pcp->dp_npages));
 984                 } else {
 985                         ASSERT(dhp->dh_roff < cp->size);
 986                         ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
 987                             dhp->dh_cvaddr < (cp->cvaddr + cp->size));
 988                         ASSERT((dhp->dh_cvaddr + len) <=
 989                             (cp->cvaddr + cp->size));
 990 
 991                         dhp->dh_cvaddr += len;
 992                 }
 993         }
 994         /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
 995         RELE_DHP_LOCK(dhp);
 996 }
 997 
 998 /*
 999  * Free devmap handle, dhp.
1000  * Return the next devmap handle on the linked list.
1001  */
1002 static devmap_handle_t *
1003 devmap_handle_unmap(devmap_handle_t *dhp)
1004 {
1005         struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1006         struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1007         devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1008 
1009         ASSERT(dhp != NULL);
1010 
1011         /*
1012          * before we free up dhp, call the driver's devmap_unmap entry point
1013          * to free resources allocated for this dhp.
1014          */
1015         if (callbackops->devmap_unmap != NULL) {
1016                 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1017                     dhp->dh_len, NULL, NULL, NULL, NULL);
1018         }
1019 
1020         if (dhpp == dhp) {      /* releasing first dhp, change sdp data */
1021                 sdp->devmap_data = dhp->dh_next;
1022         } else {
1023                 while (dhpp->dh_next != dhp) {
1024                         dhpp = dhpp->dh_next;
1025                 }
1026                 dhpp->dh_next = dhp->dh_next;
1027         }
1028         dhpp = dhp->dh_next; /* return value is next dhp in chain */
1029 
1030         if (dhp->dh_softlock != NULL)
1031                 devmap_softlock_rele(dhp);
1032 
1033         if (dhp->dh_ctx != NULL)
1034                 devmap_ctx_rele(dhp);
1035 
1036         if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1037                 mutex_destroy(&dhp->dh_lock);
1038         }
1039         kmem_free(dhp, sizeof (devmap_handle_t));
1040 
1041         return (dhpp);
1042 }
1043 
1044 /*
1045  * Free complete devmap handles from dhp for len bytes
1046  * dhp can be either the first handle or a subsequent handle
1047  */
1048 static void
1049 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1050 {
1051         struct devmap_callback_ctl *callbackops;
1052 
1053         /*
1054          * free the devmap handles covered by len.
1055          */
1056         while (len >= dhp->dh_len) {
1057                 len -= dhp->dh_len;
1058                 dhp = devmap_handle_unmap(dhp);
1059         }
1060         if (len != 0) { /* partial unmap at head of first remaining dhp */
1061                 callbackops = &dhp->dh_callbackops;
1062 
1063                 /*
1064                  * Call the unmap callback so the drivers can make
1065                  * adjustment on its private data.
1066                  */
1067                 if (callbackops->devmap_unmap != NULL)
1068                         (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1069                             dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1070                 devmap_handle_reduce_len(dhp, len);
1071         }
1072 }
1073 
1074 /*
1075  * Free devmap handles to truncate  the mapping after addr
1076  * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1077  *      Also could then use the routine in middle unmap case too
1078  */
1079 static void
1080 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1081 {
1082         register struct seg *seg = dhp->dh_seg;
1083         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1084         register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1085         struct devmap_callback_ctl *callbackops;
1086         register devmap_handle_t *dhpp;
1087         size_t maplen;
1088         ulong_t off;
1089         size_t len;
1090 
1091         maplen = (size_t)(addr - dhp->dh_uvaddr);
1092         dhph = devmap_find_handle(dhph, addr);
1093 
1094         while (dhph != NULL) {
1095                 if (maplen == 0) {
1096                         dhph =  devmap_handle_unmap(dhph);
1097                 } else {
1098                         callbackops = &dhph->dh_callbackops;
1099                         len = dhph->dh_len - maplen;
1100                         off = (ulong_t)sdp->offset + (addr - seg->s_base);
1101                         /*
1102                          * Call the unmap callback so the driver
1103                          * can make adjustments on its private data.
1104                          */
1105                         if (callbackops->devmap_unmap != NULL)
1106                                 (*callbackops->devmap_unmap)(dhph,
1107                                     dhph->dh_pvtp, off, len,
1108                                     (devmap_cookie_t *)dhph,
1109                                     &dhph->dh_pvtp, NULL, NULL);
1110                         /* XXX Reducing len needs to recalculate LARGE flag */
1111                         dhph->dh_len = maplen;
1112                         maplen = 0;
1113                         dhpp = dhph->dh_next;
1114                         dhph->dh_next = NULL;
1115                         dhph = dhpp;
1116                 }
1117         } /* end while */
1118 }
1119 
1120 /*
1121  * Free a segment.
1122  */
1123 static void
1124 segdev_free(struct seg *seg)
1125 {
1126         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1127         devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1128 
1129         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1130             "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1131         DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1132             (void *)dhp, (void *)seg));
1133 
1134         /*
1135          * Since the address space is "write" locked, we
1136          * don't need the segment lock to protect "segdev" data.
1137          */
1138         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1139 
1140         while (dhp != NULL)
1141                 dhp = devmap_handle_unmap(dhp);
1142 
1143         VN_RELE(sdp->vp);
1144         if (sdp->vpage != NULL)
1145                 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1146 
1147         rw_destroy(&sdp->lock);
1148         kmem_free(sdp, sizeof (*sdp));
1149 }
1150 
1151 static void
1152 free_devmap_handle(devmap_handle_t *dhp)
1153 {
1154         register devmap_handle_t *dhpp;
1155 
1156         /*
1157          * free up devmap handle
1158          */
1159         while (dhp != NULL) {
1160                 dhpp = dhp->dh_next;
1161                 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1162                         mutex_destroy(&dhp->dh_lock);
1163                 }
1164 
1165                 if (dhp->dh_softlock != NULL)
1166                         devmap_softlock_rele(dhp);
1167 
1168                 if (dhp->dh_ctx != NULL)
1169                         devmap_ctx_rele(dhp);
1170 
1171                 kmem_free(dhp, sizeof (devmap_handle_t));
1172                 dhp = dhpp;
1173         }
1174 }
1175 
1176 /*
1177  * routines to lock and unlock underlying segkp segment for
1178  * KMEM_PAGEABLE type cookies.
1179  * segkp only allows a single pending F_SOFTLOCK
1180  * we keep track of number of locks in the cookie so we can
1181  * have multiple pending faults and manage the calls to segkp.
1182  * RFE: if segkp supports either pagelock or can support multiple
1183  * calls to F_SOFTLOCK, then these routines can go away.
1184  *      If pagelock, segdev_faultpage can fault on a page by page basis
1185  *              and simplifies the code quite a bit.
1186  *      if multiple calls allowed but not partial ranges, then need for
1187  *      cookie->lock and locked count goes away, code can call as_fault directly
1188  */
1189 static faultcode_t
1190 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1191 {
1192         int err = 0;
1193         ASSERT(cookie_is_kpmem(cookie));
1194         /*
1195          * Fault in pages in segkp with F_SOFTLOCK.
1196          * We want to hold the lock until all pages have been loaded.
1197          * segkp only allows single caller to hold SOFTLOCK, so cookie
1198          * holds a count so we dont call into segkp multiple times
1199          */
1200         mutex_enter(&cookie->lock);
1201 
1202         /*
1203          * Check for overflow in locked field
1204          */
1205         if ((UINT32_MAX - cookie->locked) < npages) {
1206                 err = FC_MAKE_ERR(ENOMEM);
1207         } else if (cookie->locked == 0) {
1208                 /* First time locking */
1209                 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1210                     cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1211         }
1212         if (!err) {
1213                 cookie->locked += npages;
1214         }
1215         mutex_exit(&cookie->lock);
1216         return (err);
1217 }
1218 
1219 static void
1220 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1221 {
1222         mutex_enter(&cookie->lock);
1223         ASSERT(cookie_is_kpmem(cookie));
1224         ASSERT(cookie->locked >= npages);
1225         cookie->locked -= (uint_t)npages;
1226         if (cookie->locked == 0) {
1227                 /* Last unlock */
1228                 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1229                     cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1230                         panic("segdev releasing kpmem lock %p", (void *)cookie);
1231         }
1232         mutex_exit(&cookie->lock);
1233 }
1234 
1235 /*
1236  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1237  * drivers with devmap_access callbacks
1238  * slock->softlocked basically works like a rw lock
1239  *      -ve counts => F_SOFTLOCK in progress
1240  *      +ve counts => F_INVAL/F_PROT in progress
1241  * We allow only one F_SOFTLOCK at a time
1242  * but can have multiple pending F_INVAL/F_PROT calls
1243  *
1244  * This routine waits using cv_wait_sig so killing processes is more graceful
1245  * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1246  */
1247 static int devmap_softlock_enter(
1248         struct devmap_softlock *slock,
1249         size_t npages,
1250         enum fault_type type)
1251 {
1252         if (npages == 0)
1253                 return (0);
1254         mutex_enter(&(slock->lock));
1255         switch (type) {
1256         case F_SOFTLOCK :
1257                 while (slock->softlocked) {
1258                         if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1259                                 /* signalled */
1260                                 mutex_exit(&(slock->lock));
1261                                 return (EINTR);
1262                         }
1263                 }
1264                 slock->softlocked -= npages; /* -ve count => locked */
1265                 break;
1266         case F_INVAL :
1267         case F_PROT :
1268                 while (slock->softlocked < 0)
1269                         if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1270                                 /* signalled */
1271                                 mutex_exit(&(slock->lock));
1272                                 return (EINTR);
1273                         }
1274                 slock->softlocked += npages; /* +ve count => f_invals */
1275                 break;
1276         default:
1277                 ASSERT(0);
1278         }
1279         mutex_exit(&(slock->lock));
1280         return (0);
1281 }
1282 
1283 static void devmap_softlock_exit(
1284         struct devmap_softlock *slock,
1285         size_t npages,
1286         enum fault_type type)
1287 {
1288         if (slock == NULL)
1289                 return;
1290         mutex_enter(&(slock->lock));
1291         switch (type) {
1292         case F_SOFTLOCK :
1293                 ASSERT(-slock->softlocked >= npages);
1294                 slock->softlocked += npages; /* -ve count is softlocked */
1295                 if (slock->softlocked == 0)
1296                         cv_signal(&slock->cv);
1297                 break;
1298         case F_INVAL :
1299         case F_PROT:
1300                 ASSERT(slock->softlocked >= npages);
1301                 slock->softlocked -= npages;
1302                 if (slock->softlocked == 0)
1303                         cv_signal(&slock->cv);
1304                 break;
1305         default:
1306                 ASSERT(0);
1307         }
1308         mutex_exit(&(slock->lock));
1309 }
1310 
1311 /*
1312  * Do a F_SOFTUNLOCK call over the range requested.
1313  * The range must have already been F_SOFTLOCK'ed.
1314  * The segment lock should be held, (but not the segment private lock?)
1315  *  The softunlock code below does not adjust for large page sizes
1316  *      assumes the caller already did any addr/len adjustments for
1317  *      pagesize mappings before calling.
1318  */
1319 /*ARGSUSED*/
1320 static void
1321 segdev_softunlock(
1322         struct hat *hat,                /* the hat */
1323         struct seg *seg,                /* seg_dev of interest */
1324         caddr_t addr,                   /* base address of range */
1325         size_t len,                     /* number of bytes */
1326         enum seg_rw rw)                 /* type of access at fault */
1327 {
1328         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1329         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1330 
1331         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1332             "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1333             dhp_head, sdp, addr, len);
1334         DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1335             "addr %p len %lx\n",
1336             (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1337 
1338         hat_unlock(hat, addr, len);
1339 
1340         if (dhp_head != NULL) {
1341                 devmap_handle_t *dhp;
1342                 size_t mlen;
1343                 size_t tlen = len;
1344                 ulong_t off;
1345 
1346                 dhp = devmap_find_handle(dhp_head, addr);
1347                 ASSERT(dhp != NULL);
1348 
1349                 off = (ulong_t)(addr - dhp->dh_uvaddr);
1350                 while (tlen != 0) {
1351                         mlen = MIN(tlen, (dhp->dh_len - off));
1352 
1353                         /*
1354                          * unlock segkp memory, locked during F_SOFTLOCK
1355                          */
1356                         if (dhp_is_kpmem(dhp)) {
1357                                 release_kpmem_lock(
1358                                     (struct ddi_umem_cookie *)dhp->dh_cookie,
1359                                     btopr(mlen));
1360                         }
1361 
1362                         /*
1363                          * Do the softlock accounting for devmap_access
1364                          */
1365                         if (dhp->dh_callbackops.devmap_access != NULL) {
1366                                 devmap_softlock_exit(dhp->dh_softlock,
1367                                     btopr(mlen), F_SOFTLOCK);
1368                         }
1369 
1370                         tlen -= mlen;
1371                         dhp = dhp->dh_next;
1372                         off = 0;
1373                 }
1374         }
1375 
1376         mutex_enter(&freemem_lock);
1377         ASSERT(sdp->softlockcnt >= btopr(len));
1378         sdp->softlockcnt -= btopr(len);
1379         mutex_exit(&freemem_lock);
1380         if (sdp->softlockcnt == 0) {
1381                 /*
1382                  * All SOFTLOCKS are gone. Wakeup any waiting
1383                  * unmappers so they can try again to unmap.
1384                  * Check for waiters first without the mutex
1385                  * held so we don't always grab the mutex on
1386                  * softunlocks.
1387                  */
1388                 if (AS_ISUNMAPWAIT(seg->s_as)) {
1389                         mutex_enter(&seg->s_as->a_contents);
1390                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1391                                 AS_CLRUNMAPWAIT(seg->s_as);
1392                                 cv_broadcast(&seg->s_as->a_cv);
1393                         }
1394                         mutex_exit(&seg->s_as->a_contents);
1395                 }
1396         }
1397 
1398 }
1399 
1400 /*
1401  * Handle fault for a single page.
1402  * Done in a separate routine so we can handle errors more easily.
1403  * This routine is called only from segdev_faultpages()
1404  * when looping over the range of addresses requested. The segment lock is held.
1405  */
1406 static faultcode_t
1407 segdev_faultpage(
1408         struct hat *hat,                /* the hat */
1409         struct seg *seg,                /* seg_dev of interest */
1410         caddr_t addr,                   /* address in as */
1411         struct vpage *vpage,            /* pointer to vpage for seg, addr */
1412         enum fault_type type,           /* type of fault */
1413         enum seg_rw rw,                 /* type of access at fault */
1414         devmap_handle_t *dhp)           /* devmap handle if any for this page */
1415 {
1416         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1417         uint_t prot;
1418         pfn_t pfnum = PFN_INVALID;
1419         u_offset_t offset;
1420         uint_t hat_flags;
1421         dev_info_t *dip;
1422 
1423         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1424             "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1425         DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1426             (void *)dhp, (void *)seg, (void *)addr));
1427 
1428         /*
1429          * Initialize protection value for this page.
1430          * If we have per page protection values check it now.
1431          */
1432         if (sdp->pageprot) {
1433                 uint_t protchk;
1434 
1435                 switch (rw) {
1436                 case S_READ:
1437                         protchk = PROT_READ;
1438                         break;
1439                 case S_WRITE:
1440                         protchk = PROT_WRITE;
1441                         break;
1442                 case S_EXEC:
1443                         protchk = PROT_EXEC;
1444                         break;
1445                 case S_OTHER:
1446                 default:
1447                         protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1448                         break;
1449                 }
1450 
1451                 prot = VPP_PROT(vpage);
1452                 if ((prot & protchk) == 0)
1453                         return (FC_PROT);       /* illegal access type */
1454         } else {
1455                 prot = sdp->prot;
1456                 /* caller has already done segment level protection check */
1457         }
1458 
1459         if (type == F_SOFTLOCK) {
1460                 mutex_enter(&freemem_lock);
1461                 sdp->softlockcnt++;
1462                 mutex_exit(&freemem_lock);
1463         }
1464 
1465         hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1466         offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1467         /*
1468          * In the devmap framework, sdp->mapfunc is set to NULL.  we can get
1469          * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1470          * seg->s_base.
1471          */
1472         if (dhp == NULL) {
1473                 /* If segment has devmap_data, then dhp should be non-NULL */
1474                 ASSERT(sdp->devmap_data == NULL);
1475                 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1476                     (off_t)offset, prot);
1477                 prot |= sdp->hat_attr;
1478         } else {
1479                 ulong_t off;
1480                 struct ddi_umem_cookie *cp;
1481                 struct devmap_pmem_cookie *pcp;
1482 
1483                 /* ensure the dhp passed in contains addr. */
1484                 ASSERT(dhp == devmap_find_handle(
1485                     (devmap_handle_t *)sdp->devmap_data, addr));
1486 
1487                 off = addr - dhp->dh_uvaddr;
1488 
1489                 /*
1490                  * This routine assumes that the caller makes sure that the
1491                  * fields in dhp used below are unchanged due to remap during
1492                  * this call. Caller does HOLD_DHP_LOCK if neeed
1493                  */
1494                 cp = dhp->dh_cookie;
1495                 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1496                         pfnum = PFN_INVALID;
1497                 } else if (cookie_is_devmem(cp)) {
1498                         pfnum = dhp->dh_pfn + btop(off);
1499                 } else if (cookie_is_pmem(cp)) {
1500                         pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1501                         ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1502                             dhp->dh_roff < ptob(pcp->dp_npages));
1503                         pfnum = page_pptonum(
1504                             pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1505                 } else {
1506                         ASSERT(dhp->dh_roff < cp->size);
1507                         ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1508                             dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1509                         ASSERT((dhp->dh_cvaddr + off) <=
1510                             (cp->cvaddr + cp->size));
1511                         ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1512                             (cp->cvaddr + cp->size));
1513 
1514                         switch (cp->type) {
1515                         case UMEM_LOCKED :
1516                                 if (cp->pparray != NULL) {
1517                                         ASSERT((dhp->dh_roff &
1518                                             PAGEOFFSET) == 0);
1519                                         pfnum = page_pptonum(
1520                                             cp->pparray[btop(off +
1521                                             dhp->dh_roff)]);
1522                                 } else {
1523                                         pfnum = hat_getpfnum(
1524                                             ((proc_t *)cp->procp)->p_as->a_hat,
1525                                             cp->cvaddr + off);
1526                                 }
1527                         break;
1528                         case UMEM_TRASH :
1529                                 pfnum = page_pptonum(trashpp);
1530                                 /*
1531                                  * We should set hat_flags to HAT_NOFAULT also
1532                                  * However, not all hat layers implement this
1533                                  */
1534                                 break;
1535                         case KMEM_PAGEABLE:
1536                         case KMEM_NON_PAGEABLE:
1537                                 pfnum = hat_getpfnum(kas.a_hat,
1538                                     dhp->dh_cvaddr + off);
1539                                 break;
1540                         default :
1541                                 pfnum = PFN_INVALID;
1542                                 break;
1543                         }
1544                 }
1545                 prot |= dhp->dh_hat_attr;
1546         }
1547         if (pfnum == PFN_INVALID) {
1548                 return (FC_MAKE_ERR(EFAULT));
1549         }
1550         /* prot should already be OR'ed in with hat_attributes if needed */
1551 
1552         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1553             "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1554             pfnum, pf_is_memory(pfnum), prot, hat_flags);
1555         DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1556             "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1557 
1558         if (pf_is_memory(pfnum) || (dhp != NULL)) {
1559                 /*
1560                  * It's not _really_ required here to pass sdp->hat_flags
1561                  * to hat_devload even though we do it.
1562                  * This is because hat figures it out DEVMEM mappings
1563                  * are non-consistent, anyway.
1564                  */
1565                 hat_devload(hat, addr, PAGESIZE, pfnum,
1566                     prot, hat_flags | sdp->hat_flags);
1567                 return (0);
1568         }
1569 
1570         /*
1571          * Fall through to the case where devmap is not used and need to call
1572          * up the device tree to set up the mapping
1573          */
1574 
1575         dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1576         ASSERT(dip);
1577 
1578         /*
1579          * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1580          * This is because this calls drivers which may not expect
1581          * prot to have any other values than PROT_ALL
1582          * The root nexus driver has a hack to peek into the segment
1583          * structure and then OR in sdp->hat_attr.
1584          * XX In case the bus_ops interfaces are ever revisited
1585          * we need to fix this. prot should include other hat attributes
1586          */
1587         if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1588             (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1589                 return (FC_MAKE_ERR(EFAULT));
1590         }
1591         return (0);
1592 }
1593 
1594 static faultcode_t
1595 segdev_fault(
1596         struct hat *hat,                /* the hat */
1597         struct seg *seg,                /* the seg_dev of interest */
1598         caddr_t addr,                   /* the address of the fault */
1599         size_t len,                     /* the length of the range */
1600         enum fault_type type,           /* type of fault */
1601         enum seg_rw rw)                 /* type of access at fault */
1602 {
1603         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1604         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1605         devmap_handle_t *dhp;
1606         struct devmap_softlock *slock = NULL;
1607         ulong_t slpage = 0;
1608         ulong_t off;
1609         caddr_t maddr = addr;
1610         int err;
1611         int err_is_faultcode = 0;
1612 
1613         TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1614             "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1615             (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1616         DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1617             "addr %p len %lx type %x\n",
1618             (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1619 
1620         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1621 
1622         /* Handle non-devmap case */
1623         if (dhp_head == NULL)
1624                 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1625 
1626         /* Find devmap handle */
1627         if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1628                 return (FC_NOMAP);
1629 
1630         /*
1631          * The seg_dev driver does not implement copy-on-write,
1632          * and always loads translations with maximal allowed permissions
1633          * but we got an fault trying to access the device.
1634          * Servicing the fault is not going to result in any better result
1635          * RFE: If we want devmap_access callbacks to be involved in F_PROT
1636          *      faults, then the code below is written for that
1637          *      Pending resolution of the following:
1638          *      - determine if the F_INVAL/F_SOFTLOCK syncing
1639          *      is needed for F_PROT also or not. The code below assumes it does
1640          *      - If driver sees F_PROT and calls devmap_load with same type,
1641          *      then segdev_faultpages will fail with FC_PROT anyway, need to
1642          *      change that so calls from devmap_load to segdev_faultpages for
1643          *      F_PROT type are retagged to F_INVAL.
1644          * RFE: Today we dont have drivers that use devmap and want to handle
1645          *      F_PROT calls. The code in segdev_fault* is written to allow
1646          *      this case but is not tested. A driver that needs this capability
1647          *      should be able to remove the short-circuit case; resolve the
1648          *      above issues and "should" work.
1649          */
1650         if (type == F_PROT) {
1651                 return (FC_PROT);
1652         }
1653 
1654         /*
1655          * Loop through dhp list calling devmap_access or segdev_faultpages for
1656          * each devmap handle.
1657          * drivers which implement devmap_access can interpose on faults and do
1658          * device-appropriate special actions before calling devmap_load.
1659          */
1660 
1661         /*
1662          * Unfortunately, this simple loop has turned out to expose a variety
1663          * of complex problems which results in the following convoluted code.
1664          *
1665          * First, a desire to handle a serialization of F_SOFTLOCK calls
1666          * to the driver within the framework.
1667          *      This results in a dh_softlock structure that is on a per device
1668          *      (or device instance) basis and serializes devmap_access calls.
1669          *      Ideally we would need to do this for underlying
1670          *      memory/device regions that are being faulted on
1671          *      but that is hard to identify and with REMAP, harder
1672          * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1673          *      to F_SOFTLOCK calls to the driver.
1674          * These serializations are to simplify the driver programmer model.
1675          * To support these two features, the code first goes through the
1676          *      devmap handles and counts the pages (slpage) that are covered
1677          *      by devmap_access callbacks.
1678          * This part ends with a devmap_softlock_enter call
1679          *      which allows only one F_SOFTLOCK active on a device instance,
1680          *      but multiple F_INVAL/F_PROTs can be active except when a
1681          *      F_SOFTLOCK is active
1682          *
1683          * Next, we dont short-circuit the fault code upfront to call
1684          *      segdev_softunlock for F_SOFTUNLOCK, because we must use
1685          *      the same length when we softlock and softunlock.
1686          *
1687          *      -Hat layers may not support softunlocking lengths less than the
1688          *      original length when there is large page support.
1689          *      -kpmem locking is dependent on keeping the lengths same.
1690          *      -if drivers handled F_SOFTLOCK, they probably also expect to
1691          *              see an F_SOFTUNLOCK of the same length
1692          *      Hence, if extending lengths during softlock,
1693          *      softunlock has to make the same adjustments and goes through
1694          *      the same loop calling segdev_faultpages/segdev_softunlock
1695          *      But some of the synchronization and error handling is different
1696          */
1697 
1698         if (type != F_SOFTUNLOCK) {
1699                 devmap_handle_t *dhpp = dhp;
1700                 size_t slen = len;
1701 
1702                 /*
1703                  * Calculate count of pages that are :
1704                  * a) within the (potentially extended) fault region
1705                  * b) AND covered by devmap handle with devmap_access
1706                  */
1707                 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1708                 while (slen != 0) {
1709                         size_t mlen;
1710 
1711                         /*
1712                          * Softlocking on a region that allows remap is
1713                          * unsupported due to unresolved locking issues
1714                          * XXX: unclear what these are?
1715                          *      One potential is that if there is a pending
1716                          *      softlock, then a remap should not be allowed
1717                          *      until the unlock is done. This is easily
1718                          *      fixed by returning error in devmap*remap on
1719                          *      checking the dh->dh_softlock->softlocked value
1720                          */
1721                         if ((type == F_SOFTLOCK) &&
1722                             (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1723                                 return (FC_NOSUPPORT);
1724                         }
1725 
1726                         mlen = MIN(slen, (dhpp->dh_len - off));
1727                         if (dhpp->dh_callbackops.devmap_access) {
1728                                 size_t llen;
1729                                 caddr_t laddr;
1730                                 /*
1731                                  * use extended length for large page mappings
1732                                  */
1733                                 HOLD_DHP_LOCK(dhpp);
1734                                 if ((sdp->pageprot == 0) &&
1735                                     (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1736                                         devmap_get_large_pgsize(dhpp,
1737                                             mlen, maddr, &llen, &laddr);
1738                                 } else {
1739                                         llen = mlen;
1740                                 }
1741                                 RELE_DHP_LOCK(dhpp);
1742 
1743                                 slpage += btopr(llen);
1744                                 slock = dhpp->dh_softlock;
1745                         }
1746                         maddr += mlen;
1747                         ASSERT(slen >= mlen);
1748                         slen -= mlen;
1749                         dhpp = dhpp->dh_next;
1750                         off = 0;
1751                 }
1752                 /*
1753                  * synchonize with other faulting threads and wait till safe
1754                  * devmap_softlock_enter might return due to signal in cv_wait
1755                  *
1756                  * devmap_softlock_enter has to be called outside of while loop
1757                  * to prevent a deadlock if len spans over multiple dhps.
1758                  * dh_softlock is based on device instance and if multiple dhps
1759                  * use the same device instance, the second dhp's LOCK call
1760                  * will hang waiting on the first to complete.
1761                  * devmap_setup verifies that slocks in a dhp_chain are same.
1762                  * RFE: this deadlock only hold true for F_SOFTLOCK. For
1763                  *      F_INVAL/F_PROT, since we now allow multiple in parallel,
1764                  *      we could have done the softlock_enter inside the loop
1765                  *      and supported multi-dhp mappings with dissimilar devices
1766                  */
1767                 if (err = devmap_softlock_enter(slock, slpage, type))
1768                         return (FC_MAKE_ERR(err));
1769         }
1770 
1771         /* reset 'maddr' to the start addr of the range of fault. */
1772         maddr = addr;
1773 
1774         /* calculate the offset corresponds to 'addr' in the first dhp. */
1775         off = (ulong_t)(addr - dhp->dh_uvaddr);
1776 
1777         /*
1778          * The fault length may span over multiple dhps.
1779          * Loop until the total length is satisfied.
1780          */
1781         while (len != 0) {
1782                 size_t llen;
1783                 size_t mlen;
1784                 caddr_t laddr;
1785 
1786                 /*
1787                  * mlen is the smaller of 'len' and the length
1788                  * from addr to the end of mapping defined by dhp.
1789                  */
1790                 mlen = MIN(len, (dhp->dh_len - off));
1791 
1792                 HOLD_DHP_LOCK(dhp);
1793                 /*
1794                  * Pass the extended length and address to devmap_access
1795                  * if large pagesize is used for loading address translations.
1796                  */
1797                 if ((sdp->pageprot == 0) &&
1798                     (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1799                         devmap_get_large_pgsize(dhp, mlen, maddr,
1800                             &llen, &laddr);
1801                         ASSERT(maddr == addr || laddr == maddr);
1802                 } else {
1803                         llen = mlen;
1804                         laddr = maddr;
1805                 }
1806 
1807                 if (dhp->dh_callbackops.devmap_access != NULL) {
1808                         offset_t aoff;
1809 
1810                         aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1811 
1812                         /*
1813                          * call driver's devmap_access entry point which will
1814                          * call devmap_load/contextmgmt to load the translations
1815                          *
1816                          * We drop the dhp_lock before calling access so
1817                          * drivers can call devmap_*_remap within access
1818                          */
1819                         RELE_DHP_LOCK(dhp);
1820 
1821                         err = (*dhp->dh_callbackops.devmap_access)(
1822                             dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1823                 } else {
1824                         /*
1825                          * If no devmap_access entry point, then load mappings
1826                          * hold dhp_lock across faultpages if REMAP
1827                          */
1828                         err = segdev_faultpages(hat, seg, laddr, llen,
1829                             type, rw, dhp);
1830                         err_is_faultcode = 1;
1831                         RELE_DHP_LOCK(dhp);
1832                 }
1833 
1834                 if (err) {
1835                         if ((type == F_SOFTLOCK) && (maddr > addr)) {
1836                                 /*
1837                                  * If not first dhp, use
1838                                  * segdev_fault(F_SOFTUNLOCK) for prior dhps
1839                                  * While this is recursion, it is incorrect to
1840                                  * call just segdev_softunlock
1841                                  * if we are using either large pages
1842                                  * or devmap_access. It will be more right
1843                                  * to go through the same loop as above
1844                                  * rather than call segdev_softunlock directly
1845                                  * It will use the right lenghths as well as
1846                                  * call into the driver devmap_access routines.
1847                                  */
1848                                 size_t done = (size_t)(maddr - addr);
1849                                 (void) segdev_fault(hat, seg, addr, done,
1850                                     F_SOFTUNLOCK, S_OTHER);
1851                                 /*
1852                                  * reduce slpage by number of pages
1853                                  * released by segdev_softunlock
1854                                  */
1855                                 ASSERT(slpage >= btopr(done));
1856                                 devmap_softlock_exit(slock,
1857                                     slpage - btopr(done), type);
1858                         } else {
1859                                 devmap_softlock_exit(slock, slpage, type);
1860                         }
1861 
1862 
1863                         /*
1864                          * Segdev_faultpages() already returns a faultcode,
1865                          * hence, result from segdev_faultpages() should be
1866                          * returned directly.
1867                          */
1868                         if (err_is_faultcode)
1869                                 return (err);
1870                         return (FC_MAKE_ERR(err));
1871                 }
1872 
1873                 maddr += mlen;
1874                 ASSERT(len >= mlen);
1875                 len -= mlen;
1876                 dhp = dhp->dh_next;
1877                 off = 0;
1878 
1879                 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1880         }
1881         /*
1882          * release the softlock count at end of fault
1883          * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1884          */
1885         if ((type == F_INVAL) || (type == F_PROT))
1886                 devmap_softlock_exit(slock, slpage, type);
1887         return (0);
1888 }
1889 
1890 /*
1891  * segdev_faultpages
1892  *
1893  * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1894  * This routine assumes that the callers makes sure that the fields
1895  * in dhp used below are not changed due to remap during this call.
1896  * Caller does HOLD_DHP_LOCK if neeed
1897  * This routine returns a faultcode_t as a return value for segdev_fault.
1898  */
1899 static faultcode_t
1900 segdev_faultpages(
1901         struct hat *hat,                /* the hat */
1902         struct seg *seg,                /* the seg_dev of interest */
1903         caddr_t addr,                   /* the address of the fault */
1904         size_t len,                     /* the length of the range */
1905         enum fault_type type,           /* type of fault */
1906         enum seg_rw rw,                 /* type of access at fault */
1907         devmap_handle_t *dhp)           /* devmap handle */
1908 {
1909         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1910         register caddr_t a;
1911         struct vpage *vpage;
1912         struct ddi_umem_cookie *kpmem_cookie = NULL;
1913         int err;
1914 
1915         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1916             "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1917             (void *)dhp, (void *)seg, (void *)addr, len);
1918         DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1919             "dhp %p seg %p addr %p len %lx\n",
1920             (void *)dhp, (void *)seg, (void *)addr, len));
1921 
1922         /*
1923          * The seg_dev driver does not implement copy-on-write,
1924          * and always loads translations with maximal allowed permissions
1925          * but we got an fault trying to access the device.
1926          * Servicing the fault is not going to result in any better result
1927          * XXX: If we want to allow devmap_access to handle F_PROT calls,
1928          * This code should be removed and let the normal fault handling
1929          * take care of finding the error
1930          */
1931         if (type == F_PROT) {
1932                 return (FC_PROT);
1933         }
1934 
1935         if (type == F_SOFTUNLOCK) {
1936                 segdev_softunlock(hat, seg, addr, len, rw);
1937                 return (0);
1938         }
1939 
1940         /*
1941          * For kernel pageable memory, fault/lock segkp pages
1942          * We hold this until the completion of this
1943          * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1944          */
1945         if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1946                 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1947                 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1948                         return (err);
1949         }
1950 
1951         /*
1952          * If we have the same protections for the entire segment,
1953          * insure that the access being attempted is legitimate.
1954          */
1955         rw_enter(&sdp->lock, RW_READER);
1956         if (sdp->pageprot == 0) {
1957                 uint_t protchk;
1958 
1959                 switch (rw) {
1960                 case S_READ:
1961                         protchk = PROT_READ;
1962                         break;
1963                 case S_WRITE:
1964                         protchk = PROT_WRITE;
1965                         break;
1966                 case S_EXEC:
1967                         protchk = PROT_EXEC;
1968                         break;
1969                 case S_OTHER:
1970                 default:
1971                         protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1972                         break;
1973                 }
1974 
1975                 if ((sdp->prot & protchk) == 0) {
1976                         rw_exit(&sdp->lock);
1977                         /* undo kpmem locking */
1978                         if (kpmem_cookie != NULL) {
1979                                 release_kpmem_lock(kpmem_cookie, btopr(len));
1980                         }
1981                         return (FC_PROT);       /* illegal access type */
1982                 }
1983         }
1984 
1985         /*
1986          * we do a single hat_devload for the range if
1987          *   - devmap framework (dhp is not NULL),
1988          *   - pageprot == 0, i.e., no per-page protection set and
1989          *   - is device pages, irrespective of whether we are using large pages
1990          */
1991         if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1992                 pfn_t pfnum;
1993                 uint_t hat_flags;
1994 
1995                 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1996                         rw_exit(&sdp->lock);
1997                         return (FC_NOMAP);
1998                 }
1999 
2000                 if (type == F_SOFTLOCK) {
2001                         mutex_enter(&freemem_lock);
2002                         sdp->softlockcnt += btopr(len);
2003                         mutex_exit(&freemem_lock);
2004                 }
2005 
2006                 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2007                 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2008                 ASSERT(!pf_is_memory(pfnum));
2009 
2010                 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2011                     hat_flags | sdp->hat_flags);
2012                 rw_exit(&sdp->lock);
2013                 return (0);
2014         }
2015 
2016         /* Handle cases where we have to loop through fault handling per-page */
2017 
2018         if (sdp->vpage == NULL)
2019                 vpage = NULL;
2020         else
2021                 vpage = &sdp->vpage[seg_page(seg, addr)];
2022 
2023         /* loop over the address range handling each fault */
2024         for (a = addr; a < addr + len; a += PAGESIZE) {
2025                 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2026                         break;
2027                 }
2028                 if (vpage != NULL)
2029                         vpage++;
2030         }
2031         rw_exit(&sdp->lock);
2032         if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2033                 size_t done = (size_t)(a - addr); /* pages fault successfully */
2034                 if (done > 0) {
2035                         /* use softunlock for those pages */
2036                         segdev_softunlock(hat, seg, addr, done, S_OTHER);
2037                 }
2038                 if (kpmem_cookie != NULL) {
2039                         /* release kpmem lock for rest of pages */
2040                         ASSERT(len >= done);
2041                         release_kpmem_lock(kpmem_cookie, btopr(len - done));
2042                 }
2043         } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2044                 /* for non-SOFTLOCK cases, release kpmem */
2045                 release_kpmem_lock(kpmem_cookie, btopr(len));
2046         }
2047         return (err);
2048 }
2049 
2050 /*
2051  * Asynchronous page fault.  We simply do nothing since this
2052  * entry point is not supposed to load up the translation.
2053  */
2054 /*ARGSUSED*/
2055 static faultcode_t
2056 segdev_faulta(struct seg *seg, caddr_t addr)
2057 {
2058         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2059             "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2060         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2061 
2062         return (0);
2063 }
2064 
2065 static int
2066 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2067 {
2068         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2069         register devmap_handle_t *dhp;
2070         register struct vpage *vp, *evp;
2071         devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2072         ulong_t off;
2073         size_t mlen, sz;
2074 
2075         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2076             "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2077             (void *)seg, (void *)addr, len, prot);
2078         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2079 
2080         if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2081                 /*
2082                  * Fail the setprot if pages are SOFTLOCKed through this
2083                  * mapping.
2084                  * Softlockcnt is protected from change by the as read lock.
2085                  */
2086                 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2087                     "segdev_setprot:error softlockcnt=%lx", sz);
2088                 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2089                 return (EAGAIN);
2090         }
2091 
2092         if (dhp_head != NULL) {
2093                 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2094                         return (EINVAL);
2095 
2096                 /*
2097                  * check if violate maxprot.
2098                  */
2099                 off = (ulong_t)(addr - dhp->dh_uvaddr);
2100                 mlen  = len;
2101                 while (dhp) {
2102                         if ((dhp->dh_maxprot & prot) != prot)
2103                                 return (EACCES);        /* violated maxprot */
2104 
2105                         if (mlen > (dhp->dh_len - off)) {
2106                                 mlen -= dhp->dh_len - off;
2107                                 dhp = dhp->dh_next;
2108                                 off = 0;
2109                         } else
2110                                 break;
2111                 }
2112         } else {
2113                 if ((sdp->maxprot & prot) != prot)
2114                         return (EACCES);
2115         }
2116 
2117         rw_enter(&sdp->lock, RW_WRITER);
2118         if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2119                 if (sdp->prot == prot) {
2120                         rw_exit(&sdp->lock);
2121                         return (0);                     /* all done */
2122                 }
2123                 sdp->prot = (uchar_t)prot;
2124         } else {
2125                 sdp->pageprot = 1;
2126                 if (sdp->vpage == NULL) {
2127                         /*
2128                          * First time through setting per page permissions,
2129                          * initialize all the vpage structures to prot
2130                          */
2131                         sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2132                             KM_SLEEP);
2133                         evp = &sdp->vpage[seg_pages(seg)];
2134                         for (vp = sdp->vpage; vp < evp; vp++)
2135                                 VPP_SETPROT(vp, sdp->prot);
2136                 }
2137                 /*
2138                  * Now go change the needed vpages protections.
2139                  */
2140                 evp = &sdp->vpage[seg_page(seg, addr + len)];
2141                 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2142                         VPP_SETPROT(vp, prot);
2143         }
2144         rw_exit(&sdp->lock);
2145 
2146         if (dhp_head != NULL) {
2147                 devmap_handle_t *tdhp;
2148                 /*
2149                  * If large page size was used in hat_devload(),
2150                  * the same page size must be used in hat_unload().
2151                  */
2152                 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2153                 while (tdhp != NULL) {
2154                         if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2155                                 break;
2156                         }
2157                         tdhp = tdhp->dh_next;
2158                 }
2159                 if (tdhp) {
2160                         size_t slen = len;
2161                         size_t mlen;
2162                         size_t soff;
2163 
2164                         soff = (ulong_t)(addr - dhp->dh_uvaddr);
2165                         while (slen != 0) {
2166                                 mlen = MIN(slen, (dhp->dh_len - soff));
2167                                 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2168                                     dhp->dh_len, HAT_UNLOAD);
2169                                 dhp = dhp->dh_next;
2170                                 ASSERT(slen >= mlen);
2171                                 slen -= mlen;
2172                                 soff = 0;
2173                         }
2174                         return (0);
2175                 }
2176         }
2177 
2178         if ((prot & ~PROT_USER) == PROT_NONE) {
2179                 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2180         } else {
2181                 /*
2182                  * RFE: the segment should keep track of all attributes
2183                  * allowing us to remove the deprecated hat_chgprot
2184                  * and use hat_chgattr.
2185                  */
2186                 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2187         }
2188 
2189         return (0);
2190 }
2191 
2192 static int
2193 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2194 {
2195         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2196         struct vpage *vp, *evp;
2197 
2198         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2199             "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2200             (void *)seg, (void *)addr, len, prot);
2201         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2202 
2203         /*
2204          * If segment protection can be used, simply check against them
2205          */
2206         rw_enter(&sdp->lock, RW_READER);
2207         if (sdp->pageprot == 0) {
2208                 register int err;
2209 
2210                 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2211                 rw_exit(&sdp->lock);
2212                 return (err);
2213         }
2214 
2215         /*
2216          * Have to check down to the vpage level
2217          */
2218         evp = &sdp->vpage[seg_page(seg, addr + len)];
2219         for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2220                 if ((VPP_PROT(vp) & prot) != prot) {
2221                         rw_exit(&sdp->lock);
2222                         return (EACCES);
2223                 }
2224         }
2225         rw_exit(&sdp->lock);
2226         return (0);
2227 }
2228 
2229 static int
2230 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2231 {
2232         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2233         size_t pgno;
2234 
2235         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2236             "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2237             (void *)seg, (void *)addr, len, (void *)protv);
2238         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2239 
2240         pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2241         if (pgno != 0) {
2242                 rw_enter(&sdp->lock, RW_READER);
2243                 if (sdp->pageprot == 0) {
2244                         do {
2245                                 protv[--pgno] = sdp->prot;
2246                         } while (pgno != 0);
2247                 } else {
2248                         size_t pgoff = seg_page(seg, addr);
2249 
2250                         do {
2251                                 pgno--;
2252                                 protv[pgno] =
2253                                     VPP_PROT(&sdp->vpage[pgno + pgoff]);
2254                         } while (pgno != 0);
2255                 }
2256                 rw_exit(&sdp->lock);
2257         }
2258         return (0);
2259 }
2260 
2261 static u_offset_t
2262 segdev_getoffset(register struct seg *seg, caddr_t addr)
2263 {
2264         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2265 
2266         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2267             "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2268 
2269         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2270 
2271         return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2272 }
2273 
2274 /*ARGSUSED*/
2275 static int
2276 segdev_gettype(register struct seg *seg, caddr_t addr)
2277 {
2278         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2279 
2280         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2281             "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2282 
2283         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2284 
2285         return (sdp->type);
2286 }
2287 
2288 
2289 /*ARGSUSED*/
2290 static int
2291 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2292 {
2293         register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2294 
2295         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2296             "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2297 
2298         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2299 
2300         /*
2301          * Note that this vp is the common_vp of the device, where the
2302          * pages are hung ..
2303          */
2304         *vpp = VTOCVP(sdp->vp);
2305 
2306         return (0);
2307 }
2308 
2309 static void
2310 segdev_badop(void)
2311 {
2312         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2313             "segdev_badop:start");
2314         panic("segdev_badop");
2315         /*NOTREACHED*/
2316 }
2317 
2318 /*
2319  * segdev pages are not in the cache, and thus can't really be controlled.
2320  * Hence, syncs are simply always successful.
2321  */
2322 /*ARGSUSED*/
2323 static int
2324 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2325 {
2326         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2327 
2328         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2329 
2330         return (0);
2331 }
2332 
2333 /*
2334  * segdev pages are always "in core".
2335  */
2336 /*ARGSUSED*/
2337 static size_t
2338 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2339 {
2340         size_t v = 0;
2341 
2342         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2343 
2344         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2345 
2346         for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2347             v += PAGESIZE)
2348                 *vec++ = 1;
2349         return (v);
2350 }
2351 
2352 /*
2353  * segdev pages are not in the cache, and thus can't really be controlled.
2354  * Hence, locks are simply always successful.
2355  */
2356 /*ARGSUSED*/
2357 static int
2358 segdev_lockop(struct seg *seg, caddr_t addr,
2359     size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2360 {
2361         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2362 
2363         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2364 
2365         return (0);
2366 }
2367 
2368 /*
2369  * segdev pages are not in the cache, and thus can't really be controlled.
2370  * Hence, advise is simply always successful.
2371  */
2372 /*ARGSUSED*/
2373 static int
2374 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2375 {
2376         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2377 
2378         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2379 
2380         return (0);
2381 }
2382 
2383 /*
2384  * segdev pages are not dumped, so we just return
2385  */
2386 /*ARGSUSED*/
2387 static void
2388 segdev_dump(struct seg *seg)
2389 {}
2390 
2391 /*
2392  * ddi_segmap_setup:    Used by drivers who wish specify mapping attributes
2393  *                      for a segment.  Called from a drivers segmap(9E)
2394  *                      routine.
2395  */
2396 /*ARGSUSED*/
2397 int
2398 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2399     off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2400     ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2401 {
2402         struct segdev_crargs dev_a;
2403         int (*mapfunc)(dev_t dev, off_t off, int prot);
2404         uint_t hat_attr;
2405         pfn_t pfn;
2406         int     error, i;
2407 
2408         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2409             "ddi_segmap_setup:start");
2410 
2411         if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2412                 return (ENODEV);
2413 
2414         /*
2415          * Character devices that support the d_mmap
2416          * interface can only be mmap'ed shared.
2417          */
2418         if ((flags & MAP_TYPE) != MAP_SHARED)
2419                 return (EINVAL);
2420 
2421         /*
2422          * Check that this region is indeed mappable on this platform.
2423          * Use the mapping function.
2424          */
2425         if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2426                 return (ENXIO);
2427 
2428         /*
2429          * Check to ensure that the entire range is
2430          * legal and we are not trying to map in
2431          * more than the device will let us.
2432          */
2433         for (i = 0; i < len; i += PAGESIZE) {
2434                 if (i == 0) {
2435                         /*
2436                          * Save the pfn at offset here. This pfn will be
2437                          * used later to get user address.
2438                          */
2439                         if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2440                             maxprot)) == PFN_INVALID)
2441                                 return (ENXIO);
2442                 } else {
2443                         if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2444                             PFN_INVALID)
2445                                 return (ENXIO);
2446                 }
2447         }
2448 
2449         as_rangelock(as);
2450         /* Pick an address w/o worrying about any vac alignment constraints. */
2451         error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2452         if (error != 0) {
2453                 as_rangeunlock(as);
2454                 return (error);
2455         }
2456 
2457         dev_a.mapfunc = mapfunc;
2458         dev_a.dev = dev;
2459         dev_a.offset = (offset_t)offset;
2460         dev_a.type = flags & MAP_TYPE;
2461         dev_a.prot = (uchar_t)prot;
2462         dev_a.maxprot = (uchar_t)maxprot;
2463         dev_a.hat_attr = hat_attr;
2464         dev_a.hat_flags = 0;
2465         dev_a.devmap_data = NULL;
2466 
2467         error = as_map(as, *addrp, len, segdev_create, &dev_a);
2468         as_rangeunlock(as);
2469         return (error);
2470 
2471 }
2472 
2473 /*ARGSUSED*/
2474 static int
2475 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2476     struct page ***ppp, enum lock_type type, enum seg_rw rw)
2477 {
2478         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2479             "segdev_pagelock:start");
2480         return (ENOTSUP);
2481 }
2482 
2483 /*ARGSUSED*/
2484 static int
2485 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2486     uint_t szc)
2487 {
2488         return (ENOTSUP);
2489 }
2490 
2491 /*
2492  * devmap_device: Used by devmap framework to establish mapping
2493  *                called by devmap_seup(9F) during map setup time.
2494  */
2495 /*ARGSUSED*/
2496 static int
2497 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2498     offset_t off, size_t len, uint_t flags)
2499 {
2500         devmap_handle_t *rdhp, *maxdhp;
2501         struct segdev_crargs dev_a;
2502         int     err;
2503         uint_t maxprot = PROT_ALL;
2504         offset_t offset = 0;
2505         pfn_t pfn;
2506         struct devmap_pmem_cookie *pcp;
2507 
2508         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2509             "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2510             (void *)dhp, (void *)addr, off, len);
2511 
2512         DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2513             (void *)dhp, (void *)addr, off, len));
2514 
2515         as_rangelock(as);
2516         if ((flags & MAP_FIXED) == 0) {
2517                 offset_t aligned_off;
2518 
2519                 rdhp = maxdhp = dhp;
2520                 while (rdhp != NULL) {
2521                         maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2522                             maxdhp : rdhp;
2523                         rdhp = rdhp->dh_next;
2524                         maxprot |= dhp->dh_maxprot;
2525                 }
2526                 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2527 
2528                 /*
2529                  * Use the dhp that has the
2530                  * largest len to get user address.
2531                  */
2532                 /*
2533                  * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2534                  * use 0 which is as good as any other.
2535                  */
2536                 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2537                         aligned_off = (offset_t)0;
2538                 } else if (dhp_is_devmem(maxdhp)) {
2539                         aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2540                 } else if (dhp_is_pmem(maxdhp)) {
2541                         pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2542                         pfn = page_pptonum(
2543                             pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2544                         aligned_off = (offset_t)ptob(pfn) - offset;
2545                 } else {
2546                         aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2547                             offset;
2548                 }
2549 
2550                 /*
2551                  * Pick an address aligned to dh_cookie.
2552                  * for kernel memory/user memory, cookie is cvaddr.
2553                  * for device memory, cookie is physical address.
2554                  */
2555                 map_addr(addr, len, aligned_off, 1, flags);
2556                 if (*addr == NULL) {
2557                         as_rangeunlock(as);
2558                         return (ENOMEM);
2559                 }
2560         } else {
2561                 /*
2562                  * User-specified address; blow away any previous mappings.
2563                  */
2564                 (void) as_unmap(as, *addr, len);
2565         }
2566 
2567         dev_a.mapfunc = NULL;
2568         dev_a.dev = dhp->dh_dev;
2569         dev_a.type = flags & MAP_TYPE;
2570         dev_a.offset = off;
2571         /*
2572          * sdp->maxprot has the least restrict protection of all dhps.
2573          */
2574         dev_a.maxprot = maxprot;
2575         dev_a.prot = dhp->dh_prot;
2576         /*
2577          * devmap uses dhp->dh_hat_attr for hat.
2578          */
2579         dev_a.hat_flags = 0;
2580         dev_a.hat_attr = 0;
2581         dev_a.devmap_data = (void *)dhp;
2582 
2583         err = as_map(as, *addr, len, segdev_create, &dev_a);
2584         as_rangeunlock(as);
2585         return (err);
2586 }
2587 
2588 int
2589 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2590     uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2591     size_t, uint_t, uint_t))
2592 {
2593         register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2594         struct devmap_ctx *devctx;
2595         int do_timeout = 0;
2596         int ret;
2597 
2598 #ifdef lint
2599         pvtp = pvtp;
2600 #endif
2601 
2602         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2603             "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2604             (void *)dhp, off, len);
2605         DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2606             (void *)dhp, off, len));
2607 
2608         if (ctxmgt == NULL)
2609                 return (FC_HWERR);
2610 
2611         devctx = dhp->dh_ctx;
2612 
2613         /*
2614          * If we are on an MP system with more than one cpu running
2615          * and if a thread on some CPU already has the context, wait
2616          * for it to finish if there is a hysteresis timeout.
2617          *
2618          * We call cv_wait() instead of cv_wait_sig() because
2619          * it does not matter much if it returned due to a signal
2620          * or due to a cv_signal() or cv_broadcast().  In either event
2621          * we need to complete the mapping otherwise the processes
2622          * will die with a SEGV.
2623          */
2624         if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2625                 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2626                     "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2627                     devctx, dhp);
2628                 do_timeout = 1;
2629                 mutex_enter(&devctx->lock);
2630                 while (devctx->oncpu)
2631                         cv_wait(&devctx->cv, &devctx->lock);
2632                 devctx->oncpu = 1;
2633                 mutex_exit(&devctx->lock);
2634         }
2635 
2636         /*
2637          * Call the contextmgt callback so that the driver can handle
2638          * the fault.
2639          */
2640         ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2641 
2642         /*
2643          * If devmap_access() returned -1, then there was a hardware
2644          * error so we need to convert the return value to something
2645          * that trap() will understand.  Otherwise, the return value
2646          * is already a fault code generated by devmap_unload()
2647          * or devmap_load().
2648          */
2649         if (ret) {
2650                 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2651                     "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2652                     ret, dhp, devctx);
2653                 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2654                     ret, (void *)dhp));
2655                 if (devctx->oncpu) {
2656                         mutex_enter(&devctx->lock);
2657                         devctx->oncpu = 0;
2658                         cv_signal(&devctx->cv);
2659                         mutex_exit(&devctx->lock);
2660                 }
2661                 return (FC_HWERR);
2662         }
2663 
2664         /*
2665          * Setup the timeout if we need to
2666          */
2667         if (do_timeout) {
2668                 mutex_enter(&devctx->lock);
2669                 if (dhp->dh_timeout_length > 0) {
2670                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2671                             "devmap_do_ctxmgt:timeout set");
2672                         devctx->timeout = timeout(devmap_ctxto,
2673                             devctx, dhp->dh_timeout_length);
2674                 } else {
2675                         /*
2676                          * We don't want to wait so set oncpu to
2677                          * 0 and wake up anyone waiting.
2678                          */
2679                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2680                             "devmap_do_ctxmgt:timeout not set");
2681                         devctx->oncpu = 0;
2682                         cv_signal(&devctx->cv);
2683                 }
2684                 mutex_exit(&devctx->lock);
2685         }
2686 
2687         return (DDI_SUCCESS);
2688 }
2689 
2690 /*
2691  *                                       end of mapping
2692  *                    poff   fault_offset         |
2693  *            base     |        |                 |
2694  *              |      |        |                 |
2695  *              V      V        V                 V
2696  *  +-----------+---------------+-------+---------+-------+
2697  *              ^               ^       ^         ^
2698  *              |<--- offset--->|<-len->|         |
2699  *              |<--- dh_len(size of mapping) --->|
2700  *                     |<--  pg -->|
2701  *                              -->|rlen|<--
2702  */
2703 static ulong_t
2704 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2705     ulong_t *opfn, ulong_t *pagesize)
2706 {
2707         register int level;
2708         ulong_t pg;
2709         ulong_t poff;
2710         ulong_t base;
2711         caddr_t uvaddr;
2712         long rlen;
2713 
2714         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2715             "devmap_roundup:start dhp=%p off=%lx len=%lx",
2716             (void *)dhp, offset, len);
2717         DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2718             (void *)dhp, offset, len));
2719 
2720         /*
2721          * get the max. pagesize that is aligned within the range
2722          * <dh_pfn, dh_pfn+offset>.
2723          *
2724          * The calculations below use physical address to ddetermine
2725          * the page size to use. The same calculations can use the
2726          * virtual address to determine the page size.
2727          */
2728         base = (ulong_t)ptob(dhp->dh_pfn);
2729         for (level = dhp->dh_mmulevel; level >= 0; level--) {
2730                 pg = page_get_pagesize(level);
2731                 poff = ((base + offset) & ~(pg - 1));
2732                 uvaddr = dhp->dh_uvaddr + (poff - base);
2733                 if ((poff >= base) &&
2734                     ((poff + pg) <= (base + dhp->dh_len)) &&
2735                     VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2736                         break;
2737         }
2738 
2739         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2740             "devmap_roundup: base=%lx poff=%lx dhp=%p",
2741             base, poff, dhp);
2742         DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2743             base, poff, dhp->dh_pfn));
2744 
2745         ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2746         ASSERT(level >= 0);
2747 
2748         *pagesize = pg;
2749         *opfn = dhp->dh_pfn + btop(poff - base);
2750 
2751         rlen = len + offset - (poff - base + pg);
2752 
2753         ASSERT(rlen < (long)len);
2754 
2755         TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2756             "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2757             (void *)dhp, level, rlen, pagesize, opfn);
2758         DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2759             "level %x rlen %lx psize %lx opfn %lx\n",
2760             (void *)dhp, level, rlen, *pagesize, *opfn));
2761 
2762         return ((ulong_t)((rlen > 0) ? rlen : 0));
2763 }
2764 
2765 /*
2766  * find the dhp that contains addr.
2767  */
2768 static devmap_handle_t *
2769 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2770 {
2771         devmap_handle_t *dhp;
2772 
2773         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2774             "devmap_find_handle:start");
2775 
2776         dhp = dhp_head;
2777         while (dhp) {
2778                 if (addr >= dhp->dh_uvaddr &&
2779                     addr < (dhp->dh_uvaddr + dhp->dh_len))
2780                         return (dhp);
2781                 dhp = dhp->dh_next;
2782         }
2783 
2784         return ((devmap_handle_t *)NULL);
2785 }
2786 
2787 /*
2788  * devmap_unload:
2789  *                      Marks a segdev segment or pages if offset->offset+len
2790  *                      is not the entire segment as intercept and unloads the
2791  *                      pages in the range offset -> offset+len.
2792  */
2793 int
2794 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2795 {
2796         register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2797         caddr_t addr;
2798         ulong_t size;
2799         ssize_t soff;
2800 
2801         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2802             "devmap_unload:start dhp=%p offset=%llx len=%lx",
2803             (void *)dhp, offset, len);
2804         DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2805             (void *)dhp, offset, len));
2806 
2807         soff = (ssize_t)(offset - dhp->dh_uoff);
2808         soff = round_down_p2(soff, PAGESIZE);
2809         if (soff < 0 || soff >= dhp->dh_len)
2810                 return (FC_MAKE_ERR(EINVAL));
2811 
2812         /*
2813          * Address and size must be page aligned.  Len is set to the
2814          * number of bytes in the number of pages that are required to
2815          * support len.  Offset is set to the byte offset of the first byte
2816          * of the page that contains offset.
2817          */
2818         len = round_up_p2(len, PAGESIZE);
2819 
2820         /*
2821          * If len is == 0, then calculate the size by getting
2822          * the number of bytes from offset to the end of the segment.
2823          */
2824         if (len == 0)
2825                 size = dhp->dh_len - soff;
2826         else {
2827                 size = len;
2828                 if ((soff + size) > dhp->dh_len)
2829                         return (FC_MAKE_ERR(EINVAL));
2830         }
2831 
2832         /*
2833          * The address is offset bytes from the base address of
2834          * the dhp.
2835          */
2836         addr = (caddr_t)(soff + dhp->dh_uvaddr);
2837 
2838         /*
2839          * If large page size was used in hat_devload(),
2840          * the same page size must be used in hat_unload().
2841          */
2842         if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2843                 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2844                     dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2845         } else {
2846                 hat_unload(dhp->dh_seg->s_as->a_hat,  addr, size,
2847                     HAT_UNLOAD|HAT_UNLOAD_OTHER);
2848         }
2849 
2850         return (0);
2851 }
2852 
2853 /*
2854  * calculates the optimal page size that will be used for hat_devload().
2855  */
2856 static void
2857 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2858     size_t *llen, caddr_t *laddr)
2859 {
2860         ulong_t off;
2861         ulong_t pfn;
2862         ulong_t pgsize;
2863         uint_t first = 1;
2864 
2865         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2866             "devmap_get_large_pgsize:start");
2867 
2868         /*
2869          * RFE - Code only supports large page mappings for devmem
2870          * This code could be changed in future if we want to support
2871          * large page mappings for kernel exported memory.
2872          */
2873         ASSERT(dhp_is_devmem(dhp));
2874         ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2875 
2876         *llen = 0;
2877         off = (ulong_t)(addr - dhp->dh_uvaddr);
2878         while ((long)len > 0) {
2879                 /*
2880                  * get the optimal pfn to minimize address translations.
2881                  * devmap_roundup() returns residue bytes for next round
2882                  * calculations.
2883                  */
2884                 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2885 
2886                 if (first) {
2887                         *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2888                         first = 0;
2889                 }
2890 
2891                 *llen += pgsize;
2892                 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2893         }
2894         /* Large page mapping len/addr cover more range than original fault */
2895         ASSERT(*llen >= len && *laddr <= addr);
2896         ASSERT((*laddr + *llen) >= (addr + len));
2897 }
2898 
2899 /*
2900  * Initialize the devmap_softlock structure.
2901  */
2902 static struct devmap_softlock *
2903 devmap_softlock_init(dev_t dev, ulong_t id)
2904 {
2905         struct devmap_softlock *slock;
2906         struct devmap_softlock *tmp;
2907 
2908         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2909             "devmap_softlock_init:start");
2910 
2911         tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2912         mutex_enter(&devmap_slock);
2913 
2914         for (slock = devmap_slist; slock != NULL; slock = slock->next)
2915                 if ((slock->dev == dev) && (slock->id == id))
2916                         break;
2917 
2918         if (slock == NULL) {
2919                 slock = tmp;
2920                 slock->dev = dev;
2921                 slock->id = id;
2922                 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2923                 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2924                 slock->next = devmap_slist;
2925                 devmap_slist = slock;
2926         } else
2927                 kmem_free(tmp, sizeof (struct devmap_softlock));
2928 
2929         mutex_enter(&slock->lock);
2930         slock->refcnt++;
2931         mutex_exit(&slock->lock);
2932         mutex_exit(&devmap_slock);
2933 
2934         return (slock);
2935 }
2936 
2937 /*
2938  * Wake up processes that sleep on softlocked.
2939  * Free dh_softlock if refcnt is 0.
2940  */
2941 static void
2942 devmap_softlock_rele(devmap_handle_t *dhp)
2943 {
2944         struct devmap_softlock *slock = dhp->dh_softlock;
2945         struct devmap_softlock *tmp;
2946         struct devmap_softlock *parent;
2947 
2948         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2949             "devmap_softlock_rele:start");
2950 
2951         mutex_enter(&devmap_slock);
2952         mutex_enter(&slock->lock);
2953 
2954         ASSERT(slock->refcnt > 0);
2955 
2956         slock->refcnt--;
2957 
2958         /*
2959          * If no one is using the device, free up the slock data.
2960          */
2961         if (slock->refcnt == 0) {
2962                 slock->softlocked = 0;
2963                 cv_signal(&slock->cv);
2964 
2965                 if (devmap_slist == slock)
2966                         devmap_slist = slock->next;
2967                 else {
2968                         parent = devmap_slist;
2969                         for (tmp = devmap_slist->next; tmp != NULL;
2970                             tmp = tmp->next) {
2971                                 if (tmp == slock) {
2972                                         parent->next = tmp->next;
2973                                         break;
2974                                 }
2975                                 parent = tmp;
2976                         }
2977                 }
2978                 mutex_exit(&slock->lock);
2979                 mutex_destroy(&slock->lock);
2980                 cv_destroy(&slock->cv);
2981                 kmem_free(slock, sizeof (struct devmap_softlock));
2982         } else
2983                 mutex_exit(&slock->lock);
2984 
2985         mutex_exit(&devmap_slock);
2986 }
2987 
2988 /*
2989  * Wake up processes that sleep on dh_ctx->locked.
2990  * Free dh_ctx if refcnt is 0.
2991  */
2992 static void
2993 devmap_ctx_rele(devmap_handle_t *dhp)
2994 {
2995         struct devmap_ctx *devctx = dhp->dh_ctx;
2996         struct devmap_ctx *tmp;
2997         struct devmap_ctx *parent;
2998         timeout_id_t tid;
2999 
3000         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3001             "devmap_ctx_rele:start");
3002 
3003         mutex_enter(&devmapctx_lock);
3004         mutex_enter(&devctx->lock);
3005 
3006         ASSERT(devctx->refcnt > 0);
3007 
3008         devctx->refcnt--;
3009 
3010         /*
3011          * If no one is using the device, free up the devctx data.
3012          */
3013         if (devctx->refcnt == 0) {
3014                 /*
3015                  * Untimeout any threads using this mapping as they are about
3016                  * to go away.
3017                  */
3018                 if (devctx->timeout != 0) {
3019                         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3020                             "devmap_ctx_rele:untimeout ctx->timeout");
3021 
3022                         tid = devctx->timeout;
3023                         mutex_exit(&devctx->lock);
3024                         (void) untimeout(tid);
3025                         mutex_enter(&devctx->lock);
3026                 }
3027 
3028                 devctx->oncpu = 0;
3029                 cv_signal(&devctx->cv);
3030 
3031                 if (devmapctx_list == devctx)
3032                         devmapctx_list = devctx->next;
3033                 else {
3034                         parent = devmapctx_list;
3035                         for (tmp = devmapctx_list->next; tmp != NULL;
3036                             tmp = tmp->next) {
3037                                 if (tmp == devctx) {
3038                                         parent->next = tmp->next;
3039                                         break;
3040                                 }
3041                                 parent = tmp;
3042                         }
3043                 }
3044                 mutex_exit(&devctx->lock);
3045                 mutex_destroy(&devctx->lock);
3046                 cv_destroy(&devctx->cv);
3047                 kmem_free(devctx, sizeof (struct devmap_ctx));
3048         } else
3049                 mutex_exit(&devctx->lock);
3050 
3051         mutex_exit(&devmapctx_lock);
3052 }
3053 
3054 /*
3055  * devmap_load:
3056  *                      Marks a segdev segment or pages if offset->offset+len
3057  *                      is not the entire segment as nointercept and faults in
3058  *                      the pages in the range offset -> offset+len.
3059  */
3060 int
3061 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3062     uint_t rw)
3063 {
3064         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3065         struct as *asp = dhp->dh_seg->s_as;
3066         caddr_t addr;
3067         ulong_t size;
3068         ssize_t soff;   /* offset from the beginning of the segment */
3069         int rc;
3070 
3071         TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3072             "devmap_load:start dhp=%p offset=%llx len=%lx",
3073             (void *)dhp, offset, len);
3074 
3075         DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3076             (void *)dhp, offset, len));
3077 
3078         /*
3079          *      Hat layer only supports devload to process' context for which
3080          *      the as lock is held. Verify here and return error if drivers
3081          *      inadvertently call devmap_load on a wrong devmap handle.
3082          */
3083         if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3084                 return (FC_MAKE_ERR(EINVAL));
3085 
3086         soff = (ssize_t)(offset - dhp->dh_uoff);
3087         soff = round_down_p2(soff, PAGESIZE);
3088         if (soff < 0 || soff >= dhp->dh_len)
3089                 return (FC_MAKE_ERR(EINVAL));
3090 
3091         /*
3092          * Address and size must be page aligned.  Len is set to the
3093          * number of bytes in the number of pages that are required to
3094          * support len.  Offset is set to the byte offset of the first byte
3095          * of the page that contains offset.
3096          */
3097         len = round_up_p2(len, PAGESIZE);
3098 
3099         /*
3100          * If len == 0, then calculate the size by getting
3101          * the number of bytes from offset to the end of the segment.
3102          */
3103         if (len == 0)
3104                 size = dhp->dh_len - soff;
3105         else {
3106                 size = len;
3107                 if ((soff + size) > dhp->dh_len)
3108                         return (FC_MAKE_ERR(EINVAL));
3109         }
3110 
3111         /*
3112          * The address is offset bytes from the base address of
3113          * the segment.
3114          */
3115         addr = (caddr_t)(soff + dhp->dh_uvaddr);
3116 
3117         HOLD_DHP_LOCK(dhp);
3118         rc = segdev_faultpages(asp->a_hat,
3119             dhp->dh_seg, addr, size, type, rw, dhp);
3120         RELE_DHP_LOCK(dhp);
3121         return (rc);
3122 }
3123 
3124 int
3125 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3126     size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3127 {
3128         register devmap_handle_t *dhp;
3129         int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3130             size_t *, uint_t);
3131         int (*mmap)(dev_t, off_t, int);
3132         struct devmap_callback_ctl *callbackops;
3133         devmap_handle_t *dhp_head = NULL;
3134         devmap_handle_t *dhp_prev = NULL;
3135         devmap_handle_t *dhp_curr;
3136         caddr_t addr;
3137         int map_flag;
3138         int ret;
3139         ulong_t total_len;
3140         size_t map_len;
3141         size_t resid_len = len;
3142         offset_t map_off = off;
3143         struct devmap_softlock *slock = NULL;
3144 
3145 #ifdef lint
3146         cred = cred;
3147 #endif
3148 
3149         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3150             "devmap_setup:start off=%llx len=%lx", off, len);
3151         DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3152             off, len));
3153 
3154         devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3155         mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3156 
3157         /*
3158          * driver must provide devmap(9E) entry point in cb_ops to use the
3159          * devmap framework.
3160          */
3161         if (devmap == NULL || devmap == nulldev || devmap == nodev)
3162                 return (EINVAL);
3163 
3164         /*
3165          * To protect from an inadvertent entry because the devmap entry point
3166          * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3167          * mmap is NULL.
3168          */
3169         map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3170         if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3171                 return (EINVAL);
3172 
3173         /*
3174          * devmap allows mmap(2) to map multiple registers.
3175          * one devmap_handle is created for each register mapped.
3176          */
3177         for (total_len = 0; total_len < len; total_len += map_len) {
3178                 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3179 
3180                 if (dhp_prev != NULL)
3181                         dhp_prev->dh_next = dhp;
3182                 else
3183                         dhp_head = dhp;
3184                 dhp_prev = dhp;
3185 
3186                 dhp->dh_prot = prot;
3187                 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3188                 dhp->dh_dev = dev;
3189                 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3190                 dhp->dh_uoff = map_off;
3191 
3192                 /*
3193                  * Get mapping specific info from
3194                  * the driver, such as rnumber, roff, len, callbackops,
3195                  * accattrp and, if the mapping is for kernel memory,
3196                  * ddi_umem_cookie.
3197                  */
3198                 if ((ret = cdev_devmap(dev, dhp, map_off,
3199                     resid_len, &map_len, get_udatamodel())) != 0) {
3200                         free_devmap_handle(dhp_head);
3201                         return (ENXIO);
3202                 }
3203 
3204                 if (map_len & PAGEOFFSET) {
3205                         free_devmap_handle(dhp_head);
3206                         return (EINVAL);
3207                 }
3208 
3209                 callbackops = &dhp->dh_callbackops;
3210 
3211                 if ((callbackops->devmap_access == NULL) ||
3212                     (callbackops->devmap_access == nulldev) ||
3213                     (callbackops->devmap_access == nodev)) {
3214                         /*
3215                          * Normally devmap does not support MAP_PRIVATE unless
3216                          * the drivers provide a valid devmap_access routine.
3217                          */
3218                         if ((flags & MAP_PRIVATE) != 0) {
3219                                 free_devmap_handle(dhp_head);
3220                                 return (EINVAL);
3221                         }
3222                 } else {
3223                         /*
3224                          * Initialize dhp_softlock and dh_ctx if the drivers
3225                          * provide devmap_access.
3226                          */
3227                         dhp->dh_softlock = devmap_softlock_init(dev,
3228                             (ulong_t)callbackops->devmap_access);
3229                         dhp->dh_ctx = devmap_ctxinit(dev,
3230                             (ulong_t)callbackops->devmap_access);
3231 
3232                         /*
3233                          * segdev_fault can only work when all
3234                          * dh_softlock in a multi-dhp mapping
3235                          * are same. see comments in segdev_fault
3236                          * This code keeps track of the first
3237                          * dh_softlock allocated in slock and
3238                          * compares all later allocations and if
3239                          * not similar, returns an error.
3240                          */
3241                         if (slock == NULL)
3242                                 slock = dhp->dh_softlock;
3243                         if (slock != dhp->dh_softlock) {
3244                                 free_devmap_handle(dhp_head);
3245                                 return (ENOTSUP);
3246                         }
3247                 }
3248 
3249                 map_off += map_len;
3250                 resid_len -= map_len;
3251         }
3252 
3253         /*
3254          * get the user virtual address and establish the mapping between
3255          * uvaddr and device physical address.
3256          */
3257         if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3258             != 0) {
3259                 /*
3260                  * free devmap handles if error during the mapping.
3261                  */
3262                 free_devmap_handle(dhp_head);
3263 
3264                 return (ret);
3265         }
3266 
3267         /*
3268          * call the driver's devmap_map callback to do more after the mapping,
3269          * such as to allocate driver private data for context management.
3270          */
3271         dhp = dhp_head;
3272         map_off = off;
3273         addr = *addrp;
3274         while (dhp != NULL) {
3275                 callbackops = &dhp->dh_callbackops;
3276                 dhp->dh_uvaddr = addr;
3277                 dhp_curr = dhp;
3278                 if (callbackops->devmap_map != NULL) {
3279                         ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3280                             dev, flags, map_off,
3281                             dhp->dh_len, &dhp->dh_pvtp);
3282                         if (ret != 0) {
3283                                 struct segdev_data *sdp;
3284 
3285                                 /*
3286                                  * call driver's devmap_unmap entry point
3287                                  * to free driver resources.
3288                                  */
3289                                 dhp = dhp_head;
3290                                 map_off = off;
3291                                 while (dhp != dhp_curr) {
3292                                         callbackops = &dhp->dh_callbackops;
3293                                         if (callbackops->devmap_unmap != NULL) {
3294                                                 (*callbackops->devmap_unmap)(
3295                                                     dhp, dhp->dh_pvtp,
3296                                                     map_off, dhp->dh_len,
3297                                                     NULL, NULL, NULL, NULL);
3298                                         }
3299                                         map_off += dhp->dh_len;
3300                                         dhp = dhp->dh_next;
3301                                 }
3302                                 sdp = dhp_head->dh_seg->s_data;
3303                                 sdp->devmap_data = NULL;
3304                                 free_devmap_handle(dhp_head);
3305                                 return (ENXIO);
3306                         }
3307                 }
3308                 map_off += dhp->dh_len;
3309                 addr += dhp->dh_len;
3310                 dhp = dhp->dh_next;
3311         }
3312 
3313         return (0);
3314 }
3315 
3316 int
3317 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3318     off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3319 {
3320         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3321             "devmap_segmap:start");
3322         return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3323             (size_t)len, prot, maxprot, flags, cred));
3324 }
3325 
3326 /*
3327  * Called from devmap_devmem_setup/remap to see if can use large pages for
3328  * this device mapping.
3329  * Also calculate the max. page size for this mapping.
3330  * this page size will be used in fault routine for
3331  * optimal page size calculations.
3332  */
3333 static void
3334 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3335 {
3336         ASSERT(dhp_is_devmem(dhp));
3337         dhp->dh_mmulevel = 0;
3338 
3339         /*
3340          * use large page size only if:
3341          *  1. device memory.
3342          *  2. mmu supports multiple page sizes,
3343          *  3. Driver did not disallow it
3344          *  4. dhp length is at least as big as the large pagesize
3345          *  5. the uvaddr and pfn are large pagesize aligned
3346          */
3347         if (page_num_pagesizes() > 1 &&
3348             !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3349                 ulong_t base;
3350                 int level;
3351 
3352                 base = (ulong_t)ptob(dhp->dh_pfn);
3353                 for (level = 1; level < page_num_pagesizes(); level++) {
3354                         size_t pgsize = page_get_pagesize(level);
3355                         if ((dhp->dh_len < pgsize) ||
3356                             (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3357                             base, pgsize))) {
3358                                 break;
3359                         }
3360                 }
3361                 dhp->dh_mmulevel = level - 1;
3362         }
3363         if (dhp->dh_mmulevel > 0) {
3364                 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3365         } else {
3366                 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3367         }
3368 }
3369 
3370 /*
3371  * Called by driver devmap routine to pass device specific info to
3372  * the framework.    used for device memory mapping only.
3373  */
3374 int
3375 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3376     struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3377     size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3378 {
3379         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3380         ddi_acc_handle_t handle;
3381         ddi_map_req_t mr;
3382         ddi_acc_hdl_t *hp;
3383         int err;
3384 
3385         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3386             "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3387             (void *)dhp, roff, rnumber, (uint_t)len);
3388         DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3389             "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3390 
3391         /*
3392          * First to check if this function has been called for this dhp.
3393          */
3394         if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3395                 return (DDI_FAILURE);
3396 
3397         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3398                 return (DDI_FAILURE);
3399 
3400         if (flags & DEVMAP_MAPPING_INVALID) {
3401                 /*
3402                  * Don't go up the tree to get pfn if the driver specifies
3403                  * DEVMAP_MAPPING_INVALID in flags.
3404                  *
3405                  * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3406                  * remap permission.
3407                  */
3408                 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3409                         return (DDI_FAILURE);
3410                 }
3411                 dhp->dh_pfn = PFN_INVALID;
3412         } else {
3413                 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3414                 if (handle == NULL)
3415                         return (DDI_FAILURE);
3416 
3417                 hp = impl_acc_hdl_get(handle);
3418                 hp->ah_vers = VERS_ACCHDL;
3419                 hp->ah_dip = dip;
3420                 hp->ah_rnumber = rnumber;
3421                 hp->ah_offset = roff;
3422                 hp->ah_len = len;
3423                 if (accattrp != NULL)
3424                         hp->ah_acc = *accattrp;
3425 
3426                 mr.map_op = DDI_MO_MAP_LOCKED;
3427                 mr.map_type = DDI_MT_RNUMBER;
3428                 mr.map_obj.rnumber = rnumber;
3429                 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3430                 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3431                 mr.map_handlep = hp;
3432                 mr.map_vers = DDI_MAP_VERSION;
3433 
3434                 /*
3435                  * up the device tree to get pfn.
3436                  * The rootnex_map_regspec() routine in nexus drivers has been
3437                  * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3438                  */
3439                 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3440                 dhp->dh_hat_attr = hp->ah_hat_flags;
3441                 impl_acc_hdl_free(handle);
3442 
3443                 if (err)
3444                         return (DDI_FAILURE);
3445         }
3446         /* Should not be using devmem setup for memory pages */
3447         ASSERT(!pf_is_memory(dhp->dh_pfn));
3448 
3449         /* Only some of the flags bits are settable by the driver */
3450         dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3451         dhp->dh_len = ptob(btopr(len));
3452 
3453         dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3454         dhp->dh_roff = ptob(btop(roff));
3455 
3456         /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3457         devmap_devmem_large_page_setup(dhp);
3458         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3459         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3460 
3461 
3462         if (callbackops != NULL) {
3463                 bcopy(callbackops, &dhp->dh_callbackops,
3464                     sizeof (struct devmap_callback_ctl));
3465         }
3466 
3467         /*
3468          * Initialize dh_lock if we want to do remap.
3469          */
3470         if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3471                 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3472                 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3473         }
3474 
3475         dhp->dh_flags |= DEVMAP_SETUP_DONE;
3476 
3477         return (DDI_SUCCESS);
3478 }
3479 
3480 int
3481 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3482     uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3483     uint_t flags, ddi_device_acc_attr_t *accattrp)
3484 {
3485         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3486         ddi_acc_handle_t handle;
3487         ddi_map_req_t mr;
3488         ddi_acc_hdl_t *hp;
3489         pfn_t   pfn;
3490         uint_t  hat_flags;
3491         int     err;
3492 
3493         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3494             "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3495             (void *)dhp, roff, rnumber, (uint_t)len);
3496         DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3497             "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3498 
3499         /*
3500          * Return failure if setup has not been done or no remap permission
3501          * has been granted during the setup.
3502          */
3503         if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3504             (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3505                 return (DDI_FAILURE);
3506 
3507         /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3508         if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3509                 return (DDI_FAILURE);
3510 
3511         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3512                 return (DDI_FAILURE);
3513 
3514         if (!(flags & DEVMAP_MAPPING_INVALID)) {
3515                 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3516                 if (handle == NULL)
3517                         return (DDI_FAILURE);
3518         }
3519 
3520         HOLD_DHP_LOCK(dhp);
3521 
3522         /*
3523          * Unload the old mapping, so next fault will setup the new mappings
3524          * Do this while holding the dhp lock so other faults dont reestablish
3525          * the mappings
3526          */
3527         hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3528             dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3529 
3530         if (flags & DEVMAP_MAPPING_INVALID) {
3531                 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3532                 dhp->dh_pfn = PFN_INVALID;
3533         } else {
3534                 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3535                 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3536                 hp = impl_acc_hdl_get(handle);
3537                 hp->ah_vers = VERS_ACCHDL;
3538                 hp->ah_dip = dip;
3539                 hp->ah_rnumber = rnumber;
3540                 hp->ah_offset = roff;
3541                 hp->ah_len = len;
3542                 if (accattrp != NULL)
3543                         hp->ah_acc = *accattrp;
3544 
3545                 mr.map_op = DDI_MO_MAP_LOCKED;
3546                 mr.map_type = DDI_MT_RNUMBER;
3547                 mr.map_obj.rnumber = rnumber;
3548                 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3549                 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3550                 mr.map_handlep = hp;
3551                 mr.map_vers = DDI_MAP_VERSION;
3552 
3553                 /*
3554                  * up the device tree to get pfn.
3555                  * The rootnex_map_regspec() routine in nexus drivers has been
3556                  * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3557                  */
3558                 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3559                 hat_flags = hp->ah_hat_flags;
3560                 impl_acc_hdl_free(handle);
3561                 if (err) {
3562                         RELE_DHP_LOCK(dhp);
3563                         return (DDI_FAILURE);
3564                 }
3565                 /*
3566                  * Store result of ddi_map first in local variables, as we do
3567                  * not want to overwrite the existing dhp with wrong data.
3568                  */
3569                 dhp->dh_pfn = pfn;
3570                 dhp->dh_hat_attr = hat_flags;
3571         }
3572 
3573         /* clear the large page size flag */
3574         dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3575 
3576         dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3577         dhp->dh_roff = ptob(btop(roff));
3578 
3579         /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3580         devmap_devmem_large_page_setup(dhp);
3581         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3582         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3583 
3584         RELE_DHP_LOCK(dhp);
3585         return (DDI_SUCCESS);
3586 }
3587 
3588 /*
3589  * called by driver devmap routine to pass kernel virtual address  mapping
3590  * info to the framework.    used only for kernel memory
3591  * allocated from ddi_umem_alloc().
3592  */
3593 int
3594 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3595     struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3596     offset_t off, size_t len, uint_t maxprot, uint_t flags,
3597     ddi_device_acc_attr_t *accattrp)
3598 {
3599         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3600         struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3601 
3602 #ifdef lint
3603         dip = dip;
3604 #endif
3605 
3606         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3607             "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3608             (void *)dhp, off, cookie, len);
3609         DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3610             "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3611 
3612         if (cookie == NULL)
3613                 return (DDI_FAILURE);
3614 
3615         /* For UMEM_TRASH, this restriction is not needed */
3616         if ((off + len) > cp->size)
3617                 return (DDI_FAILURE);
3618 
3619         /* check if the cache attributes are supported */
3620         if (i_ddi_check_cache_attr(flags) == B_FALSE)
3621                 return (DDI_FAILURE);
3622 
3623         /*
3624          * First to check if this function has been called for this dhp.
3625          */
3626         if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3627                 return (DDI_FAILURE);
3628 
3629         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3630                 return (DDI_FAILURE);
3631 
3632         if (flags & DEVMAP_MAPPING_INVALID) {
3633                 /*
3634                  * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3635                  * remap permission.
3636                  */
3637                 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3638                         return (DDI_FAILURE);
3639                 }
3640         } else {
3641                 dhp->dh_cookie = cookie;
3642                 dhp->dh_roff = ptob(btop(off));
3643                 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3644                 /* set HAT cache attributes */
3645                 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3646                 /* set HAT endianess attributes */
3647                 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3648         }
3649 
3650         /*
3651          * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3652          * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3653          * create consistent mappings but our intention was to create
3654          * non-consistent mappings.
3655          *
3656          * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3657          * mappings.
3658          *
3659          * kernel exported memory: hat figures it out it's memory and always
3660          * creates consistent mappings.
3661          *
3662          * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3663          *
3664          * /dev/kmem: consistent mappings are created unless they are
3665          * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3666          * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3667          * mappings of /dev/kmem. See common/io/mem.c
3668          */
3669 
3670         /* Only some of the flags bits are settable by the driver */
3671         dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3672 
3673         dhp->dh_len = ptob(btopr(len));
3674         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3675         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3676 
3677         if (callbackops != NULL) {
3678                 bcopy(callbackops, &dhp->dh_callbackops,
3679                     sizeof (struct devmap_callback_ctl));
3680         }
3681         /*
3682          * Initialize dh_lock if we want to do remap.
3683          */
3684         if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3685                 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3686                 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3687         }
3688 
3689         dhp->dh_flags |= DEVMAP_SETUP_DONE;
3690 
3691         return (DDI_SUCCESS);
3692 }
3693 
3694 int
3695 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3696     ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3697     uint_t flags, ddi_device_acc_attr_t *accattrp)
3698 {
3699         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3700         struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3701 
3702         TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3703             "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3704             (void *)dhp, off, cookie, len);
3705         DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3706             "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3707 
3708 #ifdef lint
3709         dip = dip;
3710         accattrp = accattrp;
3711 #endif
3712         /*
3713          * Reture failure if setup has not been done or no remap permission
3714          * has been granted during the setup.
3715          */
3716         if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3717             (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3718                 return (DDI_FAILURE);
3719 
3720         /* No flags supported for remap yet */
3721         if (flags != 0)
3722                 return (DDI_FAILURE);
3723 
3724         /* check if the cache attributes are supported */
3725         if (i_ddi_check_cache_attr(flags) == B_FALSE)
3726                 return (DDI_FAILURE);
3727 
3728         if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3729                 return (DDI_FAILURE);
3730 
3731         /* For UMEM_TRASH, this restriction is not needed */
3732         if ((off + len) > cp->size)
3733                 return (DDI_FAILURE);
3734 
3735         HOLD_DHP_LOCK(dhp);
3736         /*
3737          * Unload the old mapping, so next fault will setup the new mappings
3738          * Do this while holding the dhp lock so other faults dont reestablish
3739          * the mappings
3740          */
3741         hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3742             dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3743 
3744         dhp->dh_cookie = cookie;
3745         dhp->dh_roff = ptob(btop(off));
3746         dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3747         /* set HAT cache attributes */
3748         i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3749         /* set HAT endianess attributes */
3750         i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3751 
3752         /* clear the large page size flag */
3753         dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3754 
3755         dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3756         ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3757         RELE_DHP_LOCK(dhp);
3758         return (DDI_SUCCESS);
3759 }
3760 
3761 /*
3762  * to set timeout value for the driver's context management callback, e.g.
3763  * devmap_access().
3764  */
3765 void
3766 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3767 {
3768         devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3769 
3770         TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3771             "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3772             (void *)dhp, ticks);
3773         dhp->dh_timeout_length = ticks;
3774 }
3775 
3776 int
3777 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3778     size_t len, uint_t type, uint_t rw)
3779 {
3780 #ifdef lint
3781         pvtp = pvtp;
3782 #endif
3783 
3784         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3785             "devmap_default_access:start");
3786         return (devmap_load(dhp, off, len, type, rw));
3787 }
3788 
3789 /*
3790  * segkmem_alloc() wrapper to allocate memory which is both
3791  * non-relocatable (for DR) and sharelocked, since the rest
3792  * of this segment driver requires it.
3793  */
3794 static void *
3795 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3796 {
3797         ASSERT(vmp != NULL);
3798         ASSERT(kvseg.s_base != NULL);
3799         vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3800         return (segkmem_alloc(vmp, size, vmflag));
3801 }
3802 
3803 /*
3804  * This is where things are a bit incestuous with seg_kmem: unlike
3805  * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3806  * we need to do a bit of a dance around that to prevent duplication of
3807  * code until we decide to bite the bullet and implement a new kernel
3808  * segment for driver-allocated memory that is exported to user space.
3809  */
3810 static void
3811 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3812 {
3813         page_t *pp;
3814         caddr_t addr = inaddr;
3815         caddr_t eaddr;
3816         pgcnt_t npages = btopr(size);
3817 
3818         ASSERT(vmp != NULL);
3819         ASSERT(kvseg.s_base != NULL);
3820         ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3821 
3822         hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3823 
3824         for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3825                 /*
3826                  * Use page_find() instead of page_lookup() to find the page
3827                  * since we know that it is hashed and has a shared lock.
3828                  */
3829                 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3830 
3831                 if (pp == NULL)
3832                         panic("devmap_free_pages: page not found");
3833                 if (!page_tryupgrade(pp)) {
3834                         page_unlock(pp);
3835                         pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3836                             SE_EXCL);
3837                         if (pp == NULL)
3838                                 panic("devmap_free_pages: page already freed");
3839                 }
3840                 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3841                 pp->p_lckcnt = 0;
3842                 page_destroy(pp, 0);
3843         }
3844         page_unresv(npages);
3845 
3846         if (vmp != NULL)
3847                 vmem_free(vmp, inaddr, size);
3848 }
3849 
3850 /*
3851  * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3852  * allocating non-pageable kmem in response to a ddi_umem_alloc()
3853  * default request. For now we allocate our own pages and we keep
3854  * them long-term sharelocked, since: A) the fault routines expect the
3855  * memory to already be locked; B) pageable umem is already long-term
3856  * locked; C) it's a lot of work to make it otherwise, particularly
3857  * since the nexus layer expects the pages to never fault. An RFE is to
3858  * not keep the pages long-term locked, but instead to be able to
3859  * take faults on them and simply look them up in kvp in case we
3860  * fault on them. Even then, we must take care not to let pageout
3861  * steal them from us since the data must remain resident; if we
3862  * do this we must come up with some way to pin the pages to prevent
3863  * faults while a driver is doing DMA to/from them.
3864  */
3865 static void *
3866 devmap_umem_alloc_np(size_t size, size_t flags)
3867 {
3868         void *buf;
3869         int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3870 
3871         buf = vmem_alloc(umem_np_arena, size, vmflags);
3872         if (buf != NULL)
3873                 bzero(buf, size);
3874         return (buf);
3875 }
3876 
3877 static void
3878 devmap_umem_free_np(void *addr, size_t size)
3879 {
3880         vmem_free(umem_np_arena, addr, size);
3881 }
3882 
3883 /*
3884  * allocate page aligned kernel memory for exporting to user land.
3885  * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3886  * to find a user virtual address that is in same color as the address
3887  * allocated here.
3888  */
3889 void *
3890 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3891 {
3892         register size_t len = ptob(btopr(size));
3893         void *buf = NULL;
3894         struct ddi_umem_cookie *cp;
3895         int iflags = 0;
3896 
3897         *cookie = NULL;
3898 
3899         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3900             "devmap_umem_alloc:start");
3901         if (len == 0)
3902                 return ((void *)NULL);
3903 
3904         /*
3905          * allocate cookie
3906          */
3907         if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3908             flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3909                 ASSERT(flags & DDI_UMEM_NOSLEEP);
3910                 return ((void *)NULL);
3911         }
3912 
3913         if (flags & DDI_UMEM_PAGEABLE) {
3914                 /* Only one of the flags is allowed */
3915                 ASSERT(!(flags & DDI_UMEM_TRASH));
3916                 /* initialize resource with 0 */
3917                 iflags = KPD_ZERO;
3918 
3919                 /*
3920                  * to allocate unlocked pageable memory, use segkp_get() to
3921                  * create a segkp segment.  Since segkp can only service kas,
3922                  * other segment drivers such as segdev have to do
3923                  * as_fault(segkp, SOFTLOCK) in its fault routine,
3924                  */
3925                 if (flags & DDI_UMEM_NOSLEEP)
3926                         iflags |= KPD_NOWAIT;
3927 
3928                 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3929                         kmem_free(cp, sizeof (struct ddi_umem_cookie));
3930                         return ((void *)NULL);
3931                 }
3932                 cp->type = KMEM_PAGEABLE;
3933                 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3934                 cp->locked = 0;
3935         } else if (flags & DDI_UMEM_TRASH) {
3936                 /* Only one of the flags is allowed */
3937                 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3938                 cp->type = UMEM_TRASH;
3939                 buf = NULL;
3940         } else {
3941                 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3942                         kmem_free(cp, sizeof (struct ddi_umem_cookie));
3943                         return ((void *)NULL);
3944                 }
3945 
3946                 cp->type = KMEM_NON_PAGEABLE;
3947         }
3948 
3949         /*
3950          * need to save size here.  size will be used when
3951          * we do kmem_free.
3952          */
3953         cp->size = len;
3954         cp->cvaddr = (caddr_t)buf;
3955 
3956         *cookie =  (void *)cp;
3957         return (buf);
3958 }
3959 
3960 void
3961 ddi_umem_free(ddi_umem_cookie_t cookie)
3962 {
3963         struct ddi_umem_cookie *cp;
3964 
3965         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3966             "devmap_umem_free:start");
3967 
3968         /*
3969          * if cookie is NULL, no effects on the system
3970          */
3971         if (cookie == NULL)
3972                 return;
3973 
3974         cp = (struct ddi_umem_cookie *)cookie;
3975 
3976         switch (cp->type) {
3977         case KMEM_PAGEABLE :
3978                 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3979                 /*
3980                  * Check if there are still any pending faults on the cookie
3981                  * while the driver is deleting it,
3982                  * XXX - could change to an ASSERT but wont catch errant drivers
3983                  */
3984                 mutex_enter(&cp->lock);
3985                 if (cp->locked) {
3986                         mutex_exit(&cp->lock);
3987                         panic("ddi_umem_free for cookie with pending faults %p",
3988                             (void *)cp);
3989                         return;
3990                 }
3991 
3992                 segkp_release(segkp, cp->cvaddr);
3993 
3994                 /*
3995                  * release mutex associated with this cookie.
3996                  */
3997                 mutex_destroy(&cp->lock);
3998                 break;
3999         case KMEM_NON_PAGEABLE :
4000                 ASSERT(cp->cvaddr != NULL && cp->size != 0);
4001                 devmap_umem_free_np(cp->cvaddr, cp->size);
4002                 break;
4003         case UMEM_TRASH :
4004                 break;
4005         case UMEM_LOCKED :
4006                 /* Callers should use ddi_umem_unlock for this type */
4007                 ddi_umem_unlock(cookie);
4008                 /* Frees the cookie too */
4009                 return;
4010         default:
4011                 /* panic so we can diagnose the underlying cause */
4012                 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4013                     cp->type);
4014         }
4015 
4016         kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4017 }
4018 
4019 
4020 static int
4021 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4022 {
4023         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4024 
4025         /*
4026          * It looks as if it is always mapped shared
4027          */
4028         TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4029             "segdev_getmemid:start");
4030         memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4031         memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4032         return (0);
4033 }
4034 
4035 /*ARGSUSED*/
4036 static lgrp_mem_policy_info_t *
4037 segdev_getpolicy(struct seg *seg, caddr_t addr)
4038 {
4039         return (NULL);
4040 }
4041 
4042 /*ARGSUSED*/
4043 static int
4044 segdev_capable(struct seg *seg, segcapability_t capability)
4045 {
4046         return (0);
4047 }
4048 
4049 /*
4050  * ddi_umem_alloc() non-pageable quantum cache max size.
4051  * This is just a SWAG.
4052  */
4053 #define DEVMAP_UMEM_QUANTUM     (8*PAGESIZE)
4054 
4055 /*
4056  * Initialize seg_dev from boot. This routine sets up the trash page
4057  * and creates the umem_np_arena used to back non-pageable memory
4058  * requests.
4059  */
4060 void
4061 segdev_init(void)
4062 {
4063         struct seg kseg;
4064 
4065         umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4066             devmap_alloc_pages, devmap_free_pages, heap_arena,
4067             DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4068 
4069         kseg.s_as = &kas;
4070         trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4071             PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4072         if (trashpp == NULL)
4073                 panic("segdev_init: failed to create trash page");
4074         pagezero(trashpp, 0, PAGESIZE);
4075         page_downgrade(trashpp);
4076 }
4077 
4078 /*
4079  * Invoke platform-dependent support routines so that /proc can have
4080  * the platform code deal with curious hardware.
4081  */
4082 int
4083 segdev_copyfrom(struct seg *seg,
4084     caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4085 {
4086         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4087         struct snode *sp = VTOS(VTOCVP(sdp->vp));
4088 
4089         return (e_ddi_copyfromdev(sp->s_dip,
4090             (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4091 }
4092 
4093 int
4094 segdev_copyto(struct seg *seg,
4095     caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4096 {
4097         struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4098         struct snode *sp = VTOS(VTOCVP(sdp->vp));
4099 
4100         return (e_ddi_copytodev(sp->s_dip,
4101             (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4102 }