Print this page
6151 use NULL setpagesize segop as a shorthand for ENOTSUP
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/vm/seg_dev.c
+++ new/usr/src/uts/common/vm/seg_dev.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 28 /* All Rights Reserved */
29 29
30 30 /*
31 31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 32 * The Regents of the University of California
33 33 * All Rights Reserved
34 34 *
35 35 * University Acknowledgment- Portions of this document are derived from
36 36 * software developed by the University of California, Berkeley, and its
37 37 * contributors.
38 38 */
39 39
40 40 /*
41 41 * VM - segment of a mapped device.
42 42 *
43 43 * This segment driver is used when mapping character special devices.
44 44 */
45 45
46 46 #include <sys/types.h>
47 47 #include <sys/t_lock.h>
48 48 #include <sys/sysmacros.h>
49 49 #include <sys/vtrace.h>
50 50 #include <sys/systm.h>
51 51 #include <sys/vmsystm.h>
52 52 #include <sys/mman.h>
53 53 #include <sys/errno.h>
54 54 #include <sys/kmem.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/vnode.h>
57 57 #include <sys/proc.h>
58 58 #include <sys/conf.h>
59 59 #include <sys/debug.h>
60 60 #include <sys/ddidevmap.h>
61 61 #include <sys/ddi_implfuncs.h>
62 62 #include <sys/lgrp.h>
63 63
64 64 #include <vm/page.h>
65 65 #include <vm/hat.h>
66 66 #include <vm/as.h>
67 67 #include <vm/seg.h>
68 68 #include <vm/seg_dev.h>
69 69 #include <vm/seg_kp.h>
70 70 #include <vm/seg_kmem.h>
71 71 #include <vm/vpage.h>
72 72
73 73 #include <sys/sunddi.h>
74 74 #include <sys/esunddi.h>
75 75 #include <sys/fs/snode.h>
76 76
77 77
78 78 #if DEBUG
79 79 int segdev_debug;
80 80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 81 #else
82 82 #define DEBUGF(level, args)
83 83 #endif
84 84
85 85 /* Default timeout for devmap context management */
86 86 #define CTX_TIMEOUT_VALUE 0
87 87
88 88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 89 { mutex_enter(&dhp->dh_lock); }
90 90
91 91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 92 { mutex_exit(&dhp->dh_lock); }
93 93
94 94 #define round_down_p2(a, s) ((a) & ~((s) - 1))
95 95 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
96 96
97 97 /*
98 98 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99 99 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100 100 */
101 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
102 102 (((uvaddr | paddr) & (pgsize - 1)) == 0)
103 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
104 104 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105 105
106 106 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
107 107
108 108 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
109 109
110 110 static struct devmap_ctx *devmapctx_list = NULL;
111 111 static struct devmap_softlock *devmap_slist = NULL;
112 112
113 113 /*
114 114 * mutex, vnode and page for the page of zeros we use for the trash mappings.
115 115 * One trash page is allocated on the first ddi_umem_setup call that uses it
116 116 * XXX Eventually, we may want to combine this with what segnf does when all
117 117 * hat layers implement HAT_NOFAULT.
118 118 *
119 119 * The trash page is used when the backing store for a userland mapping is
120 120 * removed but the application semantics do not take kindly to a SIGBUS.
121 121 * In that scenario, the applications pages are mapped to some dummy page
122 122 * which returns garbage on read and writes go into a common place.
123 123 * (Perfect for NO_FAULT semantics)
124 124 * The device driver is responsible to communicating to the app with some
125 125 * other mechanism that such remapping has happened and the app should take
126 126 * corrective action.
127 127 * We can also use an anonymous memory page as there is no requirement to
128 128 * keep the page locked, however this complicates the fault code. RFE.
129 129 */
130 130 static struct vnode trashvp;
131 131 static struct page *trashpp;
132 132
133 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 134 static vmem_t *umem_np_arena;
135 135
136 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 137 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
138 138
139 139 /*
140 140 * Macros to check if type of devmap handle
141 141 */
142 142 #define cookie_is_devmem(c) \
143 143 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144 144
145 145 #define cookie_is_pmem(c) \
146 146 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147 147
148 148 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 149 ((c)->type == KMEM_PAGEABLE))
150 150
151 151 #define dhp_is_devmem(dhp) \
152 152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153 153
154 154 #define dhp_is_pmem(dhp) \
155 155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156 156
157 157 #define dhp_is_kpmem(dhp) \
158 158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159 159
160 160 /*
161 161 * Private seg op routines.
162 162 */
163 163 static int segdev_dup(struct seg *, struct seg *);
164 164 static int segdev_unmap(struct seg *, caddr_t, size_t);
165 165 static void segdev_free(struct seg *);
166 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 167 enum fault_type, enum seg_rw);
168 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 169 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 170 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 171 static void segdev_badop(void);
172 172 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 173 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
↓ open down ↓ |
173 lines elided |
↑ open up ↑ |
174 174 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 175 ulong_t *, size_t);
176 176 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 177 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
178 178 static int segdev_gettype(struct seg *, caddr_t);
179 179 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 180 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 181 static void segdev_dump(struct seg *);
182 182 static int segdev_pagelock(struct seg *, caddr_t, size_t,
183 183 struct page ***, enum lock_type, enum seg_rw);
184 -static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 184 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 185
187 186 /*
188 187 * XXX this struct is used by rootnex_map_fault to identify
189 188 * the segment it has been passed. So if you make it
190 189 * "static" you'll need to fix rootnex_map_fault.
191 190 */
192 191 struct seg_ops segdev_ops = {
193 192 .dup = segdev_dup,
194 193 .unmap = segdev_unmap,
195 194 .free = segdev_free,
196 195 .fault = segdev_fault,
197 196 .faulta = segdev_faulta,
198 197 .setprot = segdev_setprot,
199 198 .checkprot = segdev_checkprot,
200 199 .kluster = (int (*)())segdev_badop,
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
201 200 .sync = segdev_sync,
202 201 .incore = segdev_incore,
203 202 .lockop = segdev_lockop,
204 203 .getprot = segdev_getprot,
205 204 .getoffset = segdev_getoffset,
206 205 .gettype = segdev_gettype,
207 206 .getvp = segdev_getvp,
208 207 .advise = segdev_advise,
209 208 .dump = segdev_dump,
210 209 .pagelock = segdev_pagelock,
211 - .setpagesize = segdev_setpagesize,
212 210 .getmemid = segdev_getmemid,
213 211 };
214 212
215 213 /*
216 214 * Private segdev support routines
217 215 */
218 216 static struct segdev_data *sdp_alloc(void);
219 217
220 218 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
221 219 size_t, enum seg_rw);
222 220
223 221 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
224 222 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
225 223
226 224 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
227 225 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
228 226
229 227 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
230 228 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
231 229 static void devmap_softlock_rele(devmap_handle_t *);
232 230 static void devmap_ctx_rele(devmap_handle_t *);
233 231
234 232 static void devmap_ctxto(void *);
235 233
236 234 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
237 235 caddr_t addr);
238 236
239 237 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
240 238 ulong_t *opfn, ulong_t *pagesize);
241 239
242 240 static void free_devmap_handle(devmap_handle_t *dhp);
243 241
244 242 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
245 243 struct seg *newseg);
246 244
247 245 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
248 246
249 247 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
250 248
251 249 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
252 250
253 251 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
254 252 offset_t off, size_t len, uint_t flags);
255 253
256 254 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
257 255 caddr_t addr, size_t *llen, caddr_t *laddr);
258 256
259 257 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
260 258
261 259 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
262 260 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
263 261
264 262 static void *devmap_umem_alloc_np(size_t size, size_t flags);
265 263 static void devmap_umem_free_np(void *addr, size_t size);
266 264
267 265 /*
268 266 * routines to lock and unlock underlying segkp segment for
269 267 * KMEM_PAGEABLE type cookies.
270 268 */
271 269 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
272 270 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
273 271
274 272 /*
275 273 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
276 274 * drivers with devmap_access callbacks
277 275 */
278 276 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
279 277 enum fault_type);
280 278 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
281 279 enum fault_type);
282 280
283 281 static kmutex_t devmapctx_lock;
284 282
285 283 static kmutex_t devmap_slock;
286 284
287 285 /*
288 286 * Initialize the thread callbacks and thread private data.
289 287 */
290 288 static struct devmap_ctx *
291 289 devmap_ctxinit(dev_t dev, ulong_t id)
292 290 {
293 291 struct devmap_ctx *devctx;
294 292 struct devmap_ctx *tmp;
295 293 dev_info_t *dip;
296 294
297 295 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
298 296
299 297 mutex_enter(&devmapctx_lock);
300 298
301 299 dip = e_ddi_hold_devi_by_dev(dev, 0);
302 300 ASSERT(dip != NULL);
303 301 ddi_release_devi(dip);
304 302
305 303 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
306 304 if ((devctx->dip == dip) && (devctx->id == id))
307 305 break;
308 306
309 307 if (devctx == NULL) {
310 308 devctx = tmp;
311 309 devctx->dip = dip;
312 310 devctx->id = id;
313 311 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
314 312 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
315 313 devctx->next = devmapctx_list;
316 314 devmapctx_list = devctx;
317 315 } else
318 316 kmem_free(tmp, sizeof (struct devmap_ctx));
319 317
320 318 mutex_enter(&devctx->lock);
321 319 devctx->refcnt++;
322 320 mutex_exit(&devctx->lock);
323 321 mutex_exit(&devmapctx_lock);
324 322
325 323 return (devctx);
326 324 }
327 325
328 326 /*
329 327 * Timeout callback called if a CPU has not given up the device context
330 328 * within dhp->dh_timeout_length ticks
331 329 */
332 330 static void
333 331 devmap_ctxto(void *data)
334 332 {
335 333 struct devmap_ctx *devctx = data;
336 334
337 335 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
338 336 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
339 337 mutex_enter(&devctx->lock);
340 338 /*
341 339 * Set oncpu = 0 so the next mapping trying to get the device context
342 340 * can.
343 341 */
344 342 devctx->oncpu = 0;
345 343 devctx->timeout = 0;
346 344 cv_signal(&devctx->cv);
347 345 mutex_exit(&devctx->lock);
348 346 }
349 347
350 348 /*
351 349 * Create a device segment.
352 350 */
353 351 int
354 352 segdev_create(struct seg *seg, void *argsp)
355 353 {
356 354 struct segdev_data *sdp;
357 355 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
358 356 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
359 357 int error;
360 358
361 359 /*
362 360 * Since the address space is "write" locked, we
363 361 * don't need the segment lock to protect "segdev" data.
364 362 */
365 363 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
366 364
367 365 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
368 366
369 367 sdp = sdp_alloc();
370 368
371 369 sdp->mapfunc = a->mapfunc;
372 370 sdp->offset = a->offset;
373 371 sdp->prot = a->prot;
374 372 sdp->maxprot = a->maxprot;
375 373 sdp->type = a->type;
376 374 sdp->pageprot = 0;
377 375 sdp->softlockcnt = 0;
378 376 sdp->vpage = NULL;
379 377
380 378 if (sdp->mapfunc == NULL)
381 379 sdp->devmap_data = dhp;
382 380 else
383 381 sdp->devmap_data = dhp = NULL;
384 382
385 383 sdp->hat_flags = a->hat_flags;
386 384 sdp->hat_attr = a->hat_attr;
387 385
388 386 /*
389 387 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
390 388 */
391 389 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
392 390
393 391 /*
394 392 * Hold shadow vnode -- segdev only deals with
395 393 * character (VCHR) devices. We use the common
396 394 * vp to hang pages on.
397 395 */
398 396 sdp->vp = specfind(a->dev, VCHR);
399 397 ASSERT(sdp->vp != NULL);
400 398
401 399 seg->s_ops = &segdev_ops;
402 400 seg->s_data = sdp;
403 401
404 402 while (dhp != NULL) {
405 403 dhp->dh_seg = seg;
406 404 dhp = dhp->dh_next;
407 405 }
408 406
409 407 /*
410 408 * Inform the vnode of the new mapping.
411 409 */
412 410 /*
413 411 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
414 412 * dhp specific maxprot because spec_addmap does not use maxprot.
415 413 */
416 414 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
417 415 seg->s_as, seg->s_base, seg->s_size,
418 416 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
419 417
420 418 if (error != 0) {
421 419 sdp->devmap_data = NULL;
422 420 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
423 421 HAT_UNLOAD_UNMAP);
424 422 } else {
425 423 /*
426 424 * Mappings of /dev/null don't count towards the VSZ of a
427 425 * process. Mappings of /dev/null have no mapping type.
428 426 */
429 427 if ((segop_gettype(seg, seg->s_base) & (MAP_SHARED |
430 428 MAP_PRIVATE)) == 0) {
431 429 seg->s_as->a_resvsize -= seg->s_size;
432 430 }
433 431 }
434 432
435 433 return (error);
436 434 }
437 435
438 436 static struct segdev_data *
439 437 sdp_alloc(void)
440 438 {
441 439 struct segdev_data *sdp;
442 440
443 441 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
444 442 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
445 443
446 444 return (sdp);
447 445 }
448 446
449 447 /*
450 448 * Duplicate seg and return new segment in newseg.
451 449 */
452 450 static int
453 451 segdev_dup(struct seg *seg, struct seg *newseg)
454 452 {
455 453 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
456 454 struct segdev_data *newsdp;
457 455 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
458 456 size_t npages;
459 457 int ret;
460 458
461 459 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
462 460 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
463 461
464 462 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
465 463 (void *)dhp, (void *)seg));
466 464
467 465 /*
468 466 * Since the address space is "write" locked, we
469 467 * don't need the segment lock to protect "segdev" data.
470 468 */
471 469 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
472 470
473 471 newsdp = sdp_alloc();
474 472
475 473 newseg->s_ops = seg->s_ops;
476 474 newseg->s_data = (void *)newsdp;
477 475
478 476 VN_HOLD(sdp->vp);
479 477 newsdp->vp = sdp->vp;
480 478 newsdp->mapfunc = sdp->mapfunc;
481 479 newsdp->offset = sdp->offset;
482 480 newsdp->pageprot = sdp->pageprot;
483 481 newsdp->prot = sdp->prot;
484 482 newsdp->maxprot = sdp->maxprot;
485 483 newsdp->type = sdp->type;
486 484 newsdp->hat_attr = sdp->hat_attr;
487 485 newsdp->hat_flags = sdp->hat_flags;
488 486 newsdp->softlockcnt = 0;
489 487
490 488 /*
491 489 * Initialize per page data if the segment we are
492 490 * dup'ing has per page information.
493 491 */
494 492 npages = seg_pages(newseg);
495 493
496 494 if (sdp->vpage != NULL) {
497 495 size_t nbytes = vpgtob(npages);
498 496
499 497 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
500 498 bcopy(sdp->vpage, newsdp->vpage, nbytes);
501 499 } else
502 500 newsdp->vpage = NULL;
503 501
504 502 /*
505 503 * duplicate devmap handles
506 504 */
507 505 if (dhp != NULL) {
508 506 ret = devmap_handle_dup(dhp,
509 507 (devmap_handle_t **)&newsdp->devmap_data, newseg);
510 508 if (ret != 0) {
511 509 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
512 510 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
513 511 ret, (void *)dhp, (void *)seg);
514 512 DEBUGF(1, (CE_CONT,
515 513 "segdev_dup: ret %x dhp %p seg %p\n",
516 514 ret, (void *)dhp, (void *)seg));
517 515 return (ret);
518 516 }
519 517 }
520 518
521 519 /*
522 520 * Inform the common vnode of the new mapping.
523 521 */
524 522 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
525 523 newsdp->offset, newseg->s_as,
526 524 newseg->s_base, newseg->s_size, newsdp->prot,
527 525 newsdp->maxprot, sdp->type, CRED(), NULL));
528 526 }
529 527
530 528 /*
531 529 * duplicate devmap handles
532 530 */
533 531 static int
534 532 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
535 533 struct seg *newseg)
536 534 {
537 535 devmap_handle_t *newdhp_save = NULL;
538 536 devmap_handle_t *newdhp = NULL;
539 537 struct devmap_callback_ctl *callbackops;
540 538
541 539 while (dhp != NULL) {
542 540 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
543 541
544 542 /* Need to lock the original dhp while copying if REMAP */
545 543 HOLD_DHP_LOCK(dhp);
546 544 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
547 545 RELE_DHP_LOCK(dhp);
548 546 newdhp->dh_seg = newseg;
549 547 newdhp->dh_next = NULL;
550 548 if (newdhp_save != NULL)
551 549 newdhp_save->dh_next = newdhp;
552 550 else
553 551 *new_dhp = newdhp;
554 552 newdhp_save = newdhp;
555 553
556 554 callbackops = &newdhp->dh_callbackops;
557 555
558 556 if (dhp->dh_softlock != NULL)
559 557 newdhp->dh_softlock = devmap_softlock_init(
560 558 newdhp->dh_dev,
561 559 (ulong_t)callbackops->devmap_access);
562 560 if (dhp->dh_ctx != NULL)
563 561 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
564 562 (ulong_t)callbackops->devmap_access);
565 563
566 564 /*
567 565 * Initialize dh_lock if we want to do remap.
568 566 */
569 567 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
570 568 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
571 569 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
572 570 }
573 571
574 572 if (callbackops->devmap_dup != NULL) {
575 573 int ret;
576 574
577 575 /*
578 576 * Call the dup callback so that the driver can
579 577 * duplicate its private data.
580 578 */
581 579 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
582 580 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
583 581
584 582 if (ret != 0) {
585 583 /*
586 584 * We want to free up this segment as the driver
587 585 * has indicated that we can't dup it. But we
588 586 * don't want to call the drivers, devmap_unmap,
589 587 * callback function as the driver does not
590 588 * think this segment exists. The caller of
591 589 * devmap_dup will call seg_free on newseg
592 590 * as it was the caller that allocated the
593 591 * segment.
594 592 */
595 593 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
596 594 "newdhp %p dhp %p\n", (void *)newdhp,
597 595 (void *)dhp));
598 596 callbackops->devmap_unmap = NULL;
599 597 return (ret);
600 598 }
601 599 }
602 600
603 601 dhp = dhp->dh_next;
604 602 }
605 603
606 604 return (0);
607 605 }
608 606
609 607 /*
610 608 * Split a segment at addr for length len.
611 609 */
612 610 /*ARGSUSED*/
613 611 static int
614 612 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
615 613 {
616 614 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
617 615 register struct segdev_data *nsdp;
618 616 register struct seg *nseg;
619 617 register size_t opages; /* old segment size in pages */
620 618 register size_t npages; /* new segment size in pages */
621 619 register size_t dpages; /* pages being deleted (unmapped) */
622 620 register size_t nbytes;
623 621 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
624 622 devmap_handle_t *dhpp;
625 623 devmap_handle_t *newdhp;
626 624 struct devmap_callback_ctl *callbackops;
627 625 caddr_t nbase;
628 626 offset_t off;
629 627 ulong_t nsize;
630 628 size_t mlen, sz;
631 629
632 630 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
633 631 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
634 632 (void *)dhp, (void *)seg, (void *)addr, len);
635 633
636 634 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
637 635 (void *)dhp, (void *)seg, (void *)addr, len));
638 636
639 637 /*
640 638 * Since the address space is "write" locked, we
641 639 * don't need the segment lock to protect "segdev" data.
642 640 */
643 641 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
644 642
645 643 if ((sz = sdp->softlockcnt) > 0) {
646 644 /*
647 645 * Fail the unmap if pages are SOFTLOCKed through this mapping.
648 646 * softlockcnt is protected from change by the as write lock.
649 647 */
650 648 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
651 649 "segdev_unmap:error softlockcnt = %ld", sz);
652 650 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
653 651 return (EAGAIN);
654 652 }
655 653
656 654 /*
657 655 * Check for bad sizes
658 656 */
659 657 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
660 658 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
661 659 panic("segdev_unmap");
662 660
663 661 if (dhp != NULL) {
664 662 devmap_handle_t *tdhp;
665 663 /*
666 664 * If large page size was used in hat_devload(),
667 665 * the same page size must be used in hat_unload().
668 666 */
669 667 dhpp = tdhp = devmap_find_handle(dhp, addr);
670 668 while (tdhp != NULL) {
671 669 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
672 670 break;
673 671 }
674 672 tdhp = tdhp->dh_next;
675 673 }
676 674 if (tdhp != NULL) { /* found a dhp using large pages */
677 675 size_t slen = len;
678 676 size_t mlen;
679 677 size_t soff;
680 678
681 679 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
682 680 while (slen != 0) {
683 681 mlen = MIN(slen, (dhpp->dh_len - soff));
684 682 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
685 683 dhpp->dh_len, HAT_UNLOAD_UNMAP);
686 684 dhpp = dhpp->dh_next;
687 685 ASSERT(slen >= mlen);
688 686 slen -= mlen;
689 687 soff = 0;
690 688 }
691 689 } else
692 690 hat_unload(seg->s_as->a_hat, addr, len,
693 691 HAT_UNLOAD_UNMAP);
694 692 } else {
695 693 /*
696 694 * Unload any hardware translations in the range
697 695 * to be taken out.
698 696 */
699 697 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
700 698 }
701 699
702 700 /*
703 701 * get the user offset which will used in the driver callbacks
704 702 */
705 703 off = sdp->offset + (offset_t)(addr - seg->s_base);
706 704
707 705 /*
708 706 * Inform the vnode of the unmapping.
709 707 */
710 708 ASSERT(sdp->vp != NULL);
711 709 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
712 710 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
713 711
714 712 /*
715 713 * Check for entire segment
716 714 */
717 715 if (addr == seg->s_base && len == seg->s_size) {
718 716 seg_free(seg);
719 717 return (0);
720 718 }
721 719
722 720 opages = seg_pages(seg);
723 721 dpages = btop(len);
724 722 npages = opages - dpages;
725 723
726 724 /*
727 725 * Check for beginning of segment
728 726 */
729 727 if (addr == seg->s_base) {
730 728 if (sdp->vpage != NULL) {
731 729 register struct vpage *ovpage;
732 730
733 731 ovpage = sdp->vpage; /* keep pointer to vpage */
734 732
735 733 nbytes = vpgtob(npages);
736 734 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
737 735 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
738 736
739 737 /* free up old vpage */
740 738 kmem_free(ovpage, vpgtob(opages));
741 739 }
742 740
743 741 /*
744 742 * free devmap handles from the beginning of the mapping.
745 743 */
746 744 if (dhp != NULL)
747 745 devmap_handle_unmap_head(dhp, len);
748 746
749 747 sdp->offset += (offset_t)len;
750 748
751 749 seg->s_base += len;
752 750 seg->s_size -= len;
753 751
754 752 return (0);
755 753 }
756 754
757 755 /*
758 756 * Check for end of segment
759 757 */
760 758 if (addr + len == seg->s_base + seg->s_size) {
761 759 if (sdp->vpage != NULL) {
762 760 register struct vpage *ovpage;
763 761
764 762 ovpage = sdp->vpage; /* keep pointer to vpage */
765 763
766 764 nbytes = vpgtob(npages);
767 765 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
768 766 bcopy(ovpage, sdp->vpage, nbytes);
769 767
770 768 /* free up old vpage */
771 769 kmem_free(ovpage, vpgtob(opages));
772 770 }
773 771 seg->s_size -= len;
774 772
775 773 /*
776 774 * free devmap handles from addr to the end of the mapping.
777 775 */
778 776 if (dhp != NULL)
779 777 devmap_handle_unmap_tail(dhp, addr);
780 778
781 779 return (0);
782 780 }
783 781
784 782 /*
785 783 * The section to go is in the middle of the segment,
786 784 * have to make it into two segments. nseg is made for
787 785 * the high end while seg is cut down at the low end.
788 786 */
789 787 nbase = addr + len; /* new seg base */
790 788 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
791 789 seg->s_size = addr - seg->s_base; /* shrink old seg */
792 790 nseg = seg_alloc(seg->s_as, nbase, nsize);
793 791 if (nseg == NULL)
794 792 panic("segdev_unmap seg_alloc");
795 793
796 794 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
797 795 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
798 796 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
799 797 (void *)seg, (void *)nseg));
800 798 nsdp = sdp_alloc();
801 799
802 800 nseg->s_ops = seg->s_ops;
803 801 nseg->s_data = (void *)nsdp;
804 802
805 803 VN_HOLD(sdp->vp);
806 804 nsdp->mapfunc = sdp->mapfunc;
807 805 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
808 806 nsdp->vp = sdp->vp;
809 807 nsdp->pageprot = sdp->pageprot;
810 808 nsdp->prot = sdp->prot;
811 809 nsdp->maxprot = sdp->maxprot;
812 810 nsdp->type = sdp->type;
813 811 nsdp->hat_attr = sdp->hat_attr;
814 812 nsdp->hat_flags = sdp->hat_flags;
815 813 nsdp->softlockcnt = 0;
816 814
817 815 /*
818 816 * Initialize per page data if the segment we are
819 817 * dup'ing has per page information.
820 818 */
821 819 if (sdp->vpage != NULL) {
822 820 /* need to split vpage into two arrays */
823 821 register size_t nnbytes;
824 822 register size_t nnpages;
825 823 register struct vpage *ovpage;
826 824
827 825 ovpage = sdp->vpage; /* keep pointer to vpage */
828 826
829 827 npages = seg_pages(seg); /* seg has shrunk */
830 828 nbytes = vpgtob(npages);
831 829 nnpages = seg_pages(nseg);
832 830 nnbytes = vpgtob(nnpages);
833 831
834 832 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
835 833 bcopy(ovpage, sdp->vpage, nbytes);
836 834
837 835 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
838 836 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
839 837
840 838 /* free up old vpage */
841 839 kmem_free(ovpage, vpgtob(opages));
842 840 } else
843 841 nsdp->vpage = NULL;
844 842
845 843 /*
846 844 * unmap dhps.
847 845 */
848 846 if (dhp == NULL) {
849 847 nsdp->devmap_data = NULL;
850 848 return (0);
851 849 }
852 850 while (dhp != NULL) {
853 851 callbackops = &dhp->dh_callbackops;
854 852 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
855 853 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
856 854 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
857 855 (void *)dhp, (void *)addr,
858 856 (void *)dhp->dh_uvaddr, dhp->dh_len));
859 857
860 858 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
861 859 dhpp = dhp->dh_next;
862 860 dhp->dh_next = NULL;
863 861 dhp = dhpp;
864 862 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
865 863 dhp = dhp->dh_next;
866 864 } else if (addr > dhp->dh_uvaddr &&
867 865 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
868 866 /*
869 867 * <addr, addr+len> is enclosed by dhp.
870 868 * create a newdhp that begins at addr+len and
871 869 * ends at dhp->dh_uvaddr+dhp->dh_len.
872 870 */
873 871 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
874 872 HOLD_DHP_LOCK(dhp);
875 873 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
876 874 RELE_DHP_LOCK(dhp);
877 875 newdhp->dh_seg = nseg;
878 876 newdhp->dh_next = dhp->dh_next;
879 877 if (dhp->dh_softlock != NULL)
880 878 newdhp->dh_softlock = devmap_softlock_init(
881 879 newdhp->dh_dev,
882 880 (ulong_t)callbackops->devmap_access);
883 881 if (dhp->dh_ctx != NULL)
884 882 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
885 883 (ulong_t)callbackops->devmap_access);
886 884 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
887 885 mutex_init(&newdhp->dh_lock,
888 886 NULL, MUTEX_DEFAULT, NULL);
889 887 }
890 888 if (callbackops->devmap_unmap != NULL)
891 889 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
892 890 off, len, dhp, &dhp->dh_pvtp,
893 891 newdhp, &newdhp->dh_pvtp);
894 892 mlen = len + (addr - dhp->dh_uvaddr);
895 893 devmap_handle_reduce_len(newdhp, mlen);
896 894 nsdp->devmap_data = newdhp;
897 895 /* XX Changing len should recalculate LARGE flag */
898 896 dhp->dh_len = addr - dhp->dh_uvaddr;
899 897 dhpp = dhp->dh_next;
900 898 dhp->dh_next = NULL;
901 899 dhp = dhpp;
902 900 } else if ((addr > dhp->dh_uvaddr) &&
903 901 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
904 902 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
905 903 /*
906 904 * <addr, addr+len> spans over dhps.
907 905 */
908 906 if (callbackops->devmap_unmap != NULL)
909 907 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
910 908 off, mlen, (devmap_cookie_t *)dhp,
911 909 &dhp->dh_pvtp, NULL, NULL);
912 910 /* XX Changing len should recalculate LARGE flag */
913 911 dhp->dh_len = addr - dhp->dh_uvaddr;
914 912 dhpp = dhp->dh_next;
915 913 dhp->dh_next = NULL;
916 914 dhp = dhpp;
917 915 nsdp->devmap_data = dhp;
918 916 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
919 917 /*
920 918 * dhp is enclosed by <addr, addr+len>.
921 919 */
922 920 dhp->dh_seg = nseg;
923 921 nsdp->devmap_data = dhp;
924 922 dhp = devmap_handle_unmap(dhp);
925 923 nsdp->devmap_data = dhp; /* XX redundant? */
926 924 } else if (((addr + len) > dhp->dh_uvaddr) &&
927 925 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
928 926 mlen = addr + len - dhp->dh_uvaddr;
929 927 if (callbackops->devmap_unmap != NULL)
930 928 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
931 929 dhp->dh_uoff, mlen, NULL,
932 930 NULL, dhp, &dhp->dh_pvtp);
933 931 devmap_handle_reduce_len(dhp, mlen);
934 932 nsdp->devmap_data = dhp;
935 933 dhp->dh_seg = nseg;
936 934 dhp = dhp->dh_next;
937 935 } else {
938 936 dhp->dh_seg = nseg;
939 937 dhp = dhp->dh_next;
940 938 }
941 939 }
942 940 return (0);
943 941 }
944 942
945 943 /*
946 944 * Utility function handles reducing the length of a devmap handle during unmap
947 945 * Note that is only used for unmapping the front portion of the handler,
948 946 * i.e., we are bumping up the offset/pfn etc up by len
949 947 * Do not use if reducing length at the tail.
950 948 */
951 949 static void
952 950 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
953 951 {
954 952 struct ddi_umem_cookie *cp;
955 953 struct devmap_pmem_cookie *pcp;
956 954 /*
957 955 * adjust devmap handle fields
958 956 */
959 957 ASSERT(len < dhp->dh_len);
960 958
961 959 /* Make sure only page-aligned changes are done */
962 960 ASSERT((len & PAGEOFFSET) == 0);
963 961
964 962 dhp->dh_len -= len;
965 963 dhp->dh_uoff += (offset_t)len;
966 964 dhp->dh_roff += (offset_t)len;
967 965 dhp->dh_uvaddr += len;
968 966 /* Need to grab dhp lock if REMAP */
969 967 HOLD_DHP_LOCK(dhp);
970 968 cp = dhp->dh_cookie;
971 969 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
972 970 if (cookie_is_devmem(cp)) {
973 971 dhp->dh_pfn += btop(len);
974 972 } else if (cookie_is_pmem(cp)) {
975 973 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
976 974 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
977 975 dhp->dh_roff < ptob(pcp->dp_npages));
978 976 } else {
979 977 ASSERT(dhp->dh_roff < cp->size);
980 978 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
981 979 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
982 980 ASSERT((dhp->dh_cvaddr + len) <=
983 981 (cp->cvaddr + cp->size));
984 982
985 983 dhp->dh_cvaddr += len;
986 984 }
987 985 }
988 986 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
989 987 RELE_DHP_LOCK(dhp);
990 988 }
991 989
992 990 /*
993 991 * Free devmap handle, dhp.
994 992 * Return the next devmap handle on the linked list.
995 993 */
996 994 static devmap_handle_t *
997 995 devmap_handle_unmap(devmap_handle_t *dhp)
998 996 {
999 997 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1000 998 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1001 999 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1002 1000
1003 1001 ASSERT(dhp != NULL);
1004 1002
1005 1003 /*
1006 1004 * before we free up dhp, call the driver's devmap_unmap entry point
1007 1005 * to free resources allocated for this dhp.
1008 1006 */
1009 1007 if (callbackops->devmap_unmap != NULL) {
1010 1008 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1011 1009 dhp->dh_len, NULL, NULL, NULL, NULL);
1012 1010 }
1013 1011
1014 1012 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1015 1013 sdp->devmap_data = dhp->dh_next;
1016 1014 } else {
1017 1015 while (dhpp->dh_next != dhp) {
1018 1016 dhpp = dhpp->dh_next;
1019 1017 }
1020 1018 dhpp->dh_next = dhp->dh_next;
1021 1019 }
1022 1020 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1023 1021
1024 1022 if (dhp->dh_softlock != NULL)
1025 1023 devmap_softlock_rele(dhp);
1026 1024
1027 1025 if (dhp->dh_ctx != NULL)
1028 1026 devmap_ctx_rele(dhp);
1029 1027
1030 1028 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1031 1029 mutex_destroy(&dhp->dh_lock);
1032 1030 }
1033 1031 kmem_free(dhp, sizeof (devmap_handle_t));
1034 1032
1035 1033 return (dhpp);
1036 1034 }
1037 1035
1038 1036 /*
1039 1037 * Free complete devmap handles from dhp for len bytes
1040 1038 * dhp can be either the first handle or a subsequent handle
1041 1039 */
1042 1040 static void
1043 1041 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1044 1042 {
1045 1043 struct devmap_callback_ctl *callbackops;
1046 1044
1047 1045 /*
1048 1046 * free the devmap handles covered by len.
1049 1047 */
1050 1048 while (len >= dhp->dh_len) {
1051 1049 len -= dhp->dh_len;
1052 1050 dhp = devmap_handle_unmap(dhp);
1053 1051 }
1054 1052 if (len != 0) { /* partial unmap at head of first remaining dhp */
1055 1053 callbackops = &dhp->dh_callbackops;
1056 1054
1057 1055 /*
1058 1056 * Call the unmap callback so the drivers can make
1059 1057 * adjustment on its private data.
1060 1058 */
1061 1059 if (callbackops->devmap_unmap != NULL)
1062 1060 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1063 1061 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1064 1062 devmap_handle_reduce_len(dhp, len);
1065 1063 }
1066 1064 }
1067 1065
1068 1066 /*
1069 1067 * Free devmap handles to truncate the mapping after addr
1070 1068 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1071 1069 * Also could then use the routine in middle unmap case too
1072 1070 */
1073 1071 static void
1074 1072 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1075 1073 {
1076 1074 register struct seg *seg = dhp->dh_seg;
1077 1075 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1078 1076 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1079 1077 struct devmap_callback_ctl *callbackops;
1080 1078 register devmap_handle_t *dhpp;
1081 1079 size_t maplen;
1082 1080 ulong_t off;
1083 1081 size_t len;
1084 1082
1085 1083 maplen = (size_t)(addr - dhp->dh_uvaddr);
1086 1084 dhph = devmap_find_handle(dhph, addr);
1087 1085
1088 1086 while (dhph != NULL) {
1089 1087 if (maplen == 0) {
1090 1088 dhph = devmap_handle_unmap(dhph);
1091 1089 } else {
1092 1090 callbackops = &dhph->dh_callbackops;
1093 1091 len = dhph->dh_len - maplen;
1094 1092 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1095 1093 /*
1096 1094 * Call the unmap callback so the driver
1097 1095 * can make adjustments on its private data.
1098 1096 */
1099 1097 if (callbackops->devmap_unmap != NULL)
1100 1098 (*callbackops->devmap_unmap)(dhph,
1101 1099 dhph->dh_pvtp, off, len,
1102 1100 (devmap_cookie_t *)dhph,
1103 1101 &dhph->dh_pvtp, NULL, NULL);
1104 1102 /* XXX Reducing len needs to recalculate LARGE flag */
1105 1103 dhph->dh_len = maplen;
1106 1104 maplen = 0;
1107 1105 dhpp = dhph->dh_next;
1108 1106 dhph->dh_next = NULL;
1109 1107 dhph = dhpp;
1110 1108 }
1111 1109 } /* end while */
1112 1110 }
1113 1111
1114 1112 /*
1115 1113 * Free a segment.
1116 1114 */
1117 1115 static void
1118 1116 segdev_free(struct seg *seg)
1119 1117 {
1120 1118 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1121 1119 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1122 1120
1123 1121 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1124 1122 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1125 1123 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1126 1124 (void *)dhp, (void *)seg));
1127 1125
1128 1126 /*
1129 1127 * Since the address space is "write" locked, we
1130 1128 * don't need the segment lock to protect "segdev" data.
1131 1129 */
1132 1130 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1133 1131
1134 1132 while (dhp != NULL)
1135 1133 dhp = devmap_handle_unmap(dhp);
1136 1134
1137 1135 VN_RELE(sdp->vp);
1138 1136 if (sdp->vpage != NULL)
1139 1137 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1140 1138
1141 1139 rw_destroy(&sdp->lock);
1142 1140 kmem_free(sdp, sizeof (*sdp));
1143 1141 }
1144 1142
1145 1143 static void
1146 1144 free_devmap_handle(devmap_handle_t *dhp)
1147 1145 {
1148 1146 register devmap_handle_t *dhpp;
1149 1147
1150 1148 /*
1151 1149 * free up devmap handle
1152 1150 */
1153 1151 while (dhp != NULL) {
1154 1152 dhpp = dhp->dh_next;
1155 1153 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1156 1154 mutex_destroy(&dhp->dh_lock);
1157 1155 }
1158 1156
1159 1157 if (dhp->dh_softlock != NULL)
1160 1158 devmap_softlock_rele(dhp);
1161 1159
1162 1160 if (dhp->dh_ctx != NULL)
1163 1161 devmap_ctx_rele(dhp);
1164 1162
1165 1163 kmem_free(dhp, sizeof (devmap_handle_t));
1166 1164 dhp = dhpp;
1167 1165 }
1168 1166 }
1169 1167
1170 1168 /*
1171 1169 * routines to lock and unlock underlying segkp segment for
1172 1170 * KMEM_PAGEABLE type cookies.
1173 1171 * segkp only allows a single pending F_SOFTLOCK
1174 1172 * we keep track of number of locks in the cookie so we can
1175 1173 * have multiple pending faults and manage the calls to segkp.
1176 1174 * RFE: if segkp supports either pagelock or can support multiple
1177 1175 * calls to F_SOFTLOCK, then these routines can go away.
1178 1176 * If pagelock, segdev_faultpage can fault on a page by page basis
1179 1177 * and simplifies the code quite a bit.
1180 1178 * if multiple calls allowed but not partial ranges, then need for
1181 1179 * cookie->lock and locked count goes away, code can call as_fault directly
1182 1180 */
1183 1181 static faultcode_t
1184 1182 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1185 1183 {
1186 1184 int err = 0;
1187 1185 ASSERT(cookie_is_kpmem(cookie));
1188 1186 /*
1189 1187 * Fault in pages in segkp with F_SOFTLOCK.
1190 1188 * We want to hold the lock until all pages have been loaded.
1191 1189 * segkp only allows single caller to hold SOFTLOCK, so cookie
1192 1190 * holds a count so we dont call into segkp multiple times
1193 1191 */
1194 1192 mutex_enter(&cookie->lock);
1195 1193
1196 1194 /*
1197 1195 * Check for overflow in locked field
1198 1196 */
1199 1197 if ((UINT32_MAX - cookie->locked) < npages) {
1200 1198 err = FC_MAKE_ERR(ENOMEM);
1201 1199 } else if (cookie->locked == 0) {
1202 1200 /* First time locking */
1203 1201 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1204 1202 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1205 1203 }
1206 1204 if (!err) {
1207 1205 cookie->locked += npages;
1208 1206 }
1209 1207 mutex_exit(&cookie->lock);
1210 1208 return (err);
1211 1209 }
1212 1210
1213 1211 static void
1214 1212 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1215 1213 {
1216 1214 mutex_enter(&cookie->lock);
1217 1215 ASSERT(cookie_is_kpmem(cookie));
1218 1216 ASSERT(cookie->locked >= npages);
1219 1217 cookie->locked -= (uint_t)npages;
1220 1218 if (cookie->locked == 0) {
1221 1219 /* Last unlock */
1222 1220 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1223 1221 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1224 1222 panic("segdev releasing kpmem lock %p", (void *)cookie);
1225 1223 }
1226 1224 mutex_exit(&cookie->lock);
1227 1225 }
1228 1226
1229 1227 /*
1230 1228 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1231 1229 * drivers with devmap_access callbacks
1232 1230 * slock->softlocked basically works like a rw lock
1233 1231 * -ve counts => F_SOFTLOCK in progress
1234 1232 * +ve counts => F_INVAL/F_PROT in progress
1235 1233 * We allow only one F_SOFTLOCK at a time
1236 1234 * but can have multiple pending F_INVAL/F_PROT calls
1237 1235 *
1238 1236 * This routine waits using cv_wait_sig so killing processes is more graceful
1239 1237 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1240 1238 */
1241 1239 static int devmap_softlock_enter(
1242 1240 struct devmap_softlock *slock,
1243 1241 size_t npages,
1244 1242 enum fault_type type)
1245 1243 {
1246 1244 if (npages == 0)
1247 1245 return (0);
1248 1246 mutex_enter(&(slock->lock));
1249 1247 switch (type) {
1250 1248 case F_SOFTLOCK :
1251 1249 while (slock->softlocked) {
1252 1250 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1253 1251 /* signalled */
1254 1252 mutex_exit(&(slock->lock));
1255 1253 return (EINTR);
1256 1254 }
1257 1255 }
1258 1256 slock->softlocked -= npages; /* -ve count => locked */
1259 1257 break;
1260 1258 case F_INVAL :
1261 1259 case F_PROT :
1262 1260 while (slock->softlocked < 0)
1263 1261 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1264 1262 /* signalled */
1265 1263 mutex_exit(&(slock->lock));
1266 1264 return (EINTR);
1267 1265 }
1268 1266 slock->softlocked += npages; /* +ve count => f_invals */
1269 1267 break;
1270 1268 default:
1271 1269 ASSERT(0);
1272 1270 }
1273 1271 mutex_exit(&(slock->lock));
1274 1272 return (0);
1275 1273 }
1276 1274
1277 1275 static void devmap_softlock_exit(
1278 1276 struct devmap_softlock *slock,
1279 1277 size_t npages,
1280 1278 enum fault_type type)
1281 1279 {
1282 1280 if (slock == NULL)
1283 1281 return;
1284 1282 mutex_enter(&(slock->lock));
1285 1283 switch (type) {
1286 1284 case F_SOFTLOCK :
1287 1285 ASSERT(-slock->softlocked >= npages);
1288 1286 slock->softlocked += npages; /* -ve count is softlocked */
1289 1287 if (slock->softlocked == 0)
1290 1288 cv_signal(&slock->cv);
1291 1289 break;
1292 1290 case F_INVAL :
1293 1291 case F_PROT:
1294 1292 ASSERT(slock->softlocked >= npages);
1295 1293 slock->softlocked -= npages;
1296 1294 if (slock->softlocked == 0)
1297 1295 cv_signal(&slock->cv);
1298 1296 break;
1299 1297 default:
1300 1298 ASSERT(0);
1301 1299 }
1302 1300 mutex_exit(&(slock->lock));
1303 1301 }
1304 1302
1305 1303 /*
1306 1304 * Do a F_SOFTUNLOCK call over the range requested.
1307 1305 * The range must have already been F_SOFTLOCK'ed.
1308 1306 * The segment lock should be held, (but not the segment private lock?)
1309 1307 * The softunlock code below does not adjust for large page sizes
1310 1308 * assumes the caller already did any addr/len adjustments for
1311 1309 * pagesize mappings before calling.
1312 1310 */
1313 1311 /*ARGSUSED*/
1314 1312 static void
1315 1313 segdev_softunlock(
1316 1314 struct hat *hat, /* the hat */
1317 1315 struct seg *seg, /* seg_dev of interest */
1318 1316 caddr_t addr, /* base address of range */
1319 1317 size_t len, /* number of bytes */
1320 1318 enum seg_rw rw) /* type of access at fault */
1321 1319 {
1322 1320 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1323 1321 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1324 1322
1325 1323 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1326 1324 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1327 1325 dhp_head, sdp, addr, len);
1328 1326 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1329 1327 "addr %p len %lx\n",
1330 1328 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1331 1329
1332 1330 hat_unlock(hat, addr, len);
1333 1331
1334 1332 if (dhp_head != NULL) {
1335 1333 devmap_handle_t *dhp;
1336 1334 size_t mlen;
1337 1335 size_t tlen = len;
1338 1336 ulong_t off;
1339 1337
1340 1338 dhp = devmap_find_handle(dhp_head, addr);
1341 1339 ASSERT(dhp != NULL);
1342 1340
1343 1341 off = (ulong_t)(addr - dhp->dh_uvaddr);
1344 1342 while (tlen != 0) {
1345 1343 mlen = MIN(tlen, (dhp->dh_len - off));
1346 1344
1347 1345 /*
1348 1346 * unlock segkp memory, locked during F_SOFTLOCK
1349 1347 */
1350 1348 if (dhp_is_kpmem(dhp)) {
1351 1349 release_kpmem_lock(
1352 1350 (struct ddi_umem_cookie *)dhp->dh_cookie,
1353 1351 btopr(mlen));
1354 1352 }
1355 1353
1356 1354 /*
1357 1355 * Do the softlock accounting for devmap_access
1358 1356 */
1359 1357 if (dhp->dh_callbackops.devmap_access != NULL) {
1360 1358 devmap_softlock_exit(dhp->dh_softlock,
1361 1359 btopr(mlen), F_SOFTLOCK);
1362 1360 }
1363 1361
1364 1362 tlen -= mlen;
1365 1363 dhp = dhp->dh_next;
1366 1364 off = 0;
1367 1365 }
1368 1366 }
1369 1367
1370 1368 mutex_enter(&freemem_lock);
1371 1369 ASSERT(sdp->softlockcnt >= btopr(len));
1372 1370 sdp->softlockcnt -= btopr(len);
1373 1371 mutex_exit(&freemem_lock);
1374 1372 if (sdp->softlockcnt == 0) {
1375 1373 /*
1376 1374 * All SOFTLOCKS are gone. Wakeup any waiting
1377 1375 * unmappers so they can try again to unmap.
1378 1376 * Check for waiters first without the mutex
1379 1377 * held so we don't always grab the mutex on
1380 1378 * softunlocks.
1381 1379 */
1382 1380 if (AS_ISUNMAPWAIT(seg->s_as)) {
1383 1381 mutex_enter(&seg->s_as->a_contents);
1384 1382 if (AS_ISUNMAPWAIT(seg->s_as)) {
1385 1383 AS_CLRUNMAPWAIT(seg->s_as);
1386 1384 cv_broadcast(&seg->s_as->a_cv);
1387 1385 }
1388 1386 mutex_exit(&seg->s_as->a_contents);
1389 1387 }
1390 1388 }
1391 1389
1392 1390 }
1393 1391
1394 1392 /*
1395 1393 * Handle fault for a single page.
1396 1394 * Done in a separate routine so we can handle errors more easily.
1397 1395 * This routine is called only from segdev_faultpages()
1398 1396 * when looping over the range of addresses requested. The segment lock is held.
1399 1397 */
1400 1398 static faultcode_t
1401 1399 segdev_faultpage(
1402 1400 struct hat *hat, /* the hat */
1403 1401 struct seg *seg, /* seg_dev of interest */
1404 1402 caddr_t addr, /* address in as */
1405 1403 struct vpage *vpage, /* pointer to vpage for seg, addr */
1406 1404 enum fault_type type, /* type of fault */
1407 1405 enum seg_rw rw, /* type of access at fault */
1408 1406 devmap_handle_t *dhp) /* devmap handle if any for this page */
1409 1407 {
1410 1408 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1411 1409 uint_t prot;
1412 1410 pfn_t pfnum = PFN_INVALID;
1413 1411 u_offset_t offset;
1414 1412 uint_t hat_flags;
1415 1413 dev_info_t *dip;
1416 1414
1417 1415 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1418 1416 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1419 1417 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1420 1418 (void *)dhp, (void *)seg, (void *)addr));
1421 1419
1422 1420 /*
1423 1421 * Initialize protection value for this page.
1424 1422 * If we have per page protection values check it now.
1425 1423 */
1426 1424 if (sdp->pageprot) {
1427 1425 uint_t protchk;
1428 1426
1429 1427 switch (rw) {
1430 1428 case S_READ:
1431 1429 protchk = PROT_READ;
1432 1430 break;
1433 1431 case S_WRITE:
1434 1432 protchk = PROT_WRITE;
1435 1433 break;
1436 1434 case S_EXEC:
1437 1435 protchk = PROT_EXEC;
1438 1436 break;
1439 1437 case S_OTHER:
1440 1438 default:
1441 1439 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1442 1440 break;
1443 1441 }
1444 1442
1445 1443 prot = VPP_PROT(vpage);
1446 1444 if ((prot & protchk) == 0)
1447 1445 return (FC_PROT); /* illegal access type */
1448 1446 } else {
1449 1447 prot = sdp->prot;
1450 1448 /* caller has already done segment level protection check */
1451 1449 }
1452 1450
1453 1451 if (type == F_SOFTLOCK) {
1454 1452 mutex_enter(&freemem_lock);
1455 1453 sdp->softlockcnt++;
1456 1454 mutex_exit(&freemem_lock);
1457 1455 }
1458 1456
1459 1457 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1460 1458 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1461 1459 /*
1462 1460 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1463 1461 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1464 1462 * seg->s_base.
1465 1463 */
1466 1464 if (dhp == NULL) {
1467 1465 /* If segment has devmap_data, then dhp should be non-NULL */
1468 1466 ASSERT(sdp->devmap_data == NULL);
1469 1467 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1470 1468 (off_t)offset, prot);
1471 1469 prot |= sdp->hat_attr;
1472 1470 } else {
1473 1471 ulong_t off;
1474 1472 struct ddi_umem_cookie *cp;
1475 1473 struct devmap_pmem_cookie *pcp;
1476 1474
1477 1475 /* ensure the dhp passed in contains addr. */
1478 1476 ASSERT(dhp == devmap_find_handle(
1479 1477 (devmap_handle_t *)sdp->devmap_data, addr));
1480 1478
1481 1479 off = addr - dhp->dh_uvaddr;
1482 1480
1483 1481 /*
1484 1482 * This routine assumes that the caller makes sure that the
1485 1483 * fields in dhp used below are unchanged due to remap during
1486 1484 * this call. Caller does HOLD_DHP_LOCK if neeed
1487 1485 */
1488 1486 cp = dhp->dh_cookie;
1489 1487 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1490 1488 pfnum = PFN_INVALID;
1491 1489 } else if (cookie_is_devmem(cp)) {
1492 1490 pfnum = dhp->dh_pfn + btop(off);
1493 1491 } else if (cookie_is_pmem(cp)) {
1494 1492 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1495 1493 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1496 1494 dhp->dh_roff < ptob(pcp->dp_npages));
1497 1495 pfnum = page_pptonum(
1498 1496 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1499 1497 } else {
1500 1498 ASSERT(dhp->dh_roff < cp->size);
1501 1499 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1502 1500 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1503 1501 ASSERT((dhp->dh_cvaddr + off) <=
1504 1502 (cp->cvaddr + cp->size));
1505 1503 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1506 1504 (cp->cvaddr + cp->size));
1507 1505
1508 1506 switch (cp->type) {
1509 1507 case UMEM_LOCKED :
1510 1508 if (cp->pparray != NULL) {
1511 1509 ASSERT((dhp->dh_roff &
1512 1510 PAGEOFFSET) == 0);
1513 1511 pfnum = page_pptonum(
1514 1512 cp->pparray[btop(off +
1515 1513 dhp->dh_roff)]);
1516 1514 } else {
1517 1515 pfnum = hat_getpfnum(
1518 1516 ((proc_t *)cp->procp)->p_as->a_hat,
1519 1517 cp->cvaddr + off);
1520 1518 }
1521 1519 break;
1522 1520 case UMEM_TRASH :
1523 1521 pfnum = page_pptonum(trashpp);
1524 1522 /*
1525 1523 * We should set hat_flags to HAT_NOFAULT also
1526 1524 * However, not all hat layers implement this
1527 1525 */
1528 1526 break;
1529 1527 case KMEM_PAGEABLE:
1530 1528 case KMEM_NON_PAGEABLE:
1531 1529 pfnum = hat_getpfnum(kas.a_hat,
1532 1530 dhp->dh_cvaddr + off);
1533 1531 break;
1534 1532 default :
1535 1533 pfnum = PFN_INVALID;
1536 1534 break;
1537 1535 }
1538 1536 }
1539 1537 prot |= dhp->dh_hat_attr;
1540 1538 }
1541 1539 if (pfnum == PFN_INVALID) {
1542 1540 return (FC_MAKE_ERR(EFAULT));
1543 1541 }
1544 1542 /* prot should already be OR'ed in with hat_attributes if needed */
1545 1543
1546 1544 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1547 1545 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1548 1546 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1549 1547 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1550 1548 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1551 1549
1552 1550 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1553 1551 /*
1554 1552 * It's not _really_ required here to pass sdp->hat_flags
1555 1553 * to hat_devload even though we do it.
1556 1554 * This is because hat figures it out DEVMEM mappings
1557 1555 * are non-consistent, anyway.
1558 1556 */
1559 1557 hat_devload(hat, addr, PAGESIZE, pfnum,
1560 1558 prot, hat_flags | sdp->hat_flags);
1561 1559 return (0);
1562 1560 }
1563 1561
1564 1562 /*
1565 1563 * Fall through to the case where devmap is not used and need to call
1566 1564 * up the device tree to set up the mapping
1567 1565 */
1568 1566
1569 1567 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1570 1568 ASSERT(dip);
1571 1569
1572 1570 /*
1573 1571 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1574 1572 * This is because this calls drivers which may not expect
1575 1573 * prot to have any other values than PROT_ALL
1576 1574 * The root nexus driver has a hack to peek into the segment
1577 1575 * structure and then OR in sdp->hat_attr.
1578 1576 * XX In case the bus_ops interfaces are ever revisited
1579 1577 * we need to fix this. prot should include other hat attributes
1580 1578 */
1581 1579 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1582 1580 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1583 1581 return (FC_MAKE_ERR(EFAULT));
1584 1582 }
1585 1583 return (0);
1586 1584 }
1587 1585
1588 1586 static faultcode_t
1589 1587 segdev_fault(
1590 1588 struct hat *hat, /* the hat */
1591 1589 struct seg *seg, /* the seg_dev of interest */
1592 1590 caddr_t addr, /* the address of the fault */
1593 1591 size_t len, /* the length of the range */
1594 1592 enum fault_type type, /* type of fault */
1595 1593 enum seg_rw rw) /* type of access at fault */
1596 1594 {
1597 1595 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1598 1596 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1599 1597 devmap_handle_t *dhp;
1600 1598 struct devmap_softlock *slock = NULL;
1601 1599 ulong_t slpage = 0;
1602 1600 ulong_t off;
1603 1601 caddr_t maddr = addr;
1604 1602 int err;
1605 1603 int err_is_faultcode = 0;
1606 1604
1607 1605 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1608 1606 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1609 1607 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1610 1608 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1611 1609 "addr %p len %lx type %x\n",
1612 1610 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1613 1611
1614 1612 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1615 1613
1616 1614 /* Handle non-devmap case */
1617 1615 if (dhp_head == NULL)
1618 1616 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1619 1617
1620 1618 /* Find devmap handle */
1621 1619 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1622 1620 return (FC_NOMAP);
1623 1621
1624 1622 /*
1625 1623 * The seg_dev driver does not implement copy-on-write,
1626 1624 * and always loads translations with maximal allowed permissions
1627 1625 * but we got an fault trying to access the device.
1628 1626 * Servicing the fault is not going to result in any better result
1629 1627 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1630 1628 * faults, then the code below is written for that
1631 1629 * Pending resolution of the following:
1632 1630 * - determine if the F_INVAL/F_SOFTLOCK syncing
1633 1631 * is needed for F_PROT also or not. The code below assumes it does
1634 1632 * - If driver sees F_PROT and calls devmap_load with same type,
1635 1633 * then segdev_faultpages will fail with FC_PROT anyway, need to
1636 1634 * change that so calls from devmap_load to segdev_faultpages for
1637 1635 * F_PROT type are retagged to F_INVAL.
1638 1636 * RFE: Today we dont have drivers that use devmap and want to handle
1639 1637 * F_PROT calls. The code in segdev_fault* is written to allow
1640 1638 * this case but is not tested. A driver that needs this capability
1641 1639 * should be able to remove the short-circuit case; resolve the
1642 1640 * above issues and "should" work.
1643 1641 */
1644 1642 if (type == F_PROT) {
1645 1643 return (FC_PROT);
1646 1644 }
1647 1645
1648 1646 /*
1649 1647 * Loop through dhp list calling devmap_access or segdev_faultpages for
1650 1648 * each devmap handle.
1651 1649 * drivers which implement devmap_access can interpose on faults and do
1652 1650 * device-appropriate special actions before calling devmap_load.
1653 1651 */
1654 1652
1655 1653 /*
1656 1654 * Unfortunately, this simple loop has turned out to expose a variety
1657 1655 * of complex problems which results in the following convoluted code.
1658 1656 *
1659 1657 * First, a desire to handle a serialization of F_SOFTLOCK calls
1660 1658 * to the driver within the framework.
1661 1659 * This results in a dh_softlock structure that is on a per device
1662 1660 * (or device instance) basis and serializes devmap_access calls.
1663 1661 * Ideally we would need to do this for underlying
1664 1662 * memory/device regions that are being faulted on
1665 1663 * but that is hard to identify and with REMAP, harder
1666 1664 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1667 1665 * to F_SOFTLOCK calls to the driver.
1668 1666 * These serializations are to simplify the driver programmer model.
1669 1667 * To support these two features, the code first goes through the
1670 1668 * devmap handles and counts the pages (slpage) that are covered
1671 1669 * by devmap_access callbacks.
1672 1670 * This part ends with a devmap_softlock_enter call
1673 1671 * which allows only one F_SOFTLOCK active on a device instance,
1674 1672 * but multiple F_INVAL/F_PROTs can be active except when a
1675 1673 * F_SOFTLOCK is active
1676 1674 *
1677 1675 * Next, we dont short-circuit the fault code upfront to call
1678 1676 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1679 1677 * the same length when we softlock and softunlock.
1680 1678 *
1681 1679 * -Hat layers may not support softunlocking lengths less than the
1682 1680 * original length when there is large page support.
1683 1681 * -kpmem locking is dependent on keeping the lengths same.
1684 1682 * -if drivers handled F_SOFTLOCK, they probably also expect to
1685 1683 * see an F_SOFTUNLOCK of the same length
1686 1684 * Hence, if extending lengths during softlock,
1687 1685 * softunlock has to make the same adjustments and goes through
1688 1686 * the same loop calling segdev_faultpages/segdev_softunlock
1689 1687 * But some of the synchronization and error handling is different
1690 1688 */
1691 1689
1692 1690 if (type != F_SOFTUNLOCK) {
1693 1691 devmap_handle_t *dhpp = dhp;
1694 1692 size_t slen = len;
1695 1693
1696 1694 /*
1697 1695 * Calculate count of pages that are :
1698 1696 * a) within the (potentially extended) fault region
1699 1697 * b) AND covered by devmap handle with devmap_access
1700 1698 */
1701 1699 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1702 1700 while (slen != 0) {
1703 1701 size_t mlen;
1704 1702
1705 1703 /*
1706 1704 * Softlocking on a region that allows remap is
1707 1705 * unsupported due to unresolved locking issues
1708 1706 * XXX: unclear what these are?
1709 1707 * One potential is that if there is a pending
1710 1708 * softlock, then a remap should not be allowed
1711 1709 * until the unlock is done. This is easily
1712 1710 * fixed by returning error in devmap*remap on
1713 1711 * checking the dh->dh_softlock->softlocked value
1714 1712 */
1715 1713 if ((type == F_SOFTLOCK) &&
1716 1714 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1717 1715 return (FC_NOSUPPORT);
1718 1716 }
1719 1717
1720 1718 mlen = MIN(slen, (dhpp->dh_len - off));
1721 1719 if (dhpp->dh_callbackops.devmap_access) {
1722 1720 size_t llen;
1723 1721 caddr_t laddr;
1724 1722 /*
1725 1723 * use extended length for large page mappings
1726 1724 */
1727 1725 HOLD_DHP_LOCK(dhpp);
1728 1726 if ((sdp->pageprot == 0) &&
1729 1727 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1730 1728 devmap_get_large_pgsize(dhpp,
1731 1729 mlen, maddr, &llen, &laddr);
1732 1730 } else {
1733 1731 llen = mlen;
1734 1732 }
1735 1733 RELE_DHP_LOCK(dhpp);
1736 1734
1737 1735 slpage += btopr(llen);
1738 1736 slock = dhpp->dh_softlock;
1739 1737 }
1740 1738 maddr += mlen;
1741 1739 ASSERT(slen >= mlen);
1742 1740 slen -= mlen;
1743 1741 dhpp = dhpp->dh_next;
1744 1742 off = 0;
1745 1743 }
1746 1744 /*
1747 1745 * synchonize with other faulting threads and wait till safe
1748 1746 * devmap_softlock_enter might return due to signal in cv_wait
1749 1747 *
1750 1748 * devmap_softlock_enter has to be called outside of while loop
1751 1749 * to prevent a deadlock if len spans over multiple dhps.
1752 1750 * dh_softlock is based on device instance and if multiple dhps
1753 1751 * use the same device instance, the second dhp's LOCK call
1754 1752 * will hang waiting on the first to complete.
1755 1753 * devmap_setup verifies that slocks in a dhp_chain are same.
1756 1754 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1757 1755 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1758 1756 * we could have done the softlock_enter inside the loop
1759 1757 * and supported multi-dhp mappings with dissimilar devices
1760 1758 */
1761 1759 if (err = devmap_softlock_enter(slock, slpage, type))
1762 1760 return (FC_MAKE_ERR(err));
1763 1761 }
1764 1762
1765 1763 /* reset 'maddr' to the start addr of the range of fault. */
1766 1764 maddr = addr;
1767 1765
1768 1766 /* calculate the offset corresponds to 'addr' in the first dhp. */
1769 1767 off = (ulong_t)(addr - dhp->dh_uvaddr);
1770 1768
1771 1769 /*
1772 1770 * The fault length may span over multiple dhps.
1773 1771 * Loop until the total length is satisfied.
1774 1772 */
1775 1773 while (len != 0) {
1776 1774 size_t llen;
1777 1775 size_t mlen;
1778 1776 caddr_t laddr;
1779 1777
1780 1778 /*
1781 1779 * mlen is the smaller of 'len' and the length
1782 1780 * from addr to the end of mapping defined by dhp.
1783 1781 */
1784 1782 mlen = MIN(len, (dhp->dh_len - off));
1785 1783
1786 1784 HOLD_DHP_LOCK(dhp);
1787 1785 /*
1788 1786 * Pass the extended length and address to devmap_access
1789 1787 * if large pagesize is used for loading address translations.
1790 1788 */
1791 1789 if ((sdp->pageprot == 0) &&
1792 1790 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1793 1791 devmap_get_large_pgsize(dhp, mlen, maddr,
1794 1792 &llen, &laddr);
1795 1793 ASSERT(maddr == addr || laddr == maddr);
1796 1794 } else {
1797 1795 llen = mlen;
1798 1796 laddr = maddr;
1799 1797 }
1800 1798
1801 1799 if (dhp->dh_callbackops.devmap_access != NULL) {
1802 1800 offset_t aoff;
1803 1801
1804 1802 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1805 1803
1806 1804 /*
1807 1805 * call driver's devmap_access entry point which will
1808 1806 * call devmap_load/contextmgmt to load the translations
1809 1807 *
1810 1808 * We drop the dhp_lock before calling access so
1811 1809 * drivers can call devmap_*_remap within access
1812 1810 */
1813 1811 RELE_DHP_LOCK(dhp);
1814 1812
1815 1813 err = (*dhp->dh_callbackops.devmap_access)(
1816 1814 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1817 1815 } else {
1818 1816 /*
1819 1817 * If no devmap_access entry point, then load mappings
1820 1818 * hold dhp_lock across faultpages if REMAP
1821 1819 */
1822 1820 err = segdev_faultpages(hat, seg, laddr, llen,
1823 1821 type, rw, dhp);
1824 1822 err_is_faultcode = 1;
1825 1823 RELE_DHP_LOCK(dhp);
1826 1824 }
1827 1825
1828 1826 if (err) {
1829 1827 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1830 1828 /*
1831 1829 * If not first dhp, use
1832 1830 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1833 1831 * While this is recursion, it is incorrect to
1834 1832 * call just segdev_softunlock
1835 1833 * if we are using either large pages
1836 1834 * or devmap_access. It will be more right
1837 1835 * to go through the same loop as above
1838 1836 * rather than call segdev_softunlock directly
1839 1837 * It will use the right lenghths as well as
1840 1838 * call into the driver devmap_access routines.
1841 1839 */
1842 1840 size_t done = (size_t)(maddr - addr);
1843 1841 (void) segdev_fault(hat, seg, addr, done,
1844 1842 F_SOFTUNLOCK, S_OTHER);
1845 1843 /*
1846 1844 * reduce slpage by number of pages
1847 1845 * released by segdev_softunlock
1848 1846 */
1849 1847 ASSERT(slpage >= btopr(done));
1850 1848 devmap_softlock_exit(slock,
1851 1849 slpage - btopr(done), type);
1852 1850 } else {
1853 1851 devmap_softlock_exit(slock, slpage, type);
1854 1852 }
1855 1853
1856 1854
1857 1855 /*
1858 1856 * Segdev_faultpages() already returns a faultcode,
1859 1857 * hence, result from segdev_faultpages() should be
1860 1858 * returned directly.
1861 1859 */
1862 1860 if (err_is_faultcode)
1863 1861 return (err);
1864 1862 return (FC_MAKE_ERR(err));
1865 1863 }
1866 1864
1867 1865 maddr += mlen;
1868 1866 ASSERT(len >= mlen);
1869 1867 len -= mlen;
1870 1868 dhp = dhp->dh_next;
1871 1869 off = 0;
1872 1870
1873 1871 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1874 1872 }
1875 1873 /*
1876 1874 * release the softlock count at end of fault
1877 1875 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1878 1876 */
1879 1877 if ((type == F_INVAL) || (type == F_PROT))
1880 1878 devmap_softlock_exit(slock, slpage, type);
1881 1879 return (0);
1882 1880 }
1883 1881
1884 1882 /*
1885 1883 * segdev_faultpages
1886 1884 *
1887 1885 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1888 1886 * This routine assumes that the callers makes sure that the fields
1889 1887 * in dhp used below are not changed due to remap during this call.
1890 1888 * Caller does HOLD_DHP_LOCK if neeed
1891 1889 * This routine returns a faultcode_t as a return value for segdev_fault.
1892 1890 */
1893 1891 static faultcode_t
1894 1892 segdev_faultpages(
1895 1893 struct hat *hat, /* the hat */
1896 1894 struct seg *seg, /* the seg_dev of interest */
1897 1895 caddr_t addr, /* the address of the fault */
1898 1896 size_t len, /* the length of the range */
1899 1897 enum fault_type type, /* type of fault */
1900 1898 enum seg_rw rw, /* type of access at fault */
1901 1899 devmap_handle_t *dhp) /* devmap handle */
1902 1900 {
1903 1901 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1904 1902 register caddr_t a;
1905 1903 struct vpage *vpage;
1906 1904 struct ddi_umem_cookie *kpmem_cookie = NULL;
1907 1905 int err;
1908 1906
1909 1907 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1910 1908 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1911 1909 (void *)dhp, (void *)seg, (void *)addr, len);
1912 1910 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1913 1911 "dhp %p seg %p addr %p len %lx\n",
1914 1912 (void *)dhp, (void *)seg, (void *)addr, len));
1915 1913
1916 1914 /*
1917 1915 * The seg_dev driver does not implement copy-on-write,
1918 1916 * and always loads translations with maximal allowed permissions
1919 1917 * but we got an fault trying to access the device.
1920 1918 * Servicing the fault is not going to result in any better result
1921 1919 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1922 1920 * This code should be removed and let the normal fault handling
1923 1921 * take care of finding the error
1924 1922 */
1925 1923 if (type == F_PROT) {
1926 1924 return (FC_PROT);
1927 1925 }
1928 1926
1929 1927 if (type == F_SOFTUNLOCK) {
1930 1928 segdev_softunlock(hat, seg, addr, len, rw);
1931 1929 return (0);
1932 1930 }
1933 1931
1934 1932 /*
1935 1933 * For kernel pageable memory, fault/lock segkp pages
1936 1934 * We hold this until the completion of this
1937 1935 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1938 1936 */
1939 1937 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1940 1938 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1941 1939 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1942 1940 return (err);
1943 1941 }
1944 1942
1945 1943 /*
1946 1944 * If we have the same protections for the entire segment,
1947 1945 * insure that the access being attempted is legitimate.
1948 1946 */
1949 1947 rw_enter(&sdp->lock, RW_READER);
1950 1948 if (sdp->pageprot == 0) {
1951 1949 uint_t protchk;
1952 1950
1953 1951 switch (rw) {
1954 1952 case S_READ:
1955 1953 protchk = PROT_READ;
1956 1954 break;
1957 1955 case S_WRITE:
1958 1956 protchk = PROT_WRITE;
1959 1957 break;
1960 1958 case S_EXEC:
1961 1959 protchk = PROT_EXEC;
1962 1960 break;
1963 1961 case S_OTHER:
1964 1962 default:
1965 1963 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1966 1964 break;
1967 1965 }
1968 1966
1969 1967 if ((sdp->prot & protchk) == 0) {
1970 1968 rw_exit(&sdp->lock);
1971 1969 /* undo kpmem locking */
1972 1970 if (kpmem_cookie != NULL) {
1973 1971 release_kpmem_lock(kpmem_cookie, btopr(len));
1974 1972 }
1975 1973 return (FC_PROT); /* illegal access type */
1976 1974 }
1977 1975 }
1978 1976
1979 1977 /*
1980 1978 * we do a single hat_devload for the range if
1981 1979 * - devmap framework (dhp is not NULL),
1982 1980 * - pageprot == 0, i.e., no per-page protection set and
1983 1981 * - is device pages, irrespective of whether we are using large pages
1984 1982 */
1985 1983 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1986 1984 pfn_t pfnum;
1987 1985 uint_t hat_flags;
1988 1986
1989 1987 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1990 1988 rw_exit(&sdp->lock);
1991 1989 return (FC_NOMAP);
1992 1990 }
1993 1991
1994 1992 if (type == F_SOFTLOCK) {
1995 1993 mutex_enter(&freemem_lock);
1996 1994 sdp->softlockcnt += btopr(len);
1997 1995 mutex_exit(&freemem_lock);
1998 1996 }
1999 1997
2000 1998 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2001 1999 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2002 2000 ASSERT(!pf_is_memory(pfnum));
2003 2001
2004 2002 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2005 2003 hat_flags | sdp->hat_flags);
2006 2004 rw_exit(&sdp->lock);
2007 2005 return (0);
2008 2006 }
2009 2007
2010 2008 /* Handle cases where we have to loop through fault handling per-page */
2011 2009
2012 2010 if (sdp->vpage == NULL)
2013 2011 vpage = NULL;
2014 2012 else
2015 2013 vpage = &sdp->vpage[seg_page(seg, addr)];
2016 2014
2017 2015 /* loop over the address range handling each fault */
2018 2016 for (a = addr; a < addr + len; a += PAGESIZE) {
2019 2017 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2020 2018 break;
2021 2019 }
2022 2020 if (vpage != NULL)
2023 2021 vpage++;
2024 2022 }
2025 2023 rw_exit(&sdp->lock);
2026 2024 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2027 2025 size_t done = (size_t)(a - addr); /* pages fault successfully */
2028 2026 if (done > 0) {
2029 2027 /* use softunlock for those pages */
2030 2028 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2031 2029 }
2032 2030 if (kpmem_cookie != NULL) {
2033 2031 /* release kpmem lock for rest of pages */
2034 2032 ASSERT(len >= done);
2035 2033 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2036 2034 }
2037 2035 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2038 2036 /* for non-SOFTLOCK cases, release kpmem */
2039 2037 release_kpmem_lock(kpmem_cookie, btopr(len));
2040 2038 }
2041 2039 return (err);
2042 2040 }
2043 2041
2044 2042 /*
2045 2043 * Asynchronous page fault. We simply do nothing since this
2046 2044 * entry point is not supposed to load up the translation.
2047 2045 */
2048 2046 /*ARGSUSED*/
2049 2047 static faultcode_t
2050 2048 segdev_faulta(struct seg *seg, caddr_t addr)
2051 2049 {
2052 2050 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2053 2051 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2054 2052 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2055 2053
2056 2054 return (0);
2057 2055 }
2058 2056
2059 2057 static int
2060 2058 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2061 2059 {
2062 2060 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2063 2061 register devmap_handle_t *dhp;
2064 2062 register struct vpage *vp, *evp;
2065 2063 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2066 2064 ulong_t off;
2067 2065 size_t mlen, sz;
2068 2066
2069 2067 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2070 2068 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2071 2069 (void *)seg, (void *)addr, len, prot);
2072 2070 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2073 2071
2074 2072 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2075 2073 /*
2076 2074 * Fail the setprot if pages are SOFTLOCKed through this
2077 2075 * mapping.
2078 2076 * Softlockcnt is protected from change by the as read lock.
2079 2077 */
2080 2078 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2081 2079 "segdev_setprot:error softlockcnt=%lx", sz);
2082 2080 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2083 2081 return (EAGAIN);
2084 2082 }
2085 2083
2086 2084 if (dhp_head != NULL) {
2087 2085 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2088 2086 return (EINVAL);
2089 2087
2090 2088 /*
2091 2089 * check if violate maxprot.
2092 2090 */
2093 2091 off = (ulong_t)(addr - dhp->dh_uvaddr);
2094 2092 mlen = len;
2095 2093 while (dhp) {
2096 2094 if ((dhp->dh_maxprot & prot) != prot)
2097 2095 return (EACCES); /* violated maxprot */
2098 2096
2099 2097 if (mlen > (dhp->dh_len - off)) {
2100 2098 mlen -= dhp->dh_len - off;
2101 2099 dhp = dhp->dh_next;
2102 2100 off = 0;
2103 2101 } else
2104 2102 break;
2105 2103 }
2106 2104 } else {
2107 2105 if ((sdp->maxprot & prot) != prot)
2108 2106 return (EACCES);
2109 2107 }
2110 2108
2111 2109 rw_enter(&sdp->lock, RW_WRITER);
2112 2110 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2113 2111 if (sdp->prot == prot) {
2114 2112 rw_exit(&sdp->lock);
2115 2113 return (0); /* all done */
2116 2114 }
2117 2115 sdp->prot = (uchar_t)prot;
2118 2116 } else {
2119 2117 sdp->pageprot = 1;
2120 2118 if (sdp->vpage == NULL) {
2121 2119 /*
2122 2120 * First time through setting per page permissions,
2123 2121 * initialize all the vpage structures to prot
2124 2122 */
2125 2123 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2126 2124 KM_SLEEP);
2127 2125 evp = &sdp->vpage[seg_pages(seg)];
2128 2126 for (vp = sdp->vpage; vp < evp; vp++)
2129 2127 VPP_SETPROT(vp, sdp->prot);
2130 2128 }
2131 2129 /*
2132 2130 * Now go change the needed vpages protections.
2133 2131 */
2134 2132 evp = &sdp->vpage[seg_page(seg, addr + len)];
2135 2133 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2136 2134 VPP_SETPROT(vp, prot);
2137 2135 }
2138 2136 rw_exit(&sdp->lock);
2139 2137
2140 2138 if (dhp_head != NULL) {
2141 2139 devmap_handle_t *tdhp;
2142 2140 /*
2143 2141 * If large page size was used in hat_devload(),
2144 2142 * the same page size must be used in hat_unload().
2145 2143 */
2146 2144 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2147 2145 while (tdhp != NULL) {
2148 2146 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2149 2147 break;
2150 2148 }
2151 2149 tdhp = tdhp->dh_next;
2152 2150 }
2153 2151 if (tdhp) {
2154 2152 size_t slen = len;
2155 2153 size_t mlen;
2156 2154 size_t soff;
2157 2155
2158 2156 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2159 2157 while (slen != 0) {
2160 2158 mlen = MIN(slen, (dhp->dh_len - soff));
2161 2159 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2162 2160 dhp->dh_len, HAT_UNLOAD);
2163 2161 dhp = dhp->dh_next;
2164 2162 ASSERT(slen >= mlen);
2165 2163 slen -= mlen;
2166 2164 soff = 0;
2167 2165 }
2168 2166 return (0);
2169 2167 }
2170 2168 }
2171 2169
2172 2170 if ((prot & ~PROT_USER) == PROT_NONE) {
2173 2171 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2174 2172 } else {
2175 2173 /*
2176 2174 * RFE: the segment should keep track of all attributes
2177 2175 * allowing us to remove the deprecated hat_chgprot
2178 2176 * and use hat_chgattr.
2179 2177 */
2180 2178 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2181 2179 }
2182 2180
2183 2181 return (0);
2184 2182 }
2185 2183
2186 2184 static int
2187 2185 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2188 2186 {
2189 2187 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2190 2188 struct vpage *vp, *evp;
2191 2189
2192 2190 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2193 2191 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2194 2192 (void *)seg, (void *)addr, len, prot);
2195 2193 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2196 2194
2197 2195 /*
2198 2196 * If segment protection can be used, simply check against them
2199 2197 */
2200 2198 rw_enter(&sdp->lock, RW_READER);
2201 2199 if (sdp->pageprot == 0) {
2202 2200 register int err;
2203 2201
2204 2202 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2205 2203 rw_exit(&sdp->lock);
2206 2204 return (err);
2207 2205 }
2208 2206
2209 2207 /*
2210 2208 * Have to check down to the vpage level
2211 2209 */
2212 2210 evp = &sdp->vpage[seg_page(seg, addr + len)];
2213 2211 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2214 2212 if ((VPP_PROT(vp) & prot) != prot) {
2215 2213 rw_exit(&sdp->lock);
2216 2214 return (EACCES);
2217 2215 }
2218 2216 }
2219 2217 rw_exit(&sdp->lock);
2220 2218 return (0);
2221 2219 }
2222 2220
2223 2221 static int
2224 2222 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2225 2223 {
2226 2224 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2227 2225 size_t pgno;
2228 2226
2229 2227 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2230 2228 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2231 2229 (void *)seg, (void *)addr, len, (void *)protv);
2232 2230 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2233 2231
2234 2232 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2235 2233 if (pgno != 0) {
2236 2234 rw_enter(&sdp->lock, RW_READER);
2237 2235 if (sdp->pageprot == 0) {
2238 2236 do {
2239 2237 protv[--pgno] = sdp->prot;
2240 2238 } while (pgno != 0);
2241 2239 } else {
2242 2240 size_t pgoff = seg_page(seg, addr);
2243 2241
2244 2242 do {
2245 2243 pgno--;
2246 2244 protv[pgno] =
2247 2245 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2248 2246 } while (pgno != 0);
2249 2247 }
2250 2248 rw_exit(&sdp->lock);
2251 2249 }
2252 2250 return (0);
2253 2251 }
2254 2252
2255 2253 static u_offset_t
2256 2254 segdev_getoffset(register struct seg *seg, caddr_t addr)
2257 2255 {
2258 2256 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2259 2257
2260 2258 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2261 2259 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2262 2260
2263 2261 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2264 2262
2265 2263 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2266 2264 }
2267 2265
2268 2266 /*ARGSUSED*/
2269 2267 static int
2270 2268 segdev_gettype(register struct seg *seg, caddr_t addr)
2271 2269 {
2272 2270 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2273 2271
2274 2272 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2275 2273 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2276 2274
2277 2275 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2278 2276
2279 2277 return (sdp->type);
2280 2278 }
2281 2279
2282 2280
2283 2281 /*ARGSUSED*/
2284 2282 static int
2285 2283 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2286 2284 {
2287 2285 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2288 2286
2289 2287 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2290 2288 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2291 2289
2292 2290 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2293 2291
2294 2292 /*
2295 2293 * Note that this vp is the common_vp of the device, where the
2296 2294 * pages are hung ..
2297 2295 */
2298 2296 *vpp = VTOCVP(sdp->vp);
2299 2297
2300 2298 return (0);
2301 2299 }
2302 2300
2303 2301 static void
2304 2302 segdev_badop(void)
2305 2303 {
2306 2304 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2307 2305 "segdev_badop:start");
2308 2306 panic("segdev_badop");
2309 2307 /*NOTREACHED*/
2310 2308 }
2311 2309
2312 2310 /*
2313 2311 * segdev pages are not in the cache, and thus can't really be controlled.
2314 2312 * Hence, syncs are simply always successful.
2315 2313 */
2316 2314 /*ARGSUSED*/
2317 2315 static int
2318 2316 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2319 2317 {
2320 2318 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2321 2319
2322 2320 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2323 2321
2324 2322 return (0);
2325 2323 }
2326 2324
2327 2325 /*
2328 2326 * segdev pages are always "in core".
2329 2327 */
2330 2328 /*ARGSUSED*/
2331 2329 static size_t
2332 2330 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2333 2331 {
2334 2332 size_t v = 0;
2335 2333
2336 2334 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2337 2335
2338 2336 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2339 2337
2340 2338 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2341 2339 v += PAGESIZE)
2342 2340 *vec++ = 1;
2343 2341 return (v);
2344 2342 }
2345 2343
2346 2344 /*
2347 2345 * segdev pages are not in the cache, and thus can't really be controlled.
2348 2346 * Hence, locks are simply always successful.
2349 2347 */
2350 2348 /*ARGSUSED*/
2351 2349 static int
2352 2350 segdev_lockop(struct seg *seg, caddr_t addr,
2353 2351 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2354 2352 {
2355 2353 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2356 2354
2357 2355 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2358 2356
2359 2357 return (0);
2360 2358 }
2361 2359
2362 2360 /*
2363 2361 * segdev pages are not in the cache, and thus can't really be controlled.
2364 2362 * Hence, advise is simply always successful.
2365 2363 */
2366 2364 /*ARGSUSED*/
2367 2365 static int
2368 2366 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2369 2367 {
2370 2368 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2371 2369
2372 2370 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2373 2371
2374 2372 return (0);
2375 2373 }
2376 2374
2377 2375 /*
2378 2376 * segdev pages are not dumped, so we just return
2379 2377 */
2380 2378 /*ARGSUSED*/
2381 2379 static void
2382 2380 segdev_dump(struct seg *seg)
2383 2381 {}
2384 2382
2385 2383 /*
2386 2384 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2387 2385 * for a segment. Called from a drivers segmap(9E)
2388 2386 * routine.
2389 2387 */
2390 2388 /*ARGSUSED*/
2391 2389 int
2392 2390 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2393 2391 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2394 2392 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2395 2393 {
2396 2394 struct segdev_crargs dev_a;
2397 2395 int (*mapfunc)(dev_t dev, off_t off, int prot);
2398 2396 uint_t hat_attr;
2399 2397 pfn_t pfn;
2400 2398 int error, i;
2401 2399
2402 2400 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2403 2401 "ddi_segmap_setup:start");
2404 2402
2405 2403 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2406 2404 return (ENODEV);
2407 2405
2408 2406 /*
2409 2407 * Character devices that support the d_mmap
2410 2408 * interface can only be mmap'ed shared.
2411 2409 */
2412 2410 if ((flags & MAP_TYPE) != MAP_SHARED)
2413 2411 return (EINVAL);
2414 2412
2415 2413 /*
2416 2414 * Check that this region is indeed mappable on this platform.
2417 2415 * Use the mapping function.
2418 2416 */
2419 2417 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2420 2418 return (ENXIO);
2421 2419
2422 2420 /*
2423 2421 * Check to ensure that the entire range is
2424 2422 * legal and we are not trying to map in
2425 2423 * more than the device will let us.
2426 2424 */
2427 2425 for (i = 0; i < len; i += PAGESIZE) {
2428 2426 if (i == 0) {
2429 2427 /*
2430 2428 * Save the pfn at offset here. This pfn will be
2431 2429 * used later to get user address.
2432 2430 */
2433 2431 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2434 2432 maxprot)) == PFN_INVALID)
2435 2433 return (ENXIO);
2436 2434 } else {
2437 2435 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2438 2436 PFN_INVALID)
2439 2437 return (ENXIO);
2440 2438 }
2441 2439 }
2442 2440
2443 2441 as_rangelock(as);
2444 2442 /* Pick an address w/o worrying about any vac alignment constraints. */
2445 2443 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2446 2444 if (error != 0) {
2447 2445 as_rangeunlock(as);
2448 2446 return (error);
2449 2447 }
2450 2448
2451 2449 dev_a.mapfunc = mapfunc;
2452 2450 dev_a.dev = dev;
2453 2451 dev_a.offset = (offset_t)offset;
2454 2452 dev_a.type = flags & MAP_TYPE;
2455 2453 dev_a.prot = (uchar_t)prot;
2456 2454 dev_a.maxprot = (uchar_t)maxprot;
2457 2455 dev_a.hat_attr = hat_attr;
2458 2456 dev_a.hat_flags = 0;
2459 2457 dev_a.devmap_data = NULL;
2460 2458
2461 2459 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2462 2460 as_rangeunlock(as);
2463 2461 return (error);
↓ open down ↓ |
2242 lines elided |
↑ open up ↑ |
2464 2462
2465 2463 }
2466 2464
2467 2465 /*ARGSUSED*/
2468 2466 static int
2469 2467 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2470 2468 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2471 2469 {
2472 2470 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2473 2471 "segdev_pagelock:start");
2474 - return (ENOTSUP);
2475 -}
2476 -
2477 -/*ARGSUSED*/
2478 -static int
2479 -segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2480 - uint_t szc)
2481 -{
2482 2472 return (ENOTSUP);
2483 2473 }
2484 2474
2485 2475 /*
2486 2476 * devmap_device: Used by devmap framework to establish mapping
2487 2477 * called by devmap_seup(9F) during map setup time.
2488 2478 */
2489 2479 /*ARGSUSED*/
2490 2480 static int
2491 2481 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2492 2482 offset_t off, size_t len, uint_t flags)
2493 2483 {
2494 2484 devmap_handle_t *rdhp, *maxdhp;
2495 2485 struct segdev_crargs dev_a;
2496 2486 int err;
2497 2487 uint_t maxprot = PROT_ALL;
2498 2488 offset_t offset = 0;
2499 2489 pfn_t pfn;
2500 2490 struct devmap_pmem_cookie *pcp;
2501 2491
2502 2492 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2503 2493 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2504 2494 (void *)dhp, (void *)addr, off, len);
2505 2495
2506 2496 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2507 2497 (void *)dhp, (void *)addr, off, len));
2508 2498
2509 2499 as_rangelock(as);
2510 2500 if ((flags & MAP_FIXED) == 0) {
2511 2501 offset_t aligned_off;
2512 2502
2513 2503 rdhp = maxdhp = dhp;
2514 2504 while (rdhp != NULL) {
2515 2505 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2516 2506 maxdhp : rdhp;
2517 2507 rdhp = rdhp->dh_next;
2518 2508 maxprot |= dhp->dh_maxprot;
2519 2509 }
2520 2510 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2521 2511
2522 2512 /*
2523 2513 * Use the dhp that has the
2524 2514 * largest len to get user address.
2525 2515 */
2526 2516 /*
2527 2517 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2528 2518 * use 0 which is as good as any other.
2529 2519 */
2530 2520 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2531 2521 aligned_off = (offset_t)0;
2532 2522 } else if (dhp_is_devmem(maxdhp)) {
2533 2523 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2534 2524 } else if (dhp_is_pmem(maxdhp)) {
2535 2525 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2536 2526 pfn = page_pptonum(
2537 2527 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2538 2528 aligned_off = (offset_t)ptob(pfn) - offset;
2539 2529 } else {
2540 2530 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2541 2531 offset;
2542 2532 }
2543 2533
2544 2534 /*
2545 2535 * Pick an address aligned to dh_cookie.
2546 2536 * for kernel memory/user memory, cookie is cvaddr.
2547 2537 * for device memory, cookie is physical address.
2548 2538 */
2549 2539 map_addr(addr, len, aligned_off, 1, flags);
2550 2540 if (*addr == NULL) {
2551 2541 as_rangeunlock(as);
2552 2542 return (ENOMEM);
2553 2543 }
2554 2544 } else {
2555 2545 /*
2556 2546 * User-specified address; blow away any previous mappings.
2557 2547 */
2558 2548 (void) as_unmap(as, *addr, len);
2559 2549 }
2560 2550
2561 2551 dev_a.mapfunc = NULL;
2562 2552 dev_a.dev = dhp->dh_dev;
2563 2553 dev_a.type = flags & MAP_TYPE;
2564 2554 dev_a.offset = off;
2565 2555 /*
2566 2556 * sdp->maxprot has the least restrict protection of all dhps.
2567 2557 */
2568 2558 dev_a.maxprot = maxprot;
2569 2559 dev_a.prot = dhp->dh_prot;
2570 2560 /*
2571 2561 * devmap uses dhp->dh_hat_attr for hat.
2572 2562 */
2573 2563 dev_a.hat_flags = 0;
2574 2564 dev_a.hat_attr = 0;
2575 2565 dev_a.devmap_data = (void *)dhp;
2576 2566
2577 2567 err = as_map(as, *addr, len, segdev_create, &dev_a);
2578 2568 as_rangeunlock(as);
2579 2569 return (err);
2580 2570 }
2581 2571
2582 2572 int
2583 2573 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2584 2574 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2585 2575 size_t, uint_t, uint_t))
2586 2576 {
2587 2577 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2588 2578 struct devmap_ctx *devctx;
2589 2579 int do_timeout = 0;
2590 2580 int ret;
2591 2581
2592 2582 #ifdef lint
2593 2583 pvtp = pvtp;
2594 2584 #endif
2595 2585
2596 2586 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2597 2587 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2598 2588 (void *)dhp, off, len);
2599 2589 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2600 2590 (void *)dhp, off, len));
2601 2591
2602 2592 if (ctxmgt == NULL)
2603 2593 return (FC_HWERR);
2604 2594
2605 2595 devctx = dhp->dh_ctx;
2606 2596
2607 2597 /*
2608 2598 * If we are on an MP system with more than one cpu running
2609 2599 * and if a thread on some CPU already has the context, wait
2610 2600 * for it to finish if there is a hysteresis timeout.
2611 2601 *
2612 2602 * We call cv_wait() instead of cv_wait_sig() because
2613 2603 * it does not matter much if it returned due to a signal
2614 2604 * or due to a cv_signal() or cv_broadcast(). In either event
2615 2605 * we need to complete the mapping otherwise the processes
2616 2606 * will die with a SEGV.
2617 2607 */
2618 2608 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2619 2609 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2620 2610 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2621 2611 devctx, dhp);
2622 2612 do_timeout = 1;
2623 2613 mutex_enter(&devctx->lock);
2624 2614 while (devctx->oncpu)
2625 2615 cv_wait(&devctx->cv, &devctx->lock);
2626 2616 devctx->oncpu = 1;
2627 2617 mutex_exit(&devctx->lock);
2628 2618 }
2629 2619
2630 2620 /*
2631 2621 * Call the contextmgt callback so that the driver can handle
2632 2622 * the fault.
2633 2623 */
2634 2624 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2635 2625
2636 2626 /*
2637 2627 * If devmap_access() returned -1, then there was a hardware
2638 2628 * error so we need to convert the return value to something
2639 2629 * that trap() will understand. Otherwise, the return value
2640 2630 * is already a fault code generated by devmap_unload()
2641 2631 * or devmap_load().
2642 2632 */
2643 2633 if (ret) {
2644 2634 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2645 2635 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2646 2636 ret, dhp, devctx);
2647 2637 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2648 2638 ret, (void *)dhp));
2649 2639 if (devctx->oncpu) {
2650 2640 mutex_enter(&devctx->lock);
2651 2641 devctx->oncpu = 0;
2652 2642 cv_signal(&devctx->cv);
2653 2643 mutex_exit(&devctx->lock);
2654 2644 }
2655 2645 return (FC_HWERR);
2656 2646 }
2657 2647
2658 2648 /*
2659 2649 * Setup the timeout if we need to
2660 2650 */
2661 2651 if (do_timeout) {
2662 2652 mutex_enter(&devctx->lock);
2663 2653 if (dhp->dh_timeout_length > 0) {
2664 2654 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2665 2655 "devmap_do_ctxmgt:timeout set");
2666 2656 devctx->timeout = timeout(devmap_ctxto,
2667 2657 devctx, dhp->dh_timeout_length);
2668 2658 } else {
2669 2659 /*
2670 2660 * We don't want to wait so set oncpu to
2671 2661 * 0 and wake up anyone waiting.
2672 2662 */
2673 2663 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2674 2664 "devmap_do_ctxmgt:timeout not set");
2675 2665 devctx->oncpu = 0;
2676 2666 cv_signal(&devctx->cv);
2677 2667 }
2678 2668 mutex_exit(&devctx->lock);
2679 2669 }
2680 2670
2681 2671 return (DDI_SUCCESS);
2682 2672 }
2683 2673
2684 2674 /*
2685 2675 * end of mapping
2686 2676 * poff fault_offset |
2687 2677 * base | | |
2688 2678 * | | | |
2689 2679 * V V V V
2690 2680 * +-----------+---------------+-------+---------+-------+
2691 2681 * ^ ^ ^ ^
2692 2682 * |<--- offset--->|<-len->| |
2693 2683 * |<--- dh_len(size of mapping) --->|
2694 2684 * |<-- pg -->|
2695 2685 * -->|rlen|<--
2696 2686 */
2697 2687 static ulong_t
2698 2688 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2699 2689 ulong_t *opfn, ulong_t *pagesize)
2700 2690 {
2701 2691 register int level;
2702 2692 ulong_t pg;
2703 2693 ulong_t poff;
2704 2694 ulong_t base;
2705 2695 caddr_t uvaddr;
2706 2696 long rlen;
2707 2697
2708 2698 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2709 2699 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2710 2700 (void *)dhp, offset, len);
2711 2701 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2712 2702 (void *)dhp, offset, len));
2713 2703
2714 2704 /*
2715 2705 * get the max. pagesize that is aligned within the range
2716 2706 * <dh_pfn, dh_pfn+offset>.
2717 2707 *
2718 2708 * The calculations below use physical address to ddetermine
2719 2709 * the page size to use. The same calculations can use the
2720 2710 * virtual address to determine the page size.
2721 2711 */
2722 2712 base = (ulong_t)ptob(dhp->dh_pfn);
2723 2713 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2724 2714 pg = page_get_pagesize(level);
2725 2715 poff = ((base + offset) & ~(pg - 1));
2726 2716 uvaddr = dhp->dh_uvaddr + (poff - base);
2727 2717 if ((poff >= base) &&
2728 2718 ((poff + pg) <= (base + dhp->dh_len)) &&
2729 2719 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2730 2720 break;
2731 2721 }
2732 2722
2733 2723 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2734 2724 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2735 2725 base, poff, dhp);
2736 2726 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2737 2727 base, poff, dhp->dh_pfn));
2738 2728
2739 2729 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2740 2730 ASSERT(level >= 0);
2741 2731
2742 2732 *pagesize = pg;
2743 2733 *opfn = dhp->dh_pfn + btop(poff - base);
2744 2734
2745 2735 rlen = len + offset - (poff - base + pg);
2746 2736
2747 2737 ASSERT(rlen < (long)len);
2748 2738
2749 2739 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2750 2740 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2751 2741 (void *)dhp, level, rlen, pagesize, opfn);
2752 2742 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2753 2743 "level %x rlen %lx psize %lx opfn %lx\n",
2754 2744 (void *)dhp, level, rlen, *pagesize, *opfn));
2755 2745
2756 2746 return ((ulong_t)((rlen > 0) ? rlen : 0));
2757 2747 }
2758 2748
2759 2749 /*
2760 2750 * find the dhp that contains addr.
2761 2751 */
2762 2752 static devmap_handle_t *
2763 2753 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2764 2754 {
2765 2755 devmap_handle_t *dhp;
2766 2756
2767 2757 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2768 2758 "devmap_find_handle:start");
2769 2759
2770 2760 dhp = dhp_head;
2771 2761 while (dhp) {
2772 2762 if (addr >= dhp->dh_uvaddr &&
2773 2763 addr < (dhp->dh_uvaddr + dhp->dh_len))
2774 2764 return (dhp);
2775 2765 dhp = dhp->dh_next;
2776 2766 }
2777 2767
2778 2768 return ((devmap_handle_t *)NULL);
2779 2769 }
2780 2770
2781 2771 /*
2782 2772 * devmap_unload:
2783 2773 * Marks a segdev segment or pages if offset->offset+len
2784 2774 * is not the entire segment as intercept and unloads the
2785 2775 * pages in the range offset -> offset+len.
2786 2776 */
2787 2777 int
2788 2778 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2789 2779 {
2790 2780 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2791 2781 caddr_t addr;
2792 2782 ulong_t size;
2793 2783 ssize_t soff;
2794 2784
2795 2785 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2796 2786 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2797 2787 (void *)dhp, offset, len);
2798 2788 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2799 2789 (void *)dhp, offset, len));
2800 2790
2801 2791 soff = (ssize_t)(offset - dhp->dh_uoff);
2802 2792 soff = round_down_p2(soff, PAGESIZE);
2803 2793 if (soff < 0 || soff >= dhp->dh_len)
2804 2794 return (FC_MAKE_ERR(EINVAL));
2805 2795
2806 2796 /*
2807 2797 * Address and size must be page aligned. Len is set to the
2808 2798 * number of bytes in the number of pages that are required to
2809 2799 * support len. Offset is set to the byte offset of the first byte
2810 2800 * of the page that contains offset.
2811 2801 */
2812 2802 len = round_up_p2(len, PAGESIZE);
2813 2803
2814 2804 /*
2815 2805 * If len is == 0, then calculate the size by getting
2816 2806 * the number of bytes from offset to the end of the segment.
2817 2807 */
2818 2808 if (len == 0)
2819 2809 size = dhp->dh_len - soff;
2820 2810 else {
2821 2811 size = len;
2822 2812 if ((soff + size) > dhp->dh_len)
2823 2813 return (FC_MAKE_ERR(EINVAL));
2824 2814 }
2825 2815
2826 2816 /*
2827 2817 * The address is offset bytes from the base address of
2828 2818 * the dhp.
2829 2819 */
2830 2820 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2831 2821
2832 2822 /*
2833 2823 * If large page size was used in hat_devload(),
2834 2824 * the same page size must be used in hat_unload().
2835 2825 */
2836 2826 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2837 2827 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2838 2828 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2839 2829 } else {
2840 2830 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2841 2831 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2842 2832 }
2843 2833
2844 2834 return (0);
2845 2835 }
2846 2836
2847 2837 /*
2848 2838 * calculates the optimal page size that will be used for hat_devload().
2849 2839 */
2850 2840 static void
2851 2841 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2852 2842 size_t *llen, caddr_t *laddr)
2853 2843 {
2854 2844 ulong_t off;
2855 2845 ulong_t pfn;
2856 2846 ulong_t pgsize;
2857 2847 uint_t first = 1;
2858 2848
2859 2849 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2860 2850 "devmap_get_large_pgsize:start");
2861 2851
2862 2852 /*
2863 2853 * RFE - Code only supports large page mappings for devmem
2864 2854 * This code could be changed in future if we want to support
2865 2855 * large page mappings for kernel exported memory.
2866 2856 */
2867 2857 ASSERT(dhp_is_devmem(dhp));
2868 2858 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2869 2859
2870 2860 *llen = 0;
2871 2861 off = (ulong_t)(addr - dhp->dh_uvaddr);
2872 2862 while ((long)len > 0) {
2873 2863 /*
2874 2864 * get the optimal pfn to minimize address translations.
2875 2865 * devmap_roundup() returns residue bytes for next round
2876 2866 * calculations.
2877 2867 */
2878 2868 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2879 2869
2880 2870 if (first) {
2881 2871 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2882 2872 first = 0;
2883 2873 }
2884 2874
2885 2875 *llen += pgsize;
2886 2876 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2887 2877 }
2888 2878 /* Large page mapping len/addr cover more range than original fault */
2889 2879 ASSERT(*llen >= len && *laddr <= addr);
2890 2880 ASSERT((*laddr + *llen) >= (addr + len));
2891 2881 }
2892 2882
2893 2883 /*
2894 2884 * Initialize the devmap_softlock structure.
2895 2885 */
2896 2886 static struct devmap_softlock *
2897 2887 devmap_softlock_init(dev_t dev, ulong_t id)
2898 2888 {
2899 2889 struct devmap_softlock *slock;
2900 2890 struct devmap_softlock *tmp;
2901 2891
2902 2892 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2903 2893 "devmap_softlock_init:start");
2904 2894
2905 2895 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2906 2896 mutex_enter(&devmap_slock);
2907 2897
2908 2898 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2909 2899 if ((slock->dev == dev) && (slock->id == id))
2910 2900 break;
2911 2901
2912 2902 if (slock == NULL) {
2913 2903 slock = tmp;
2914 2904 slock->dev = dev;
2915 2905 slock->id = id;
2916 2906 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2917 2907 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2918 2908 slock->next = devmap_slist;
2919 2909 devmap_slist = slock;
2920 2910 } else
2921 2911 kmem_free(tmp, sizeof (struct devmap_softlock));
2922 2912
2923 2913 mutex_enter(&slock->lock);
2924 2914 slock->refcnt++;
2925 2915 mutex_exit(&slock->lock);
2926 2916 mutex_exit(&devmap_slock);
2927 2917
2928 2918 return (slock);
2929 2919 }
2930 2920
2931 2921 /*
2932 2922 * Wake up processes that sleep on softlocked.
2933 2923 * Free dh_softlock if refcnt is 0.
2934 2924 */
2935 2925 static void
2936 2926 devmap_softlock_rele(devmap_handle_t *dhp)
2937 2927 {
2938 2928 struct devmap_softlock *slock = dhp->dh_softlock;
2939 2929 struct devmap_softlock *tmp;
2940 2930 struct devmap_softlock *parent;
2941 2931
2942 2932 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2943 2933 "devmap_softlock_rele:start");
2944 2934
2945 2935 mutex_enter(&devmap_slock);
2946 2936 mutex_enter(&slock->lock);
2947 2937
2948 2938 ASSERT(slock->refcnt > 0);
2949 2939
2950 2940 slock->refcnt--;
2951 2941
2952 2942 /*
2953 2943 * If no one is using the device, free up the slock data.
2954 2944 */
2955 2945 if (slock->refcnt == 0) {
2956 2946 slock->softlocked = 0;
2957 2947 cv_signal(&slock->cv);
2958 2948
2959 2949 if (devmap_slist == slock)
2960 2950 devmap_slist = slock->next;
2961 2951 else {
2962 2952 parent = devmap_slist;
2963 2953 for (tmp = devmap_slist->next; tmp != NULL;
2964 2954 tmp = tmp->next) {
2965 2955 if (tmp == slock) {
2966 2956 parent->next = tmp->next;
2967 2957 break;
2968 2958 }
2969 2959 parent = tmp;
2970 2960 }
2971 2961 }
2972 2962 mutex_exit(&slock->lock);
2973 2963 mutex_destroy(&slock->lock);
2974 2964 cv_destroy(&slock->cv);
2975 2965 kmem_free(slock, sizeof (struct devmap_softlock));
2976 2966 } else
2977 2967 mutex_exit(&slock->lock);
2978 2968
2979 2969 mutex_exit(&devmap_slock);
2980 2970 }
2981 2971
2982 2972 /*
2983 2973 * Wake up processes that sleep on dh_ctx->locked.
2984 2974 * Free dh_ctx if refcnt is 0.
2985 2975 */
2986 2976 static void
2987 2977 devmap_ctx_rele(devmap_handle_t *dhp)
2988 2978 {
2989 2979 struct devmap_ctx *devctx = dhp->dh_ctx;
2990 2980 struct devmap_ctx *tmp;
2991 2981 struct devmap_ctx *parent;
2992 2982 timeout_id_t tid;
2993 2983
2994 2984 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
2995 2985 "devmap_ctx_rele:start");
2996 2986
2997 2987 mutex_enter(&devmapctx_lock);
2998 2988 mutex_enter(&devctx->lock);
2999 2989
3000 2990 ASSERT(devctx->refcnt > 0);
3001 2991
3002 2992 devctx->refcnt--;
3003 2993
3004 2994 /*
3005 2995 * If no one is using the device, free up the devctx data.
3006 2996 */
3007 2997 if (devctx->refcnt == 0) {
3008 2998 /*
3009 2999 * Untimeout any threads using this mapping as they are about
3010 3000 * to go away.
3011 3001 */
3012 3002 if (devctx->timeout != 0) {
3013 3003 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3014 3004 "devmap_ctx_rele:untimeout ctx->timeout");
3015 3005
3016 3006 tid = devctx->timeout;
3017 3007 mutex_exit(&devctx->lock);
3018 3008 (void) untimeout(tid);
3019 3009 mutex_enter(&devctx->lock);
3020 3010 }
3021 3011
3022 3012 devctx->oncpu = 0;
3023 3013 cv_signal(&devctx->cv);
3024 3014
3025 3015 if (devmapctx_list == devctx)
3026 3016 devmapctx_list = devctx->next;
3027 3017 else {
3028 3018 parent = devmapctx_list;
3029 3019 for (tmp = devmapctx_list->next; tmp != NULL;
3030 3020 tmp = tmp->next) {
3031 3021 if (tmp == devctx) {
3032 3022 parent->next = tmp->next;
3033 3023 break;
3034 3024 }
3035 3025 parent = tmp;
3036 3026 }
3037 3027 }
3038 3028 mutex_exit(&devctx->lock);
3039 3029 mutex_destroy(&devctx->lock);
3040 3030 cv_destroy(&devctx->cv);
3041 3031 kmem_free(devctx, sizeof (struct devmap_ctx));
3042 3032 } else
3043 3033 mutex_exit(&devctx->lock);
3044 3034
3045 3035 mutex_exit(&devmapctx_lock);
3046 3036 }
3047 3037
3048 3038 /*
3049 3039 * devmap_load:
3050 3040 * Marks a segdev segment or pages if offset->offset+len
3051 3041 * is not the entire segment as nointercept and faults in
3052 3042 * the pages in the range offset -> offset+len.
3053 3043 */
3054 3044 int
3055 3045 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3056 3046 uint_t rw)
3057 3047 {
3058 3048 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3059 3049 struct as *asp = dhp->dh_seg->s_as;
3060 3050 caddr_t addr;
3061 3051 ulong_t size;
3062 3052 ssize_t soff; /* offset from the beginning of the segment */
3063 3053 int rc;
3064 3054
3065 3055 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3066 3056 "devmap_load:start dhp=%p offset=%llx len=%lx",
3067 3057 (void *)dhp, offset, len);
3068 3058
3069 3059 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3070 3060 (void *)dhp, offset, len));
3071 3061
3072 3062 /*
3073 3063 * Hat layer only supports devload to process' context for which
3074 3064 * the as lock is held. Verify here and return error if drivers
3075 3065 * inadvertently call devmap_load on a wrong devmap handle.
3076 3066 */
3077 3067 if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3078 3068 return (FC_MAKE_ERR(EINVAL));
3079 3069
3080 3070 soff = (ssize_t)(offset - dhp->dh_uoff);
3081 3071 soff = round_down_p2(soff, PAGESIZE);
3082 3072 if (soff < 0 || soff >= dhp->dh_len)
3083 3073 return (FC_MAKE_ERR(EINVAL));
3084 3074
3085 3075 /*
3086 3076 * Address and size must be page aligned. Len is set to the
3087 3077 * number of bytes in the number of pages that are required to
3088 3078 * support len. Offset is set to the byte offset of the first byte
3089 3079 * of the page that contains offset.
3090 3080 */
3091 3081 len = round_up_p2(len, PAGESIZE);
3092 3082
3093 3083 /*
3094 3084 * If len == 0, then calculate the size by getting
3095 3085 * the number of bytes from offset to the end of the segment.
3096 3086 */
3097 3087 if (len == 0)
3098 3088 size = dhp->dh_len - soff;
3099 3089 else {
3100 3090 size = len;
3101 3091 if ((soff + size) > dhp->dh_len)
3102 3092 return (FC_MAKE_ERR(EINVAL));
3103 3093 }
3104 3094
3105 3095 /*
3106 3096 * The address is offset bytes from the base address of
3107 3097 * the segment.
3108 3098 */
3109 3099 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3110 3100
3111 3101 HOLD_DHP_LOCK(dhp);
3112 3102 rc = segdev_faultpages(asp->a_hat,
3113 3103 dhp->dh_seg, addr, size, type, rw, dhp);
3114 3104 RELE_DHP_LOCK(dhp);
3115 3105 return (rc);
3116 3106 }
3117 3107
3118 3108 int
3119 3109 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3120 3110 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3121 3111 {
3122 3112 register devmap_handle_t *dhp;
3123 3113 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3124 3114 size_t *, uint_t);
3125 3115 int (*mmap)(dev_t, off_t, int);
3126 3116 struct devmap_callback_ctl *callbackops;
3127 3117 devmap_handle_t *dhp_head = NULL;
3128 3118 devmap_handle_t *dhp_prev = NULL;
3129 3119 devmap_handle_t *dhp_curr;
3130 3120 caddr_t addr;
3131 3121 int map_flag;
3132 3122 int ret;
3133 3123 ulong_t total_len;
3134 3124 size_t map_len;
3135 3125 size_t resid_len = len;
3136 3126 offset_t map_off = off;
3137 3127 struct devmap_softlock *slock = NULL;
3138 3128
3139 3129 #ifdef lint
3140 3130 cred = cred;
3141 3131 #endif
3142 3132
3143 3133 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3144 3134 "devmap_setup:start off=%llx len=%lx", off, len);
3145 3135 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3146 3136 off, len));
3147 3137
3148 3138 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3149 3139 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3150 3140
3151 3141 /*
3152 3142 * driver must provide devmap(9E) entry point in cb_ops to use the
3153 3143 * devmap framework.
3154 3144 */
3155 3145 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3156 3146 return (EINVAL);
3157 3147
3158 3148 /*
3159 3149 * To protect from an inadvertent entry because the devmap entry point
3160 3150 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3161 3151 * mmap is NULL.
3162 3152 */
3163 3153 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3164 3154 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3165 3155 return (EINVAL);
3166 3156
3167 3157 /*
3168 3158 * devmap allows mmap(2) to map multiple registers.
3169 3159 * one devmap_handle is created for each register mapped.
3170 3160 */
3171 3161 for (total_len = 0; total_len < len; total_len += map_len) {
3172 3162 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3173 3163
3174 3164 if (dhp_prev != NULL)
3175 3165 dhp_prev->dh_next = dhp;
3176 3166 else
3177 3167 dhp_head = dhp;
3178 3168 dhp_prev = dhp;
3179 3169
3180 3170 dhp->dh_prot = prot;
3181 3171 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3182 3172 dhp->dh_dev = dev;
3183 3173 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3184 3174 dhp->dh_uoff = map_off;
3185 3175
3186 3176 /*
3187 3177 * Get mapping specific info from
3188 3178 * the driver, such as rnumber, roff, len, callbackops,
3189 3179 * accattrp and, if the mapping is for kernel memory,
3190 3180 * ddi_umem_cookie.
3191 3181 */
3192 3182 if ((ret = cdev_devmap(dev, dhp, map_off,
3193 3183 resid_len, &map_len, get_udatamodel())) != 0) {
3194 3184 free_devmap_handle(dhp_head);
3195 3185 return (ENXIO);
3196 3186 }
3197 3187
3198 3188 if (map_len & PAGEOFFSET) {
3199 3189 free_devmap_handle(dhp_head);
3200 3190 return (EINVAL);
3201 3191 }
3202 3192
3203 3193 callbackops = &dhp->dh_callbackops;
3204 3194
3205 3195 if ((callbackops->devmap_access == NULL) ||
3206 3196 (callbackops->devmap_access == nulldev) ||
3207 3197 (callbackops->devmap_access == nodev)) {
3208 3198 /*
3209 3199 * Normally devmap does not support MAP_PRIVATE unless
3210 3200 * the drivers provide a valid devmap_access routine.
3211 3201 */
3212 3202 if ((flags & MAP_PRIVATE) != 0) {
3213 3203 free_devmap_handle(dhp_head);
3214 3204 return (EINVAL);
3215 3205 }
3216 3206 } else {
3217 3207 /*
3218 3208 * Initialize dhp_softlock and dh_ctx if the drivers
3219 3209 * provide devmap_access.
3220 3210 */
3221 3211 dhp->dh_softlock = devmap_softlock_init(dev,
3222 3212 (ulong_t)callbackops->devmap_access);
3223 3213 dhp->dh_ctx = devmap_ctxinit(dev,
3224 3214 (ulong_t)callbackops->devmap_access);
3225 3215
3226 3216 /*
3227 3217 * segdev_fault can only work when all
3228 3218 * dh_softlock in a multi-dhp mapping
3229 3219 * are same. see comments in segdev_fault
3230 3220 * This code keeps track of the first
3231 3221 * dh_softlock allocated in slock and
3232 3222 * compares all later allocations and if
3233 3223 * not similar, returns an error.
3234 3224 */
3235 3225 if (slock == NULL)
3236 3226 slock = dhp->dh_softlock;
3237 3227 if (slock != dhp->dh_softlock) {
3238 3228 free_devmap_handle(dhp_head);
3239 3229 return (ENOTSUP);
3240 3230 }
3241 3231 }
3242 3232
3243 3233 map_off += map_len;
3244 3234 resid_len -= map_len;
3245 3235 }
3246 3236
3247 3237 /*
3248 3238 * get the user virtual address and establish the mapping between
3249 3239 * uvaddr and device physical address.
3250 3240 */
3251 3241 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3252 3242 != 0) {
3253 3243 /*
3254 3244 * free devmap handles if error during the mapping.
3255 3245 */
3256 3246 free_devmap_handle(dhp_head);
3257 3247
3258 3248 return (ret);
3259 3249 }
3260 3250
3261 3251 /*
3262 3252 * call the driver's devmap_map callback to do more after the mapping,
3263 3253 * such as to allocate driver private data for context management.
3264 3254 */
3265 3255 dhp = dhp_head;
3266 3256 map_off = off;
3267 3257 addr = *addrp;
3268 3258 while (dhp != NULL) {
3269 3259 callbackops = &dhp->dh_callbackops;
3270 3260 dhp->dh_uvaddr = addr;
3271 3261 dhp_curr = dhp;
3272 3262 if (callbackops->devmap_map != NULL) {
3273 3263 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3274 3264 dev, flags, map_off,
3275 3265 dhp->dh_len, &dhp->dh_pvtp);
3276 3266 if (ret != 0) {
3277 3267 struct segdev_data *sdp;
3278 3268
3279 3269 /*
3280 3270 * call driver's devmap_unmap entry point
3281 3271 * to free driver resources.
3282 3272 */
3283 3273 dhp = dhp_head;
3284 3274 map_off = off;
3285 3275 while (dhp != dhp_curr) {
3286 3276 callbackops = &dhp->dh_callbackops;
3287 3277 if (callbackops->devmap_unmap != NULL) {
3288 3278 (*callbackops->devmap_unmap)(
3289 3279 dhp, dhp->dh_pvtp,
3290 3280 map_off, dhp->dh_len,
3291 3281 NULL, NULL, NULL, NULL);
3292 3282 }
3293 3283 map_off += dhp->dh_len;
3294 3284 dhp = dhp->dh_next;
3295 3285 }
3296 3286 sdp = dhp_head->dh_seg->s_data;
3297 3287 sdp->devmap_data = NULL;
3298 3288 free_devmap_handle(dhp_head);
3299 3289 return (ENXIO);
3300 3290 }
3301 3291 }
3302 3292 map_off += dhp->dh_len;
3303 3293 addr += dhp->dh_len;
3304 3294 dhp = dhp->dh_next;
3305 3295 }
3306 3296
3307 3297 return (0);
3308 3298 }
3309 3299
3310 3300 int
3311 3301 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3312 3302 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3313 3303 {
3314 3304 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3315 3305 "devmap_segmap:start");
3316 3306 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3317 3307 (size_t)len, prot, maxprot, flags, cred));
3318 3308 }
3319 3309
3320 3310 /*
3321 3311 * Called from devmap_devmem_setup/remap to see if can use large pages for
3322 3312 * this device mapping.
3323 3313 * Also calculate the max. page size for this mapping.
3324 3314 * this page size will be used in fault routine for
3325 3315 * optimal page size calculations.
3326 3316 */
3327 3317 static void
3328 3318 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3329 3319 {
3330 3320 ASSERT(dhp_is_devmem(dhp));
3331 3321 dhp->dh_mmulevel = 0;
3332 3322
3333 3323 /*
3334 3324 * use large page size only if:
3335 3325 * 1. device memory.
3336 3326 * 2. mmu supports multiple page sizes,
3337 3327 * 3. Driver did not disallow it
3338 3328 * 4. dhp length is at least as big as the large pagesize
3339 3329 * 5. the uvaddr and pfn are large pagesize aligned
3340 3330 */
3341 3331 if (page_num_pagesizes() > 1 &&
3342 3332 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3343 3333 ulong_t base;
3344 3334 int level;
3345 3335
3346 3336 base = (ulong_t)ptob(dhp->dh_pfn);
3347 3337 for (level = 1; level < page_num_pagesizes(); level++) {
3348 3338 size_t pgsize = page_get_pagesize(level);
3349 3339 if ((dhp->dh_len < pgsize) ||
3350 3340 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3351 3341 base, pgsize))) {
3352 3342 break;
3353 3343 }
3354 3344 }
3355 3345 dhp->dh_mmulevel = level - 1;
3356 3346 }
3357 3347 if (dhp->dh_mmulevel > 0) {
3358 3348 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3359 3349 } else {
3360 3350 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3361 3351 }
3362 3352 }
3363 3353
3364 3354 /*
3365 3355 * Called by driver devmap routine to pass device specific info to
3366 3356 * the framework. used for device memory mapping only.
3367 3357 */
3368 3358 int
3369 3359 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3370 3360 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3371 3361 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3372 3362 {
3373 3363 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3374 3364 ddi_acc_handle_t handle;
3375 3365 ddi_map_req_t mr;
3376 3366 ddi_acc_hdl_t *hp;
3377 3367 int err;
3378 3368
3379 3369 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3380 3370 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3381 3371 (void *)dhp, roff, rnumber, (uint_t)len);
3382 3372 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3383 3373 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3384 3374
3385 3375 /*
3386 3376 * First to check if this function has been called for this dhp.
3387 3377 */
3388 3378 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3389 3379 return (DDI_FAILURE);
3390 3380
3391 3381 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3392 3382 return (DDI_FAILURE);
3393 3383
3394 3384 if (flags & DEVMAP_MAPPING_INVALID) {
3395 3385 /*
3396 3386 * Don't go up the tree to get pfn if the driver specifies
3397 3387 * DEVMAP_MAPPING_INVALID in flags.
3398 3388 *
3399 3389 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3400 3390 * remap permission.
3401 3391 */
3402 3392 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3403 3393 return (DDI_FAILURE);
3404 3394 }
3405 3395 dhp->dh_pfn = PFN_INVALID;
3406 3396 } else {
3407 3397 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3408 3398 if (handle == NULL)
3409 3399 return (DDI_FAILURE);
3410 3400
3411 3401 hp = impl_acc_hdl_get(handle);
3412 3402 hp->ah_vers = VERS_ACCHDL;
3413 3403 hp->ah_dip = dip;
3414 3404 hp->ah_rnumber = rnumber;
3415 3405 hp->ah_offset = roff;
3416 3406 hp->ah_len = len;
3417 3407 if (accattrp != NULL)
3418 3408 hp->ah_acc = *accattrp;
3419 3409
3420 3410 mr.map_op = DDI_MO_MAP_LOCKED;
3421 3411 mr.map_type = DDI_MT_RNUMBER;
3422 3412 mr.map_obj.rnumber = rnumber;
3423 3413 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3424 3414 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3425 3415 mr.map_handlep = hp;
3426 3416 mr.map_vers = DDI_MAP_VERSION;
3427 3417
3428 3418 /*
3429 3419 * up the device tree to get pfn.
3430 3420 * The rootnex_map_regspec() routine in nexus drivers has been
3431 3421 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3432 3422 */
3433 3423 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3434 3424 dhp->dh_hat_attr = hp->ah_hat_flags;
3435 3425 impl_acc_hdl_free(handle);
3436 3426
3437 3427 if (err)
3438 3428 return (DDI_FAILURE);
3439 3429 }
3440 3430 /* Should not be using devmem setup for memory pages */
3441 3431 ASSERT(!pf_is_memory(dhp->dh_pfn));
3442 3432
3443 3433 /* Only some of the flags bits are settable by the driver */
3444 3434 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3445 3435 dhp->dh_len = ptob(btopr(len));
3446 3436
3447 3437 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3448 3438 dhp->dh_roff = ptob(btop(roff));
3449 3439
3450 3440 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3451 3441 devmap_devmem_large_page_setup(dhp);
3452 3442 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3453 3443 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3454 3444
3455 3445
3456 3446 if (callbackops != NULL) {
3457 3447 bcopy(callbackops, &dhp->dh_callbackops,
3458 3448 sizeof (struct devmap_callback_ctl));
3459 3449 }
3460 3450
3461 3451 /*
3462 3452 * Initialize dh_lock if we want to do remap.
3463 3453 */
3464 3454 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3465 3455 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3466 3456 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3467 3457 }
3468 3458
3469 3459 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3470 3460
3471 3461 return (DDI_SUCCESS);
3472 3462 }
3473 3463
3474 3464 int
3475 3465 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3476 3466 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3477 3467 uint_t flags, ddi_device_acc_attr_t *accattrp)
3478 3468 {
3479 3469 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3480 3470 ddi_acc_handle_t handle;
3481 3471 ddi_map_req_t mr;
3482 3472 ddi_acc_hdl_t *hp;
3483 3473 pfn_t pfn;
3484 3474 uint_t hat_flags;
3485 3475 int err;
3486 3476
3487 3477 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3488 3478 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3489 3479 (void *)dhp, roff, rnumber, (uint_t)len);
3490 3480 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3491 3481 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3492 3482
3493 3483 /*
3494 3484 * Return failure if setup has not been done or no remap permission
3495 3485 * has been granted during the setup.
3496 3486 */
3497 3487 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3498 3488 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3499 3489 return (DDI_FAILURE);
3500 3490
3501 3491 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3502 3492 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3503 3493 return (DDI_FAILURE);
3504 3494
3505 3495 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3506 3496 return (DDI_FAILURE);
3507 3497
3508 3498 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3509 3499 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3510 3500 if (handle == NULL)
3511 3501 return (DDI_FAILURE);
3512 3502 }
3513 3503
3514 3504 HOLD_DHP_LOCK(dhp);
3515 3505
3516 3506 /*
3517 3507 * Unload the old mapping, so next fault will setup the new mappings
3518 3508 * Do this while holding the dhp lock so other faults dont reestablish
3519 3509 * the mappings
3520 3510 */
3521 3511 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3522 3512 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3523 3513
3524 3514 if (flags & DEVMAP_MAPPING_INVALID) {
3525 3515 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3526 3516 dhp->dh_pfn = PFN_INVALID;
3527 3517 } else {
3528 3518 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3529 3519 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3530 3520 hp = impl_acc_hdl_get(handle);
3531 3521 hp->ah_vers = VERS_ACCHDL;
3532 3522 hp->ah_dip = dip;
3533 3523 hp->ah_rnumber = rnumber;
3534 3524 hp->ah_offset = roff;
3535 3525 hp->ah_len = len;
3536 3526 if (accattrp != NULL)
3537 3527 hp->ah_acc = *accattrp;
3538 3528
3539 3529 mr.map_op = DDI_MO_MAP_LOCKED;
3540 3530 mr.map_type = DDI_MT_RNUMBER;
3541 3531 mr.map_obj.rnumber = rnumber;
3542 3532 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3543 3533 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3544 3534 mr.map_handlep = hp;
3545 3535 mr.map_vers = DDI_MAP_VERSION;
3546 3536
3547 3537 /*
3548 3538 * up the device tree to get pfn.
3549 3539 * The rootnex_map_regspec() routine in nexus drivers has been
3550 3540 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3551 3541 */
3552 3542 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3553 3543 hat_flags = hp->ah_hat_flags;
3554 3544 impl_acc_hdl_free(handle);
3555 3545 if (err) {
3556 3546 RELE_DHP_LOCK(dhp);
3557 3547 return (DDI_FAILURE);
3558 3548 }
3559 3549 /*
3560 3550 * Store result of ddi_map first in local variables, as we do
3561 3551 * not want to overwrite the existing dhp with wrong data.
3562 3552 */
3563 3553 dhp->dh_pfn = pfn;
3564 3554 dhp->dh_hat_attr = hat_flags;
3565 3555 }
3566 3556
3567 3557 /* clear the large page size flag */
3568 3558 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3569 3559
3570 3560 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3571 3561 dhp->dh_roff = ptob(btop(roff));
3572 3562
3573 3563 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3574 3564 devmap_devmem_large_page_setup(dhp);
3575 3565 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3576 3566 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3577 3567
3578 3568 RELE_DHP_LOCK(dhp);
3579 3569 return (DDI_SUCCESS);
3580 3570 }
3581 3571
3582 3572 /*
3583 3573 * called by driver devmap routine to pass kernel virtual address mapping
3584 3574 * info to the framework. used only for kernel memory
3585 3575 * allocated from ddi_umem_alloc().
3586 3576 */
3587 3577 int
3588 3578 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3589 3579 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3590 3580 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3591 3581 ddi_device_acc_attr_t *accattrp)
3592 3582 {
3593 3583 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3594 3584 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3595 3585
3596 3586 #ifdef lint
3597 3587 dip = dip;
3598 3588 #endif
3599 3589
3600 3590 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3601 3591 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3602 3592 (void *)dhp, off, cookie, len);
3603 3593 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3604 3594 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3605 3595
3606 3596 if (cookie == NULL)
3607 3597 return (DDI_FAILURE);
3608 3598
3609 3599 /* For UMEM_TRASH, this restriction is not needed */
3610 3600 if ((off + len) > cp->size)
3611 3601 return (DDI_FAILURE);
3612 3602
3613 3603 /* check if the cache attributes are supported */
3614 3604 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3615 3605 return (DDI_FAILURE);
3616 3606
3617 3607 /*
3618 3608 * First to check if this function has been called for this dhp.
3619 3609 */
3620 3610 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3621 3611 return (DDI_FAILURE);
3622 3612
3623 3613 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3624 3614 return (DDI_FAILURE);
3625 3615
3626 3616 if (flags & DEVMAP_MAPPING_INVALID) {
3627 3617 /*
3628 3618 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3629 3619 * remap permission.
3630 3620 */
3631 3621 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3632 3622 return (DDI_FAILURE);
3633 3623 }
3634 3624 } else {
3635 3625 dhp->dh_cookie = cookie;
3636 3626 dhp->dh_roff = ptob(btop(off));
3637 3627 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3638 3628 /* set HAT cache attributes */
3639 3629 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3640 3630 /* set HAT endianess attributes */
3641 3631 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3642 3632 }
3643 3633
3644 3634 /*
3645 3635 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3646 3636 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3647 3637 * create consistent mappings but our intention was to create
3648 3638 * non-consistent mappings.
3649 3639 *
3650 3640 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3651 3641 * mappings.
3652 3642 *
3653 3643 * kernel exported memory: hat figures it out it's memory and always
3654 3644 * creates consistent mappings.
3655 3645 *
3656 3646 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3657 3647 *
3658 3648 * /dev/kmem: consistent mappings are created unless they are
3659 3649 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3660 3650 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3661 3651 * mappings of /dev/kmem. See common/io/mem.c
3662 3652 */
3663 3653
3664 3654 /* Only some of the flags bits are settable by the driver */
3665 3655 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3666 3656
3667 3657 dhp->dh_len = ptob(btopr(len));
3668 3658 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3669 3659 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3670 3660
3671 3661 if (callbackops != NULL) {
3672 3662 bcopy(callbackops, &dhp->dh_callbackops,
3673 3663 sizeof (struct devmap_callback_ctl));
3674 3664 }
3675 3665 /*
3676 3666 * Initialize dh_lock if we want to do remap.
3677 3667 */
3678 3668 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3679 3669 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3680 3670 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3681 3671 }
3682 3672
3683 3673 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3684 3674
3685 3675 return (DDI_SUCCESS);
3686 3676 }
3687 3677
3688 3678 int
3689 3679 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3690 3680 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3691 3681 uint_t flags, ddi_device_acc_attr_t *accattrp)
3692 3682 {
3693 3683 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3694 3684 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3695 3685
3696 3686 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3697 3687 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3698 3688 (void *)dhp, off, cookie, len);
3699 3689 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3700 3690 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3701 3691
3702 3692 #ifdef lint
3703 3693 dip = dip;
3704 3694 accattrp = accattrp;
3705 3695 #endif
3706 3696 /*
3707 3697 * Reture failure if setup has not been done or no remap permission
3708 3698 * has been granted during the setup.
3709 3699 */
3710 3700 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3711 3701 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3712 3702 return (DDI_FAILURE);
3713 3703
3714 3704 /* No flags supported for remap yet */
3715 3705 if (flags != 0)
3716 3706 return (DDI_FAILURE);
3717 3707
3718 3708 /* check if the cache attributes are supported */
3719 3709 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3720 3710 return (DDI_FAILURE);
3721 3711
3722 3712 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3723 3713 return (DDI_FAILURE);
3724 3714
3725 3715 /* For UMEM_TRASH, this restriction is not needed */
3726 3716 if ((off + len) > cp->size)
3727 3717 return (DDI_FAILURE);
3728 3718
3729 3719 HOLD_DHP_LOCK(dhp);
3730 3720 /*
3731 3721 * Unload the old mapping, so next fault will setup the new mappings
3732 3722 * Do this while holding the dhp lock so other faults dont reestablish
3733 3723 * the mappings
3734 3724 */
3735 3725 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3736 3726 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3737 3727
3738 3728 dhp->dh_cookie = cookie;
3739 3729 dhp->dh_roff = ptob(btop(off));
3740 3730 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3741 3731 /* set HAT cache attributes */
3742 3732 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3743 3733 /* set HAT endianess attributes */
3744 3734 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3745 3735
3746 3736 /* clear the large page size flag */
3747 3737 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3748 3738
3749 3739 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3750 3740 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3751 3741 RELE_DHP_LOCK(dhp);
3752 3742 return (DDI_SUCCESS);
3753 3743 }
3754 3744
3755 3745 /*
3756 3746 * to set timeout value for the driver's context management callback, e.g.
3757 3747 * devmap_access().
3758 3748 */
3759 3749 void
3760 3750 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3761 3751 {
3762 3752 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3763 3753
3764 3754 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3765 3755 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3766 3756 (void *)dhp, ticks);
3767 3757 dhp->dh_timeout_length = ticks;
3768 3758 }
3769 3759
3770 3760 int
3771 3761 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3772 3762 size_t len, uint_t type, uint_t rw)
3773 3763 {
3774 3764 #ifdef lint
3775 3765 pvtp = pvtp;
3776 3766 #endif
3777 3767
3778 3768 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3779 3769 "devmap_default_access:start");
3780 3770 return (devmap_load(dhp, off, len, type, rw));
3781 3771 }
3782 3772
3783 3773 /*
3784 3774 * segkmem_alloc() wrapper to allocate memory which is both
3785 3775 * non-relocatable (for DR) and sharelocked, since the rest
3786 3776 * of this segment driver requires it.
3787 3777 */
3788 3778 static void *
3789 3779 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3790 3780 {
3791 3781 ASSERT(vmp != NULL);
3792 3782 ASSERT(kvseg.s_base != NULL);
3793 3783 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3794 3784 return (segkmem_alloc(vmp, size, vmflag));
3795 3785 }
3796 3786
3797 3787 /*
3798 3788 * This is where things are a bit incestuous with seg_kmem: unlike
3799 3789 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3800 3790 * we need to do a bit of a dance around that to prevent duplication of
3801 3791 * code until we decide to bite the bullet and implement a new kernel
3802 3792 * segment for driver-allocated memory that is exported to user space.
3803 3793 */
3804 3794 static void
3805 3795 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3806 3796 {
3807 3797 page_t *pp;
3808 3798 caddr_t addr = inaddr;
3809 3799 caddr_t eaddr;
3810 3800 pgcnt_t npages = btopr(size);
3811 3801
3812 3802 ASSERT(vmp != NULL);
3813 3803 ASSERT(kvseg.s_base != NULL);
3814 3804 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3815 3805
3816 3806 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3817 3807
3818 3808 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3819 3809 /*
3820 3810 * Use page_find() instead of page_lookup() to find the page
3821 3811 * since we know that it is hashed and has a shared lock.
3822 3812 */
3823 3813 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3824 3814
3825 3815 if (pp == NULL)
3826 3816 panic("devmap_free_pages: page not found");
3827 3817 if (!page_tryupgrade(pp)) {
3828 3818 page_unlock(pp);
3829 3819 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3830 3820 SE_EXCL);
3831 3821 if (pp == NULL)
3832 3822 panic("devmap_free_pages: page already freed");
3833 3823 }
3834 3824 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3835 3825 pp->p_lckcnt = 0;
3836 3826 page_destroy(pp, 0);
3837 3827 }
3838 3828 page_unresv(npages);
3839 3829
3840 3830 if (vmp != NULL)
3841 3831 vmem_free(vmp, inaddr, size);
3842 3832 }
3843 3833
3844 3834 /*
3845 3835 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3846 3836 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3847 3837 * default request. For now we allocate our own pages and we keep
3848 3838 * them long-term sharelocked, since: A) the fault routines expect the
3849 3839 * memory to already be locked; B) pageable umem is already long-term
3850 3840 * locked; C) it's a lot of work to make it otherwise, particularly
3851 3841 * since the nexus layer expects the pages to never fault. An RFE is to
3852 3842 * not keep the pages long-term locked, but instead to be able to
3853 3843 * take faults on them and simply look them up in kvp in case we
3854 3844 * fault on them. Even then, we must take care not to let pageout
3855 3845 * steal them from us since the data must remain resident; if we
3856 3846 * do this we must come up with some way to pin the pages to prevent
3857 3847 * faults while a driver is doing DMA to/from them.
3858 3848 */
3859 3849 static void *
3860 3850 devmap_umem_alloc_np(size_t size, size_t flags)
3861 3851 {
3862 3852 void *buf;
3863 3853 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3864 3854
3865 3855 buf = vmem_alloc(umem_np_arena, size, vmflags);
3866 3856 if (buf != NULL)
3867 3857 bzero(buf, size);
3868 3858 return (buf);
3869 3859 }
3870 3860
3871 3861 static void
3872 3862 devmap_umem_free_np(void *addr, size_t size)
3873 3863 {
3874 3864 vmem_free(umem_np_arena, addr, size);
3875 3865 }
3876 3866
3877 3867 /*
3878 3868 * allocate page aligned kernel memory for exporting to user land.
3879 3869 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3880 3870 * to find a user virtual address that is in same color as the address
3881 3871 * allocated here.
3882 3872 */
3883 3873 void *
3884 3874 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3885 3875 {
3886 3876 register size_t len = ptob(btopr(size));
3887 3877 void *buf = NULL;
3888 3878 struct ddi_umem_cookie *cp;
3889 3879 int iflags = 0;
3890 3880
3891 3881 *cookie = NULL;
3892 3882
3893 3883 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3894 3884 "devmap_umem_alloc:start");
3895 3885 if (len == 0)
3896 3886 return ((void *)NULL);
3897 3887
3898 3888 /*
3899 3889 * allocate cookie
3900 3890 */
3901 3891 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3902 3892 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3903 3893 ASSERT(flags & DDI_UMEM_NOSLEEP);
3904 3894 return ((void *)NULL);
3905 3895 }
3906 3896
3907 3897 if (flags & DDI_UMEM_PAGEABLE) {
3908 3898 /* Only one of the flags is allowed */
3909 3899 ASSERT(!(flags & DDI_UMEM_TRASH));
3910 3900 /* initialize resource with 0 */
3911 3901 iflags = KPD_ZERO;
3912 3902
3913 3903 /*
3914 3904 * to allocate unlocked pageable memory, use segkp_get() to
3915 3905 * create a segkp segment. Since segkp can only service kas,
3916 3906 * other segment drivers such as segdev have to do
3917 3907 * as_fault(segkp, SOFTLOCK) in its fault routine,
3918 3908 */
3919 3909 if (flags & DDI_UMEM_NOSLEEP)
3920 3910 iflags |= KPD_NOWAIT;
3921 3911
3922 3912 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3923 3913 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3924 3914 return ((void *)NULL);
3925 3915 }
3926 3916 cp->type = KMEM_PAGEABLE;
3927 3917 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3928 3918 cp->locked = 0;
3929 3919 } else if (flags & DDI_UMEM_TRASH) {
3930 3920 /* Only one of the flags is allowed */
3931 3921 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3932 3922 cp->type = UMEM_TRASH;
3933 3923 buf = NULL;
3934 3924 } else {
3935 3925 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3936 3926 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3937 3927 return ((void *)NULL);
3938 3928 }
3939 3929
3940 3930 cp->type = KMEM_NON_PAGEABLE;
3941 3931 }
3942 3932
3943 3933 /*
3944 3934 * need to save size here. size will be used when
3945 3935 * we do kmem_free.
3946 3936 */
3947 3937 cp->size = len;
3948 3938 cp->cvaddr = (caddr_t)buf;
3949 3939
3950 3940 *cookie = (void *)cp;
3951 3941 return (buf);
3952 3942 }
3953 3943
3954 3944 void
3955 3945 ddi_umem_free(ddi_umem_cookie_t cookie)
3956 3946 {
3957 3947 struct ddi_umem_cookie *cp;
3958 3948
3959 3949 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3960 3950 "devmap_umem_free:start");
3961 3951
3962 3952 /*
3963 3953 * if cookie is NULL, no effects on the system
3964 3954 */
3965 3955 if (cookie == NULL)
3966 3956 return;
3967 3957
3968 3958 cp = (struct ddi_umem_cookie *)cookie;
3969 3959
3970 3960 switch (cp->type) {
3971 3961 case KMEM_PAGEABLE :
3972 3962 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3973 3963 /*
3974 3964 * Check if there are still any pending faults on the cookie
3975 3965 * while the driver is deleting it,
3976 3966 * XXX - could change to an ASSERT but wont catch errant drivers
3977 3967 */
3978 3968 mutex_enter(&cp->lock);
3979 3969 if (cp->locked) {
3980 3970 mutex_exit(&cp->lock);
3981 3971 panic("ddi_umem_free for cookie with pending faults %p",
3982 3972 (void *)cp);
3983 3973 return;
3984 3974 }
3985 3975
3986 3976 segkp_release(segkp, cp->cvaddr);
3987 3977
3988 3978 /*
3989 3979 * release mutex associated with this cookie.
3990 3980 */
3991 3981 mutex_destroy(&cp->lock);
3992 3982 break;
3993 3983 case KMEM_NON_PAGEABLE :
3994 3984 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3995 3985 devmap_umem_free_np(cp->cvaddr, cp->size);
3996 3986 break;
3997 3987 case UMEM_TRASH :
3998 3988 break;
3999 3989 case UMEM_LOCKED :
4000 3990 /* Callers should use ddi_umem_unlock for this type */
4001 3991 ddi_umem_unlock(cookie);
4002 3992 /* Frees the cookie too */
4003 3993 return;
4004 3994 default:
4005 3995 /* panic so we can diagnose the underlying cause */
4006 3996 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4007 3997 cp->type);
4008 3998 }
4009 3999
4010 4000 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4011 4001 }
4012 4002
4013 4003
4014 4004 static int
4015 4005 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4016 4006 {
4017 4007 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4018 4008
4019 4009 /*
4020 4010 * It looks as if it is always mapped shared
4021 4011 */
4022 4012 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4023 4013 "segdev_getmemid:start");
4024 4014 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4025 4015 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4026 4016 return (0);
4027 4017 }
4028 4018
4029 4019 /*
4030 4020 * ddi_umem_alloc() non-pageable quantum cache max size.
4031 4021 * This is just a SWAG.
4032 4022 */
4033 4023 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4034 4024
4035 4025 /*
4036 4026 * Initialize seg_dev from boot. This routine sets up the trash page
4037 4027 * and creates the umem_np_arena used to back non-pageable memory
4038 4028 * requests.
4039 4029 */
4040 4030 void
4041 4031 segdev_init(void)
4042 4032 {
4043 4033 struct seg kseg;
4044 4034
4045 4035 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4046 4036 devmap_alloc_pages, devmap_free_pages, heap_arena,
4047 4037 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4048 4038
4049 4039 kseg.s_as = &kas;
4050 4040 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4051 4041 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4052 4042 if (trashpp == NULL)
4053 4043 panic("segdev_init: failed to create trash page");
4054 4044 pagezero(trashpp, 0, PAGESIZE);
4055 4045 page_downgrade(trashpp);
4056 4046 }
4057 4047
4058 4048 /*
4059 4049 * Invoke platform-dependent support routines so that /proc can have
4060 4050 * the platform code deal with curious hardware.
4061 4051 */
4062 4052 int
4063 4053 segdev_copyfrom(struct seg *seg,
4064 4054 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4065 4055 {
4066 4056 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4067 4057 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4068 4058
4069 4059 return (e_ddi_copyfromdev(sp->s_dip,
4070 4060 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4071 4061 }
4072 4062
4073 4063 int
4074 4064 segdev_copyto(struct seg *seg,
4075 4065 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4076 4066 {
4077 4067 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4078 4068 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4079 4069
4080 4070 return (e_ddi_copytodev(sp->s_dip,
4081 4071 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4082 4072 }
↓ open down ↓ |
1591 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX