Print this page
patch as-lock-macro-simplification
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/vm/seg_dev.c
+++ new/usr/src/uts/common/vm/seg_dev.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 28 /* All Rights Reserved */
29 29
30 30 /*
31 31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 32 * The Regents of the University of California
33 33 * All Rights Reserved
34 34 *
35 35 * University Acknowledgment- Portions of this document are derived from
36 36 * software developed by the University of California, Berkeley, and its
37 37 * contributors.
38 38 */
39 39
40 40 /*
41 41 * VM - segment of a mapped device.
42 42 *
43 43 * This segment driver is used when mapping character special devices.
44 44 */
45 45
46 46 #include <sys/types.h>
47 47 #include <sys/t_lock.h>
48 48 #include <sys/sysmacros.h>
49 49 #include <sys/vtrace.h>
50 50 #include <sys/systm.h>
51 51 #include <sys/vmsystm.h>
52 52 #include <sys/mman.h>
53 53 #include <sys/errno.h>
54 54 #include <sys/kmem.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/vnode.h>
57 57 #include <sys/proc.h>
58 58 #include <sys/conf.h>
59 59 #include <sys/debug.h>
60 60 #include <sys/ddidevmap.h>
61 61 #include <sys/ddi_implfuncs.h>
62 62 #include <sys/lgrp.h>
63 63
64 64 #include <vm/page.h>
65 65 #include <vm/hat.h>
66 66 #include <vm/as.h>
67 67 #include <vm/seg.h>
68 68 #include <vm/seg_dev.h>
69 69 #include <vm/seg_kp.h>
70 70 #include <vm/seg_kmem.h>
71 71 #include <vm/vpage.h>
72 72
73 73 #include <sys/sunddi.h>
74 74 #include <sys/esunddi.h>
75 75 #include <sys/fs/snode.h>
76 76
77 77
78 78 #if DEBUG
79 79 int segdev_debug;
80 80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 81 #else
82 82 #define DEBUGF(level, args)
83 83 #endif
84 84
85 85 /* Default timeout for devmap context management */
86 86 #define CTX_TIMEOUT_VALUE 0
87 87
88 88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 89 { mutex_enter(&dhp->dh_lock); }
90 90
91 91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 92 { mutex_exit(&dhp->dh_lock); }
93 93
94 94 #define round_down_p2(a, s) ((a) & ~((s) - 1))
95 95 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
96 96
97 97 /*
98 98 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99 99 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100 100 */
101 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
102 102 (((uvaddr | paddr) & (pgsize - 1)) == 0)
103 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
104 104 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105 105
106 106 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
107 107
108 108 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
109 109
110 110 static struct devmap_ctx *devmapctx_list = NULL;
111 111 static struct devmap_softlock *devmap_slist = NULL;
112 112
113 113 /*
114 114 * mutex, vnode and page for the page of zeros we use for the trash mappings.
115 115 * One trash page is allocated on the first ddi_umem_setup call that uses it
116 116 * XXX Eventually, we may want to combine this with what segnf does when all
117 117 * hat layers implement HAT_NOFAULT.
118 118 *
119 119 * The trash page is used when the backing store for a userland mapping is
120 120 * removed but the application semantics do not take kindly to a SIGBUS.
121 121 * In that scenario, the applications pages are mapped to some dummy page
122 122 * which returns garbage on read and writes go into a common place.
123 123 * (Perfect for NO_FAULT semantics)
124 124 * The device driver is responsible to communicating to the app with some
125 125 * other mechanism that such remapping has happened and the app should take
126 126 * corrective action.
127 127 * We can also use an anonymous memory page as there is no requirement to
128 128 * keep the page locked, however this complicates the fault code. RFE.
129 129 */
130 130 static struct vnode trashvp;
131 131 static struct page *trashpp;
132 132
133 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 134 static vmem_t *umem_np_arena;
135 135
136 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 137 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
138 138
139 139 /*
140 140 * Macros to check if type of devmap handle
141 141 */
142 142 #define cookie_is_devmem(c) \
143 143 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144 144
145 145 #define cookie_is_pmem(c) \
146 146 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147 147
148 148 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 149 ((c)->type == KMEM_PAGEABLE))
150 150
151 151 #define dhp_is_devmem(dhp) \
152 152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153 153
154 154 #define dhp_is_pmem(dhp) \
155 155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156 156
157 157 #define dhp_is_kpmem(dhp) \
158 158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159 159
160 160 /*
161 161 * Private seg op routines.
162 162 */
163 163 static int segdev_dup(struct seg *, struct seg *);
164 164 static int segdev_unmap(struct seg *, caddr_t, size_t);
165 165 static void segdev_free(struct seg *);
166 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 167 enum fault_type, enum seg_rw);
168 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 169 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 170 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 171 static void segdev_badop(void);
172 172 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 173 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
174 174 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 175 ulong_t *, size_t);
176 176 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 177 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
178 178 static int segdev_gettype(struct seg *, caddr_t);
179 179 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 180 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 181 static void segdev_dump(struct seg *);
182 182 static int segdev_pagelock(struct seg *, caddr_t, size_t,
183 183 struct page ***, enum lock_type, enum seg_rw);
184 184 static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 185 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 186 static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t);
187 187 static int segdev_capable(struct seg *, segcapability_t);
188 188
189 189 /*
190 190 * XXX this struct is used by rootnex_map_fault to identify
191 191 * the segment it has been passed. So if you make it
192 192 * "static" you'll need to fix rootnex_map_fault.
193 193 */
194 194 struct seg_ops segdev_ops = {
195 195 segdev_dup,
196 196 segdev_unmap,
197 197 segdev_free,
198 198 segdev_fault,
199 199 segdev_faulta,
200 200 segdev_setprot,
201 201 segdev_checkprot,
202 202 (int (*)())segdev_badop, /* kluster */
203 203 (size_t (*)(struct seg *))NULL, /* swapout */
204 204 segdev_sync, /* sync */
205 205 segdev_incore,
206 206 segdev_lockop, /* lockop */
207 207 segdev_getprot,
208 208 segdev_getoffset,
209 209 segdev_gettype,
210 210 segdev_getvp,
211 211 segdev_advise,
212 212 segdev_dump,
213 213 segdev_pagelock,
214 214 segdev_setpagesize,
215 215 segdev_getmemid,
216 216 segdev_getpolicy,
217 217 segdev_capable,
218 218 seg_inherit_notsup
219 219 };
220 220
221 221 /*
222 222 * Private segdev support routines
223 223 */
224 224 static struct segdev_data *sdp_alloc(void);
225 225
226 226 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
227 227 size_t, enum seg_rw);
228 228
229 229 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
230 230 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
231 231
232 232 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
233 233 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
234 234
235 235 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
236 236 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
237 237 static void devmap_softlock_rele(devmap_handle_t *);
238 238 static void devmap_ctx_rele(devmap_handle_t *);
239 239
240 240 static void devmap_ctxto(void *);
241 241
242 242 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
243 243 caddr_t addr);
244 244
245 245 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
246 246 ulong_t *opfn, ulong_t *pagesize);
247 247
248 248 static void free_devmap_handle(devmap_handle_t *dhp);
249 249
250 250 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
251 251 struct seg *newseg);
252 252
253 253 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
254 254
255 255 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
256 256
257 257 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
258 258
259 259 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
260 260 offset_t off, size_t len, uint_t flags);
261 261
262 262 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
263 263 caddr_t addr, size_t *llen, caddr_t *laddr);
264 264
265 265 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
266 266
267 267 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
268 268 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
269 269
270 270 static void *devmap_umem_alloc_np(size_t size, size_t flags);
271 271 static void devmap_umem_free_np(void *addr, size_t size);
272 272
273 273 /*
274 274 * routines to lock and unlock underlying segkp segment for
275 275 * KMEM_PAGEABLE type cookies.
276 276 */
277 277 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
278 278 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
279 279
280 280 /*
281 281 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
282 282 * drivers with devmap_access callbacks
283 283 */
284 284 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
285 285 enum fault_type);
286 286 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
287 287 enum fault_type);
288 288
289 289 static kmutex_t devmapctx_lock;
290 290
291 291 static kmutex_t devmap_slock;
292 292
293 293 /*
294 294 * Initialize the thread callbacks and thread private data.
295 295 */
296 296 static struct devmap_ctx *
297 297 devmap_ctxinit(dev_t dev, ulong_t id)
298 298 {
299 299 struct devmap_ctx *devctx;
300 300 struct devmap_ctx *tmp;
301 301 dev_info_t *dip;
302 302
303 303 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
304 304
305 305 mutex_enter(&devmapctx_lock);
306 306
307 307 dip = e_ddi_hold_devi_by_dev(dev, 0);
308 308 ASSERT(dip != NULL);
309 309 ddi_release_devi(dip);
310 310
311 311 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
312 312 if ((devctx->dip == dip) && (devctx->id == id))
313 313 break;
314 314
315 315 if (devctx == NULL) {
316 316 devctx = tmp;
317 317 devctx->dip = dip;
318 318 devctx->id = id;
319 319 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
320 320 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
321 321 devctx->next = devmapctx_list;
322 322 devmapctx_list = devctx;
323 323 } else
324 324 kmem_free(tmp, sizeof (struct devmap_ctx));
325 325
326 326 mutex_enter(&devctx->lock);
327 327 devctx->refcnt++;
328 328 mutex_exit(&devctx->lock);
329 329 mutex_exit(&devmapctx_lock);
330 330
331 331 return (devctx);
332 332 }
333 333
334 334 /*
335 335 * Timeout callback called if a CPU has not given up the device context
336 336 * within dhp->dh_timeout_length ticks
337 337 */
338 338 static void
339 339 devmap_ctxto(void *data)
340 340 {
341 341 struct devmap_ctx *devctx = data;
342 342
343 343 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
344 344 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
345 345 mutex_enter(&devctx->lock);
346 346 /*
347 347 * Set oncpu = 0 so the next mapping trying to get the device context
348 348 * can.
349 349 */
350 350 devctx->oncpu = 0;
351 351 devctx->timeout = 0;
352 352 cv_signal(&devctx->cv);
353 353 mutex_exit(&devctx->lock);
354 354 }
355 355
356 356 /*
357 357 * Create a device segment.
358 358 */
359 359 int
360 360 segdev_create(struct seg *seg, void *argsp)
↓ open down ↓ |
360 lines elided |
↑ open up ↑ |
361 361 {
362 362 struct segdev_data *sdp;
363 363 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
364 364 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
365 365 int error;
366 366
367 367 /*
368 368 * Since the address space is "write" locked, we
369 369 * don't need the segment lock to protect "segdev" data.
370 370 */
371 - ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
371 + ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
372 372
373 373 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
374 374
375 375 sdp = sdp_alloc();
376 376
377 377 sdp->mapfunc = a->mapfunc;
378 378 sdp->offset = a->offset;
379 379 sdp->prot = a->prot;
380 380 sdp->maxprot = a->maxprot;
381 381 sdp->type = a->type;
382 382 sdp->pageprot = 0;
383 383 sdp->softlockcnt = 0;
384 384 sdp->vpage = NULL;
385 385
386 386 if (sdp->mapfunc == NULL)
387 387 sdp->devmap_data = dhp;
388 388 else
389 389 sdp->devmap_data = dhp = NULL;
390 390
391 391 sdp->hat_flags = a->hat_flags;
392 392 sdp->hat_attr = a->hat_attr;
393 393
394 394 /*
395 395 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
396 396 */
397 397 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
398 398
399 399 /*
400 400 * Hold shadow vnode -- segdev only deals with
401 401 * character (VCHR) devices. We use the common
402 402 * vp to hang pages on.
403 403 */
404 404 sdp->vp = specfind(a->dev, VCHR);
405 405 ASSERT(sdp->vp != NULL);
406 406
407 407 seg->s_ops = &segdev_ops;
408 408 seg->s_data = sdp;
409 409
410 410 while (dhp != NULL) {
411 411 dhp->dh_seg = seg;
412 412 dhp = dhp->dh_next;
413 413 }
414 414
415 415 /*
416 416 * Inform the vnode of the new mapping.
417 417 */
418 418 /*
419 419 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
420 420 * dhp specific maxprot because spec_addmap does not use maxprot.
421 421 */
422 422 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
423 423 seg->s_as, seg->s_base, seg->s_size,
424 424 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
425 425
426 426 if (error != 0) {
427 427 sdp->devmap_data = NULL;
428 428 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
429 429 HAT_UNLOAD_UNMAP);
430 430 } else {
431 431 /*
432 432 * Mappings of /dev/null don't count towards the VSZ of a
433 433 * process. Mappings of /dev/null have no mapping type.
434 434 */
435 435 if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
436 436 MAP_PRIVATE)) == 0) {
437 437 seg->s_as->a_resvsize -= seg->s_size;
438 438 }
439 439 }
440 440
441 441 return (error);
442 442 }
443 443
444 444 static struct segdev_data *
445 445 sdp_alloc(void)
446 446 {
447 447 struct segdev_data *sdp;
448 448
449 449 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
450 450 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
451 451
452 452 return (sdp);
453 453 }
454 454
455 455 /*
456 456 * Duplicate seg and return new segment in newseg.
457 457 */
458 458 static int
459 459 segdev_dup(struct seg *seg, struct seg *newseg)
460 460 {
461 461 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
462 462 struct segdev_data *newsdp;
463 463 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
464 464 size_t npages;
465 465 int ret;
466 466
↓ open down ↓ |
85 lines elided |
↑ open up ↑ |
467 467 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
468 468 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
469 469
470 470 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
471 471 (void *)dhp, (void *)seg));
472 472
473 473 /*
474 474 * Since the address space is "write" locked, we
475 475 * don't need the segment lock to protect "segdev" data.
476 476 */
477 - ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
477 + ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
478 478
479 479 newsdp = sdp_alloc();
480 480
481 481 newseg->s_ops = seg->s_ops;
482 482 newseg->s_data = (void *)newsdp;
483 483
484 484 VN_HOLD(sdp->vp);
485 485 newsdp->vp = sdp->vp;
486 486 newsdp->mapfunc = sdp->mapfunc;
487 487 newsdp->offset = sdp->offset;
488 488 newsdp->pageprot = sdp->pageprot;
489 489 newsdp->prot = sdp->prot;
490 490 newsdp->maxprot = sdp->maxprot;
491 491 newsdp->type = sdp->type;
492 492 newsdp->hat_attr = sdp->hat_attr;
493 493 newsdp->hat_flags = sdp->hat_flags;
494 494 newsdp->softlockcnt = 0;
495 495
496 496 /*
497 497 * Initialize per page data if the segment we are
498 498 * dup'ing has per page information.
499 499 */
500 500 npages = seg_pages(newseg);
501 501
502 502 if (sdp->vpage != NULL) {
503 503 size_t nbytes = vpgtob(npages);
504 504
505 505 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
506 506 bcopy(sdp->vpage, newsdp->vpage, nbytes);
507 507 } else
508 508 newsdp->vpage = NULL;
509 509
510 510 /*
511 511 * duplicate devmap handles
512 512 */
513 513 if (dhp != NULL) {
514 514 ret = devmap_handle_dup(dhp,
515 515 (devmap_handle_t **)&newsdp->devmap_data, newseg);
516 516 if (ret != 0) {
517 517 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
518 518 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
519 519 ret, (void *)dhp, (void *)seg);
520 520 DEBUGF(1, (CE_CONT,
521 521 "segdev_dup: ret %x dhp %p seg %p\n",
522 522 ret, (void *)dhp, (void *)seg));
523 523 return (ret);
524 524 }
525 525 }
526 526
527 527 /*
528 528 * Inform the common vnode of the new mapping.
529 529 */
530 530 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
531 531 newsdp->offset, newseg->s_as,
532 532 newseg->s_base, newseg->s_size, newsdp->prot,
533 533 newsdp->maxprot, sdp->type, CRED(), NULL));
534 534 }
535 535
536 536 /*
537 537 * duplicate devmap handles
538 538 */
539 539 static int
540 540 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
541 541 struct seg *newseg)
542 542 {
543 543 devmap_handle_t *newdhp_save = NULL;
544 544 devmap_handle_t *newdhp = NULL;
545 545 struct devmap_callback_ctl *callbackops;
546 546
547 547 while (dhp != NULL) {
548 548 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
549 549
550 550 /* Need to lock the original dhp while copying if REMAP */
551 551 HOLD_DHP_LOCK(dhp);
552 552 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
553 553 RELE_DHP_LOCK(dhp);
554 554 newdhp->dh_seg = newseg;
555 555 newdhp->dh_next = NULL;
556 556 if (newdhp_save != NULL)
557 557 newdhp_save->dh_next = newdhp;
558 558 else
559 559 *new_dhp = newdhp;
560 560 newdhp_save = newdhp;
561 561
562 562 callbackops = &newdhp->dh_callbackops;
563 563
564 564 if (dhp->dh_softlock != NULL)
565 565 newdhp->dh_softlock = devmap_softlock_init(
566 566 newdhp->dh_dev,
567 567 (ulong_t)callbackops->devmap_access);
568 568 if (dhp->dh_ctx != NULL)
569 569 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
570 570 (ulong_t)callbackops->devmap_access);
571 571
572 572 /*
573 573 * Initialize dh_lock if we want to do remap.
574 574 */
575 575 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
576 576 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
577 577 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
578 578 }
579 579
580 580 if (callbackops->devmap_dup != NULL) {
581 581 int ret;
582 582
583 583 /*
584 584 * Call the dup callback so that the driver can
585 585 * duplicate its private data.
586 586 */
587 587 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
588 588 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
589 589
590 590 if (ret != 0) {
591 591 /*
592 592 * We want to free up this segment as the driver
593 593 * has indicated that we can't dup it. But we
594 594 * don't want to call the drivers, devmap_unmap,
595 595 * callback function as the driver does not
596 596 * think this segment exists. The caller of
597 597 * devmap_dup will call seg_free on newseg
598 598 * as it was the caller that allocated the
599 599 * segment.
600 600 */
601 601 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
602 602 "newdhp %p dhp %p\n", (void *)newdhp,
603 603 (void *)dhp));
604 604 callbackops->devmap_unmap = NULL;
605 605 return (ret);
606 606 }
607 607 }
608 608
609 609 dhp = dhp->dh_next;
610 610 }
611 611
612 612 return (0);
613 613 }
614 614
615 615 /*
616 616 * Split a segment at addr for length len.
617 617 */
618 618 /*ARGSUSED*/
619 619 static int
620 620 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
621 621 {
622 622 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
623 623 register struct segdev_data *nsdp;
624 624 register struct seg *nseg;
625 625 register size_t opages; /* old segment size in pages */
626 626 register size_t npages; /* new segment size in pages */
627 627 register size_t dpages; /* pages being deleted (unmapped) */
628 628 register size_t nbytes;
629 629 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
630 630 devmap_handle_t *dhpp;
631 631 devmap_handle_t *newdhp;
632 632 struct devmap_callback_ctl *callbackops;
633 633 caddr_t nbase;
634 634 offset_t off;
635 635 ulong_t nsize;
636 636 size_t mlen, sz;
637 637
638 638 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
↓ open down ↓ |
151 lines elided |
↑ open up ↑ |
639 639 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
640 640 (void *)dhp, (void *)seg, (void *)addr, len);
641 641
642 642 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
643 643 (void *)dhp, (void *)seg, (void *)addr, len));
644 644
645 645 /*
646 646 * Since the address space is "write" locked, we
647 647 * don't need the segment lock to protect "segdev" data.
648 648 */
649 - ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
649 + ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
650 650
651 651 if ((sz = sdp->softlockcnt) > 0) {
652 652 /*
653 653 * Fail the unmap if pages are SOFTLOCKed through this mapping.
654 654 * softlockcnt is protected from change by the as write lock.
655 655 */
656 656 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
657 657 "segdev_unmap:error softlockcnt = %ld", sz);
658 658 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
659 659 return (EAGAIN);
660 660 }
661 661
662 662 /*
663 663 * Check for bad sizes
664 664 */
665 665 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
666 666 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
667 667 panic("segdev_unmap");
668 668
669 669 if (dhp != NULL) {
670 670 devmap_handle_t *tdhp;
671 671 /*
672 672 * If large page size was used in hat_devload(),
673 673 * the same page size must be used in hat_unload().
674 674 */
675 675 dhpp = tdhp = devmap_find_handle(dhp, addr);
676 676 while (tdhp != NULL) {
677 677 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
678 678 break;
679 679 }
680 680 tdhp = tdhp->dh_next;
681 681 }
682 682 if (tdhp != NULL) { /* found a dhp using large pages */
683 683 size_t slen = len;
684 684 size_t mlen;
685 685 size_t soff;
686 686
687 687 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
688 688 while (slen != 0) {
689 689 mlen = MIN(slen, (dhpp->dh_len - soff));
690 690 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
691 691 dhpp->dh_len, HAT_UNLOAD_UNMAP);
692 692 dhpp = dhpp->dh_next;
693 693 ASSERT(slen >= mlen);
694 694 slen -= mlen;
695 695 soff = 0;
696 696 }
697 697 } else
698 698 hat_unload(seg->s_as->a_hat, addr, len,
699 699 HAT_UNLOAD_UNMAP);
700 700 } else {
701 701 /*
702 702 * Unload any hardware translations in the range
703 703 * to be taken out.
704 704 */
705 705 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
706 706 }
707 707
708 708 /*
709 709 * get the user offset which will used in the driver callbacks
710 710 */
711 711 off = sdp->offset + (offset_t)(addr - seg->s_base);
712 712
713 713 /*
714 714 * Inform the vnode of the unmapping.
715 715 */
716 716 ASSERT(sdp->vp != NULL);
717 717 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
718 718 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
719 719
720 720 /*
721 721 * Check for entire segment
722 722 */
723 723 if (addr == seg->s_base && len == seg->s_size) {
724 724 seg_free(seg);
725 725 return (0);
726 726 }
727 727
728 728 opages = seg_pages(seg);
729 729 dpages = btop(len);
730 730 npages = opages - dpages;
731 731
732 732 /*
733 733 * Check for beginning of segment
734 734 */
735 735 if (addr == seg->s_base) {
736 736 if (sdp->vpage != NULL) {
737 737 register struct vpage *ovpage;
738 738
739 739 ovpage = sdp->vpage; /* keep pointer to vpage */
740 740
741 741 nbytes = vpgtob(npages);
742 742 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
743 743 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
744 744
745 745 /* free up old vpage */
746 746 kmem_free(ovpage, vpgtob(opages));
747 747 }
748 748
749 749 /*
750 750 * free devmap handles from the beginning of the mapping.
751 751 */
752 752 if (dhp != NULL)
753 753 devmap_handle_unmap_head(dhp, len);
754 754
755 755 sdp->offset += (offset_t)len;
756 756
757 757 seg->s_base += len;
758 758 seg->s_size -= len;
759 759
760 760 return (0);
761 761 }
762 762
763 763 /*
764 764 * Check for end of segment
765 765 */
766 766 if (addr + len == seg->s_base + seg->s_size) {
767 767 if (sdp->vpage != NULL) {
768 768 register struct vpage *ovpage;
769 769
770 770 ovpage = sdp->vpage; /* keep pointer to vpage */
771 771
772 772 nbytes = vpgtob(npages);
773 773 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
774 774 bcopy(ovpage, sdp->vpage, nbytes);
775 775
776 776 /* free up old vpage */
777 777 kmem_free(ovpage, vpgtob(opages));
778 778 }
779 779 seg->s_size -= len;
780 780
781 781 /*
782 782 * free devmap handles from addr to the end of the mapping.
783 783 */
784 784 if (dhp != NULL)
785 785 devmap_handle_unmap_tail(dhp, addr);
786 786
787 787 return (0);
788 788 }
789 789
790 790 /*
791 791 * The section to go is in the middle of the segment,
792 792 * have to make it into two segments. nseg is made for
793 793 * the high end while seg is cut down at the low end.
794 794 */
795 795 nbase = addr + len; /* new seg base */
796 796 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
797 797 seg->s_size = addr - seg->s_base; /* shrink old seg */
798 798 nseg = seg_alloc(seg->s_as, nbase, nsize);
799 799 if (nseg == NULL)
800 800 panic("segdev_unmap seg_alloc");
801 801
802 802 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
803 803 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
804 804 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
805 805 (void *)seg, (void *)nseg));
806 806 nsdp = sdp_alloc();
807 807
808 808 nseg->s_ops = seg->s_ops;
809 809 nseg->s_data = (void *)nsdp;
810 810
811 811 VN_HOLD(sdp->vp);
812 812 nsdp->mapfunc = sdp->mapfunc;
813 813 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
814 814 nsdp->vp = sdp->vp;
815 815 nsdp->pageprot = sdp->pageprot;
816 816 nsdp->prot = sdp->prot;
817 817 nsdp->maxprot = sdp->maxprot;
818 818 nsdp->type = sdp->type;
819 819 nsdp->hat_attr = sdp->hat_attr;
820 820 nsdp->hat_flags = sdp->hat_flags;
821 821 nsdp->softlockcnt = 0;
822 822
823 823 /*
824 824 * Initialize per page data if the segment we are
825 825 * dup'ing has per page information.
826 826 */
827 827 if (sdp->vpage != NULL) {
828 828 /* need to split vpage into two arrays */
829 829 register size_t nnbytes;
830 830 register size_t nnpages;
831 831 register struct vpage *ovpage;
832 832
833 833 ovpage = sdp->vpage; /* keep pointer to vpage */
834 834
835 835 npages = seg_pages(seg); /* seg has shrunk */
836 836 nbytes = vpgtob(npages);
837 837 nnpages = seg_pages(nseg);
838 838 nnbytes = vpgtob(nnpages);
839 839
840 840 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
841 841 bcopy(ovpage, sdp->vpage, nbytes);
842 842
843 843 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
844 844 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
845 845
846 846 /* free up old vpage */
847 847 kmem_free(ovpage, vpgtob(opages));
848 848 } else
849 849 nsdp->vpage = NULL;
850 850
851 851 /*
852 852 * unmap dhps.
853 853 */
854 854 if (dhp == NULL) {
855 855 nsdp->devmap_data = NULL;
856 856 return (0);
857 857 }
858 858 while (dhp != NULL) {
859 859 callbackops = &dhp->dh_callbackops;
860 860 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
861 861 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
862 862 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
863 863 (void *)dhp, (void *)addr,
864 864 (void *)dhp->dh_uvaddr, dhp->dh_len));
865 865
866 866 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
867 867 dhpp = dhp->dh_next;
868 868 dhp->dh_next = NULL;
869 869 dhp = dhpp;
870 870 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
871 871 dhp = dhp->dh_next;
872 872 } else if (addr > dhp->dh_uvaddr &&
873 873 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
874 874 /*
875 875 * <addr, addr+len> is enclosed by dhp.
876 876 * create a newdhp that begins at addr+len and
877 877 * ends at dhp->dh_uvaddr+dhp->dh_len.
878 878 */
879 879 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
880 880 HOLD_DHP_LOCK(dhp);
881 881 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
882 882 RELE_DHP_LOCK(dhp);
883 883 newdhp->dh_seg = nseg;
884 884 newdhp->dh_next = dhp->dh_next;
885 885 if (dhp->dh_softlock != NULL)
886 886 newdhp->dh_softlock = devmap_softlock_init(
887 887 newdhp->dh_dev,
888 888 (ulong_t)callbackops->devmap_access);
889 889 if (dhp->dh_ctx != NULL)
890 890 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
891 891 (ulong_t)callbackops->devmap_access);
892 892 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
893 893 mutex_init(&newdhp->dh_lock,
894 894 NULL, MUTEX_DEFAULT, NULL);
895 895 }
896 896 if (callbackops->devmap_unmap != NULL)
897 897 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
898 898 off, len, dhp, &dhp->dh_pvtp,
899 899 newdhp, &newdhp->dh_pvtp);
900 900 mlen = len + (addr - dhp->dh_uvaddr);
901 901 devmap_handle_reduce_len(newdhp, mlen);
902 902 nsdp->devmap_data = newdhp;
903 903 /* XX Changing len should recalculate LARGE flag */
904 904 dhp->dh_len = addr - dhp->dh_uvaddr;
905 905 dhpp = dhp->dh_next;
906 906 dhp->dh_next = NULL;
907 907 dhp = dhpp;
908 908 } else if ((addr > dhp->dh_uvaddr) &&
909 909 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
910 910 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
911 911 /*
912 912 * <addr, addr+len> spans over dhps.
913 913 */
914 914 if (callbackops->devmap_unmap != NULL)
915 915 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
916 916 off, mlen, (devmap_cookie_t *)dhp,
917 917 &dhp->dh_pvtp, NULL, NULL);
918 918 /* XX Changing len should recalculate LARGE flag */
919 919 dhp->dh_len = addr - dhp->dh_uvaddr;
920 920 dhpp = dhp->dh_next;
921 921 dhp->dh_next = NULL;
922 922 dhp = dhpp;
923 923 nsdp->devmap_data = dhp;
924 924 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
925 925 /*
926 926 * dhp is enclosed by <addr, addr+len>.
927 927 */
928 928 dhp->dh_seg = nseg;
929 929 nsdp->devmap_data = dhp;
930 930 dhp = devmap_handle_unmap(dhp);
931 931 nsdp->devmap_data = dhp; /* XX redundant? */
932 932 } else if (((addr + len) > dhp->dh_uvaddr) &&
933 933 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
934 934 mlen = addr + len - dhp->dh_uvaddr;
935 935 if (callbackops->devmap_unmap != NULL)
936 936 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
937 937 dhp->dh_uoff, mlen, NULL,
938 938 NULL, dhp, &dhp->dh_pvtp);
939 939 devmap_handle_reduce_len(dhp, mlen);
940 940 nsdp->devmap_data = dhp;
941 941 dhp->dh_seg = nseg;
942 942 dhp = dhp->dh_next;
943 943 } else {
944 944 dhp->dh_seg = nseg;
945 945 dhp = dhp->dh_next;
946 946 }
947 947 }
948 948 return (0);
949 949 }
950 950
951 951 /*
952 952 * Utility function handles reducing the length of a devmap handle during unmap
953 953 * Note that is only used for unmapping the front portion of the handler,
954 954 * i.e., we are bumping up the offset/pfn etc up by len
955 955 * Do not use if reducing length at the tail.
956 956 */
957 957 static void
958 958 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
959 959 {
960 960 struct ddi_umem_cookie *cp;
961 961 struct devmap_pmem_cookie *pcp;
962 962 /*
963 963 * adjust devmap handle fields
964 964 */
965 965 ASSERT(len < dhp->dh_len);
966 966
967 967 /* Make sure only page-aligned changes are done */
968 968 ASSERT((len & PAGEOFFSET) == 0);
969 969
970 970 dhp->dh_len -= len;
971 971 dhp->dh_uoff += (offset_t)len;
972 972 dhp->dh_roff += (offset_t)len;
973 973 dhp->dh_uvaddr += len;
974 974 /* Need to grab dhp lock if REMAP */
975 975 HOLD_DHP_LOCK(dhp);
976 976 cp = dhp->dh_cookie;
977 977 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
978 978 if (cookie_is_devmem(cp)) {
979 979 dhp->dh_pfn += btop(len);
980 980 } else if (cookie_is_pmem(cp)) {
981 981 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
982 982 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
983 983 dhp->dh_roff < ptob(pcp->dp_npages));
984 984 } else {
985 985 ASSERT(dhp->dh_roff < cp->size);
986 986 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
987 987 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
988 988 ASSERT((dhp->dh_cvaddr + len) <=
989 989 (cp->cvaddr + cp->size));
990 990
991 991 dhp->dh_cvaddr += len;
992 992 }
993 993 }
994 994 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
995 995 RELE_DHP_LOCK(dhp);
996 996 }
997 997
998 998 /*
999 999 * Free devmap handle, dhp.
1000 1000 * Return the next devmap handle on the linked list.
1001 1001 */
1002 1002 static devmap_handle_t *
1003 1003 devmap_handle_unmap(devmap_handle_t *dhp)
1004 1004 {
1005 1005 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1006 1006 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1007 1007 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1008 1008
1009 1009 ASSERT(dhp != NULL);
1010 1010
1011 1011 /*
1012 1012 * before we free up dhp, call the driver's devmap_unmap entry point
1013 1013 * to free resources allocated for this dhp.
1014 1014 */
1015 1015 if (callbackops->devmap_unmap != NULL) {
1016 1016 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1017 1017 dhp->dh_len, NULL, NULL, NULL, NULL);
1018 1018 }
1019 1019
1020 1020 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1021 1021 sdp->devmap_data = dhp->dh_next;
1022 1022 } else {
1023 1023 while (dhpp->dh_next != dhp) {
1024 1024 dhpp = dhpp->dh_next;
1025 1025 }
1026 1026 dhpp->dh_next = dhp->dh_next;
1027 1027 }
1028 1028 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1029 1029
1030 1030 if (dhp->dh_softlock != NULL)
1031 1031 devmap_softlock_rele(dhp);
1032 1032
1033 1033 if (dhp->dh_ctx != NULL)
1034 1034 devmap_ctx_rele(dhp);
1035 1035
1036 1036 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1037 1037 mutex_destroy(&dhp->dh_lock);
1038 1038 }
1039 1039 kmem_free(dhp, sizeof (devmap_handle_t));
1040 1040
1041 1041 return (dhpp);
1042 1042 }
1043 1043
1044 1044 /*
1045 1045 * Free complete devmap handles from dhp for len bytes
1046 1046 * dhp can be either the first handle or a subsequent handle
1047 1047 */
1048 1048 static void
1049 1049 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1050 1050 {
1051 1051 struct devmap_callback_ctl *callbackops;
1052 1052
1053 1053 /*
1054 1054 * free the devmap handles covered by len.
1055 1055 */
1056 1056 while (len >= dhp->dh_len) {
1057 1057 len -= dhp->dh_len;
1058 1058 dhp = devmap_handle_unmap(dhp);
1059 1059 }
1060 1060 if (len != 0) { /* partial unmap at head of first remaining dhp */
1061 1061 callbackops = &dhp->dh_callbackops;
1062 1062
1063 1063 /*
1064 1064 * Call the unmap callback so the drivers can make
1065 1065 * adjustment on its private data.
1066 1066 */
1067 1067 if (callbackops->devmap_unmap != NULL)
1068 1068 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1069 1069 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1070 1070 devmap_handle_reduce_len(dhp, len);
1071 1071 }
1072 1072 }
1073 1073
1074 1074 /*
1075 1075 * Free devmap handles to truncate the mapping after addr
1076 1076 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1077 1077 * Also could then use the routine in middle unmap case too
1078 1078 */
1079 1079 static void
1080 1080 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1081 1081 {
1082 1082 register struct seg *seg = dhp->dh_seg;
1083 1083 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1084 1084 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1085 1085 struct devmap_callback_ctl *callbackops;
1086 1086 register devmap_handle_t *dhpp;
1087 1087 size_t maplen;
1088 1088 ulong_t off;
1089 1089 size_t len;
1090 1090
1091 1091 maplen = (size_t)(addr - dhp->dh_uvaddr);
1092 1092 dhph = devmap_find_handle(dhph, addr);
1093 1093
1094 1094 while (dhph != NULL) {
1095 1095 if (maplen == 0) {
1096 1096 dhph = devmap_handle_unmap(dhph);
1097 1097 } else {
1098 1098 callbackops = &dhph->dh_callbackops;
1099 1099 len = dhph->dh_len - maplen;
1100 1100 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1101 1101 /*
1102 1102 * Call the unmap callback so the driver
1103 1103 * can make adjustments on its private data.
1104 1104 */
1105 1105 if (callbackops->devmap_unmap != NULL)
1106 1106 (*callbackops->devmap_unmap)(dhph,
1107 1107 dhph->dh_pvtp, off, len,
1108 1108 (devmap_cookie_t *)dhph,
1109 1109 &dhph->dh_pvtp, NULL, NULL);
1110 1110 /* XXX Reducing len needs to recalculate LARGE flag */
1111 1111 dhph->dh_len = maplen;
1112 1112 maplen = 0;
1113 1113 dhpp = dhph->dh_next;
1114 1114 dhph->dh_next = NULL;
1115 1115 dhph = dhpp;
1116 1116 }
1117 1117 } /* end while */
1118 1118 }
1119 1119
1120 1120 /*
1121 1121 * Free a segment.
1122 1122 */
1123 1123 static void
1124 1124 segdev_free(struct seg *seg)
1125 1125 {
1126 1126 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1127 1127 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
↓ open down ↓ |
468 lines elided |
↑ open up ↑ |
1128 1128
1129 1129 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1130 1130 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1131 1131 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1132 1132 (void *)dhp, (void *)seg));
1133 1133
1134 1134 /*
1135 1135 * Since the address space is "write" locked, we
1136 1136 * don't need the segment lock to protect "segdev" data.
1137 1137 */
1138 - ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1138 + ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1139 1139
1140 1140 while (dhp != NULL)
1141 1141 dhp = devmap_handle_unmap(dhp);
1142 1142
1143 1143 VN_RELE(sdp->vp);
1144 1144 if (sdp->vpage != NULL)
1145 1145 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1146 1146
1147 1147 rw_destroy(&sdp->lock);
1148 1148 kmem_free(sdp, sizeof (*sdp));
1149 1149 }
1150 1150
1151 1151 static void
1152 1152 free_devmap_handle(devmap_handle_t *dhp)
1153 1153 {
1154 1154 register devmap_handle_t *dhpp;
1155 1155
1156 1156 /*
1157 1157 * free up devmap handle
1158 1158 */
1159 1159 while (dhp != NULL) {
1160 1160 dhpp = dhp->dh_next;
1161 1161 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1162 1162 mutex_destroy(&dhp->dh_lock);
1163 1163 }
1164 1164
1165 1165 if (dhp->dh_softlock != NULL)
1166 1166 devmap_softlock_rele(dhp);
1167 1167
1168 1168 if (dhp->dh_ctx != NULL)
1169 1169 devmap_ctx_rele(dhp);
1170 1170
1171 1171 kmem_free(dhp, sizeof (devmap_handle_t));
1172 1172 dhp = dhpp;
1173 1173 }
1174 1174 }
1175 1175
1176 1176 /*
1177 1177 * routines to lock and unlock underlying segkp segment for
1178 1178 * KMEM_PAGEABLE type cookies.
1179 1179 * segkp only allows a single pending F_SOFTLOCK
1180 1180 * we keep track of number of locks in the cookie so we can
1181 1181 * have multiple pending faults and manage the calls to segkp.
1182 1182 * RFE: if segkp supports either pagelock or can support multiple
1183 1183 * calls to F_SOFTLOCK, then these routines can go away.
1184 1184 * If pagelock, segdev_faultpage can fault on a page by page basis
1185 1185 * and simplifies the code quite a bit.
1186 1186 * if multiple calls allowed but not partial ranges, then need for
1187 1187 * cookie->lock and locked count goes away, code can call as_fault directly
1188 1188 */
1189 1189 static faultcode_t
1190 1190 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1191 1191 {
1192 1192 int err = 0;
1193 1193 ASSERT(cookie_is_kpmem(cookie));
1194 1194 /*
1195 1195 * Fault in pages in segkp with F_SOFTLOCK.
1196 1196 * We want to hold the lock until all pages have been loaded.
1197 1197 * segkp only allows single caller to hold SOFTLOCK, so cookie
1198 1198 * holds a count so we dont call into segkp multiple times
1199 1199 */
1200 1200 mutex_enter(&cookie->lock);
1201 1201
1202 1202 /*
1203 1203 * Check for overflow in locked field
1204 1204 */
1205 1205 if ((UINT32_MAX - cookie->locked) < npages) {
1206 1206 err = FC_MAKE_ERR(ENOMEM);
1207 1207 } else if (cookie->locked == 0) {
1208 1208 /* First time locking */
1209 1209 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1210 1210 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1211 1211 }
1212 1212 if (!err) {
1213 1213 cookie->locked += npages;
1214 1214 }
1215 1215 mutex_exit(&cookie->lock);
1216 1216 return (err);
1217 1217 }
1218 1218
1219 1219 static void
1220 1220 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1221 1221 {
1222 1222 mutex_enter(&cookie->lock);
1223 1223 ASSERT(cookie_is_kpmem(cookie));
1224 1224 ASSERT(cookie->locked >= npages);
1225 1225 cookie->locked -= (uint_t)npages;
1226 1226 if (cookie->locked == 0) {
1227 1227 /* Last unlock */
1228 1228 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1229 1229 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1230 1230 panic("segdev releasing kpmem lock %p", (void *)cookie);
1231 1231 }
1232 1232 mutex_exit(&cookie->lock);
1233 1233 }
1234 1234
1235 1235 /*
1236 1236 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1237 1237 * drivers with devmap_access callbacks
1238 1238 * slock->softlocked basically works like a rw lock
1239 1239 * -ve counts => F_SOFTLOCK in progress
1240 1240 * +ve counts => F_INVAL/F_PROT in progress
1241 1241 * We allow only one F_SOFTLOCK at a time
1242 1242 * but can have multiple pending F_INVAL/F_PROT calls
1243 1243 *
1244 1244 * This routine waits using cv_wait_sig so killing processes is more graceful
1245 1245 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1246 1246 */
1247 1247 static int devmap_softlock_enter(
1248 1248 struct devmap_softlock *slock,
1249 1249 size_t npages,
1250 1250 enum fault_type type)
1251 1251 {
1252 1252 if (npages == 0)
1253 1253 return (0);
1254 1254 mutex_enter(&(slock->lock));
1255 1255 switch (type) {
1256 1256 case F_SOFTLOCK :
1257 1257 while (slock->softlocked) {
1258 1258 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1259 1259 /* signalled */
1260 1260 mutex_exit(&(slock->lock));
1261 1261 return (EINTR);
1262 1262 }
1263 1263 }
1264 1264 slock->softlocked -= npages; /* -ve count => locked */
1265 1265 break;
1266 1266 case F_INVAL :
1267 1267 case F_PROT :
1268 1268 while (slock->softlocked < 0)
1269 1269 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1270 1270 /* signalled */
1271 1271 mutex_exit(&(slock->lock));
1272 1272 return (EINTR);
1273 1273 }
1274 1274 slock->softlocked += npages; /* +ve count => f_invals */
1275 1275 break;
1276 1276 default:
1277 1277 ASSERT(0);
1278 1278 }
1279 1279 mutex_exit(&(slock->lock));
1280 1280 return (0);
1281 1281 }
1282 1282
1283 1283 static void devmap_softlock_exit(
1284 1284 struct devmap_softlock *slock,
1285 1285 size_t npages,
1286 1286 enum fault_type type)
1287 1287 {
1288 1288 if (slock == NULL)
1289 1289 return;
1290 1290 mutex_enter(&(slock->lock));
1291 1291 switch (type) {
1292 1292 case F_SOFTLOCK :
1293 1293 ASSERT(-slock->softlocked >= npages);
1294 1294 slock->softlocked += npages; /* -ve count is softlocked */
1295 1295 if (slock->softlocked == 0)
1296 1296 cv_signal(&slock->cv);
1297 1297 break;
1298 1298 case F_INVAL :
1299 1299 case F_PROT:
1300 1300 ASSERT(slock->softlocked >= npages);
1301 1301 slock->softlocked -= npages;
1302 1302 if (slock->softlocked == 0)
1303 1303 cv_signal(&slock->cv);
1304 1304 break;
1305 1305 default:
1306 1306 ASSERT(0);
1307 1307 }
1308 1308 mutex_exit(&(slock->lock));
1309 1309 }
1310 1310
1311 1311 /*
1312 1312 * Do a F_SOFTUNLOCK call over the range requested.
1313 1313 * The range must have already been F_SOFTLOCK'ed.
1314 1314 * The segment lock should be held, (but not the segment private lock?)
1315 1315 * The softunlock code below does not adjust for large page sizes
1316 1316 * assumes the caller already did any addr/len adjustments for
1317 1317 * pagesize mappings before calling.
1318 1318 */
1319 1319 /*ARGSUSED*/
1320 1320 static void
1321 1321 segdev_softunlock(
1322 1322 struct hat *hat, /* the hat */
1323 1323 struct seg *seg, /* seg_dev of interest */
1324 1324 caddr_t addr, /* base address of range */
1325 1325 size_t len, /* number of bytes */
1326 1326 enum seg_rw rw) /* type of access at fault */
1327 1327 {
1328 1328 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1329 1329 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1330 1330
1331 1331 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1332 1332 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1333 1333 dhp_head, sdp, addr, len);
1334 1334 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1335 1335 "addr %p len %lx\n",
1336 1336 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1337 1337
1338 1338 hat_unlock(hat, addr, len);
1339 1339
1340 1340 if (dhp_head != NULL) {
1341 1341 devmap_handle_t *dhp;
1342 1342 size_t mlen;
1343 1343 size_t tlen = len;
1344 1344 ulong_t off;
1345 1345
1346 1346 dhp = devmap_find_handle(dhp_head, addr);
1347 1347 ASSERT(dhp != NULL);
1348 1348
1349 1349 off = (ulong_t)(addr - dhp->dh_uvaddr);
1350 1350 while (tlen != 0) {
1351 1351 mlen = MIN(tlen, (dhp->dh_len - off));
1352 1352
1353 1353 /*
1354 1354 * unlock segkp memory, locked during F_SOFTLOCK
1355 1355 */
1356 1356 if (dhp_is_kpmem(dhp)) {
1357 1357 release_kpmem_lock(
1358 1358 (struct ddi_umem_cookie *)dhp->dh_cookie,
1359 1359 btopr(mlen));
1360 1360 }
1361 1361
1362 1362 /*
1363 1363 * Do the softlock accounting for devmap_access
1364 1364 */
1365 1365 if (dhp->dh_callbackops.devmap_access != NULL) {
1366 1366 devmap_softlock_exit(dhp->dh_softlock,
1367 1367 btopr(mlen), F_SOFTLOCK);
1368 1368 }
1369 1369
1370 1370 tlen -= mlen;
1371 1371 dhp = dhp->dh_next;
1372 1372 off = 0;
1373 1373 }
1374 1374 }
1375 1375
1376 1376 mutex_enter(&freemem_lock);
1377 1377 ASSERT(sdp->softlockcnt >= btopr(len));
1378 1378 sdp->softlockcnt -= btopr(len);
1379 1379 mutex_exit(&freemem_lock);
1380 1380 if (sdp->softlockcnt == 0) {
1381 1381 /*
1382 1382 * All SOFTLOCKS are gone. Wakeup any waiting
1383 1383 * unmappers so they can try again to unmap.
1384 1384 * Check for waiters first without the mutex
1385 1385 * held so we don't always grab the mutex on
1386 1386 * softunlocks.
1387 1387 */
1388 1388 if (AS_ISUNMAPWAIT(seg->s_as)) {
1389 1389 mutex_enter(&seg->s_as->a_contents);
1390 1390 if (AS_ISUNMAPWAIT(seg->s_as)) {
1391 1391 AS_CLRUNMAPWAIT(seg->s_as);
1392 1392 cv_broadcast(&seg->s_as->a_cv);
1393 1393 }
1394 1394 mutex_exit(&seg->s_as->a_contents);
1395 1395 }
1396 1396 }
1397 1397
1398 1398 }
1399 1399
1400 1400 /*
1401 1401 * Handle fault for a single page.
1402 1402 * Done in a separate routine so we can handle errors more easily.
1403 1403 * This routine is called only from segdev_faultpages()
1404 1404 * when looping over the range of addresses requested. The segment lock is held.
1405 1405 */
1406 1406 static faultcode_t
1407 1407 segdev_faultpage(
1408 1408 struct hat *hat, /* the hat */
1409 1409 struct seg *seg, /* seg_dev of interest */
1410 1410 caddr_t addr, /* address in as */
1411 1411 struct vpage *vpage, /* pointer to vpage for seg, addr */
1412 1412 enum fault_type type, /* type of fault */
1413 1413 enum seg_rw rw, /* type of access at fault */
1414 1414 devmap_handle_t *dhp) /* devmap handle if any for this page */
1415 1415 {
1416 1416 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1417 1417 uint_t prot;
1418 1418 pfn_t pfnum = PFN_INVALID;
1419 1419 u_offset_t offset;
1420 1420 uint_t hat_flags;
1421 1421 dev_info_t *dip;
1422 1422
1423 1423 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1424 1424 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1425 1425 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1426 1426 (void *)dhp, (void *)seg, (void *)addr));
1427 1427
1428 1428 /*
1429 1429 * Initialize protection value for this page.
1430 1430 * If we have per page protection values check it now.
1431 1431 */
1432 1432 if (sdp->pageprot) {
1433 1433 uint_t protchk;
1434 1434
1435 1435 switch (rw) {
1436 1436 case S_READ:
1437 1437 protchk = PROT_READ;
1438 1438 break;
1439 1439 case S_WRITE:
1440 1440 protchk = PROT_WRITE;
1441 1441 break;
1442 1442 case S_EXEC:
1443 1443 protchk = PROT_EXEC;
1444 1444 break;
1445 1445 case S_OTHER:
1446 1446 default:
1447 1447 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1448 1448 break;
1449 1449 }
1450 1450
1451 1451 prot = VPP_PROT(vpage);
1452 1452 if ((prot & protchk) == 0)
1453 1453 return (FC_PROT); /* illegal access type */
1454 1454 } else {
1455 1455 prot = sdp->prot;
1456 1456 /* caller has already done segment level protection check */
1457 1457 }
1458 1458
1459 1459 if (type == F_SOFTLOCK) {
1460 1460 mutex_enter(&freemem_lock);
1461 1461 sdp->softlockcnt++;
1462 1462 mutex_exit(&freemem_lock);
1463 1463 }
1464 1464
1465 1465 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1466 1466 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1467 1467 /*
1468 1468 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1469 1469 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1470 1470 * seg->s_base.
1471 1471 */
1472 1472 if (dhp == NULL) {
1473 1473 /* If segment has devmap_data, then dhp should be non-NULL */
1474 1474 ASSERT(sdp->devmap_data == NULL);
1475 1475 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1476 1476 (off_t)offset, prot);
1477 1477 prot |= sdp->hat_attr;
1478 1478 } else {
1479 1479 ulong_t off;
1480 1480 struct ddi_umem_cookie *cp;
1481 1481 struct devmap_pmem_cookie *pcp;
1482 1482
1483 1483 /* ensure the dhp passed in contains addr. */
1484 1484 ASSERT(dhp == devmap_find_handle(
1485 1485 (devmap_handle_t *)sdp->devmap_data, addr));
1486 1486
1487 1487 off = addr - dhp->dh_uvaddr;
1488 1488
1489 1489 /*
1490 1490 * This routine assumes that the caller makes sure that the
1491 1491 * fields in dhp used below are unchanged due to remap during
1492 1492 * this call. Caller does HOLD_DHP_LOCK if neeed
1493 1493 */
1494 1494 cp = dhp->dh_cookie;
1495 1495 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1496 1496 pfnum = PFN_INVALID;
1497 1497 } else if (cookie_is_devmem(cp)) {
1498 1498 pfnum = dhp->dh_pfn + btop(off);
1499 1499 } else if (cookie_is_pmem(cp)) {
1500 1500 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1501 1501 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1502 1502 dhp->dh_roff < ptob(pcp->dp_npages));
1503 1503 pfnum = page_pptonum(
1504 1504 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1505 1505 } else {
1506 1506 ASSERT(dhp->dh_roff < cp->size);
1507 1507 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1508 1508 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1509 1509 ASSERT((dhp->dh_cvaddr + off) <=
1510 1510 (cp->cvaddr + cp->size));
1511 1511 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1512 1512 (cp->cvaddr + cp->size));
1513 1513
1514 1514 switch (cp->type) {
1515 1515 case UMEM_LOCKED :
1516 1516 if (cp->pparray != NULL) {
1517 1517 ASSERT((dhp->dh_roff &
1518 1518 PAGEOFFSET) == 0);
1519 1519 pfnum = page_pptonum(
1520 1520 cp->pparray[btop(off +
1521 1521 dhp->dh_roff)]);
1522 1522 } else {
1523 1523 pfnum = hat_getpfnum(
1524 1524 ((proc_t *)cp->procp)->p_as->a_hat,
1525 1525 cp->cvaddr + off);
1526 1526 }
1527 1527 break;
1528 1528 case UMEM_TRASH :
1529 1529 pfnum = page_pptonum(trashpp);
1530 1530 /*
1531 1531 * We should set hat_flags to HAT_NOFAULT also
1532 1532 * However, not all hat layers implement this
1533 1533 */
1534 1534 break;
1535 1535 case KMEM_PAGEABLE:
1536 1536 case KMEM_NON_PAGEABLE:
1537 1537 pfnum = hat_getpfnum(kas.a_hat,
1538 1538 dhp->dh_cvaddr + off);
1539 1539 break;
1540 1540 default :
1541 1541 pfnum = PFN_INVALID;
1542 1542 break;
1543 1543 }
1544 1544 }
1545 1545 prot |= dhp->dh_hat_attr;
1546 1546 }
1547 1547 if (pfnum == PFN_INVALID) {
1548 1548 return (FC_MAKE_ERR(EFAULT));
1549 1549 }
1550 1550 /* prot should already be OR'ed in with hat_attributes if needed */
1551 1551
1552 1552 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1553 1553 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1554 1554 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1555 1555 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1556 1556 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1557 1557
1558 1558 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1559 1559 /*
1560 1560 * It's not _really_ required here to pass sdp->hat_flags
1561 1561 * to hat_devload even though we do it.
1562 1562 * This is because hat figures it out DEVMEM mappings
1563 1563 * are non-consistent, anyway.
1564 1564 */
1565 1565 hat_devload(hat, addr, PAGESIZE, pfnum,
1566 1566 prot, hat_flags | sdp->hat_flags);
1567 1567 return (0);
1568 1568 }
1569 1569
1570 1570 /*
1571 1571 * Fall through to the case where devmap is not used and need to call
1572 1572 * up the device tree to set up the mapping
1573 1573 */
1574 1574
1575 1575 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1576 1576 ASSERT(dip);
1577 1577
1578 1578 /*
1579 1579 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1580 1580 * This is because this calls drivers which may not expect
1581 1581 * prot to have any other values than PROT_ALL
1582 1582 * The root nexus driver has a hack to peek into the segment
1583 1583 * structure and then OR in sdp->hat_attr.
1584 1584 * XX In case the bus_ops interfaces are ever revisited
1585 1585 * we need to fix this. prot should include other hat attributes
1586 1586 */
1587 1587 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1588 1588 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1589 1589 return (FC_MAKE_ERR(EFAULT));
1590 1590 }
1591 1591 return (0);
1592 1592 }
1593 1593
1594 1594 static faultcode_t
1595 1595 segdev_fault(
1596 1596 struct hat *hat, /* the hat */
1597 1597 struct seg *seg, /* the seg_dev of interest */
1598 1598 caddr_t addr, /* the address of the fault */
1599 1599 size_t len, /* the length of the range */
1600 1600 enum fault_type type, /* type of fault */
1601 1601 enum seg_rw rw) /* type of access at fault */
1602 1602 {
1603 1603 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1604 1604 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1605 1605 devmap_handle_t *dhp;
1606 1606 struct devmap_softlock *slock = NULL;
1607 1607 ulong_t slpage = 0;
1608 1608 ulong_t off;
1609 1609 caddr_t maddr = addr;
↓ open down ↓ |
461 lines elided |
↑ open up ↑ |
1610 1610 int err;
1611 1611 int err_is_faultcode = 0;
1612 1612
1613 1613 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1614 1614 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1615 1615 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1616 1616 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1617 1617 "addr %p len %lx type %x\n",
1618 1618 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1619 1619
1620 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1620 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1621 1621
1622 1622 /* Handle non-devmap case */
1623 1623 if (dhp_head == NULL)
1624 1624 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1625 1625
1626 1626 /* Find devmap handle */
1627 1627 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1628 1628 return (FC_NOMAP);
1629 1629
1630 1630 /*
1631 1631 * The seg_dev driver does not implement copy-on-write,
1632 1632 * and always loads translations with maximal allowed permissions
1633 1633 * but we got an fault trying to access the device.
1634 1634 * Servicing the fault is not going to result in any better result
1635 1635 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1636 1636 * faults, then the code below is written for that
1637 1637 * Pending resolution of the following:
1638 1638 * - determine if the F_INVAL/F_SOFTLOCK syncing
1639 1639 * is needed for F_PROT also or not. The code below assumes it does
1640 1640 * - If driver sees F_PROT and calls devmap_load with same type,
1641 1641 * then segdev_faultpages will fail with FC_PROT anyway, need to
1642 1642 * change that so calls from devmap_load to segdev_faultpages for
1643 1643 * F_PROT type are retagged to F_INVAL.
1644 1644 * RFE: Today we dont have drivers that use devmap and want to handle
1645 1645 * F_PROT calls. The code in segdev_fault* is written to allow
1646 1646 * this case but is not tested. A driver that needs this capability
1647 1647 * should be able to remove the short-circuit case; resolve the
1648 1648 * above issues and "should" work.
1649 1649 */
1650 1650 if (type == F_PROT) {
1651 1651 return (FC_PROT);
1652 1652 }
1653 1653
1654 1654 /*
1655 1655 * Loop through dhp list calling devmap_access or segdev_faultpages for
1656 1656 * each devmap handle.
1657 1657 * drivers which implement devmap_access can interpose on faults and do
1658 1658 * device-appropriate special actions before calling devmap_load.
1659 1659 */
1660 1660
1661 1661 /*
1662 1662 * Unfortunately, this simple loop has turned out to expose a variety
1663 1663 * of complex problems which results in the following convoluted code.
1664 1664 *
1665 1665 * First, a desire to handle a serialization of F_SOFTLOCK calls
1666 1666 * to the driver within the framework.
1667 1667 * This results in a dh_softlock structure that is on a per device
1668 1668 * (or device instance) basis and serializes devmap_access calls.
1669 1669 * Ideally we would need to do this for underlying
1670 1670 * memory/device regions that are being faulted on
1671 1671 * but that is hard to identify and with REMAP, harder
1672 1672 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1673 1673 * to F_SOFTLOCK calls to the driver.
1674 1674 * These serializations are to simplify the driver programmer model.
1675 1675 * To support these two features, the code first goes through the
1676 1676 * devmap handles and counts the pages (slpage) that are covered
1677 1677 * by devmap_access callbacks.
1678 1678 * This part ends with a devmap_softlock_enter call
1679 1679 * which allows only one F_SOFTLOCK active on a device instance,
1680 1680 * but multiple F_INVAL/F_PROTs can be active except when a
1681 1681 * F_SOFTLOCK is active
1682 1682 *
1683 1683 * Next, we dont short-circuit the fault code upfront to call
1684 1684 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1685 1685 * the same length when we softlock and softunlock.
1686 1686 *
1687 1687 * -Hat layers may not support softunlocking lengths less than the
1688 1688 * original length when there is large page support.
1689 1689 * -kpmem locking is dependent on keeping the lengths same.
1690 1690 * -if drivers handled F_SOFTLOCK, they probably also expect to
1691 1691 * see an F_SOFTUNLOCK of the same length
1692 1692 * Hence, if extending lengths during softlock,
1693 1693 * softunlock has to make the same adjustments and goes through
1694 1694 * the same loop calling segdev_faultpages/segdev_softunlock
1695 1695 * But some of the synchronization and error handling is different
1696 1696 */
1697 1697
1698 1698 if (type != F_SOFTUNLOCK) {
1699 1699 devmap_handle_t *dhpp = dhp;
1700 1700 size_t slen = len;
1701 1701
1702 1702 /*
1703 1703 * Calculate count of pages that are :
1704 1704 * a) within the (potentially extended) fault region
1705 1705 * b) AND covered by devmap handle with devmap_access
1706 1706 */
1707 1707 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1708 1708 while (slen != 0) {
1709 1709 size_t mlen;
1710 1710
1711 1711 /*
1712 1712 * Softlocking on a region that allows remap is
1713 1713 * unsupported due to unresolved locking issues
1714 1714 * XXX: unclear what these are?
1715 1715 * One potential is that if there is a pending
1716 1716 * softlock, then a remap should not be allowed
1717 1717 * until the unlock is done. This is easily
1718 1718 * fixed by returning error in devmap*remap on
1719 1719 * checking the dh->dh_softlock->softlocked value
1720 1720 */
1721 1721 if ((type == F_SOFTLOCK) &&
1722 1722 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1723 1723 return (FC_NOSUPPORT);
1724 1724 }
1725 1725
1726 1726 mlen = MIN(slen, (dhpp->dh_len - off));
1727 1727 if (dhpp->dh_callbackops.devmap_access) {
1728 1728 size_t llen;
1729 1729 caddr_t laddr;
1730 1730 /*
1731 1731 * use extended length for large page mappings
1732 1732 */
1733 1733 HOLD_DHP_LOCK(dhpp);
1734 1734 if ((sdp->pageprot == 0) &&
1735 1735 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1736 1736 devmap_get_large_pgsize(dhpp,
1737 1737 mlen, maddr, &llen, &laddr);
1738 1738 } else {
1739 1739 llen = mlen;
1740 1740 }
1741 1741 RELE_DHP_LOCK(dhpp);
1742 1742
1743 1743 slpage += btopr(llen);
1744 1744 slock = dhpp->dh_softlock;
1745 1745 }
1746 1746 maddr += mlen;
1747 1747 ASSERT(slen >= mlen);
1748 1748 slen -= mlen;
1749 1749 dhpp = dhpp->dh_next;
1750 1750 off = 0;
1751 1751 }
1752 1752 /*
1753 1753 * synchonize with other faulting threads and wait till safe
1754 1754 * devmap_softlock_enter might return due to signal in cv_wait
1755 1755 *
1756 1756 * devmap_softlock_enter has to be called outside of while loop
1757 1757 * to prevent a deadlock if len spans over multiple dhps.
1758 1758 * dh_softlock is based on device instance and if multiple dhps
1759 1759 * use the same device instance, the second dhp's LOCK call
1760 1760 * will hang waiting on the first to complete.
1761 1761 * devmap_setup verifies that slocks in a dhp_chain are same.
1762 1762 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1763 1763 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1764 1764 * we could have done the softlock_enter inside the loop
1765 1765 * and supported multi-dhp mappings with dissimilar devices
1766 1766 */
1767 1767 if (err = devmap_softlock_enter(slock, slpage, type))
1768 1768 return (FC_MAKE_ERR(err));
1769 1769 }
1770 1770
1771 1771 /* reset 'maddr' to the start addr of the range of fault. */
1772 1772 maddr = addr;
1773 1773
1774 1774 /* calculate the offset corresponds to 'addr' in the first dhp. */
1775 1775 off = (ulong_t)(addr - dhp->dh_uvaddr);
1776 1776
1777 1777 /*
1778 1778 * The fault length may span over multiple dhps.
1779 1779 * Loop until the total length is satisfied.
1780 1780 */
1781 1781 while (len != 0) {
1782 1782 size_t llen;
1783 1783 size_t mlen;
1784 1784 caddr_t laddr;
1785 1785
1786 1786 /*
1787 1787 * mlen is the smaller of 'len' and the length
1788 1788 * from addr to the end of mapping defined by dhp.
1789 1789 */
1790 1790 mlen = MIN(len, (dhp->dh_len - off));
1791 1791
1792 1792 HOLD_DHP_LOCK(dhp);
1793 1793 /*
1794 1794 * Pass the extended length and address to devmap_access
1795 1795 * if large pagesize is used for loading address translations.
1796 1796 */
1797 1797 if ((sdp->pageprot == 0) &&
1798 1798 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1799 1799 devmap_get_large_pgsize(dhp, mlen, maddr,
1800 1800 &llen, &laddr);
1801 1801 ASSERT(maddr == addr || laddr == maddr);
1802 1802 } else {
1803 1803 llen = mlen;
1804 1804 laddr = maddr;
1805 1805 }
1806 1806
1807 1807 if (dhp->dh_callbackops.devmap_access != NULL) {
1808 1808 offset_t aoff;
1809 1809
1810 1810 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1811 1811
1812 1812 /*
1813 1813 * call driver's devmap_access entry point which will
1814 1814 * call devmap_load/contextmgmt to load the translations
1815 1815 *
1816 1816 * We drop the dhp_lock before calling access so
1817 1817 * drivers can call devmap_*_remap within access
1818 1818 */
1819 1819 RELE_DHP_LOCK(dhp);
1820 1820
1821 1821 err = (*dhp->dh_callbackops.devmap_access)(
1822 1822 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1823 1823 } else {
1824 1824 /*
1825 1825 * If no devmap_access entry point, then load mappings
1826 1826 * hold dhp_lock across faultpages if REMAP
1827 1827 */
1828 1828 err = segdev_faultpages(hat, seg, laddr, llen,
1829 1829 type, rw, dhp);
1830 1830 err_is_faultcode = 1;
1831 1831 RELE_DHP_LOCK(dhp);
1832 1832 }
1833 1833
1834 1834 if (err) {
1835 1835 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1836 1836 /*
1837 1837 * If not first dhp, use
1838 1838 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1839 1839 * While this is recursion, it is incorrect to
1840 1840 * call just segdev_softunlock
1841 1841 * if we are using either large pages
1842 1842 * or devmap_access. It will be more right
1843 1843 * to go through the same loop as above
1844 1844 * rather than call segdev_softunlock directly
1845 1845 * It will use the right lenghths as well as
1846 1846 * call into the driver devmap_access routines.
1847 1847 */
1848 1848 size_t done = (size_t)(maddr - addr);
1849 1849 (void) segdev_fault(hat, seg, addr, done,
1850 1850 F_SOFTUNLOCK, S_OTHER);
1851 1851 /*
1852 1852 * reduce slpage by number of pages
1853 1853 * released by segdev_softunlock
1854 1854 */
1855 1855 ASSERT(slpage >= btopr(done));
1856 1856 devmap_softlock_exit(slock,
1857 1857 slpage - btopr(done), type);
1858 1858 } else {
1859 1859 devmap_softlock_exit(slock, slpage, type);
1860 1860 }
1861 1861
1862 1862
1863 1863 /*
1864 1864 * Segdev_faultpages() already returns a faultcode,
1865 1865 * hence, result from segdev_faultpages() should be
1866 1866 * returned directly.
1867 1867 */
1868 1868 if (err_is_faultcode)
1869 1869 return (err);
1870 1870 return (FC_MAKE_ERR(err));
1871 1871 }
1872 1872
1873 1873 maddr += mlen;
1874 1874 ASSERT(len >= mlen);
1875 1875 len -= mlen;
1876 1876 dhp = dhp->dh_next;
1877 1877 off = 0;
1878 1878
1879 1879 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1880 1880 }
1881 1881 /*
1882 1882 * release the softlock count at end of fault
1883 1883 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1884 1884 */
1885 1885 if ((type == F_INVAL) || (type == F_PROT))
1886 1886 devmap_softlock_exit(slock, slpage, type);
1887 1887 return (0);
1888 1888 }
1889 1889
1890 1890 /*
1891 1891 * segdev_faultpages
1892 1892 *
1893 1893 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1894 1894 * This routine assumes that the callers makes sure that the fields
1895 1895 * in dhp used below are not changed due to remap during this call.
1896 1896 * Caller does HOLD_DHP_LOCK if neeed
1897 1897 * This routine returns a faultcode_t as a return value for segdev_fault.
1898 1898 */
1899 1899 static faultcode_t
1900 1900 segdev_faultpages(
1901 1901 struct hat *hat, /* the hat */
1902 1902 struct seg *seg, /* the seg_dev of interest */
1903 1903 caddr_t addr, /* the address of the fault */
1904 1904 size_t len, /* the length of the range */
1905 1905 enum fault_type type, /* type of fault */
1906 1906 enum seg_rw rw, /* type of access at fault */
1907 1907 devmap_handle_t *dhp) /* devmap handle */
1908 1908 {
1909 1909 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1910 1910 register caddr_t a;
1911 1911 struct vpage *vpage;
1912 1912 struct ddi_umem_cookie *kpmem_cookie = NULL;
1913 1913 int err;
1914 1914
1915 1915 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1916 1916 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1917 1917 (void *)dhp, (void *)seg, (void *)addr, len);
1918 1918 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1919 1919 "dhp %p seg %p addr %p len %lx\n",
1920 1920 (void *)dhp, (void *)seg, (void *)addr, len));
1921 1921
1922 1922 /*
1923 1923 * The seg_dev driver does not implement copy-on-write,
1924 1924 * and always loads translations with maximal allowed permissions
1925 1925 * but we got an fault trying to access the device.
1926 1926 * Servicing the fault is not going to result in any better result
1927 1927 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1928 1928 * This code should be removed and let the normal fault handling
1929 1929 * take care of finding the error
1930 1930 */
1931 1931 if (type == F_PROT) {
1932 1932 return (FC_PROT);
1933 1933 }
1934 1934
1935 1935 if (type == F_SOFTUNLOCK) {
1936 1936 segdev_softunlock(hat, seg, addr, len, rw);
1937 1937 return (0);
1938 1938 }
1939 1939
1940 1940 /*
1941 1941 * For kernel pageable memory, fault/lock segkp pages
1942 1942 * We hold this until the completion of this
1943 1943 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1944 1944 */
1945 1945 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1946 1946 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1947 1947 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1948 1948 return (err);
1949 1949 }
1950 1950
1951 1951 /*
1952 1952 * If we have the same protections for the entire segment,
1953 1953 * insure that the access being attempted is legitimate.
1954 1954 */
1955 1955 rw_enter(&sdp->lock, RW_READER);
1956 1956 if (sdp->pageprot == 0) {
1957 1957 uint_t protchk;
1958 1958
1959 1959 switch (rw) {
1960 1960 case S_READ:
1961 1961 protchk = PROT_READ;
1962 1962 break;
1963 1963 case S_WRITE:
1964 1964 protchk = PROT_WRITE;
1965 1965 break;
1966 1966 case S_EXEC:
1967 1967 protchk = PROT_EXEC;
1968 1968 break;
1969 1969 case S_OTHER:
1970 1970 default:
1971 1971 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1972 1972 break;
1973 1973 }
1974 1974
1975 1975 if ((sdp->prot & protchk) == 0) {
1976 1976 rw_exit(&sdp->lock);
1977 1977 /* undo kpmem locking */
1978 1978 if (kpmem_cookie != NULL) {
1979 1979 release_kpmem_lock(kpmem_cookie, btopr(len));
1980 1980 }
1981 1981 return (FC_PROT); /* illegal access type */
1982 1982 }
1983 1983 }
1984 1984
1985 1985 /*
1986 1986 * we do a single hat_devload for the range if
1987 1987 * - devmap framework (dhp is not NULL),
1988 1988 * - pageprot == 0, i.e., no per-page protection set and
1989 1989 * - is device pages, irrespective of whether we are using large pages
1990 1990 */
1991 1991 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1992 1992 pfn_t pfnum;
1993 1993 uint_t hat_flags;
1994 1994
1995 1995 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1996 1996 rw_exit(&sdp->lock);
1997 1997 return (FC_NOMAP);
1998 1998 }
1999 1999
2000 2000 if (type == F_SOFTLOCK) {
2001 2001 mutex_enter(&freemem_lock);
2002 2002 sdp->softlockcnt += btopr(len);
2003 2003 mutex_exit(&freemem_lock);
2004 2004 }
2005 2005
2006 2006 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2007 2007 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2008 2008 ASSERT(!pf_is_memory(pfnum));
2009 2009
2010 2010 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2011 2011 hat_flags | sdp->hat_flags);
2012 2012 rw_exit(&sdp->lock);
2013 2013 return (0);
2014 2014 }
2015 2015
2016 2016 /* Handle cases where we have to loop through fault handling per-page */
2017 2017
2018 2018 if (sdp->vpage == NULL)
2019 2019 vpage = NULL;
2020 2020 else
2021 2021 vpage = &sdp->vpage[seg_page(seg, addr)];
2022 2022
2023 2023 /* loop over the address range handling each fault */
2024 2024 for (a = addr; a < addr + len; a += PAGESIZE) {
2025 2025 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2026 2026 break;
2027 2027 }
2028 2028 if (vpage != NULL)
2029 2029 vpage++;
2030 2030 }
2031 2031 rw_exit(&sdp->lock);
2032 2032 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2033 2033 size_t done = (size_t)(a - addr); /* pages fault successfully */
2034 2034 if (done > 0) {
2035 2035 /* use softunlock for those pages */
2036 2036 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2037 2037 }
2038 2038 if (kpmem_cookie != NULL) {
2039 2039 /* release kpmem lock for rest of pages */
2040 2040 ASSERT(len >= done);
2041 2041 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2042 2042 }
2043 2043 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2044 2044 /* for non-SOFTLOCK cases, release kpmem */
2045 2045 release_kpmem_lock(kpmem_cookie, btopr(len));
2046 2046 }
2047 2047 return (err);
2048 2048 }
2049 2049
↓ open down ↓ |
419 lines elided |
↑ open up ↑ |
2050 2050 /*
2051 2051 * Asynchronous page fault. We simply do nothing since this
2052 2052 * entry point is not supposed to load up the translation.
2053 2053 */
2054 2054 /*ARGSUSED*/
2055 2055 static faultcode_t
2056 2056 segdev_faulta(struct seg *seg, caddr_t addr)
2057 2057 {
2058 2058 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2059 2059 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2060 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2060 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2061 2061
2062 2062 return (0);
2063 2063 }
2064 2064
2065 2065 static int
2066 2066 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2067 2067 {
2068 2068 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2069 2069 register devmap_handle_t *dhp;
2070 2070 register struct vpage *vp, *evp;
2071 2071 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2072 2072 ulong_t off;
2073 2073 size_t mlen, sz;
2074 2074
2075 2075 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2076 2076 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2077 2077 (void *)seg, (void *)addr, len, prot);
2078 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2078 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2079 2079
2080 2080 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2081 2081 /*
2082 2082 * Fail the setprot if pages are SOFTLOCKed through this
2083 2083 * mapping.
2084 2084 * Softlockcnt is protected from change by the as read lock.
2085 2085 */
2086 2086 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2087 2087 "segdev_setprot:error softlockcnt=%lx", sz);
2088 2088 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2089 2089 return (EAGAIN);
2090 2090 }
2091 2091
2092 2092 if (dhp_head != NULL) {
2093 2093 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2094 2094 return (EINVAL);
2095 2095
2096 2096 /*
2097 2097 * check if violate maxprot.
2098 2098 */
2099 2099 off = (ulong_t)(addr - dhp->dh_uvaddr);
2100 2100 mlen = len;
2101 2101 while (dhp) {
2102 2102 if ((dhp->dh_maxprot & prot) != prot)
2103 2103 return (EACCES); /* violated maxprot */
2104 2104
2105 2105 if (mlen > (dhp->dh_len - off)) {
2106 2106 mlen -= dhp->dh_len - off;
2107 2107 dhp = dhp->dh_next;
2108 2108 off = 0;
2109 2109 } else
2110 2110 break;
2111 2111 }
2112 2112 } else {
2113 2113 if ((sdp->maxprot & prot) != prot)
2114 2114 return (EACCES);
2115 2115 }
2116 2116
2117 2117 rw_enter(&sdp->lock, RW_WRITER);
2118 2118 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2119 2119 if (sdp->prot == prot) {
2120 2120 rw_exit(&sdp->lock);
2121 2121 return (0); /* all done */
2122 2122 }
2123 2123 sdp->prot = (uchar_t)prot;
2124 2124 } else {
2125 2125 sdp->pageprot = 1;
2126 2126 if (sdp->vpage == NULL) {
2127 2127 /*
2128 2128 * First time through setting per page permissions,
2129 2129 * initialize all the vpage structures to prot
2130 2130 */
2131 2131 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2132 2132 KM_SLEEP);
2133 2133 evp = &sdp->vpage[seg_pages(seg)];
2134 2134 for (vp = sdp->vpage; vp < evp; vp++)
2135 2135 VPP_SETPROT(vp, sdp->prot);
2136 2136 }
2137 2137 /*
2138 2138 * Now go change the needed vpages protections.
2139 2139 */
2140 2140 evp = &sdp->vpage[seg_page(seg, addr + len)];
2141 2141 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2142 2142 VPP_SETPROT(vp, prot);
2143 2143 }
2144 2144 rw_exit(&sdp->lock);
2145 2145
2146 2146 if (dhp_head != NULL) {
2147 2147 devmap_handle_t *tdhp;
2148 2148 /*
2149 2149 * If large page size was used in hat_devload(),
2150 2150 * the same page size must be used in hat_unload().
2151 2151 */
2152 2152 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2153 2153 while (tdhp != NULL) {
2154 2154 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2155 2155 break;
2156 2156 }
2157 2157 tdhp = tdhp->dh_next;
2158 2158 }
2159 2159 if (tdhp) {
2160 2160 size_t slen = len;
2161 2161 size_t mlen;
2162 2162 size_t soff;
2163 2163
2164 2164 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2165 2165 while (slen != 0) {
2166 2166 mlen = MIN(slen, (dhp->dh_len - soff));
2167 2167 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2168 2168 dhp->dh_len, HAT_UNLOAD);
2169 2169 dhp = dhp->dh_next;
2170 2170 ASSERT(slen >= mlen);
2171 2171 slen -= mlen;
2172 2172 soff = 0;
2173 2173 }
2174 2174 return (0);
2175 2175 }
2176 2176 }
2177 2177
2178 2178 if ((prot & ~PROT_USER) == PROT_NONE) {
2179 2179 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2180 2180 } else {
2181 2181 /*
2182 2182 * RFE: the segment should keep track of all attributes
2183 2183 * allowing us to remove the deprecated hat_chgprot
2184 2184 * and use hat_chgattr.
2185 2185 */
2186 2186 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2187 2187 }
2188 2188
2189 2189 return (0);
2190 2190 }
↓ open down ↓ |
102 lines elided |
↑ open up ↑ |
2191 2191
2192 2192 static int
2193 2193 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2194 2194 {
2195 2195 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2196 2196 struct vpage *vp, *evp;
2197 2197
2198 2198 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2199 2199 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2200 2200 (void *)seg, (void *)addr, len, prot);
2201 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2201 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2202 2202
2203 2203 /*
2204 2204 * If segment protection can be used, simply check against them
2205 2205 */
2206 2206 rw_enter(&sdp->lock, RW_READER);
2207 2207 if (sdp->pageprot == 0) {
2208 2208 register int err;
2209 2209
2210 2210 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2211 2211 rw_exit(&sdp->lock);
2212 2212 return (err);
2213 2213 }
2214 2214
2215 2215 /*
2216 2216 * Have to check down to the vpage level
2217 2217 */
2218 2218 evp = &sdp->vpage[seg_page(seg, addr + len)];
2219 2219 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2220 2220 if ((VPP_PROT(vp) & prot) != prot) {
2221 2221 rw_exit(&sdp->lock);
2222 2222 return (EACCES);
2223 2223 }
2224 2224 }
2225 2225 rw_exit(&sdp->lock);
2226 2226 return (0);
2227 2227 }
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
2228 2228
2229 2229 static int
2230 2230 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2231 2231 {
2232 2232 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2233 2233 size_t pgno;
2234 2234
2235 2235 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2236 2236 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2237 2237 (void *)seg, (void *)addr, len, (void *)protv);
2238 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2238 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2239 2239
2240 2240 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2241 2241 if (pgno != 0) {
2242 2242 rw_enter(&sdp->lock, RW_READER);
2243 2243 if (sdp->pageprot == 0) {
2244 2244 do {
2245 2245 protv[--pgno] = sdp->prot;
2246 2246 } while (pgno != 0);
2247 2247 } else {
2248 2248 size_t pgoff = seg_page(seg, addr);
2249 2249
2250 2250 do {
2251 2251 pgno--;
2252 2252 protv[pgno] =
2253 2253 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2254 2254 } while (pgno != 0);
2255 2255 }
2256 2256 rw_exit(&sdp->lock);
2257 2257 }
2258 2258 return (0);
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
2259 2259 }
2260 2260
2261 2261 static u_offset_t
2262 2262 segdev_getoffset(register struct seg *seg, caddr_t addr)
2263 2263 {
2264 2264 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2265 2265
2266 2266 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2267 2267 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2268 2268
2269 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2269 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2270 2270
2271 2271 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2272 2272 }
2273 2273
2274 2274 /*ARGSUSED*/
2275 2275 static int
2276 2276 segdev_gettype(register struct seg *seg, caddr_t addr)
2277 2277 {
2278 2278 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2279 2279
2280 2280 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2281 2281 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2282 2282
2283 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2283 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2284 2284
2285 2285 return (sdp->type);
2286 2286 }
2287 2287
2288 2288
2289 2289 /*ARGSUSED*/
2290 2290 static int
2291 2291 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2292 2292 {
2293 2293 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2294 2294
2295 2295 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2296 2296 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2297 2297
2298 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2298 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2299 2299
2300 2300 /*
2301 2301 * Note that this vp is the common_vp of the device, where the
2302 2302 * pages are hung ..
2303 2303 */
2304 2304 *vpp = VTOCVP(sdp->vp);
2305 2305
2306 2306 return (0);
2307 2307 }
2308 2308
2309 2309 static void
2310 2310 segdev_badop(void)
2311 2311 {
2312 2312 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2313 2313 "segdev_badop:start");
2314 2314 panic("segdev_badop");
2315 2315 /*NOTREACHED*/
2316 2316 }
2317 2317
↓ open down ↓ |
9 lines elided |
↑ open up ↑ |
2318 2318 /*
2319 2319 * segdev pages are not in the cache, and thus can't really be controlled.
2320 2320 * Hence, syncs are simply always successful.
2321 2321 */
2322 2322 /*ARGSUSED*/
2323 2323 static int
2324 2324 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2325 2325 {
2326 2326 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2327 2327
2328 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2328 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2329 2329
2330 2330 return (0);
2331 2331 }
2332 2332
2333 2333 /*
2334 2334 * segdev pages are always "in core".
2335 2335 */
2336 2336 /*ARGSUSED*/
2337 2337 static size_t
2338 2338 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2339 2339 {
2340 2340 size_t v = 0;
2341 2341
2342 2342 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2343 2343
2344 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2344 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2345 2345
2346 2346 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2347 2347 v += PAGESIZE)
2348 2348 *vec++ = 1;
2349 2349 return (v);
2350 2350 }
2351 2351
2352 2352 /*
2353 2353 * segdev pages are not in the cache, and thus can't really be controlled.
2354 2354 * Hence, locks are simply always successful.
2355 2355 */
2356 2356 /*ARGSUSED*/
2357 2357 static int
2358 2358 segdev_lockop(struct seg *seg, caddr_t addr,
2359 2359 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2360 2360 {
2361 2361 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2362 2362
2363 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2363 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2364 2364
2365 2365 return (0);
2366 2366 }
2367 2367
2368 2368 /*
2369 2369 * segdev pages are not in the cache, and thus can't really be controlled.
2370 2370 * Hence, advise is simply always successful.
2371 2371 */
2372 2372 /*ARGSUSED*/
2373 2373 static int
2374 2374 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2375 2375 {
2376 2376 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2377 2377
2378 - ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2378 + ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2379 2379
2380 2380 return (0);
2381 2381 }
2382 2382
2383 2383 /*
2384 2384 * segdev pages are not dumped, so we just return
2385 2385 */
2386 2386 /*ARGSUSED*/
2387 2387 static void
2388 2388 segdev_dump(struct seg *seg)
2389 2389 {}
2390 2390
2391 2391 /*
2392 2392 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2393 2393 * for a segment. Called from a drivers segmap(9E)
2394 2394 * routine.
2395 2395 */
2396 2396 /*ARGSUSED*/
2397 2397 int
2398 2398 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2399 2399 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2400 2400 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2401 2401 {
2402 2402 struct segdev_crargs dev_a;
2403 2403 int (*mapfunc)(dev_t dev, off_t off, int prot);
2404 2404 uint_t hat_attr;
2405 2405 pfn_t pfn;
2406 2406 int error, i;
2407 2407
2408 2408 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2409 2409 "ddi_segmap_setup:start");
2410 2410
2411 2411 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2412 2412 return (ENODEV);
2413 2413
2414 2414 /*
2415 2415 * Character devices that support the d_mmap
2416 2416 * interface can only be mmap'ed shared.
2417 2417 */
2418 2418 if ((flags & MAP_TYPE) != MAP_SHARED)
2419 2419 return (EINVAL);
2420 2420
2421 2421 /*
2422 2422 * Check that this region is indeed mappable on this platform.
2423 2423 * Use the mapping function.
2424 2424 */
2425 2425 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2426 2426 return (ENXIO);
2427 2427
2428 2428 /*
2429 2429 * Check to ensure that the entire range is
2430 2430 * legal and we are not trying to map in
2431 2431 * more than the device will let us.
2432 2432 */
2433 2433 for (i = 0; i < len; i += PAGESIZE) {
2434 2434 if (i == 0) {
2435 2435 /*
2436 2436 * Save the pfn at offset here. This pfn will be
2437 2437 * used later to get user address.
2438 2438 */
2439 2439 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2440 2440 maxprot)) == PFN_INVALID)
2441 2441 return (ENXIO);
2442 2442 } else {
2443 2443 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2444 2444 PFN_INVALID)
2445 2445 return (ENXIO);
2446 2446 }
2447 2447 }
2448 2448
2449 2449 as_rangelock(as);
2450 2450 /* Pick an address w/o worrying about any vac alignment constraints. */
2451 2451 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2452 2452 if (error != 0) {
2453 2453 as_rangeunlock(as);
2454 2454 return (error);
2455 2455 }
2456 2456
2457 2457 dev_a.mapfunc = mapfunc;
2458 2458 dev_a.dev = dev;
2459 2459 dev_a.offset = (offset_t)offset;
2460 2460 dev_a.type = flags & MAP_TYPE;
2461 2461 dev_a.prot = (uchar_t)prot;
2462 2462 dev_a.maxprot = (uchar_t)maxprot;
2463 2463 dev_a.hat_attr = hat_attr;
2464 2464 dev_a.hat_flags = 0;
2465 2465 dev_a.devmap_data = NULL;
2466 2466
2467 2467 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2468 2468 as_rangeunlock(as);
2469 2469 return (error);
2470 2470
2471 2471 }
2472 2472
2473 2473 /*ARGSUSED*/
2474 2474 static int
2475 2475 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2476 2476 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2477 2477 {
2478 2478 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2479 2479 "segdev_pagelock:start");
2480 2480 return (ENOTSUP);
2481 2481 }
2482 2482
2483 2483 /*ARGSUSED*/
2484 2484 static int
2485 2485 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2486 2486 uint_t szc)
2487 2487 {
2488 2488 return (ENOTSUP);
2489 2489 }
2490 2490
2491 2491 /*
2492 2492 * devmap_device: Used by devmap framework to establish mapping
2493 2493 * called by devmap_seup(9F) during map setup time.
2494 2494 */
2495 2495 /*ARGSUSED*/
2496 2496 static int
2497 2497 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2498 2498 offset_t off, size_t len, uint_t flags)
2499 2499 {
2500 2500 devmap_handle_t *rdhp, *maxdhp;
2501 2501 struct segdev_crargs dev_a;
2502 2502 int err;
2503 2503 uint_t maxprot = PROT_ALL;
2504 2504 offset_t offset = 0;
2505 2505 pfn_t pfn;
2506 2506 struct devmap_pmem_cookie *pcp;
2507 2507
2508 2508 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2509 2509 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2510 2510 (void *)dhp, (void *)addr, off, len);
2511 2511
2512 2512 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2513 2513 (void *)dhp, (void *)addr, off, len));
2514 2514
2515 2515 as_rangelock(as);
2516 2516 if ((flags & MAP_FIXED) == 0) {
2517 2517 offset_t aligned_off;
2518 2518
2519 2519 rdhp = maxdhp = dhp;
2520 2520 while (rdhp != NULL) {
2521 2521 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2522 2522 maxdhp : rdhp;
2523 2523 rdhp = rdhp->dh_next;
2524 2524 maxprot |= dhp->dh_maxprot;
2525 2525 }
2526 2526 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2527 2527
2528 2528 /*
2529 2529 * Use the dhp that has the
2530 2530 * largest len to get user address.
2531 2531 */
2532 2532 /*
2533 2533 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2534 2534 * use 0 which is as good as any other.
2535 2535 */
2536 2536 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2537 2537 aligned_off = (offset_t)0;
2538 2538 } else if (dhp_is_devmem(maxdhp)) {
2539 2539 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2540 2540 } else if (dhp_is_pmem(maxdhp)) {
2541 2541 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2542 2542 pfn = page_pptonum(
2543 2543 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2544 2544 aligned_off = (offset_t)ptob(pfn) - offset;
2545 2545 } else {
2546 2546 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2547 2547 offset;
2548 2548 }
2549 2549
2550 2550 /*
2551 2551 * Pick an address aligned to dh_cookie.
2552 2552 * for kernel memory/user memory, cookie is cvaddr.
2553 2553 * for device memory, cookie is physical address.
2554 2554 */
2555 2555 map_addr(addr, len, aligned_off, 1, flags);
2556 2556 if (*addr == NULL) {
2557 2557 as_rangeunlock(as);
2558 2558 return (ENOMEM);
2559 2559 }
2560 2560 } else {
2561 2561 /*
2562 2562 * User-specified address; blow away any previous mappings.
2563 2563 */
2564 2564 (void) as_unmap(as, *addr, len);
2565 2565 }
2566 2566
2567 2567 dev_a.mapfunc = NULL;
2568 2568 dev_a.dev = dhp->dh_dev;
2569 2569 dev_a.type = flags & MAP_TYPE;
2570 2570 dev_a.offset = off;
2571 2571 /*
2572 2572 * sdp->maxprot has the least restrict protection of all dhps.
2573 2573 */
2574 2574 dev_a.maxprot = maxprot;
2575 2575 dev_a.prot = dhp->dh_prot;
2576 2576 /*
2577 2577 * devmap uses dhp->dh_hat_attr for hat.
2578 2578 */
2579 2579 dev_a.hat_flags = 0;
2580 2580 dev_a.hat_attr = 0;
2581 2581 dev_a.devmap_data = (void *)dhp;
2582 2582
2583 2583 err = as_map(as, *addr, len, segdev_create, &dev_a);
2584 2584 as_rangeunlock(as);
2585 2585 return (err);
2586 2586 }
2587 2587
2588 2588 int
2589 2589 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2590 2590 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2591 2591 size_t, uint_t, uint_t))
2592 2592 {
2593 2593 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2594 2594 struct devmap_ctx *devctx;
2595 2595 int do_timeout = 0;
2596 2596 int ret;
2597 2597
2598 2598 #ifdef lint
2599 2599 pvtp = pvtp;
2600 2600 #endif
2601 2601
2602 2602 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2603 2603 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2604 2604 (void *)dhp, off, len);
2605 2605 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2606 2606 (void *)dhp, off, len));
2607 2607
2608 2608 if (ctxmgt == NULL)
2609 2609 return (FC_HWERR);
2610 2610
2611 2611 devctx = dhp->dh_ctx;
2612 2612
2613 2613 /*
2614 2614 * If we are on an MP system with more than one cpu running
2615 2615 * and if a thread on some CPU already has the context, wait
2616 2616 * for it to finish if there is a hysteresis timeout.
2617 2617 *
2618 2618 * We call cv_wait() instead of cv_wait_sig() because
2619 2619 * it does not matter much if it returned due to a signal
2620 2620 * or due to a cv_signal() or cv_broadcast(). In either event
2621 2621 * we need to complete the mapping otherwise the processes
2622 2622 * will die with a SEGV.
2623 2623 */
2624 2624 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2625 2625 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2626 2626 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2627 2627 devctx, dhp);
2628 2628 do_timeout = 1;
2629 2629 mutex_enter(&devctx->lock);
2630 2630 while (devctx->oncpu)
2631 2631 cv_wait(&devctx->cv, &devctx->lock);
2632 2632 devctx->oncpu = 1;
2633 2633 mutex_exit(&devctx->lock);
2634 2634 }
2635 2635
2636 2636 /*
2637 2637 * Call the contextmgt callback so that the driver can handle
2638 2638 * the fault.
2639 2639 */
2640 2640 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2641 2641
2642 2642 /*
2643 2643 * If devmap_access() returned -1, then there was a hardware
2644 2644 * error so we need to convert the return value to something
2645 2645 * that trap() will understand. Otherwise, the return value
2646 2646 * is already a fault code generated by devmap_unload()
2647 2647 * or devmap_load().
2648 2648 */
2649 2649 if (ret) {
2650 2650 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2651 2651 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2652 2652 ret, dhp, devctx);
2653 2653 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2654 2654 ret, (void *)dhp));
2655 2655 if (devctx->oncpu) {
2656 2656 mutex_enter(&devctx->lock);
2657 2657 devctx->oncpu = 0;
2658 2658 cv_signal(&devctx->cv);
2659 2659 mutex_exit(&devctx->lock);
2660 2660 }
2661 2661 return (FC_HWERR);
2662 2662 }
2663 2663
2664 2664 /*
2665 2665 * Setup the timeout if we need to
2666 2666 */
2667 2667 if (do_timeout) {
2668 2668 mutex_enter(&devctx->lock);
2669 2669 if (dhp->dh_timeout_length > 0) {
2670 2670 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2671 2671 "devmap_do_ctxmgt:timeout set");
2672 2672 devctx->timeout = timeout(devmap_ctxto,
2673 2673 devctx, dhp->dh_timeout_length);
2674 2674 } else {
2675 2675 /*
2676 2676 * We don't want to wait so set oncpu to
2677 2677 * 0 and wake up anyone waiting.
2678 2678 */
2679 2679 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2680 2680 "devmap_do_ctxmgt:timeout not set");
2681 2681 devctx->oncpu = 0;
2682 2682 cv_signal(&devctx->cv);
2683 2683 }
2684 2684 mutex_exit(&devctx->lock);
2685 2685 }
2686 2686
2687 2687 return (DDI_SUCCESS);
2688 2688 }
2689 2689
2690 2690 /*
2691 2691 * end of mapping
2692 2692 * poff fault_offset |
2693 2693 * base | | |
2694 2694 * | | | |
2695 2695 * V V V V
2696 2696 * +-----------+---------------+-------+---------+-------+
2697 2697 * ^ ^ ^ ^
2698 2698 * |<--- offset--->|<-len->| |
2699 2699 * |<--- dh_len(size of mapping) --->|
2700 2700 * |<-- pg -->|
2701 2701 * -->|rlen|<--
2702 2702 */
2703 2703 static ulong_t
2704 2704 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2705 2705 ulong_t *opfn, ulong_t *pagesize)
2706 2706 {
2707 2707 register int level;
2708 2708 ulong_t pg;
2709 2709 ulong_t poff;
2710 2710 ulong_t base;
2711 2711 caddr_t uvaddr;
2712 2712 long rlen;
2713 2713
2714 2714 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2715 2715 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2716 2716 (void *)dhp, offset, len);
2717 2717 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2718 2718 (void *)dhp, offset, len));
2719 2719
2720 2720 /*
2721 2721 * get the max. pagesize that is aligned within the range
2722 2722 * <dh_pfn, dh_pfn+offset>.
2723 2723 *
2724 2724 * The calculations below use physical address to ddetermine
2725 2725 * the page size to use. The same calculations can use the
2726 2726 * virtual address to determine the page size.
2727 2727 */
2728 2728 base = (ulong_t)ptob(dhp->dh_pfn);
2729 2729 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2730 2730 pg = page_get_pagesize(level);
2731 2731 poff = ((base + offset) & ~(pg - 1));
2732 2732 uvaddr = dhp->dh_uvaddr + (poff - base);
2733 2733 if ((poff >= base) &&
2734 2734 ((poff + pg) <= (base + dhp->dh_len)) &&
2735 2735 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2736 2736 break;
2737 2737 }
2738 2738
2739 2739 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2740 2740 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2741 2741 base, poff, dhp);
2742 2742 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2743 2743 base, poff, dhp->dh_pfn));
2744 2744
2745 2745 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2746 2746 ASSERT(level >= 0);
2747 2747
2748 2748 *pagesize = pg;
2749 2749 *opfn = dhp->dh_pfn + btop(poff - base);
2750 2750
2751 2751 rlen = len + offset - (poff - base + pg);
2752 2752
2753 2753 ASSERT(rlen < (long)len);
2754 2754
2755 2755 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2756 2756 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2757 2757 (void *)dhp, level, rlen, pagesize, opfn);
2758 2758 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2759 2759 "level %x rlen %lx psize %lx opfn %lx\n",
2760 2760 (void *)dhp, level, rlen, *pagesize, *opfn));
2761 2761
2762 2762 return ((ulong_t)((rlen > 0) ? rlen : 0));
2763 2763 }
2764 2764
2765 2765 /*
2766 2766 * find the dhp that contains addr.
2767 2767 */
2768 2768 static devmap_handle_t *
2769 2769 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2770 2770 {
2771 2771 devmap_handle_t *dhp;
2772 2772
2773 2773 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2774 2774 "devmap_find_handle:start");
2775 2775
2776 2776 dhp = dhp_head;
2777 2777 while (dhp) {
2778 2778 if (addr >= dhp->dh_uvaddr &&
2779 2779 addr < (dhp->dh_uvaddr + dhp->dh_len))
2780 2780 return (dhp);
2781 2781 dhp = dhp->dh_next;
2782 2782 }
2783 2783
2784 2784 return ((devmap_handle_t *)NULL);
2785 2785 }
2786 2786
2787 2787 /*
2788 2788 * devmap_unload:
2789 2789 * Marks a segdev segment or pages if offset->offset+len
2790 2790 * is not the entire segment as intercept and unloads the
2791 2791 * pages in the range offset -> offset+len.
2792 2792 */
2793 2793 int
2794 2794 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2795 2795 {
2796 2796 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2797 2797 caddr_t addr;
2798 2798 ulong_t size;
2799 2799 ssize_t soff;
2800 2800
2801 2801 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2802 2802 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2803 2803 (void *)dhp, offset, len);
2804 2804 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2805 2805 (void *)dhp, offset, len));
2806 2806
2807 2807 soff = (ssize_t)(offset - dhp->dh_uoff);
2808 2808 soff = round_down_p2(soff, PAGESIZE);
2809 2809 if (soff < 0 || soff >= dhp->dh_len)
2810 2810 return (FC_MAKE_ERR(EINVAL));
2811 2811
2812 2812 /*
2813 2813 * Address and size must be page aligned. Len is set to the
2814 2814 * number of bytes in the number of pages that are required to
2815 2815 * support len. Offset is set to the byte offset of the first byte
2816 2816 * of the page that contains offset.
2817 2817 */
2818 2818 len = round_up_p2(len, PAGESIZE);
2819 2819
2820 2820 /*
2821 2821 * If len is == 0, then calculate the size by getting
2822 2822 * the number of bytes from offset to the end of the segment.
2823 2823 */
2824 2824 if (len == 0)
2825 2825 size = dhp->dh_len - soff;
2826 2826 else {
2827 2827 size = len;
2828 2828 if ((soff + size) > dhp->dh_len)
2829 2829 return (FC_MAKE_ERR(EINVAL));
2830 2830 }
2831 2831
2832 2832 /*
2833 2833 * The address is offset bytes from the base address of
2834 2834 * the dhp.
2835 2835 */
2836 2836 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2837 2837
2838 2838 /*
2839 2839 * If large page size was used in hat_devload(),
2840 2840 * the same page size must be used in hat_unload().
2841 2841 */
2842 2842 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2843 2843 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2844 2844 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2845 2845 } else {
2846 2846 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2847 2847 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2848 2848 }
2849 2849
2850 2850 return (0);
2851 2851 }
2852 2852
2853 2853 /*
2854 2854 * calculates the optimal page size that will be used for hat_devload().
2855 2855 */
2856 2856 static void
2857 2857 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2858 2858 size_t *llen, caddr_t *laddr)
2859 2859 {
2860 2860 ulong_t off;
2861 2861 ulong_t pfn;
2862 2862 ulong_t pgsize;
2863 2863 uint_t first = 1;
2864 2864
2865 2865 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2866 2866 "devmap_get_large_pgsize:start");
2867 2867
2868 2868 /*
2869 2869 * RFE - Code only supports large page mappings for devmem
2870 2870 * This code could be changed in future if we want to support
2871 2871 * large page mappings for kernel exported memory.
2872 2872 */
2873 2873 ASSERT(dhp_is_devmem(dhp));
2874 2874 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2875 2875
2876 2876 *llen = 0;
2877 2877 off = (ulong_t)(addr - dhp->dh_uvaddr);
2878 2878 while ((long)len > 0) {
2879 2879 /*
2880 2880 * get the optimal pfn to minimize address translations.
2881 2881 * devmap_roundup() returns residue bytes for next round
2882 2882 * calculations.
2883 2883 */
2884 2884 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2885 2885
2886 2886 if (first) {
2887 2887 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2888 2888 first = 0;
2889 2889 }
2890 2890
2891 2891 *llen += pgsize;
2892 2892 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2893 2893 }
2894 2894 /* Large page mapping len/addr cover more range than original fault */
2895 2895 ASSERT(*llen >= len && *laddr <= addr);
2896 2896 ASSERT((*laddr + *llen) >= (addr + len));
2897 2897 }
2898 2898
2899 2899 /*
2900 2900 * Initialize the devmap_softlock structure.
2901 2901 */
2902 2902 static struct devmap_softlock *
2903 2903 devmap_softlock_init(dev_t dev, ulong_t id)
2904 2904 {
2905 2905 struct devmap_softlock *slock;
2906 2906 struct devmap_softlock *tmp;
2907 2907
2908 2908 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2909 2909 "devmap_softlock_init:start");
2910 2910
2911 2911 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2912 2912 mutex_enter(&devmap_slock);
2913 2913
2914 2914 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2915 2915 if ((slock->dev == dev) && (slock->id == id))
2916 2916 break;
2917 2917
2918 2918 if (slock == NULL) {
2919 2919 slock = tmp;
2920 2920 slock->dev = dev;
2921 2921 slock->id = id;
2922 2922 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2923 2923 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2924 2924 slock->next = devmap_slist;
2925 2925 devmap_slist = slock;
2926 2926 } else
2927 2927 kmem_free(tmp, sizeof (struct devmap_softlock));
2928 2928
2929 2929 mutex_enter(&slock->lock);
2930 2930 slock->refcnt++;
2931 2931 mutex_exit(&slock->lock);
2932 2932 mutex_exit(&devmap_slock);
2933 2933
2934 2934 return (slock);
2935 2935 }
2936 2936
2937 2937 /*
2938 2938 * Wake up processes that sleep on softlocked.
2939 2939 * Free dh_softlock if refcnt is 0.
2940 2940 */
2941 2941 static void
2942 2942 devmap_softlock_rele(devmap_handle_t *dhp)
2943 2943 {
2944 2944 struct devmap_softlock *slock = dhp->dh_softlock;
2945 2945 struct devmap_softlock *tmp;
2946 2946 struct devmap_softlock *parent;
2947 2947
2948 2948 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2949 2949 "devmap_softlock_rele:start");
2950 2950
2951 2951 mutex_enter(&devmap_slock);
2952 2952 mutex_enter(&slock->lock);
2953 2953
2954 2954 ASSERT(slock->refcnt > 0);
2955 2955
2956 2956 slock->refcnt--;
2957 2957
2958 2958 /*
2959 2959 * If no one is using the device, free up the slock data.
2960 2960 */
2961 2961 if (slock->refcnt == 0) {
2962 2962 slock->softlocked = 0;
2963 2963 cv_signal(&slock->cv);
2964 2964
2965 2965 if (devmap_slist == slock)
2966 2966 devmap_slist = slock->next;
2967 2967 else {
2968 2968 parent = devmap_slist;
2969 2969 for (tmp = devmap_slist->next; tmp != NULL;
2970 2970 tmp = tmp->next) {
2971 2971 if (tmp == slock) {
2972 2972 parent->next = tmp->next;
2973 2973 break;
2974 2974 }
2975 2975 parent = tmp;
2976 2976 }
2977 2977 }
2978 2978 mutex_exit(&slock->lock);
2979 2979 mutex_destroy(&slock->lock);
2980 2980 cv_destroy(&slock->cv);
2981 2981 kmem_free(slock, sizeof (struct devmap_softlock));
2982 2982 } else
2983 2983 mutex_exit(&slock->lock);
2984 2984
2985 2985 mutex_exit(&devmap_slock);
2986 2986 }
2987 2987
2988 2988 /*
2989 2989 * Wake up processes that sleep on dh_ctx->locked.
2990 2990 * Free dh_ctx if refcnt is 0.
2991 2991 */
2992 2992 static void
2993 2993 devmap_ctx_rele(devmap_handle_t *dhp)
2994 2994 {
2995 2995 struct devmap_ctx *devctx = dhp->dh_ctx;
2996 2996 struct devmap_ctx *tmp;
2997 2997 struct devmap_ctx *parent;
2998 2998 timeout_id_t tid;
2999 2999
3000 3000 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3001 3001 "devmap_ctx_rele:start");
3002 3002
3003 3003 mutex_enter(&devmapctx_lock);
3004 3004 mutex_enter(&devctx->lock);
3005 3005
3006 3006 ASSERT(devctx->refcnt > 0);
3007 3007
3008 3008 devctx->refcnt--;
3009 3009
3010 3010 /*
3011 3011 * If no one is using the device, free up the devctx data.
3012 3012 */
3013 3013 if (devctx->refcnt == 0) {
3014 3014 /*
3015 3015 * Untimeout any threads using this mapping as they are about
3016 3016 * to go away.
3017 3017 */
3018 3018 if (devctx->timeout != 0) {
3019 3019 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3020 3020 "devmap_ctx_rele:untimeout ctx->timeout");
3021 3021
3022 3022 tid = devctx->timeout;
3023 3023 mutex_exit(&devctx->lock);
3024 3024 (void) untimeout(tid);
3025 3025 mutex_enter(&devctx->lock);
3026 3026 }
3027 3027
3028 3028 devctx->oncpu = 0;
3029 3029 cv_signal(&devctx->cv);
3030 3030
3031 3031 if (devmapctx_list == devctx)
3032 3032 devmapctx_list = devctx->next;
3033 3033 else {
3034 3034 parent = devmapctx_list;
3035 3035 for (tmp = devmapctx_list->next; tmp != NULL;
3036 3036 tmp = tmp->next) {
3037 3037 if (tmp == devctx) {
3038 3038 parent->next = tmp->next;
3039 3039 break;
3040 3040 }
3041 3041 parent = tmp;
3042 3042 }
3043 3043 }
3044 3044 mutex_exit(&devctx->lock);
3045 3045 mutex_destroy(&devctx->lock);
3046 3046 cv_destroy(&devctx->cv);
3047 3047 kmem_free(devctx, sizeof (struct devmap_ctx));
3048 3048 } else
3049 3049 mutex_exit(&devctx->lock);
3050 3050
3051 3051 mutex_exit(&devmapctx_lock);
3052 3052 }
3053 3053
3054 3054 /*
3055 3055 * devmap_load:
3056 3056 * Marks a segdev segment or pages if offset->offset+len
3057 3057 * is not the entire segment as nointercept and faults in
3058 3058 * the pages in the range offset -> offset+len.
3059 3059 */
3060 3060 int
3061 3061 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3062 3062 uint_t rw)
3063 3063 {
3064 3064 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3065 3065 struct as *asp = dhp->dh_seg->s_as;
3066 3066 caddr_t addr;
3067 3067 ulong_t size;
3068 3068 ssize_t soff; /* offset from the beginning of the segment */
3069 3069 int rc;
3070 3070
3071 3071 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3072 3072 "devmap_load:start dhp=%p offset=%llx len=%lx",
↓ open down ↓ |
684 lines elided |
↑ open up ↑ |
3073 3073 (void *)dhp, offset, len);
3074 3074
3075 3075 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3076 3076 (void *)dhp, offset, len));
3077 3077
3078 3078 /*
3079 3079 * Hat layer only supports devload to process' context for which
3080 3080 * the as lock is held. Verify here and return error if drivers
3081 3081 * inadvertently call devmap_load on a wrong devmap handle.
3082 3082 */
3083 - if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3083 + if ((asp != &kas) && !AS_LOCK_HELD(asp))
3084 3084 return (FC_MAKE_ERR(EINVAL));
3085 3085
3086 3086 soff = (ssize_t)(offset - dhp->dh_uoff);
3087 3087 soff = round_down_p2(soff, PAGESIZE);
3088 3088 if (soff < 0 || soff >= dhp->dh_len)
3089 3089 return (FC_MAKE_ERR(EINVAL));
3090 3090
3091 3091 /*
3092 3092 * Address and size must be page aligned. Len is set to the
3093 3093 * number of bytes in the number of pages that are required to
3094 3094 * support len. Offset is set to the byte offset of the first byte
3095 3095 * of the page that contains offset.
3096 3096 */
3097 3097 len = round_up_p2(len, PAGESIZE);
3098 3098
3099 3099 /*
3100 3100 * If len == 0, then calculate the size by getting
3101 3101 * the number of bytes from offset to the end of the segment.
3102 3102 */
3103 3103 if (len == 0)
3104 3104 size = dhp->dh_len - soff;
3105 3105 else {
3106 3106 size = len;
3107 3107 if ((soff + size) > dhp->dh_len)
3108 3108 return (FC_MAKE_ERR(EINVAL));
3109 3109 }
3110 3110
3111 3111 /*
3112 3112 * The address is offset bytes from the base address of
3113 3113 * the segment.
3114 3114 */
3115 3115 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3116 3116
3117 3117 HOLD_DHP_LOCK(dhp);
3118 3118 rc = segdev_faultpages(asp->a_hat,
3119 3119 dhp->dh_seg, addr, size, type, rw, dhp);
3120 3120 RELE_DHP_LOCK(dhp);
3121 3121 return (rc);
3122 3122 }
3123 3123
3124 3124 int
3125 3125 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3126 3126 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3127 3127 {
3128 3128 register devmap_handle_t *dhp;
3129 3129 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3130 3130 size_t *, uint_t);
3131 3131 int (*mmap)(dev_t, off_t, int);
3132 3132 struct devmap_callback_ctl *callbackops;
3133 3133 devmap_handle_t *dhp_head = NULL;
3134 3134 devmap_handle_t *dhp_prev = NULL;
3135 3135 devmap_handle_t *dhp_curr;
3136 3136 caddr_t addr;
3137 3137 int map_flag;
3138 3138 int ret;
3139 3139 ulong_t total_len;
3140 3140 size_t map_len;
3141 3141 size_t resid_len = len;
3142 3142 offset_t map_off = off;
3143 3143 struct devmap_softlock *slock = NULL;
3144 3144
3145 3145 #ifdef lint
3146 3146 cred = cred;
3147 3147 #endif
3148 3148
3149 3149 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3150 3150 "devmap_setup:start off=%llx len=%lx", off, len);
3151 3151 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3152 3152 off, len));
3153 3153
3154 3154 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3155 3155 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3156 3156
3157 3157 /*
3158 3158 * driver must provide devmap(9E) entry point in cb_ops to use the
3159 3159 * devmap framework.
3160 3160 */
3161 3161 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3162 3162 return (EINVAL);
3163 3163
3164 3164 /*
3165 3165 * To protect from an inadvertent entry because the devmap entry point
3166 3166 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3167 3167 * mmap is NULL.
3168 3168 */
3169 3169 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3170 3170 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3171 3171 return (EINVAL);
3172 3172
3173 3173 /*
3174 3174 * devmap allows mmap(2) to map multiple registers.
3175 3175 * one devmap_handle is created for each register mapped.
3176 3176 */
3177 3177 for (total_len = 0; total_len < len; total_len += map_len) {
3178 3178 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3179 3179
3180 3180 if (dhp_prev != NULL)
3181 3181 dhp_prev->dh_next = dhp;
3182 3182 else
3183 3183 dhp_head = dhp;
3184 3184 dhp_prev = dhp;
3185 3185
3186 3186 dhp->dh_prot = prot;
3187 3187 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3188 3188 dhp->dh_dev = dev;
3189 3189 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3190 3190 dhp->dh_uoff = map_off;
3191 3191
3192 3192 /*
3193 3193 * Get mapping specific info from
3194 3194 * the driver, such as rnumber, roff, len, callbackops,
3195 3195 * accattrp and, if the mapping is for kernel memory,
3196 3196 * ddi_umem_cookie.
3197 3197 */
3198 3198 if ((ret = cdev_devmap(dev, dhp, map_off,
3199 3199 resid_len, &map_len, get_udatamodel())) != 0) {
3200 3200 free_devmap_handle(dhp_head);
3201 3201 return (ENXIO);
3202 3202 }
3203 3203
3204 3204 if (map_len & PAGEOFFSET) {
3205 3205 free_devmap_handle(dhp_head);
3206 3206 return (EINVAL);
3207 3207 }
3208 3208
3209 3209 callbackops = &dhp->dh_callbackops;
3210 3210
3211 3211 if ((callbackops->devmap_access == NULL) ||
3212 3212 (callbackops->devmap_access == nulldev) ||
3213 3213 (callbackops->devmap_access == nodev)) {
3214 3214 /*
3215 3215 * Normally devmap does not support MAP_PRIVATE unless
3216 3216 * the drivers provide a valid devmap_access routine.
3217 3217 */
3218 3218 if ((flags & MAP_PRIVATE) != 0) {
3219 3219 free_devmap_handle(dhp_head);
3220 3220 return (EINVAL);
3221 3221 }
3222 3222 } else {
3223 3223 /*
3224 3224 * Initialize dhp_softlock and dh_ctx if the drivers
3225 3225 * provide devmap_access.
3226 3226 */
3227 3227 dhp->dh_softlock = devmap_softlock_init(dev,
3228 3228 (ulong_t)callbackops->devmap_access);
3229 3229 dhp->dh_ctx = devmap_ctxinit(dev,
3230 3230 (ulong_t)callbackops->devmap_access);
3231 3231
3232 3232 /*
3233 3233 * segdev_fault can only work when all
3234 3234 * dh_softlock in a multi-dhp mapping
3235 3235 * are same. see comments in segdev_fault
3236 3236 * This code keeps track of the first
3237 3237 * dh_softlock allocated in slock and
3238 3238 * compares all later allocations and if
3239 3239 * not similar, returns an error.
3240 3240 */
3241 3241 if (slock == NULL)
3242 3242 slock = dhp->dh_softlock;
3243 3243 if (slock != dhp->dh_softlock) {
3244 3244 free_devmap_handle(dhp_head);
3245 3245 return (ENOTSUP);
3246 3246 }
3247 3247 }
3248 3248
3249 3249 map_off += map_len;
3250 3250 resid_len -= map_len;
3251 3251 }
3252 3252
3253 3253 /*
3254 3254 * get the user virtual address and establish the mapping between
3255 3255 * uvaddr and device physical address.
3256 3256 */
3257 3257 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3258 3258 != 0) {
3259 3259 /*
3260 3260 * free devmap handles if error during the mapping.
3261 3261 */
3262 3262 free_devmap_handle(dhp_head);
3263 3263
3264 3264 return (ret);
3265 3265 }
3266 3266
3267 3267 /*
3268 3268 * call the driver's devmap_map callback to do more after the mapping,
3269 3269 * such as to allocate driver private data for context management.
3270 3270 */
3271 3271 dhp = dhp_head;
3272 3272 map_off = off;
3273 3273 addr = *addrp;
3274 3274 while (dhp != NULL) {
3275 3275 callbackops = &dhp->dh_callbackops;
3276 3276 dhp->dh_uvaddr = addr;
3277 3277 dhp_curr = dhp;
3278 3278 if (callbackops->devmap_map != NULL) {
3279 3279 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3280 3280 dev, flags, map_off,
3281 3281 dhp->dh_len, &dhp->dh_pvtp);
3282 3282 if (ret != 0) {
3283 3283 struct segdev_data *sdp;
3284 3284
3285 3285 /*
3286 3286 * call driver's devmap_unmap entry point
3287 3287 * to free driver resources.
3288 3288 */
3289 3289 dhp = dhp_head;
3290 3290 map_off = off;
3291 3291 while (dhp != dhp_curr) {
3292 3292 callbackops = &dhp->dh_callbackops;
3293 3293 if (callbackops->devmap_unmap != NULL) {
3294 3294 (*callbackops->devmap_unmap)(
3295 3295 dhp, dhp->dh_pvtp,
3296 3296 map_off, dhp->dh_len,
3297 3297 NULL, NULL, NULL, NULL);
3298 3298 }
3299 3299 map_off += dhp->dh_len;
3300 3300 dhp = dhp->dh_next;
3301 3301 }
3302 3302 sdp = dhp_head->dh_seg->s_data;
3303 3303 sdp->devmap_data = NULL;
3304 3304 free_devmap_handle(dhp_head);
3305 3305 return (ENXIO);
3306 3306 }
3307 3307 }
3308 3308 map_off += dhp->dh_len;
3309 3309 addr += dhp->dh_len;
3310 3310 dhp = dhp->dh_next;
3311 3311 }
3312 3312
3313 3313 return (0);
3314 3314 }
3315 3315
3316 3316 int
3317 3317 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3318 3318 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3319 3319 {
3320 3320 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3321 3321 "devmap_segmap:start");
3322 3322 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3323 3323 (size_t)len, prot, maxprot, flags, cred));
3324 3324 }
3325 3325
3326 3326 /*
3327 3327 * Called from devmap_devmem_setup/remap to see if can use large pages for
3328 3328 * this device mapping.
3329 3329 * Also calculate the max. page size for this mapping.
3330 3330 * this page size will be used in fault routine for
3331 3331 * optimal page size calculations.
3332 3332 */
3333 3333 static void
3334 3334 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3335 3335 {
3336 3336 ASSERT(dhp_is_devmem(dhp));
3337 3337 dhp->dh_mmulevel = 0;
3338 3338
3339 3339 /*
3340 3340 * use large page size only if:
3341 3341 * 1. device memory.
3342 3342 * 2. mmu supports multiple page sizes,
3343 3343 * 3. Driver did not disallow it
3344 3344 * 4. dhp length is at least as big as the large pagesize
3345 3345 * 5. the uvaddr and pfn are large pagesize aligned
3346 3346 */
3347 3347 if (page_num_pagesizes() > 1 &&
3348 3348 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3349 3349 ulong_t base;
3350 3350 int level;
3351 3351
3352 3352 base = (ulong_t)ptob(dhp->dh_pfn);
3353 3353 for (level = 1; level < page_num_pagesizes(); level++) {
3354 3354 size_t pgsize = page_get_pagesize(level);
3355 3355 if ((dhp->dh_len < pgsize) ||
3356 3356 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3357 3357 base, pgsize))) {
3358 3358 break;
3359 3359 }
3360 3360 }
3361 3361 dhp->dh_mmulevel = level - 1;
3362 3362 }
3363 3363 if (dhp->dh_mmulevel > 0) {
3364 3364 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3365 3365 } else {
3366 3366 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3367 3367 }
3368 3368 }
3369 3369
3370 3370 /*
3371 3371 * Called by driver devmap routine to pass device specific info to
3372 3372 * the framework. used for device memory mapping only.
3373 3373 */
3374 3374 int
3375 3375 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3376 3376 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3377 3377 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3378 3378 {
3379 3379 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3380 3380 ddi_acc_handle_t handle;
3381 3381 ddi_map_req_t mr;
3382 3382 ddi_acc_hdl_t *hp;
3383 3383 int err;
3384 3384
3385 3385 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3386 3386 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3387 3387 (void *)dhp, roff, rnumber, (uint_t)len);
3388 3388 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3389 3389 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3390 3390
3391 3391 /*
3392 3392 * First to check if this function has been called for this dhp.
3393 3393 */
3394 3394 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3395 3395 return (DDI_FAILURE);
3396 3396
3397 3397 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3398 3398 return (DDI_FAILURE);
3399 3399
3400 3400 if (flags & DEVMAP_MAPPING_INVALID) {
3401 3401 /*
3402 3402 * Don't go up the tree to get pfn if the driver specifies
3403 3403 * DEVMAP_MAPPING_INVALID in flags.
3404 3404 *
3405 3405 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3406 3406 * remap permission.
3407 3407 */
3408 3408 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3409 3409 return (DDI_FAILURE);
3410 3410 }
3411 3411 dhp->dh_pfn = PFN_INVALID;
3412 3412 } else {
3413 3413 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3414 3414 if (handle == NULL)
3415 3415 return (DDI_FAILURE);
3416 3416
3417 3417 hp = impl_acc_hdl_get(handle);
3418 3418 hp->ah_vers = VERS_ACCHDL;
3419 3419 hp->ah_dip = dip;
3420 3420 hp->ah_rnumber = rnumber;
3421 3421 hp->ah_offset = roff;
3422 3422 hp->ah_len = len;
3423 3423 if (accattrp != NULL)
3424 3424 hp->ah_acc = *accattrp;
3425 3425
3426 3426 mr.map_op = DDI_MO_MAP_LOCKED;
3427 3427 mr.map_type = DDI_MT_RNUMBER;
3428 3428 mr.map_obj.rnumber = rnumber;
3429 3429 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3430 3430 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3431 3431 mr.map_handlep = hp;
3432 3432 mr.map_vers = DDI_MAP_VERSION;
3433 3433
3434 3434 /*
3435 3435 * up the device tree to get pfn.
3436 3436 * The rootnex_map_regspec() routine in nexus drivers has been
3437 3437 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3438 3438 */
3439 3439 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3440 3440 dhp->dh_hat_attr = hp->ah_hat_flags;
3441 3441 impl_acc_hdl_free(handle);
3442 3442
3443 3443 if (err)
3444 3444 return (DDI_FAILURE);
3445 3445 }
3446 3446 /* Should not be using devmem setup for memory pages */
3447 3447 ASSERT(!pf_is_memory(dhp->dh_pfn));
3448 3448
3449 3449 /* Only some of the flags bits are settable by the driver */
3450 3450 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3451 3451 dhp->dh_len = ptob(btopr(len));
3452 3452
3453 3453 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3454 3454 dhp->dh_roff = ptob(btop(roff));
3455 3455
3456 3456 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3457 3457 devmap_devmem_large_page_setup(dhp);
3458 3458 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3459 3459 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3460 3460
3461 3461
3462 3462 if (callbackops != NULL) {
3463 3463 bcopy(callbackops, &dhp->dh_callbackops,
3464 3464 sizeof (struct devmap_callback_ctl));
3465 3465 }
3466 3466
3467 3467 /*
3468 3468 * Initialize dh_lock if we want to do remap.
3469 3469 */
3470 3470 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3471 3471 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3472 3472 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3473 3473 }
3474 3474
3475 3475 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3476 3476
3477 3477 return (DDI_SUCCESS);
3478 3478 }
3479 3479
3480 3480 int
3481 3481 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3482 3482 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3483 3483 uint_t flags, ddi_device_acc_attr_t *accattrp)
3484 3484 {
3485 3485 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3486 3486 ddi_acc_handle_t handle;
3487 3487 ddi_map_req_t mr;
3488 3488 ddi_acc_hdl_t *hp;
3489 3489 pfn_t pfn;
3490 3490 uint_t hat_flags;
3491 3491 int err;
3492 3492
3493 3493 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3494 3494 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3495 3495 (void *)dhp, roff, rnumber, (uint_t)len);
3496 3496 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3497 3497 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3498 3498
3499 3499 /*
3500 3500 * Return failure if setup has not been done or no remap permission
3501 3501 * has been granted during the setup.
3502 3502 */
3503 3503 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3504 3504 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3505 3505 return (DDI_FAILURE);
3506 3506
3507 3507 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3508 3508 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3509 3509 return (DDI_FAILURE);
3510 3510
3511 3511 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3512 3512 return (DDI_FAILURE);
3513 3513
3514 3514 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3515 3515 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3516 3516 if (handle == NULL)
3517 3517 return (DDI_FAILURE);
3518 3518 }
3519 3519
3520 3520 HOLD_DHP_LOCK(dhp);
3521 3521
3522 3522 /*
3523 3523 * Unload the old mapping, so next fault will setup the new mappings
3524 3524 * Do this while holding the dhp lock so other faults dont reestablish
3525 3525 * the mappings
3526 3526 */
3527 3527 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3528 3528 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3529 3529
3530 3530 if (flags & DEVMAP_MAPPING_INVALID) {
3531 3531 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3532 3532 dhp->dh_pfn = PFN_INVALID;
3533 3533 } else {
3534 3534 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3535 3535 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3536 3536 hp = impl_acc_hdl_get(handle);
3537 3537 hp->ah_vers = VERS_ACCHDL;
3538 3538 hp->ah_dip = dip;
3539 3539 hp->ah_rnumber = rnumber;
3540 3540 hp->ah_offset = roff;
3541 3541 hp->ah_len = len;
3542 3542 if (accattrp != NULL)
3543 3543 hp->ah_acc = *accattrp;
3544 3544
3545 3545 mr.map_op = DDI_MO_MAP_LOCKED;
3546 3546 mr.map_type = DDI_MT_RNUMBER;
3547 3547 mr.map_obj.rnumber = rnumber;
3548 3548 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3549 3549 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3550 3550 mr.map_handlep = hp;
3551 3551 mr.map_vers = DDI_MAP_VERSION;
3552 3552
3553 3553 /*
3554 3554 * up the device tree to get pfn.
3555 3555 * The rootnex_map_regspec() routine in nexus drivers has been
3556 3556 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3557 3557 */
3558 3558 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3559 3559 hat_flags = hp->ah_hat_flags;
3560 3560 impl_acc_hdl_free(handle);
3561 3561 if (err) {
3562 3562 RELE_DHP_LOCK(dhp);
3563 3563 return (DDI_FAILURE);
3564 3564 }
3565 3565 /*
3566 3566 * Store result of ddi_map first in local variables, as we do
3567 3567 * not want to overwrite the existing dhp with wrong data.
3568 3568 */
3569 3569 dhp->dh_pfn = pfn;
3570 3570 dhp->dh_hat_attr = hat_flags;
3571 3571 }
3572 3572
3573 3573 /* clear the large page size flag */
3574 3574 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3575 3575
3576 3576 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3577 3577 dhp->dh_roff = ptob(btop(roff));
3578 3578
3579 3579 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3580 3580 devmap_devmem_large_page_setup(dhp);
3581 3581 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3582 3582 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3583 3583
3584 3584 RELE_DHP_LOCK(dhp);
3585 3585 return (DDI_SUCCESS);
3586 3586 }
3587 3587
3588 3588 /*
3589 3589 * called by driver devmap routine to pass kernel virtual address mapping
3590 3590 * info to the framework. used only for kernel memory
3591 3591 * allocated from ddi_umem_alloc().
3592 3592 */
3593 3593 int
3594 3594 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3595 3595 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3596 3596 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3597 3597 ddi_device_acc_attr_t *accattrp)
3598 3598 {
3599 3599 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3600 3600 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3601 3601
3602 3602 #ifdef lint
3603 3603 dip = dip;
3604 3604 #endif
3605 3605
3606 3606 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3607 3607 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3608 3608 (void *)dhp, off, cookie, len);
3609 3609 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3610 3610 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3611 3611
3612 3612 if (cookie == NULL)
3613 3613 return (DDI_FAILURE);
3614 3614
3615 3615 /* For UMEM_TRASH, this restriction is not needed */
3616 3616 if ((off + len) > cp->size)
3617 3617 return (DDI_FAILURE);
3618 3618
3619 3619 /* check if the cache attributes are supported */
3620 3620 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3621 3621 return (DDI_FAILURE);
3622 3622
3623 3623 /*
3624 3624 * First to check if this function has been called for this dhp.
3625 3625 */
3626 3626 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3627 3627 return (DDI_FAILURE);
3628 3628
3629 3629 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3630 3630 return (DDI_FAILURE);
3631 3631
3632 3632 if (flags & DEVMAP_MAPPING_INVALID) {
3633 3633 /*
3634 3634 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3635 3635 * remap permission.
3636 3636 */
3637 3637 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3638 3638 return (DDI_FAILURE);
3639 3639 }
3640 3640 } else {
3641 3641 dhp->dh_cookie = cookie;
3642 3642 dhp->dh_roff = ptob(btop(off));
3643 3643 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3644 3644 /* set HAT cache attributes */
3645 3645 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3646 3646 /* set HAT endianess attributes */
3647 3647 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3648 3648 }
3649 3649
3650 3650 /*
3651 3651 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3652 3652 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3653 3653 * create consistent mappings but our intention was to create
3654 3654 * non-consistent mappings.
3655 3655 *
3656 3656 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3657 3657 * mappings.
3658 3658 *
3659 3659 * kernel exported memory: hat figures it out it's memory and always
3660 3660 * creates consistent mappings.
3661 3661 *
3662 3662 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3663 3663 *
3664 3664 * /dev/kmem: consistent mappings are created unless they are
3665 3665 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3666 3666 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3667 3667 * mappings of /dev/kmem. See common/io/mem.c
3668 3668 */
3669 3669
3670 3670 /* Only some of the flags bits are settable by the driver */
3671 3671 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3672 3672
3673 3673 dhp->dh_len = ptob(btopr(len));
3674 3674 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3675 3675 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3676 3676
3677 3677 if (callbackops != NULL) {
3678 3678 bcopy(callbackops, &dhp->dh_callbackops,
3679 3679 sizeof (struct devmap_callback_ctl));
3680 3680 }
3681 3681 /*
3682 3682 * Initialize dh_lock if we want to do remap.
3683 3683 */
3684 3684 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3685 3685 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3686 3686 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3687 3687 }
3688 3688
3689 3689 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3690 3690
3691 3691 return (DDI_SUCCESS);
3692 3692 }
3693 3693
3694 3694 int
3695 3695 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3696 3696 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3697 3697 uint_t flags, ddi_device_acc_attr_t *accattrp)
3698 3698 {
3699 3699 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3700 3700 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3701 3701
3702 3702 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3703 3703 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3704 3704 (void *)dhp, off, cookie, len);
3705 3705 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3706 3706 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3707 3707
3708 3708 #ifdef lint
3709 3709 dip = dip;
3710 3710 accattrp = accattrp;
3711 3711 #endif
3712 3712 /*
3713 3713 * Reture failure if setup has not been done or no remap permission
3714 3714 * has been granted during the setup.
3715 3715 */
3716 3716 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3717 3717 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3718 3718 return (DDI_FAILURE);
3719 3719
3720 3720 /* No flags supported for remap yet */
3721 3721 if (flags != 0)
3722 3722 return (DDI_FAILURE);
3723 3723
3724 3724 /* check if the cache attributes are supported */
3725 3725 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3726 3726 return (DDI_FAILURE);
3727 3727
3728 3728 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3729 3729 return (DDI_FAILURE);
3730 3730
3731 3731 /* For UMEM_TRASH, this restriction is not needed */
3732 3732 if ((off + len) > cp->size)
3733 3733 return (DDI_FAILURE);
3734 3734
3735 3735 HOLD_DHP_LOCK(dhp);
3736 3736 /*
3737 3737 * Unload the old mapping, so next fault will setup the new mappings
3738 3738 * Do this while holding the dhp lock so other faults dont reestablish
3739 3739 * the mappings
3740 3740 */
3741 3741 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3742 3742 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3743 3743
3744 3744 dhp->dh_cookie = cookie;
3745 3745 dhp->dh_roff = ptob(btop(off));
3746 3746 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3747 3747 /* set HAT cache attributes */
3748 3748 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3749 3749 /* set HAT endianess attributes */
3750 3750 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3751 3751
3752 3752 /* clear the large page size flag */
3753 3753 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3754 3754
3755 3755 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3756 3756 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3757 3757 RELE_DHP_LOCK(dhp);
3758 3758 return (DDI_SUCCESS);
3759 3759 }
3760 3760
3761 3761 /*
3762 3762 * to set timeout value for the driver's context management callback, e.g.
3763 3763 * devmap_access().
3764 3764 */
3765 3765 void
3766 3766 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3767 3767 {
3768 3768 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3769 3769
3770 3770 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3771 3771 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3772 3772 (void *)dhp, ticks);
3773 3773 dhp->dh_timeout_length = ticks;
3774 3774 }
3775 3775
3776 3776 int
3777 3777 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3778 3778 size_t len, uint_t type, uint_t rw)
3779 3779 {
3780 3780 #ifdef lint
3781 3781 pvtp = pvtp;
3782 3782 #endif
3783 3783
3784 3784 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3785 3785 "devmap_default_access:start");
3786 3786 return (devmap_load(dhp, off, len, type, rw));
3787 3787 }
3788 3788
3789 3789 /*
3790 3790 * segkmem_alloc() wrapper to allocate memory which is both
3791 3791 * non-relocatable (for DR) and sharelocked, since the rest
3792 3792 * of this segment driver requires it.
3793 3793 */
3794 3794 static void *
3795 3795 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3796 3796 {
3797 3797 ASSERT(vmp != NULL);
3798 3798 ASSERT(kvseg.s_base != NULL);
3799 3799 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3800 3800 return (segkmem_alloc(vmp, size, vmflag));
3801 3801 }
3802 3802
3803 3803 /*
3804 3804 * This is where things are a bit incestuous with seg_kmem: unlike
3805 3805 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3806 3806 * we need to do a bit of a dance around that to prevent duplication of
3807 3807 * code until we decide to bite the bullet and implement a new kernel
3808 3808 * segment for driver-allocated memory that is exported to user space.
3809 3809 */
3810 3810 static void
3811 3811 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3812 3812 {
3813 3813 page_t *pp;
3814 3814 caddr_t addr = inaddr;
3815 3815 caddr_t eaddr;
3816 3816 pgcnt_t npages = btopr(size);
3817 3817
3818 3818 ASSERT(vmp != NULL);
3819 3819 ASSERT(kvseg.s_base != NULL);
3820 3820 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3821 3821
3822 3822 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3823 3823
3824 3824 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3825 3825 /*
3826 3826 * Use page_find() instead of page_lookup() to find the page
3827 3827 * since we know that it is hashed and has a shared lock.
3828 3828 */
3829 3829 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3830 3830
3831 3831 if (pp == NULL)
3832 3832 panic("devmap_free_pages: page not found");
3833 3833 if (!page_tryupgrade(pp)) {
3834 3834 page_unlock(pp);
3835 3835 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3836 3836 SE_EXCL);
3837 3837 if (pp == NULL)
3838 3838 panic("devmap_free_pages: page already freed");
3839 3839 }
3840 3840 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3841 3841 pp->p_lckcnt = 0;
3842 3842 page_destroy(pp, 0);
3843 3843 }
3844 3844 page_unresv(npages);
3845 3845
3846 3846 if (vmp != NULL)
3847 3847 vmem_free(vmp, inaddr, size);
3848 3848 }
3849 3849
3850 3850 /*
3851 3851 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3852 3852 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3853 3853 * default request. For now we allocate our own pages and we keep
3854 3854 * them long-term sharelocked, since: A) the fault routines expect the
3855 3855 * memory to already be locked; B) pageable umem is already long-term
3856 3856 * locked; C) it's a lot of work to make it otherwise, particularly
3857 3857 * since the nexus layer expects the pages to never fault. An RFE is to
3858 3858 * not keep the pages long-term locked, but instead to be able to
3859 3859 * take faults on them and simply look them up in kvp in case we
3860 3860 * fault on them. Even then, we must take care not to let pageout
3861 3861 * steal them from us since the data must remain resident; if we
3862 3862 * do this we must come up with some way to pin the pages to prevent
3863 3863 * faults while a driver is doing DMA to/from them.
3864 3864 */
3865 3865 static void *
3866 3866 devmap_umem_alloc_np(size_t size, size_t flags)
3867 3867 {
3868 3868 void *buf;
3869 3869 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3870 3870
3871 3871 buf = vmem_alloc(umem_np_arena, size, vmflags);
3872 3872 if (buf != NULL)
3873 3873 bzero(buf, size);
3874 3874 return (buf);
3875 3875 }
3876 3876
3877 3877 static void
3878 3878 devmap_umem_free_np(void *addr, size_t size)
3879 3879 {
3880 3880 vmem_free(umem_np_arena, addr, size);
3881 3881 }
3882 3882
3883 3883 /*
3884 3884 * allocate page aligned kernel memory for exporting to user land.
3885 3885 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3886 3886 * to find a user virtual address that is in same color as the address
3887 3887 * allocated here.
3888 3888 */
3889 3889 void *
3890 3890 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3891 3891 {
3892 3892 register size_t len = ptob(btopr(size));
3893 3893 void *buf = NULL;
3894 3894 struct ddi_umem_cookie *cp;
3895 3895 int iflags = 0;
3896 3896
3897 3897 *cookie = NULL;
3898 3898
3899 3899 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3900 3900 "devmap_umem_alloc:start");
3901 3901 if (len == 0)
3902 3902 return ((void *)NULL);
3903 3903
3904 3904 /*
3905 3905 * allocate cookie
3906 3906 */
3907 3907 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3908 3908 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3909 3909 ASSERT(flags & DDI_UMEM_NOSLEEP);
3910 3910 return ((void *)NULL);
3911 3911 }
3912 3912
3913 3913 if (flags & DDI_UMEM_PAGEABLE) {
3914 3914 /* Only one of the flags is allowed */
3915 3915 ASSERT(!(flags & DDI_UMEM_TRASH));
3916 3916 /* initialize resource with 0 */
3917 3917 iflags = KPD_ZERO;
3918 3918
3919 3919 /*
3920 3920 * to allocate unlocked pageable memory, use segkp_get() to
3921 3921 * create a segkp segment. Since segkp can only service kas,
3922 3922 * other segment drivers such as segdev have to do
3923 3923 * as_fault(segkp, SOFTLOCK) in its fault routine,
3924 3924 */
3925 3925 if (flags & DDI_UMEM_NOSLEEP)
3926 3926 iflags |= KPD_NOWAIT;
3927 3927
3928 3928 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3929 3929 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3930 3930 return ((void *)NULL);
3931 3931 }
3932 3932 cp->type = KMEM_PAGEABLE;
3933 3933 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3934 3934 cp->locked = 0;
3935 3935 } else if (flags & DDI_UMEM_TRASH) {
3936 3936 /* Only one of the flags is allowed */
3937 3937 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3938 3938 cp->type = UMEM_TRASH;
3939 3939 buf = NULL;
3940 3940 } else {
3941 3941 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3942 3942 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3943 3943 return ((void *)NULL);
3944 3944 }
3945 3945
3946 3946 cp->type = KMEM_NON_PAGEABLE;
3947 3947 }
3948 3948
3949 3949 /*
3950 3950 * need to save size here. size will be used when
3951 3951 * we do kmem_free.
3952 3952 */
3953 3953 cp->size = len;
3954 3954 cp->cvaddr = (caddr_t)buf;
3955 3955
3956 3956 *cookie = (void *)cp;
3957 3957 return (buf);
3958 3958 }
3959 3959
3960 3960 void
3961 3961 ddi_umem_free(ddi_umem_cookie_t cookie)
3962 3962 {
3963 3963 struct ddi_umem_cookie *cp;
3964 3964
3965 3965 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3966 3966 "devmap_umem_free:start");
3967 3967
3968 3968 /*
3969 3969 * if cookie is NULL, no effects on the system
3970 3970 */
3971 3971 if (cookie == NULL)
3972 3972 return;
3973 3973
3974 3974 cp = (struct ddi_umem_cookie *)cookie;
3975 3975
3976 3976 switch (cp->type) {
3977 3977 case KMEM_PAGEABLE :
3978 3978 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3979 3979 /*
3980 3980 * Check if there are still any pending faults on the cookie
3981 3981 * while the driver is deleting it,
3982 3982 * XXX - could change to an ASSERT but wont catch errant drivers
3983 3983 */
3984 3984 mutex_enter(&cp->lock);
3985 3985 if (cp->locked) {
3986 3986 mutex_exit(&cp->lock);
3987 3987 panic("ddi_umem_free for cookie with pending faults %p",
3988 3988 (void *)cp);
3989 3989 return;
3990 3990 }
3991 3991
3992 3992 segkp_release(segkp, cp->cvaddr);
3993 3993
3994 3994 /*
3995 3995 * release mutex associated with this cookie.
3996 3996 */
3997 3997 mutex_destroy(&cp->lock);
3998 3998 break;
3999 3999 case KMEM_NON_PAGEABLE :
4000 4000 ASSERT(cp->cvaddr != NULL && cp->size != 0);
4001 4001 devmap_umem_free_np(cp->cvaddr, cp->size);
4002 4002 break;
4003 4003 case UMEM_TRASH :
4004 4004 break;
4005 4005 case UMEM_LOCKED :
4006 4006 /* Callers should use ddi_umem_unlock for this type */
4007 4007 ddi_umem_unlock(cookie);
4008 4008 /* Frees the cookie too */
4009 4009 return;
4010 4010 default:
4011 4011 /* panic so we can diagnose the underlying cause */
4012 4012 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4013 4013 cp->type);
4014 4014 }
4015 4015
4016 4016 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4017 4017 }
4018 4018
4019 4019
4020 4020 static int
4021 4021 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4022 4022 {
4023 4023 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4024 4024
4025 4025 /*
4026 4026 * It looks as if it is always mapped shared
4027 4027 */
4028 4028 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4029 4029 "segdev_getmemid:start");
4030 4030 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4031 4031 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4032 4032 return (0);
4033 4033 }
4034 4034
4035 4035 /*ARGSUSED*/
4036 4036 static lgrp_mem_policy_info_t *
4037 4037 segdev_getpolicy(struct seg *seg, caddr_t addr)
4038 4038 {
4039 4039 return (NULL);
4040 4040 }
4041 4041
4042 4042 /*ARGSUSED*/
4043 4043 static int
4044 4044 segdev_capable(struct seg *seg, segcapability_t capability)
4045 4045 {
4046 4046 return (0);
4047 4047 }
4048 4048
4049 4049 /*
4050 4050 * ddi_umem_alloc() non-pageable quantum cache max size.
4051 4051 * This is just a SWAG.
4052 4052 */
4053 4053 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4054 4054
4055 4055 /*
4056 4056 * Initialize seg_dev from boot. This routine sets up the trash page
4057 4057 * and creates the umem_np_arena used to back non-pageable memory
4058 4058 * requests.
4059 4059 */
4060 4060 void
4061 4061 segdev_init(void)
4062 4062 {
4063 4063 struct seg kseg;
4064 4064
4065 4065 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4066 4066 devmap_alloc_pages, devmap_free_pages, heap_arena,
4067 4067 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4068 4068
4069 4069 kseg.s_as = &kas;
4070 4070 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4071 4071 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4072 4072 if (trashpp == NULL)
4073 4073 panic("segdev_init: failed to create trash page");
4074 4074 pagezero(trashpp, 0, PAGESIZE);
4075 4075 page_downgrade(trashpp);
4076 4076 }
4077 4077
4078 4078 /*
4079 4079 * Invoke platform-dependent support routines so that /proc can have
4080 4080 * the platform code deal with curious hardware.
4081 4081 */
4082 4082 int
4083 4083 segdev_copyfrom(struct seg *seg,
4084 4084 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4085 4085 {
4086 4086 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4087 4087 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4088 4088
4089 4089 return (e_ddi_copyfromdev(sp->s_dip,
4090 4090 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4091 4091 }
4092 4092
4093 4093 int
4094 4094 segdev_copyto(struct seg *seg,
4095 4095 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4096 4096 {
4097 4097 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4098 4098 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4099 4099
4100 4100 return (e_ddi_copytodev(sp->s_dip,
4101 4101 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4102 4102 }
↓ open down ↓ |
1009 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX