Print this page
patch SEGOP_SWAPOUT-delete
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/vm/seg_map.c
+++ new/usr/src/uts/common/vm/seg_map.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 27 /* All Rights Reserved */
28 28
29 29 /*
30 30 * Portions of this source code were derived from Berkeley 4.3 BSD
31 31 * under license from the Regents of the University of California.
32 32 */
33 33
34 34 /*
35 35 * VM - generic vnode mapping segment.
36 36 *
37 37 * The segmap driver is used only by the kernel to get faster (than seg_vn)
38 38 * mappings [lower routine overhead; more persistent cache] to random
39 39 * vnode/offsets. Note than the kernel may (and does) use seg_vn as well.
40 40 */
41 41
42 42 #include <sys/types.h>
43 43 #include <sys/t_lock.h>
44 44 #include <sys/param.h>
45 45 #include <sys/sysmacros.h>
46 46 #include <sys/buf.h>
47 47 #include <sys/systm.h>
48 48 #include <sys/vnode.h>
49 49 #include <sys/mman.h>
50 50 #include <sys/errno.h>
51 51 #include <sys/cred.h>
52 52 #include <sys/kmem.h>
53 53 #include <sys/vtrace.h>
54 54 #include <sys/cmn_err.h>
55 55 #include <sys/debug.h>
56 56 #include <sys/thread.h>
57 57 #include <sys/dumphdr.h>
58 58 #include <sys/bitmap.h>
59 59 #include <sys/lgrp.h>
60 60
61 61 #include <vm/seg_kmem.h>
62 62 #include <vm/hat.h>
63 63 #include <vm/as.h>
64 64 #include <vm/seg.h>
65 65 #include <vm/seg_kpm.h>
66 66 #include <vm/seg_map.h>
67 67 #include <vm/page.h>
68 68 #include <vm/pvn.h>
69 69 #include <vm/rm.h>
70 70
71 71 /*
72 72 * Private seg op routines.
73 73 */
74 74 static void segmap_free(struct seg *seg);
75 75 faultcode_t segmap_fault(struct hat *hat, struct seg *seg, caddr_t addr,
76 76 size_t len, enum fault_type type, enum seg_rw rw);
77 77 static faultcode_t segmap_faulta(struct seg *seg, caddr_t addr);
78 78 static int segmap_checkprot(struct seg *seg, caddr_t addr, size_t len,
79 79 uint_t prot);
80 80 static int segmap_kluster(struct seg *seg, caddr_t addr, ssize_t);
81 81 static int segmap_getprot(struct seg *seg, caddr_t addr, size_t len,
82 82 uint_t *protv);
83 83 static u_offset_t segmap_getoffset(struct seg *seg, caddr_t addr);
84 84 static int segmap_gettype(struct seg *seg, caddr_t addr);
85 85 static int segmap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
86 86 static void segmap_dump(struct seg *seg);
87 87 static int segmap_pagelock(struct seg *seg, caddr_t addr, size_t len,
88 88 struct page ***ppp, enum lock_type type,
89 89 enum seg_rw rw);
90 90 static void segmap_badop(void);
91 91 static int segmap_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
92 92 static lgrp_mem_policy_info_t *segmap_getpolicy(struct seg *seg,
93 93 caddr_t addr);
94 94 static int segmap_capable(struct seg *seg, segcapability_t capability);
95 95
96 96 /* segkpm support */
97 97 static caddr_t segmap_pagecreate_kpm(struct seg *, vnode_t *, u_offset_t,
98 98 struct smap *, enum seg_rw);
99 99 struct smap *get_smap_kpm(caddr_t, page_t **);
100 100
101 101 #define SEGMAP_BADOP(t) (t(*)())segmap_badop
↓ open down ↓ |
101 lines elided |
↑ open up ↑ |
102 102
103 103 static struct seg_ops segmap_ops = {
104 104 SEGMAP_BADOP(int), /* dup */
105 105 SEGMAP_BADOP(int), /* unmap */
106 106 segmap_free,
107 107 segmap_fault,
108 108 segmap_faulta,
109 109 SEGMAP_BADOP(int), /* setprot */
110 110 segmap_checkprot,
111 111 segmap_kluster,
112 - SEGMAP_BADOP(size_t), /* swapout */
113 112 SEGMAP_BADOP(int), /* sync */
114 113 SEGMAP_BADOP(size_t), /* incore */
115 114 SEGMAP_BADOP(int), /* lockop */
116 115 segmap_getprot,
117 116 segmap_getoffset,
118 117 segmap_gettype,
119 118 segmap_getvp,
120 119 SEGMAP_BADOP(int), /* advise */
121 120 segmap_dump,
122 121 segmap_pagelock, /* pagelock */
123 122 SEGMAP_BADOP(int), /* setpgsz */
124 123 segmap_getmemid, /* getmemid */
125 124 segmap_getpolicy, /* getpolicy */
126 125 segmap_capable, /* capable */
127 126 };
128 127
129 128 /*
130 129 * Private segmap routines.
131 130 */
132 131 static void segmap_unlock(struct hat *hat, struct seg *seg, caddr_t addr,
133 132 size_t len, enum seg_rw rw, struct smap *smp);
134 133 static void segmap_smapadd(struct smap *smp);
135 134 static struct smap *segmap_hashin(struct smap *smp, struct vnode *vp,
136 135 u_offset_t off, int hashid);
137 136 static void segmap_hashout(struct smap *smp);
138 137
139 138
140 139 /*
141 140 * Statistics for segmap operations.
142 141 *
143 142 * No explicit locking to protect these stats.
144 143 */
145 144 struct segmapcnt segmapcnt = {
146 145 { "fault", KSTAT_DATA_ULONG },
147 146 { "faulta", KSTAT_DATA_ULONG },
148 147 { "getmap", KSTAT_DATA_ULONG },
149 148 { "get_use", KSTAT_DATA_ULONG },
150 149 { "get_reclaim", KSTAT_DATA_ULONG },
151 150 { "get_reuse", KSTAT_DATA_ULONG },
152 151 { "get_unused", KSTAT_DATA_ULONG },
153 152 { "get_nofree", KSTAT_DATA_ULONG },
154 153 { "rel_async", KSTAT_DATA_ULONG },
155 154 { "rel_write", KSTAT_DATA_ULONG },
156 155 { "rel_free", KSTAT_DATA_ULONG },
157 156 { "rel_abort", KSTAT_DATA_ULONG },
158 157 { "rel_dontneed", KSTAT_DATA_ULONG },
159 158 { "release", KSTAT_DATA_ULONG },
160 159 { "pagecreate", KSTAT_DATA_ULONG },
161 160 { "free_notfree", KSTAT_DATA_ULONG },
162 161 { "free_dirty", KSTAT_DATA_ULONG },
163 162 { "free", KSTAT_DATA_ULONG },
164 163 { "stolen", KSTAT_DATA_ULONG },
165 164 { "get_nomtx", KSTAT_DATA_ULONG }
166 165 };
167 166
168 167 kstat_named_t *segmapcnt_ptr = (kstat_named_t *)&segmapcnt;
169 168 uint_t segmapcnt_ndata = sizeof (segmapcnt) / sizeof (kstat_named_t);
170 169
171 170 /*
172 171 * Return number of map pages in segment.
173 172 */
174 173 #define MAP_PAGES(seg) ((seg)->s_size >> MAXBSHIFT)
175 174
176 175 /*
177 176 * Translate addr into smap number within segment.
178 177 */
179 178 #define MAP_PAGE(seg, addr) (((addr) - (seg)->s_base) >> MAXBSHIFT)
180 179
181 180 /*
182 181 * Translate addr in seg into struct smap pointer.
183 182 */
184 183 #define GET_SMAP(seg, addr) \
185 184 &(((struct segmap_data *)((seg)->s_data))->smd_sm[MAP_PAGE(seg, addr)])
186 185
187 186 /*
188 187 * Bit in map (16 bit bitmap).
189 188 */
190 189 #define SMAP_BIT_MASK(bitindex) (1 << ((bitindex) & 0xf))
191 190
192 191 static int smd_colormsk = 0;
193 192 static int smd_ncolor = 0;
194 193 static int smd_nfree = 0;
195 194 static int smd_freemsk = 0;
196 195 #ifdef DEBUG
197 196 static int *colors_used;
198 197 #endif
199 198 static struct smap *smd_smap;
200 199 static struct smaphash *smd_hash;
201 200 #ifdef SEGMAP_HASHSTATS
202 201 static unsigned int *smd_hash_len;
203 202 #endif
204 203 static struct smfree *smd_free;
205 204 static ulong_t smd_hashmsk = 0;
206 205
207 206 #define SEGMAP_MAXCOLOR 2
208 207 #define SEGMAP_CACHE_PAD 64
209 208
210 209 union segmap_cpu {
211 210 struct {
212 211 uint32_t scpu_free_ndx[SEGMAP_MAXCOLOR];
213 212 struct smap *scpu_last_smap;
214 213 ulong_t scpu_getmap;
215 214 ulong_t scpu_release;
216 215 ulong_t scpu_get_reclaim;
217 216 ulong_t scpu_fault;
218 217 ulong_t scpu_pagecreate;
219 218 ulong_t scpu_get_reuse;
220 219 } scpu;
221 220 char scpu_pad[SEGMAP_CACHE_PAD];
222 221 };
223 222 static union segmap_cpu *smd_cpu;
224 223
225 224 /*
226 225 * There are three locks in seg_map:
227 226 * - per freelist mutexes
228 227 * - per hashchain mutexes
229 228 * - per smap mutexes
230 229 *
231 230 * The lock ordering is to get the smap mutex to lock down the slot
232 231 * first then the hash lock (for hash in/out (vp, off) list) or the
233 232 * freelist lock to put the slot back on the free list.
234 233 *
235 234 * The hash search is done by only holding the hashchain lock, when a wanted
236 235 * slot is found, we drop the hashchain lock then lock the slot so there
237 236 * is no overlapping of hashchain and smap locks. After the slot is
238 237 * locked, we verify again if the slot is still what we are looking
239 238 * for.
240 239 *
241 240 * Allocation of a free slot is done by holding the freelist lock,
242 241 * then locking the smap slot at the head of the freelist. This is
243 242 * in reversed lock order so mutex_tryenter() is used.
244 243 *
245 244 * The smap lock protects all fields in smap structure except for
246 245 * the link fields for hash/free lists which are protected by
247 246 * hashchain and freelist locks.
248 247 */
249 248
250 249 #define SHASHMTX(hashid) (&smd_hash[hashid].sh_mtx)
251 250
252 251 #define SMP2SMF(smp) (&smd_free[(smp - smd_smap) & smd_freemsk])
253 252 #define SMP2SMF_NDX(smp) (ushort_t)((smp - smd_smap) & smd_freemsk)
254 253
255 254 #define SMAPMTX(smp) (&smp->sm_mtx)
256 255
257 256 #define SMAP_HASHFUNC(vp, off, hashid) \
258 257 { \
259 258 hashid = ((((uintptr_t)(vp) >> 6) + ((uintptr_t)(vp) >> 3) + \
260 259 ((off) >> MAXBSHIFT)) & smd_hashmsk); \
261 260 }
262 261
263 262 /*
264 263 * The most frequently updated kstat counters are kept in the
265 264 * per cpu array to avoid hot cache blocks. The update function
266 265 * sums the cpu local counters to update the global counters.
267 266 */
268 267
269 268 /* ARGSUSED */
270 269 int
271 270 segmap_kstat_update(kstat_t *ksp, int rw)
272 271 {
273 272 int i;
274 273 ulong_t getmap, release, get_reclaim;
275 274 ulong_t fault, pagecreate, get_reuse;
276 275
277 276 if (rw == KSTAT_WRITE)
278 277 return (EACCES);
279 278 getmap = release = get_reclaim = (ulong_t)0;
280 279 fault = pagecreate = get_reuse = (ulong_t)0;
281 280 for (i = 0; i < max_ncpus; i++) {
282 281 getmap += smd_cpu[i].scpu.scpu_getmap;
283 282 release += smd_cpu[i].scpu.scpu_release;
284 283 get_reclaim += smd_cpu[i].scpu.scpu_get_reclaim;
285 284 fault += smd_cpu[i].scpu.scpu_fault;
286 285 pagecreate += smd_cpu[i].scpu.scpu_pagecreate;
287 286 get_reuse += smd_cpu[i].scpu.scpu_get_reuse;
288 287 }
289 288 segmapcnt.smp_getmap.value.ul = getmap;
290 289 segmapcnt.smp_release.value.ul = release;
291 290 segmapcnt.smp_get_reclaim.value.ul = get_reclaim;
292 291 segmapcnt.smp_fault.value.ul = fault;
293 292 segmapcnt.smp_pagecreate.value.ul = pagecreate;
294 293 segmapcnt.smp_get_reuse.value.ul = get_reuse;
295 294 return (0);
296 295 }
297 296
298 297 int
299 298 segmap_create(struct seg *seg, void *argsp)
300 299 {
301 300 struct segmap_data *smd;
302 301 struct smap *smp;
303 302 struct smfree *sm;
304 303 struct segmap_crargs *a = (struct segmap_crargs *)argsp;
305 304 struct smaphash *shashp;
306 305 union segmap_cpu *scpu;
307 306 long i, npages;
308 307 size_t hashsz;
309 308 uint_t nfreelist;
310 309 extern void prefetch_smap_w(void *);
311 310 extern int max_ncpus;
312 311
313 312 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
314 313
315 314 if (((uintptr_t)seg->s_base | seg->s_size) & MAXBOFFSET) {
316 315 panic("segkmap not MAXBSIZE aligned");
317 316 /*NOTREACHED*/
318 317 }
319 318
320 319 smd = kmem_zalloc(sizeof (struct segmap_data), KM_SLEEP);
321 320
322 321 seg->s_data = (void *)smd;
323 322 seg->s_ops = &segmap_ops;
324 323 smd->smd_prot = a->prot;
325 324
326 325 /*
327 326 * Scale the number of smap freelists to be
328 327 * proportional to max_ncpus * number of virtual colors.
329 328 * The caller can over-ride this scaling by providing
330 329 * a non-zero a->nfreelist argument.
331 330 */
332 331 nfreelist = a->nfreelist;
333 332 if (nfreelist == 0)
334 333 nfreelist = max_ncpus;
335 334 else if (nfreelist < 0 || nfreelist > 4 * max_ncpus) {
336 335 cmn_err(CE_WARN, "segmap_create: nfreelist out of range "
337 336 "%d, using %d", nfreelist, max_ncpus);
338 337 nfreelist = max_ncpus;
339 338 }
340 339 if (nfreelist & (nfreelist - 1)) {
341 340 /* round up nfreelist to the next power of two. */
342 341 nfreelist = 1 << (highbit(nfreelist));
343 342 }
344 343
345 344 /*
346 345 * Get the number of virtual colors - must be a power of 2.
347 346 */
348 347 if (a->shmsize)
349 348 smd_ncolor = a->shmsize >> MAXBSHIFT;
350 349 else
351 350 smd_ncolor = 1;
352 351 ASSERT((smd_ncolor & (smd_ncolor - 1)) == 0);
353 352 ASSERT(smd_ncolor <= SEGMAP_MAXCOLOR);
354 353 smd_colormsk = smd_ncolor - 1;
355 354 smd->smd_nfree = smd_nfree = smd_ncolor * nfreelist;
356 355 smd_freemsk = smd_nfree - 1;
357 356
358 357 /*
359 358 * Allocate and initialize the freelist headers.
360 359 * Note that sm_freeq[1] starts out as the release queue. This
361 360 * is known when the smap structures are initialized below.
362 361 */
363 362 smd_free = smd->smd_free =
364 363 kmem_zalloc(smd_nfree * sizeof (struct smfree), KM_SLEEP);
365 364 for (i = 0; i < smd_nfree; i++) {
366 365 sm = &smd->smd_free[i];
367 366 mutex_init(&sm->sm_freeq[0].smq_mtx, NULL, MUTEX_DEFAULT, NULL);
368 367 mutex_init(&sm->sm_freeq[1].smq_mtx, NULL, MUTEX_DEFAULT, NULL);
369 368 sm->sm_allocq = &sm->sm_freeq[0];
370 369 sm->sm_releq = &sm->sm_freeq[1];
371 370 }
372 371
373 372 /*
374 373 * Allocate and initialize the smap hash chain headers.
375 374 * Compute hash size rounding down to the next power of two.
376 375 */
377 376 npages = MAP_PAGES(seg);
378 377 smd->smd_npages = npages;
379 378 hashsz = npages / SMAP_HASHAVELEN;
380 379 hashsz = 1 << (highbit(hashsz)-1);
381 380 smd_hashmsk = hashsz - 1;
382 381 smd_hash = smd->smd_hash =
383 382 kmem_alloc(hashsz * sizeof (struct smaphash), KM_SLEEP);
384 383 #ifdef SEGMAP_HASHSTATS
385 384 smd_hash_len =
386 385 kmem_zalloc(hashsz * sizeof (unsigned int), KM_SLEEP);
387 386 #endif
388 387 for (i = 0, shashp = smd_hash; i < hashsz; i++, shashp++) {
389 388 shashp->sh_hash_list = NULL;
390 389 mutex_init(&shashp->sh_mtx, NULL, MUTEX_DEFAULT, NULL);
391 390 }
392 391
393 392 /*
394 393 * Allocate and initialize the smap structures.
395 394 * Link all slots onto the appropriate freelist.
396 395 * The smap array is large enough to affect boot time
397 396 * on large systems, so use memory prefetching and only
398 397 * go through the array 1 time. Inline a optimized version
399 398 * of segmap_smapadd to add structures to freelists with
400 399 * knowledge that no locks are needed here.
401 400 */
402 401 smd_smap = smd->smd_sm =
403 402 kmem_alloc(sizeof (struct smap) * npages, KM_SLEEP);
404 403
405 404 for (smp = &smd->smd_sm[MAP_PAGES(seg) - 1];
406 405 smp >= smd->smd_sm; smp--) {
407 406 struct smap *smpfreelist;
408 407 struct sm_freeq *releq;
409 408
410 409 prefetch_smap_w((char *)smp);
411 410
412 411 smp->sm_vp = NULL;
413 412 smp->sm_hash = NULL;
414 413 smp->sm_off = 0;
415 414 smp->sm_bitmap = 0;
416 415 smp->sm_refcnt = 0;
417 416 mutex_init(&smp->sm_mtx, NULL, MUTEX_DEFAULT, NULL);
418 417 smp->sm_free_ndx = SMP2SMF_NDX(smp);
419 418
420 419 sm = SMP2SMF(smp);
421 420 releq = sm->sm_releq;
422 421
423 422 smpfreelist = releq->smq_free;
424 423 if (smpfreelist == 0) {
425 424 releq->smq_free = smp->sm_next = smp->sm_prev = smp;
426 425 } else {
427 426 smp->sm_next = smpfreelist;
428 427 smp->sm_prev = smpfreelist->sm_prev;
429 428 smpfreelist->sm_prev = smp;
430 429 smp->sm_prev->sm_next = smp;
431 430 releq->smq_free = smp->sm_next;
432 431 }
433 432
434 433 /*
435 434 * sm_flag = 0 (no SM_QNDX_ZERO) implies smap on sm_freeq[1]
436 435 */
437 436 smp->sm_flags = 0;
438 437
439 438 #ifdef SEGKPM_SUPPORT
440 439 /*
441 440 * Due to the fragile prefetch loop no
442 441 * separate function is used here.
443 442 */
444 443 smp->sm_kpme_next = NULL;
445 444 smp->sm_kpme_prev = NULL;
446 445 smp->sm_kpme_page = NULL;
447 446 #endif
448 447 }
449 448
450 449 /*
451 450 * Allocate the per color indices that distribute allocation
452 451 * requests over the free lists. Each cpu will have a private
453 452 * rotor index to spread the allocations even across the available
454 453 * smap freelists. Init the scpu_last_smap field to the first
455 454 * smap element so there is no need to check for NULL.
456 455 */
457 456 smd_cpu =
458 457 kmem_zalloc(sizeof (union segmap_cpu) * max_ncpus, KM_SLEEP);
459 458 for (i = 0, scpu = smd_cpu; i < max_ncpus; i++, scpu++) {
460 459 int j;
461 460 for (j = 0; j < smd_ncolor; j++)
462 461 scpu->scpu.scpu_free_ndx[j] = j;
463 462 scpu->scpu.scpu_last_smap = smd_smap;
464 463 }
465 464
466 465 vpm_init();
467 466
468 467 #ifdef DEBUG
469 468 /*
470 469 * Keep track of which colors are used more often.
471 470 */
472 471 colors_used = kmem_zalloc(smd_nfree * sizeof (int), KM_SLEEP);
473 472 #endif /* DEBUG */
474 473
475 474 return (0);
476 475 }
477 476
478 477 static void
479 478 segmap_free(seg)
480 479 struct seg *seg;
481 480 {
482 481 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
483 482 }
484 483
485 484 /*
486 485 * Do a F_SOFTUNLOCK call over the range requested.
487 486 * The range must have already been F_SOFTLOCK'ed.
488 487 */
489 488 static void
490 489 segmap_unlock(
491 490 struct hat *hat,
492 491 struct seg *seg,
493 492 caddr_t addr,
494 493 size_t len,
495 494 enum seg_rw rw,
496 495 struct smap *smp)
497 496 {
498 497 page_t *pp;
499 498 caddr_t adr;
500 499 u_offset_t off;
501 500 struct vnode *vp;
502 501 kmutex_t *smtx;
503 502
504 503 ASSERT(smp->sm_refcnt > 0);
505 504
506 505 #ifdef lint
507 506 seg = seg;
508 507 #endif
509 508
510 509 if (segmap_kpm && IS_KPM_ADDR(addr)) {
511 510
512 511 /*
513 512 * We're called only from segmap_fault and this was a
514 513 * NOP in case of a kpm based smap, so dangerous things
515 514 * must have happened in the meantime. Pages are prefaulted
516 515 * and locked in segmap_getmapflt and they will not be
517 516 * unlocked until segmap_release.
518 517 */
519 518 panic("segmap_unlock: called with kpm addr %p", (void *)addr);
520 519 /*NOTREACHED*/
521 520 }
522 521
523 522 vp = smp->sm_vp;
524 523 off = smp->sm_off + (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
525 524
526 525 hat_unlock(hat, addr, P2ROUNDUP(len, PAGESIZE));
527 526 for (adr = addr; adr < addr + len; adr += PAGESIZE, off += PAGESIZE) {
528 527 ushort_t bitmask;
529 528
530 529 /*
531 530 * Use page_find() instead of page_lookup() to
532 531 * find the page since we know that it has
533 532 * "shared" lock.
534 533 */
535 534 pp = page_find(vp, off);
536 535 if (pp == NULL) {
537 536 panic("segmap_unlock: page not found");
538 537 /*NOTREACHED*/
539 538 }
540 539
541 540 if (rw == S_WRITE) {
542 541 hat_setrefmod(pp);
543 542 } else if (rw != S_OTHER) {
544 543 TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
545 544 "segmap_fault:pp %p vp %p offset %llx", pp, vp, off);
546 545 hat_setref(pp);
547 546 }
548 547
549 548 /*
550 549 * Clear bitmap, if the bit corresponding to "off" is set,
551 550 * since the page and translation are being unlocked.
552 551 */
553 552 bitmask = SMAP_BIT_MASK((off - smp->sm_off) >> PAGESHIFT);
554 553
555 554 /*
556 555 * Large Files: Following assertion is to verify
557 556 * the correctness of the cast to (int) above.
558 557 */
559 558 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
560 559 smtx = SMAPMTX(smp);
561 560 mutex_enter(smtx);
562 561 if (smp->sm_bitmap & bitmask) {
563 562 smp->sm_bitmap &= ~bitmask;
564 563 }
565 564 mutex_exit(smtx);
566 565
567 566 page_unlock(pp);
568 567 }
569 568 }
570 569
571 570 #define MAXPPB (MAXBSIZE/4096) /* assumes minimum page size of 4k */
572 571
573 572 /*
574 573 * This routine is called via a machine specific fault handling
575 574 * routine. It is also called by software routines wishing to
576 575 * lock or unlock a range of addresses.
577 576 *
578 577 * Note that this routine expects a page-aligned "addr".
579 578 */
580 579 faultcode_t
581 580 segmap_fault(
582 581 struct hat *hat,
583 582 struct seg *seg,
584 583 caddr_t addr,
585 584 size_t len,
586 585 enum fault_type type,
587 586 enum seg_rw rw)
588 587 {
589 588 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
590 589 struct smap *smp;
591 590 page_t *pp, **ppp;
592 591 struct vnode *vp;
593 592 u_offset_t off;
594 593 page_t *pl[MAXPPB + 1];
595 594 uint_t prot;
596 595 u_offset_t addroff;
597 596 caddr_t adr;
598 597 int err;
599 598 u_offset_t sm_off;
600 599 int hat_flag;
601 600
602 601 if (segmap_kpm && IS_KPM_ADDR(addr)) {
603 602 int newpage;
604 603 kmutex_t *smtx;
605 604
606 605 /*
607 606 * Pages are successfully prefaulted and locked in
608 607 * segmap_getmapflt and can't be unlocked until
609 608 * segmap_release. No hat mappings have to be locked
610 609 * and they also can't be unlocked as long as the
611 610 * caller owns an active kpm addr.
612 611 */
613 612 #ifndef DEBUG
614 613 if (type != F_SOFTUNLOCK)
615 614 return (0);
616 615 #endif
617 616
618 617 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
619 618 panic("segmap_fault: smap not found "
620 619 "for addr %p", (void *)addr);
621 620 /*NOTREACHED*/
622 621 }
623 622
624 623 smtx = SMAPMTX(smp);
625 624 #ifdef DEBUG
626 625 newpage = smp->sm_flags & SM_KPM_NEWPAGE;
627 626 if (newpage) {
628 627 cmn_err(CE_WARN, "segmap_fault: newpage? smp %p",
629 628 (void *)smp);
630 629 }
631 630
632 631 if (type != F_SOFTUNLOCK) {
633 632 mutex_exit(smtx);
634 633 return (0);
635 634 }
636 635 #endif
637 636 mutex_exit(smtx);
638 637 vp = smp->sm_vp;
639 638 sm_off = smp->sm_off;
640 639
641 640 if (vp == NULL)
642 641 return (FC_MAKE_ERR(EIO));
643 642
644 643 ASSERT(smp->sm_refcnt > 0);
645 644
646 645 addroff = (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
647 646 if (addroff + len > MAXBSIZE)
648 647 panic("segmap_fault: endaddr %p exceeds MAXBSIZE chunk",
649 648 (void *)(addr + len));
650 649
651 650 off = sm_off + addroff;
652 651
653 652 pp = page_find(vp, off);
654 653
655 654 if (pp == NULL)
656 655 panic("segmap_fault: softunlock page not found");
657 656
658 657 /*
659 658 * Set ref bit also here in case of S_OTHER to avoid the
660 659 * overhead of supporting other cases than F_SOFTUNLOCK
661 660 * with segkpm. We can do this because the underlying
662 661 * pages are locked anyway.
663 662 */
664 663 if (rw == S_WRITE) {
665 664 hat_setrefmod(pp);
666 665 } else {
667 666 TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
668 667 "segmap_fault:pp %p vp %p offset %llx",
669 668 pp, vp, off);
670 669 hat_setref(pp);
671 670 }
672 671
673 672 return (0);
674 673 }
675 674
676 675 smd_cpu[CPU->cpu_seqid].scpu.scpu_fault++;
677 676 smp = GET_SMAP(seg, addr);
678 677 vp = smp->sm_vp;
679 678 sm_off = smp->sm_off;
680 679
681 680 if (vp == NULL)
682 681 return (FC_MAKE_ERR(EIO));
683 682
684 683 ASSERT(smp->sm_refcnt > 0);
685 684
686 685 addroff = (u_offset_t)((uintptr_t)addr & MAXBOFFSET);
687 686 if (addroff + len > MAXBSIZE) {
688 687 panic("segmap_fault: endaddr %p "
689 688 "exceeds MAXBSIZE chunk", (void *)(addr + len));
690 689 /*NOTREACHED*/
691 690 }
692 691 off = sm_off + addroff;
693 692
694 693 /*
695 694 * First handle the easy stuff
696 695 */
697 696 if (type == F_SOFTUNLOCK) {
698 697 segmap_unlock(hat, seg, addr, len, rw, smp);
699 698 return (0);
700 699 }
701 700
702 701 TRACE_3(TR_FAC_VM, TR_SEGMAP_GETPAGE,
703 702 "segmap_getpage:seg %p addr %p vp %p", seg, addr, vp);
704 703 err = VOP_GETPAGE(vp, (offset_t)off, len, &prot, pl, MAXBSIZE,
705 704 seg, addr, rw, CRED(), NULL);
706 705
707 706 if (err)
708 707 return (FC_MAKE_ERR(err));
709 708
710 709 prot &= smd->smd_prot;
711 710
712 711 /*
713 712 * Handle all pages returned in the pl[] array.
714 713 * This loop is coded on the assumption that if
715 714 * there was no error from the VOP_GETPAGE routine,
716 715 * that the page list returned will contain all the
717 716 * needed pages for the vp from [off..off + len].
718 717 */
719 718 ppp = pl;
720 719 while ((pp = *ppp++) != NULL) {
721 720 u_offset_t poff;
722 721 ASSERT(pp->p_vnode == vp);
723 722 hat_flag = HAT_LOAD;
724 723
725 724 /*
726 725 * Verify that the pages returned are within the range
727 726 * of this segmap region. Note that it is theoretically
728 727 * possible for pages outside this range to be returned,
729 728 * but it is not very likely. If we cannot use the
730 729 * page here, just release it and go on to the next one.
731 730 */
732 731 if (pp->p_offset < sm_off ||
733 732 pp->p_offset >= sm_off + MAXBSIZE) {
734 733 (void) page_release(pp, 1);
735 734 continue;
736 735 }
737 736
738 737 ASSERT(hat == kas.a_hat);
739 738 poff = pp->p_offset;
740 739 adr = addr + (poff - off);
741 740 if (adr >= addr && adr < addr + len) {
742 741 hat_setref(pp);
743 742 TRACE_3(TR_FAC_VM, TR_SEGMAP_FAULT,
744 743 "segmap_fault:pp %p vp %p offset %llx",
745 744 pp, vp, poff);
746 745 if (type == F_SOFTLOCK)
747 746 hat_flag = HAT_LOAD_LOCK;
748 747 }
749 748
750 749 /*
751 750 * Deal with VMODSORT pages here. If we know this is a write
752 751 * do the setmod now and allow write protection.
753 752 * As long as it's modified or not S_OTHER, remove write
754 753 * protection. With S_OTHER it's up to the FS to deal with this.
755 754 */
756 755 if (IS_VMODSORT(vp)) {
757 756 if (rw == S_WRITE)
758 757 hat_setmod(pp);
759 758 else if (rw != S_OTHER && !hat_ismod(pp))
760 759 prot &= ~PROT_WRITE;
761 760 }
762 761
763 762 hat_memload(hat, adr, pp, prot, hat_flag);
764 763 if (hat_flag != HAT_LOAD_LOCK)
765 764 page_unlock(pp);
766 765 }
767 766 return (0);
768 767 }
769 768
770 769 /*
771 770 * This routine is used to start I/O on pages asynchronously.
772 771 */
773 772 static faultcode_t
774 773 segmap_faulta(struct seg *seg, caddr_t addr)
775 774 {
776 775 struct smap *smp;
777 776 struct vnode *vp;
778 777 u_offset_t off;
779 778 int err;
780 779
781 780 if (segmap_kpm && IS_KPM_ADDR(addr)) {
782 781 int newpage;
783 782 kmutex_t *smtx;
784 783
785 784 /*
786 785 * Pages are successfully prefaulted and locked in
787 786 * segmap_getmapflt and can't be unlocked until
788 787 * segmap_release. No hat mappings have to be locked
789 788 * and they also can't be unlocked as long as the
790 789 * caller owns an active kpm addr.
791 790 */
792 791 #ifdef DEBUG
793 792 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
794 793 panic("segmap_faulta: smap not found "
795 794 "for addr %p", (void *)addr);
796 795 /*NOTREACHED*/
797 796 }
798 797
799 798 smtx = SMAPMTX(smp);
800 799 newpage = smp->sm_flags & SM_KPM_NEWPAGE;
801 800 mutex_exit(smtx);
802 801 if (newpage)
803 802 cmn_err(CE_WARN, "segmap_faulta: newpage? smp %p",
804 803 (void *)smp);
805 804 #endif
806 805 return (0);
807 806 }
808 807
809 808 segmapcnt.smp_faulta.value.ul++;
810 809 smp = GET_SMAP(seg, addr);
811 810
812 811 ASSERT(smp->sm_refcnt > 0);
813 812
814 813 vp = smp->sm_vp;
815 814 off = smp->sm_off;
816 815
817 816 if (vp == NULL) {
818 817 cmn_err(CE_WARN, "segmap_faulta - no vp");
819 818 return (FC_MAKE_ERR(EIO));
820 819 }
821 820
822 821 TRACE_3(TR_FAC_VM, TR_SEGMAP_GETPAGE,
823 822 "segmap_getpage:seg %p addr %p vp %p", seg, addr, vp);
824 823
825 824 err = VOP_GETPAGE(vp, (offset_t)(off + ((offset_t)((uintptr_t)addr
826 825 & MAXBOFFSET))), PAGESIZE, (uint_t *)NULL, (page_t **)NULL, 0,
827 826 seg, addr, S_READ, CRED(), NULL);
828 827
829 828 if (err)
830 829 return (FC_MAKE_ERR(err));
831 830 return (0);
832 831 }
833 832
834 833 /*ARGSUSED*/
835 834 static int
836 835 segmap_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
837 836 {
838 837 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
839 838
840 839 ASSERT(seg->s_as && RW_LOCK_HELD(&seg->s_as->a_lock));
841 840
842 841 /*
843 842 * Need not acquire the segment lock since
844 843 * "smd_prot" is a read-only field.
845 844 */
846 845 return (((smd->smd_prot & prot) != prot) ? EACCES : 0);
847 846 }
848 847
849 848 static int
850 849 segmap_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
851 850 {
852 851 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
853 852 size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
854 853
855 854 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
856 855
857 856 if (pgno != 0) {
858 857 do {
859 858 protv[--pgno] = smd->smd_prot;
860 859 } while (pgno != 0);
861 860 }
862 861 return (0);
863 862 }
864 863
865 864 static u_offset_t
866 865 segmap_getoffset(struct seg *seg, caddr_t addr)
867 866 {
868 867 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
869 868
870 869 ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));
871 870
872 871 return ((u_offset_t)smd->smd_sm->sm_off + (addr - seg->s_base));
873 872 }
874 873
875 874 /*ARGSUSED*/
876 875 static int
877 876 segmap_gettype(struct seg *seg, caddr_t addr)
878 877 {
879 878 ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));
880 879
881 880 return (MAP_SHARED);
882 881 }
883 882
884 883 /*ARGSUSED*/
885 884 static int
886 885 segmap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
887 886 {
888 887 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
889 888
890 889 ASSERT(seg->s_as && RW_READ_HELD(&seg->s_as->a_lock));
891 890
892 891 /* XXX - This doesn't make any sense */
893 892 *vpp = smd->smd_sm->sm_vp;
894 893 return (0);
895 894 }
896 895
897 896 /*
898 897 * Check to see if it makes sense to do kluster/read ahead to
899 898 * addr + delta relative to the mapping at addr. We assume here
900 899 * that delta is a signed PAGESIZE'd multiple (which can be negative).
901 900 *
902 901 * For segmap we always "approve" of this action from our standpoint.
903 902 */
904 903 /*ARGSUSED*/
905 904 static int
906 905 segmap_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
907 906 {
908 907 return (0);
909 908 }
910 909
911 910 static void
912 911 segmap_badop()
913 912 {
914 913 panic("segmap_badop");
915 914 /*NOTREACHED*/
916 915 }
917 916
918 917 /*
919 918 * Special private segmap operations
920 919 */
921 920
922 921 /*
923 922 * Add smap to the appropriate free list.
924 923 */
925 924 static void
926 925 segmap_smapadd(struct smap *smp)
927 926 {
928 927 struct smfree *sm;
929 928 struct smap *smpfreelist;
930 929 struct sm_freeq *releq;
931 930
932 931 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
933 932
934 933 if (smp->sm_refcnt != 0) {
935 934 panic("segmap_smapadd");
936 935 /*NOTREACHED*/
937 936 }
938 937
939 938 sm = &smd_free[smp->sm_free_ndx];
940 939 /*
941 940 * Add to the tail of the release queue
942 941 * Note that sm_releq and sm_allocq could toggle
943 942 * before we get the lock. This does not affect
944 943 * correctness as the 2 queues are only maintained
945 944 * to reduce lock pressure.
946 945 */
947 946 releq = sm->sm_releq;
948 947 if (releq == &sm->sm_freeq[0])
949 948 smp->sm_flags |= SM_QNDX_ZERO;
950 949 else
951 950 smp->sm_flags &= ~SM_QNDX_ZERO;
952 951 mutex_enter(&releq->smq_mtx);
953 952 smpfreelist = releq->smq_free;
954 953 if (smpfreelist == 0) {
955 954 int want;
956 955
957 956 releq->smq_free = smp->sm_next = smp->sm_prev = smp;
958 957 /*
959 958 * Both queue mutexes held to set sm_want;
960 959 * snapshot the value before dropping releq mutex.
961 960 * If sm_want appears after the releq mutex is dropped,
962 961 * then the smap just freed is already gone.
963 962 */
964 963 want = sm->sm_want;
965 964 mutex_exit(&releq->smq_mtx);
966 965 /*
967 966 * See if there was a waiter before dropping the releq mutex
968 967 * then recheck after obtaining sm_freeq[0] mutex as
969 968 * the another thread may have already signaled.
970 969 */
971 970 if (want) {
972 971 mutex_enter(&sm->sm_freeq[0].smq_mtx);
973 972 if (sm->sm_want)
974 973 cv_signal(&sm->sm_free_cv);
975 974 mutex_exit(&sm->sm_freeq[0].smq_mtx);
976 975 }
977 976 } else {
978 977 smp->sm_next = smpfreelist;
979 978 smp->sm_prev = smpfreelist->sm_prev;
980 979 smpfreelist->sm_prev = smp;
981 980 smp->sm_prev->sm_next = smp;
982 981 mutex_exit(&releq->smq_mtx);
983 982 }
984 983 }
985 984
986 985
987 986 static struct smap *
988 987 segmap_hashin(struct smap *smp, struct vnode *vp, u_offset_t off, int hashid)
989 988 {
990 989 struct smap **hpp;
991 990 struct smap *tmp;
992 991 kmutex_t *hmtx;
993 992
994 993 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
995 994 ASSERT(smp->sm_vp == NULL);
996 995 ASSERT(smp->sm_hash == NULL);
997 996 ASSERT(smp->sm_prev == NULL);
998 997 ASSERT(smp->sm_next == NULL);
999 998 ASSERT(hashid >= 0 && hashid <= smd_hashmsk);
1000 999
1001 1000 hmtx = SHASHMTX(hashid);
1002 1001
1003 1002 mutex_enter(hmtx);
1004 1003 /*
1005 1004 * First we need to verify that no one has created a smp
1006 1005 * with (vp,off) as its tag before we us.
1007 1006 */
1008 1007 for (tmp = smd_hash[hashid].sh_hash_list;
1009 1008 tmp != NULL; tmp = tmp->sm_hash)
1010 1009 if (tmp->sm_vp == vp && tmp->sm_off == off)
1011 1010 break;
1012 1011
1013 1012 if (tmp == NULL) {
1014 1013 /*
1015 1014 * No one created one yet.
1016 1015 *
1017 1016 * Funniness here - we don't increment the ref count on the
1018 1017 * vnode * even though we have another pointer to it here.
1019 1018 * The reason for this is that we don't want the fact that
1020 1019 * a seg_map entry somewhere refers to a vnode to prevent the
1021 1020 * vnode * itself from going away. This is because this
1022 1021 * reference to the vnode is a "soft one". In the case where
1023 1022 * a mapping is being used by a rdwr [or directory routine?]
1024 1023 * there already has to be a non-zero ref count on the vnode.
1025 1024 * In the case where the vp has been freed and the the smap
1026 1025 * structure is on the free list, there are no pages in memory
1027 1026 * that can refer to the vnode. Thus even if we reuse the same
1028 1027 * vnode/smap structure for a vnode which has the same
1029 1028 * address but represents a different object, we are ok.
1030 1029 */
1031 1030 smp->sm_vp = vp;
1032 1031 smp->sm_off = off;
1033 1032
1034 1033 hpp = &smd_hash[hashid].sh_hash_list;
1035 1034 smp->sm_hash = *hpp;
1036 1035 *hpp = smp;
1037 1036 #ifdef SEGMAP_HASHSTATS
1038 1037 smd_hash_len[hashid]++;
1039 1038 #endif
1040 1039 }
1041 1040 mutex_exit(hmtx);
1042 1041
1043 1042 return (tmp);
1044 1043 }
1045 1044
1046 1045 static void
1047 1046 segmap_hashout(struct smap *smp)
1048 1047 {
1049 1048 struct smap **hpp, *hp;
1050 1049 struct vnode *vp;
1051 1050 kmutex_t *mtx;
1052 1051 int hashid;
1053 1052 u_offset_t off;
1054 1053
1055 1054 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
1056 1055
1057 1056 vp = smp->sm_vp;
1058 1057 off = smp->sm_off;
1059 1058
1060 1059 SMAP_HASHFUNC(vp, off, hashid); /* macro assigns hashid */
1061 1060 mtx = SHASHMTX(hashid);
1062 1061 mutex_enter(mtx);
1063 1062
1064 1063 hpp = &smd_hash[hashid].sh_hash_list;
1065 1064 for (;;) {
1066 1065 hp = *hpp;
1067 1066 if (hp == NULL) {
1068 1067 panic("segmap_hashout");
1069 1068 /*NOTREACHED*/
1070 1069 }
1071 1070 if (hp == smp)
1072 1071 break;
1073 1072 hpp = &hp->sm_hash;
1074 1073 }
1075 1074
1076 1075 *hpp = smp->sm_hash;
1077 1076 smp->sm_hash = NULL;
1078 1077 #ifdef SEGMAP_HASHSTATS
1079 1078 smd_hash_len[hashid]--;
1080 1079 #endif
1081 1080 mutex_exit(mtx);
1082 1081
1083 1082 smp->sm_vp = NULL;
1084 1083 smp->sm_off = (u_offset_t)0;
1085 1084
1086 1085 }
1087 1086
1088 1087 /*
1089 1088 * Attempt to free unmodified, unmapped, and non locked segmap
1090 1089 * pages.
1091 1090 */
1092 1091 void
1093 1092 segmap_pagefree(struct vnode *vp, u_offset_t off)
1094 1093 {
1095 1094 u_offset_t pgoff;
1096 1095 page_t *pp;
1097 1096
1098 1097 for (pgoff = off; pgoff < off + MAXBSIZE; pgoff += PAGESIZE) {
1099 1098
1100 1099 if ((pp = page_lookup_nowait(vp, pgoff, SE_EXCL)) == NULL)
1101 1100 continue;
1102 1101
1103 1102 switch (page_release(pp, 1)) {
1104 1103 case PGREL_NOTREL:
1105 1104 segmapcnt.smp_free_notfree.value.ul++;
1106 1105 break;
1107 1106 case PGREL_MOD:
1108 1107 segmapcnt.smp_free_dirty.value.ul++;
1109 1108 break;
1110 1109 case PGREL_CLEAN:
1111 1110 segmapcnt.smp_free.value.ul++;
1112 1111 break;
1113 1112 }
1114 1113 }
1115 1114 }
1116 1115
1117 1116 /*
1118 1117 * Locks held on entry: smap lock
1119 1118 * Locks held on exit : smap lock.
1120 1119 */
1121 1120
1122 1121 static void
1123 1122 grab_smp(struct smap *smp, page_t *pp)
1124 1123 {
1125 1124 ASSERT(MUTEX_HELD(SMAPMTX(smp)));
1126 1125 ASSERT(smp->sm_refcnt == 0);
1127 1126
1128 1127 if (smp->sm_vp != (struct vnode *)NULL) {
1129 1128 struct vnode *vp = smp->sm_vp;
1130 1129 u_offset_t off = smp->sm_off;
1131 1130 /*
1132 1131 * Destroy old vnode association and
1133 1132 * unload any hardware translations to
1134 1133 * the old object.
1135 1134 */
1136 1135 smd_cpu[CPU->cpu_seqid].scpu.scpu_get_reuse++;
1137 1136 segmap_hashout(smp);
1138 1137
1139 1138 /*
1140 1139 * This node is off freelist and hashlist,
1141 1140 * so there is no reason to drop/reacquire sm_mtx
1142 1141 * across calls to hat_unload.
1143 1142 */
1144 1143 if (segmap_kpm) {
1145 1144 caddr_t vaddr;
1146 1145 int hat_unload_needed = 0;
1147 1146
1148 1147 /*
1149 1148 * unload kpm mapping
1150 1149 */
1151 1150 if (pp != NULL) {
1152 1151 vaddr = hat_kpm_page2va(pp, 1);
1153 1152 hat_kpm_mapout(pp, GET_KPME(smp), vaddr);
1154 1153 page_unlock(pp);
1155 1154 }
1156 1155
1157 1156 /*
1158 1157 * Check if we have (also) the rare case of a
1159 1158 * non kpm mapping.
1160 1159 */
1161 1160 if (smp->sm_flags & SM_NOTKPM_RELEASED) {
1162 1161 hat_unload_needed = 1;
1163 1162 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
1164 1163 }
1165 1164
1166 1165 if (hat_unload_needed) {
1167 1166 hat_unload(kas.a_hat, segkmap->s_base +
1168 1167 ((smp - smd_smap) * MAXBSIZE),
1169 1168 MAXBSIZE, HAT_UNLOAD);
1170 1169 }
1171 1170
1172 1171 } else {
1173 1172 ASSERT(smp->sm_flags & SM_NOTKPM_RELEASED);
1174 1173 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
1175 1174 hat_unload(kas.a_hat, segkmap->s_base +
1176 1175 ((smp - smd_smap) * MAXBSIZE),
1177 1176 MAXBSIZE, HAT_UNLOAD);
1178 1177 }
1179 1178 segmap_pagefree(vp, off);
1180 1179 }
1181 1180 }
1182 1181
1183 1182 static struct smap *
1184 1183 get_free_smp(int free_ndx)
1185 1184 {
1186 1185 struct smfree *sm;
1187 1186 kmutex_t *smtx;
1188 1187 struct smap *smp, *first;
1189 1188 struct sm_freeq *allocq, *releq;
1190 1189 struct kpme *kpme;
1191 1190 page_t *pp = NULL;
1192 1191 int end_ndx, page_locked = 0;
1193 1192
1194 1193 end_ndx = free_ndx;
1195 1194 sm = &smd_free[free_ndx];
1196 1195
1197 1196 retry_queue:
1198 1197 allocq = sm->sm_allocq;
1199 1198 mutex_enter(&allocq->smq_mtx);
1200 1199
1201 1200 if ((smp = allocq->smq_free) == NULL) {
1202 1201
1203 1202 skip_queue:
1204 1203 /*
1205 1204 * The alloc list is empty or this queue is being skipped;
1206 1205 * first see if the allocq toggled.
1207 1206 */
1208 1207 if (sm->sm_allocq != allocq) {
1209 1208 /* queue changed */
1210 1209 mutex_exit(&allocq->smq_mtx);
1211 1210 goto retry_queue;
1212 1211 }
1213 1212 releq = sm->sm_releq;
1214 1213 if (!mutex_tryenter(&releq->smq_mtx)) {
1215 1214 /* cannot get releq; a free smp may be there now */
1216 1215 mutex_exit(&allocq->smq_mtx);
1217 1216
1218 1217 /*
1219 1218 * This loop could spin forever if this thread has
1220 1219 * higher priority than the thread that is holding
1221 1220 * releq->smq_mtx. In order to force the other thread
1222 1221 * to run, we'll lock/unlock the mutex which is safe
1223 1222 * since we just unlocked the allocq mutex.
1224 1223 */
1225 1224 mutex_enter(&releq->smq_mtx);
1226 1225 mutex_exit(&releq->smq_mtx);
1227 1226 goto retry_queue;
1228 1227 }
1229 1228 if (releq->smq_free == NULL) {
1230 1229 /*
1231 1230 * This freelist is empty.
1232 1231 * This should not happen unless clients
1233 1232 * are failing to release the segmap
1234 1233 * window after accessing the data.
1235 1234 * Before resorting to sleeping, try
1236 1235 * the next list of the same color.
1237 1236 */
1238 1237 free_ndx = (free_ndx + smd_ncolor) & smd_freemsk;
1239 1238 if (free_ndx != end_ndx) {
1240 1239 mutex_exit(&releq->smq_mtx);
1241 1240 mutex_exit(&allocq->smq_mtx);
1242 1241 sm = &smd_free[free_ndx];
1243 1242 goto retry_queue;
1244 1243 }
1245 1244 /*
1246 1245 * Tried all freelists of the same color once,
1247 1246 * wait on this list and hope something gets freed.
1248 1247 */
1249 1248 segmapcnt.smp_get_nofree.value.ul++;
1250 1249 sm->sm_want++;
1251 1250 mutex_exit(&sm->sm_freeq[1].smq_mtx);
1252 1251 cv_wait(&sm->sm_free_cv,
1253 1252 &sm->sm_freeq[0].smq_mtx);
1254 1253 sm->sm_want--;
1255 1254 mutex_exit(&sm->sm_freeq[0].smq_mtx);
1256 1255 sm = &smd_free[free_ndx];
1257 1256 goto retry_queue;
1258 1257 } else {
1259 1258 /*
1260 1259 * Something on the rele queue; flip the alloc
1261 1260 * and rele queues and retry.
1262 1261 */
1263 1262 sm->sm_allocq = releq;
1264 1263 sm->sm_releq = allocq;
1265 1264 mutex_exit(&allocq->smq_mtx);
1266 1265 mutex_exit(&releq->smq_mtx);
1267 1266 if (page_locked) {
1268 1267 delay(hz >> 2);
1269 1268 page_locked = 0;
1270 1269 }
1271 1270 goto retry_queue;
1272 1271 }
1273 1272 } else {
1274 1273 /*
1275 1274 * Fastpath the case we get the smap mutex
1276 1275 * on the first try.
1277 1276 */
1278 1277 first = smp;
1279 1278 next_smap:
1280 1279 smtx = SMAPMTX(smp);
1281 1280 if (!mutex_tryenter(smtx)) {
1282 1281 /*
1283 1282 * Another thread is trying to reclaim this slot.
1284 1283 * Skip to the next queue or smap.
1285 1284 */
1286 1285 if ((smp = smp->sm_next) == first) {
1287 1286 goto skip_queue;
1288 1287 } else {
1289 1288 goto next_smap;
1290 1289 }
1291 1290 } else {
1292 1291 /*
1293 1292 * if kpme exists, get shared lock on the page
1294 1293 */
1295 1294 if (segmap_kpm && smp->sm_vp != NULL) {
1296 1295
1297 1296 kpme = GET_KPME(smp);
1298 1297 pp = kpme->kpe_page;
1299 1298
1300 1299 if (pp != NULL) {
1301 1300 if (!page_trylock(pp, SE_SHARED)) {
1302 1301 smp = smp->sm_next;
1303 1302 mutex_exit(smtx);
1304 1303 page_locked = 1;
1305 1304
1306 1305 pp = NULL;
1307 1306
1308 1307 if (smp == first) {
1309 1308 goto skip_queue;
1310 1309 } else {
1311 1310 goto next_smap;
1312 1311 }
1313 1312 } else {
1314 1313 if (kpme->kpe_page == NULL) {
1315 1314 page_unlock(pp);
1316 1315 pp = NULL;
1317 1316 }
1318 1317 }
1319 1318 }
1320 1319 }
1321 1320
1322 1321 /*
1323 1322 * At this point, we've selected smp. Remove smp
1324 1323 * from its freelist. If smp is the first one in
1325 1324 * the freelist, update the head of the freelist.
1326 1325 */
1327 1326 if (first == smp) {
1328 1327 ASSERT(first == allocq->smq_free);
1329 1328 allocq->smq_free = smp->sm_next;
1330 1329 }
1331 1330
1332 1331 /*
1333 1332 * if the head of the freelist still points to smp,
1334 1333 * then there are no more free smaps in that list.
1335 1334 */
1336 1335 if (allocq->smq_free == smp)
1337 1336 /*
1338 1337 * Took the last one
1339 1338 */
1340 1339 allocq->smq_free = NULL;
1341 1340 else {
1342 1341 smp->sm_prev->sm_next = smp->sm_next;
1343 1342 smp->sm_next->sm_prev = smp->sm_prev;
1344 1343 }
1345 1344 mutex_exit(&allocq->smq_mtx);
1346 1345 smp->sm_prev = smp->sm_next = NULL;
1347 1346
1348 1347 /*
1349 1348 * if pp != NULL, pp must have been locked;
1350 1349 * grab_smp() unlocks pp.
1351 1350 */
1352 1351 ASSERT((pp == NULL) || PAGE_LOCKED(pp));
1353 1352 grab_smp(smp, pp);
1354 1353 /* return smp locked. */
1355 1354 ASSERT(SMAPMTX(smp) == smtx);
1356 1355 ASSERT(MUTEX_HELD(smtx));
1357 1356 return (smp);
1358 1357 }
1359 1358 }
1360 1359 }
1361 1360
1362 1361 /*
1363 1362 * Special public segmap operations
1364 1363 */
1365 1364
1366 1365 /*
1367 1366 * Create pages (without using VOP_GETPAGE) and load up translations to them.
1368 1367 * If softlock is TRUE, then set things up so that it looks like a call
1369 1368 * to segmap_fault with F_SOFTLOCK.
1370 1369 *
1371 1370 * Returns 1, if a page is created by calling page_create_va(), or 0 otherwise.
1372 1371 *
1373 1372 * All fields in the generic segment (struct seg) are considered to be
1374 1373 * read-only for "segmap" even though the kernel address space (kas) may
1375 1374 * not be locked, hence no lock is needed to access them.
1376 1375 */
1377 1376 int
1378 1377 segmap_pagecreate(struct seg *seg, caddr_t addr, size_t len, int softlock)
1379 1378 {
1380 1379 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
1381 1380 page_t *pp;
1382 1381 u_offset_t off;
1383 1382 struct smap *smp;
1384 1383 struct vnode *vp;
1385 1384 caddr_t eaddr;
1386 1385 int newpage = 0;
1387 1386 uint_t prot;
1388 1387 kmutex_t *smtx;
1389 1388 int hat_flag;
1390 1389
1391 1390 ASSERT(seg->s_as == &kas);
1392 1391
1393 1392 if (segmap_kpm && IS_KPM_ADDR(addr)) {
1394 1393 /*
1395 1394 * Pages are successfully prefaulted and locked in
1396 1395 * segmap_getmapflt and can't be unlocked until
1397 1396 * segmap_release. The SM_KPM_NEWPAGE flag is set
1398 1397 * in segmap_pagecreate_kpm when new pages are created.
1399 1398 * and it is returned as "newpage" indication here.
1400 1399 */
1401 1400 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
1402 1401 panic("segmap_pagecreate: smap not found "
1403 1402 "for addr %p", (void *)addr);
1404 1403 /*NOTREACHED*/
1405 1404 }
1406 1405
1407 1406 smtx = SMAPMTX(smp);
1408 1407 newpage = smp->sm_flags & SM_KPM_NEWPAGE;
1409 1408 smp->sm_flags &= ~SM_KPM_NEWPAGE;
1410 1409 mutex_exit(smtx);
1411 1410
1412 1411 return (newpage);
1413 1412 }
1414 1413
1415 1414 smd_cpu[CPU->cpu_seqid].scpu.scpu_pagecreate++;
1416 1415
1417 1416 eaddr = addr + len;
1418 1417 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1419 1418
1420 1419 smp = GET_SMAP(seg, addr);
1421 1420
1422 1421 /*
1423 1422 * We don't grab smp mutex here since we assume the smp
1424 1423 * has a refcnt set already which prevents the slot from
1425 1424 * changing its id.
1426 1425 */
1427 1426 ASSERT(smp->sm_refcnt > 0);
1428 1427
1429 1428 vp = smp->sm_vp;
1430 1429 off = smp->sm_off + ((u_offset_t)((uintptr_t)addr & MAXBOFFSET));
1431 1430 prot = smd->smd_prot;
1432 1431
1433 1432 for (; addr < eaddr; addr += PAGESIZE, off += PAGESIZE) {
1434 1433 hat_flag = HAT_LOAD;
1435 1434 pp = page_lookup(vp, off, SE_SHARED);
1436 1435 if (pp == NULL) {
1437 1436 ushort_t bitindex;
1438 1437
1439 1438 if ((pp = page_create_va(vp, off,
1440 1439 PAGESIZE, PG_WAIT, seg, addr)) == NULL) {
1441 1440 panic("segmap_pagecreate: page_create failed");
1442 1441 /*NOTREACHED*/
1443 1442 }
1444 1443 newpage = 1;
1445 1444 page_io_unlock(pp);
1446 1445
1447 1446 /*
1448 1447 * Since pages created here do not contain valid
1449 1448 * data until the caller writes into them, the
1450 1449 * "exclusive" lock will not be dropped to prevent
1451 1450 * other users from accessing the page. We also
1452 1451 * have to lock the translation to prevent a fault
1453 1452 * from occurring when the virtual address mapped by
1454 1453 * this page is written into. This is necessary to
1455 1454 * avoid a deadlock since we haven't dropped the
1456 1455 * "exclusive" lock.
1457 1456 */
1458 1457 bitindex = (ushort_t)((off - smp->sm_off) >> PAGESHIFT);
1459 1458
1460 1459 /*
1461 1460 * Large Files: The following assertion is to
1462 1461 * verify the cast above.
1463 1462 */
1464 1463 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
1465 1464 smtx = SMAPMTX(smp);
1466 1465 mutex_enter(smtx);
1467 1466 smp->sm_bitmap |= SMAP_BIT_MASK(bitindex);
1468 1467 mutex_exit(smtx);
1469 1468
1470 1469 hat_flag = HAT_LOAD_LOCK;
1471 1470 } else if (softlock) {
1472 1471 hat_flag = HAT_LOAD_LOCK;
1473 1472 }
1474 1473
1475 1474 if (IS_VMODSORT(pp->p_vnode) && (prot & PROT_WRITE))
1476 1475 hat_setmod(pp);
1477 1476
1478 1477 hat_memload(kas.a_hat, addr, pp, prot, hat_flag);
1479 1478
1480 1479 if (hat_flag != HAT_LOAD_LOCK)
1481 1480 page_unlock(pp);
1482 1481
1483 1482 TRACE_5(TR_FAC_VM, TR_SEGMAP_PAGECREATE,
1484 1483 "segmap_pagecreate:seg %p addr %p pp %p vp %p offset %llx",
1485 1484 seg, addr, pp, vp, off);
1486 1485 }
1487 1486
1488 1487 return (newpage);
1489 1488 }
1490 1489
1491 1490 void
1492 1491 segmap_pageunlock(struct seg *seg, caddr_t addr, size_t len, enum seg_rw rw)
1493 1492 {
1494 1493 struct smap *smp;
1495 1494 ushort_t bitmask;
1496 1495 page_t *pp;
1497 1496 struct vnode *vp;
1498 1497 u_offset_t off;
1499 1498 caddr_t eaddr;
1500 1499 kmutex_t *smtx;
1501 1500
1502 1501 ASSERT(seg->s_as == &kas);
1503 1502
1504 1503 eaddr = addr + len;
1505 1504 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1506 1505
1507 1506 if (segmap_kpm && IS_KPM_ADDR(addr)) {
1508 1507 /*
1509 1508 * Pages are successfully prefaulted and locked in
1510 1509 * segmap_getmapflt and can't be unlocked until
1511 1510 * segmap_release, so no pages or hat mappings have
1512 1511 * to be unlocked at this point.
1513 1512 */
1514 1513 #ifdef DEBUG
1515 1514 if ((smp = get_smap_kpm(addr, NULL)) == NULL) {
1516 1515 panic("segmap_pageunlock: smap not found "
1517 1516 "for addr %p", (void *)addr);
1518 1517 /*NOTREACHED*/
1519 1518 }
1520 1519
1521 1520 ASSERT(smp->sm_refcnt > 0);
1522 1521 mutex_exit(SMAPMTX(smp));
1523 1522 #endif
1524 1523 return;
1525 1524 }
1526 1525
1527 1526 smp = GET_SMAP(seg, addr);
1528 1527 smtx = SMAPMTX(smp);
1529 1528
1530 1529 ASSERT(smp->sm_refcnt > 0);
1531 1530
1532 1531 vp = smp->sm_vp;
1533 1532 off = smp->sm_off + ((u_offset_t)((uintptr_t)addr & MAXBOFFSET));
1534 1533
1535 1534 for (; addr < eaddr; addr += PAGESIZE, off += PAGESIZE) {
1536 1535 bitmask = SMAP_BIT_MASK((int)(off - smp->sm_off) >> PAGESHIFT);
1537 1536
1538 1537 /*
1539 1538 * Large Files: Following assertion is to verify
1540 1539 * the correctness of the cast to (int) above.
1541 1540 */
1542 1541 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
1543 1542
1544 1543 /*
1545 1544 * If the bit corresponding to "off" is set,
1546 1545 * clear this bit in the bitmap, unlock translations,
1547 1546 * and release the "exclusive" lock on the page.
1548 1547 */
1549 1548 if (smp->sm_bitmap & bitmask) {
1550 1549 mutex_enter(smtx);
1551 1550 smp->sm_bitmap &= ~bitmask;
1552 1551 mutex_exit(smtx);
1553 1552
1554 1553 hat_unlock(kas.a_hat, addr, PAGESIZE);
1555 1554
1556 1555 /*
1557 1556 * Use page_find() instead of page_lookup() to
1558 1557 * find the page since we know that it has
1559 1558 * "exclusive" lock.
1560 1559 */
1561 1560 pp = page_find(vp, off);
1562 1561 if (pp == NULL) {
1563 1562 panic("segmap_pageunlock: page not found");
1564 1563 /*NOTREACHED*/
1565 1564 }
1566 1565 if (rw == S_WRITE) {
1567 1566 hat_setrefmod(pp);
1568 1567 } else if (rw != S_OTHER) {
1569 1568 hat_setref(pp);
1570 1569 }
1571 1570
1572 1571 page_unlock(pp);
1573 1572 }
1574 1573 }
1575 1574 }
1576 1575
1577 1576 caddr_t
1578 1577 segmap_getmap(struct seg *seg, struct vnode *vp, u_offset_t off)
1579 1578 {
1580 1579 return (segmap_getmapflt(seg, vp, off, MAXBSIZE, 0, S_OTHER));
1581 1580 }
1582 1581
1583 1582 /*
1584 1583 * This is the magic virtual address that offset 0 of an ELF
1585 1584 * file gets mapped to in user space. This is used to pick
1586 1585 * the vac color on the freelist.
1587 1586 */
1588 1587 #define ELF_OFFZERO_VA (0x10000)
1589 1588 /*
1590 1589 * segmap_getmap allocates a MAXBSIZE big slot to map the vnode vp
1591 1590 * in the range <off, off + len). off doesn't need to be MAXBSIZE aligned.
1592 1591 * The return address is always MAXBSIZE aligned.
1593 1592 *
1594 1593 * If forcefault is nonzero and the MMU translations haven't yet been created,
1595 1594 * segmap_getmap will call segmap_fault(..., F_INVAL, rw) to create them.
1596 1595 */
1597 1596 caddr_t
1598 1597 segmap_getmapflt(
1599 1598 struct seg *seg,
1600 1599 struct vnode *vp,
1601 1600 u_offset_t off,
1602 1601 size_t len,
1603 1602 int forcefault,
1604 1603 enum seg_rw rw)
1605 1604 {
1606 1605 struct smap *smp, *nsmp;
1607 1606 extern struct vnode *common_specvp();
1608 1607 caddr_t baseaddr; /* MAXBSIZE aligned */
1609 1608 u_offset_t baseoff;
1610 1609 int newslot;
1611 1610 caddr_t vaddr;
1612 1611 int color, hashid;
1613 1612 kmutex_t *hashmtx, *smapmtx;
1614 1613 struct smfree *sm;
1615 1614 page_t *pp;
1616 1615 struct kpme *kpme;
1617 1616 uint_t prot;
1618 1617 caddr_t base;
1619 1618 page_t *pl[MAXPPB + 1];
1620 1619 int error;
1621 1620 int is_kpm = 1;
1622 1621
1623 1622 ASSERT(seg->s_as == &kas);
1624 1623 ASSERT(seg == segkmap);
1625 1624
1626 1625 baseoff = off & (offset_t)MAXBMASK;
1627 1626 if (off + len > baseoff + MAXBSIZE) {
1628 1627 panic("segmap_getmap bad len");
1629 1628 /*NOTREACHED*/
1630 1629 }
1631 1630
1632 1631 /*
1633 1632 * If this is a block device we have to be sure to use the
1634 1633 * "common" block device vnode for the mapping.
1635 1634 */
1636 1635 if (vp->v_type == VBLK)
1637 1636 vp = common_specvp(vp);
1638 1637
1639 1638 smd_cpu[CPU->cpu_seqid].scpu.scpu_getmap++;
1640 1639
1641 1640 if (segmap_kpm == 0 ||
1642 1641 (forcefault == SM_PAGECREATE && rw != S_WRITE)) {
1643 1642 is_kpm = 0;
1644 1643 }
1645 1644
1646 1645 SMAP_HASHFUNC(vp, off, hashid); /* macro assigns hashid */
1647 1646 hashmtx = SHASHMTX(hashid);
1648 1647
1649 1648 retry_hash:
1650 1649 mutex_enter(hashmtx);
1651 1650 for (smp = smd_hash[hashid].sh_hash_list;
1652 1651 smp != NULL; smp = smp->sm_hash)
1653 1652 if (smp->sm_vp == vp && smp->sm_off == baseoff)
1654 1653 break;
1655 1654 mutex_exit(hashmtx);
1656 1655
1657 1656 vrfy_smp:
1658 1657 if (smp != NULL) {
1659 1658
1660 1659 ASSERT(vp->v_count != 0);
1661 1660
1662 1661 /*
1663 1662 * Get smap lock and recheck its tag. The hash lock
1664 1663 * is dropped since the hash is based on (vp, off)
1665 1664 * and (vp, off) won't change when we have smap mtx.
1666 1665 */
1667 1666 smapmtx = SMAPMTX(smp);
1668 1667 mutex_enter(smapmtx);
1669 1668 if (smp->sm_vp != vp || smp->sm_off != baseoff) {
1670 1669 mutex_exit(smapmtx);
1671 1670 goto retry_hash;
1672 1671 }
1673 1672
1674 1673 if (smp->sm_refcnt == 0) {
1675 1674
1676 1675 smd_cpu[CPU->cpu_seqid].scpu.scpu_get_reclaim++;
1677 1676
1678 1677 /*
1679 1678 * Could still be on the free list. However, this
1680 1679 * could also be an smp that is transitioning from
1681 1680 * the free list when we have too much contention
1682 1681 * for the smapmtx's. In this case, we have an
1683 1682 * unlocked smp that is not on the free list any
1684 1683 * longer, but still has a 0 refcnt. The only way
1685 1684 * to be sure is to check the freelist pointers.
1686 1685 * Since we now have the smapmtx, we are guaranteed
1687 1686 * that the (vp, off) won't change, so we are safe
1688 1687 * to reclaim it. get_free_smp() knows that this
1689 1688 * can happen, and it will check the refcnt.
1690 1689 */
1691 1690
1692 1691 if ((smp->sm_next != NULL)) {
1693 1692 struct sm_freeq *freeq;
1694 1693
1695 1694 ASSERT(smp->sm_prev != NULL);
1696 1695 sm = &smd_free[smp->sm_free_ndx];
1697 1696
1698 1697 if (smp->sm_flags & SM_QNDX_ZERO)
1699 1698 freeq = &sm->sm_freeq[0];
1700 1699 else
1701 1700 freeq = &sm->sm_freeq[1];
1702 1701
1703 1702 mutex_enter(&freeq->smq_mtx);
1704 1703 if (freeq->smq_free != smp) {
1705 1704 /*
1706 1705 * fastpath normal case
1707 1706 */
1708 1707 smp->sm_prev->sm_next = smp->sm_next;
1709 1708 smp->sm_next->sm_prev = smp->sm_prev;
1710 1709 } else if (smp == smp->sm_next) {
1711 1710 /*
1712 1711 * Taking the last smap on freelist
1713 1712 */
1714 1713 freeq->smq_free = NULL;
1715 1714 } else {
1716 1715 /*
1717 1716 * Reclaiming 1st smap on list
1718 1717 */
1719 1718 freeq->smq_free = smp->sm_next;
1720 1719 smp->sm_prev->sm_next = smp->sm_next;
1721 1720 smp->sm_next->sm_prev = smp->sm_prev;
1722 1721 }
1723 1722 mutex_exit(&freeq->smq_mtx);
1724 1723 smp->sm_prev = smp->sm_next = NULL;
1725 1724 } else {
1726 1725 ASSERT(smp->sm_prev == NULL);
1727 1726 segmapcnt.smp_stolen.value.ul++;
1728 1727 }
1729 1728
1730 1729 } else {
1731 1730 segmapcnt.smp_get_use.value.ul++;
1732 1731 }
1733 1732 smp->sm_refcnt++; /* another user */
1734 1733
1735 1734 /*
1736 1735 * We don't invoke segmap_fault via TLB miss, so we set ref
1737 1736 * and mod bits in advance. For S_OTHER we set them in
1738 1737 * segmap_fault F_SOFTUNLOCK.
1739 1738 */
1740 1739 if (is_kpm) {
1741 1740 if (rw == S_WRITE) {
1742 1741 smp->sm_flags |= SM_WRITE_DATA;
1743 1742 } else if (rw == S_READ) {
1744 1743 smp->sm_flags |= SM_READ_DATA;
1745 1744 }
1746 1745 }
1747 1746 mutex_exit(smapmtx);
1748 1747
1749 1748 newslot = 0;
1750 1749 } else {
1751 1750
1752 1751 uint32_t free_ndx, *free_ndxp;
1753 1752 union segmap_cpu *scpu;
1754 1753
1755 1754 /*
1756 1755 * On a PAC machine or a machine with anti-alias
1757 1756 * hardware, smd_colormsk will be zero.
1758 1757 *
1759 1758 * On a VAC machine- pick color by offset in the file
1760 1759 * so we won't get VAC conflicts on elf files.
1761 1760 * On data files, color does not matter but we
1762 1761 * don't know what kind of file it is so we always
1763 1762 * pick color by offset. This causes color
1764 1763 * corresponding to file offset zero to be used more
1765 1764 * heavily.
1766 1765 */
1767 1766 color = (baseoff >> MAXBSHIFT) & smd_colormsk;
1768 1767 scpu = smd_cpu+CPU->cpu_seqid;
1769 1768 free_ndxp = &scpu->scpu.scpu_free_ndx[color];
1770 1769 free_ndx = (*free_ndxp += smd_ncolor) & smd_freemsk;
1771 1770 #ifdef DEBUG
1772 1771 colors_used[free_ndx]++;
1773 1772 #endif /* DEBUG */
1774 1773
1775 1774 /*
1776 1775 * Get a locked smp slot from the free list.
1777 1776 */
1778 1777 smp = get_free_smp(free_ndx);
1779 1778 smapmtx = SMAPMTX(smp);
1780 1779
1781 1780 ASSERT(smp->sm_vp == NULL);
1782 1781
1783 1782 if ((nsmp = segmap_hashin(smp, vp, baseoff, hashid)) != NULL) {
1784 1783 /*
1785 1784 * Failed to hashin, there exists one now.
1786 1785 * Return the smp we just allocated.
1787 1786 */
1788 1787 segmap_smapadd(smp);
1789 1788 mutex_exit(smapmtx);
1790 1789
1791 1790 smp = nsmp;
1792 1791 goto vrfy_smp;
1793 1792 }
1794 1793 smp->sm_refcnt++; /* another user */
1795 1794
1796 1795 /*
1797 1796 * We don't invoke segmap_fault via TLB miss, so we set ref
1798 1797 * and mod bits in advance. For S_OTHER we set them in
1799 1798 * segmap_fault F_SOFTUNLOCK.
1800 1799 */
1801 1800 if (is_kpm) {
1802 1801 if (rw == S_WRITE) {
1803 1802 smp->sm_flags |= SM_WRITE_DATA;
1804 1803 } else if (rw == S_READ) {
1805 1804 smp->sm_flags |= SM_READ_DATA;
1806 1805 }
1807 1806 }
1808 1807 mutex_exit(smapmtx);
1809 1808
1810 1809 newslot = 1;
1811 1810 }
1812 1811
1813 1812 if (!is_kpm)
1814 1813 goto use_segmap_range;
1815 1814
1816 1815 /*
1817 1816 * Use segkpm
1818 1817 */
1819 1818 /* Lint directive required until 6746211 is fixed */
1820 1819 /*CONSTCOND*/
1821 1820 ASSERT(PAGESIZE == MAXBSIZE);
1822 1821
1823 1822 /*
1824 1823 * remember the last smp faulted on this cpu.
1825 1824 */
1826 1825 (smd_cpu+CPU->cpu_seqid)->scpu.scpu_last_smap = smp;
1827 1826
1828 1827 if (forcefault == SM_PAGECREATE) {
1829 1828 baseaddr = segmap_pagecreate_kpm(seg, vp, baseoff, smp, rw);
1830 1829 return (baseaddr);
1831 1830 }
1832 1831
1833 1832 if (newslot == 0 &&
1834 1833 (pp = GET_KPME(smp)->kpe_page) != NULL) {
1835 1834
1836 1835 /* fastpath */
1837 1836 switch (rw) {
1838 1837 case S_READ:
1839 1838 case S_WRITE:
1840 1839 if (page_trylock(pp, SE_SHARED)) {
1841 1840 if (PP_ISFREE(pp) ||
1842 1841 !(pp->p_vnode == vp &&
1843 1842 pp->p_offset == baseoff)) {
1844 1843 page_unlock(pp);
1845 1844 pp = page_lookup(vp, baseoff,
1846 1845 SE_SHARED);
1847 1846 }
1848 1847 } else {
1849 1848 pp = page_lookup(vp, baseoff, SE_SHARED);
1850 1849 }
1851 1850
1852 1851 if (pp == NULL) {
1853 1852 ASSERT(GET_KPME(smp)->kpe_page == NULL);
1854 1853 break;
1855 1854 }
1856 1855
1857 1856 if (rw == S_WRITE &&
1858 1857 hat_page_getattr(pp, P_MOD | P_REF) !=
1859 1858 (P_MOD | P_REF)) {
1860 1859 page_unlock(pp);
1861 1860 break;
1862 1861 }
1863 1862
1864 1863 /*
1865 1864 * We have the p_selock as reader, grab_smp
1866 1865 * can't hit us, we have bumped the smap
1867 1866 * refcnt and hat_pageunload needs the
1868 1867 * p_selock exclusive.
1869 1868 */
1870 1869 kpme = GET_KPME(smp);
1871 1870 if (kpme->kpe_page == pp) {
1872 1871 baseaddr = hat_kpm_page2va(pp, 0);
1873 1872 } else if (kpme->kpe_page == NULL) {
1874 1873 baseaddr = hat_kpm_mapin(pp, kpme);
1875 1874 } else {
1876 1875 panic("segmap_getmapflt: stale "
1877 1876 "kpme page, kpme %p", (void *)kpme);
1878 1877 /*NOTREACHED*/
1879 1878 }
1880 1879
1881 1880 /*
1882 1881 * We don't invoke segmap_fault via TLB miss,
1883 1882 * so we set ref and mod bits in advance.
1884 1883 * For S_OTHER and we set them in segmap_fault
1885 1884 * F_SOFTUNLOCK.
1886 1885 */
1887 1886 if (rw == S_READ && !hat_isref(pp))
1888 1887 hat_setref(pp);
1889 1888
1890 1889 return (baseaddr);
1891 1890 default:
1892 1891 break;
1893 1892 }
1894 1893 }
1895 1894
1896 1895 base = segkpm_create_va(baseoff);
1897 1896 error = VOP_GETPAGE(vp, (offset_t)baseoff, len, &prot, pl, MAXBSIZE,
1898 1897 seg, base, rw, CRED(), NULL);
1899 1898
1900 1899 pp = pl[0];
1901 1900 if (error || pp == NULL) {
1902 1901 /*
1903 1902 * Use segmap address slot and let segmap_fault deal
1904 1903 * with the error cases. There is no error return
1905 1904 * possible here.
1906 1905 */
1907 1906 goto use_segmap_range;
1908 1907 }
1909 1908
1910 1909 ASSERT(pl[1] == NULL);
1911 1910
1912 1911 /*
1913 1912 * When prot is not returned w/ PROT_ALL the returned pages
1914 1913 * are not backed by fs blocks. For most of the segmap users
1915 1914 * this is no problem, they don't write to the pages in the
1916 1915 * same request and therefore don't rely on a following
1917 1916 * trap driven segmap_fault. With SM_LOCKPROTO users it
1918 1917 * is more secure to use segkmap adresses to allow
1919 1918 * protection segmap_fault's.
1920 1919 */
1921 1920 if (prot != PROT_ALL && forcefault == SM_LOCKPROTO) {
1922 1921 /*
1923 1922 * Use segmap address slot and let segmap_fault
1924 1923 * do the error return.
1925 1924 */
1926 1925 ASSERT(rw != S_WRITE);
1927 1926 ASSERT(PAGE_LOCKED(pp));
1928 1927 page_unlock(pp);
1929 1928 forcefault = 0;
1930 1929 goto use_segmap_range;
1931 1930 }
1932 1931
1933 1932 /*
1934 1933 * We have the p_selock as reader, grab_smp can't hit us, we
1935 1934 * have bumped the smap refcnt and hat_pageunload needs the
1936 1935 * p_selock exclusive.
1937 1936 */
1938 1937 kpme = GET_KPME(smp);
1939 1938 if (kpme->kpe_page == pp) {
1940 1939 baseaddr = hat_kpm_page2va(pp, 0);
1941 1940 } else if (kpme->kpe_page == NULL) {
1942 1941 baseaddr = hat_kpm_mapin(pp, kpme);
1943 1942 } else {
1944 1943 panic("segmap_getmapflt: stale kpme page after "
1945 1944 "VOP_GETPAGE, kpme %p", (void *)kpme);
1946 1945 /*NOTREACHED*/
1947 1946 }
1948 1947
1949 1948 smd_cpu[CPU->cpu_seqid].scpu.scpu_fault++;
1950 1949
1951 1950 return (baseaddr);
1952 1951
1953 1952
1954 1953 use_segmap_range:
1955 1954 baseaddr = seg->s_base + ((smp - smd_smap) * MAXBSIZE);
1956 1955 TRACE_4(TR_FAC_VM, TR_SEGMAP_GETMAP,
1957 1956 "segmap_getmap:seg %p addr %p vp %p offset %llx",
1958 1957 seg, baseaddr, vp, baseoff);
1959 1958
1960 1959 /*
1961 1960 * Prefault the translations
1962 1961 */
1963 1962 vaddr = baseaddr + (off - baseoff);
1964 1963 if (forcefault && (newslot || !hat_probe(kas.a_hat, vaddr))) {
1965 1964
1966 1965 caddr_t pgaddr = (caddr_t)((uintptr_t)vaddr &
1967 1966 (uintptr_t)PAGEMASK);
1968 1967
1969 1968 (void) segmap_fault(kas.a_hat, seg, pgaddr,
1970 1969 (vaddr + len - pgaddr + PAGESIZE - 1) & (uintptr_t)PAGEMASK,
1971 1970 F_INVAL, rw);
1972 1971 }
1973 1972
1974 1973 return (baseaddr);
1975 1974 }
1976 1975
1977 1976 int
1978 1977 segmap_release(struct seg *seg, caddr_t addr, uint_t flags)
1979 1978 {
1980 1979 struct smap *smp;
1981 1980 int error;
1982 1981 int bflags = 0;
1983 1982 struct vnode *vp;
1984 1983 u_offset_t offset;
1985 1984 kmutex_t *smtx;
1986 1985 int is_kpm = 0;
1987 1986 page_t *pp;
1988 1987
1989 1988 if (segmap_kpm && IS_KPM_ADDR(addr)) {
1990 1989
1991 1990 if (((uintptr_t)addr & MAXBOFFSET) != 0) {
1992 1991 panic("segmap_release: addr %p not "
1993 1992 "MAXBSIZE aligned", (void *)addr);
1994 1993 /*NOTREACHED*/
1995 1994 }
1996 1995
1997 1996 if ((smp = get_smap_kpm(addr, &pp)) == NULL) {
1998 1997 panic("segmap_release: smap not found "
1999 1998 "for addr %p", (void *)addr);
2000 1999 /*NOTREACHED*/
2001 2000 }
2002 2001
2003 2002 TRACE_3(TR_FAC_VM, TR_SEGMAP_RELMAP,
2004 2003 "segmap_relmap:seg %p addr %p smp %p",
2005 2004 seg, addr, smp);
2006 2005
2007 2006 smtx = SMAPMTX(smp);
2008 2007
2009 2008 /*
2010 2009 * For compatibility reasons segmap_pagecreate_kpm sets this
2011 2010 * flag to allow a following segmap_pagecreate to return
2012 2011 * this as "newpage" flag. When segmap_pagecreate is not
2013 2012 * called at all we clear it now.
2014 2013 */
2015 2014 smp->sm_flags &= ~SM_KPM_NEWPAGE;
2016 2015 is_kpm = 1;
2017 2016 if (smp->sm_flags & SM_WRITE_DATA) {
2018 2017 hat_setrefmod(pp);
2019 2018 } else if (smp->sm_flags & SM_READ_DATA) {
2020 2019 hat_setref(pp);
2021 2020 }
2022 2021 } else {
2023 2022 if (addr < seg->s_base || addr >= seg->s_base + seg->s_size ||
2024 2023 ((uintptr_t)addr & MAXBOFFSET) != 0) {
2025 2024 panic("segmap_release: bad addr %p", (void *)addr);
2026 2025 /*NOTREACHED*/
2027 2026 }
2028 2027 smp = GET_SMAP(seg, addr);
2029 2028
2030 2029 TRACE_3(TR_FAC_VM, TR_SEGMAP_RELMAP,
2031 2030 "segmap_relmap:seg %p addr %p smp %p",
2032 2031 seg, addr, smp);
2033 2032
2034 2033 smtx = SMAPMTX(smp);
2035 2034 mutex_enter(smtx);
2036 2035 smp->sm_flags |= SM_NOTKPM_RELEASED;
2037 2036 }
2038 2037
2039 2038 ASSERT(smp->sm_refcnt > 0);
2040 2039
2041 2040 /*
2042 2041 * Need to call VOP_PUTPAGE() if any flags (except SM_DONTNEED)
2043 2042 * are set.
2044 2043 */
2045 2044 if ((flags & ~SM_DONTNEED) != 0) {
2046 2045 if (flags & SM_WRITE)
2047 2046 segmapcnt.smp_rel_write.value.ul++;
2048 2047 if (flags & SM_ASYNC) {
2049 2048 bflags |= B_ASYNC;
2050 2049 segmapcnt.smp_rel_async.value.ul++;
2051 2050 }
2052 2051 if (flags & SM_INVAL) {
2053 2052 bflags |= B_INVAL;
2054 2053 segmapcnt.smp_rel_abort.value.ul++;
2055 2054 }
2056 2055 if (flags & SM_DESTROY) {
2057 2056 bflags |= (B_INVAL|B_TRUNC);
2058 2057 segmapcnt.smp_rel_abort.value.ul++;
2059 2058 }
2060 2059 if (smp->sm_refcnt == 1) {
2061 2060 /*
2062 2061 * We only bother doing the FREE and DONTNEED flags
2063 2062 * if no one else is still referencing this mapping.
2064 2063 */
2065 2064 if (flags & SM_FREE) {
2066 2065 bflags |= B_FREE;
2067 2066 segmapcnt.smp_rel_free.value.ul++;
2068 2067 }
2069 2068 if (flags & SM_DONTNEED) {
2070 2069 bflags |= B_DONTNEED;
2071 2070 segmapcnt.smp_rel_dontneed.value.ul++;
2072 2071 }
2073 2072 }
2074 2073 } else {
2075 2074 smd_cpu[CPU->cpu_seqid].scpu.scpu_release++;
2076 2075 }
2077 2076
2078 2077 vp = smp->sm_vp;
2079 2078 offset = smp->sm_off;
2080 2079
2081 2080 if (--smp->sm_refcnt == 0) {
2082 2081
2083 2082 smp->sm_flags &= ~(SM_WRITE_DATA | SM_READ_DATA);
2084 2083
2085 2084 if (flags & (SM_INVAL|SM_DESTROY)) {
2086 2085 segmap_hashout(smp); /* remove map info */
2087 2086 if (is_kpm) {
2088 2087 hat_kpm_mapout(pp, GET_KPME(smp), addr);
2089 2088 if (smp->sm_flags & SM_NOTKPM_RELEASED) {
2090 2089 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
2091 2090 hat_unload(kas.a_hat, segkmap->s_base +
2092 2091 ((smp - smd_smap) * MAXBSIZE),
2093 2092 MAXBSIZE, HAT_UNLOAD);
2094 2093 }
2095 2094
2096 2095 } else {
2097 2096 if (segmap_kpm)
2098 2097 segkpm_mapout_validkpme(GET_KPME(smp));
2099 2098
2100 2099 smp->sm_flags &= ~SM_NOTKPM_RELEASED;
2101 2100 hat_unload(kas.a_hat, addr, MAXBSIZE,
2102 2101 HAT_UNLOAD);
2103 2102 }
2104 2103 }
2105 2104 segmap_smapadd(smp); /* add to free list */
2106 2105 }
2107 2106
2108 2107 mutex_exit(smtx);
2109 2108
2110 2109 if (is_kpm)
2111 2110 page_unlock(pp);
2112 2111 /*
2113 2112 * Now invoke VOP_PUTPAGE() if any flags (except SM_DONTNEED)
2114 2113 * are set.
2115 2114 */
2116 2115 if ((flags & ~SM_DONTNEED) != 0) {
2117 2116 error = VOP_PUTPAGE(vp, offset, MAXBSIZE,
2118 2117 bflags, CRED(), NULL);
2119 2118 } else {
2120 2119 error = 0;
2121 2120 }
2122 2121
2123 2122 return (error);
2124 2123 }
2125 2124
2126 2125 /*
2127 2126 * Dump the pages belonging to this segmap segment.
2128 2127 */
2129 2128 static void
2130 2129 segmap_dump(struct seg *seg)
2131 2130 {
2132 2131 struct segmap_data *smd;
2133 2132 struct smap *smp, *smp_end;
2134 2133 page_t *pp;
2135 2134 pfn_t pfn;
2136 2135 u_offset_t off;
2137 2136 caddr_t addr;
2138 2137
2139 2138 smd = (struct segmap_data *)seg->s_data;
2140 2139 addr = seg->s_base;
2141 2140 for (smp = smd->smd_sm, smp_end = smp + smd->smd_npages;
2142 2141 smp < smp_end; smp++) {
2143 2142
2144 2143 if (smp->sm_refcnt) {
2145 2144 for (off = 0; off < MAXBSIZE; off += PAGESIZE) {
2146 2145 int we_own_it = 0;
2147 2146
2148 2147 /*
2149 2148 * If pp == NULL, the page either does
2150 2149 * not exist or is exclusively locked.
2151 2150 * So determine if it exists before
2152 2151 * searching for it.
2153 2152 */
2154 2153 if ((pp = page_lookup_nowait(smp->sm_vp,
2155 2154 smp->sm_off + off, SE_SHARED)))
2156 2155 we_own_it = 1;
2157 2156 else
2158 2157 pp = page_exists(smp->sm_vp,
2159 2158 smp->sm_off + off);
2160 2159
2161 2160 if (pp) {
2162 2161 pfn = page_pptonum(pp);
2163 2162 dump_addpage(seg->s_as,
2164 2163 addr + off, pfn);
2165 2164 if (we_own_it)
2166 2165 page_unlock(pp);
2167 2166 }
2168 2167 dump_timeleft = dump_timeout;
2169 2168 }
2170 2169 }
2171 2170 addr += MAXBSIZE;
2172 2171 }
2173 2172 }
2174 2173
2175 2174 /*ARGSUSED*/
2176 2175 static int
2177 2176 segmap_pagelock(struct seg *seg, caddr_t addr, size_t len,
2178 2177 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2179 2178 {
2180 2179 return (ENOTSUP);
2181 2180 }
2182 2181
2183 2182 static int
2184 2183 segmap_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
2185 2184 {
2186 2185 struct segmap_data *smd = (struct segmap_data *)seg->s_data;
2187 2186
2188 2187 memidp->val[0] = (uintptr_t)smd->smd_sm->sm_vp;
2189 2188 memidp->val[1] = smd->smd_sm->sm_off + (uintptr_t)(addr - seg->s_base);
2190 2189 return (0);
2191 2190 }
2192 2191
2193 2192 /*ARGSUSED*/
2194 2193 static lgrp_mem_policy_info_t *
2195 2194 segmap_getpolicy(struct seg *seg, caddr_t addr)
2196 2195 {
2197 2196 return (NULL);
2198 2197 }
2199 2198
2200 2199 /*ARGSUSED*/
2201 2200 static int
2202 2201 segmap_capable(struct seg *seg, segcapability_t capability)
2203 2202 {
2204 2203 return (0);
2205 2204 }
2206 2205
2207 2206
2208 2207 #ifdef SEGKPM_SUPPORT
2209 2208
2210 2209 /*
2211 2210 * segkpm support routines
2212 2211 */
2213 2212
2214 2213 static caddr_t
2215 2214 segmap_pagecreate_kpm(struct seg *seg, vnode_t *vp, u_offset_t off,
2216 2215 struct smap *smp, enum seg_rw rw)
2217 2216 {
2218 2217 caddr_t base;
2219 2218 page_t *pp;
2220 2219 int newpage = 0;
2221 2220 struct kpme *kpme;
2222 2221
2223 2222 ASSERT(smp->sm_refcnt > 0);
2224 2223
2225 2224 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) {
2226 2225 kmutex_t *smtx;
2227 2226
2228 2227 base = segkpm_create_va(off);
2229 2228
2230 2229 if ((pp = page_create_va(vp, off, PAGESIZE, PG_WAIT,
2231 2230 seg, base)) == NULL) {
2232 2231 panic("segmap_pagecreate_kpm: "
2233 2232 "page_create failed");
2234 2233 /*NOTREACHED*/
2235 2234 }
2236 2235
2237 2236 newpage = 1;
2238 2237 page_io_unlock(pp);
2239 2238 ASSERT((u_offset_t)(off - smp->sm_off) <= INT_MAX);
2240 2239
2241 2240 /*
2242 2241 * Mark this here until the following segmap_pagecreate
2243 2242 * or segmap_release.
2244 2243 */
2245 2244 smtx = SMAPMTX(smp);
2246 2245 mutex_enter(smtx);
2247 2246 smp->sm_flags |= SM_KPM_NEWPAGE;
2248 2247 mutex_exit(smtx);
2249 2248 }
2250 2249
2251 2250 kpme = GET_KPME(smp);
2252 2251 if (!newpage && kpme->kpe_page == pp)
2253 2252 base = hat_kpm_page2va(pp, 0);
2254 2253 else
2255 2254 base = hat_kpm_mapin(pp, kpme);
2256 2255
2257 2256 /*
2258 2257 * FS code may decide not to call segmap_pagecreate and we
2259 2258 * don't invoke segmap_fault via TLB miss, so we have to set
2260 2259 * ref and mod bits in advance.
2261 2260 */
2262 2261 if (rw == S_WRITE) {
2263 2262 hat_setrefmod(pp);
2264 2263 } else {
2265 2264 ASSERT(rw == S_READ);
2266 2265 hat_setref(pp);
2267 2266 }
2268 2267
2269 2268 smd_cpu[CPU->cpu_seqid].scpu.scpu_pagecreate++;
2270 2269
2271 2270 return (base);
2272 2271 }
2273 2272
2274 2273 /*
2275 2274 * Find the smap structure corresponding to the
2276 2275 * KPM addr and return it locked.
2277 2276 */
2278 2277 struct smap *
2279 2278 get_smap_kpm(caddr_t addr, page_t **ppp)
2280 2279 {
2281 2280 struct smap *smp;
2282 2281 struct vnode *vp;
2283 2282 u_offset_t offset;
2284 2283 caddr_t baseaddr = (caddr_t)((uintptr_t)addr & MAXBMASK);
2285 2284 int hashid;
2286 2285 kmutex_t *hashmtx;
2287 2286 page_t *pp;
2288 2287 union segmap_cpu *scpu;
2289 2288
2290 2289 pp = hat_kpm_vaddr2page(baseaddr);
2291 2290
2292 2291 ASSERT(pp && !PP_ISFREE(pp));
2293 2292 ASSERT(PAGE_LOCKED(pp));
2294 2293 ASSERT(((uintptr_t)pp->p_offset & MAXBOFFSET) == 0);
2295 2294
2296 2295 vp = pp->p_vnode;
2297 2296 offset = pp->p_offset;
2298 2297 ASSERT(vp != NULL);
2299 2298
2300 2299 /*
2301 2300 * Assume the last smap used on this cpu is the one needed.
2302 2301 */
2303 2302 scpu = smd_cpu+CPU->cpu_seqid;
2304 2303 smp = scpu->scpu.scpu_last_smap;
2305 2304 mutex_enter(&smp->sm_mtx);
2306 2305 if (smp->sm_vp == vp && smp->sm_off == offset) {
2307 2306 ASSERT(smp->sm_refcnt > 0);
2308 2307 } else {
2309 2308 /*
2310 2309 * Assumption wrong, find the smap on the hash chain.
2311 2310 */
2312 2311 mutex_exit(&smp->sm_mtx);
2313 2312 SMAP_HASHFUNC(vp, offset, hashid); /* macro assigns hashid */
2314 2313 hashmtx = SHASHMTX(hashid);
2315 2314
2316 2315 mutex_enter(hashmtx);
2317 2316 smp = smd_hash[hashid].sh_hash_list;
2318 2317 for (; smp != NULL; smp = smp->sm_hash) {
2319 2318 if (smp->sm_vp == vp && smp->sm_off == offset)
2320 2319 break;
2321 2320 }
2322 2321 mutex_exit(hashmtx);
2323 2322 if (smp) {
2324 2323 mutex_enter(&smp->sm_mtx);
2325 2324 ASSERT(smp->sm_vp == vp && smp->sm_off == offset);
2326 2325 }
2327 2326 }
2328 2327
2329 2328 if (ppp)
2330 2329 *ppp = smp ? pp : NULL;
2331 2330
2332 2331 return (smp);
2333 2332 }
2334 2333
2335 2334 #else /* SEGKPM_SUPPORT */
2336 2335
2337 2336 /* segkpm stubs */
2338 2337
2339 2338 /*ARGSUSED*/
2340 2339 static caddr_t
2341 2340 segmap_pagecreate_kpm(struct seg *seg, vnode_t *vp, u_offset_t off,
2342 2341 struct smap *smp, enum seg_rw rw)
2343 2342 {
2344 2343 return (NULL);
2345 2344 }
2346 2345
2347 2346 /*ARGSUSED*/
2348 2347 struct smap *
2349 2348 get_smap_kpm(caddr_t addr, page_t **ppp)
2350 2349 {
2351 2350 return (NULL);
2352 2351 }
2353 2352
2354 2353 #endif /* SEGKPM_SUPPORT */
↓ open down ↓ |
2232 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX