1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
27
28 /*
29 * Portions of this source code were derived from Berkeley 4.3 BSD
30 * under license from the Regents of the University of California.
31 */
32
33 /*
34 * segkp is a segment driver that administers the allocation and deallocation
35 * of pageable variable size chunks of kernel virtual address space. Each
36 * allocated resource is page-aligned.
37 *
38 * The user may specify whether the resource should be initialized to 0,
39 * include a redzone, or locked in memory.
40 */
41
42 #include <sys/types.h>
43 #include <sys/t_lock.h>
44 #include <sys/thread.h>
45 #include <sys/param.h>
46 #include <sys/errno.h>
47 #include <sys/sysmacros.h>
48 #include <sys/systm.h>
49 #include <sys/buf.h>
50 #include <sys/mman.h>
51 #include <sys/vnode.h>
52 #include <sys/cmn_err.h>
53 #include <sys/swap.h>
54 #include <sys/tuneable.h>
55 #include <sys/kmem.h>
56 #include <sys/vmem.h>
57 #include <sys/cred.h>
58 #include <sys/dumphdr.h>
59 #include <sys/debug.h>
60 #include <sys/vtrace.h>
61 #include <sys/stack.h>
62 #include <sys/atomic.h>
63 #include <sys/archsystm.h>
64 #include <sys/lgrp.h>
65
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_kp.h>
69 #include <vm/seg_kmem.h>
70 #include <vm/anon.h>
71 #include <vm/page.h>
72 #include <vm/hat.h>
73 #include <sys/bitmap.h>
74
75 /*
76 * Private seg op routines
77 */
78 static void segkp_dump(struct seg *seg);
79 static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len,
80 uint_t prot);
81 static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta);
82 static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
83 struct page ***page, enum lock_type type,
84 enum seg_rw rw);
85 static void segkp_insert(struct seg *seg, struct segkp_data *kpd);
86 static void segkp_delete(struct seg *seg, struct segkp_data *kpd);
87 static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags,
88 struct segkp_data **tkpd, struct anon_map *amp);
89 static void segkp_release_internal(struct seg *seg,
90 struct segkp_data *kpd, size_t len);
91 static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr,
92 size_t len, struct segkp_data *kpd, uint_t flags);
93 static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr,
94 size_t len, struct segkp_data *kpd, uint_t flags);
95 static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr);
96
97 /*
98 * Lock used to protect the hash table(s) and caches.
99 */
100 static kmutex_t segkp_lock;
101
102 /*
103 * The segkp caches
104 */
105 static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE];
106
107 /*
108 * When there are fewer than red_minavail bytes left on the stack,
109 * segkp_map_red() will map in the redzone (if called). 5000 seems
110 * to work reasonably well...
111 */
112 long red_minavail = 5000;
113
114 /*
115 * will be set to 1 for 32 bit x86 systems only, in startup.c
116 */
117 int segkp_fromheap = 0;
118 ulong_t *segkp_bitmap;
119
120 /*
121 * If segkp_map_red() is called with the redzone already mapped and
122 * with less than RED_DEEP_THRESHOLD bytes available on the stack,
123 * then the stack situation has become quite serious; if much more stack
124 * is consumed, we have the potential of scrogging the next thread/LWP
125 * structure. To help debug the "can't happen" panics which may
126 * result from this condition, we record hrestime and the calling thread
127 * in red_deep_hires and red_deep_thread respectively.
128 */
129 #define RED_DEEP_THRESHOLD 2000
130
131 hrtime_t red_deep_hires;
132 kthread_t *red_deep_thread;
133
134 uint32_t red_nmapped;
135 uint32_t red_closest = UINT_MAX;
136 uint32_t red_ndoubles;
137
138 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */
139 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */
140
141 static struct seg_ops segkp_ops = {
142 .fault = segkp_fault,
143 .checkprot = segkp_checkprot,
144 .kluster = segkp_kluster,
145 .dump = segkp_dump,
146 .pagelock = segkp_pagelock,
147 };
148
149
150 static void segkpinit_mem_config(struct seg *);
151
152 static uint32_t segkp_indel;
153
154 /*
155 * Allocate the segment specific private data struct and fill it in
156 * with the per kp segment mutex, anon ptr. array and hash table.
157 */
158 int
159 segkp_create(struct seg *seg)
160 {
161 struct segkp_segdata *kpsd;
162 size_t np;
163
164 ASSERT(seg != NULL && seg->s_as == &kas);
165 ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock));
166
167 if (seg->s_size & PAGEOFFSET) {
168 panic("Bad segkp size");
169 /*NOTREACHED*/
170 }
171
172 kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP);
173
174 /*
175 * Allocate the virtual memory for segkp and initialize it
176 */
177 if (segkp_fromheap) {
178 np = btop(kvseg.s_size);
179 segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP);
180 kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE,
181 vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP);
182 } else {
183 segkp_bitmap = NULL;
184 np = btop(seg->s_size);
185 kpsd->kpsd_arena = vmem_create("segkp", seg->s_base,
186 seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE,
187 VM_SLEEP);
188 }
189
190 kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE);
191
192 kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *),
193 KM_SLEEP);
194 seg->s_data = (void *)kpsd;
195 seg->s_ops = &segkp_ops;
196 segkpinit_mem_config(seg);
197 return (0);
198 }
199
200
201 /*
202 * Find a free 'freelist' and initialize it with the appropriate attributes
203 */
204 void *
205 segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags)
206 {
207 int i;
208
209 if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED))
210 return ((void *)-1);
211
212 mutex_enter(&segkp_lock);
213 for (i = 0; i < SEGKP_MAX_CACHE; i++) {
214 if (segkp_cache[i].kpf_inuse)
215 continue;
216 segkp_cache[i].kpf_inuse = 1;
217 segkp_cache[i].kpf_max = maxsize;
218 segkp_cache[i].kpf_flags = flags;
219 segkp_cache[i].kpf_seg = seg;
220 segkp_cache[i].kpf_len = len;
221 mutex_exit(&segkp_lock);
222 return ((void *)(uintptr_t)i);
223 }
224 mutex_exit(&segkp_lock);
225 return ((void *)-1);
226 }
227
228 /*
229 * Free all the cache resources.
230 */
231 void
232 segkp_cache_free(void)
233 {
234 struct segkp_data *kpd;
235 struct seg *seg;
236 int i;
237
238 mutex_enter(&segkp_lock);
239 for (i = 0; i < SEGKP_MAX_CACHE; i++) {
240 if (!segkp_cache[i].kpf_inuse)
241 continue;
242 /*
243 * Disconnect the freelist and process each element
244 */
245 kpd = segkp_cache[i].kpf_list;
246 seg = segkp_cache[i].kpf_seg;
247 segkp_cache[i].kpf_list = NULL;
248 segkp_cache[i].kpf_count = 0;
249 mutex_exit(&segkp_lock);
250
251 while (kpd != NULL) {
252 struct segkp_data *next;
253
254 next = kpd->kp_next;
255 segkp_release_internal(seg, kpd, kpd->kp_len);
256 kpd = next;
257 }
258 mutex_enter(&segkp_lock);
259 }
260 mutex_exit(&segkp_lock);
261 }
262
263 /*
264 * There are 2 entries into segkp_get_internal. The first includes a cookie
265 * used to access a pool of cached segkp resources. The second does not
266 * use the cache.
267 */
268 caddr_t
269 segkp_get(struct seg *seg, size_t len, uint_t flags)
270 {
271 struct segkp_data *kpd = NULL;
272
273 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
274 kpd->kp_cookie = -1;
275 return (stom(kpd->kp_base, flags));
276 }
277 return (NULL);
278 }
279
280 /*
281 * Return a 'cached' segkp address
282 */
283 caddr_t
284 segkp_cache_get(void *cookie)
285 {
286 struct segkp_cache *freelist = NULL;
287 struct segkp_data *kpd = NULL;
288 int index = (int)(uintptr_t)cookie;
289 struct seg *seg;
290 size_t len;
291 uint_t flags;
292
293 if (index < 0 || index >= SEGKP_MAX_CACHE)
294 return (NULL);
295 freelist = &segkp_cache[index];
296
297 mutex_enter(&segkp_lock);
298 seg = freelist->kpf_seg;
299 flags = freelist->kpf_flags;
300 if (freelist->kpf_list != NULL) {
301 kpd = freelist->kpf_list;
302 freelist->kpf_list = kpd->kp_next;
303 freelist->kpf_count--;
304 mutex_exit(&segkp_lock);
305 kpd->kp_next = NULL;
306 segkp_insert(seg, kpd);
307 return (stom(kpd->kp_base, flags));
308 }
309 len = freelist->kpf_len;
310 mutex_exit(&segkp_lock);
311 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
312 kpd->kp_cookie = index;
313 return (stom(kpd->kp_base, flags));
314 }
315 return (NULL);
316 }
317
318 caddr_t
319 segkp_get_withanonmap(
320 struct seg *seg,
321 size_t len,
322 uint_t flags,
323 struct anon_map *amp)
324 {
325 struct segkp_data *kpd = NULL;
326
327 ASSERT(amp != NULL);
328 flags |= KPD_HASAMP;
329 if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) {
330 kpd->kp_cookie = -1;
331 return (stom(kpd->kp_base, flags));
332 }
333 return (NULL);
334 }
335
336 /*
337 * This does the real work of segkp allocation.
338 * Return to client base addr. len must be page-aligned. A null value is
339 * returned if there are no more vm resources (e.g. pages, swap). The len
340 * and base recorded in the private data structure include the redzone
341 * and the redzone length (if applicable). If the user requests a redzone
342 * either the first or last page is left unmapped depending whether stacks
343 * grow to low or high memory.
344 *
345 * The client may also specify a no-wait flag. If that is set then the
346 * request will choose a non-blocking path when requesting resources.
347 * The default is make the client wait.
348 */
349 static caddr_t
350 segkp_get_internal(
351 struct seg *seg,
352 size_t len,
353 uint_t flags,
354 struct segkp_data **tkpd,
355 struct anon_map *amp)
356 {
357 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
358 struct segkp_data *kpd;
359 caddr_t vbase = NULL; /* always first virtual, may not be mapped */
360 pgcnt_t np = 0; /* number of pages in the resource */
361 pgcnt_t segkpindex;
362 long i;
363 caddr_t va;
364 pgcnt_t pages = 0;
365 ulong_t anon_idx = 0;
366 int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
367 caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base;
368
369 if (len & PAGEOFFSET) {
370 panic("segkp_get: len is not page-aligned");
371 /*NOTREACHED*/
372 }
373
374 ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL));
375
376 /* Only allow KPD_NO_ANON if we are going to lock it down */
377 if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON)
378 return (NULL);
379
380 if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL)
381 return (NULL);
382 /*
383 * Fix up the len to reflect the REDZONE if applicable
384 */
385 if (flags & KPD_HASREDZONE)
386 len += PAGESIZE;
387 np = btop(len);
388
389 vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT);
390 if (vbase == NULL) {
391 kmem_free(kpd, sizeof (struct segkp_data));
392 return (NULL);
393 }
394
395 /* If locking, reserve physical memory */
396 if (flags & KPD_LOCKED) {
397 pages = btop(SEGKP_MAPLEN(len, flags));
398 if (page_resv(pages, kmflag) == 0) {
399 vmem_free(SEGKP_VMEM(seg), vbase, len);
400 kmem_free(kpd, sizeof (struct segkp_data));
401 return (NULL);
402 }
403 if ((flags & KPD_NO_ANON) == 0)
404 atomic_add_long(&anon_segkp_pages_locked, pages);
405 }
406
407 /*
408 * Reserve sufficient swap space for this vm resource. We'll
409 * actually allocate it in the loop below, but reserving it
410 * here allows us to back out more gracefully than if we
411 * had an allocation failure in the body of the loop.
412 *
413 * Note that we don't need swap space for the red zone page.
414 */
415 if (amp != NULL) {
416 /*
417 * The swap reservation has been done, if required, and the
418 * anon_hdr is separate.
419 */
420 anon_idx = 0;
421 kpd->kp_anon_idx = anon_idx;
422 kpd->kp_anon = amp->ahp;
423
424 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
425 kpd, vbase, len, flags, 1);
426
427 } else if ((flags & KPD_NO_ANON) == 0) {
428 if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) {
429 if (flags & KPD_LOCKED) {
430 atomic_add_long(&anon_segkp_pages_locked,
431 -pages);
432 page_unresv(pages);
433 }
434 vmem_free(SEGKP_VMEM(seg), vbase, len);
435 kmem_free(kpd, sizeof (struct segkp_data));
436 return (NULL);
437 }
438 atomic_add_long(&anon_segkp_pages_resv,
439 btop(SEGKP_MAPLEN(len, flags)));
440 anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT;
441 kpd->kp_anon_idx = anon_idx;
442 kpd->kp_anon = kpsd->kpsd_anon;
443
444 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
445 kpd, vbase, len, flags, 1);
446 } else {
447 kpd->kp_anon = NULL;
448 kpd->kp_anon_idx = 0;
449 }
450
451 /*
452 * Allocate page and anon resources for the virtual address range
453 * except the redzone
454 */
455 if (segkp_fromheap)
456 segkpindex = btop((uintptr_t)(vbase - kvseg.s_base));
457 for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) {
458 page_t *pl[2];
459 struct vnode *vp;
460 anoff_t off;
461 int err;
462 page_t *pp = NULL;
463
464 /*
465 * Mark this page to be a segkp page in the bitmap.
466 */
467 if (segkp_fromheap) {
468 BT_ATOMIC_SET(segkp_bitmap, segkpindex);
469 segkpindex++;
470 }
471
472 /*
473 * If this page is the red zone page, we don't need swap
474 * space for it. Note that we skip over the code that
475 * establishes MMU mappings, so that the page remains
476 * invalid.
477 */
478 if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i)
479 continue;
480
481 if (kpd->kp_anon != NULL) {
482 struct anon *ap;
483
484 ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i)
485 == NULL);
486 /*
487 * Determine the "vp" and "off" of the anon slot.
488 */
489 ap = anon_alloc(NULL, 0);
490 if (amp != NULL)
491 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
492 (void) anon_set_ptr(kpd->kp_anon, anon_idx + i,
493 ap, ANON_SLEEP);
494 if (amp != NULL)
495 ANON_LOCK_EXIT(&->a_rwlock);
496 swap_xlate(ap, &vp, &off);
497
498 /*
499 * Create a page with the specified identity. The
500 * page is returned with the "shared" lock held.
501 */
502 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE,
503 NULL, pl, PAGESIZE, seg, va, S_CREATE,
504 kcred, NULL);
505 if (err) {
506 /*
507 * XXX - This should not fail.
508 */
509 panic("segkp_get: no pages");
510 /*NOTREACHED*/
511 }
512 pp = pl[0];
513 } else {
514 ASSERT(page_exists(&kvp,
515 (u_offset_t)(uintptr_t)va) == NULL);
516
517 if ((pp = page_create_va(&kvp,
518 (u_offset_t)(uintptr_t)va, PAGESIZE,
519 (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL |
520 PG_NORELOC, seg, va)) == NULL) {
521 /*
522 * Legitimize resource; then destroy it.
523 * Easier than trying to unwind here.
524 */
525 kpd->kp_flags = flags;
526 kpd->kp_base = vbase;
527 kpd->kp_len = len;
528 segkp_release_internal(seg, kpd, va - vbase);
529 return (NULL);
530 }
531 page_io_unlock(pp);
532 }
533
534 if (flags & KPD_ZERO)
535 pagezero(pp, 0, PAGESIZE);
536
537 /*
538 * Load and lock an MMU translation for the page.
539 */
540 hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE),
541 ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD));
542
543 /*
544 * Now, release lock on the page.
545 */
546 if (flags & KPD_LOCKED) {
547 /*
548 * Indicate to page_retire framework that this
549 * page can only be retired when it is freed.
550 */
551 PP_SETRAF(pp);
552 page_downgrade(pp);
553 } else
554 page_unlock(pp);
555 }
556
557 kpd->kp_flags = flags;
558 kpd->kp_base = vbase;
559 kpd->kp_len = len;
560 segkp_insert(seg, kpd);
561 *tkpd = kpd;
562 return (stom(kpd->kp_base, flags));
563 }
564
565 /*
566 * Release the resource to cache if the pool(designate by the cookie)
567 * has less than the maximum allowable. If inserted in cache,
568 * segkp_delete insures element is taken off of active list.
569 */
570 void
571 segkp_release(struct seg *seg, caddr_t vaddr)
572 {
573 struct segkp_cache *freelist;
574 struct segkp_data *kpd = NULL;
575
576 if ((kpd = segkp_find(seg, vaddr)) == NULL) {
577 panic("segkp_release: null kpd");
578 /*NOTREACHED*/
579 }
580
581 if (kpd->kp_cookie != -1) {
582 freelist = &segkp_cache[kpd->kp_cookie];
583 mutex_enter(&segkp_lock);
584 if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) {
585 segkp_delete(seg, kpd);
586 kpd->kp_next = freelist->kpf_list;
587 freelist->kpf_list = kpd;
588 freelist->kpf_count++;
589 mutex_exit(&segkp_lock);
590 return;
591 } else {
592 mutex_exit(&segkp_lock);
593 kpd->kp_cookie = -1;
594 }
595 }
596 segkp_release_internal(seg, kpd, kpd->kp_len);
597 }
598
599 /*
600 * Free the entire resource. segkp_unlock gets called with the start of the
601 * mapped portion of the resource. The length is the size of the mapped
602 * portion
603 */
604 static void
605 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len)
606 {
607 caddr_t va;
608 long i;
609 long redzone;
610 size_t np;
611 page_t *pp;
612 struct vnode *vp;
613 anoff_t off;
614 struct anon *ap;
615 pgcnt_t segkpindex;
616
617 ASSERT(kpd != NULL);
618 ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1);
619 np = btop(len);
620
621 /* Remove from active hash list */
622 if (kpd->kp_cookie == -1) {
623 mutex_enter(&segkp_lock);
624 segkp_delete(seg, kpd);
625 mutex_exit(&segkp_lock);
626 }
627
628 /*
629 * Precompute redzone page index.
630 */
631 redzone = -1;
632 if (kpd->kp_flags & KPD_HASREDZONE)
633 redzone = KPD_REDZONE(kpd);
634
635
636 va = kpd->kp_base;
637
638 hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT),
639 ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
640 /*
641 * Free up those anon resources that are quiescent.
642 */
643 if (segkp_fromheap)
644 segkpindex = btop((uintptr_t)(va - kvseg.s_base));
645 for (i = 0; i < np; i++, va += PAGESIZE) {
646
647 /*
648 * Clear the bit for this page from the bitmap.
649 */
650 if (segkp_fromheap) {
651 BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex);
652 segkpindex++;
653 }
654
655 if (i == redzone)
656 continue;
657 if (kpd->kp_anon) {
658 /*
659 * Free up anon resources and destroy the
660 * associated pages.
661 *
662 * Release the lock if there is one. Have to get the
663 * page to do this, unfortunately.
664 */
665 if (kpd->kp_flags & KPD_LOCKED) {
666 ap = anon_get_ptr(kpd->kp_anon,
667 kpd->kp_anon_idx + i);
668 swap_xlate(ap, &vp, &off);
669 /* Find the shared-locked page. */
670 pp = page_find(vp, (u_offset_t)off);
671 if (pp == NULL) {
672 panic("segkp_release: "
673 "kp_anon: no page to unlock ");
674 /*NOTREACHED*/
675 }
676 if (PP_ISRAF(pp))
677 PP_CLRRAF(pp);
678
679 page_unlock(pp);
680 }
681 if ((kpd->kp_flags & KPD_HASAMP) == 0) {
682 anon_free(kpd->kp_anon, kpd->kp_anon_idx + i,
683 PAGESIZE);
684 anon_unresv_zone(PAGESIZE, NULL);
685 atomic_dec_ulong(&anon_segkp_pages_resv);
686 }
687 TRACE_5(TR_FAC_VM,
688 TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
689 kpd, va, PAGESIZE, 0, 0);
690 } else {
691 if (kpd->kp_flags & KPD_LOCKED) {
692 pp = page_find(&kvp, (u_offset_t)(uintptr_t)va);
693 if (pp == NULL) {
694 panic("segkp_release: "
695 "no page to unlock");
696 /*NOTREACHED*/
697 }
698 if (PP_ISRAF(pp))
699 PP_CLRRAF(pp);
700 /*
701 * We should just upgrade the lock here
702 * but there is no upgrade that waits.
703 */
704 page_unlock(pp);
705 }
706 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va,
707 SE_EXCL);
708 if (pp != NULL)
709 page_destroy(pp, 0);
710 }
711 }
712
713 /* If locked, release physical memory reservation */
714 if (kpd->kp_flags & KPD_LOCKED) {
715 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
716 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
717 atomic_add_long(&anon_segkp_pages_locked, -pages);
718 page_unresv(pages);
719 }
720
721 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
722 kmem_free(kpd, sizeof (struct segkp_data));
723 }
724
725 /*
726 * segkp_map_red() will check the current frame pointer against the
727 * stack base. If the amount of stack remaining is questionable
728 * (less than red_minavail), then segkp_map_red() will map in the redzone
729 * and return 1. Otherwise, it will return 0. segkp_map_red() can
730 * _only_ be called when it is safe to sleep on page_create_va().
731 *
732 * It is up to the caller to remember whether segkp_map_red() successfully
733 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
734 * time.
735 *
736 * Currently, this routine is only called from pagefault() (which necessarily
737 * satisfies the above conditions).
738 */
739 #if defined(STACK_GROWTH_DOWN)
740 int
741 segkp_map_red(void)
742 {
743 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
744 #ifndef _LP64
745 caddr_t stkbase;
746 #endif
747
748 /*
749 * Optimize for the common case where we simply return.
750 */
751 if ((curthread->t_red_pp == NULL) &&
752 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
753 return (0);
754
755 #if defined(_LP64)
756 /*
757 * XXX We probably need something better than this.
758 */
759 panic("kernel stack overflow");
760 /*NOTREACHED*/
761 #else /* _LP64 */
762 if (curthread->t_red_pp == NULL) {
763 page_t *red_pp;
764 struct seg kseg;
765
766 caddr_t red_va = (caddr_t)
767 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
768 PAGESIZE);
769
770 ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) ==
771 NULL);
772
773 /*
774 * Allocate the physical for the red page.
775 */
776 /*
777 * No PG_NORELOC here to avoid waits. Unlikely to get
778 * a relocate happening in the short time the page exists
779 * and it will be OK anyway.
780 */
781
782 kseg.s_as = &kas;
783 red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va,
784 PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va);
785 ASSERT(red_pp != NULL);
786
787 /*
788 * So we now have a page to jam into the redzone...
789 */
790 page_io_unlock(red_pp);
791
792 hat_memload(kas.a_hat, red_va, red_pp,
793 (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK);
794 page_downgrade(red_pp);
795
796 /*
797 * The page is left SE_SHARED locked so we can hold on to
798 * the page_t pointer.
799 */
800 curthread->t_red_pp = red_pp;
801
802 atomic_inc_32(&red_nmapped);
803 while (fp - (uintptr_t)curthread->t_stkbase < red_closest) {
804 (void) atomic_cas_32(&red_closest, red_closest,
805 (uint32_t)(fp - (uintptr_t)curthread->t_stkbase));
806 }
807 return (1);
808 }
809
810 stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase &
811 (uintptr_t)PAGEMASK) - PAGESIZE);
812
813 atomic_inc_32(&red_ndoubles);
814
815 if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) {
816 /*
817 * Oh boy. We're already deep within the mapped-in
818 * redzone page, and the caller is trying to prepare
819 * for a deep stack run. We're running without a
820 * redzone right now: if the caller plows off the
821 * end of the stack, it'll plow another thread or
822 * LWP structure. That situation could result in
823 * a very hard-to-debug panic, so, in the spirit of
824 * recording the name of one's killer in one's own
825 * blood, we're going to record hrestime and the calling
826 * thread.
827 */
828 red_deep_hires = hrestime.tv_nsec;
829 red_deep_thread = curthread;
830 }
831
832 /*
833 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
834 */
835 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
836 return (0);
837 #endif /* _LP64 */
838 }
839
840 void
841 segkp_unmap_red(void)
842 {
843 page_t *pp;
844 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
845 (uintptr_t)PAGEMASK) - PAGESIZE);
846
847 ASSERT(curthread->t_red_pp != NULL);
848
849 /*
850 * Because we locked the mapping down, we can't simply rely
851 * on page_destroy() to clean everything up; we need to call
852 * hat_unload() to explicitly unlock the mapping resources.
853 */
854 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
855
856 pp = curthread->t_red_pp;
857
858 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
859
860 /*
861 * Need to upgrade the SE_SHARED lock to SE_EXCL.
862 */
863 if (!page_tryupgrade(pp)) {
864 /*
865 * As there is now wait for upgrade, release the
866 * SE_SHARED lock and wait for SE_EXCL.
867 */
868 page_unlock(pp);
869 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL);
870 /* pp may be NULL here, hence the test below */
871 }
872
873 /*
874 * Destroy the page, with dontfree set to zero (i.e. free it).
875 */
876 if (pp != NULL)
877 page_destroy(pp, 0);
878 curthread->t_red_pp = NULL;
879 }
880 #else
881 #error Red stacks only supported with downwards stack growth.
882 #endif
883
884 /*
885 * Handle a fault on an address corresponding to one of the
886 * resources in the segkp segment.
887 */
888 faultcode_t
889 segkp_fault(
890 struct hat *hat,
891 struct seg *seg,
892 caddr_t vaddr,
893 size_t len,
894 enum fault_type type,
895 enum seg_rw rw)
896 {
897 struct segkp_data *kpd = NULL;
898 int err;
899
900 ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock));
901
902 /*
903 * Sanity checks.
904 */
905 if (type == F_PROT) {
906 panic("segkp_fault: unexpected F_PROT fault");
907 /*NOTREACHED*/
908 }
909
910 if ((kpd = segkp_find(seg, vaddr)) == NULL)
911 return (FC_NOMAP);
912
913 mutex_enter(&kpd->kp_lock);
914
915 if (type == F_SOFTLOCK) {
916 ASSERT(!(kpd->kp_flags & KPD_LOCKED));
917 /*
918 * The F_SOFTLOCK case has more stringent
919 * range requirements: the given range must exactly coincide
920 * with the resource's mapped portion. Note reference to
921 * redzone is handled since vaddr would not equal base
922 */
923 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
924 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
925 mutex_exit(&kpd->kp_lock);
926 return (FC_MAKE_ERR(EFAULT));
927 }
928
929 if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) {
930 mutex_exit(&kpd->kp_lock);
931 return (FC_MAKE_ERR(err));
932 }
933 kpd->kp_flags |= KPD_LOCKED;
934 mutex_exit(&kpd->kp_lock);
935 return (0);
936 }
937
938 if (type == F_INVAL) {
939 ASSERT(!(kpd->kp_flags & KPD_NO_ANON));
940
941 /*
942 * Check if we touched the redzone. Somewhat optimistic
943 * here if we are touching the redzone of our own stack
944 * since we wouldn't have a stack to get this far...
945 */
946 if ((kpd->kp_flags & KPD_HASREDZONE) &&
947 btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd))
948 panic("segkp_fault: accessing redzone");
949
950 /*
951 * This fault may occur while the page is being F_SOFTLOCK'ed.
952 * Return since a 2nd segkp_load is unnecessary and also would
953 * result in the page being locked twice and eventually
954 * hang the thread_reaper thread.
955 */
956 if (kpd->kp_flags & KPD_LOCKED) {
957 mutex_exit(&kpd->kp_lock);
958 return (0);
959 }
960
961 err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags);
962 mutex_exit(&kpd->kp_lock);
963 return (err ? FC_MAKE_ERR(err) : 0);
964 }
965
966 if (type == F_SOFTUNLOCK) {
967 uint_t flags;
968
969 /*
970 * Make sure the addr is LOCKED and it has anon backing
971 * before unlocking
972 */
973 if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) {
974 panic("segkp_fault: bad unlock");
975 /*NOTREACHED*/
976 }
977
978 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
979 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
980 panic("segkp_fault: bad range");
981 /*NOTREACHED*/
982 }
983
984 if (rw == S_WRITE)
985 flags = kpd->kp_flags | KPD_WRITEDIRTY;
986 else
987 flags = kpd->kp_flags;
988 err = segkp_unlock(hat, seg, vaddr, len, kpd, flags);
989 kpd->kp_flags &= ~KPD_LOCKED;
990 mutex_exit(&kpd->kp_lock);
991 return (err ? FC_MAKE_ERR(err) : 0);
992 }
993 mutex_exit(&kpd->kp_lock);
994 panic("segkp_fault: bogus fault type: %d\n", type);
995 /*NOTREACHED*/
996 }
997
998 /*
999 * Check that the given protections suffice over the range specified by
1000 * vaddr and len. For this segment type, the only issue is whether or
1001 * not the range lies completely within the mapped part of an allocated
1002 * resource.
1003 */
1004 /* ARGSUSED */
1005 static int
1006 segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot)
1007 {
1008 struct segkp_data *kpd = NULL;
1009 caddr_t mbase;
1010 size_t mlen;
1011
1012 if ((kpd = segkp_find(seg, vaddr)) == NULL)
1013 return (EACCES);
1014
1015 mutex_enter(&kpd->kp_lock);
1016 mbase = stom(kpd->kp_base, kpd->kp_flags);
1017 mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags);
1018 if (len > mlen || vaddr < mbase ||
1019 ((vaddr + len) > (mbase + mlen))) {
1020 mutex_exit(&kpd->kp_lock);
1021 return (EACCES);
1022 }
1023 mutex_exit(&kpd->kp_lock);
1024 return (0);
1025 }
1026
1027
1028 /*
1029 * Check to see if it makes sense to do kluster/read ahead to
1030 * addr + delta relative to the mapping at addr. We assume here
1031 * that delta is a signed PAGESIZE'd multiple (which can be negative).
1032 *
1033 * For seg_u we always "approve" of this action from our standpoint.
1034 */
1035 /*ARGSUSED*/
1036 static int
1037 segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
1038 {
1039 return (0);
1040 }
1041
1042 /*
1043 * Load and possibly lock intra-slot resources in the range given by
1044 * vaddr and len.
1045 */
1046 static int
1047 segkp_load(
1048 struct hat *hat,
1049 struct seg *seg,
1050 caddr_t vaddr,
1051 size_t len,
1052 struct segkp_data *kpd,
1053 uint_t flags)
1054 {
1055 caddr_t va;
1056 caddr_t vlim;
1057 ulong_t i;
1058 uint_t lock;
1059
1060 ASSERT(MUTEX_HELD(&kpd->kp_lock));
1061
1062 len = P2ROUNDUP(len, PAGESIZE);
1063
1064 /* If locking, reserve physical memory */
1065 if (flags & KPD_LOCKED) {
1066 pgcnt_t pages = btop(len);
1067 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1068 atomic_add_long(&anon_segkp_pages_locked, pages);
1069 (void) page_resv(pages, KM_SLEEP);
1070 }
1071
1072 /*
1073 * Loop through the pages in the given range.
1074 */
1075 va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
1076 vaddr = va;
1077 vlim = va + len;
1078 lock = flags & KPD_LOCKED;
1079 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1080 for (; va < vlim; va += PAGESIZE, i++) {
1081 page_t *pl[2]; /* second element NULL terminator */
1082 struct vnode *vp;
1083 anoff_t off;
1084 int err;
1085 struct anon *ap;
1086
1087 /*
1088 * Summon the page. If it's not resident, arrange
1089 * for synchronous i/o to pull it in.
1090 */
1091 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1092 swap_xlate(ap, &vp, &off);
1093
1094 /*
1095 * The returned page list will have exactly one entry,
1096 * which is returned to us already kept.
1097 */
1098 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL,
1099 pl, PAGESIZE, seg, va, S_READ, kcred, NULL);
1100
1101 if (err) {
1102 /*
1103 * Back out of what we've done so far.
1104 */
1105 (void) segkp_unlock(hat, seg, vaddr,
1106 (va - vaddr), kpd, flags);
1107 return (err);
1108 }
1109
1110 /*
1111 * Load an MMU translation for the page.
1112 */
1113 hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE),
1114 lock ? HAT_LOAD_LOCK : HAT_LOAD);
1115
1116 if (!lock) {
1117 /*
1118 * Now, release "shared" lock on the page.
1119 */
1120 page_unlock(pl[0]);
1121 }
1122 }
1123 return (0);
1124 }
1125
1126 /*
1127 * At the very least unload the mmu-translations and unlock the range if locked
1128 * Can be called with the following flag value KPD_WRITEDIRTY which specifies
1129 * any dirty pages should be written to disk.
1130 */
1131 static int
1132 segkp_unlock(
1133 struct hat *hat,
1134 struct seg *seg,
1135 caddr_t vaddr,
1136 size_t len,
1137 struct segkp_data *kpd,
1138 uint_t flags)
1139 {
1140 caddr_t va;
1141 caddr_t vlim;
1142 ulong_t i;
1143 struct page *pp;
1144 struct vnode *vp;
1145 anoff_t off;
1146 struct anon *ap;
1147
1148 #ifdef lint
1149 seg = seg;
1150 #endif /* lint */
1151
1152 ASSERT(MUTEX_HELD(&kpd->kp_lock));
1153
1154 /*
1155 * Loop through the pages in the given range. It is assumed
1156 * segkp_unlock is called with page aligned base
1157 */
1158 va = vaddr;
1159 vlim = va + len;
1160 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1161 hat_unload(hat, va, len,
1162 ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
1163 for (; va < vlim; va += PAGESIZE, i++) {
1164 /*
1165 * Find the page associated with this part of the
1166 * slot, tracking it down through its associated swap
1167 * space.
1168 */
1169 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1170 swap_xlate(ap, &vp, &off);
1171
1172 if (flags & KPD_LOCKED) {
1173 if ((pp = page_find(vp, off)) == NULL) {
1174 if (flags & KPD_LOCKED) {
1175 panic("segkp_softunlock: missing page");
1176 /*NOTREACHED*/
1177 }
1178 }
1179 } else {
1180 /*
1181 * Nothing to do if the slot is not locked and the
1182 * page doesn't exist.
1183 */
1184 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL)
1185 continue;
1186 }
1187
1188 /*
1189 * If the page doesn't have any translations, is
1190 * dirty and not being shared, then push it out
1191 * asynchronously and avoid waiting for the
1192 * pageout daemon to do it for us.
1193 *
1194 * XXX - Do we really need to get the "exclusive"
1195 * lock via an upgrade?
1196 */
1197 if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) &&
1198 hat_ismod(pp) && page_tryupgrade(pp)) {
1199 /*
1200 * Hold the vnode before releasing the page lock to
1201 * prevent it from being freed and re-used by some
1202 * other thread.
1203 */
1204 VN_HOLD(vp);
1205 page_unlock(pp);
1206
1207 /*
1208 * Want most powerful credentials we can get so
1209 * use kcred.
1210 */
1211 (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE,
1212 B_ASYNC | B_FREE, kcred, NULL);
1213 VN_RELE(vp);
1214 } else {
1215 page_unlock(pp);
1216 }
1217 }
1218
1219 /* If unlocking, release physical memory */
1220 if (flags & KPD_LOCKED) {
1221 pgcnt_t pages = btopr(len);
1222 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1223 atomic_add_long(&anon_segkp_pages_locked, -pages);
1224 page_unresv(pages);
1225 }
1226 return (0);
1227 }
1228
1229 /*
1230 * Insert the kpd in the hash table.
1231 */
1232 static void
1233 segkp_insert(struct seg *seg, struct segkp_data *kpd)
1234 {
1235 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1236 int index;
1237
1238 /*
1239 * Insert the kpd based on the address that will be returned
1240 * via segkp_release.
1241 */
1242 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1243 mutex_enter(&segkp_lock);
1244 kpd->kp_next = kpsd->kpsd_hash[index];
1245 kpsd->kpsd_hash[index] = kpd;
1246 mutex_exit(&segkp_lock);
1247 }
1248
1249 /*
1250 * Remove kpd from the hash table.
1251 */
1252 static void
1253 segkp_delete(struct seg *seg, struct segkp_data *kpd)
1254 {
1255 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1256 struct segkp_data **kpp;
1257 int index;
1258
1259 ASSERT(MUTEX_HELD(&segkp_lock));
1260
1261 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1262 for (kpp = &kpsd->kpsd_hash[index];
1263 *kpp != NULL; kpp = &((*kpp)->kp_next)) {
1264 if (*kpp == kpd) {
1265 *kpp = kpd->kp_next;
1266 return;
1267 }
1268 }
1269 panic("segkp_delete: unable to find element to delete");
1270 /*NOTREACHED*/
1271 }
1272
1273 /*
1274 * Find the kpd associated with a vaddr.
1275 *
1276 * Most of the callers of segkp_find will pass the vaddr that
1277 * hashes to the desired index, but there are cases where
1278 * this is not true in which case we have to (potentially) scan
1279 * the whole table looking for it. This should be very rare
1280 * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the
1281 * middle of the segkp_data region).
1282 */
1283 static struct segkp_data *
1284 segkp_find(struct seg *seg, caddr_t vaddr)
1285 {
1286 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1287 struct segkp_data *kpd;
1288 int i;
1289 int stop;
1290
1291 i = stop = SEGKP_HASH(vaddr);
1292 mutex_enter(&segkp_lock);
1293 do {
1294 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
1295 kpd = kpd->kp_next) {
1296 if (vaddr >= kpd->kp_base &&
1297 vaddr < kpd->kp_base + kpd->kp_len) {
1298 mutex_exit(&segkp_lock);
1299 return (kpd);
1300 }
1301 }
1302 if (--i < 0)
1303 i = SEGKP_HASHSZ - 1; /* Wrap */
1304 } while (i != stop);
1305 mutex_exit(&segkp_lock);
1306 return (NULL); /* Not found */
1307 }
1308
1309 /*
1310 * returns size of swappable area.
1311 */
1312 size_t
1313 swapsize(caddr_t v)
1314 {
1315 struct segkp_data *kpd;
1316
1317 if ((kpd = segkp_find(segkp, v)) != NULL)
1318 return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
1319 else
1320 return (NULL);
1321 }
1322
1323 /*
1324 * Dump out all the active segkp pages
1325 */
1326 static void
1327 segkp_dump(struct seg *seg)
1328 {
1329 int i;
1330 struct segkp_data *kpd;
1331 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1332
1333 for (i = 0; i < SEGKP_HASHSZ; i++) {
1334 for (kpd = kpsd->kpsd_hash[i];
1335 kpd != NULL; kpd = kpd->kp_next) {
1336 pfn_t pfn;
1337 caddr_t addr;
1338 caddr_t eaddr;
1339
1340 addr = kpd->kp_base;
1341 eaddr = addr + kpd->kp_len;
1342 while (addr < eaddr) {
1343 ASSERT(seg->s_as == &kas);
1344 pfn = hat_getpfnum(seg->s_as->a_hat, addr);
1345 if (pfn != PFN_INVALID)
1346 dump_addpage(seg->s_as, addr, pfn);
1347 addr += PAGESIZE;
1348 dump_timeleft = dump_timeout;
1349 }
1350 }
1351 }
1352 }
1353
1354 /*ARGSUSED*/
1355 static int
1356 segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
1357 struct page ***ppp, enum lock_type type, enum seg_rw rw)
1358 {
1359 return (ENOTSUP);
1360 }
1361
1362 #include <sys/mem_config.h>
1363
1364 /*ARGSUSED*/
1365 static void
1366 segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages)
1367 {}
1368
1369 /*
1370 * During memory delete, turn off caches so that pages are not held.
1371 * A better solution may be to unlock the pages while they are
1372 * in the cache so that they may be collected naturally.
1373 */
1374
1375 /*ARGSUSED*/
1376 static int
1377 segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages)
1378 {
1379 atomic_inc_32(&segkp_indel);
1380 segkp_cache_free();
1381 return (0);
1382 }
1383
1384 /*ARGSUSED*/
1385 static void
1386 segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled)
1387 {
1388 atomic_dec_32(&segkp_indel);
1389 }
1390
1391 static kphysm_setup_vector_t segkp_mem_config_vec = {
1392 KPHYSM_SETUP_VECTOR_VERSION,
1393 segkp_mem_config_post_add,
1394 segkp_mem_config_pre_del,
1395 segkp_mem_config_post_del,
1396 };
1397
1398 static void
1399 segkpinit_mem_config(struct seg *seg)
1400 {
1401 int ret;
1402
1403 ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg);
1404 ASSERT(ret == 0);
1405 }