1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/types.h>
26 #include <sys/t_lock.h>
27 #include <sys/param.h>
28 #include <sys/sysmacros.h>
29 #include <sys/tuneable.h>
30 #include <sys/systm.h>
31 #include <sys/vm.h>
32 #include <sys/kmem.h>
33 #include <sys/vmem.h>
34 #include <sys/mman.h>
35 #include <sys/cmn_err.h>
36 #include <sys/debug.h>
37 #include <sys/dumphdr.h>
38 #include <sys/bootconf.h>
39 #include <sys/lgrp.h>
40 #include <vm/seg_kmem.h>
41 #include <vm/hat.h>
42 #include <vm/page.h>
43 #include <vm/vm_dep.h>
44 #include <vm/faultcode.h>
45 #include <sys/promif.h>
46 #include <vm/seg_kp.h>
47 #include <sys/bitmap.h>
48 #include <sys/mem_cage.h>
49
50 #ifdef __sparc
51 #include <sys/ivintr.h>
52 #include <sys/panic.h>
53 #endif
54
55 /*
56 * seg_kmem is the primary kernel memory segment driver. It
57 * maps the kernel heap [kernelheap, ekernelheap), module text,
58 * and all memory which was allocated before the VM was initialized
59 * into kas.
60 *
61 * Pages which belong to seg_kmem are hashed into &kvp vnode at
62 * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
63 * They must never be paged out since segkmem_fault() is a no-op to
64 * prevent recursive faults.
65 *
66 * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
67 * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86
68 * supports relocation the #ifdef kludges can be removed.
69 *
70 * seg_kmem pages may be subject to relocation by page_relocate(),
71 * provided that the HAT supports it; if this is so, segkmem_reloc
72 * will be set to a nonzero value. All boot time allocated memory as
73 * well as static memory is considered off limits to relocation.
74 * Pages are "relocatable" if p_state does not have P_NORELOC set, so
75 * we request P_NORELOC pages for memory that isn't safe to relocate.
76 *
77 * The kernel heap is logically divided up into four pieces:
78 *
79 * heap32_arena is for allocations that require 32-bit absolute
80 * virtual addresses (e.g. code that uses 32-bit pointers/offsets).
81 *
82 * heap_core is for allocations that require 2GB *relative*
83 * offsets; in other words all memory from heap_core is within
84 * 2GB of all other memory from the same arena. This is a requirement
85 * of the addressing modes of some processors in supervisor code.
86 *
87 * heap_arena is the general heap arena.
88 *
89 * static_arena is the static memory arena. Allocations from it
90 * are not subject to relocation so it is safe to use the memory
91 * physical address as well as the virtual address (e.g. the VA to
92 * PA translations are static). Caches may import from static_arena;
93 * all other static memory allocations should use static_alloc_arena.
94 *
95 * On some platforms which have limited virtual address space, seg_kmem
96 * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
97 * segkp_bitmap is non-NULL, and each bit represents a page of virtual
98 * address space which is actually seg_kp mapped.
99 */
100
101 extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */
102
103 char *kernelheap; /* start of primary kernel heap */
104 char *ekernelheap; /* end of primary kernel heap */
105 struct seg kvseg; /* primary kernel heap segment */
106 struct seg kvseg_core; /* "core" kernel heap segment */
107 struct seg kzioseg; /* Segment for zio mappings */
108 vmem_t *heap_arena; /* primary kernel heap arena */
109 vmem_t *heap_core_arena; /* core kernel heap arena */
110 char *heap_core_base; /* start of core kernel heap arena */
111 char *heap_lp_base; /* start of kernel large page heap arena */
112 char *heap_lp_end; /* end of kernel large page heap arena */
113 vmem_t *hat_memload_arena; /* HAT translation data */
114 struct seg kvseg32; /* 32-bit kernel heap segment */
115 vmem_t *heap32_arena; /* 32-bit kernel heap arena */
116 vmem_t *heaptext_arena; /* heaptext arena */
117 struct as kas; /* kernel address space */
118 int segkmem_reloc; /* enable/disable relocatable segkmem pages */
119 vmem_t *static_arena; /* arena for caches to import static memory */
120 vmem_t *static_alloc_arena; /* arena for allocating static memory */
121 vmem_t *zio_arena = NULL; /* arena for allocating zio memory */
122 vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */
123
124 /*
125 * seg_kmem driver can map part of the kernel heap with large pages.
126 * Currently this functionality is implemented for sparc platforms only.
127 *
128 * The large page size "segkmem_lpsize" for kernel heap is selected in the
129 * platform specific code. It can also be modified via /etc/system file.
130 * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
131 * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
132 * match segkmem_lpsize.
133 *
134 * At boot time we carve from kernel heap arena a range of virtual addresses
135 * that will be used for large page mappings. This range [heap_lp_base,
136 * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
137 * create "kmem_lp_arena" that caches memory already backed up by large
138 * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
139 */
140
141 size_t segkmem_lpsize;
142 static uint_t segkmem_lpshift = PAGESHIFT;
143 int segkmem_lpszc = 0;
144
145 size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */
146 size_t segkmem_heaplp_quantum;
147 vmem_t *heap_lp_arena;
148 static vmem_t *kmem_lp_arena;
149 static vmem_t *segkmem_ppa_arena;
150 static segkmem_lpcb_t segkmem_lpcb;
151
152 /*
153 * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
154 * consumed by the large page heap. By default this parameter is set to 1/8 of
155 * physmem but can be adjusted through /etc/system either directly or
156 * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
157 * we allow for large page heap.
158 */
159 size_t segkmem_kmemlp_max;
160 static uint_t segkmem_kmemlp_pcnt;
161
162 /*
163 * Getting large pages for kernel heap could be problematic due to
164 * physical memory fragmentation. That's why we allow to preallocate
165 * "segkmem_kmemlp_min" bytes at boot time.
166 */
167 static size_t segkmem_kmemlp_min;
168
169 /*
170 * Throttling is used to avoid expensive tries to allocate large pages
171 * for kernel heap when a lot of succesive attempts to do so fail.
172 */
173 static ulong_t segkmem_lpthrottle_max = 0x400000;
174 static ulong_t segkmem_lpthrottle_start = 0x40;
175 static ulong_t segkmem_use_lpthrottle = 1;
176
177 /*
178 * Freed pages accumulate on a garbage list until segkmem is ready,
179 * at which point we call segkmem_gc() to free it all.
180 */
181 typedef struct segkmem_gc_list {
182 struct segkmem_gc_list *gc_next;
183 vmem_t *gc_arena;
184 size_t gc_size;
185 } segkmem_gc_list_t;
186
187 static segkmem_gc_list_t *segkmem_gc_list;
188
189 /*
190 * Allocations from the hat_memload arena add VM_MEMLOAD to their
191 * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
192 * to take steps to prevent infinite recursion. HAT allocations also
193 * must be non-relocatable to prevent recursive page faults.
194 */
195 static void *
196 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
197 {
198 flags |= (VM_MEMLOAD | VM_NORELOC);
199 return (segkmem_alloc(vmp, size, flags));
200 }
201
202 /*
203 * Allocations from static_arena arena (or any other arena that uses
204 * segkmem_alloc_permanent()) require non-relocatable (permanently
205 * wired) memory pages, since these pages are referenced by physical
206 * as well as virtual address.
207 */
208 void *
209 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
210 {
211 return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
212 }
213
214 /*
215 * Initialize kernel heap boundaries.
216 */
217 void
218 kernelheap_init(
219 void *heap_start,
220 void *heap_end,
221 char *first_avail,
222 void *core_start,
223 void *core_end)
224 {
225 uintptr_t textbase;
226 size_t core_size;
227 size_t heap_size;
228 vmem_t *heaptext_parent;
229 size_t heap_lp_size = 0;
230 #ifdef __sparc
231 size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
232 #endif /* __sparc */
233
234 kernelheap = heap_start;
235 ekernelheap = heap_end;
236
237 #ifdef __sparc
238 heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
239 /*
240 * Bias heap_lp start address by kmem64_sz to reduce collisions
241 * in 4M kernel TSB between kmem64 area and heap_lp
242 */
243 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
244 if (kmem64_sz <= heap_lp_size / 2)
245 heap_lp_size -= kmem64_sz;
246 heap_lp_base = ekernelheap - heap_lp_size;
247 heap_lp_end = heap_lp_base + heap_lp_size;
248 #endif /* __sparc */
249
250 /*
251 * If this platform has a 'core' heap area, then the space for
252 * overflow module text should be carved out of the end of that
253 * heap. Otherwise, it gets carved out of the general purpose
254 * heap.
255 */
256 core_size = (uintptr_t)core_end - (uintptr_t)core_start;
257 if (core_size > 0) {
258 ASSERT(core_size >= HEAPTEXT_SIZE);
259 textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
260 core_size -= HEAPTEXT_SIZE;
261 }
262 #ifndef __sparc
263 else {
264 ekernelheap -= HEAPTEXT_SIZE;
265 textbase = (uintptr_t)ekernelheap;
266 }
267 #endif
268
269 heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
270 heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
271 segkmem_alloc, segkmem_free);
272
273 if (core_size > 0) {
274 heap_core_arena = vmem_create("heap_core", core_start,
275 core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
276 heap_core_base = core_start;
277 } else {
278 heap_core_arena = heap_arena;
279 heap_core_base = kernelheap;
280 }
281
282 /*
283 * reserve space for the large page heap. If large pages for kernel
284 * heap is enabled large page heap arean will be created later in the
285 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
286 * range will be returned back to the heap_arena.
287 */
288 if (heap_lp_size) {
289 (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
290 heap_lp_base, heap_lp_end,
291 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
292 }
293
294 /*
295 * Remove the already-spoken-for memory range [kernelheap, first_avail).
296 */
297 (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
298 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
299
300 #ifdef __sparc
301 heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
302 SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
303 NULL, NULL, 0, VM_SLEEP);
304 /*
305 * Prom claims the physical and virtual resources used by panicbuf
306 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
307 * reserved interrupt vector data structures from 32-bit heap.
308 */
309 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
310 panicbuf, panicbuf + PANICBUFSIZE,
311 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
312
313 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
314 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
315 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
316
317 textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
318 heaptext_parent = NULL;
319 #else /* __sparc */
320 heap32_arena = heap_core_arena;
321 heaptext_parent = heap_core_arena;
322 #endif /* __sparc */
323
324 heaptext_arena = vmem_create("heaptext", (void *)textbase,
325 HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
326
327 /*
328 * Create a set of arenas for memory with static translations
329 * (e.g. VA -> PA translations cannot change). Since using
330 * kernel pages by physical address implies it isn't safe to
331 * walk across page boundaries, the static_arena quantum must
332 * be PAGESIZE. Any kmem caches that require static memory
333 * should source from static_arena, while direct allocations
334 * should only use static_alloc_arena.
335 */
336 static_arena = vmem_create("static", NULL, 0, PAGESIZE,
337 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
338 static_alloc_arena = vmem_create("static_alloc", NULL, 0,
339 sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
340 0, VM_SLEEP);
341
342 /*
343 * Create an arena for translation data (ptes, hmes, or hblks).
344 * We need an arena for this because hat_memload() is essential
345 * to vmem_populate() (see comments in common/os/vmem.c).
346 *
347 * Note: any kmem cache that allocates from hat_memload_arena
348 * must be created as a KMC_NOHASH cache (i.e. no external slab
349 * and bufctl structures to allocate) so that slab creation doesn't
350 * require anything more than a single vmem_alloc().
351 */
352 hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
353 hat_memload_alloc, segkmem_free, heap_arena, 0,
354 VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
355 }
356
357 void
358 boot_mapin(caddr_t addr, size_t size)
359 {
360 caddr_t eaddr;
361 page_t *pp;
362 pfn_t pfnum;
363
364 if (page_resv(btop(size), KM_NOSLEEP) == 0)
365 panic("boot_mapin: page_resv failed");
366
367 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
368 pfnum = va_to_pfn(addr);
369 if (pfnum == PFN_INVALID)
370 continue;
371 if ((pp = page_numtopp_nolock(pfnum)) == NULL)
372 panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
373
374 /*
375 * must break up any large pages that may have constituent
376 * pages being utilized for BOP_ALLOC()'s before calling
377 * page_numtopp().The locking code (ie. page_reclaim())
378 * can't handle them
379 */
380 if (pp->p_szc != 0)
381 page_boot_demote(pp);
382
383 pp = page_numtopp(pfnum, SE_EXCL);
384 if (pp == NULL || PP_ISFREE(pp))
385 panic("boot_alloc: pp is NULL or free");
386
387 /*
388 * If the cage is on but doesn't yet contain this page,
389 * mark it as non-relocatable.
390 */
391 if (kcage_on && !PP_ISNORELOC(pp)) {
392 PP_SETNORELOC(pp);
393 PLCNT_XFER_NORELOC(pp);
394 }
395
396 (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
397 pp->p_lckcnt = 1;
398 #if defined(__x86)
399 page_downgrade(pp);
400 #else
401 page_unlock(pp);
402 #endif
403 }
404 }
405
406 /*
407 * Get pages from boot and hash them into the kernel's vp.
408 * Used after page structs have been allocated, but before segkmem is ready.
409 */
410 void *
411 boot_alloc(void *inaddr, size_t size, uint_t align)
412 {
413 caddr_t addr = inaddr;
414
415 if (bootops == NULL)
416 prom_panic("boot_alloc: attempt to allocate memory after "
417 "BOP_GONE");
418
419 size = ptob(btopr(size));
420 #ifdef __sparc
421 if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
422 panic("boot_alloc: bop_alloc_chunk failed");
423 #else
424 if (BOP_ALLOC(bootops, addr, size, align) != addr)
425 panic("boot_alloc: BOP_ALLOC failed");
426 #endif
427 boot_mapin((caddr_t)addr, size);
428 return (addr);
429 }
430
431 /*ARGSUSED*/
432 static faultcode_t
433 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
434 enum fault_type type, enum seg_rw rw)
435 {
436 pgcnt_t npages;
437 spgcnt_t pg;
438 page_t *pp;
439 struct vnode *vp = seg->s_data;
440
441 ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
442
443 if (seg->s_as != &kas || size > seg->s_size ||
444 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
445 panic("segkmem_fault: bad args");
446
447 /*
448 * If it is one of segkp pages, call segkp_fault.
449 */
450 if (segkp_bitmap && seg == &kvseg &&
451 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
452 return (segop_fault(hat, segkp, addr, size, type, rw));
453
454 if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
455 return (FC_NOSUPPORT);
456
457 npages = btopr(size);
458
459 switch (type) {
460 case F_SOFTLOCK: /* lock down already-loaded translations */
461 for (pg = 0; pg < npages; pg++) {
462 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
463 SE_SHARED);
464 if (pp == NULL) {
465 /*
466 * Hmm, no page. Does a kernel mapping
467 * exist for it?
468 */
469 if (!hat_probe(kas.a_hat, addr)) {
470 addr -= PAGESIZE;
471 while (--pg >= 0) {
472 pp = page_find(vp, (u_offset_t)
473 (uintptr_t)addr);
474 if (pp)
475 page_unlock(pp);
476 addr -= PAGESIZE;
477 }
478 return (FC_NOMAP);
479 }
480 }
481 addr += PAGESIZE;
482 }
483 if (rw == S_OTHER)
484 hat_reserve(seg->s_as, addr, size);
485 return (0);
486 case F_SOFTUNLOCK:
487 while (npages--) {
488 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
489 if (pp)
490 page_unlock(pp);
491 addr += PAGESIZE;
492 }
493 return (0);
494 default:
495 return (FC_NOSUPPORT);
496 }
497 /*NOTREACHED*/
498 }
499
500 static int
501 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
502 {
503 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
504
505 if (seg->s_as != &kas || size > seg->s_size ||
506 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
507 panic("segkmem_setprot: bad args");
508
509 /*
510 * If it is one of segkp pages, call segkp.
511 */
512 if (segkp_bitmap && seg == &kvseg &&
513 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
514 return (segop_setprot(segkp, addr, size, prot));
515
516 if (prot == 0)
517 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
518 else
519 hat_chgprot(kas.a_hat, addr, size, prot);
520 return (0);
521 }
522
523 /*
524 * This is a dummy segkmem function overloaded to call segkp
525 * when segkp is under the heap.
526 */
527 /* ARGSUSED */
528 static int
529 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
530 {
531 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
532
533 if (seg->s_as != &kas)
534 panic("segkmem badop");
535
536 /*
537 * If it is one of segkp pages, call into segkp.
538 */
539 if (segkp_bitmap && seg == &kvseg &&
540 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
541 return (segop_checkprot(segkp, addr, size, prot));
542
543 panic("segkmem badop");
544 return (0);
545 }
546
547 /*
548 * This is a dummy segkmem function overloaded to call segkp
549 * when segkp is under the heap.
550 */
551 /* ARGSUSED */
552 static int
553 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
554 {
555 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
556
557 if (seg->s_as != &kas)
558 panic("segkmem badop");
559
560 /*
561 * If it is one of segkp pages, call into segkp.
562 */
563 if (segkp_bitmap && seg == &kvseg &&
564 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
565 return (segop_kluster(segkp, addr, delta));
566
567 panic("segkmem badop");
568 return (0);
569 }
570
571 static void
572 segkmem_xdump_range(void *arg, void *start, size_t size)
573 {
574 struct as *as = arg;
575 caddr_t addr = start;
576 caddr_t addr_end = addr + size;
577
578 while (addr < addr_end) {
579 pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
580 if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
581 dump_addpage(as, addr, pfn);
582 addr += PAGESIZE;
583 dump_timeleft = dump_timeout;
584 }
585 }
586
587 static void
588 segkmem_dump_range(void *arg, void *start, size_t size)
589 {
590 caddr_t addr = start;
591 caddr_t addr_end = addr + size;
592
593 /*
594 * If we are about to start dumping the range of addresses we
595 * carved out of the kernel heap for the large page heap walk
596 * heap_lp_arena to find what segments are actually populated
597 */
598 if (SEGKMEM_USE_LARGEPAGES &&
599 addr == heap_lp_base && addr_end == heap_lp_end &&
600 vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
601 vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
602 segkmem_xdump_range, arg);
603 } else {
604 segkmem_xdump_range(arg, start, size);
605 }
606 }
607
608 static void
609 segkmem_dump(struct seg *seg)
610 {
611 /*
612 * The kernel's heap_arena (represented by kvseg) is a very large
613 * VA space, most of which is typically unused. To speed up dumping
614 * we use vmem_walk() to quickly find the pieces of heap_arena that
615 * are actually in use. We do the same for heap32_arena and
616 * heap_core.
617 *
618 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
619 * may ultimately need to allocate memory. Reentrant walks are
620 * necessarily imperfect snapshots. The kernel heap continues
621 * to change during a live crash dump, for example. For a normal
622 * crash dump, however, we know that there won't be any other threads
623 * messing with the heap. Therefore, at worst, we may fail to dump
624 * the pages that get allocated by the act of dumping; but we will
625 * always dump every page that was allocated when the walk began.
626 *
627 * The other segkmem segments are dense (fully populated), so there's
628 * no need to use this technique when dumping them.
629 *
630 * Note: when adding special dump handling for any new sparsely-
631 * populated segments, be sure to add similar handling to the ::kgrep
632 * code in mdb.
633 */
634 if (seg == &kvseg) {
635 vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
636 segkmem_dump_range, seg->s_as);
637 #ifndef __sparc
638 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
639 segkmem_dump_range, seg->s_as);
640 #endif
641 } else if (seg == &kvseg_core) {
642 vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
643 segkmem_dump_range, seg->s_as);
644 } else if (seg == &kvseg32) {
645 vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
646 segkmem_dump_range, seg->s_as);
647 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
648 segkmem_dump_range, seg->s_as);
649 } else if (seg == &kzioseg) {
650 /*
651 * We don't want to dump pages attached to kzioseg since they
652 * contain file data from ZFS. If this page's segment is
653 * kzioseg return instead of writing it to the dump device.
654 */
655 return;
656 } else {
657 segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
658 }
659 }
660
661 /*
662 * lock/unlock kmem pages over a given range [addr, addr+len).
663 * Returns a shadow list of pages in ppp. If there are holes
664 * in the range (e.g. some of the kernel mappings do not have
665 * underlying page_ts) returns ENOTSUP so that as_pagelock()
666 * will handle the range via as_fault(F_SOFTLOCK).
667 */
668 /*ARGSUSED*/
669 static int
670 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
671 page_t ***ppp, enum lock_type type, enum seg_rw rw)
672 {
673 page_t **pplist, *pp;
674 pgcnt_t npages;
675 spgcnt_t pg;
676 size_t nb;
677 struct vnode *vp = seg->s_data;
678
679 ASSERT(ppp != NULL);
680
681 /*
682 * If it is one of segkp pages, call into segkp.
683 */
684 if (segkp_bitmap && seg == &kvseg &&
685 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
686 return (segop_pagelock(segkp, addr, len, ppp, type, rw));
687
688 npages = btopr(len);
689 nb = sizeof (page_t *) * npages;
690
691 if (type == L_PAGEUNLOCK) {
692 pplist = *ppp;
693 ASSERT(pplist != NULL);
694
695 for (pg = 0; pg < npages; pg++) {
696 pp = pplist[pg];
697 page_unlock(pp);
698 }
699 kmem_free(pplist, nb);
700 return (0);
701 }
702
703 ASSERT(type == L_PAGELOCK);
704
705 pplist = kmem_alloc(nb, KM_NOSLEEP);
706 if (pplist == NULL) {
707 *ppp = NULL;
708 return (ENOTSUP); /* take the slow path */
709 }
710
711 for (pg = 0; pg < npages; pg++) {
712 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
713 if (pp == NULL) {
714 while (--pg >= 0)
715 page_unlock(pplist[pg]);
716 kmem_free(pplist, nb);
717 *ppp = NULL;
718 return (ENOTSUP);
719 }
720 pplist[pg] = pp;
721 addr += PAGESIZE;
722 }
723
724 *ppp = pplist;
725 return (0);
726 }
727
728 /*
729 * This is a dummy segkmem function overloaded to call segkp
730 * when segkp is under the heap.
731 */
732 /* ARGSUSED */
733 static int
734 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
735 {
736 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
737
738 if (seg->s_as != &kas)
739 panic("segkmem badop");
740
741 /*
742 * If it is one of segkp pages, call into segkp.
743 */
744 if (segkp_bitmap && seg == &kvseg &&
745 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
746 return (segop_getmemid(segkp, addr, memidp));
747
748 panic("segkmem badop");
749 return (0);
750 }
751
752 /*ARGSUSED*/
753 static lgrp_mem_policy_info_t *
754 segkmem_getpolicy(struct seg *seg, caddr_t addr)
755 {
756 return (NULL);
757 }
758
759 /*ARGSUSED*/
760 static int
761 segkmem_capable(struct seg *seg, segcapability_t capability)
762 {
763 if (capability == S_CAPABILITY_NOMINFLT)
764 return (1);
765 return (0);
766 }
767
768 static struct seg_ops segkmem_ops = {
769 .fault = segkmem_fault,
770 .setprot = segkmem_setprot,
771 .checkprot = segkmem_checkprot,
772 .kluster = segkmem_kluster,
773 .dump = segkmem_dump,
774 .pagelock = segkmem_pagelock,
775 .getmemid = segkmem_getmemid,
776 .getpolicy = segkmem_getpolicy,
777 .capable = segkmem_capable,
778 .inherit = seg_inherit_notsup,
779 };
780
781 int
782 segkmem_zio_create(struct seg *seg)
783 {
784 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
785 seg->s_ops = &segkmem_ops;
786 seg->s_data = &zvp;
787 kas.a_size += seg->s_size;
788 return (0);
789 }
790
791 int
792 segkmem_create(struct seg *seg)
793 {
794 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
795 seg->s_ops = &segkmem_ops;
796 seg->s_data = &kvp;
797 kas.a_size += seg->s_size;
798 return (0);
799 }
800
801 /*ARGSUSED*/
802 page_t *
803 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
804 {
805 struct seg kseg;
806 int pgflags;
807 struct vnode *vp = arg;
808
809 if (vp == NULL)
810 vp = &kvp;
811
812 kseg.s_as = &kas;
813 pgflags = PG_EXCL;
814
815 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
816 pgflags |= PG_NORELOC;
817 if ((vmflag & VM_NOSLEEP) == 0)
818 pgflags |= PG_WAIT;
819 if (vmflag & VM_PANIC)
820 pgflags |= PG_PANIC;
821 if (vmflag & VM_PUSHPAGE)
822 pgflags |= PG_PUSHPAGE;
823 if (vmflag & VM_NORMALPRI) {
824 ASSERT(vmflag & VM_NOSLEEP);
825 pgflags |= PG_NORMALPRI;
826 }
827
828 return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
829 pgflags, &kseg, addr));
830 }
831
832 /*
833 * Allocate pages to back the virtual address range [addr, addr + size).
834 * If addr is NULL, allocate the virtual address space as well.
835 */
836 void *
837 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
838 page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
839 {
840 page_t *ppl;
841 caddr_t addr = inaddr;
842 pgcnt_t npages = btopr(size);
843 int allocflag;
844
845 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
846 return (NULL);
847
848 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
849
850 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
851 if (inaddr == NULL)
852 vmem_free(vmp, addr, size);
853 return (NULL);
854 }
855
856 ppl = page_create_func(addr, size, vmflag, pcarg);
857 if (ppl == NULL) {
858 if (inaddr == NULL)
859 vmem_free(vmp, addr, size);
860 page_unresv(npages);
861 return (NULL);
862 }
863
864 /*
865 * Under certain conditions, we need to let the HAT layer know
866 * that it cannot safely allocate memory. Allocations from
867 * the hat_memload vmem arena always need this, to prevent
868 * infinite recursion.
869 *
870 * In addition, the x86 hat cannot safely do memory
871 * allocations while in vmem_populate(), because there
872 * is no simple bound on its usage.
873 */
874 if (vmflag & VM_MEMLOAD)
875 allocflag = HAT_NO_KALLOC;
876 #if defined(__x86)
877 else if (vmem_is_populator())
878 allocflag = HAT_NO_KALLOC;
879 #endif
880 else
881 allocflag = 0;
882
883 while (ppl != NULL) {
884 page_t *pp = ppl;
885 page_sub(&ppl, pp);
886 ASSERT(page_iolock_assert(pp));
887 ASSERT(PAGE_EXCL(pp));
888 page_io_unlock(pp);
889 hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
890 (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
891 HAT_LOAD_LOCK | allocflag);
892 pp->p_lckcnt = 1;
893 #if defined(__x86)
894 page_downgrade(pp);
895 #else
896 if (vmflag & SEGKMEM_SHARELOCKED)
897 page_downgrade(pp);
898 else
899 page_unlock(pp);
900 #endif
901 }
902
903 return (addr);
904 }
905
906 static void *
907 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
908 {
909 void *addr;
910 segkmem_gc_list_t *gcp, **prev_gcpp;
911
912 ASSERT(vp != NULL);
913
914 if (kvseg.s_base == NULL) {
915 #ifndef __sparc
916 if (bootops->bsys_alloc == NULL)
917 halt("Memory allocation between bop_alloc() and "
918 "kmem_alloc().\n");
919 #endif
920
921 /*
922 * There's not a lot of memory to go around during boot,
923 * so recycle it if we can.
924 */
925 for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
926 prev_gcpp = &gcp->gc_next) {
927 if (gcp->gc_arena == vmp && gcp->gc_size == size) {
928 *prev_gcpp = gcp->gc_next;
929 return (gcp);
930 }
931 }
932
933 addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
934 if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
935 panic("segkmem_alloc: boot_alloc failed");
936 return (addr);
937 }
938 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
939 segkmem_page_create, vp));
940 }
941
942 void *
943 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
944 {
945 return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
946 }
947
948 void *
949 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
950 {
951 return (segkmem_alloc_vn(vmp, size, vmflag, &zvp));
952 }
953
954 /*
955 * Any changes to this routine must also be carried over to
956 * devmap_free_pages() in the seg_dev driver. This is because
957 * we currently don't have a special kernel segment for non-paged
958 * kernel memory that is exported by drivers to user space.
959 */
960 static void
961 segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
962 void (*func)(page_t *))
963 {
964 page_t *pp;
965 caddr_t addr = inaddr;
966 caddr_t eaddr;
967 pgcnt_t npages = btopr(size);
968
969 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
970 ASSERT(vp != NULL);
971
972 if (kvseg.s_base == NULL) {
973 segkmem_gc_list_t *gc = inaddr;
974 gc->gc_arena = vmp;
975 gc->gc_size = size;
976 gc->gc_next = segkmem_gc_list;
977 segkmem_gc_list = gc;
978 return;
979 }
980
981 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
982
983 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
984 #if defined(__x86)
985 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
986 if (pp == NULL)
987 panic("segkmem_free: page not found");
988 if (!page_tryupgrade(pp)) {
989 /*
990 * Some other thread has a sharelock. Wait for
991 * it to drop the lock so we can free this page.
992 */
993 page_unlock(pp);
994 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
995 SE_EXCL);
996 }
997 #else
998 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
999 #endif
1000 if (pp == NULL)
1001 panic("segkmem_free: page not found");
1002 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1003 pp->p_lckcnt = 0;
1004 if (func)
1005 func(pp);
1006 else
1007 page_destroy(pp, 0);
1008 }
1009 if (func == NULL)
1010 page_unresv(npages);
1011
1012 if (vmp != NULL)
1013 vmem_free(vmp, inaddr, size);
1014
1015 }
1016
1017 void
1018 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *))
1019 {
1020 segkmem_free_vn(vmp, inaddr, size, &kvp, func);
1021 }
1022
1023 void
1024 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1025 {
1026 segkmem_free_vn(vmp, inaddr, size, &kvp, NULL);
1027 }
1028
1029 void
1030 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1031 {
1032 segkmem_free_vn(vmp, inaddr, size, &zvp, NULL);
1033 }
1034
1035 void
1036 segkmem_gc(void)
1037 {
1038 ASSERT(kvseg.s_base != NULL);
1039 while (segkmem_gc_list != NULL) {
1040 segkmem_gc_list_t *gc = segkmem_gc_list;
1041 segkmem_gc_list = gc->gc_next;
1042 segkmem_free(gc->gc_arena, gc, gc->gc_size);
1043 }
1044 }
1045
1046 /*
1047 * Legacy entry points from here to end of file.
1048 */
1049 void
1050 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1051 pfn_t pfn, uint_t flags)
1052 {
1053 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1054 hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1055 flags | HAT_LOAD_LOCK);
1056 }
1057
1058 void
1059 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1060 {
1061 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1062 }
1063
1064 void *
1065 kmem_getpages(pgcnt_t npages, int kmflag)
1066 {
1067 return (kmem_alloc(ptob(npages), kmflag));
1068 }
1069
1070 void
1071 kmem_freepages(void *addr, pgcnt_t npages)
1072 {
1073 kmem_free(addr, ptob(npages));
1074 }
1075
1076 /*
1077 * segkmem_page_create_large() allocates a large page to be used for the kmem
1078 * caches. If kpr is enabled we ask for a relocatable page unless requested
1079 * otherwise. If kpr is disabled we have to ask for a non-reloc page
1080 */
1081 static page_t *
1082 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1083 {
1084 int pgflags;
1085
1086 pgflags = PG_EXCL;
1087
1088 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1089 pgflags |= PG_NORELOC;
1090 if (!(vmflag & VM_NOSLEEP))
1091 pgflags |= PG_WAIT;
1092 if (vmflag & VM_PUSHPAGE)
1093 pgflags |= PG_PUSHPAGE;
1094 if (vmflag & VM_NORMALPRI)
1095 pgflags |= PG_NORMALPRI;
1096
1097 return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1098 pgflags, &kvseg, addr, arg));
1099 }
1100
1101 /*
1102 * Allocate a large page to back the virtual address range
1103 * [addr, addr + size). If addr is NULL, allocate the virtual address
1104 * space as well.
1105 */
1106 static void *
1107 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1108 uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1109 void *pcarg)
1110 {
1111 caddr_t addr = inaddr, pa;
1112 size_t lpsize = segkmem_lpsize;
1113 pgcnt_t npages = btopr(size);
1114 pgcnt_t nbpages = btop(lpsize);
1115 pgcnt_t nlpages = size >> segkmem_lpshift;
1116 size_t ppasize = nbpages * sizeof (page_t *);
1117 page_t *pp, *rootpp, **ppa, *pplist = NULL;
1118 int i;
1119
1120 vmflag |= VM_NOSLEEP;
1121
1122 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1123 return (NULL);
1124 }
1125
1126 /*
1127 * allocate an array we need for hat_memload_array.
1128 * we use a separate arena to avoid recursion.
1129 * we will not need this array when hat_memload_array learns pp++
1130 */
1131 if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1132 goto fail_array_alloc;
1133 }
1134
1135 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1136 goto fail_vmem_alloc;
1137
1138 ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1139
1140 /* create all the pages */
1141 for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1142 if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1143 goto fail_page_create;
1144 page_list_concat(&pplist, &pp);
1145 }
1146
1147 /* at this point we have all the resource to complete the request */
1148 while ((rootpp = pplist) != NULL) {
1149 for (i = 0; i < nbpages; i++) {
1150 ASSERT(pplist != NULL);
1151 pp = pplist;
1152 page_sub(&pplist, pp);
1153 ASSERT(page_iolock_assert(pp));
1154 page_io_unlock(pp);
1155 ppa[i] = pp;
1156 }
1157 /*
1158 * Load the locked entry. It's OK to preload the entry into the
1159 * TSB since we now support large mappings in the kernel TSB.
1160 */
1161 hat_memload_array(kas.a_hat,
1162 (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1163 ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1164 HAT_LOAD_LOCK);
1165
1166 for (--i; i >= 0; --i) {
1167 ppa[i]->p_lckcnt = 1;
1168 page_unlock(ppa[i]);
1169 }
1170 }
1171
1172 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1173 return (addr);
1174
1175 fail_page_create:
1176 while ((rootpp = pplist) != NULL) {
1177 for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1178 ASSERT(pp != NULL);
1179 page_sub(&pplist, pp);
1180 ASSERT(page_iolock_assert(pp));
1181 page_io_unlock(pp);
1182 }
1183 page_destroy_pages(rootpp);
1184 }
1185
1186 if (inaddr == NULL)
1187 vmem_free(vmp, addr, size);
1188
1189 fail_vmem_alloc:
1190 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1191
1192 fail_array_alloc:
1193 page_unresv(npages);
1194
1195 return (NULL);
1196 }
1197
1198 static void
1199 segkmem_free_one_lp(caddr_t addr, size_t size)
1200 {
1201 page_t *pp, *rootpp = NULL;
1202 pgcnt_t pgs_left = btopr(size);
1203
1204 ASSERT(size == segkmem_lpsize);
1205
1206 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1207
1208 for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1209 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1210 if (pp == NULL)
1211 panic("segkmem_free_one_lp: page not found");
1212 ASSERT(PAGE_EXCL(pp));
1213 pp->p_lckcnt = 0;
1214 if (rootpp == NULL)
1215 rootpp = pp;
1216 }
1217 ASSERT(rootpp != NULL);
1218 page_destroy_pages(rootpp);
1219
1220 /* page_unresv() is done by the caller */
1221 }
1222
1223 /*
1224 * This function is called to import new spans into the vmem arenas like
1225 * kmem_default_arena and kmem_oversize_arena. It first tries to import
1226 * spans from large page arena - kmem_lp_arena. In order to do this it might
1227 * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1228 * it was not able to satisfy the upgraded request it then calls regular
1229 * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1230 */
1231 /*ARGSUSED*/
1232 void *
1233 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1234 {
1235 size_t size;
1236 kthread_t *t = curthread;
1237 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1238
1239 ASSERT(sizep != NULL);
1240
1241 size = *sizep;
1242
1243 if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1244 !(vmflag & SEGKMEM_SHARELOCKED)) {
1245
1246 size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1247 size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1248 void *addr = NULL;
1249 ulong_t *lpthrtp = &lpcb->lp_throttle;
1250 ulong_t lpthrt = *lpthrtp;
1251 int dowakeup = 0;
1252 int doalloc = 1;
1253
1254 ASSERT(kmem_lp_arena != NULL);
1255 ASSERT(asize >= size);
1256
1257 if (lpthrt != 0) {
1258 /* try to update the throttle value */
1259 lpthrt = atomic_inc_ulong_nv(lpthrtp);
1260 if (lpthrt >= segkmem_lpthrottle_max) {
1261 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1262 segkmem_lpthrottle_max / 4);
1263 }
1264
1265 /*
1266 * when we get above throttle start do an exponential
1267 * backoff at trying large pages and reaping
1268 */
1269 if (lpthrt > segkmem_lpthrottle_start &&
1270 !ISP2(lpthrt)) {
1271 lpcb->allocs_throttled++;
1272 lpthrt--;
1273 if (ISP2(lpthrt))
1274 kmem_reap();
1275 return (segkmem_alloc(vmp, size, vmflag));
1276 }
1277 }
1278
1279 if (!(vmflag & VM_NOSLEEP) &&
1280 segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1281 vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1282 asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1283
1284 /*
1285 * we are low on free memory in kmem_lp_arena
1286 * we let only one guy to allocate heap_lp
1287 * quantum size chunk that everybody is going to
1288 * share
1289 */
1290 mutex_enter(&lpcb->lp_lock);
1291
1292 if (lpcb->lp_wait) {
1293
1294 /* we are not the first one - wait */
1295 cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1296 if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1297 kmemlp_qnt) {
1298 doalloc = 0;
1299 }
1300 } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1301 kmemlp_qnt) {
1302
1303 /*
1304 * we are the first one, make sure we import
1305 * a large page
1306 */
1307 if (asize == kmemlp_qnt)
1308 asize += kmemlp_qnt;
1309 dowakeup = 1;
1310 lpcb->lp_wait = 1;
1311 }
1312
1313 mutex_exit(&lpcb->lp_lock);
1314 }
1315
1316 /*
1317 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1318 * large pages are not available. In that case this allocation
1319 * attempt will fail and we will retry allocation with small
1320 * pages. We also do not want to panic if this allocation fails
1321 * because we are going to retry.
1322 */
1323 if (doalloc) {
1324 addr = vmem_alloc(kmem_lp_arena, asize,
1325 (vmflag | VM_ABORT) & ~VM_PANIC);
1326
1327 if (dowakeup) {
1328 mutex_enter(&lpcb->lp_lock);
1329 ASSERT(lpcb->lp_wait != 0);
1330 lpcb->lp_wait = 0;
1331 cv_broadcast(&lpcb->lp_cv);
1332 mutex_exit(&lpcb->lp_lock);
1333 }
1334 }
1335
1336 if (addr != NULL) {
1337 *sizep = asize;
1338 *lpthrtp = 0;
1339 return (addr);
1340 }
1341
1342 if (vmflag & VM_NOSLEEP)
1343 lpcb->nosleep_allocs_failed++;
1344 else
1345 lpcb->sleep_allocs_failed++;
1346 lpcb->alloc_bytes_failed += size;
1347
1348 /* if large page throttling is not started yet do it */
1349 if (segkmem_use_lpthrottle && lpthrt == 0) {
1350 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1351 }
1352 }
1353 return (segkmem_alloc(vmp, size, vmflag));
1354 }
1355
1356 void
1357 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1358 {
1359 if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1360 segkmem_free(vmp, inaddr, size);
1361 } else {
1362 vmem_free(kmem_lp_arena, inaddr, size);
1363 }
1364 }
1365
1366 /*
1367 * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1368 * into kmem_lp arena. In the process it maps the imported segment with
1369 * large pages
1370 */
1371 static void *
1372 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1373 {
1374 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1375 void *addr;
1376
1377 ASSERT(size != 0);
1378 ASSERT(vmp == heap_lp_arena);
1379
1380 /* do not allow large page heap grow beyound limits */
1381 if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1382 lpcb->allocs_limited++;
1383 return (NULL);
1384 }
1385
1386 addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1387 segkmem_page_create_large, NULL);
1388 return (addr);
1389 }
1390
1391 /*
1392 * segkmem_free_lpi() returns virtual memory back into large page heap arena
1393 * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1394 * large pages used to map it.
1395 */
1396 static void
1397 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1398 {
1399 pgcnt_t nlpages = size >> segkmem_lpshift;
1400 size_t lpsize = segkmem_lpsize;
1401 caddr_t addr = inaddr;
1402 pgcnt_t npages = btopr(size);
1403 int i;
1404
1405 ASSERT(vmp == heap_lp_arena);
1406 ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1407 ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1408
1409 for (i = 0; i < nlpages; i++) {
1410 segkmem_free_one_lp(addr, lpsize);
1411 addr += lpsize;
1412 }
1413
1414 page_unresv(npages);
1415
1416 vmem_free(vmp, inaddr, size);
1417 }
1418
1419 /*
1420 * This function is called at system boot time by kmem_init right after
1421 * /etc/system file has been read. It checks based on hardware configuration
1422 * and /etc/system settings if system is going to use large pages. The
1423 * initialiazation necessary to actually start using large pages
1424 * happens later in the process after segkmem_heap_lp_init() is called.
1425 */
1426 int
1427 segkmem_lpsetup()
1428 {
1429 int use_large_pages = 0;
1430
1431 #ifdef __sparc
1432
1433 size_t memtotal = physmem * PAGESIZE;
1434
1435 if (heap_lp_base == NULL) {
1436 segkmem_lpsize = PAGESIZE;
1437 return (0);
1438 }
1439
1440 /* get a platform dependent value of large page size for kernel heap */
1441 segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1442
1443 if (segkmem_lpsize <= PAGESIZE) {
1444 /*
1445 * put virtual space reserved for the large page kernel
1446 * back to the regular heap
1447 */
1448 vmem_xfree(heap_arena, heap_lp_base,
1449 heap_lp_end - heap_lp_base);
1450 heap_lp_base = NULL;
1451 heap_lp_end = NULL;
1452 segkmem_lpsize = PAGESIZE;
1453 return (0);
1454 }
1455
1456 /* set heap_lp quantum if necessary */
1457 if (segkmem_heaplp_quantum == 0 || !ISP2(segkmem_heaplp_quantum) ||
1458 P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1459 segkmem_heaplp_quantum = segkmem_lpsize;
1460 }
1461
1462 /* set kmem_lp quantum if necessary */
1463 if (segkmem_kmemlp_quantum == 0 || !ISP2(segkmem_kmemlp_quantum) ||
1464 segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1465 segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1466 }
1467
1468 /* set total amount of memory allowed for large page kernel heap */
1469 if (segkmem_kmemlp_max == 0) {
1470 if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1471 segkmem_kmemlp_pcnt = 12;
1472 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1473 }
1474 segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1475 segkmem_heaplp_quantum);
1476
1477 /* fix lp kmem preallocation request if necesssary */
1478 if (segkmem_kmemlp_min) {
1479 segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1480 segkmem_heaplp_quantum);
1481 if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1482 segkmem_kmemlp_min = segkmem_kmemlp_max;
1483 }
1484
1485 use_large_pages = 1;
1486 segkmem_lpszc = page_szc(segkmem_lpsize);
1487 segkmem_lpshift = page_get_shift(segkmem_lpszc);
1488
1489 #endif
1490 return (use_large_pages);
1491 }
1492
1493 void
1494 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1495 {
1496 ASSERT(zio_mem_base != NULL);
1497 ASSERT(zio_mem_size != 0);
1498
1499 /*
1500 * To reduce VA space fragmentation, we set up quantum caches for the
1501 * smaller sizes; we chose 32k because that translates to 128k VA
1502 * slabs, which matches nicely with the common 128k zio_data bufs.
1503 */
1504 zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1505 PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1506
1507 zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1508 segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1509
1510 ASSERT(zio_arena != NULL);
1511 ASSERT(zio_alloc_arena != NULL);
1512 }
1513
1514 #ifdef __sparc
1515
1516
1517 static void *
1518 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1519 {
1520 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1521 void *addr;
1522
1523 if (ppaquantum <= PAGESIZE)
1524 return (segkmem_alloc(vmp, size, vmflag));
1525
1526 ASSERT((size & (ppaquantum - 1)) == 0);
1527
1528 addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1529 if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1530 segkmem_page_create, NULL) == NULL) {
1531 vmem_xfree(vmp, addr, size);
1532 addr = NULL;
1533 }
1534
1535 return (addr);
1536 }
1537
1538 static void
1539 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1540 {
1541 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1542
1543 ASSERT(addr != NULL);
1544
1545 if (ppaquantum <= PAGESIZE) {
1546 segkmem_free(vmp, addr, size);
1547 } else {
1548 segkmem_free(NULL, addr, size);
1549 vmem_xfree(vmp, addr, size);
1550 }
1551 }
1552
1553 void
1554 segkmem_heap_lp_init()
1555 {
1556 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1557 size_t heap_lp_size = heap_lp_end - heap_lp_base;
1558 size_t lpsize = segkmem_lpsize;
1559 size_t ppaquantum;
1560 void *addr;
1561
1562 if (segkmem_lpsize <= PAGESIZE) {
1563 ASSERT(heap_lp_base == NULL);
1564 ASSERT(heap_lp_end == NULL);
1565 return;
1566 }
1567
1568 ASSERT(segkmem_heaplp_quantum >= lpsize);
1569 ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1570 ASSERT(lpcb->lp_uselp == 0);
1571 ASSERT(heap_lp_base != NULL);
1572 ASSERT(heap_lp_end != NULL);
1573 ASSERT(heap_lp_base < heap_lp_end);
1574 ASSERT(heap_lp_arena == NULL);
1575 ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1576 ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1577
1578 /* create large page heap arena */
1579 heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1580 segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1581
1582 ASSERT(heap_lp_arena != NULL);
1583
1584 /* This arena caches memory already mapped by large pages */
1585 kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1586 segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1587
1588 ASSERT(kmem_lp_arena != NULL);
1589
1590 mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1591 cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1592
1593 /*
1594 * this arena is used for the array of page_t pointers necessary
1595 * to call hat_mem_load_array
1596 */
1597 ppaquantum = btopr(lpsize) * sizeof (page_t *);
1598 segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1599 segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1600 VM_SLEEP);
1601
1602 ASSERT(segkmem_ppa_arena != NULL);
1603
1604 /* prealloacate some memory for the lp kernel heap */
1605 if (segkmem_kmemlp_min) {
1606
1607 ASSERT(P2PHASE(segkmem_kmemlp_min,
1608 segkmem_heaplp_quantum) == 0);
1609
1610 if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1611 segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1612
1613 addr = vmem_add(kmem_lp_arena, addr,
1614 segkmem_kmemlp_min, VM_SLEEP);
1615 ASSERT(addr != NULL);
1616 }
1617 }
1618
1619 lpcb->lp_uselp = 1;
1620 }
1621
1622 #endif