1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Kernel Physical Mapping (kpm) segment driver (segkpm).
29 *
30 * This driver delivers along with the hat_kpm* interfaces an alternative
31 * mechanism for kernel mappings within the 64-bit Solaris operating system,
32 * which allows the mapping of all physical memory into the kernel address
33 * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II
34 * and beyond processors, since the available VA range is much larger than
35 * possible physical memory. Momentarily all physical memory is supported,
36 * that is represented by the list of memory segments (memsegs).
37 *
38 * Segkpm mappings have also very low overhead and large pages are used
39 * (when possible) to minimize the TLB and TSB footprint. It is also
40 * extentable for other than Sparc architectures (e.g. AMD64). Main
41 * advantage is the avoidance of the TLB-shootdown X-calls, which are
42 * normally needed when a kernel (global) mapping has to be removed.
43 *
44 * First example of a kernel facility that uses the segkpm mapping scheme
45 * is seg_map, where it is used as an alternative to hat_memload().
46 * See also hat layer for more information about the hat_kpm* routines.
47 * The kpm facilty can be turned off at boot time (e.g. /etc/system).
48 */
49
50 #include <sys/types.h>
51 #include <sys/param.h>
52 #include <sys/sysmacros.h>
53 #include <sys/systm.h>
54 #include <sys/vnode.h>
55 #include <sys/cmn_err.h>
56 #include <sys/debug.h>
57 #include <sys/thread.h>
58 #include <sys/cpuvar.h>
59 #include <sys/bitmap.h>
60 #include <sys/atomic.h>
61 #include <sys/lgrp.h>
62
63 #include <vm/seg_kmem.h>
64 #include <vm/seg_kpm.h>
65 #include <vm/hat.h>
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/page.h>
69
70 /*
71 * Global kpm controls.
72 * See also platform and mmu specific controls.
73 *
74 * kpm_enable -- global on/off switch for segkpm.
75 * . Set by default on 64bit platforms that have kpm support.
76 * . Will be disabled from platform layer if not supported.
77 * . Can be disabled via /etc/system.
78 *
79 * kpm_smallpages -- use only regular/system pagesize for kpm mappings.
80 * . Can be useful for critical debugging of kpm clients.
81 * . Set to zero by default for platforms that support kpm large pages.
82 * The use of kpm large pages reduces the footprint of kpm meta data
83 * and has all the other advantages of using large pages (e.g TLB
84 * miss reduction).
85 * . Set by default for platforms that don't support kpm large pages or
86 * where large pages cannot be used for other reasons (e.g. there are
87 * only few full associative TLB entries available for large pages).
88 *
89 * segmap_kpm -- separate on/off switch for segmap using segkpm:
90 * . Set by default.
91 * . Will be disabled when kpm_enable is zero.
92 * . Will be disabled when MAXBSIZE != PAGESIZE.
93 * . Can be disabled via /etc/system.
94 *
95 */
96 int kpm_enable = 1;
97 int kpm_smallpages = 0;
98 int segmap_kpm = 1;
99
100 /*
101 * Private seg op routines.
102 */
103 faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr,
104 size_t len, enum fault_type type, enum seg_rw rw);
105 static int segkpm_pagelock(struct seg *seg, caddr_t addr, size_t len,
106 struct page ***page, enum lock_type type,
107 enum seg_rw rw);
108
109 static const struct seg_ops segkpm_ops = {
110 .fault = segkpm_fault,
111 .pagelock = segkpm_pagelock,
112 //#ifndef SEGKPM_SUPPORT
113 #if 0
114 #error FIXME: define nop
115 .dup = nop,
116 .unmap = nop,
117 .free = nop,
118 .faulta = nop,
119 .setprot = nop,
120 .checkprot = nop,
121 .kluster = nop,
122 .sync = nop,
123 .incore = nop,
124 .lockop = nop,
125 .getprot = nop,
126 .getoffset = nop,
127 .gettype = nop,
128 .getvp = nop,
129 .advise = nop,
130 .getpolicy = nop,
131 #endif
132 };
133
134 /*
135 * kpm_pgsz and kpm_pgshft are set by platform layer.
136 */
137 size_t kpm_pgsz; /* kpm page size */
138 uint_t kpm_pgshft; /* kpm page shift */
139 u_offset_t kpm_pgoff; /* kpm page offset mask */
140 uint_t kpmp2pshft; /* kpm page to page shift */
141 pgcnt_t kpmpnpgs; /* how many pages per kpm page */
142
143
144 #ifdef SEGKPM_SUPPORT
145
146 int
147 segkpm_create(struct seg *seg, void *argsp)
148 {
149 struct segkpm_data *skd;
150 struct segkpm_crargs *b = (struct segkpm_crargs *)argsp;
151 ushort_t *p;
152 int i, j;
153
154 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
155 ASSERT(btokpmp(seg->s_size) >= 1 &&
156 kpmpageoff((uintptr_t)seg->s_base) == 0 &&
157 kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0);
158
159 skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP);
160
161 seg->s_data = (void *)skd;
162 seg->s_ops = &segkpm_ops;
163 skd->skd_prot = b->prot;
164
165 /*
166 * (1) Segkpm virtual addresses are based on physical adresses.
167 * From this and in opposite to other segment drivers it is
168 * often required to allocate a page first to be able to
169 * calculate the final segkpm virtual address.
170 * (2) Page allocation is done by calling page_create_va(),
171 * one important input argument is a virtual address (also
172 * expressed by the "va" in the function name). This function
173 * is highly optimized to select the right page for an optimal
174 * processor and platform support (e.g. virtual addressed
175 * caches (VAC), physical addressed caches, NUMA).
176 *
177 * Because of (1) the approach is to generate a faked virtual
178 * address for calling page_create_va(). In order to exploit
179 * the abilities of (2), especially to utilize the cache
180 * hierarchy (3) and to avoid VAC alias conflicts (4) the
181 * selection has to be done carefully. For each virtual color
182 * a separate counter is provided (4). The count values are
183 * used for the utilization of all cache lines (3) and are
184 * corresponding to the cache bins.
185 */
186 skd->skd_nvcolors = b->nvcolors;
187
188 p = skd->skd_va_select =
189 kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP);
190
191 for (i = 0; i < NCPU; i++)
192 for (j = 0; j < b->nvcolors; j++, p++)
193 *p = j;
194
195 return (0);
196 }
197
198 /*
199 * This routine is called via a machine specific fault handling
200 * routine.
201 */
202 /* ARGSUSED */
203 faultcode_t
204 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
205 enum fault_type type, enum seg_rw rw)
206 {
207 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
208
209 switch (type) {
210 case F_INVAL:
211 return (hat_kpm_fault(hat, addr));
212 case F_SOFTLOCK:
213 case F_SOFTUNLOCK:
214 return (0);
215 default:
216 return (FC_NOSUPPORT);
217 }
218 /*NOTREACHED*/
219 }
220
221 #define addr_to_vcolor(addr, vcolors) \
222 ((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT))
223
224 /*
225 * Create a virtual address that can be used for invocations of
226 * page_create_va. Goal is to utilize the cache hierarchy (round
227 * robin bins) and to select the right color for virtual indexed
228 * caches. It isn't exact since we also increment the bin counter
229 * when the caller uses VOP_GETPAGE and gets a hit in the page
230 * cache, but we keep the bins turning for cache distribution
231 * (see also segkpm_create block comment).
232 */
233 caddr_t
234 segkpm_create_va(u_offset_t off)
235 {
236 int vcolor;
237 ushort_t *p;
238 struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data;
239 int nvcolors = skd->skd_nvcolors;
240 caddr_t va;
241
242 vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0;
243 p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor];
244 va = (caddr_t)ptob(*p);
245
246 atomic_add_16(p, nvcolors);
247
248 return (va);
249 }
250
251 /*
252 * Unload mapping if the instance has an active kpm mapping.
253 */
254 void
255 segkpm_mapout_validkpme(struct kpme *kpme)
256 {
257 caddr_t vaddr;
258 page_t *pp;
259
260 retry:
261 if ((pp = kpme->kpe_page) == NULL) {
262 return;
263 }
264
265 if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0)
266 goto retry;
267
268 /*
269 * Check if segkpm mapping is not unloaded in the meantime
270 */
271 if (kpme->kpe_page == NULL) {
272 page_unlock(pp);
273 return;
274 }
275
276 vaddr = hat_kpm_page2va(pp, 1);
277 hat_kpm_mapout(pp, kpme, vaddr);
278 page_unlock(pp);
279 }
280
281 #else /* SEGKPM_SUPPORT */
282
283 /* segkpm stubs */
284
285 /*ARGSUSED*/
286 int segkpm_create(struct seg *seg, void *argsp)
287 {
288 return (0);
289 }
290
291 /* ARGSUSED */
292 faultcode_t
293 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
294 enum fault_type type, enum seg_rw rw)
295 {
296 return (0);
297 }
298
299 /* ARGSUSED */
300 caddr_t segkpm_create_va(u_offset_t off)
301 {
302 return (NULL);
303 }
304
305 /* ARGSUSED */
306 void segkpm_mapout_validkpme(struct kpme *kpme)
307 {
308 }
309
310 #endif /* SEGKPM_SUPPORT */
311
312 /* ARGSUSED */
313 static int
314 segkpm_pagelock(struct seg *seg, caddr_t addr, size_t len,
315 struct page ***page, enum lock_type type, enum seg_rw rw)
316 {
317 return (ENOTSUP);
318 }