Print this page
use C99 initializers in segment ops structures
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/seg_kpm.c
+++ new/usr/src/uts/common/vm/seg_kpm.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License, Version 1.0 only
6 6 * (the "License"). You may not use this file except in compliance
7 7 * with the License.
8 8 *
9 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 10 * or http://www.opensolaris.org/os/licensing.
11 11 * See the License for the specific language governing permissions
12 12 * and limitations under the License.
13 13 *
14 14 * When distributing Covered Code, include this CDDL HEADER in each
15 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 16 * If applicable, add the following below this CDDL HEADER, with the
17 17 * fields enclosed by brackets "[]" replaced with your own identifying
18 18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 19 *
20 20 * CDDL HEADER END
21 21 */
22 22 /*
23 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /*
28 28 * Kernel Physical Mapping (kpm) segment driver (segkpm).
29 29 *
30 30 * This driver delivers along with the hat_kpm* interfaces an alternative
31 31 * mechanism for kernel mappings within the 64-bit Solaris operating system,
32 32 * which allows the mapping of all physical memory into the kernel address
33 33 * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II
34 34 * and beyond processors, since the available VA range is much larger than
35 35 * possible physical memory. Momentarily all physical memory is supported,
36 36 * that is represented by the list of memory segments (memsegs).
37 37 *
38 38 * Segkpm mappings have also very low overhead and large pages are used
39 39 * (when possible) to minimize the TLB and TSB footprint. It is also
40 40 * extentable for other than Sparc architectures (e.g. AMD64). Main
41 41 * advantage is the avoidance of the TLB-shootdown X-calls, which are
42 42 * normally needed when a kernel (global) mapping has to be removed.
43 43 *
44 44 * First example of a kernel facility that uses the segkpm mapping scheme
45 45 * is seg_map, where it is used as an alternative to hat_memload().
46 46 * See also hat layer for more information about the hat_kpm* routines.
47 47 * The kpm facilty can be turned off at boot time (e.g. /etc/system).
48 48 */
49 49
50 50 #include <sys/types.h>
51 51 #include <sys/param.h>
52 52 #include <sys/sysmacros.h>
53 53 #include <sys/systm.h>
54 54 #include <sys/vnode.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/debug.h>
57 57 #include <sys/thread.h>
58 58 #include <sys/cpuvar.h>
59 59 #include <sys/bitmap.h>
60 60 #include <sys/atomic.h>
61 61 #include <sys/lgrp.h>
62 62
63 63 #include <vm/seg_kmem.h>
64 64 #include <vm/seg_kpm.h>
65 65 #include <vm/hat.h>
66 66 #include <vm/as.h>
67 67 #include <vm/seg.h>
68 68 #include <vm/page.h>
69 69
70 70 /*
71 71 * Global kpm controls.
72 72 * See also platform and mmu specific controls.
73 73 *
74 74 * kpm_enable -- global on/off switch for segkpm.
75 75 * . Set by default on 64bit platforms that have kpm support.
76 76 * . Will be disabled from platform layer if not supported.
77 77 * . Can be disabled via /etc/system.
78 78 *
79 79 * kpm_smallpages -- use only regular/system pagesize for kpm mappings.
80 80 * . Can be useful for critical debugging of kpm clients.
81 81 * . Set to zero by default for platforms that support kpm large pages.
82 82 * The use of kpm large pages reduces the footprint of kpm meta data
83 83 * and has all the other advantages of using large pages (e.g TLB
84 84 * miss reduction).
85 85 * . Set by default for platforms that don't support kpm large pages or
86 86 * where large pages cannot be used for other reasons (e.g. there are
87 87 * only few full associative TLB entries available for large pages).
88 88 *
89 89 * segmap_kpm -- separate on/off switch for segmap using segkpm:
90 90 * . Set by default.
91 91 * . Will be disabled when kpm_enable is zero.
92 92 * . Will be disabled when MAXBSIZE != PAGESIZE.
93 93 * . Can be disabled via /etc/system.
94 94 *
95 95 */
96 96 int kpm_enable = 1;
97 97 int kpm_smallpages = 0;
98 98 int segmap_kpm = 1;
99 99
100 100 /*
101 101 * Private seg op routines.
102 102 */
103 103 faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr,
|
↓ open down ↓ |
103 lines elided |
↑ open up ↑ |
104 104 size_t len, enum fault_type type, enum seg_rw rw);
105 105 static void segkpm_dump(struct seg *);
106 106 static void segkpm_badop(void);
107 107 static int segkpm_notsup(void);
108 108 static int segkpm_capable(struct seg *, segcapability_t);
109 109
110 110 #define SEGKPM_BADOP(t) (t(*)())segkpm_badop
111 111 #define SEGKPM_NOTSUP (int(*)())segkpm_notsup
112 112
113 113 static struct seg_ops segkpm_ops = {
114 - SEGKPM_BADOP(int), /* dup */
115 - SEGKPM_BADOP(int), /* unmap */
116 - SEGKPM_BADOP(void), /* free */
117 - segkpm_fault,
118 - SEGKPM_BADOP(int), /* faulta */
119 - SEGKPM_BADOP(int), /* setprot */
120 - SEGKPM_BADOP(int), /* checkprot */
121 - SEGKPM_BADOP(int), /* kluster */
122 - SEGKPM_BADOP(int), /* sync */
123 - SEGKPM_BADOP(size_t), /* incore */
124 - SEGKPM_BADOP(int), /* lockop */
125 - SEGKPM_BADOP(int), /* getprot */
126 - SEGKPM_BADOP(u_offset_t), /* getoffset */
127 - SEGKPM_BADOP(int), /* gettype */
128 - SEGKPM_BADOP(int), /* getvp */
129 - SEGKPM_BADOP(int), /* advise */
130 - segkpm_dump, /* dump */
131 - SEGKPM_NOTSUP, /* pagelock */
132 - SEGKPM_BADOP(int), /* setpgsz */
133 - SEGKPM_BADOP(int), /* getmemid */
134 - SEGKPM_BADOP(lgrp_mem_policy_info_t *), /* getpolicy */
135 - segkpm_capable, /* capable */
136 - seg_inherit_notsup /* inherit */
114 + .dup = SEGKPM_BADOP(int),
115 + .unmap = SEGKPM_BADOP(int),
116 + .free = SEGKPM_BADOP(void),
117 + .fault = segkpm_fault,
118 + .faulta = SEGKPM_BADOP(int),
119 + .setprot = SEGKPM_BADOP(int),
120 + .checkprot = SEGKPM_BADOP(int),
121 + .kluster = SEGKPM_BADOP(int),
122 + .sync = SEGKPM_BADOP(int),
123 + .incore = SEGKPM_BADOP(size_t),
124 + .lockop = SEGKPM_BADOP(int),
125 + .getprot = SEGKPM_BADOP(int),
126 + .getoffset = SEGKPM_BADOP(u_offset_t),
127 + .gettype = SEGKPM_BADOP(int),
128 + .getvp = SEGKPM_BADOP(int),
129 + .advise = SEGKPM_BADOP(int),
130 + .dump = segkpm_dump,
131 + .pagelock = SEGKPM_NOTSUP,
132 + .setpagesize = SEGKPM_BADOP(int),
133 + .getmemid = SEGKPM_BADOP(int),
134 + .getpolicy = SEGKPM_BADOP(lgrp_mem_policy_info_t *),
135 + .capable = segkpm_capable,
136 + .inherit = seg_inherit_notsup,
137 137 };
138 138
139 139 /*
140 140 * kpm_pgsz and kpm_pgshft are set by platform layer.
141 141 */
142 142 size_t kpm_pgsz; /* kpm page size */
143 143 uint_t kpm_pgshft; /* kpm page shift */
144 144 u_offset_t kpm_pgoff; /* kpm page offset mask */
145 145 uint_t kpmp2pshft; /* kpm page to page shift */
146 146 pgcnt_t kpmpnpgs; /* how many pages per kpm page */
147 147
148 148
149 149 #ifdef SEGKPM_SUPPORT
150 150
151 151 int
152 152 segkpm_create(struct seg *seg, void *argsp)
153 153 {
154 154 struct segkpm_data *skd;
155 155 struct segkpm_crargs *b = (struct segkpm_crargs *)argsp;
156 156 ushort_t *p;
157 157 int i, j;
158 158
159 159 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
160 160 ASSERT(btokpmp(seg->s_size) >= 1 &&
161 161 kpmpageoff((uintptr_t)seg->s_base) == 0 &&
162 162 kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0);
163 163
164 164 skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP);
165 165
166 166 seg->s_data = (void *)skd;
167 167 seg->s_ops = &segkpm_ops;
168 168 skd->skd_prot = b->prot;
169 169
170 170 /*
171 171 * (1) Segkpm virtual addresses are based on physical adresses.
172 172 * From this and in opposite to other segment drivers it is
173 173 * often required to allocate a page first to be able to
174 174 * calculate the final segkpm virtual address.
175 175 * (2) Page allocation is done by calling page_create_va(),
176 176 * one important input argument is a virtual address (also
177 177 * expressed by the "va" in the function name). This function
178 178 * is highly optimized to select the right page for an optimal
179 179 * processor and platform support (e.g. virtual addressed
180 180 * caches (VAC), physical addressed caches, NUMA).
181 181 *
182 182 * Because of (1) the approach is to generate a faked virtual
183 183 * address for calling page_create_va(). In order to exploit
184 184 * the abilities of (2), especially to utilize the cache
185 185 * hierarchy (3) and to avoid VAC alias conflicts (4) the
186 186 * selection has to be done carefully. For each virtual color
187 187 * a separate counter is provided (4). The count values are
188 188 * used for the utilization of all cache lines (3) and are
189 189 * corresponding to the cache bins.
190 190 */
191 191 skd->skd_nvcolors = b->nvcolors;
192 192
193 193 p = skd->skd_va_select =
194 194 kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP);
195 195
196 196 for (i = 0; i < NCPU; i++)
197 197 for (j = 0; j < b->nvcolors; j++, p++)
198 198 *p = j;
199 199
200 200 return (0);
201 201 }
202 202
203 203 /*
204 204 * This routine is called via a machine specific fault handling
205 205 * routine.
206 206 */
207 207 /* ARGSUSED */
208 208 faultcode_t
209 209 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
210 210 enum fault_type type, enum seg_rw rw)
211 211 {
212 212 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
213 213
214 214 switch (type) {
215 215 case F_INVAL:
216 216 return (hat_kpm_fault(hat, addr));
217 217 case F_SOFTLOCK:
218 218 case F_SOFTUNLOCK:
219 219 return (0);
220 220 default:
221 221 return (FC_NOSUPPORT);
222 222 }
223 223 /*NOTREACHED*/
224 224 }
225 225
226 226 #define addr_to_vcolor(addr, vcolors) \
227 227 ((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT))
228 228
229 229 /*
230 230 * Create a virtual address that can be used for invocations of
231 231 * page_create_va. Goal is to utilize the cache hierarchy (round
232 232 * robin bins) and to select the right color for virtual indexed
233 233 * caches. It isn't exact since we also increment the bin counter
234 234 * when the caller uses VOP_GETPAGE and gets a hit in the page
235 235 * cache, but we keep the bins turning for cache distribution
236 236 * (see also segkpm_create block comment).
237 237 */
238 238 caddr_t
239 239 segkpm_create_va(u_offset_t off)
240 240 {
241 241 int vcolor;
242 242 ushort_t *p;
243 243 struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data;
244 244 int nvcolors = skd->skd_nvcolors;
245 245 caddr_t va;
246 246
247 247 vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0;
248 248 p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor];
249 249 va = (caddr_t)ptob(*p);
250 250
251 251 atomic_add_16(p, nvcolors);
252 252
253 253 return (va);
254 254 }
255 255
256 256 /*
257 257 * Unload mapping if the instance has an active kpm mapping.
258 258 */
259 259 void
260 260 segkpm_mapout_validkpme(struct kpme *kpme)
261 261 {
262 262 caddr_t vaddr;
263 263 page_t *pp;
264 264
265 265 retry:
266 266 if ((pp = kpme->kpe_page) == NULL) {
267 267 return;
268 268 }
269 269
270 270 if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0)
271 271 goto retry;
272 272
273 273 /*
274 274 * Check if segkpm mapping is not unloaded in the meantime
275 275 */
276 276 if (kpme->kpe_page == NULL) {
277 277 page_unlock(pp);
278 278 return;
279 279 }
280 280
281 281 vaddr = hat_kpm_page2va(pp, 1);
282 282 hat_kpm_mapout(pp, kpme, vaddr);
283 283 page_unlock(pp);
284 284 }
285 285
286 286 static void
287 287 segkpm_badop()
288 288 {
289 289 panic("segkpm_badop");
290 290 }
291 291
292 292 #else /* SEGKPM_SUPPORT */
293 293
294 294 /* segkpm stubs */
295 295
296 296 /*ARGSUSED*/
297 297 int segkpm_create(struct seg *seg, void *argsp) { return (0); }
298 298
299 299 /* ARGSUSED */
300 300 faultcode_t
301 301 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
302 302 enum fault_type type, enum seg_rw rw)
303 303 {
304 304 return ((faultcode_t)0);
305 305 }
306 306
307 307 /* ARGSUSED */
308 308 caddr_t segkpm_create_va(u_offset_t off) { return (NULL); }
309 309
310 310 /* ARGSUSED */
311 311 void segkpm_mapout_validkpme(struct kpme *kpme) {}
312 312
313 313 static void
314 314 segkpm_badop() {}
315 315
316 316 #endif /* SEGKPM_SUPPORT */
317 317
318 318 static int
319 319 segkpm_notsup()
320 320 {
321 321 return (ENOTSUP);
322 322 }
323 323
324 324 /*
325 325 * segkpm pages are not dumped, so we just return
326 326 */
327 327 /*ARGSUSED*/
328 328 static void
329 329 segkpm_dump(struct seg *seg)
330 330 {}
331 331
332 332 /*
333 333 * We claim to have no special capabilities.
334 334 */
335 335 /*ARGSUSED*/
336 336 static int
337 337 segkpm_capable(struct seg *seg, segcapability_t capability)
338 338 {
339 339 return (0);
340 340 }
|
↓ open down ↓ |
194 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX