1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28
29 /*
30 * Portions of this source code were derived from Berkeley 4.3 BSD
31 * under license from the Regents of the University of California.
32 */
33
34 /*
35 * VM - segment for non-faulting loads.
36 */
37
38 #include <sys/types.h>
39 #include <sys/t_lock.h>
40 #include <sys/param.h>
41 #include <sys/mman.h>
42 #include <sys/errno.h>
43 #include <sys/kmem.h>
44 #include <sys/cmn_err.h>
45 #include <sys/vnode.h>
46 #include <sys/proc.h>
47 #include <sys/conf.h>
48 #include <sys/debug.h>
49 #include <sys/archsystm.h>
50 #include <sys/lgrp.h>
51
52 #include <vm/page.h>
53 #include <vm/hat.h>
54 #include <vm/as.h>
55 #include <vm/seg.h>
56 #include <vm/vpage.h>
57
58 /*
59 * Private seg op routines.
60 */
61 static int segnf_dup(struct seg *seg, struct seg *newseg);
62 static int segnf_unmap(struct seg *seg, caddr_t addr, size_t len);
63 static void segnf_free(struct seg *seg);
64 static faultcode_t segnf_nomap(void);
65 static int segnf_setprot(struct seg *seg, caddr_t addr,
66 size_t len, uint_t prot);
67 static int segnf_checkprot(struct seg *seg, caddr_t addr,
68 size_t len, uint_t prot);
69 static int segnf_nop(void);
70 static int segnf_getprot(struct seg *seg, caddr_t addr,
71 size_t len, uint_t *protv);
72 static u_offset_t segnf_getoffset(struct seg *seg, caddr_t addr);
73 static int segnf_gettype(struct seg *seg, caddr_t addr);
74 static int segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
75 static int segnf_pagelock(struct seg *seg, caddr_t addr, size_t len,
76 struct page ***ppp, enum lock_type type, enum seg_rw rw);
77
78
79 const struct seg_ops segnf_ops = {
80 .dup = segnf_dup,
81 .unmap = segnf_unmap,
82 .free = segnf_free,
83 .fault = (faultcode_t (*)(struct hat *, struct seg *, caddr_t,
84 size_t, enum fault_type, enum seg_rw))segnf_nomap,
85 .faulta = (faultcode_t (*)(struct seg *, caddr_t)) segnf_nomap,
86 .setprot = segnf_setprot,
87 .checkprot = segnf_checkprot,
88 .sync = (int (*)(struct seg *, caddr_t, size_t, int, uint_t))
89 segnf_nop,
90 .incore = (size_t (*)(struct seg *, caddr_t, size_t, char *))
91 segnf_nop,
92 .lockop = (int (*)(struct seg *, caddr_t, size_t, int, int,
93 ulong_t *, size_t))segnf_nop,
94 .getprot = segnf_getprot,
95 .getoffset = segnf_getoffset,
96 .gettype = segnf_gettype,
97 .getvp = segnf_getvp,
98 .advise = (int (*)(struct seg *, caddr_t, size_t, uint_t))
99 segnf_nop,
100 .pagelock = segnf_pagelock,
101 };
102
103 /*
104 * vnode and page for the page of zeros we use for the nf mappings.
105 */
106 static kmutex_t segnf_lock;
107 static struct vnode nfvp;
108 static struct page **nfpp;
109
110 #define addr_to_vcolor(addr) \
111 (shm_alignment) ? \
112 ((int)(((uintptr_t)(addr) & (shm_alignment - 1)) >> PAGESHIFT)) : 0
113
114 /*
115 * We try to limit the number of Non-fault segments created.
116 * Non fault segments are created to optimize sparc V9 code which uses
117 * the sparc nonfaulting load ASI (ASI_PRIMARY_NOFAULT).
118 *
119 * There are several reasons why creating too many non-fault segments
120 * could cause problems.
121 *
122 * First, excessive allocation of kernel resources for the seg
123 * structures and the HAT data to map the zero pages.
124 *
125 * Secondly, creating nofault segments actually uses up user virtual
126 * address space. This makes it unavailable for subsequent mmap(0, ...)
127 * calls which use as_gap() to find empty va regions. Creation of too
128 * many nofault segments could thus interfere with the ability of the
129 * runtime linker to load a shared object.
130 */
131 #define MAXSEGFORNF (10000)
132 #define MAXNFSEARCH (5)
133
134
135 /*
136 * Must be called from startup()
137 */
138 void
139 segnf_init()
140 {
141 mutex_init(&segnf_lock, NULL, MUTEX_DEFAULT, NULL);
142 }
143
144
145 /*
146 * Create a no-fault segment.
147 *
148 * The no-fault segment is not technically necessary, as the code in
149 * nfload() in trap.c will emulate the SPARC instruction and load
150 * a value of zero in the destination register.
151 *
152 * However, this code tries to put a page of zero's at the nofault address
153 * so that subsequent non-faulting loads to the same page will not
154 * trap with a tlb miss.
155 *
156 * In order to help limit the number of segments we merge adjacent nofault
157 * segments into a single segment. If we get a large number of segments
158 * we'll also try to delete a random other nf segment.
159 */
160 /* ARGSUSED */
161 int
162 segnf_create(struct seg *seg, void *argsp)
163 {
164 uint_t prot;
165 pgcnt_t vacpgs;
166 u_offset_t off = 0;
167 caddr_t vaddr = NULL;
168 int i, color;
169 struct seg *s1;
170 struct seg *s2;
171 size_t size;
172 struct as *as = seg->s_as;
173
174 ASSERT(as && AS_WRITE_HELD(as, &as->a_lock));
175
176 /*
177 * Need a page per virtual color or just 1 if no vac.
178 */
179 mutex_enter(&segnf_lock);
180 if (nfpp == NULL) {
181 struct seg kseg;
182
183 vacpgs = 1;
184 if (shm_alignment > PAGESIZE) {
185 vacpgs = shm_alignment >> PAGESHIFT;
186 }
187
188 nfpp = kmem_alloc(sizeof (*nfpp) * vacpgs, KM_SLEEP);
189
190 kseg.s_as = &kas;
191 for (i = 0; i < vacpgs; i++, off += PAGESIZE,
192 vaddr += PAGESIZE) {
193 nfpp[i] = page_create_va(&nfvp, off, PAGESIZE,
194 PG_WAIT | PG_NORELOC, &kseg, vaddr);
195 page_io_unlock(nfpp[i]);
196 page_downgrade(nfpp[i]);
197 pagezero(nfpp[i], 0, PAGESIZE);
198 }
199 }
200 mutex_exit(&segnf_lock);
201
202 hat_map(as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
203
204 /*
205 * s_data can't be NULL because of ASSERTS in the common vm code.
206 */
207 seg->s_ops = &segnf_ops;
208 seg->s_data = seg;
209 seg->s_flags |= S_PURGE;
210
211 mutex_enter(&as->a_contents);
212 as->a_flags |= AS_NEEDSPURGE;
213 mutex_exit(&as->a_contents);
214
215 prot = PROT_READ;
216 color = addr_to_vcolor(seg->s_base);
217 if (as != &kas)
218 prot |= PROT_USER;
219 hat_memload(as->a_hat, seg->s_base, nfpp[color],
220 prot | HAT_NOFAULT, HAT_LOAD);
221
222 /*
223 * At this point see if we can concatenate a segment to
224 * a non-fault segment immediately before and/or after it.
225 */
226 if ((s1 = AS_SEGPREV(as, seg)) != NULL &&
227 s1->s_ops == &segnf_ops &&
228 s1->s_base + s1->s_size == seg->s_base) {
229 size = s1->s_size;
230 seg_free(s1);
231 seg->s_base -= size;
232 seg->s_size += size;
233 }
234
235 if ((s2 = AS_SEGNEXT(as, seg)) != NULL &&
236 s2->s_ops == &segnf_ops &&
237 seg->s_base + seg->s_size == s2->s_base) {
238 size = s2->s_size;
239 seg_free(s2);
240 seg->s_size += size;
241 }
242
243 /*
244 * if we already have a lot of segments, try to delete some other
245 * nofault segment to reduce the probability of uncontrolled segment
246 * creation.
247 *
248 * the code looks around quickly (no more than MAXNFSEARCH segments
249 * each way) for another NF segment and then deletes it.
250 */
251 if (avl_numnodes(&as->a_segtree) > MAXSEGFORNF) {
252 size = 0;
253 s2 = NULL;
254 s1 = AS_SEGPREV(as, seg);
255 while (size++ < MAXNFSEARCH && s1 != NULL) {
256 if (s1->s_ops == &segnf_ops)
257 s2 = s1;
258 s1 = AS_SEGPREV(s1->s_as, seg);
259 }
260 if (s2 == NULL) {
261 s1 = AS_SEGNEXT(as, seg);
262 while (size-- > 0 && s1 != NULL) {
263 if (s1->s_ops == &segnf_ops)
264 s2 = s1;
265 s1 = AS_SEGNEXT(as, seg);
266 }
267 }
268 if (s2 != NULL)
269 seg_unmap(s2);
270 }
271
272 return (0);
273 }
274
275 /*
276 * Never really need "No fault" segments, so they aren't dup'd.
277 */
278 /* ARGSUSED */
279 static int
280 segnf_dup(struct seg *seg, struct seg *newseg)
281 {
282 panic("segnf_dup");
283 return (0);
284 }
285
286 /*
287 * Split a segment at addr for length len.
288 */
289 static int
290 segnf_unmap(struct seg *seg, caddr_t addr, size_t len)
291 {
292 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
293
294 /*
295 * Check for bad sizes.
296 */
297 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
298 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) {
299 cmn_err(CE_PANIC, "segnf_unmap: bad unmap size");
300 }
301
302 /*
303 * Unload any hardware translations in the range to be taken out.
304 */
305 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
306
307 if (addr == seg->s_base && len == seg->s_size) {
308 /*
309 * Freeing entire segment.
310 */
311 seg_free(seg);
312 } else if (addr == seg->s_base) {
313 /*
314 * Freeing the beginning of the segment.
315 */
316 seg->s_base += len;
317 seg->s_size -= len;
318 } else if (addr + len == seg->s_base + seg->s_size) {
319 /*
320 * Freeing the end of the segment.
321 */
322 seg->s_size -= len;
323 } else {
324 /*
325 * The section to go is in the middle of the segment, so we
326 * have to cut it into two segments. We shrink the existing
327 * "seg" at the low end, and create "nseg" for the high end.
328 */
329 caddr_t nbase = addr + len;
330 size_t nsize = (seg->s_base + seg->s_size) - nbase;
331 struct seg *nseg;
332
333 /*
334 * Trim down "seg" before trying to stick "nseg" into the as.
335 */
336 seg->s_size = addr - seg->s_base;
337 nseg = seg_alloc(seg->s_as, nbase, nsize);
338 if (nseg == NULL)
339 cmn_err(CE_PANIC, "segnf_unmap: seg_alloc failed");
340
341 /*
342 * s_data can't be NULL because of ASSERTs in common VM code.
343 */
344 nseg->s_ops = seg->s_ops;
345 nseg->s_data = nseg;
346 nseg->s_flags |= S_PURGE;
347 mutex_enter(&seg->s_as->a_contents);
348 seg->s_as->a_flags |= AS_NEEDSPURGE;
349 mutex_exit(&seg->s_as->a_contents);
350 }
351
352 return (0);
353 }
354
355 /*
356 * Free a segment.
357 */
358 static void
359 segnf_free(struct seg *seg)
360 {
361 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
362 }
363
364 /*
365 * No faults allowed on segnf.
366 */
367 static faultcode_t
368 segnf_nomap(void)
369 {
370 return (FC_NOMAP);
371 }
372
373 /* ARGSUSED */
374 static int
375 segnf_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
376 {
377 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
378 return (EACCES);
379 }
380
381 /* ARGSUSED */
382 static int
383 segnf_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
384 {
385 uint_t sprot;
386 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
387
388 sprot = seg->s_as == &kas ? PROT_READ : PROT_READ|PROT_USER;
389 return ((prot & sprot) == prot ? 0 : EACCES);
390 }
391
392 static int
393 segnf_nop(void)
394 {
395 return (0);
396 }
397
398 static int
399 segnf_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
400 {
401 size_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
402 size_t p;
403 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
404
405 for (p = 0; p < pgno; ++p)
406 protv[p] = PROT_READ;
407 return (0);
408 }
409
410 /* ARGSUSED */
411 static u_offset_t
412 segnf_getoffset(struct seg *seg, caddr_t addr)
413 {
414 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
415
416 return ((u_offset_t)0);
417 }
418
419 /* ARGSUSED */
420 static int
421 segnf_gettype(struct seg *seg, caddr_t addr)
422 {
423 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
424
425 return (MAP_SHARED);
426 }
427
428 /* ARGSUSED */
429 static int
430 segnf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
431 {
432 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
433
434 *vpp = &nfvp;
435 return (0);
436 }
437
438 /*ARGSUSED*/
439 static int
440 segnf_pagelock(struct seg *seg, caddr_t addr, size_t len,
441 struct page ***ppp, enum lock_type type, enum seg_rw rw)
442 {
443 return (ENOTSUP);
444 }