Print this page
5042 stop using deprecated atomic functions
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/sun4v/os/ppage.c
+++ new/usr/src/uts/sun4v/os/ppage.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 -#pragma ident "%Z%%M% %I% %E% SMI"
27 -
28 26 #include <sys/types.h>
29 27 #include <sys/systm.h>
30 28 #include <sys/archsystm.h>
31 29 #include <sys/machsystm.h>
32 30 #include <sys/t_lock.h>
33 31 #include <sys/vmem.h>
34 32 #include <sys/mman.h>
35 33 #include <sys/vm.h>
36 34 #include <sys/cpu.h>
37 35 #include <sys/cmn_err.h>
38 36 #include <sys/cpuvar.h>
39 37 #include <sys/atomic.h>
40 38 #include <vm/as.h>
41 39 #include <vm/hat.h>
42 40 #include <vm/as.h>
43 41 #include <vm/page.h>
44 42 #include <vm/seg.h>
45 43 #include <vm/seg_kmem.h>
46 44 #include <vm/seg_kpm.h>
47 45 #include <vm/hat_sfmmu.h>
48 46 #include <sys/debug.h>
49 47 #include <sys/cpu_module.h>
50 48
51 49 /*
52 50 * A quick way to generate a cache consistent address to map in a page.
53 51 * users: ppcopy, pagezero, /proc, dev/mem
54 52 *
55 53 * The ppmapin/ppmapout routines provide a quick way of generating a cache
56 54 * consistent address by reserving a given amount of kernel address space.
57 55 * The base is PPMAPBASE and its size is PPMAPSIZE. This memory is divided
58 56 * into x number of sets, where x is the number of colors for the virtual
59 57 * cache. The number of colors is how many times a page can be mapped
60 58 * simulatenously in the cache. For direct map caches this translates to
61 59 * the number of pages in the cache.
62 60 * Each set will be assigned a group of virtual pages from the reserved memory
63 61 * depending on its virtual color.
64 62 * When trying to assign a virtual address we will find out the color for the
65 63 * physical page in question (if applicable). Then we will try to find an
66 64 * available virtual page from the set of the appropiate color.
67 65 */
68 66
69 67 int pp_slots = 4; /* small default, tuned by cpu module */
70 68
71 69 /* tuned by cpu module, default is "safe" */
72 70 int pp_consistent_coloring = PPAGE_STORES_POLLUTE | PPAGE_LOADS_POLLUTE;
73 71
74 72 static caddr_t ppmap_vaddrs[PPMAPSIZE / MMU_PAGESIZE];
75 73 static int nsets; /* number of sets */
76 74 static int ppmap_shift; /* set selector */
77 75
78 76 #ifdef PPDEBUG
79 77 #define MAXCOLORS 16 /* for debug only */
80 78 static int ppalloc_noslot = 0; /* # of allocations from kernelmap */
81 79 static int align_hits;
82 80 static int pp_allocs; /* # of ppmapin requests */
83 81 #endif /* PPDEBUG */
84 82
85 83 /*
86 84 * There are only 64 TLB entries on spitfire, 16 on cheetah
87 85 * (fully-associative TLB) so we allow the cpu module to tune the
88 86 * number to use here via pp_slots.
89 87 */
90 88 static struct ppmap_va {
91 89 caddr_t ppmap_slots[MAXPP_SLOTS];
92 90 } ppmap_va[NCPU];
93 91
94 92 /* prevent compilation with VAC defined */
95 93 #ifdef VAC
96 94 #error "sun4v ppmapin and ppmapout do not support VAC"
97 95 #endif
98 96
99 97 void
100 98 ppmapinit(void)
101 99 {
102 100 int nset;
103 101 caddr_t va;
104 102
105 103 ASSERT(pp_slots <= MAXPP_SLOTS);
106 104
107 105 va = (caddr_t)PPMAPBASE;
108 106
109 107 /*
110 108 * sun4v does not have a virtual indexed cache and simply
111 109 * has only one set containing all pages.
112 110 */
113 111 nsets = mmu_btop(PPMAPSIZE);
114 112 ppmap_shift = MMU_PAGESHIFT;
115 113
116 114 for (nset = 0; nset < nsets; nset++) {
117 115 ppmap_vaddrs[nset] =
118 116 (caddr_t)((uintptr_t)va + (nset * MMU_PAGESIZE));
119 117 }
120 118 }
121 119
122 120 /*
123 121 * Allocate a cache consistent virtual address to map a page, pp,
124 122 * with protection, vprot; and map it in the MMU, using the most
125 123 * efficient means possible. The argument avoid is a virtual address
126 124 * hint which when masked yields an offset into a virtual cache
127 125 * that should be avoided when allocating an address to map in a
128 126 * page. An avoid arg of -1 means you don't care, for instance pagezero.
129 127 *
130 128 * machine dependent, depends on virtual address space layout,
131 129 * understands that all kernel addresses have bit 31 set.
132 130 *
133 131 * NOTE: For sun4 platforms the meaning of the hint argument is opposite from
134 132 * that found in other architectures. In other architectures the hint
135 133 * (called avoid) was used to ask ppmapin to NOT use the specified cache color.
136 134 * This was used to avoid virtual cache trashing in the bcopy. Unfortunately
137 135 * in the case of a COW, this later on caused a cache aliasing conflict. In
138 136 * sun4, the bcopy routine uses the block ld/st instructions so we don't have
139 137 * to worry about virtual cache trashing. Actually, by using the hint to choose
140 138 * the right color we can almost guarantee a cache conflict will not occur.
141 139 */
142 140
143 141 /*ARGSUSED2*/
144 142 caddr_t
145 143 ppmapin(page_t *pp, uint_t vprot, caddr_t hint)
146 144 {
147 145 int nset;
148 146 caddr_t va;
149 147
150 148 #ifdef PPDEBUG
151 149 pp_allocs++;
152 150 #endif /* PPDEBUG */
153 151
|
↓ open down ↓ |
116 lines elided |
↑ open up ↑ |
154 152 /*
155 153 * For sun4v caches are physical caches, we can pick any address
156 154 * we want.
157 155 */
158 156 for (nset = 0; nset < nsets; nset++) {
159 157 va = ppmap_vaddrs[nset];
160 158 if (va != NULL) {
161 159 #ifdef PPDEBUG
162 160 align_hits++;
163 161 #endif /* PPDEBUG */
164 - if (casptr(&ppmap_vaddrs[nset], va, NULL) == va) {
162 + if (atomic_cas_ptr(&ppmap_vaddrs[nset], va, NULL) ==
163 + va) {
165 164 hat_memload(kas.a_hat, va, pp,
166 165 vprot | HAT_NOSYNC,
167 166 HAT_LOAD_LOCK);
168 167 return (va);
169 168 }
170 169 }
171 170 }
172 171
173 172 #ifdef PPDEBUG
174 173 ppalloc_noslot++;
175 174 #endif /* PPDEBUG */
176 175
177 176 /*
178 177 * No free slots; get a random one from the kernel heap area.
179 178 */
180 179 va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
181 180
182 181 hat_memload(kas.a_hat, va, pp, vprot | HAT_NOSYNC, HAT_LOAD_LOCK);
183 182
184 183 return (va);
185 184
186 185 }
187 186
188 187 void
189 188 ppmapout(caddr_t va)
190 189 {
191 190 int nset;
192 191
193 192 if (va >= kernelheap && va < ekernelheap) {
194 193 /*
195 194 * Space came from kernelmap, flush the page and
196 195 * return the space.
197 196 */
198 197 hat_unload(kas.a_hat, va, PAGESIZE,
199 198 (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK));
200 199 vmem_free(heap_arena, va, PAGESIZE);
201 200 } else {
202 201 /*
203 202 * Space came from ppmap_vaddrs[], give it back.
204 203 */
205 204 nset = ((uintptr_t)va >> ppmap_shift) & (nsets - 1);
206 205 hat_unload(kas.a_hat, va, PAGESIZE,
207 206 (HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK));
208 207
209 208 ASSERT(ppmap_vaddrs[nset] == NULL);
210 209 ppmap_vaddrs[nset] = va;
211 210 }
212 211 }
213 212
214 213 #ifdef DEBUG
215 214 #define PP_STAT_ADD(stat) (stat)++
216 215 uint_t pload, ploadfail;
217 216 uint_t ppzero, ppzero_short;
218 217 #else
219 218 #define PP_STAT_ADD(stat)
220 219 #endif /* DEBUG */
221 220
222 221 static void
223 222 pp_unload_tlb(caddr_t *pslot, caddr_t va)
224 223 {
225 224 ASSERT(*pslot == va);
226 225
227 226 vtag_flushpage(va, (uint64_t)ksfmmup);
228 227 *pslot = NULL; /* release the slot */
229 228 }
230 229
231 230 /*
232 231 * Routine to copy kernel pages during relocation. It will copy one
233 232 * PAGESIZE page to another PAGESIZE page. This function may be called
234 233 * above LOCK_LEVEL so it should not grab any locks.
235 234 */
236 235 void
237 236 ppcopy_kernel__relocatable(page_t *fm_pp, page_t *to_pp)
238 237 {
239 238 uint64_t fm_pa, to_pa;
240 239 size_t nbytes;
241 240
242 241 fm_pa = (uint64_t)(fm_pp->p_pagenum) << MMU_PAGESHIFT;
243 242 to_pa = (uint64_t)(to_pp->p_pagenum) << MMU_PAGESHIFT;
244 243
245 244 nbytes = MMU_PAGESIZE;
246 245
247 246 for (; nbytes > 0; fm_pa += 32, to_pa += 32, nbytes -= 32)
248 247 hw_pa_bcopy32(fm_pa, to_pa);
249 248 }
250 249
251 250 /*
252 251 * Copy the data from the physical page represented by "frompp" to
253 252 * that represented by "topp".
254 253 *
255 254 * Try to use per cpu mapping first, if that fails then call pp_mapin
256 255 * to load it.
257 256 * Returns one on success or zero on some sort of fault while doing the copy.
258 257 */
259 258 int
260 259 ppcopy(page_t *fm_pp, page_t *to_pp)
261 260 {
262 261 caddr_t fm_va = NULL;
263 262 caddr_t to_va;
264 263 boolean_t fast;
265 264 label_t ljb;
266 265 int ret = 1;
267 266
268 267 ASSERT(PAGE_LOCKED(fm_pp));
269 268 ASSERT(PAGE_LOCKED(to_pp));
270 269
271 270 /*
272 271 * Try to map using KPM if enabled. If it fails, fall
273 272 * back to ppmapin/ppmapout.
274 273 */
275 274 if ((kpm_enable == 0) ||
276 275 (fm_va = hat_kpm_mapin(fm_pp, NULL)) == NULL ||
277 276 (to_va = hat_kpm_mapin(to_pp, NULL)) == NULL) {
278 277 if (fm_va != NULL)
279 278 hat_kpm_mapout(fm_pp, NULL, fm_va);
280 279 fm_va = ppmapin(fm_pp, PROT_READ, (caddr_t)-1);
281 280 to_va = ppmapin(to_pp, PROT_READ | PROT_WRITE, fm_va);
282 281 fast = B_FALSE;
283 282 } else
284 283 fast = B_TRUE;
285 284
286 285 if (on_fault(&ljb)) {
287 286 ret = 0;
288 287 goto faulted;
289 288 }
290 289 bcopy(fm_va, to_va, PAGESIZE);
291 290 no_fault();
292 291 faulted:
293 292
294 293 /* Unmap */
295 294 if (fast) {
296 295 hat_kpm_mapout(fm_pp, NULL, fm_va);
297 296 hat_kpm_mapout(to_pp, NULL, to_va);
298 297 } else {
299 298 ppmapout(fm_va);
300 299 ppmapout(to_va);
301 300 }
302 301 return (ret);
303 302 }
304 303
305 304 /*
306 305 * Zero the physical page from off to off + len given by `pp'
307 306 * without changing the reference and modified bits of page.
308 307 *
309 308 * Again, we'll try per cpu mapping first.
310 309 */
311 310
312 311 void
313 312 pagezero(page_t *pp, uint_t off, uint_t len)
314 313 {
315 314 caddr_t va;
316 315 extern int hwblkclr(void *, size_t);
317 316 extern int use_hw_bzero;
318 317 boolean_t fast;
319 318
320 319 ASSERT((int)len > 0 && (int)off >= 0 && off + len <= PAGESIZE);
321 320 ASSERT(PAGE_LOCKED(pp));
322 321
323 322 PP_STAT_ADD(ppzero);
324 323
325 324 if (len != MMU_PAGESIZE || !use_hw_bzero) {
326 325 PP_STAT_ADD(ppzero_short);
327 326 }
328 327
329 328 kpreempt_disable();
330 329
331 330 /*
332 331 * Try to use KPM if enabled. If that fails, fall back to
333 332 * ppmapin/ppmapout.
334 333 */
335 334
336 335 if (kpm_enable != 0) {
337 336 fast = B_TRUE;
338 337 va = hat_kpm_mapin(pp, NULL);
339 338 } else
340 339 va = NULL;
341 340
342 341 if (va == NULL) {
343 342 fast = B_FALSE;
344 343 va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1);
345 344 }
346 345
347 346 if (!use_hw_bzero) {
348 347 bzero(va + off, len);
349 348 sync_icache(va + off, len);
350 349 } else if (hwblkclr(va + off, len)) {
351 350 /*
352 351 * We may not have used block commit asi.
353 352 * So flush the I-$ manually
354 353 */
355 354 sync_icache(va + off, len);
356 355 } else {
357 356 /*
358 357 * We have used blk commit, and flushed the I-$.
359 358 * However we still may have an instruction in the
360 359 * pipeline. Only a flush will invalidate that.
361 360 */
362 361 doflush(va);
363 362 }
364 363
365 364 if (fast) {
366 365 hat_kpm_mapout(pp, NULL, va);
367 366 } else {
368 367 ppmapout(va);
369 368 }
370 369 kpreempt_enable();
371 370 }
|
↓ open down ↓ |
197 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX