Print this page
5045 use atomic_{inc,dec}_* instead of atomic_add_*
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/vm/htable.h
+++ new/usr/src/uts/i86pc/vm/htable.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
|
↓ open down ↓ |
18 lines elided |
↑ open up ↑ |
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #ifndef _VM_HTABLE_H
27 27 #define _VM_HTABLE_H
28 28
29 -#pragma ident "%Z%%M% %I% %E% SMI"
30 -
31 29 #ifdef __cplusplus
32 30 extern "C" {
33 31 #endif
34 32
35 33 #if defined(__GNUC__) && defined(_ASM_INLINES) && defined(_KERNEL)
36 34 #include <asm/htable.h>
37 35 #endif
38 36
39 37 extern void atomic_andb(uint8_t *addr, uint8_t value);
40 38 extern void atomic_orb(uint8_t *addr, uint8_t value);
41 39 extern void atomic_inc16(uint16_t *addr);
42 40 extern void atomic_dec16(uint16_t *addr);
43 41 extern void mmu_tlbflush_entry(caddr_t addr);
44 42
45 43 /*
46 44 * Each hardware page table has an htable_t describing it.
47 45 *
48 46 * We use a reference counter mechanism to detect when we can free an htable.
49 47 * In the implmentation the reference count is split into 2 separate counters:
50 48 *
51 49 * ht_busy is a traditional reference count of uses of the htable pointer
52 50 *
53 51 * ht_valid_cnt is a count of how references are implied by valid PTE/PTP
54 52 * entries in the pagetable
55 53 *
56 54 * ht_busy is only incremented by htable_lookup() or htable_create()
57 55 * while holding the appropriate hash_table mutex. While installing a new
58 56 * valid PTE or PTP, in order to increment ht_valid_cnt a thread must have
59 57 * done an htable_lookup() or htable_create() but not the htable_release yet.
60 58 *
61 59 * htable_release(), while holding the mutex, can know that if
62 60 * busy == 1 and valid_cnt == 0, the htable can be free'd.
63 61 *
64 62 * The fields have been ordered to make htable_lookup() fast. Hence,
65 63 * ht_hat, ht_vaddr, ht_level and ht_next need to be clustered together.
66 64 */
67 65 struct htable {
68 66 struct htable *ht_next; /* forward link for hash table */
69 67 struct hat *ht_hat; /* hat this mapping comes from */
70 68 uintptr_t ht_vaddr; /* virt addr at start of this table */
71 69 int8_t ht_level; /* page table level: 0=4K, 1=2M, ... */
72 70 uint8_t ht_flags; /* see below */
73 71 int16_t ht_busy; /* implements locking protocol */
74 72 int16_t ht_valid_cnt; /* # of valid entries in this table */
75 73 uint32_t ht_lock_cnt; /* # of locked entries in this table */
76 74 /* never used for kernel hat */
77 75 pfn_t ht_pfn; /* pfn of page of the pagetable */
78 76 struct htable *ht_prev; /* backward link for hash table */
79 77 struct htable *ht_parent; /* htable that points to this htable */
80 78 struct htable *ht_shares; /* for HTABLE_SHARED_PFN only */
81 79 };
82 80 typedef struct htable htable_t;
83 81
84 82 /*
85 83 * Flags values for htable ht_flags field:
86 84 *
87 85 * HTABLE_VLP - this is the top level htable of a VLP HAT.
88 86 *
89 87 * HTABLE_SHARED_PFN - this htable had its PFN assigned from sharing another
90 88 * htable. Used by hat_share() for ISM.
91 89 */
92 90 #define HTABLE_VLP (0x01)
93 91 #define HTABLE_SHARED_PFN (0x02)
94 92
95 93 /*
96 94 * The htable hash table hashing function. The 28 is so that high
97 95 * order bits are include in the hash index to skew the wrap
98 96 * around of addresses. Even though the hash buckets are stored per
99 97 * hat we include the value of hat pointer in the hash function so
100 98 * that the secondary hash for the htable mutex winds up begin different in
101 99 * every address space.
102 100 */
103 101 #define HTABLE_HASH(hat, va, lvl) \
104 102 ((((va) >> LEVEL_SHIFT(1)) + ((va) >> 28) + (lvl) + \
105 103 ((uintptr_t)(hat) >> 4)) & ((hat)->hat_num_hash - 1))
106 104
107 105 /*
108 106 * Each CPU gets a unique hat_cpu_info structure in cpu_hat_info.
109 107 */
110 108 struct hat_cpu_info {
111 109 kmutex_t hci_mutex; /* mutex to ensure sequential usage */
112 110 #if defined(__amd64)
113 111 pfn_t hci_vlp_pfn; /* pfn of hci_vlp_l3ptes */
114 112 x86pte_t *hci_vlp_l3ptes; /* VLP Level==3 pagetable (top) */
115 113 x86pte_t *hci_vlp_l2ptes; /* VLP Level==2 pagetable */
116 114 #endif /* __amd64 */
117 115 };
118 116
119 117
120 118 /*
121 119 * Compute the last page aligned VA mapped by an htable.
122 120 *
123 121 * Given a va and a level, compute the virtual address of the start of the
124 122 * next page at that level.
125 123 *
126 124 * XX64 - The check for the VA hole needs to be better generalized.
127 125 */
128 126 #if defined(__amd64)
129 127 #define HTABLE_NUM_PTES(ht) (((ht)->ht_flags & HTABLE_VLP) ? 4 : 512)
130 128
131 129 #define HTABLE_LAST_PAGE(ht) \
132 130 ((ht)->ht_level == mmu.max_level ? ((uintptr_t)0UL - MMU_PAGESIZE) :\
133 131 ((ht)->ht_vaddr - MMU_PAGESIZE + \
134 132 ((uintptr_t)HTABLE_NUM_PTES(ht) << LEVEL_SHIFT((ht)->ht_level))))
135 133
136 134 #define NEXT_ENTRY_VA(va, l) \
137 135 ((va & LEVEL_MASK(l)) + LEVEL_SIZE(l) == mmu.hole_start ? \
138 136 mmu.hole_end : (va & LEVEL_MASK(l)) + LEVEL_SIZE(l))
139 137
140 138 #elif defined(__i386)
141 139
142 140 #define HTABLE_NUM_PTES(ht) \
143 141 (!mmu.pae_hat ? 1024 : ((ht)->ht_level == 2 ? 4 : 512))
144 142
145 143 #define HTABLE_LAST_PAGE(ht) ((ht)->ht_vaddr - MMU_PAGESIZE + \
146 144 ((uintptr_t)HTABLE_NUM_PTES(ht) << LEVEL_SHIFT((ht)->ht_level)))
147 145
148 146 #define NEXT_ENTRY_VA(va, l) ((va & LEVEL_MASK(l)) + LEVEL_SIZE(l))
149 147
150 148 #endif
151 149
152 150 #if defined(_KERNEL)
153 151
154 152 /*
155 153 * initialization function called from hat_init()
156 154 */
157 155 extern void htable_init(void);
158 156
159 157 /*
160 158 * Functions to lookup, or "lookup and create", the htable corresponding
161 159 * to the virtual address "vaddr" in the "hat" at the given "level" of
162 160 * page tables. htable_lookup() may return NULL if no such entry exists.
163 161 *
164 162 * On return the given htable is marked busy (a shared lock) - this prevents
165 163 * the htable from being stolen or freed) until htable_release() is called.
166 164 *
167 165 * If kalloc_flag is set on an htable_create() we can't call kmem allocation
168 166 * routines for this htable, since it's for the kernel hat itself.
169 167 *
170 168 * htable_acquire() is used when an htable pointer has been extracted from
171 169 * an hment and we need to get a reference to the htable.
172 170 */
173 171 extern htable_t *htable_lookup(struct hat *hat, uintptr_t vaddr, level_t level);
174 172 extern htable_t *htable_create(struct hat *hat, uintptr_t vaddr, level_t level,
175 173 htable_t *shared);
176 174 extern void htable_acquire(htable_t *);
177 175
178 176 extern void htable_release(htable_t *ht);
179 177 extern void htable_destroy(htable_t *ht);
180 178
181 179 /*
182 180 * Code to free all remaining htables for a hat. Called after the hat is no
183 181 * longer in use by any thread.
184 182 */
185 183 extern void htable_purge_hat(struct hat *hat);
186 184
187 185 /*
188 186 * Find the htable, page table entry index, and PTE of the given virtual
189 187 * address. If not found returns NULL. When found, returns the htable_t *,
190 188 * sets entry, and has a hold on the htable.
191 189 */
192 190 extern htable_t *htable_getpte(struct hat *, uintptr_t, uint_t *, x86pte_t *,
193 191 level_t);
194 192
195 193 /*
196 194 * Similar to hat_getpte(), except that this only succeeds if a valid
197 195 * page mapping is present.
198 196 */
199 197 extern htable_t *htable_getpage(struct hat *hat, uintptr_t va, uint_t *entry);
200 198
201 199 /*
202 200 * Called to allocate initial/additional htables for reserve.
203 201 */
204 202 extern void htable_initial_reserve(uint_t);
205 203 extern void htable_reserve(uint_t);
206 204
207 205 /*
208 206 * Used to readjust the htable reserve after the reserve list has been used.
209 207 * Also called after boot to release left over boot reserves.
210 208 */
211 209 extern void htable_adjust_reserve(void);
212 210
213 211 /*
214 212 * return number of bytes mapped by all the htables in a given hat
215 213 */
216 214 extern size_t htable_mapped(struct hat *);
217 215
218 216
219 217 /*
220 218 * Attach initial pagetables as htables
221 219 */
222 220 extern void htable_attach(struct hat *, uintptr_t, level_t, struct htable *,
223 221 pfn_t);
224 222
225 223 /*
226 224 * Routine to find the next populated htable at or above a given virtual
227 225 * address. Can specify an upper limit, or HTABLE_WALK_TO_END to indicate
228 226 * that it should search the entire address space. Similar to
229 227 * hat_getpte(), but used for walking through address ranges. It can be
230 228 * used like this:
231 229 *
232 230 * va = ...
233 231 * ht = NULL;
234 232 * while (va < end_va) {
235 233 * pte = htable_walk(hat, &ht, &va, end_va);
236 234 * if (!pte)
237 235 * break;
238 236 *
239 237 * ... code to operate on page at va ...
240 238 *
241 239 * va += LEVEL_SIZE(ht->ht_level);
242 240 * }
243 241 * if (ht)
244 242 * htable_release(ht);
245 243 *
246 244 */
247 245 extern x86pte_t htable_walk(struct hat *hat, htable_t **ht, uintptr_t *va,
248 246 uintptr_t eaddr);
249 247
250 248 #define HTABLE_WALK_TO_END ((uintptr_t)-1)
251 249
252 250 /*
253 251 * Utilities convert between virtual addresses and page table entry indeces.
254 252 */
255 253 extern uint_t htable_va2entry(uintptr_t va, htable_t *ht);
256 254 extern uintptr_t htable_e2va(htable_t *ht, uint_t entry);
257 255
258 256 /*
259 257 * Interfaces that provide access to page table entries via the htable.
260 258 *
261 259 * Note that all accesses except x86pte_copy() and x86pte_zero() are atomic.
262 260 */
263 261 extern void x86pte_cpu_init(cpu_t *);
264 262 extern void x86pte_cpu_fini(cpu_t *);
265 263
266 264 extern x86pte_t x86pte_get(htable_t *, uint_t entry);
267 265
268 266 /*
269 267 * x86pte_set returns LPAGE_ERROR if it's asked to overwrite a page table
270 268 * link with a large page mapping.
271 269 */
272 270 #define LPAGE_ERROR (-(x86pte_t)1)
273 271 extern x86pte_t x86pte_set(htable_t *, uint_t entry, x86pte_t new, void *);
274 272
275 273 extern x86pte_t x86pte_inval(htable_t *ht, uint_t entry,
276 274 x86pte_t old, x86pte_t *ptr);
277 275
278 276 extern x86pte_t x86pte_update(htable_t *ht, uint_t entry,
279 277 x86pte_t old, x86pte_t new);
280 278
281 279 extern void x86pte_copy(htable_t *src, htable_t *dest, uint_t entry,
282 280 uint_t cnt);
283 281
284 282 /*
|
↓ open down ↓ |
244 lines elided |
↑ open up ↑ |
285 283 * access to a pagetable knowing only the pfn
286 284 */
287 285 extern x86pte_t *x86pte_mapin(pfn_t, uint_t, htable_t *);
288 286 extern void x86pte_mapout(void);
289 287
290 288 /*
291 289 * these are actually inlines for "lock; incw", "lock; decw", etc. instructions.
292 290 */
293 291 #define HTABLE_INC(x) atomic_inc16((uint16_t *)&x)
294 292 #define HTABLE_DEC(x) atomic_dec16((uint16_t *)&x)
295 -#define HTABLE_LOCK_INC(ht) atomic_add_32(&(ht)->ht_lock_cnt, 1)
296 -#define HTABLE_LOCK_DEC(ht) atomic_add_32(&(ht)->ht_lock_cnt, -1)
293 +#define HTABLE_LOCK_INC(ht) atomic_inc_32(&(ht)->ht_lock_cnt)
294 +#define HTABLE_LOCK_DEC(ht) atomic_dec_32(&(ht)->ht_lock_cnt)
297 295
298 296 #ifdef __xpv
299 297 extern void xen_flush_va(caddr_t va);
300 298 extern void xen_gflush_va(caddr_t va, cpuset_t);
301 299 extern void xen_flush_tlb(void);
302 300 extern void xen_gflush_tlb(cpuset_t);
303 301 extern void xen_pin(pfn_t, level_t);
304 302 extern void xen_unpin(pfn_t);
305 303 extern int xen_kpm_page(pfn_t, uint_t);
306 304
307 305 /*
308 306 * The hypervisor maps all page tables into our address space read-only.
309 307 * Under normal circumstances, the hypervisor then handles all updates to
310 308 * the page tables underneath the covers for us. However, when we are
311 309 * trying to dump core after a hypervisor panic, the hypervisor is no
312 310 * longer available to do these updates. To work around the protection
313 311 * problem, we simply disable write-protect checking for the duration of a
314 312 * pagetable update operation.
315 313 */
316 314 #define XPV_ALLOW_PAGETABLE_UPDATES() \
317 315 { \
318 316 if (IN_XPV_PANIC()) \
319 317 setcr0((getcr0() & ~CR0_WP) & 0xffffffff); \
320 318 }
321 319 #define XPV_DISALLOW_PAGETABLE_UPDATES() \
322 320 { \
323 321 if (IN_XPV_PANIC() > 0) \
324 322 setcr0((getcr0() | CR0_WP) & 0xffffffff); \
325 323 }
326 324
327 325 #else /* __xpv */
328 326
329 327 #define XPV_ALLOW_PAGETABLE_UPDATES()
330 328 #define XPV_DISALLOW_PAGETABLE_UPDATES()
331 329
332 330 #endif
333 331
334 332 #endif /* _KERNEL */
335 333
336 334
337 335 #ifdef __cplusplus
338 336 }
339 337 #endif
340 338
341 339 #endif /* _VM_HTABLE_H */
|
↓ open down ↓ |
35 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX