Print this page
6138 don't abuse atomic_cas_*
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/sun4/os/memnode.c
+++ new/usr/src/uts/sun4/os/memnode.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 #include <sys/systm.h>
27 27 #include <sys/platform_module.h>
28 28 #include <sys/sysmacros.h>
29 29 #include <sys/atomic.h>
30 30 #include <sys/memlist.h>
31 31 #include <sys/memnode.h>
32 32 #include <vm/vm_dep.h>
33 33
34 34 int max_mem_nodes = 1; /* max memory nodes on this system */
35 35
36 36 struct mem_node_conf mem_node_config[MAX_MEM_NODES];
37 37 int mem_node_pfn_shift;
38 38 /*
39 39 * num_memnodes should be updated atomically and always >=
40 40 * the number of bits in memnodes_mask or the algorithm may fail.
41 41 */
42 42 uint16_t num_memnodes;
43 43 mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */
44 44
45 45 /*
46 46 * If set, mem_node_physalign should be a power of two, and
47 47 * should reflect the minimum address alignment of each node.
48 48 */
49 49 uint64_t mem_node_physalign;
50 50
51 51 /*
52 52 * Platform hooks we will need.
53 53 */
54 54
55 55 #pragma weak plat_build_mem_nodes
56 56 #pragma weak plat_slice_add
57 57 #pragma weak plat_slice_del
58 58
59 59 /*
60 60 * Adjust the memnode config after a DR operation.
↓ open down ↓ |
60 lines elided |
↑ open up ↑ |
61 61 *
62 62 * It is rather tricky to do these updates since we can't
63 63 * protect the memnode structures with locks, so we must
64 64 * be mindful of the order in which updates and reads to
65 65 * these values can occur.
66 66 */
67 67 void
68 68 mem_node_add_slice(pfn_t start, pfn_t end)
69 69 {
70 70 int mnode;
71 - mnodeset_t newmask, oldmask;
72 71
73 72 /*
74 73 * DR will pass us the first pfn that is allocatable.
75 74 * We need to round down to get the real start of
76 75 * the slice.
77 76 */
78 77 if (mem_node_physalign) {
79 78 start &= ~(btop(mem_node_physalign) - 1);
80 79 end = roundup(end, btop(mem_node_physalign)) - 1;
81 80 }
82 81
83 82 mnode = PFN_2_MEM_NODE(start);
84 83 ASSERT(mnode < max_mem_nodes);
85 84
86 85 if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
87 86 /*
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
88 87 * Add slice to existing node.
89 88 */
90 89 if (start < mem_node_config[mnode].physbase)
91 90 mem_node_config[mnode].physbase = start;
92 91 if (end > mem_node_config[mnode].physmax)
93 92 mem_node_config[mnode].physmax = end;
94 93 } else {
95 94 mem_node_config[mnode].physbase = start;
96 95 mem_node_config[mnode].physmax = end;
97 96 atomic_inc_16(&num_memnodes);
98 - do {
99 - oldmask = memnodes_mask;
100 - newmask = memnodes_mask | (1ull << mnode);
101 - } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) !=
102 - oldmask);
97 + atomic_or_64(&memnodes_mask, 1ull << mnode);
103 98 }
104 99 /*
105 100 * Let the common lgrp framework know about the new memory
106 101 */
107 102 lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
108 103 }
109 104
110 105 /*
111 106 * Remove a PFN range from a memnode. On some platforms,
112 107 * the memnode will be created with physbase at the first
113 108 * allocatable PFN, but later deleted with the MC slice
114 109 * base address converted to a PFN, in which case we need
115 110 * to assume physbase and up.
116 111 */
117 112 void
118 113 mem_node_del_slice(pfn_t start, pfn_t end)
119 114 {
120 115 int mnode;
121 116 pgcnt_t delta_pgcnt, node_size;
122 - mnodeset_t omask, nmask;
123 117
124 118 if (mem_node_physalign) {
125 119 start &= ~(btop(mem_node_physalign) - 1);
126 120 end = roundup(end, btop(mem_node_physalign)) - 1;
127 121 }
128 122 mnode = PFN_2_MEM_NODE(start);
129 123
130 124 ASSERT(mnode < max_mem_nodes);
131 125 ASSERT(mem_node_config[mnode].exists == 1);
132 126
133 127 delta_pgcnt = end - start;
134 128 node_size = mem_node_config[mnode].physmax -
135 129 mem_node_config[mnode].physbase;
136 130
137 131 if (node_size > delta_pgcnt) {
138 132 /*
139 133 * Subtract the slice from the memnode.
140 134 */
141 135 if (start <= mem_node_config[mnode].physbase)
142 136 mem_node_config[mnode].physbase = end + 1;
143 137 ASSERT(end <= mem_node_config[mnode].physmax);
144 138 if (end == mem_node_config[mnode].physmax)
145 139 mem_node_config[mnode].physmax = start - 1;
146 140 } else {
147 141
148 142 /*
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
149 143 * Let the common lgrp framework know the mnode is
150 144 * leaving
151 145 */
152 146 lgrp_config(LGRP_CONFIG_MEM_DEL, mnode,
153 147 MEM_NODE_2_LGRPHAND(mnode));
154 148
155 149 /*
156 150 * Delete the whole node.
157 151 */
158 152 ASSERT(MNODE_PGCNT(mnode) == 0);
159 - do {
160 - omask = memnodes_mask;
161 - nmask = omask & ~(1ull << mnode);
162 - } while (atomic_cas_64(&memnodes_mask, omask, nmask) != omask);
153 + atomic_and_64(&memnodes_mask, ~(1ull << mnode));
163 154 atomic_dec_16(&num_memnodes);
164 155 mem_node_config[mnode].exists = 0;
165 156 }
166 157 }
167 158
168 159 void
169 160 mem_node_add_range(pfn_t start, pfn_t end)
170 161 {
171 162 if (&plat_slice_add != NULL)
172 163 plat_slice_add(start, end);
173 164 else
174 165 mem_node_add_slice(start, end);
175 166 }
176 167
177 168 void
178 169 mem_node_del_range(pfn_t start, pfn_t end)
179 170 {
180 171 if (&plat_slice_del != NULL)
181 172 plat_slice_del(start, end);
182 173 else
183 174 mem_node_del_slice(start, end);
184 175 }
185 176
186 177 void
187 178 startup_build_mem_nodes(prom_memlist_t *list, size_t nelems)
188 179 {
189 180 size_t elem;
190 181 pfn_t basepfn;
191 182 pgcnt_t npgs;
192 183
193 184 /* LINTED: ASSERT will always true or false */
194 185 ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes);
195 186
196 187 if (&plat_build_mem_nodes != NULL) {
197 188 plat_build_mem_nodes(list, nelems);
198 189 } else {
199 190 /*
200 191 * Boot install lists are arranged <addr, len>, ...
201 192 */
202 193 for (elem = 0; elem < nelems; list++, elem++) {
203 194 basepfn = btop(list->addr);
204 195 npgs = btop(list->size);
205 196 mem_node_add_range(basepfn, basepfn + npgs - 1);
206 197 }
↓ open down ↓ |
34 lines elided |
↑ open up ↑ |
207 198 }
208 199 }
209 200
210 201 /*
211 202 * Allocate an unassigned memnode.
212 203 */
213 204 int
214 205 mem_node_alloc()
215 206 {
216 207 int mnode;
217 - mnodeset_t newmask, oldmask;
218 208
219 209 /*
220 210 * Find an unused memnode. Update it atomically to prevent
221 211 * a first time memnode creation race.
222 212 */
223 213 for (mnode = 0; mnode < max_mem_nodes; mnode++)
224 214 if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists,
225 215 0, 1) == 0)
226 216 break;
227 217
228 218 if (mnode >= max_mem_nodes)
229 219 panic("Out of free memnodes\n");
230 220
231 221 mem_node_config[mnode].physbase = (uint64_t)-1;
232 222 mem_node_config[mnode].physmax = 0;
233 223 atomic_inc_16(&num_memnodes);
234 - do {
235 - oldmask = memnodes_mask;
236 - newmask = memnodes_mask | (1ull << mnode);
237 - } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask);
224 + atomic_or_64(&memnodes_mask, 1ull << mnode);
238 225
239 226 return (mnode);
240 227 }
241 228
242 229 /*
243 230 * Find the intersection between a memnode and a memlist
244 231 * and returns the number of pages that overlap.
245 232 *
246 233 * Grab the memlist lock to protect the list from DR operations.
247 234 */
248 235 pgcnt_t
249 236 mem_node_memlist_pages(int mnode, struct memlist *mlist)
250 237 {
251 238 pfn_t base, end;
252 239 pfn_t cur_base, cur_end;
253 240 pgcnt_t npgs = 0;
254 241 pgcnt_t pages;
255 242 struct memlist *pmem;
256 243
257 244 if (&plat_mem_node_intersect_range != NULL) {
258 245 memlist_read_lock();
259 246
260 247 for (pmem = mlist; pmem; pmem = pmem->ml_next) {
261 248 plat_mem_node_intersect_range(btop(pmem->ml_address),
262 249 btop(pmem->ml_size), mnode, &pages);
263 250 npgs += pages;
264 251 }
265 252
266 253 memlist_read_unlock();
267 254 return (npgs);
268 255 }
269 256
270 257 base = mem_node_config[mnode].physbase;
271 258 end = mem_node_config[mnode].physmax;
272 259
273 260 memlist_read_lock();
274 261
275 262 for (pmem = mlist; pmem; pmem = pmem->ml_next) {
276 263 cur_base = btop(pmem->ml_address);
277 264 cur_end = cur_base + btop(pmem->ml_size) - 1;
278 265 if (end < cur_base || base > cur_end)
279 266 continue;
280 267 npgs = npgs + (MIN(cur_end, end) -
281 268 MAX(cur_base, base)) + 1;
282 269 }
283 270
284 271 memlist_read_unlock();
285 272
286 273 return (npgs);
287 274 }
288 275
289 276 /*
290 277 * Find MIN(physbase) and MAX(physmax) over all mnodes
291 278 *
292 279 * Called during startup and DR to find hpm_counters limits when
293 280 * interleaved_mnodes is set.
294 281 * NOTE: there is a race condition with DR if it tries to change more than
295 282 * one mnode in parallel. Sizing shared hpm_counters depends on finding the
296 283 * min(physbase) and max(physmax) across all mnodes. Therefore, the caller of
297 284 * page_ctrs_adjust must ensure that mem_node_config does not change while it
298 285 * is running.
299 286 */
300 287 void
301 288 mem_node_max_range(pfn_t *basep, pfn_t *maxp)
302 289 {
303 290 int mnode;
304 291 pfn_t max = 0;
305 292 pfn_t base = (pfn_t)-1;
306 293
307 294 for (mnode = 0; mnode < max_mem_nodes; mnode++) {
308 295 if (mem_node_config[mnode].exists == 0)
309 296 continue;
310 297 if (max < mem_node_config[mnode].physmax)
311 298 max = mem_node_config[mnode].physmax;
312 299 if (base > mem_node_config[mnode].physbase)
313 300 base = mem_node_config[mnode].physbase;
314 301 }
315 302 ASSERT(base != (pfn_t)-1 && max != 0);
316 303 *basep = base;
317 304 *maxp = max;
318 305 }
↓ open down ↓ |
71 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX