1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Kernel Physical Mapping (kpm) segment driver (segkpm). 29 * 30 * This driver delivers along with the hat_kpm* interfaces an alternative 31 * mechanism for kernel mappings within the 64-bit Solaris operating system, 32 * which allows the mapping of all physical memory into the kernel address 33 * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II 34 * and beyond processors, since the available VA range is much larger than 35 * possible physical memory. Momentarily all physical memory is supported, 36 * that is represented by the list of memory segments (memsegs). 37 * 38 * Segkpm mappings have also very low overhead and large pages are used 39 * (when possible) to minimize the TLB and TSB footprint. It is also 40 * extentable for other than Sparc architectures (e.g. AMD64). Main 41 * advantage is the avoidance of the TLB-shootdown X-calls, which are 42 * normally needed when a kernel (global) mapping has to be removed. 43 * 44 * First example of a kernel facility that uses the segkpm mapping scheme 45 * is seg_map, where it is used as an alternative to hat_memload(). 46 * See also hat layer for more information about the hat_kpm* routines. 47 * The kpm facilty can be turned off at boot time (e.g. /etc/system). 48 */ 49 50 #include <sys/types.h> 51 #include <sys/param.h> 52 #include <sys/sysmacros.h> 53 #include <sys/systm.h> 54 #include <sys/vnode.h> 55 #include <sys/cmn_err.h> 56 #include <sys/debug.h> 57 #include <sys/thread.h> 58 #include <sys/cpuvar.h> 59 #include <sys/bitmap.h> 60 #include <sys/atomic.h> 61 #include <sys/lgrp.h> 62 63 #include <vm/seg_kmem.h> 64 #include <vm/seg_kpm.h> 65 #include <vm/hat.h> 66 #include <vm/as.h> 67 #include <vm/seg.h> 68 #include <vm/page.h> 69 70 /* 71 * Global kpm controls. 72 * See also platform and mmu specific controls. 73 * 74 * kpm_enable -- global on/off switch for segkpm. 75 * . Set by default on 64bit platforms that have kpm support. 76 * . Will be disabled from platform layer if not supported. 77 * . Can be disabled via /etc/system. 78 * 79 * kpm_smallpages -- use only regular/system pagesize for kpm mappings. 80 * . Can be useful for critical debugging of kpm clients. 81 * . Set to zero by default for platforms that support kpm large pages. 82 * The use of kpm large pages reduces the footprint of kpm meta data 83 * and has all the other advantages of using large pages (e.g TLB 84 * miss reduction). 85 * . Set by default for platforms that don't support kpm large pages or 86 * where large pages cannot be used for other reasons (e.g. there are 87 * only few full associative TLB entries available for large pages). 88 * 89 * segmap_kpm -- separate on/off switch for segmap using segkpm: 90 * . Set by default. 91 * . Will be disabled when kpm_enable is zero. 92 * . Will be disabled when MAXBSIZE != PAGESIZE. 93 * . Can be disabled via /etc/system. 94 * 95 */ 96 int kpm_enable = 1; 97 int kpm_smallpages = 0; 98 int segmap_kpm = 1; 99 100 /* 101 * Private seg op routines. 102 */ 103 faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, 104 size_t len, enum fault_type type, enum seg_rw rw); 105 static void segkpm_dump(struct seg *); 106 static int segkpm_pagelock(struct seg *seg, caddr_t addr, size_t len, 107 struct page ***page, enum lock_type type, 108 enum seg_rw rw); 109 static int segkpm_capable(struct seg *, segcapability_t); 110 111 static struct seg_ops segkpm_ops = { 112 .fault = segkpm_fault, 113 .dump = segkpm_dump, 114 .pagelock = segkpm_pagelock, 115 .capable = segkpm_capable, 116 //#ifndef SEGKPM_SUPPORT 117 #if 0 118 #error FIXME: define nop 119 .dup = nop, 120 .unmap = nop, 121 .free = nop, 122 .faulta = nop, 123 .setprot = nop, 124 .checkprot = nop, 125 .kluster = nop, 126 .sync = nop, 127 .incore = nop, 128 .lockop = nop, 129 .getprot = nop, 130 .getoffset = nop, 131 .gettype = nop, 132 .getvp = nop, 133 .advise = nop, 134 .setpagesize = nop, 135 .getmemid = nop, 136 .getpolicy = nop, 137 #endif 138 }; 139 140 /* 141 * kpm_pgsz and kpm_pgshft are set by platform layer. 142 */ 143 size_t kpm_pgsz; /* kpm page size */ 144 uint_t kpm_pgshft; /* kpm page shift */ 145 u_offset_t kpm_pgoff; /* kpm page offset mask */ 146 uint_t kpmp2pshft; /* kpm page to page shift */ 147 pgcnt_t kpmpnpgs; /* how many pages per kpm page */ 148 149 150 #ifdef SEGKPM_SUPPORT 151 152 int 153 segkpm_create(struct seg *seg, void *argsp) 154 { 155 struct segkpm_data *skd; 156 struct segkpm_crargs *b = (struct segkpm_crargs *)argsp; 157 ushort_t *p; 158 int i, j; 159 160 ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock)); 161 ASSERT(btokpmp(seg->s_size) >= 1 && 162 kpmpageoff((uintptr_t)seg->s_base) == 0 && 163 kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0); 164 165 skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP); 166 167 seg->s_data = (void *)skd; 168 seg->s_ops = &segkpm_ops; 169 skd->skd_prot = b->prot; 170 171 /* 172 * (1) Segkpm virtual addresses are based on physical adresses. 173 * From this and in opposite to other segment drivers it is 174 * often required to allocate a page first to be able to 175 * calculate the final segkpm virtual address. 176 * (2) Page allocation is done by calling page_create_va(), 177 * one important input argument is a virtual address (also 178 * expressed by the "va" in the function name). This function 179 * is highly optimized to select the right page for an optimal 180 * processor and platform support (e.g. virtual addressed 181 * caches (VAC), physical addressed caches, NUMA). 182 * 183 * Because of (1) the approach is to generate a faked virtual 184 * address for calling page_create_va(). In order to exploit 185 * the abilities of (2), especially to utilize the cache 186 * hierarchy (3) and to avoid VAC alias conflicts (4) the 187 * selection has to be done carefully. For each virtual color 188 * a separate counter is provided (4). The count values are 189 * used for the utilization of all cache lines (3) and are 190 * corresponding to the cache bins. 191 */ 192 skd->skd_nvcolors = b->nvcolors; 193 194 p = skd->skd_va_select = 195 kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP); 196 197 for (i = 0; i < NCPU; i++) 198 for (j = 0; j < b->nvcolors; j++, p++) 199 *p = j; 200 201 return (0); 202 } 203 204 /* 205 * This routine is called via a machine specific fault handling 206 * routine. 207 */ 208 /* ARGSUSED */ 209 faultcode_t 210 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len, 211 enum fault_type type, enum seg_rw rw) 212 { 213 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 214 215 switch (type) { 216 case F_INVAL: 217 return (hat_kpm_fault(hat, addr)); 218 case F_SOFTLOCK: 219 case F_SOFTUNLOCK: 220 return (0); 221 default: 222 return (FC_NOSUPPORT); 223 } 224 /*NOTREACHED*/ 225 } 226 227 #define addr_to_vcolor(addr, vcolors) \ 228 ((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT)) 229 230 /* 231 * Create a virtual address that can be used for invocations of 232 * page_create_va. Goal is to utilize the cache hierarchy (round 233 * robin bins) and to select the right color for virtual indexed 234 * caches. It isn't exact since we also increment the bin counter 235 * when the caller uses VOP_GETPAGE and gets a hit in the page 236 * cache, but we keep the bins turning for cache distribution 237 * (see also segkpm_create block comment). 238 */ 239 caddr_t 240 segkpm_create_va(u_offset_t off) 241 { 242 int vcolor; 243 ushort_t *p; 244 struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data; 245 int nvcolors = skd->skd_nvcolors; 246 caddr_t va; 247 248 vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0; 249 p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor]; 250 va = (caddr_t)ptob(*p); 251 252 atomic_add_16(p, nvcolors); 253 254 return (va); 255 } 256 257 /* 258 * Unload mapping if the instance has an active kpm mapping. 259 */ 260 void 261 segkpm_mapout_validkpme(struct kpme *kpme) 262 { 263 caddr_t vaddr; 264 page_t *pp; 265 266 retry: 267 if ((pp = kpme->kpe_page) == NULL) { 268 return; 269 } 270 271 if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0) 272 goto retry; 273 274 /* 275 * Check if segkpm mapping is not unloaded in the meantime 276 */ 277 if (kpme->kpe_page == NULL) { 278 page_unlock(pp); 279 return; 280 } 281 282 vaddr = hat_kpm_page2va(pp, 1); 283 hat_kpm_mapout(pp, kpme, vaddr); 284 page_unlock(pp); 285 } 286 287 #else /* SEGKPM_SUPPORT */ 288 289 /* segkpm stubs */ 290 291 /*ARGSUSED*/ 292 int segkpm_create(struct seg *seg, void *argsp) 293 { 294 return (0); 295 } 296 297 /* ARGSUSED */ 298 faultcode_t 299 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len, 300 enum fault_type type, enum seg_rw rw) 301 { 302 return (0); 303 } 304 305 /* ARGSUSED */ 306 caddr_t segkpm_create_va(u_offset_t off) 307 { 308 return (NULL); 309 } 310 311 /* ARGSUSED */ 312 void segkpm_mapout_validkpme(struct kpme *kpme) 313 { 314 } 315 316 #endif /* SEGKPM_SUPPORT */ 317 318 /* ARGSUSED */ 319 static int 320 segkpm_pagelock(struct seg *seg, caddr_t addr, size_t len, 321 struct page ***page, enum lock_type type, enum seg_rw rw) 322 { 323 return (ENOTSUP); 324 } 325 326 /* 327 * segkpm pages are not dumped, so we just return 328 */ 329 /*ARGSUSED*/ 330 static void 331 segkpm_dump(struct seg *seg) 332 { 333 } 334 335 /* 336 * We claim to have no special capabilities. 337 */ 338 /*ARGSUSED*/ 339 static int 340 segkpm_capable(struct seg *seg, segcapability_t capability) 341 { 342 return (0); 343 }