1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 29 */ 30 31 #include <sys/atomic.h> 32 #include <sys/errno.h> 33 #include <sys/stat.h> 34 #include <sys/modctl.h> 35 #include <sys/conf.h> 36 #include <sys/systm.h> 37 #include <sys/ddi.h> 38 #include <sys/sunddi.h> 39 #include <sys/cpuvar.h> 40 #include <sys/kmem.h> 41 #include <sys/strsubr.h> 42 #include <sys/fasttrap.h> 43 #include <sys/fasttrap_impl.h> 44 #include <sys/fasttrap_isa.h> 45 #include <sys/dtrace.h> 46 #include <sys/dtrace_impl.h> 47 #include <sys/sysmacros.h> 48 #include <sys/proc.h> 49 #include <sys/priv.h> 50 #include <sys/policy.h> 51 #include <util/qsort.h> 52 53 /* 54 * User-Land Trap-Based Tracing 55 * ---------------------------- 56 * 57 * The fasttrap provider allows DTrace consumers to instrument any user-level 58 * instruction to gather data; this includes probes with semantic 59 * signifigance like entry and return as well as simple offsets into the 60 * function. While the specific techniques used are very ISA specific, the 61 * methodology is generalizable to any architecture. 62 * 63 * 64 * The General Methodology 65 * ----------------------- 66 * 67 * With the primary goal of tracing every user-land instruction and the 68 * limitation that we can't trust user space so don't want to rely on much 69 * information there, we begin by replacing the instructions we want to trace 70 * with trap instructions. Each instruction we overwrite is saved into a hash 71 * table keyed by process ID and pc address. When we enter the kernel due to 72 * this trap instruction, we need the effects of the replaced instruction to 73 * appear to have occurred before we proceed with the user thread's 74 * execution. 75 * 76 * Each user level thread is represented by a ulwp_t structure which is 77 * always easily accessible through a register. The most basic way to produce 78 * the effects of the instruction we replaced is to copy that instruction out 79 * to a bit of scratch space reserved in the user thread's ulwp_t structure 80 * (a sort of kernel-private thread local storage), set the PC to that 81 * scratch space and single step. When we reenter the kernel after single 82 * stepping the instruction we must then adjust the PC to point to what would 83 * normally be the next instruction. Of course, special care must be taken 84 * for branches and jumps, but these represent such a small fraction of any 85 * instruction set that writing the code to emulate these in the kernel is 86 * not too difficult. 87 * 88 * Return probes may require several tracepoints to trace every return site, 89 * and, conversely, each tracepoint may activate several probes (the entry 90 * and offset 0 probes, for example). To solve this muliplexing problem, 91 * tracepoints contain lists of probes to activate and probes contain lists 92 * of tracepoints to enable. If a probe is activated, it adds its ID to 93 * existing tracepoints or creates new ones as necessary. 94 * 95 * Most probes are activated _before_ the instruction is executed, but return 96 * probes are activated _after_ the effects of the last instruction of the 97 * function are visible. Return probes must be fired _after_ we have 98 * single-stepped the instruction whereas all other probes are fired 99 * beforehand. 100 * 101 * 102 * Lock Ordering 103 * ------------- 104 * 105 * The lock ordering below -- both internally and with respect to the DTrace 106 * framework -- is a little tricky and bears some explanation. Each provider 107 * has a lock (ftp_mtx) that protects its members including reference counts 108 * for enabled probes (ftp_rcount), consumers actively creating probes 109 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 110 * from being freed. A provider is looked up by taking the bucket lock for the 111 * provider hash table, and is returned with its lock held. The provider lock 112 * may be taken in functions invoked by the DTrace framework, but may not be 113 * held while calling functions in the DTrace framework. 114 * 115 * To ensure consistency over multiple calls to the DTrace framework, the 116 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 117 * not be taken when holding the provider lock as that would create a cyclic 118 * lock ordering. In situations where one would naturally take the provider 119 * lock and then the creation lock, we instead up a reference count to prevent 120 * the provider from disappearing, drop the provider lock, and acquire the 121 * creation lock. 122 * 123 * Briefly: 124 * bucket lock before provider lock 125 * DTrace before provider lock 126 * creation lock before DTrace 127 * never hold the provider lock and creation lock simultaneously 128 */ 129 130 static dev_info_t *fasttrap_devi; 131 static dtrace_meta_provider_id_t fasttrap_meta_id; 132 133 static timeout_id_t fasttrap_timeout; 134 static kmutex_t fasttrap_cleanup_mtx; 135 static uint_t fasttrap_cleanup_work; 136 137 /* 138 * Generation count on modifications to the global tracepoint lookup table. 139 */ 140 static volatile uint64_t fasttrap_mod_gen; 141 142 /* 143 * When the fasttrap provider is loaded, fasttrap_max is set to either 144 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 145 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 146 * incremented by the number of tracepoints that may be associated with that 147 * probe; fasttrap_total is capped at fasttrap_max. 148 */ 149 #define FASTTRAP_MAX_DEFAULT 250000 150 static uint32_t fasttrap_max; 151 static uint32_t fasttrap_total; 152 153 154 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 155 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 156 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 157 158 #define FASTTRAP_PID_NAME "pid" 159 160 fasttrap_hash_t fasttrap_tpoints; 161 static fasttrap_hash_t fasttrap_provs; 162 static fasttrap_hash_t fasttrap_procs; 163 164 static uint64_t fasttrap_pid_count; /* pid ref count */ 165 static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 166 167 #define FASTTRAP_ENABLE_FAIL 1 168 #define FASTTRAP_ENABLE_PARTIAL 2 169 170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 172 173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 174 const dtrace_pattr_t *); 175 static void fasttrap_provider_retire(pid_t, const char *, int); 176 static void fasttrap_provider_free(fasttrap_provider_t *); 177 178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 179 static void fasttrap_proc_release(fasttrap_proc_t *); 180 181 #define FASTTRAP_PROVS_INDEX(pid, name) \ 182 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 183 184 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 185 186 static int 187 fasttrap_highbit(ulong_t i) 188 { 189 int h = 1; 190 191 if (i == 0) 192 return (0); 193 #ifdef _LP64 194 if (i & 0xffffffff00000000ul) { 195 h += 32; i >>= 32; 196 } 197 #endif 198 if (i & 0xffff0000) { 199 h += 16; i >>= 16; 200 } 201 if (i & 0xff00) { 202 h += 8; i >>= 8; 203 } 204 if (i & 0xf0) { 205 h += 4; i >>= 4; 206 } 207 if (i & 0xc) { 208 h += 2; i >>= 2; 209 } 210 if (i & 0x2) { 211 h += 1; 212 } 213 return (h); 214 } 215 216 static uint_t 217 fasttrap_hash_str(const char *p) 218 { 219 unsigned int g; 220 uint_t hval = 0; 221 222 while (*p) { 223 hval = (hval << 4) + *p++; 224 if ((g = (hval & 0xf0000000)) != 0) 225 hval ^= g >> 24; 226 hval &= ~g; 227 } 228 return (hval); 229 } 230 231 void 232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 233 { 234 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 235 236 sqp->sq_info.si_signo = SIGTRAP; 237 sqp->sq_info.si_code = TRAP_DTRACE; 238 sqp->sq_info.si_addr = (caddr_t)pc; 239 240 mutex_enter(&p->p_lock); 241 sigaddqa(p, t, sqp); 242 mutex_exit(&p->p_lock); 243 244 if (t != NULL) 245 aston(t); 246 } 247 248 /* 249 * This function ensures that no threads are actively using the memory 250 * associated with probes that were formerly live. 251 */ 252 static void 253 fasttrap_mod_barrier(uint64_t gen) 254 { 255 int i; 256 257 if (gen < fasttrap_mod_gen) 258 return; 259 260 fasttrap_mod_gen++; 261 262 for (i = 0; i < NCPU; i++) { 263 mutex_enter(&cpu_core[i].cpuc_pid_lock); 264 mutex_exit(&cpu_core[i].cpuc_pid_lock); 265 } 266 } 267 268 /* 269 * This is the timeout's callback for cleaning up the providers and their 270 * probes. 271 */ 272 /*ARGSUSED*/ 273 static void 274 fasttrap_pid_cleanup_cb(void *data) 275 { 276 fasttrap_provider_t **fpp, *fp; 277 fasttrap_bucket_t *bucket; 278 dtrace_provider_id_t provid; 279 int i, later, rval; 280 281 static volatile int in = 0; 282 ASSERT(in == 0); 283 in = 1; 284 285 mutex_enter(&fasttrap_cleanup_mtx); 286 while (fasttrap_cleanup_work) { 287 fasttrap_cleanup_work = 0; 288 mutex_exit(&fasttrap_cleanup_mtx); 289 290 later = 0; 291 292 /* 293 * Iterate over all the providers trying to remove the marked 294 * ones. If a provider is marked but not retired, we just 295 * have to take a crack at removing it -- it's no big deal if 296 * we can't. 297 */ 298 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 299 bucket = &fasttrap_provs.fth_table[i]; 300 mutex_enter(&bucket->ftb_mtx); 301 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 302 303 while ((fp = *fpp) != NULL) { 304 if (!fp->ftp_marked) { 305 fpp = &fp->ftp_next; 306 continue; 307 } 308 309 mutex_enter(&fp->ftp_mtx); 310 311 /* 312 * If this provider has consumers actively 313 * creating probes (ftp_ccount) or is a USDT 314 * provider (ftp_mcount), we can't unregister 315 * or even condense. 316 */ 317 if (fp->ftp_ccount != 0 || 318 fp->ftp_mcount != 0) { 319 mutex_exit(&fp->ftp_mtx); 320 fp->ftp_marked = 0; 321 continue; 322 } 323 324 if (!fp->ftp_retired || fp->ftp_rcount != 0) 325 fp->ftp_marked = 0; 326 327 mutex_exit(&fp->ftp_mtx); 328 329 /* 330 * If we successfully unregister this 331 * provider we can remove it from the hash 332 * chain and free the memory. If our attempt 333 * to unregister fails and this is a retired 334 * provider, increment our flag to try again 335 * pretty soon. If we've consumed more than 336 * half of our total permitted number of 337 * probes call dtrace_condense() to try to 338 * clean out the unenabled probes. 339 */ 340 provid = fp->ftp_provid; 341 if ((rval = dtrace_unregister(provid)) != 0) { 342 if (fasttrap_total > fasttrap_max / 2) 343 (void) dtrace_condense(provid); 344 345 if (rval == EAGAIN) 346 fp->ftp_marked = 1; 347 348 later += fp->ftp_marked; 349 fpp = &fp->ftp_next; 350 } else { 351 *fpp = fp->ftp_next; 352 fasttrap_provider_free(fp); 353 } 354 } 355 mutex_exit(&bucket->ftb_mtx); 356 } 357 358 mutex_enter(&fasttrap_cleanup_mtx); 359 } 360 361 ASSERT(fasttrap_timeout != 0); 362 363 /* 364 * If we were unable to remove a retired provider, try again after 365 * a second. This situation can occur in certain circumstances where 366 * providers cannot be unregistered even though they have no probes 367 * enabled because of an execution of dtrace -l or something similar. 368 * If the timeout has been disabled (set to 1 because we're trying 369 * to detach), we set fasttrap_cleanup_work to ensure that we'll 370 * get a chance to do that work if and when the timeout is reenabled 371 * (if detach fails). 372 */ 373 if (later > 0) { 374 if (fasttrap_timeout != (timeout_id_t)1) { 375 fasttrap_timeout = 376 timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 377 } 378 379 fasttrap_cleanup_work = 1; 380 } else { 381 fasttrap_timeout = 0; 382 } 383 384 mutex_exit(&fasttrap_cleanup_mtx); 385 in = 0; 386 } 387 388 /* 389 * Activates the asynchronous cleanup mechanism. 390 */ 391 static void 392 fasttrap_pid_cleanup(void) 393 { 394 mutex_enter(&fasttrap_cleanup_mtx); 395 fasttrap_cleanup_work = 1; 396 if (fasttrap_timeout == 0) 397 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 398 mutex_exit(&fasttrap_cleanup_mtx); 399 } 400 401 /* 402 * This is called from cfork() via dtrace_fasttrap_fork(). The child 403 * process's address space is (roughly) a copy of the parent process's so 404 * we have to remove all the instrumentation we had previously enabled in the 405 * parent. 406 */ 407 static void 408 fasttrap_fork(proc_t *p, proc_t *cp) 409 { 410 pid_t ppid = p->p_pid; 411 int i; 412 413 ASSERT(curproc == p); 414 ASSERT(p->p_proc_flag & P_PR_LOCK); 415 ASSERT(p->p_dtrace_count > 0); 416 ASSERT(cp->p_dtrace_count == 0); 417 418 /* 419 * This would be simpler and faster if we maintained per-process 420 * hash tables of enabled tracepoints. It could, however, potentially 421 * slow down execution of a tracepoint since we'd need to go 422 * through two levels of indirection. In the future, we should 423 * consider either maintaining per-process ancillary lists of 424 * enabled tracepoints or hanging a pointer to a per-process hash 425 * table of enabled tracepoints off the proc structure. 426 */ 427 428 /* 429 * We don't have to worry about the child process disappearing 430 * because we're in fork(). 431 */ 432 mutex_enter(&cp->p_lock); 433 sprlock_proc(cp); 434 mutex_exit(&cp->p_lock); 435 436 /* 437 * Iterate over every tracepoint looking for ones that belong to the 438 * parent process, and remove each from the child process. 439 */ 440 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 441 fasttrap_tracepoint_t *tp; 442 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 443 444 mutex_enter(&bucket->ftb_mtx); 445 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 446 if (tp->ftt_pid == ppid && 447 tp->ftt_proc->ftpc_acount != 0) { 448 int ret = fasttrap_tracepoint_remove(cp, tp); 449 ASSERT(ret == 0); 450 451 /* 452 * The count of active providers can only be 453 * decremented (i.e. to zero) during exec, 454 * exit, and removal of a meta provider so it 455 * should be impossible to drop the count 456 * mid-fork. 457 */ 458 ASSERT(tp->ftt_proc->ftpc_acount != 0); 459 } 460 } 461 mutex_exit(&bucket->ftb_mtx); 462 } 463 464 mutex_enter(&cp->p_lock); 465 sprunlock(cp); 466 } 467 468 /* 469 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 470 * is set on the proc structure to indicate that there is a pid provider 471 * associated with this process. 472 */ 473 static void 474 fasttrap_exec_exit(proc_t *p) 475 { 476 ASSERT(p == curproc); 477 ASSERT(MUTEX_HELD(&p->p_lock)); 478 479 mutex_exit(&p->p_lock); 480 481 /* 482 * We clean up the pid provider for this process here; user-land 483 * static probes are handled by the meta-provider remove entry point. 484 */ 485 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 486 487 mutex_enter(&p->p_lock); 488 } 489 490 491 /*ARGSUSED*/ 492 static void 493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc) 494 { 495 /* 496 * There are no "default" pid probes. 497 */ 498 } 499 500 static int 501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 502 { 503 fasttrap_tracepoint_t *tp, *new_tp = NULL; 504 fasttrap_bucket_t *bucket; 505 fasttrap_id_t *id; 506 pid_t pid; 507 uintptr_t pc; 508 509 ASSERT(index < probe->ftp_ntps); 510 511 pid = probe->ftp_pid; 512 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 513 id = &probe->ftp_tps[index].fit_id; 514 515 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 516 517 ASSERT(!(p->p_flag & SVFORK)); 518 519 /* 520 * Before we make any modifications, make sure we've imposed a barrier 521 * on the generation in which this probe was last modified. 522 */ 523 fasttrap_mod_barrier(probe->ftp_gen); 524 525 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 526 527 /* 528 * If the tracepoint has already been enabled, just add our id to the 529 * list of interested probes. This may be our second time through 530 * this path in which case we'll have constructed the tracepoint we'd 531 * like to install. If we can't find a match, and have an allocated 532 * tracepoint ready to go, enable that one now. 533 * 534 * A tracepoint whose process is defunct is also considered defunct. 535 */ 536 again: 537 mutex_enter(&bucket->ftb_mtx); 538 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 539 /* 540 * Note that it's safe to access the active count on the 541 * associated proc structure because we know that at least one 542 * provider (this one) will still be around throughout this 543 * operation. 544 */ 545 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 546 tp->ftt_proc->ftpc_acount == 0) 547 continue; 548 549 /* 550 * Now that we've found a matching tracepoint, it would be 551 * a decent idea to confirm that the tracepoint is still 552 * enabled and the trap instruction hasn't been overwritten. 553 * Since this is a little hairy, we'll punt for now. 554 */ 555 556 /* 557 * This can't be the first interested probe. We don't have 558 * to worry about another thread being in the midst of 559 * deleting this tracepoint (which would be the only valid 560 * reason for a tracepoint to have no interested probes) 561 * since we're holding P_PR_LOCK for this process. 562 */ 563 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 564 565 switch (id->fti_ptype) { 566 case DTFTP_ENTRY: 567 case DTFTP_OFFSETS: 568 case DTFTP_IS_ENABLED: 569 id->fti_next = tp->ftt_ids; 570 membar_producer(); 571 tp->ftt_ids = id; 572 membar_producer(); 573 break; 574 575 case DTFTP_RETURN: 576 case DTFTP_POST_OFFSETS: 577 id->fti_next = tp->ftt_retids; 578 membar_producer(); 579 tp->ftt_retids = id; 580 membar_producer(); 581 break; 582 583 default: 584 ASSERT(0); 585 } 586 587 mutex_exit(&bucket->ftb_mtx); 588 589 if (new_tp != NULL) { 590 new_tp->ftt_ids = NULL; 591 new_tp->ftt_retids = NULL; 592 } 593 594 return (0); 595 } 596 597 /* 598 * If we have a good tracepoint ready to go, install it now while 599 * we have the lock held and no one can screw with us. 600 */ 601 if (new_tp != NULL) { 602 int rc = 0; 603 604 new_tp->ftt_next = bucket->ftb_data; 605 membar_producer(); 606 bucket->ftb_data = new_tp; 607 membar_producer(); 608 mutex_exit(&bucket->ftb_mtx); 609 610 /* 611 * Activate the tracepoint in the ISA-specific manner. 612 * If this fails, we need to report the failure, but 613 * indicate that this tracepoint must still be disabled 614 * by calling fasttrap_tracepoint_disable(). 615 */ 616 if (fasttrap_tracepoint_install(p, new_tp) != 0) 617 rc = FASTTRAP_ENABLE_PARTIAL; 618 619 /* 620 * Increment the count of the number of tracepoints active in 621 * the victim process. 622 */ 623 ASSERT(p->p_proc_flag & P_PR_LOCK); 624 p->p_dtrace_count++; 625 626 return (rc); 627 } 628 629 mutex_exit(&bucket->ftb_mtx); 630 631 /* 632 * Initialize the tracepoint that's been preallocated with the probe. 633 */ 634 new_tp = probe->ftp_tps[index].fit_tp; 635 636 ASSERT(new_tp->ftt_pid == pid); 637 ASSERT(new_tp->ftt_pc == pc); 638 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 639 ASSERT(new_tp->ftt_ids == NULL); 640 ASSERT(new_tp->ftt_retids == NULL); 641 642 switch (id->fti_ptype) { 643 case DTFTP_ENTRY: 644 case DTFTP_OFFSETS: 645 case DTFTP_IS_ENABLED: 646 id->fti_next = NULL; 647 new_tp->ftt_ids = id; 648 break; 649 650 case DTFTP_RETURN: 651 case DTFTP_POST_OFFSETS: 652 id->fti_next = NULL; 653 new_tp->ftt_retids = id; 654 break; 655 656 default: 657 ASSERT(0); 658 } 659 660 /* 661 * If the ISA-dependent initialization goes to plan, go back to the 662 * beginning and try to install this freshly made tracepoint. 663 */ 664 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 665 goto again; 666 667 new_tp->ftt_ids = NULL; 668 new_tp->ftt_retids = NULL; 669 670 return (FASTTRAP_ENABLE_FAIL); 671 } 672 673 static void 674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 675 { 676 fasttrap_bucket_t *bucket; 677 fasttrap_provider_t *provider = probe->ftp_prov; 678 fasttrap_tracepoint_t **pp, *tp; 679 fasttrap_id_t *id, **idp; 680 pid_t pid; 681 uintptr_t pc; 682 683 ASSERT(index < probe->ftp_ntps); 684 685 pid = probe->ftp_pid; 686 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 687 id = &probe->ftp_tps[index].fit_id; 688 689 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 690 691 /* 692 * Find the tracepoint and make sure that our id is one of the 693 * ones registered with it. 694 */ 695 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 696 mutex_enter(&bucket->ftb_mtx); 697 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 698 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 699 tp->ftt_proc == provider->ftp_proc) 700 break; 701 } 702 703 /* 704 * If we somehow lost this tracepoint, we're in a world of hurt. 705 */ 706 ASSERT(tp != NULL); 707 708 switch (id->fti_ptype) { 709 case DTFTP_ENTRY: 710 case DTFTP_OFFSETS: 711 case DTFTP_IS_ENABLED: 712 ASSERT(tp->ftt_ids != NULL); 713 idp = &tp->ftt_ids; 714 break; 715 716 case DTFTP_RETURN: 717 case DTFTP_POST_OFFSETS: 718 ASSERT(tp->ftt_retids != NULL); 719 idp = &tp->ftt_retids; 720 break; 721 722 default: 723 ASSERT(0); 724 } 725 726 while ((*idp)->fti_probe != probe) { 727 idp = &(*idp)->fti_next; 728 ASSERT(*idp != NULL); 729 } 730 731 id = *idp; 732 *idp = id->fti_next; 733 membar_producer(); 734 735 ASSERT(id->fti_probe == probe); 736 737 /* 738 * If there are other registered enablings of this tracepoint, we're 739 * all done, but if this was the last probe assocated with this 740 * this tracepoint, we need to remove and free it. 741 */ 742 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 743 744 /* 745 * If the current probe's tracepoint is in use, swap it 746 * for an unused tracepoint. 747 */ 748 if (tp == probe->ftp_tps[index].fit_tp) { 749 fasttrap_probe_t *tmp_probe; 750 fasttrap_tracepoint_t **tmp_tp; 751 uint_t tmp_index; 752 753 if (tp->ftt_ids != NULL) { 754 tmp_probe = tp->ftt_ids->fti_probe; 755 /* LINTED - alignment */ 756 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 757 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 758 } else { 759 tmp_probe = tp->ftt_retids->fti_probe; 760 /* LINTED - alignment */ 761 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 762 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 763 } 764 765 ASSERT(*tmp_tp != NULL); 766 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 767 ASSERT((*tmp_tp)->ftt_ids == NULL); 768 ASSERT((*tmp_tp)->ftt_retids == NULL); 769 770 probe->ftp_tps[index].fit_tp = *tmp_tp; 771 *tmp_tp = tp; 772 } 773 774 mutex_exit(&bucket->ftb_mtx); 775 776 /* 777 * Tag the modified probe with the generation in which it was 778 * changed. 779 */ 780 probe->ftp_gen = fasttrap_mod_gen; 781 return; 782 } 783 784 mutex_exit(&bucket->ftb_mtx); 785 786 /* 787 * We can't safely remove the tracepoint from the set of active 788 * tracepoints until we've actually removed the fasttrap instruction 789 * from the process's text. We can, however, operate on this 790 * tracepoint secure in the knowledge that no other thread is going to 791 * be looking at it since we hold P_PR_LOCK on the process if it's 792 * live or we hold the provider lock on the process if it's dead and 793 * gone. 794 */ 795 796 /* 797 * We only need to remove the actual instruction if we're looking 798 * at an existing process 799 */ 800 if (p != NULL) { 801 /* 802 * If we fail to restore the instruction we need to kill 803 * this process since it's in a completely unrecoverable 804 * state. 805 */ 806 if (fasttrap_tracepoint_remove(p, tp) != 0) 807 fasttrap_sigtrap(p, NULL, pc); 808 809 /* 810 * Decrement the count of the number of tracepoints active 811 * in the victim process. 812 */ 813 ASSERT(p->p_proc_flag & P_PR_LOCK); 814 p->p_dtrace_count--; 815 } 816 817 /* 818 * Remove the probe from the hash table of active tracepoints. 819 */ 820 mutex_enter(&bucket->ftb_mtx); 821 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 822 ASSERT(*pp != NULL); 823 while (*pp != tp) { 824 pp = &(*pp)->ftt_next; 825 ASSERT(*pp != NULL); 826 } 827 828 *pp = tp->ftt_next; 829 membar_producer(); 830 831 mutex_exit(&bucket->ftb_mtx); 832 833 /* 834 * Tag the modified probe with the generation in which it was changed. 835 */ 836 probe->ftp_gen = fasttrap_mod_gen; 837 } 838 839 static void 840 fasttrap_enable_callbacks(void) 841 { 842 /* 843 * We don't have to play the rw lock game here because we're 844 * providing something rather than taking something away -- 845 * we can be sure that no threads have tried to follow this 846 * function pointer yet. 847 */ 848 mutex_enter(&fasttrap_count_mtx); 849 if (fasttrap_pid_count == 0) { 850 ASSERT(dtrace_pid_probe_ptr == NULL); 851 ASSERT(dtrace_return_probe_ptr == NULL); 852 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 853 dtrace_return_probe_ptr = &fasttrap_return_probe; 854 } 855 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 856 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 857 fasttrap_pid_count++; 858 mutex_exit(&fasttrap_count_mtx); 859 } 860 861 static void 862 fasttrap_disable_callbacks(void) 863 { 864 ASSERT(MUTEX_HELD(&cpu_lock)); 865 866 mutex_enter(&fasttrap_count_mtx); 867 ASSERT(fasttrap_pid_count > 0); 868 fasttrap_pid_count--; 869 if (fasttrap_pid_count == 0) { 870 cpu_t *cur, *cpu = CPU; 871 872 for (cur = cpu->cpu_next_onln; cur != cpu; 873 cur = cur->cpu_next_onln) { 874 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 875 } 876 877 dtrace_pid_probe_ptr = NULL; 878 dtrace_return_probe_ptr = NULL; 879 880 for (cur = cpu->cpu_next_onln; cur != cpu; 881 cur = cur->cpu_next_onln) { 882 rw_exit(&cur->cpu_ft_lock); 883 } 884 } 885 mutex_exit(&fasttrap_count_mtx); 886 } 887 888 /*ARGSUSED*/ 889 static int 890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 891 { 892 fasttrap_probe_t *probe = parg; 893 proc_t *p; 894 int i, rc; 895 896 ASSERT(probe != NULL); 897 ASSERT(!probe->ftp_enabled); 898 ASSERT(id == probe->ftp_id); 899 ASSERT(MUTEX_HELD(&cpu_lock)); 900 901 /* 902 * Increment the count of enabled probes on this probe's provider; 903 * the provider can't go away while the probe still exists. We 904 * must increment this even if we aren't able to properly enable 905 * this probe. 906 */ 907 mutex_enter(&probe->ftp_prov->ftp_mtx); 908 probe->ftp_prov->ftp_rcount++; 909 mutex_exit(&probe->ftp_prov->ftp_mtx); 910 911 /* 912 * If this probe's provider is retired (meaning it was valid in a 913 * previously exec'ed incarnation of this address space), bail out. The 914 * provider can't go away while we're in this code path. 915 */ 916 if (probe->ftp_prov->ftp_retired) 917 return (0); 918 919 /* 920 * If we can't find the process, it may be that we're in the context of 921 * a fork in which the traced process is being born and we're copying 922 * USDT probes. Otherwise, the process is gone so bail. 923 */ 924 if ((p = sprlock(probe->ftp_pid)) == NULL) { 925 if ((curproc->p_flag & SFORKING) == 0) 926 return (0); 927 928 mutex_enter(&pidlock); 929 p = prfind(probe->ftp_pid); 930 931 /* 932 * Confirm that curproc is indeed forking the process in which 933 * we're trying to enable probes. 934 */ 935 ASSERT(p != NULL); 936 ASSERT(p->p_parent == curproc); 937 ASSERT(p->p_stat == SIDL); 938 939 mutex_enter(&p->p_lock); 940 mutex_exit(&pidlock); 941 942 sprlock_proc(p); 943 } 944 945 ASSERT(!(p->p_flag & SVFORK)); 946 mutex_exit(&p->p_lock); 947 948 /* 949 * We have to enable the trap entry point before any user threads have 950 * the chance to execute the trap instruction we're about to place 951 * in their process's text. 952 */ 953 fasttrap_enable_callbacks(); 954 955 /* 956 * Enable all the tracepoints and add this probe's id to each 957 * tracepoint's list of active probes. 958 */ 959 for (i = 0; i < probe->ftp_ntps; i++) { 960 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 961 /* 962 * If enabling the tracepoint failed completely, 963 * we don't have to disable it; if the failure 964 * was only partial we must disable it. 965 */ 966 if (rc == FASTTRAP_ENABLE_FAIL) 967 i--; 968 else 969 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 970 971 /* 972 * Back up and pull out all the tracepoints we've 973 * created so far for this probe. 974 */ 975 while (i >= 0) { 976 fasttrap_tracepoint_disable(p, probe, i); 977 i--; 978 } 979 980 mutex_enter(&p->p_lock); 981 sprunlock(p); 982 983 /* 984 * Since we're not actually enabling this probe, 985 * drop our reference on the trap table entry. 986 */ 987 fasttrap_disable_callbacks(); 988 return (0); 989 } 990 } 991 992 mutex_enter(&p->p_lock); 993 sprunlock(p); 994 995 probe->ftp_enabled = 1; 996 return (0); 997 } 998 999 /*ARGSUSED*/ 1000 static void 1001 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1002 { 1003 fasttrap_probe_t *probe = parg; 1004 fasttrap_provider_t *provider = probe->ftp_prov; 1005 proc_t *p; 1006 int i, whack = 0; 1007 1008 ASSERT(id == probe->ftp_id); 1009 1010 /* 1011 * We won't be able to acquire a /proc-esque lock on the process 1012 * iff the process is dead and gone. In this case, we rely on the 1013 * provider lock as a point of mutual exclusion to prevent other 1014 * DTrace consumers from disabling this probe. 1015 */ 1016 if ((p = sprlock(probe->ftp_pid)) != NULL) { 1017 ASSERT(!(p->p_flag & SVFORK)); 1018 mutex_exit(&p->p_lock); 1019 } 1020 1021 mutex_enter(&provider->ftp_mtx); 1022 1023 /* 1024 * Disable all the associated tracepoints (for fully enabled probes). 1025 */ 1026 if (probe->ftp_enabled) { 1027 for (i = 0; i < probe->ftp_ntps; i++) { 1028 fasttrap_tracepoint_disable(p, probe, i); 1029 } 1030 } 1031 1032 ASSERT(provider->ftp_rcount > 0); 1033 provider->ftp_rcount--; 1034 1035 if (p != NULL) { 1036 /* 1037 * Even though we may not be able to remove it entirely, we 1038 * mark this retired provider to get a chance to remove some 1039 * of the associated probes. 1040 */ 1041 if (provider->ftp_retired && !provider->ftp_marked) 1042 whack = provider->ftp_marked = 1; 1043 mutex_exit(&provider->ftp_mtx); 1044 1045 mutex_enter(&p->p_lock); 1046 sprunlock(p); 1047 } else { 1048 /* 1049 * If the process is dead, we're just waiting for the 1050 * last probe to be disabled to be able to free it. 1051 */ 1052 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1053 whack = provider->ftp_marked = 1; 1054 mutex_exit(&provider->ftp_mtx); 1055 } 1056 1057 if (whack) 1058 fasttrap_pid_cleanup(); 1059 1060 if (!probe->ftp_enabled) 1061 return; 1062 1063 probe->ftp_enabled = 0; 1064 1065 ASSERT(MUTEX_HELD(&cpu_lock)); 1066 fasttrap_disable_callbacks(); 1067 } 1068 1069 /*ARGSUSED*/ 1070 static void 1071 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1072 dtrace_argdesc_t *desc) 1073 { 1074 fasttrap_probe_t *probe = parg; 1075 char *str; 1076 int i, ndx; 1077 1078 desc->dtargd_native[0] = '\0'; 1079 desc->dtargd_xlate[0] = '\0'; 1080 1081 if (probe->ftp_prov->ftp_retired != 0 || 1082 desc->dtargd_ndx >= probe->ftp_nargs) { 1083 desc->dtargd_ndx = DTRACE_ARGNONE; 1084 return; 1085 } 1086 1087 ndx = (probe->ftp_argmap != NULL) ? 1088 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1089 1090 str = probe->ftp_ntypes; 1091 for (i = 0; i < ndx; i++) { 1092 str += strlen(str) + 1; 1093 } 1094 1095 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1096 (void) strcpy(desc->dtargd_native, str); 1097 1098 if (probe->ftp_xtypes == NULL) 1099 return; 1100 1101 str = probe->ftp_xtypes; 1102 for (i = 0; i < desc->dtargd_ndx; i++) { 1103 str += strlen(str) + 1; 1104 } 1105 1106 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1107 (void) strcpy(desc->dtargd_xlate, str); 1108 } 1109 1110 /*ARGSUSED*/ 1111 static void 1112 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1113 { 1114 fasttrap_probe_t *probe = parg; 1115 int i; 1116 size_t size; 1117 1118 ASSERT(probe != NULL); 1119 ASSERT(!probe->ftp_enabled); 1120 ASSERT(fasttrap_total >= probe->ftp_ntps); 1121 1122 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1123 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1124 1125 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1126 fasttrap_mod_barrier(probe->ftp_gen); 1127 1128 for (i = 0; i < probe->ftp_ntps; i++) { 1129 kmem_free(probe->ftp_tps[i].fit_tp, 1130 sizeof (fasttrap_tracepoint_t)); 1131 } 1132 1133 kmem_free(probe, size); 1134 } 1135 1136 1137 static const dtrace_pattr_t pid_attr = { 1138 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1139 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1140 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1141 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1142 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1143 }; 1144 1145 static dtrace_pops_t pid_pops = { 1146 fasttrap_pid_provide, 1147 NULL, 1148 fasttrap_pid_enable, 1149 fasttrap_pid_disable, 1150 NULL, 1151 NULL, 1152 fasttrap_pid_getargdesc, 1153 fasttrap_pid_getarg, 1154 NULL, 1155 fasttrap_pid_destroy 1156 }; 1157 1158 static dtrace_pops_t usdt_pops = { 1159 fasttrap_pid_provide, 1160 NULL, 1161 fasttrap_pid_enable, 1162 fasttrap_pid_disable, 1163 NULL, 1164 NULL, 1165 fasttrap_pid_getargdesc, 1166 fasttrap_usdt_getarg, 1167 NULL, 1168 fasttrap_pid_destroy 1169 }; 1170 1171 static fasttrap_proc_t * 1172 fasttrap_proc_lookup(pid_t pid) 1173 { 1174 fasttrap_bucket_t *bucket; 1175 fasttrap_proc_t *fprc, *new_fprc; 1176 1177 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1178 mutex_enter(&bucket->ftb_mtx); 1179 1180 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1181 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1182 mutex_enter(&fprc->ftpc_mtx); 1183 mutex_exit(&bucket->ftb_mtx); 1184 fprc->ftpc_rcount++; 1185 atomic_inc_64(&fprc->ftpc_acount); 1186 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1187 mutex_exit(&fprc->ftpc_mtx); 1188 1189 return (fprc); 1190 } 1191 } 1192 1193 /* 1194 * Drop the bucket lock so we don't try to perform a sleeping 1195 * allocation under it. 1196 */ 1197 mutex_exit(&bucket->ftb_mtx); 1198 1199 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1200 new_fprc->ftpc_pid = pid; 1201 new_fprc->ftpc_rcount = 1; 1202 new_fprc->ftpc_acount = 1; 1203 1204 mutex_enter(&bucket->ftb_mtx); 1205 1206 /* 1207 * Take another lap through the list to make sure a proc hasn't 1208 * been created for this pid while we weren't under the bucket lock. 1209 */ 1210 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1211 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1212 mutex_enter(&fprc->ftpc_mtx); 1213 mutex_exit(&bucket->ftb_mtx); 1214 fprc->ftpc_rcount++; 1215 atomic_inc_64(&fprc->ftpc_acount); 1216 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1217 mutex_exit(&fprc->ftpc_mtx); 1218 1219 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1220 1221 return (fprc); 1222 } 1223 } 1224 1225 new_fprc->ftpc_next = bucket->ftb_data; 1226 bucket->ftb_data = new_fprc; 1227 1228 mutex_exit(&bucket->ftb_mtx); 1229 1230 return (new_fprc); 1231 } 1232 1233 static void 1234 fasttrap_proc_release(fasttrap_proc_t *proc) 1235 { 1236 fasttrap_bucket_t *bucket; 1237 fasttrap_proc_t *fprc, **fprcp; 1238 pid_t pid = proc->ftpc_pid; 1239 1240 mutex_enter(&proc->ftpc_mtx); 1241 1242 ASSERT(proc->ftpc_rcount != 0); 1243 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1244 1245 if (--proc->ftpc_rcount != 0) { 1246 mutex_exit(&proc->ftpc_mtx); 1247 return; 1248 } 1249 1250 mutex_exit(&proc->ftpc_mtx); 1251 1252 /* 1253 * There should definitely be no live providers associated with this 1254 * process at this point. 1255 */ 1256 ASSERT(proc->ftpc_acount == 0); 1257 1258 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1259 mutex_enter(&bucket->ftb_mtx); 1260 1261 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1262 while ((fprc = *fprcp) != NULL) { 1263 if (fprc == proc) 1264 break; 1265 1266 fprcp = &fprc->ftpc_next; 1267 } 1268 1269 /* 1270 * Something strange has happened if we can't find the proc. 1271 */ 1272 ASSERT(fprc != NULL); 1273 1274 *fprcp = fprc->ftpc_next; 1275 1276 mutex_exit(&bucket->ftb_mtx); 1277 1278 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1279 } 1280 1281 /* 1282 * Lookup a fasttrap-managed provider based on its name and associated pid. 1283 * If the pattr argument is non-NULL, this function instantiates the provider 1284 * if it doesn't exist otherwise it returns NULL. The provider is returned 1285 * with its lock held. 1286 */ 1287 static fasttrap_provider_t * 1288 fasttrap_provider_lookup(pid_t pid, const char *name, 1289 const dtrace_pattr_t *pattr) 1290 { 1291 fasttrap_provider_t *fp, *new_fp = NULL; 1292 fasttrap_bucket_t *bucket; 1293 char provname[DTRACE_PROVNAMELEN]; 1294 proc_t *p; 1295 cred_t *cred; 1296 1297 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1298 ASSERT(pattr != NULL); 1299 1300 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1301 mutex_enter(&bucket->ftb_mtx); 1302 1303 /* 1304 * Take a lap through the list and return the match if we find it. 1305 */ 1306 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1307 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1308 !fp->ftp_retired) { 1309 mutex_enter(&fp->ftp_mtx); 1310 mutex_exit(&bucket->ftb_mtx); 1311 return (fp); 1312 } 1313 } 1314 1315 /* 1316 * Drop the bucket lock so we don't try to perform a sleeping 1317 * allocation under it. 1318 */ 1319 mutex_exit(&bucket->ftb_mtx); 1320 1321 /* 1322 * Make sure the process exists, isn't a child created as the result 1323 * of a vfork(2), and isn't a zombie (but may be in fork). 1324 */ 1325 mutex_enter(&pidlock); 1326 if ((p = prfind(pid)) == NULL) { 1327 mutex_exit(&pidlock); 1328 return (NULL); 1329 } 1330 mutex_enter(&p->p_lock); 1331 mutex_exit(&pidlock); 1332 if (p->p_flag & (SVFORK | SEXITING)) { 1333 mutex_exit(&p->p_lock); 1334 return (NULL); 1335 } 1336 1337 /* 1338 * Increment p_dtrace_probes so that the process knows to inform us 1339 * when it exits or execs. fasttrap_provider_free() decrements this 1340 * when we're done with this provider. 1341 */ 1342 p->p_dtrace_probes++; 1343 1344 /* 1345 * Grab the credentials for this process so we have 1346 * something to pass to dtrace_register(). 1347 */ 1348 mutex_enter(&p->p_crlock); 1349 crhold(p->p_cred); 1350 cred = p->p_cred; 1351 mutex_exit(&p->p_crlock); 1352 mutex_exit(&p->p_lock); 1353 1354 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1355 new_fp->ftp_pid = pid; 1356 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1357 1358 ASSERT(new_fp->ftp_proc != NULL); 1359 1360 mutex_enter(&bucket->ftb_mtx); 1361 1362 /* 1363 * Take another lap through the list to make sure a provider hasn't 1364 * been created for this pid while we weren't under the bucket lock. 1365 */ 1366 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1367 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1368 !fp->ftp_retired) { 1369 mutex_enter(&fp->ftp_mtx); 1370 mutex_exit(&bucket->ftb_mtx); 1371 fasttrap_provider_free(new_fp); 1372 crfree(cred); 1373 return (fp); 1374 } 1375 } 1376 1377 (void) strcpy(new_fp->ftp_name, name); 1378 1379 /* 1380 * Fail and return NULL if either the provider name is too long 1381 * or we fail to register this new provider with the DTrace 1382 * framework. Note that this is the only place we ever construct 1383 * the full provider name -- we keep it in pieces in the provider 1384 * structure. 1385 */ 1386 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1387 sizeof (provname) || 1388 dtrace_register(provname, pattr, 1389 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1390 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1391 &new_fp->ftp_provid) != 0) { 1392 mutex_exit(&bucket->ftb_mtx); 1393 fasttrap_provider_free(new_fp); 1394 crfree(cred); 1395 return (NULL); 1396 } 1397 1398 new_fp->ftp_next = bucket->ftb_data; 1399 bucket->ftb_data = new_fp; 1400 1401 mutex_enter(&new_fp->ftp_mtx); 1402 mutex_exit(&bucket->ftb_mtx); 1403 1404 crfree(cred); 1405 return (new_fp); 1406 } 1407 1408 static void 1409 fasttrap_provider_free(fasttrap_provider_t *provider) 1410 { 1411 pid_t pid = provider->ftp_pid; 1412 proc_t *p; 1413 1414 /* 1415 * There need to be no associated enabled probes, no consumers 1416 * creating probes, and no meta providers referencing this provider. 1417 */ 1418 ASSERT(provider->ftp_rcount == 0); 1419 ASSERT(provider->ftp_ccount == 0); 1420 ASSERT(provider->ftp_mcount == 0); 1421 1422 /* 1423 * If this provider hasn't been retired, we need to explicitly drop the 1424 * count of active providers on the associated process structure. 1425 */ 1426 if (!provider->ftp_retired) { 1427 atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1428 ASSERT(provider->ftp_proc->ftpc_acount < 1429 provider->ftp_proc->ftpc_rcount); 1430 } 1431 1432 fasttrap_proc_release(provider->ftp_proc); 1433 1434 kmem_free(provider, sizeof (fasttrap_provider_t)); 1435 1436 /* 1437 * Decrement p_dtrace_probes on the process whose provider we're 1438 * freeing. We don't have to worry about clobbering somone else's 1439 * modifications to it because we have locked the bucket that 1440 * corresponds to this process's hash chain in the provider hash 1441 * table. Don't sweat it if we can't find the process. 1442 */ 1443 mutex_enter(&pidlock); 1444 if ((p = prfind(pid)) == NULL) { 1445 mutex_exit(&pidlock); 1446 return; 1447 } 1448 1449 mutex_enter(&p->p_lock); 1450 mutex_exit(&pidlock); 1451 1452 p->p_dtrace_probes--; 1453 mutex_exit(&p->p_lock); 1454 } 1455 1456 static void 1457 fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1458 { 1459 fasttrap_provider_t *fp; 1460 fasttrap_bucket_t *bucket; 1461 dtrace_provider_id_t provid; 1462 1463 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1464 1465 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1466 mutex_enter(&bucket->ftb_mtx); 1467 1468 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1469 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1470 !fp->ftp_retired) 1471 break; 1472 } 1473 1474 if (fp == NULL) { 1475 mutex_exit(&bucket->ftb_mtx); 1476 return; 1477 } 1478 1479 mutex_enter(&fp->ftp_mtx); 1480 ASSERT(!mprov || fp->ftp_mcount > 0); 1481 if (mprov && --fp->ftp_mcount != 0) { 1482 mutex_exit(&fp->ftp_mtx); 1483 mutex_exit(&bucket->ftb_mtx); 1484 return; 1485 } 1486 1487 /* 1488 * Mark the provider to be removed in our post-processing step, mark it 1489 * retired, and drop the active count on its proc. Marking it indicates 1490 * that we should try to remove it; setting the retired flag indicates 1491 * that we're done with this provider; dropping the active the proc 1492 * releases our hold, and when this reaches zero (as it will during 1493 * exit or exec) the proc and associated providers become defunct. 1494 * 1495 * We obviously need to take the bucket lock before the provider lock 1496 * to perform the lookup, but we need to drop the provider lock 1497 * before calling into the DTrace framework since we acquire the 1498 * provider lock in callbacks invoked from the DTrace framework. The 1499 * bucket lock therefore protects the integrity of the provider hash 1500 * table. 1501 */ 1502 atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1503 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1504 1505 fp->ftp_retired = 1; 1506 fp->ftp_marked = 1; 1507 provid = fp->ftp_provid; 1508 mutex_exit(&fp->ftp_mtx); 1509 1510 /* 1511 * We don't have to worry about invalidating the same provider twice 1512 * since fasttrap_provider_lookup() will ignore provider that have 1513 * been marked as retired. 1514 */ 1515 dtrace_invalidate(provid); 1516 1517 mutex_exit(&bucket->ftb_mtx); 1518 1519 fasttrap_pid_cleanup(); 1520 } 1521 1522 static int 1523 fasttrap_uint32_cmp(const void *ap, const void *bp) 1524 { 1525 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1526 } 1527 1528 static int 1529 fasttrap_uint64_cmp(const void *ap, const void *bp) 1530 { 1531 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1532 } 1533 1534 static int 1535 fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1536 { 1537 fasttrap_provider_t *provider; 1538 fasttrap_probe_t *pp; 1539 fasttrap_tracepoint_t *tp; 1540 char *name; 1541 int i, aframes, whack; 1542 1543 /* 1544 * There needs to be at least one desired trace point. 1545 */ 1546 if (pdata->ftps_noffs == 0) 1547 return (EINVAL); 1548 1549 switch (pdata->ftps_type) { 1550 case DTFTP_ENTRY: 1551 name = "entry"; 1552 aframes = FASTTRAP_ENTRY_AFRAMES; 1553 break; 1554 case DTFTP_RETURN: 1555 name = "return"; 1556 aframes = FASTTRAP_RETURN_AFRAMES; 1557 break; 1558 case DTFTP_OFFSETS: 1559 name = NULL; 1560 break; 1561 default: 1562 return (EINVAL); 1563 } 1564 1565 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1566 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1567 return (ESRCH); 1568 1569 /* 1570 * Increment this reference count to indicate that a consumer is 1571 * actively adding a new probe associated with this provider. This 1572 * prevents the provider from being deleted -- we'll need to check 1573 * for pending deletions when we drop this reference count. 1574 */ 1575 provider->ftp_ccount++; 1576 mutex_exit(&provider->ftp_mtx); 1577 1578 /* 1579 * Grab the creation lock to ensure consistency between calls to 1580 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1581 * other threads creating probes. We must drop the provider lock 1582 * before taking this lock to avoid a three-way deadlock with the 1583 * DTrace framework. 1584 */ 1585 mutex_enter(&provider->ftp_cmtx); 1586 1587 if (name == NULL) { 1588 for (i = 0; i < pdata->ftps_noffs; i++) { 1589 char name_str[17]; 1590 1591 (void) sprintf(name_str, "%llx", 1592 (unsigned long long)pdata->ftps_offs[i]); 1593 1594 if (dtrace_probe_lookup(provider->ftp_provid, 1595 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1596 continue; 1597 1598 atomic_inc_32(&fasttrap_total); 1599 1600 if (fasttrap_total > fasttrap_max) { 1601 atomic_dec_32(&fasttrap_total); 1602 goto no_mem; 1603 } 1604 1605 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1606 1607 pp->ftp_prov = provider; 1608 pp->ftp_faddr = pdata->ftps_pc; 1609 pp->ftp_fsize = pdata->ftps_size; 1610 pp->ftp_pid = pdata->ftps_pid; 1611 pp->ftp_ntps = 1; 1612 1613 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1614 KM_SLEEP); 1615 1616 tp->ftt_proc = provider->ftp_proc; 1617 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1618 tp->ftt_pid = pdata->ftps_pid; 1619 1620 pp->ftp_tps[0].fit_tp = tp; 1621 pp->ftp_tps[0].fit_id.fti_probe = pp; 1622 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1623 1624 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1625 pdata->ftps_mod, pdata->ftps_func, name_str, 1626 FASTTRAP_OFFSET_AFRAMES, pp); 1627 } 1628 1629 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1630 pdata->ftps_func, name) == 0) { 1631 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1632 1633 if (fasttrap_total > fasttrap_max) { 1634 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1635 goto no_mem; 1636 } 1637 1638 /* 1639 * Make sure all tracepoint program counter values are unique. 1640 * We later assume that each probe has exactly one tracepoint 1641 * for a given pc. 1642 */ 1643 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1644 sizeof (uint64_t), fasttrap_uint64_cmp); 1645 for (i = 1; i < pdata->ftps_noffs; i++) { 1646 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1647 continue; 1648 1649 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1650 goto no_mem; 1651 } 1652 1653 ASSERT(pdata->ftps_noffs > 0); 1654 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1655 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1656 1657 pp->ftp_prov = provider; 1658 pp->ftp_faddr = pdata->ftps_pc; 1659 pp->ftp_fsize = pdata->ftps_size; 1660 pp->ftp_pid = pdata->ftps_pid; 1661 pp->ftp_ntps = pdata->ftps_noffs; 1662 1663 for (i = 0; i < pdata->ftps_noffs; i++) { 1664 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1665 KM_SLEEP); 1666 1667 tp->ftt_proc = provider->ftp_proc; 1668 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1669 tp->ftt_pid = pdata->ftps_pid; 1670 1671 pp->ftp_tps[i].fit_tp = tp; 1672 pp->ftp_tps[i].fit_id.fti_probe = pp; 1673 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1674 } 1675 1676 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1677 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1678 } 1679 1680 mutex_exit(&provider->ftp_cmtx); 1681 1682 /* 1683 * We know that the provider is still valid since we incremented the 1684 * creation reference count. If someone tried to clean up this provider 1685 * while we were using it (e.g. because the process called exec(2) or 1686 * exit(2)), take note of that and try to clean it up now. 1687 */ 1688 mutex_enter(&provider->ftp_mtx); 1689 provider->ftp_ccount--; 1690 whack = provider->ftp_retired; 1691 mutex_exit(&provider->ftp_mtx); 1692 1693 if (whack) 1694 fasttrap_pid_cleanup(); 1695 1696 return (0); 1697 1698 no_mem: 1699 /* 1700 * If we've exhausted the allowable resources, we'll try to remove 1701 * this provider to free some up. This is to cover the case where 1702 * the user has accidentally created many more probes than was 1703 * intended (e.g. pid123:::). 1704 */ 1705 mutex_exit(&provider->ftp_cmtx); 1706 mutex_enter(&provider->ftp_mtx); 1707 provider->ftp_ccount--; 1708 provider->ftp_marked = 1; 1709 mutex_exit(&provider->ftp_mtx); 1710 1711 fasttrap_pid_cleanup(); 1712 1713 return (ENOMEM); 1714 } 1715 1716 /*ARGSUSED*/ 1717 static void * 1718 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1719 { 1720 fasttrap_provider_t *provider; 1721 1722 /* 1723 * A 32-bit unsigned integer (like a pid for example) can be 1724 * expressed in 10 or fewer decimal digits. Make sure that we'll 1725 * have enough space for the provider name. 1726 */ 1727 if (strlen(dhpv->dthpv_provname) + 10 >= 1728 sizeof (provider->ftp_name)) { 1729 cmn_err(CE_WARN, "failed to instantiate provider %s: " 1730 "name too long to accomodate pid", dhpv->dthpv_provname); 1731 return (NULL); 1732 } 1733 1734 /* 1735 * Don't let folks spoof the true pid provider. 1736 */ 1737 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1738 cmn_err(CE_WARN, "failed to instantiate provider %s: " 1739 "%s is an invalid name", dhpv->dthpv_provname, 1740 FASTTRAP_PID_NAME); 1741 return (NULL); 1742 } 1743 1744 /* 1745 * The highest stability class that fasttrap supports is ISA; cap 1746 * the stability of the new provider accordingly. 1747 */ 1748 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1749 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1750 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1751 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1752 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1753 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1754 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1755 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1756 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1757 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1758 1759 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1760 &dhpv->dthpv_pattr)) == NULL) { 1761 cmn_err(CE_WARN, "failed to instantiate provider %s for " 1762 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1763 return (NULL); 1764 } 1765 1766 /* 1767 * Up the meta provider count so this provider isn't removed until 1768 * the meta provider has been told to remove it. 1769 */ 1770 provider->ftp_mcount++; 1771 1772 mutex_exit(&provider->ftp_mtx); 1773 1774 return (provider); 1775 } 1776 1777 /*ARGSUSED*/ 1778 static void 1779 fasttrap_meta_create_probe(void *arg, void *parg, 1780 dtrace_helper_probedesc_t *dhpb) 1781 { 1782 fasttrap_provider_t *provider = parg; 1783 fasttrap_probe_t *pp; 1784 fasttrap_tracepoint_t *tp; 1785 int i, j; 1786 uint32_t ntps; 1787 1788 /* 1789 * Since the meta provider count is non-zero we don't have to worry 1790 * about this provider disappearing. 1791 */ 1792 ASSERT(provider->ftp_mcount > 0); 1793 1794 /* 1795 * The offsets must be unique. 1796 */ 1797 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1798 fasttrap_uint32_cmp); 1799 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1800 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1801 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1802 return; 1803 } 1804 1805 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1806 fasttrap_uint32_cmp); 1807 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1808 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1809 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1810 return; 1811 } 1812 1813 /* 1814 * Grab the creation lock to ensure consistency between calls to 1815 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1816 * other threads creating probes. 1817 */ 1818 mutex_enter(&provider->ftp_cmtx); 1819 1820 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1821 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1822 mutex_exit(&provider->ftp_cmtx); 1823 return; 1824 } 1825 1826 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1827 ASSERT(ntps > 0); 1828 1829 atomic_add_32(&fasttrap_total, ntps); 1830 1831 if (fasttrap_total > fasttrap_max) { 1832 atomic_add_32(&fasttrap_total, -ntps); 1833 mutex_exit(&provider->ftp_cmtx); 1834 return; 1835 } 1836 1837 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1838 1839 pp->ftp_prov = provider; 1840 pp->ftp_pid = provider->ftp_pid; 1841 pp->ftp_ntps = ntps; 1842 pp->ftp_nargs = dhpb->dthpb_xargc; 1843 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1844 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1845 1846 /* 1847 * First create a tracepoint for each actual point of interest. 1848 */ 1849 for (i = 0; i < dhpb->dthpb_noffs; i++) { 1850 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1851 1852 tp->ftt_proc = provider->ftp_proc; 1853 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1854 tp->ftt_pid = provider->ftp_pid; 1855 1856 pp->ftp_tps[i].fit_tp = tp; 1857 pp->ftp_tps[i].fit_id.fti_probe = pp; 1858 #ifdef __sparc 1859 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1860 #else 1861 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1862 #endif 1863 } 1864 1865 /* 1866 * Then create a tracepoint for each is-enabled point. 1867 */ 1868 for (j = 0; i < ntps; i++, j++) { 1869 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1870 1871 tp->ftt_proc = provider->ftp_proc; 1872 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1873 tp->ftt_pid = provider->ftp_pid; 1874 1875 pp->ftp_tps[i].fit_tp = tp; 1876 pp->ftp_tps[i].fit_id.fti_probe = pp; 1877 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1878 } 1879 1880 /* 1881 * If the arguments are shuffled around we set the argument remapping 1882 * table. Later, when the probe fires, we only remap the arguments 1883 * if the table is non-NULL. 1884 */ 1885 for (i = 0; i < dhpb->dthpb_xargc; i++) { 1886 if (dhpb->dthpb_args[i] != i) { 1887 pp->ftp_argmap = dhpb->dthpb_args; 1888 break; 1889 } 1890 } 1891 1892 /* 1893 * The probe is fully constructed -- register it with DTrace. 1894 */ 1895 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 1896 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 1897 1898 mutex_exit(&provider->ftp_cmtx); 1899 } 1900 1901 /*ARGSUSED*/ 1902 static void 1903 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1904 { 1905 /* 1906 * Clean up the USDT provider. There may be active consumers of the 1907 * provider busy adding probes, no damage will actually befall the 1908 * provider until that count has dropped to zero. This just puts 1909 * the provider on death row. 1910 */ 1911 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 1912 } 1913 1914 static dtrace_mops_t fasttrap_mops = { 1915 fasttrap_meta_create_probe, 1916 fasttrap_meta_provide, 1917 fasttrap_meta_remove 1918 }; 1919 1920 /*ARGSUSED*/ 1921 static int 1922 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 1923 { 1924 return (0); 1925 } 1926 1927 /*ARGSUSED*/ 1928 static int 1929 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 1930 { 1931 if (!dtrace_attached()) 1932 return (EAGAIN); 1933 1934 if (cmd == FASTTRAPIOC_MAKEPROBE) { 1935 fasttrap_probe_spec_t *uprobe = (void *)arg; 1936 fasttrap_probe_spec_t *probe; 1937 uint64_t noffs; 1938 size_t size; 1939 int ret, err; 1940 1941 if (copyin(&uprobe->ftps_noffs, &noffs, 1942 sizeof (uprobe->ftps_noffs))) 1943 return (EFAULT); 1944 1945 /* 1946 * Probes must have at least one tracepoint. 1947 */ 1948 if (noffs == 0) 1949 return (EINVAL); 1950 1951 size = sizeof (fasttrap_probe_spec_t) + 1952 sizeof (probe->ftps_offs[0]) * (noffs - 1); 1953 1954 if (size > 1024 * 1024) 1955 return (ENOMEM); 1956 1957 probe = kmem_alloc(size, KM_SLEEP); 1958 1959 if (copyin(uprobe, probe, size) != 0 || 1960 probe->ftps_noffs != noffs) { 1961 kmem_free(probe, size); 1962 return (EFAULT); 1963 } 1964 1965 /* 1966 * Verify that the function and module strings contain no 1967 * funny characters. 1968 */ 1969 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 1970 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 1971 ret = EINVAL; 1972 goto err; 1973 } 1974 1975 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 1976 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 1977 ret = EINVAL; 1978 goto err; 1979 } 1980 1981 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 1982 proc_t *p; 1983 pid_t pid = probe->ftps_pid; 1984 1985 mutex_enter(&pidlock); 1986 /* 1987 * Report an error if the process doesn't exist 1988 * or is actively being birthed. 1989 */ 1990 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 1991 mutex_exit(&pidlock); 1992 return (ESRCH); 1993 } 1994 mutex_enter(&p->p_lock); 1995 mutex_exit(&pidlock); 1996 1997 if ((ret = priv_proc_cred_perm(cr, p, NULL, 1998 VREAD | VWRITE)) != 0) { 1999 mutex_exit(&p->p_lock); 2000 return (ret); 2001 } 2002 2003 mutex_exit(&p->p_lock); 2004 } 2005 2006 ret = fasttrap_add_probe(probe); 2007 err: 2008 kmem_free(probe, size); 2009 2010 return (ret); 2011 2012 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2013 fasttrap_instr_query_t instr; 2014 fasttrap_tracepoint_t *tp; 2015 uint_t index; 2016 int ret; 2017 2018 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2019 return (EFAULT); 2020 2021 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2022 proc_t *p; 2023 pid_t pid = instr.ftiq_pid; 2024 2025 mutex_enter(&pidlock); 2026 /* 2027 * Report an error if the process doesn't exist 2028 * or is actively being birthed. 2029 */ 2030 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 2031 mutex_exit(&pidlock); 2032 return (ESRCH); 2033 } 2034 mutex_enter(&p->p_lock); 2035 mutex_exit(&pidlock); 2036 2037 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2038 VREAD)) != 0) { 2039 mutex_exit(&p->p_lock); 2040 return (ret); 2041 } 2042 2043 mutex_exit(&p->p_lock); 2044 } 2045 2046 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2047 2048 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2049 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2050 while (tp != NULL) { 2051 if (instr.ftiq_pid == tp->ftt_pid && 2052 instr.ftiq_pc == tp->ftt_pc && 2053 tp->ftt_proc->ftpc_acount != 0) 2054 break; 2055 2056 tp = tp->ftt_next; 2057 } 2058 2059 if (tp == NULL) { 2060 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2061 return (ENOENT); 2062 } 2063 2064 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2065 sizeof (instr.ftiq_instr)); 2066 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2067 2068 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2069 return (EFAULT); 2070 2071 return (0); 2072 } 2073 2074 return (EINVAL); 2075 } 2076 2077 static struct cb_ops fasttrap_cb_ops = { 2078 fasttrap_open, /* open */ 2079 nodev, /* close */ 2080 nulldev, /* strategy */ 2081 nulldev, /* print */ 2082 nodev, /* dump */ 2083 nodev, /* read */ 2084 nodev, /* write */ 2085 fasttrap_ioctl, /* ioctl */ 2086 nodev, /* devmap */ 2087 nodev, /* mmap */ 2088 nodev, /* segmap */ 2089 nochpoll, /* poll */ 2090 ddi_prop_op, /* cb_prop_op */ 2091 0, /* streamtab */ 2092 D_NEW | D_MP /* Driver compatibility flag */ 2093 }; 2094 2095 /*ARGSUSED*/ 2096 static int 2097 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 2098 { 2099 int error; 2100 2101 switch (infocmd) { 2102 case DDI_INFO_DEVT2DEVINFO: 2103 *result = (void *)fasttrap_devi; 2104 error = DDI_SUCCESS; 2105 break; 2106 case DDI_INFO_DEVT2INSTANCE: 2107 *result = (void *)0; 2108 error = DDI_SUCCESS; 2109 break; 2110 default: 2111 error = DDI_FAILURE; 2112 } 2113 return (error); 2114 } 2115 2116 static int 2117 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 2118 { 2119 ulong_t nent; 2120 2121 switch (cmd) { 2122 case DDI_ATTACH: 2123 break; 2124 case DDI_RESUME: 2125 return (DDI_SUCCESS); 2126 default: 2127 return (DDI_FAILURE); 2128 } 2129 2130 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 2131 DDI_PSEUDO, NULL) == DDI_FAILURE) { 2132 ddi_remove_minor_node(devi, NULL); 2133 return (DDI_FAILURE); 2134 } 2135 2136 ddi_report_dev(devi); 2137 fasttrap_devi = devi; 2138 2139 /* 2140 * Install our hooks into fork(2), exec(2), and exit(2). 2141 */ 2142 dtrace_fasttrap_fork_ptr = &fasttrap_fork; 2143 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 2144 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 2145 2146 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2147 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2148 fasttrap_total = 0; 2149 2150 /* 2151 * Conjure up the tracepoints hashtable... 2152 */ 2153 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2154 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2155 2156 if (nent == 0 || nent > 0x1000000) 2157 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2158 2159 if (ISP2(nent)) 2160 fasttrap_tpoints.fth_nent = nent; 2161 else 2162 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2163 ASSERT(fasttrap_tpoints.fth_nent > 0); 2164 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2165 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2166 sizeof (fasttrap_bucket_t), KM_SLEEP); 2167 2168 /* 2169 * ... and the providers hash table... 2170 */ 2171 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2172 if (ISP2(nent)) 2173 fasttrap_provs.fth_nent = nent; 2174 else 2175 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2176 ASSERT(fasttrap_provs.fth_nent > 0); 2177 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2178 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2179 sizeof (fasttrap_bucket_t), KM_SLEEP); 2180 2181 /* 2182 * ... and the procs hash table. 2183 */ 2184 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2185 if (ISP2(nent)) 2186 fasttrap_procs.fth_nent = nent; 2187 else 2188 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2189 ASSERT(fasttrap_procs.fth_nent > 0); 2190 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2191 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2192 sizeof (fasttrap_bucket_t), KM_SLEEP); 2193 2194 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2195 &fasttrap_meta_id); 2196 2197 return (DDI_SUCCESS); 2198 } 2199 2200 static int 2201 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 2202 { 2203 int i, fail = 0; 2204 timeout_id_t tmp; 2205 2206 switch (cmd) { 2207 case DDI_DETACH: 2208 break; 2209 case DDI_SUSPEND: 2210 return (DDI_SUCCESS); 2211 default: 2212 return (DDI_FAILURE); 2213 } 2214 2215 /* 2216 * Unregister the meta-provider to make sure no new fasttrap- 2217 * managed providers come along while we're trying to close up 2218 * shop. If we fail to detach, we'll need to re-register as a 2219 * meta-provider. We can fail to unregister as a meta-provider 2220 * if providers we manage still exist. 2221 */ 2222 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2223 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2224 return (DDI_FAILURE); 2225 2226 /* 2227 * Prevent any new timeouts from running by setting fasttrap_timeout 2228 * to a non-zero value, and wait for the current timeout to complete. 2229 */ 2230 mutex_enter(&fasttrap_cleanup_mtx); 2231 fasttrap_cleanup_work = 0; 2232 2233 while (fasttrap_timeout != (timeout_id_t)1) { 2234 tmp = fasttrap_timeout; 2235 fasttrap_timeout = (timeout_id_t)1; 2236 2237 if (tmp != 0) { 2238 mutex_exit(&fasttrap_cleanup_mtx); 2239 (void) untimeout(tmp); 2240 mutex_enter(&fasttrap_cleanup_mtx); 2241 } 2242 } 2243 2244 fasttrap_cleanup_work = 0; 2245 mutex_exit(&fasttrap_cleanup_mtx); 2246 2247 /* 2248 * Iterate over all of our providers. If there's still a process 2249 * that corresponds to that pid, fail to detach. 2250 */ 2251 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2252 fasttrap_provider_t **fpp, *fp; 2253 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2254 2255 mutex_enter(&bucket->ftb_mtx); 2256 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2257 while ((fp = *fpp) != NULL) { 2258 /* 2259 * Acquire and release the lock as a simple way of 2260 * waiting for any other consumer to finish with 2261 * this provider. A thread must first acquire the 2262 * bucket lock so there's no chance of another thread 2263 * blocking on the provider's lock. 2264 */ 2265 mutex_enter(&fp->ftp_mtx); 2266 mutex_exit(&fp->ftp_mtx); 2267 2268 if (dtrace_unregister(fp->ftp_provid) != 0) { 2269 fail = 1; 2270 fpp = &fp->ftp_next; 2271 } else { 2272 *fpp = fp->ftp_next; 2273 fasttrap_provider_free(fp); 2274 } 2275 } 2276 2277 mutex_exit(&bucket->ftb_mtx); 2278 } 2279 2280 if (fail) { 2281 uint_t work; 2282 /* 2283 * If we're failing to detach, we need to unblock timeouts 2284 * and start a new timeout if any work has accumulated while 2285 * we've been unsuccessfully trying to detach. 2286 */ 2287 mutex_enter(&fasttrap_cleanup_mtx); 2288 fasttrap_timeout = 0; 2289 work = fasttrap_cleanup_work; 2290 mutex_exit(&fasttrap_cleanup_mtx); 2291 2292 if (work) 2293 fasttrap_pid_cleanup(); 2294 2295 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2296 &fasttrap_meta_id); 2297 2298 return (DDI_FAILURE); 2299 } 2300 2301 #ifdef DEBUG 2302 mutex_enter(&fasttrap_count_mtx); 2303 ASSERT(fasttrap_pid_count == 0); 2304 mutex_exit(&fasttrap_count_mtx); 2305 #endif 2306 2307 kmem_free(fasttrap_tpoints.fth_table, 2308 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2309 fasttrap_tpoints.fth_nent = 0; 2310 2311 kmem_free(fasttrap_provs.fth_table, 2312 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2313 fasttrap_provs.fth_nent = 0; 2314 2315 kmem_free(fasttrap_procs.fth_table, 2316 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2317 fasttrap_procs.fth_nent = 0; 2318 2319 /* 2320 * We know there are no tracepoints in any process anywhere in 2321 * the system so there is no process which has its p_dtrace_count 2322 * greater than zero, therefore we know that no thread can actively 2323 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 2324 * and fasttrap_exec() and fasttrap_exit(). 2325 */ 2326 ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 2327 dtrace_fasttrap_fork_ptr = NULL; 2328 2329 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 2330 dtrace_fasttrap_exec_ptr = NULL; 2331 2332 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 2333 dtrace_fasttrap_exit_ptr = NULL; 2334 2335 ddi_remove_minor_node(devi, NULL); 2336 2337 return (DDI_SUCCESS); 2338 } 2339 2340 static struct dev_ops fasttrap_ops = { 2341 DEVO_REV, /* devo_rev */ 2342 0, /* refcnt */ 2343 fasttrap_info, /* get_dev_info */ 2344 nulldev, /* identify */ 2345 nulldev, /* probe */ 2346 fasttrap_attach, /* attach */ 2347 fasttrap_detach, /* detach */ 2348 nodev, /* reset */ 2349 &fasttrap_cb_ops, /* driver operations */ 2350 NULL, /* bus operations */ 2351 nodev, /* dev power */ 2352 ddi_quiesce_not_needed, /* quiesce */ 2353 }; 2354 2355 /* 2356 * Module linkage information for the kernel. 2357 */ 2358 static struct modldrv modldrv = { 2359 &mod_driverops, /* module type (this is a pseudo driver) */ 2360 "Fasttrap Tracing", /* name of module */ 2361 &fasttrap_ops, /* driver ops */ 2362 }; 2363 2364 static struct modlinkage modlinkage = { 2365 MODREV_1, 2366 (void *)&modldrv, 2367 NULL 2368 }; 2369 2370 int 2371 _init(void) 2372 { 2373 return (mod_install(&modlinkage)); 2374 } 2375 2376 int 2377 _info(struct modinfo *modinfop) 2378 { 2379 return (mod_info(&modlinkage, modinfop)); 2380 } 2381 2382 int 2383 _fini(void) 2384 { 2385 return (mod_remove(&modlinkage)); 2386 }