1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 #include <mdb/mdb_modapi.h>
  27 #include <mdb/mdb_ctf.h>
  28 
  29 #include <sys/types.h>
  30 #include <sys/regset.h>
  31 #include <sys/stack.h>
  32 #include <sys/thread.h>
  33 #include <sys/modctl.h>
  34 #include <assert.h>
  35 
  36 #include "findstack.h"
  37 #include "thread.h"
  38 #include "sobj.h"
  39 
  40 int findstack_debug_on = 0;
  41 
  42 /*
  43  * "sp" is a kernel VA.
  44  */
  45 static int
  46 print_stack(uintptr_t sp, uintptr_t pc, uintptr_t addr,
  47     int argc, const mdb_arg_t *argv, int free_state)
  48 {
  49         int showargs = 0, count, err;
  50 
  51         count = mdb_getopts(argc, argv,
  52             'v', MDB_OPT_SETBITS, TRUE, &showargs, NULL);
  53         argc -= count;
  54         argv += count;
  55 
  56         if (argc > 1 || (argc == 1 && argv->a_type != MDB_TYPE_STRING))
  57                 return (DCMD_USAGE);
  58 
  59         mdb_printf("stack pointer for thread %p%s: %p\n",
  60             addr, (free_state ? " (TS_FREE)" : ""), sp);
  61         if (pc != 0)
  62                 mdb_printf("[ %0?lr %a() ]\n", sp, pc);
  63 
  64         mdb_inc_indent(2);
  65         mdb_set_dot(sp);
  66 
  67         if (argc == 1)
  68                 err = mdb_eval(argv->a_un.a_str);
  69         else if (showargs)
  70                 err = mdb_eval("<.$C");
  71         else
  72                 err = mdb_eval("<.$C0");
  73 
  74         mdb_dec_indent(2);
  75 
  76         return ((err == -1) ? DCMD_ABORT : DCMD_OK);
  77 }
  78 
  79 int
  80 findstack(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
  81 {
  82         findstack_info_t fsi;
  83         int retval;
  84 
  85         if (!(flags & DCMD_ADDRSPEC))
  86                 return (DCMD_USAGE);
  87 
  88         bzero(&fsi, sizeof (fsi));
  89 
  90         if ((retval = stacks_findstack(addr, &fsi, 1)) != DCMD_OK ||
  91             fsi.fsi_failed)
  92                 return (retval);
  93 
  94         return (print_stack(fsi.fsi_sp, fsi.fsi_pc, addr,
  95             argc, argv, fsi.fsi_tstate == TS_FREE));
  96 }
  97 
  98 /*ARGSUSED*/
  99 int
 100 findstack_debug(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *av)
 101 {
 102         findstack_debug_on ^= 1;
 103 
 104         mdb_printf("findstack: debugging is now %s\n",
 105             findstack_debug_on ? "on" : "off");
 106 
 107         return (DCMD_OK);
 108 }
 109 
 110 static void
 111 uppercase(char *p)
 112 {
 113         for (; *p != '\0'; p++) {
 114                 if (*p >= 'a' && *p <= 'z')
 115                         *p += 'A' - 'a';
 116         }
 117 }
 118 
 119 static void
 120 sobj_to_text(uintptr_t addr, char *out, size_t out_sz)
 121 {
 122         sobj_ops_to_text(addr, out, out_sz);
 123         uppercase(out);
 124 }
 125 
 126 #define SOBJ_ALL        1
 127 
 128 static int
 129 text_to_sobj(const char *text, uintptr_t *out)
 130 {
 131         if (strcasecmp(text, "ALL") == 0) {
 132                 *out = SOBJ_ALL;
 133                 return (0);
 134         }
 135 
 136         return (sobj_text_to_ops(text, out));
 137 }
 138 
 139 #define TSTATE_PANIC    -2U
 140 static int
 141 text_to_tstate(const char *text, uint_t *out)
 142 {
 143         if (strcasecmp(text, "panic") == 0)
 144                 *out = TSTATE_PANIC;
 145         else if (thread_text_to_state(text, out) != 0) {
 146                 mdb_warn("tstate \"%s\" not recognized\n", text);
 147                 return (-1);
 148         }
 149         return (0);
 150 }
 151 
 152 static void
 153 tstate_to_text(uint_t tstate, uint_t paniced, char *out, size_t out_sz)
 154 {
 155         if (paniced)
 156                 mdb_snprintf(out, out_sz, "panic");
 157         else
 158                 thread_state_to_text(tstate, out, out_sz);
 159         uppercase(out);
 160 }
 161 
 162 typedef struct stacks_entry {
 163         struct stacks_entry     *se_next;
 164         struct stacks_entry     *se_dup;        /* dups of this stack */
 165         uintptr_t               se_thread;
 166         uintptr_t               se_sp;
 167         uintptr_t               se_sobj_ops;
 168         uint32_t                se_tstate;
 169         uint32_t                se_count;       /* # threads w/ this stack */
 170         uint8_t                 se_overflow;
 171         uint8_t                 se_depth;
 172         uint8_t                 se_failed;      /* failure reason; FSI_FAIL_* */
 173         uint8_t                 se_panic;
 174         uintptr_t               se_stack[1];
 175 } stacks_entry_t;
 176 #define STACKS_ENTRY_SIZE(x) OFFSETOF(stacks_entry_t, se_stack[(x)])
 177 
 178 #define STACKS_HSIZE 127
 179 
 180 /* Maximum stack depth reported in stacks */
 181 #define STACKS_MAX_DEPTH        254
 182 
 183 typedef struct stacks_info {
 184         size_t          si_count;       /* total stacks_entry_ts (incl dups) */
 185         size_t          si_entries;     /* # entries in hash table */
 186         stacks_entry_t  **si_hash;      /* hash table */
 187         findstack_info_t si_fsi;        /* transient callback state */
 188 } stacks_info_t;
 189 
 190 /* global state cached between invocations */
 191 #define STACKS_STATE_CLEAN      0
 192 #define STACKS_STATE_DIRTY      1
 193 #define STACKS_STATE_DONE       2
 194 static uint_t stacks_state = STACKS_STATE_CLEAN;
 195 static stacks_entry_t **stacks_hash;
 196 static stacks_entry_t **stacks_array;
 197 static size_t stacks_array_size;
 198 
 199 size_t
 200 stacks_hash_entry(stacks_entry_t *sep)
 201 {
 202         size_t depth = sep->se_depth;
 203         uintptr_t *stack = sep->se_stack;
 204 
 205         uint64_t total = depth;
 206 
 207         while (depth > 0) {
 208                 total += *stack;
 209                 stack++; depth--;
 210         }
 211 
 212         return (total % STACKS_HSIZE);
 213 }
 214 
 215 /*
 216  * This is used to both compare stacks for equality and to sort the final
 217  * list of unique stacks.  forsort specifies the latter behavior, which
 218  * additionally:
 219  *      compares se_count, and
 220  *      sorts the stacks by text function name.
 221  *
 222  * The equality test is independent of se_count, and doesn't care about
 223  * relative ordering, so we don't do the extra work of looking up symbols
 224  * for the stack addresses.
 225  */
 226 int
 227 stacks_entry_comp_impl(stacks_entry_t *l, stacks_entry_t *r,
 228     uint_t forsort)
 229 {
 230         int idx;
 231 
 232         int depth = MIN(l->se_depth, r->se_depth);
 233 
 234         /* no matter what, panic stacks come last. */
 235         if (l->se_panic > r->se_panic)
 236                 return (1);
 237         if (l->se_panic < r->se_panic)
 238                 return (-1);
 239 
 240         if (forsort) {
 241                 /* put large counts earlier */
 242                 if (l->se_count > r->se_count)
 243                         return (-1);
 244                 if (l->se_count < r->se_count)
 245                         return (1);
 246         }
 247 
 248         if (l->se_tstate > r->se_tstate)
 249                 return (1);
 250         if (l->se_tstate < r->se_tstate)
 251                 return (-1);
 252 
 253         if (l->se_failed > r->se_failed)
 254                 return (1);
 255         if (l->se_failed < r->se_failed)
 256                 return (-1);
 257 
 258         for (idx = 0; idx < depth; idx++) {
 259                 char lbuf[MDB_SYM_NAMLEN];
 260                 char rbuf[MDB_SYM_NAMLEN];
 261 
 262                 int rval;
 263                 uintptr_t laddr = l->se_stack[idx];
 264                 uintptr_t raddr = r->se_stack[idx];
 265 
 266                 if (laddr == raddr)
 267                         continue;
 268 
 269                 if (forsort &&
 270                     mdb_lookup_by_addr(laddr, MDB_SYM_FUZZY,
 271                     lbuf, sizeof (lbuf), NULL) != -1 &&
 272                     mdb_lookup_by_addr(raddr, MDB_SYM_FUZZY,
 273                     rbuf, sizeof (rbuf), NULL) != -1 &&
 274                     (rval = strcmp(lbuf, rbuf)) != 0)
 275                         return (rval);
 276 
 277                 if (laddr > raddr)
 278                         return (1);
 279                 return (-1);
 280         }
 281 
 282         if (l->se_overflow > r->se_overflow)
 283                 return (-1);
 284         if (l->se_overflow < r->se_overflow)
 285                 return (1);
 286 
 287         if (l->se_depth > r->se_depth)
 288                 return (1);
 289         if (l->se_depth < r->se_depth)
 290                 return (-1);
 291 
 292         if (l->se_sobj_ops > r->se_sobj_ops)
 293                 return (1);
 294         if (l->se_sobj_ops < r->se_sobj_ops)
 295                 return (-1);
 296 
 297         return (0);
 298 }
 299 
 300 int
 301 stacks_entry_comp(const void *l_arg, const void *r_arg)
 302 {
 303         stacks_entry_t * const *lp = l_arg;
 304         stacks_entry_t * const *rp = r_arg;
 305 
 306         return (stacks_entry_comp_impl(*lp, *rp, 1));
 307 }
 308 
 309 void
 310 stacks_cleanup(int force)
 311 {
 312         int idx = 0;
 313         stacks_entry_t *cur, *next;
 314 
 315         if (stacks_state == STACKS_STATE_CLEAN)
 316                 return;
 317 
 318         if (!force && stacks_state == STACKS_STATE_DONE)
 319                 return;
 320 
 321         /*
 322          * Until the array is sorted and stable, stacks_hash will be non-NULL.
 323          * This way, we can get at all of the data, even if qsort() was
 324          * interrupted while mucking with the array.
 325          */
 326         if (stacks_hash != NULL) {
 327                 for (idx = 0; idx < STACKS_HSIZE; idx++) {
 328                         while ((cur = stacks_hash[idx]) != NULL) {
 329                                 while ((next = cur->se_dup) != NULL) {
 330                                         cur->se_dup = next->se_dup;
 331                                         mdb_free(next,
 332                                             STACKS_ENTRY_SIZE(next->se_depth));
 333                                 }
 334                                 next = cur->se_next;
 335                                 stacks_hash[idx] = next;
 336                                 mdb_free(cur, STACKS_ENTRY_SIZE(cur->se_depth));
 337                         }
 338                 }
 339                 if (stacks_array != NULL)
 340                         mdb_free(stacks_array,
 341                             stacks_array_size * sizeof (*stacks_array));
 342 
 343         } else if (stacks_array != NULL) {
 344                 for (idx = 0; idx < stacks_array_size; idx++) {
 345                         if ((cur = stacks_array[idx]) != NULL) {
 346                                 while ((next = cur->se_dup) != NULL) {
 347                                         cur->se_dup = next->se_dup;
 348                                         mdb_free(next,
 349                                             STACKS_ENTRY_SIZE(next->se_depth));
 350                                 }
 351                                 stacks_array[idx] = NULL;
 352                                 mdb_free(cur, STACKS_ENTRY_SIZE(cur->se_depth));
 353                         }
 354                 }
 355                 mdb_free(stacks_array,
 356                     stacks_array_size * sizeof (*stacks_array));
 357         }
 358 
 359         stacks_findstack_cleanup();
 360 
 361         stacks_array_size = 0;
 362         stacks_state = STACKS_STATE_CLEAN;
 363 }
 364 
 365 /*ARGSUSED*/
 366 int
 367 stacks_thread_cb(uintptr_t addr, const void *ignored, void *cbarg)
 368 {
 369         stacks_info_t *sip = cbarg;
 370         findstack_info_t *fsip = &sip->si_fsi;
 371 
 372         stacks_entry_t **sepp, *nsep, *sep;
 373         int idx;
 374         size_t depth;
 375 
 376         if (stacks_findstack(addr, fsip, 0) != DCMD_OK &&
 377             fsip->fsi_failed == FSI_FAIL_BADTHREAD) {
 378                 mdb_warn("couldn't read thread at %p\n", addr);
 379                 return (WALK_NEXT);
 380         }
 381 
 382         sip->si_count++;
 383 
 384         depth = fsip->fsi_depth;
 385         nsep = mdb_zalloc(STACKS_ENTRY_SIZE(depth), UM_SLEEP);
 386         nsep->se_thread = addr;
 387         nsep->se_sp = fsip->fsi_sp;
 388         nsep->se_sobj_ops = fsip->fsi_sobj_ops;
 389         nsep->se_tstate = fsip->fsi_tstate;
 390         nsep->se_count = 1;
 391         nsep->se_overflow = fsip->fsi_overflow;
 392         nsep->se_depth = depth;
 393         nsep->se_failed = fsip->fsi_failed;
 394         nsep->se_panic = fsip->fsi_panic;
 395 
 396         for (idx = 0; idx < depth; idx++)
 397                 nsep->se_stack[idx] = fsip->fsi_stack[idx];
 398 
 399         for (sepp = &sip->si_hash[stacks_hash_entry(nsep)];
 400             (sep = *sepp) != NULL;
 401             sepp = &sep->se_next) {
 402 
 403                 if (stacks_entry_comp_impl(sep, nsep, 0) != 0)
 404                         continue;
 405 
 406                 nsep->se_dup = sep->se_dup;
 407                 sep->se_dup = nsep;
 408                 sep->se_count++;
 409                 return (WALK_NEXT);
 410         }
 411 
 412         nsep->se_next = NULL;
 413         *sepp = nsep;
 414         sip->si_entries++;
 415 
 416         return (WALK_NEXT);
 417 }
 418 
 419 int
 420 stacks_run_tlist(mdb_pipe_t *tlist, stacks_info_t *si)
 421 {
 422         size_t idx;
 423         size_t found = 0;
 424         int ret;
 425 
 426         for (idx = 0; idx < tlist->pipe_len; idx++) {
 427                 uintptr_t addr = tlist->pipe_data[idx];
 428 
 429                 found++;
 430 
 431                 ret = stacks_thread_cb(addr, NULL, si);
 432                 if (ret == WALK_DONE)
 433                         break;
 434                 if (ret != WALK_NEXT)
 435                         return (-1);
 436         }
 437 
 438         if (found)
 439                 return (0);
 440         return (-1);
 441 }
 442 
 443 int
 444 stacks_run(int verbose, mdb_pipe_t *tlist)
 445 {
 446         stacks_info_t si;
 447         findstack_info_t *fsip = &si.si_fsi;
 448         size_t idx;
 449         stacks_entry_t **cur;
 450 
 451         bzero(&si, sizeof (si));
 452 
 453         stacks_state = STACKS_STATE_DIRTY;
 454 
 455         stacks_hash = si.si_hash =
 456             mdb_zalloc(STACKS_HSIZE * sizeof (*si.si_hash), UM_SLEEP);
 457         si.si_entries = 0;
 458         si.si_count = 0;
 459 
 460         fsip->fsi_max_depth = STACKS_MAX_DEPTH;
 461         fsip->fsi_stack =
 462             mdb_alloc(fsip->fsi_max_depth * sizeof (*fsip->fsi_stack),
 463             UM_SLEEP | UM_GC);
 464 
 465         if (verbose)
 466                 mdb_warn("stacks: processing kernel threads\n");
 467 
 468         if (tlist != NULL) {
 469                 if (stacks_run_tlist(tlist, &si))
 470                         return (DCMD_ERR);
 471         } else {
 472                 if (mdb_walk("thread", stacks_thread_cb, &si) != 0) {
 473                         mdb_warn("cannot walk \"thread\"");
 474                         return (DCMD_ERR);
 475                 }
 476         }
 477 
 478         if (verbose)
 479                 mdb_warn("stacks: %d unique stacks / %d threads\n",
 480                     si.si_entries, si.si_count);
 481 
 482         stacks_array_size = si.si_entries;
 483         stacks_array =
 484             mdb_zalloc(si.si_entries * sizeof (*stacks_array), UM_SLEEP);
 485         cur = stacks_array;
 486         for (idx = 0; idx < STACKS_HSIZE; idx++) {
 487                 stacks_entry_t *sep;
 488                 for (sep = si.si_hash[idx]; sep != NULL; sep = sep->se_next)
 489                         *(cur++) = sep;
 490         }
 491 
 492         if (cur != stacks_array + si.si_entries) {
 493                 mdb_warn("stacks: miscounted array size (%d != size: %d)\n",
 494                     (cur - stacks_array), stacks_array_size);
 495                 return (DCMD_ERR);
 496         }
 497         qsort(stacks_array, si.si_entries, sizeof (*stacks_array),
 498             stacks_entry_comp);
 499 
 500         /* Now that we're done, free the hash table */
 501         stacks_hash = NULL;
 502         mdb_free(si.si_hash, STACKS_HSIZE * sizeof (*si.si_hash));
 503 
 504         if (tlist == NULL)
 505                 stacks_state = STACKS_STATE_DONE;
 506 
 507         if (verbose)
 508                 mdb_warn("stacks: done\n");
 509 
 510         return (DCMD_OK);
 511 }
 512 
 513 static int
 514 stacks_has_caller(stacks_entry_t *sep, uintptr_t addr)
 515 {
 516         uintptr_t laddr = addr;
 517         uintptr_t haddr = addr + 1;
 518         int idx;
 519         char c[MDB_SYM_NAMLEN];
 520         GElf_Sym sym;
 521 
 522         if (mdb_lookup_by_addr(addr, MDB_SYM_FUZZY,
 523             c, sizeof (c), &sym) != -1 &&
 524             addr == (uintptr_t)sym.st_value) {
 525                 laddr = (uintptr_t)sym.st_value;
 526                 haddr = (uintptr_t)sym.st_value + sym.st_size;
 527         }
 528 
 529         for (idx = 0; idx < sep->se_depth; idx++)
 530                 if (sep->se_stack[idx] >= laddr && sep->se_stack[idx] < haddr)
 531                         return (1);
 532 
 533         return (0);
 534 }
 535 
 536 static int
 537 stacks_has_module(stacks_entry_t *sep, stacks_module_t *mp)
 538 {
 539         int idx;
 540 
 541         for (idx = 0; idx < sep->se_depth; idx++) {
 542                 if (sep->se_stack[idx] >= mp->sm_text &&
 543                     sep->se_stack[idx] < mp->sm_text + mp->sm_size)
 544                         return (1);
 545         }
 546 
 547         return (0);
 548 }
 549 
 550 static int
 551 stacks_module_find(const char *name, stacks_module_t *mp)
 552 {
 553         (void) strncpy(mp->sm_name, name, sizeof (mp->sm_name));
 554 
 555         if (stacks_module(mp) != 0)
 556                 return (-1);
 557 
 558         if (mp->sm_size == 0) {
 559                 mdb_warn("stacks: module \"%s\" is unknown\n", name);
 560                 return (-1);
 561         }
 562 
 563         return (0);
 564 }
 565 
 566 static int
 567 uintptrcomp(const void *lp, const void *rp)
 568 {
 569         uintptr_t lhs = *(const uintptr_t *)lp;
 570         uintptr_t rhs = *(const uintptr_t *)rp;
 571         if (lhs > rhs)
 572                 return (1);
 573         if (lhs < rhs)
 574                 return (-1);
 575         return (0);
 576 }
 577 
 578 /*ARGSUSED*/
 579 int
 580 stacks(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
 581 {
 582         size_t idx;
 583 
 584         char *seen = NULL;
 585 
 586         const char *caller_str = NULL;
 587         const char *excl_caller_str = NULL;
 588         uintptr_t caller = 0, excl_caller = 0;
 589         const char *module_str = NULL;
 590         const char *excl_module_str = NULL;
 591         stacks_module_t module, excl_module;
 592         const char *sobj = NULL;
 593         const char *excl_sobj = NULL;
 594         uintptr_t sobj_ops = 0, excl_sobj_ops = 0;
 595         const char *tstate_str = NULL;
 596         const char *excl_tstate_str = NULL;
 597         uint_t tstate = -1U;
 598         uint_t excl_tstate = -1U;
 599         uint_t printed = 0;
 600 
 601         uint_t all = 0;
 602         uint_t force = 0;
 603         uint_t interesting = 0;
 604         uint_t verbose = 0;
 605 
 606         /*
 607          * We have a slight behavior difference between having piped
 608          * input and 'addr::stacks'.  Without a pipe, we assume the
 609          * thread pointer given is a representative thread, and so
 610          * we include all similar threads in the system in our output.
 611          *
 612          * With a pipe, we filter down to just the threads in our
 613          * input.
 614          */
 615         uint_t addrspec = (flags & DCMD_ADDRSPEC);
 616         uint_t only_matching = addrspec && (flags & DCMD_PIPE);
 617 
 618         mdb_pipe_t p;
 619 
 620         bzero(&module, sizeof (module));
 621         bzero(&excl_module, sizeof (excl_module));
 622 
 623         if (mdb_getopts(argc, argv,
 624             'a', MDB_OPT_SETBITS, TRUE, &all,
 625             'f', MDB_OPT_SETBITS, TRUE, &force,
 626             'i', MDB_OPT_SETBITS, TRUE, &interesting,
 627             'v', MDB_OPT_SETBITS, TRUE, &verbose,
 628             'c', MDB_OPT_STR, &caller_str,
 629             'C', MDB_OPT_STR, &excl_caller_str,
 630             'm', MDB_OPT_STR, &module_str,
 631             'M', MDB_OPT_STR, &excl_module_str,
 632             's', MDB_OPT_STR, &sobj,
 633             'S', MDB_OPT_STR, &excl_sobj,
 634             't', MDB_OPT_STR, &tstate_str,
 635             'T', MDB_OPT_STR, &excl_tstate_str,
 636             NULL) != argc)
 637                 return (DCMD_USAGE);
 638 
 639         if (interesting) {
 640                 if (sobj != NULL || excl_sobj != NULL ||
 641                     tstate_str != NULL || excl_tstate_str != NULL) {
 642                         mdb_warn(
 643                             "stacks: -i is incompatible with -[sStT]\n");
 644                         return (DCMD_USAGE);
 645                 }
 646                 excl_sobj = "CV";
 647                 excl_tstate_str = "FREE";
 648         }
 649 
 650         if (caller_str != NULL) {
 651                 mdb_set_dot(0);
 652                 if (mdb_eval(caller_str) != 0) {
 653                         mdb_warn("stacks: evaluation of \"%s\" failed",
 654                             caller_str);
 655                         return (DCMD_ABORT);
 656                 }
 657                 caller = mdb_get_dot();
 658         }
 659 
 660         if (excl_caller_str != NULL) {
 661                 mdb_set_dot(0);
 662                 if (mdb_eval(excl_caller_str) != 0) {
 663                         mdb_warn("stacks: evaluation of \"%s\" failed",
 664                             excl_caller_str);
 665                         return (DCMD_ABORT);
 666                 }
 667                 excl_caller = mdb_get_dot();
 668         }
 669         mdb_set_dot(addr);
 670 
 671         if (module_str != NULL && stacks_module_find(module_str, &module) != 0)
 672                 return (DCMD_ABORT);
 673 
 674         if (excl_module_str != NULL &&
 675             stacks_module_find(excl_module_str, &excl_module) != 0)
 676                 return (DCMD_ABORT);
 677 
 678         if (sobj != NULL && text_to_sobj(sobj, &sobj_ops) != 0)
 679                 return (DCMD_USAGE);
 680 
 681         if (excl_sobj != NULL && text_to_sobj(excl_sobj, &excl_sobj_ops) != 0)
 682                 return (DCMD_USAGE);
 683 
 684         if (sobj_ops != 0 && excl_sobj_ops != 0) {
 685                 mdb_warn("stacks: only one of -s and -S can be specified\n");
 686                 return (DCMD_USAGE);
 687         }
 688 
 689         if (tstate_str != NULL && text_to_tstate(tstate_str, &tstate) != 0)
 690                 return (DCMD_USAGE);
 691 
 692         if (excl_tstate_str != NULL &&
 693             text_to_tstate(excl_tstate_str, &excl_tstate) != 0)
 694                 return (DCMD_USAGE);
 695 
 696         if (tstate != -1U && excl_tstate != -1U) {
 697                 mdb_warn("stacks: only one of -t and -T can be specified\n");
 698                 return (DCMD_USAGE);
 699         }
 700 
 701         /*
 702          * If there's an address specified, we're going to further filter
 703          * to only entries which have an address in the input.  To reduce
 704          * overhead (and make the sorted output come out right), we
 705          * use mdb_get_pipe() to grab the entire pipeline of input, then
 706          * use qsort() and bsearch() to speed up the search.
 707          */
 708         if (addrspec) {
 709                 mdb_get_pipe(&p);
 710                 if (p.pipe_data == NULL || p.pipe_len == 0) {
 711                         p.pipe_data = &addr;
 712                         p.pipe_len = 1;
 713                 }
 714                 qsort(p.pipe_data, p.pipe_len, sizeof (uintptr_t),
 715                     uintptrcomp);
 716 
 717                 /* remove any duplicates in the data */
 718                 idx = 0;
 719                 while (idx < p.pipe_len - 1) {
 720                         uintptr_t *data = &p.pipe_data[idx];
 721                         size_t len = p.pipe_len - idx;
 722 
 723                         if (data[0] == data[1]) {
 724                                 memmove(data, data + 1,
 725                                     (len - 1) * sizeof (*data));
 726                                 p.pipe_len--;
 727                                 continue; /* repeat without incrementing idx */
 728                         }
 729                         idx++;
 730                 }
 731 
 732                 seen = mdb_zalloc(p.pipe_len, UM_SLEEP | UM_GC);
 733         }
 734 
 735         /*
 736          * Force a cleanup if we're connected to a live system. Never
 737          * do a cleanup after the first invocation around the loop.
 738          */
 739         force |= (mdb_get_state() == MDB_STATE_RUNNING);
 740         if (force && (flags & (DCMD_LOOPFIRST|DCMD_LOOP)) == DCMD_LOOP)
 741                 force = 0;
 742 
 743         stacks_cleanup(force);
 744 
 745         if (stacks_state == STACKS_STATE_CLEAN) {
 746                 int res = stacks_run(verbose, addrspec ? &p : NULL);
 747                 if (res != DCMD_OK)
 748                         return (res);
 749         }
 750 
 751         for (idx = 0; idx < stacks_array_size; idx++) {
 752                 stacks_entry_t *sep = stacks_array[idx];
 753                 stacks_entry_t *cur = sep;
 754                 int frame;
 755                 size_t count = sep->se_count;
 756 
 757                 if (addrspec) {
 758                         stacks_entry_t *head = NULL, *tail = NULL, *sp;
 759                         size_t foundcount = 0;
 760                         /*
 761                          * We use the now-unused hash chain field se_next to
 762                          * link together the dups which match our list.
 763                          */
 764                         for (sp = sep; sp != NULL; sp = sp->se_dup) {
 765                                 uintptr_t *entry = bsearch(&sp->se_thread,
 766                                     p.pipe_data, p.pipe_len, sizeof (uintptr_t),
 767                                     uintptrcomp);
 768                                 if (entry != NULL) {
 769                                         foundcount++;
 770                                         seen[entry - p.pipe_data]++;
 771                                         if (head == NULL)
 772                                                 head = sp;
 773                                         else
 774                                                 tail->se_next = sp;
 775                                         tail = sp;
 776                                         sp->se_next = NULL;
 777                                 }
 778                         }
 779                         if (head == NULL)
 780                                 continue;       /* no match, skip entry */
 781 
 782                         if (only_matching) {
 783                                 cur = sep = head;
 784                                 count = foundcount;
 785                         }
 786                 }
 787 
 788                 if (caller != 0 && !stacks_has_caller(sep, caller))
 789                         continue;
 790 
 791                 if (excl_caller != 0 && stacks_has_caller(sep, excl_caller))
 792                         continue;
 793 
 794                 if (module.sm_size != 0 && !stacks_has_module(sep, &module))
 795                         continue;
 796 
 797                 if (excl_module.sm_size != 0 &&
 798                     stacks_has_module(sep, &excl_module))
 799                         continue;
 800 
 801                 if (tstate != -1U) {
 802                         if (tstate == TSTATE_PANIC) {
 803                                 if (!sep->se_panic)
 804                                         continue;
 805                         } else if (sep->se_panic || sep->se_tstate != tstate)
 806                                 continue;
 807                 }
 808                 if (excl_tstate != -1U) {
 809                         if (excl_tstate == TSTATE_PANIC) {
 810                                 if (sep->se_panic)
 811                                         continue;
 812                         } else if (!sep->se_panic &&
 813                             sep->se_tstate == excl_tstate)
 814                                 continue;
 815                 }
 816 
 817                 if (sobj_ops == SOBJ_ALL) {
 818                         if (sep->se_sobj_ops == 0)
 819                                 continue;
 820                 } else if (sobj_ops != 0) {
 821                         if (sobj_ops != sep->se_sobj_ops)
 822                                 continue;
 823                 }
 824 
 825                 if (!(interesting && sep->se_panic)) {
 826                         if (excl_sobj_ops == SOBJ_ALL) {
 827                                 if (sep->se_sobj_ops != 0)
 828                                         continue;
 829                         } else if (excl_sobj_ops != 0) {
 830                                 if (excl_sobj_ops == sep->se_sobj_ops)
 831                                         continue;
 832                         }
 833                 }
 834 
 835                 if (flags & DCMD_PIPE_OUT) {
 836                         while (sep != NULL) {
 837                                 mdb_printf("%lr\n", sep->se_thread);
 838                                 sep = only_matching ?
 839                                     sep->se_next : sep->se_dup;
 840                         }
 841                         continue;
 842                 }
 843 
 844                 if (all || !printed) {
 845                         mdb_printf("%<u>%-?s %-8s %-?s %8s%</u>\n",
 846                             "THREAD", "STATE", "SOBJ", "COUNT");
 847                         printed = 1;
 848                 }
 849 
 850                 do {
 851                         char state[20];
 852                         char sobj[100];
 853 
 854                         tstate_to_text(cur->se_tstate, cur->se_panic,
 855                             state, sizeof (state));
 856                         sobj_to_text(cur->se_sobj_ops,
 857                             sobj, sizeof (sobj));
 858 
 859                         if (cur == sep)
 860                                 mdb_printf("%-?p %-8s %-?s %8d\n",
 861                                     cur->se_thread, state, sobj, count);
 862                         else
 863                                 mdb_printf("%-?p %-8s %-?s %8s\n",
 864                                     cur->se_thread, state, sobj, "-");
 865 
 866                         cur = only_matching ? cur->se_next : cur->se_dup;
 867                 } while (all && cur != NULL);
 868 
 869                 if (sep->se_failed != 0) {
 870                         char *reason;
 871                         switch (sep->se_failed) {
 872                         case FSI_FAIL_NOTINMEMORY:
 873                                 reason = "thread not in memory";
 874                                 break;
 875                         case FSI_FAIL_THREADCORRUPT:
 876                                 reason = "thread structure stack info corrupt";
 877                                 break;
 878                         case FSI_FAIL_STACKNOTFOUND:
 879                                 reason = "no consistent stack found";
 880                                 break;
 881                         default:
 882                                 reason = "unknown failure";
 883                                 break;
 884                         }
 885                         mdb_printf("%?s <%s>\n", "", reason);
 886                 }
 887 
 888                 for (frame = 0; frame < sep->se_depth; frame++)
 889                         mdb_printf("%?s %a\n", "", sep->se_stack[frame]);
 890                 if (sep->se_overflow)
 891                         mdb_printf("%?s ... truncated ...\n", "");
 892                 mdb_printf("\n");
 893         }
 894 
 895         if (flags & DCMD_ADDRSPEC) {
 896                 for (idx = 0; idx < p.pipe_len; idx++)
 897                         if (seen[idx] == 0)
 898                                 mdb_warn("stacks: %p not in thread list\n",
 899                                     p.pipe_data[idx]);
 900         }
 901         return (DCMD_OK);
 902 }