1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * Panic software-diagnosis subsidiary
28 *
29 * We model a system panic as a defect diagnosis in FMA. When a system
30 * panicks, savecore publishes events which we subscribe to here.
31 *
32 * Our driving events are all raised by savecore, run either from
33 * startup of the dumpadm service or interactively at the command line.
34 * The following describes the logic for the handling of these events.
35 *
36 * On reboot after panic we will run savecore as part of the dumpadm
37 * service startup; we run savecore even if savecore is otherwise
38 * disabled (ie dumpadm -n in effect) - we run savecore -c to check for
39 * a valid dump and raise the initial event.
40 *
41 * If savecore (or savecore -c) observes a valid dump pending on the
42 * device, it raises a "dump_pending_on_device" event provided this
43 * was not an FMA-initiated panic (for those we will replay ereports
44 * from the dump device as usual and make a diagnosis from those; we do
45 * not need to open a case for the panic). We subscribe to the
46 * "dump_pending_on_device" event and use that to open a case; we
47 * open a case requesting the same case uuid as the panic dump image
48 * has for the OS instance uuid - if that fails because of a duplicate
49 * uuid then we have already opened a case for this panic so no need
50 * to open another.
51 *
52 * Included in the "dump_pending_on_device" event is an indication of
53 * whether or not dumpadm is enabled. If not (dumpadm -n in effect)
54 * then we do not expect any further events regarding this panic
55 * until such time as the admin runs savecore manually (if ever).
56 * So in this case we solve the case immediately after open. If/when
57 * subsequent events arrive when savecore is run manually, we will toss
58 * them.
59 *
60 * If dumpadm is enabled then savecore, run from dumpadm service startup,
61 * will attempt to process the dump - either to copy it off the dump
62 * device (if saving compressed) or to uncompress it off the dump device.
63 * If this succeeds savecore raises a "dump_available" event which
64 * includes information on the directory it was saved in, the instance
65 * number, image uuid, compressed form or not, and whether the dump
66 * was complete (as per the dumphdr). If the savecore fails for
67 * some reason then it exits and raises a "savecore_failure" event.
68 * These two events are raised even for FMA-initiated panics.
69 *
70 * We subscribe to both the "dump_available" and "savecore_failed" events,
71 * and in the handling thereof we will close the case opened earlier (if
72 * this is not an FMA-initiated panic). On receipt of the initial
73 * "dump_available" event we also arm a timer for +10 minutes if
74 * dumpadm is enabled - if no "dump_available" or "savecore_failed" arrives
75 * in that time we will solve the case on timeout.
76 *
77 * When the timer fires we check whether the initial event for each panic
78 * case was received more than 30 minutes ago; if it was we solve the case
79 * with what we have. If we're still within the waiting period we rearm
80 * for a further 10 minutes. The timer is shared by all cases that we
81 * create, which is why the fire interval is shorter than the maximum time
82 * we are prepared to wait.
83 */
84
85 #include <strings.h>
86 #include <sys/panic.h>
87 #include <alloca.h>
88 #include <zone.h>
89
90 #include "../../common/sw.h"
91 #include "panic.h"
92
93 #define MAX_STRING_LEN 160
94
95 static id_t myid;
96
97 static id_t mytimerid;
98
99 /*
100 * Our serialization structure type.
101 */
102 #define SWDE_PANIC_CASEDATA_VERS 1
103
104 typedef struct swde_panic_casedata {
105 uint32_t scd_vers; /* must be first member */
106 uint64_t scd_receive_time; /* when we first knew of this panic */
107 size_t scd_nvlbufsz; /* size of following buffer */
108 /* packed attr nvlist follows */
109 } swde_panic_casedata_t;
110
111 static struct {
112 fmd_stat_t swde_panic_diagnosed;
113 fmd_stat_t swde_panic_badclass;
114 fmd_stat_t swde_panic_noattr;
115 fmd_stat_t swde_panic_unexpected_fm_panic;
116 fmd_stat_t swde_panic_badattr;
117 fmd_stat_t swde_panic_badfmri;
118 fmd_stat_t swde_panic_noinstance;
119 fmd_stat_t swde_panic_nouuid;
120 fmd_stat_t swde_panic_dupuuid;
121 fmd_stat_t swde_panic_nocase;
122 fmd_stat_t swde_panic_notime;
123 fmd_stat_t swde_panic_nopanicstr;
124 fmd_stat_t swde_panic_nodumpdir;
125 fmd_stat_t swde_panic_nostack;
126 fmd_stat_t swde_panic_incomplete;
127 fmd_stat_t swde_panic_failed;
128 fmd_stat_t swde_panic_basecasedata;
129 fmd_stat_t swde_panic_failsrlz;
130 } swde_panic_stats = {
131 { "swde_panic_diagnosed", FMD_TYPE_UINT64,
132 "panic defects published" },
133 { "swde_panic_badclass", FMD_TYPE_UINT64,
134 "incorrect event class received" },
135 { "swde_panic_noattr", FMD_TYPE_UINT64,
136 "malformed event - missing attr nvlist" },
137 { "swde_panic_unexpected_fm_panic", FMD_TYPE_UINT64,
138 "dump available for an fm_panic()" },
139 { "swde_panic_badattr", FMD_TYPE_UINT64,
140 "malformed event - invalid attr list" },
141 { "swde_panic_badfmri", FMD_TYPE_UINT64,
142 "malformed event - fmri2str fails" },
143 { "swde_panic_noinstance", FMD_TYPE_UINT64,
144 "malformed event - no instance number" },
145 { "swde_panic_nouuid", FMD_TYPE_UINT64,
146 "malformed event - missing uuid" },
147 { "swde_panic_dupuuid", FMD_TYPE_UINT64,
148 "duplicate events received" },
149 { "swde_panic_nocase", FMD_TYPE_UINT64,
150 "case missing for uuid" },
151 { "swde_panic_notime", FMD_TYPE_UINT64,
152 "missing crash dump time" },
153 { "swde_panic_nopanicstr", FMD_TYPE_UINT64,
154 "missing panic string" },
155 { "swde_panic_nodumpdir", FMD_TYPE_UINT64,
156 "missing crashdump save directory" },
157 { "swde_panic_nostack", FMD_TYPE_UINT64,
158 "missing panic stack" },
159 { "swde_panic_incomplete", FMD_TYPE_UINT64,
160 "missing panic incomplete" },
161 { "swde_panic_failed", FMD_TYPE_UINT64,
162 "missing panic failed" },
163 { "swde_panic_badcasedata", FMD_TYPE_UINT64,
164 "bad case data during timeout" },
165 { "swde_panic_failsrlz", FMD_TYPE_UINT64,
166 "failures to serialize case data" },
167 };
168
169 #define BUMPSTAT(stat) swde_panic_stats.stat.fmds_value.ui64++
170
171 static nvlist_t *
172 panic_sw_fmri(fmd_hdl_t *hdl, char *object)
173 {
174 nvlist_t *fmri;
175 nvlist_t *sw_obj;
176 int err = 0;
177
178 fmri = fmd_nvl_alloc(hdl, FMD_SLEEP);
179 err |= nvlist_add_uint8(fmri, FM_VERSION, FM_SW_SCHEME_VERSION);
180 err |= nvlist_add_string(fmri, FM_FMRI_SCHEME, FM_FMRI_SCHEME_SW);
181
182 sw_obj = fmd_nvl_alloc(hdl, FMD_SLEEP);
183 err |= nvlist_add_string(sw_obj, FM_FMRI_SW_OBJ_PATH, object);
184 err |= nvlist_add_nvlist(fmri, FM_FMRI_SW_OBJ, sw_obj);
185 nvlist_free(sw_obj);
186 if (!err)
187 return (fmri);
188 else
189 return (0);
190 }
191
192 static const char *dumpfiles[2] = { "unix.%lld", "vmcore.%lld" };
193 static const char *dumpfiles_comp[2] = { "vmdump.%lld", NULL};
194
195 static void
196 swde_panic_solve(fmd_hdl_t *hdl, fmd_case_t *cp,
197 nvlist_t *attr, fmd_event_t *ep, boolean_t savecore_success)
198 {
199 char *dumpdir, *path, *uuid;
200 nvlist_t *defect, *rsrc;
201 nvpair_t *nvp;
202 int i;
203
204 /*
205 * Attribute members to include in event-specific defect
206 * payload. Some attributes will not be present for some
207 * cases - e.g., if we timed out and solved the case without
208 * a "dump_available" report.
209 */
210 const char *toadd[] = {
211 "os-instance-uuid", /* same as case uuid */
212 "panicstr", /* for initial classification work */
213 "panicstack", /* for initial classification work */
214 "crashtime", /* in epoch time */
215 "panic-time", /* Formatted crash time */
216 };
217
218 if (ep != NULL)
219 fmd_case_add_ereport(hdl, cp, ep);
220 /*
221 * As a temporary solution we create and fmri in the sw scheme
222 * in panic_sw_fmri. This should become a generic fmri constructor
223 *
224 * We need to user a resource FMRI which will have a sufficiently
225 * unique string representation such that fmd will not see
226 * repeated panic diagnoses (all using the same defect class)
227 * as duplicates and discard later cases. We can't actually diagnose
228 * the panic to anything specific (e.g., a path to a module and
229 * function/line etc therein). We could pick on a generic
230 * representative such as /kernel/genunix but that could lead
231 * to misunderstanding. So we choose a path based on <dumpdir>
232 * and the OS instance UUID - "<dumpdir>/.<os-instance-uuid>".
233 * There's no file at that path (*) but no matter. We can't use
234 * <dumpdir>/vmdump.N or similar because if savecore is disabled
235 * or failed we don't have any file or instance number.
236 *
237 * (*) Some day it would seem tidier to keep all files to do
238 * with a single crash (unix/vmcore/vmdump, analysis output etc)
239 * in a distinct directory, and <dumpdir>/.<uuid> seems like a good
240 * choice. For compatability we'd symlink into it. So that is
241 * another reason for this choice - some day it may exist!
242 */
243 (void) nvlist_lookup_string(attr, "dumpdir", &dumpdir);
244 (void) nvlist_lookup_string(attr, "os-instance-uuid", &uuid);
245 path = alloca(strlen(dumpdir) + 1 + 1 + 36 + 1);
246 /* LINTED: E_SEC_SPRINTF_UNBOUNDED_COPY */
247 (void) sprintf(path, "%s/.%s", dumpdir, uuid);
248 rsrc = panic_sw_fmri(hdl, path);
249
250 defect = fmd_nvl_create_defect(hdl, SW_SUNOS_PANIC_DEFECT,
251 100, rsrc, NULL, rsrc);
252 nvlist_free(rsrc);
253
254 (void) nvlist_add_boolean_value(defect, "savecore-succcess",
255 savecore_success);
256
257 if (savecore_success) {
258 boolean_t compressed;
259 int64_t instance;
260 const char **pathfmts;
261 char buf[2][32];
262 int files = 0;
263 char *arr[2];
264 int i;
265
266 (void) nvlist_lookup_int64(attr, "instance", &instance);
267 (void) nvlist_lookup_boolean_value(attr, "compressed",
268 &compressed);
269
270 pathfmts = compressed ? &dumpfiles_comp[0] : &dumpfiles[0];
271
272 for (i = 0; i < 2; i++) {
273 if (pathfmts[i] == NULL) {
274 arr[i] = NULL;
275 continue;
276 }
277
278 (void) snprintf(buf[i], 32, pathfmts[i], instance);
279 arr[i] = buf[i];
280 files++;
281 }
282
283 (void) nvlist_add_string(defect, "dump-dir", dumpdir);
284 (void) nvlist_add_string_array(defect, "dump-files", arr,
285 files);
286 } else {
287 char *rsn;
288
289 if (nvlist_lookup_string(attr, "failure-reason", &rsn) == 0)
290 (void) nvlist_add_string(defect, "failure-reason", rsn);
291 }
292
293 /*
294 * Not all attributes will necessarily be available - eg if
295 * dumpadm was not enabled there'll be no instance and dumpdir.
296 */
297 for (i = 0; i < sizeof (toadd) / sizeof (toadd[0]); i++) {
298 if (nvlist_lookup_nvpair(attr, toadd[i], &nvp) == 0)
299 (void) nvlist_add_nvpair(defect, nvp);
300 }
301
302 fmd_case_add_suspect(hdl, cp, defect);
303 fmd_case_solve(hdl, cp);
304
305 /*
306 * Close the case. Do no free casedata - framework does that for us
307 * on closure callback.
308 */
309 fmd_case_close(hdl, cp);
310 BUMPSTAT(swde_panic_diagnosed);
311 }
312
313 /*ARGSUSED*/
314 static void
315 swde_panic_timeout(fmd_hdl_t *hdl, id_t timerid, void *data)
316 {
317 fmd_case_t *cp = swde_case_first(hdl, myid);
318 swde_panic_casedata_t *cdp;
319 time_t now = time(NULL);
320 nvlist_t *attr;
321 int remain = 0;
322 uint32_t vers;
323
324 while (cp != NULL) {
325 cdp = swde_case_data(hdl, cp, &vers);
326 if (vers != SWDE_PANIC_CASEDATA_VERS)
327 fmd_hdl_abort(hdl, "case data version confused\n");
328
329 if (now > cdp->scd_receive_time + 30 * 60) {
330 if (nvlist_unpack((char *)cdp + sizeof (*cdp),
331 cdp->scd_nvlbufsz, &attr, 0) == 0) {
332 swde_panic_solve(hdl, cp, attr, NULL, B_FALSE);
333 nvlist_free(attr);
334 } else {
335 BUMPSTAT(swde_panic_basecasedata);
336 fmd_case_close(hdl, cp);
337 }
338 } else {
339 remain++;
340 }
341
342
343 cp = swde_case_next(hdl, cp);
344 }
345
346 if (remain) {
347 mytimerid = sw_timer_install(hdl, myid, NULL, NULL,
348 10ULL * NANOSEC * 60);
349 }
350 }
351
352 /*
353 * Our verify entry point is called for each of our open cases during
354 * module load. We must return 0 for the case to be closed by our caller,
355 * or 1 to keep it (or if we have already closed it during this call).
356 */
357 static int
358 swde_panic_vrfy(fmd_hdl_t *hdl, fmd_case_t *cp)
359 {
360 swde_panic_casedata_t *cdp;
361 time_t now = time(NULL);
362 nvlist_t *attr;
363 uint32_t vers;
364
365 cdp = swde_case_data(hdl, cp, &vers);
366
367 if (vers != SWDE_PANIC_CASEDATA_VERS)
368 return (0); /* case will be closed */
369
370 if (now > cdp->scd_receive_time + 30 * 60) {
371 if (nvlist_unpack((char *)cdp + sizeof (*cdp),
372 cdp->scd_nvlbufsz, &attr, 0) == 0) {
373 swde_panic_solve(hdl, cp, attr, NULL, B_FALSE);
374 nvlist_free(attr);
375 return (1); /* case already closed */
376 } else {
377 return (0); /* close case */
378 }
379 }
380
381 if (mytimerid != 0)
382 mytimerid = sw_timer_install(hdl, myid,
383 NULL, NULL, 10ULL * NANOSEC * 60);
384
385 return (1); /* retain case */
386 }
387
388 /*
389 * Handler for ireport.os.sunos.panic.dump_pending_on_device.
390 *
391 * A future RFE should try adding a means of avoiding diagnosing repeated
392 * defects on panic loops, which would just add to the mayhem and potentially
393 * log lots of calls through ASR. Panics with similar enough panic
394 * strings and/or stacks should not diagnose to new defects with some
395 * period of time, for example.
396 */
397
398 /*ARGSUSED*/
399 void
400 swde_panic_detected(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
401 const char *class, void *arg)
402 {
403 boolean_t fm_panic, expect_savecore;
404 swde_panic_casedata_t *cdp;
405 nvlist_t *attr;
406 fmd_case_t *cp;
407 char *fmribuf;
408 char *uuid;
409 size_t sz;
410
411 fmd_hdl_debug(hdl, "swde_panic_detected\n");
412
413 if (nvlist_lookup_nvlist(nvl, FM_IREPORT_ATTRIBUTES, &attr) != 0) {
414 BUMPSTAT(swde_panic_noattr);
415 return;
416 }
417
418 if (nvlist_lookup_string(attr, "os-instance-uuid", &uuid) != 0) {
419 BUMPSTAT(swde_panic_nouuid);
420 return;
421 }
422
423 fmd_hdl_debug(hdl, "swde_panic_detected: OS instance %s\n", uuid);
424
425 if (nvlist_lookup_boolean_value(attr, "fm-panic", &fm_panic) != 0 ||
426 fm_panic == B_TRUE) {
427 BUMPSTAT(swde_panic_unexpected_fm_panic);
428 return;
429 }
430
431 /*
432 * Prepare serialization data to be associated with a new
433 * case. Our serialization data consists of a swde_panic_casedata_t
434 * structure followed by a packed nvlist of the attributes of
435 * the initial event.
436 */
437 if (nvlist_size(attr, &sz, NV_ENCODE_NATIVE) != 0) {
438 BUMPSTAT(swde_panic_failsrlz);
439 return;
440 }
441
442 cdp = fmd_hdl_zalloc(hdl, sizeof (*cdp) + sz, FMD_SLEEP);
443 fmribuf = (char *)cdp + sizeof (*cdp);
444 cdp->scd_vers = SWDE_PANIC_CASEDATA_VERS;
445 cdp->scd_receive_time = time(NULL);
446 cdp->scd_nvlbufsz = sz;
447
448 /*
449 * Open a case with UUID matching the the panicking kernel, add this
450 * event to the case.
451 */
452 if ((cp = swde_case_open(hdl, myid, uuid, SWDE_PANIC_CASEDATA_VERS,
453 cdp, sizeof (*cdp) + sz)) == NULL) {
454 BUMPSTAT(swde_panic_dupuuid);
455 fmd_hdl_debug(hdl, "swde_case_open returned NULL - dup?\n");
456 fmd_hdl_free(hdl, cdp, sizeof (*cdp) + sz);
457 return;
458 }
459
460 fmd_case_setprincipal(hdl, cp, ep);
461
462 if (nvlist_lookup_boolean_value(attr, "will-attempt-savecore",
463 &expect_savecore) != 0 || expect_savecore == B_FALSE) {
464 fmd_hdl_debug(hdl, "savecore not being attempted - "
465 "solve now\n");
466 swde_panic_solve(hdl, cp, attr, ep, B_FALSE);
467 return;
468 }
469
470 /*
471 * We expect to see either a "dump_available" or a "savecore_failed"
472 * event before too long. In case that never shows up, for whatever
473 * reason, we want to be able to solve the case anyway.
474 */
475 fmd_case_add_ereport(hdl, cp, ep);
476 (void) nvlist_pack(attr, &fmribuf, &sz, NV_ENCODE_NATIVE, 0);
477 swde_case_data_write(hdl, cp);
478
479 if (mytimerid == 0) {
480 mytimerid = sw_timer_install(hdl, myid, NULL, ep,
481 10ULL * NANOSEC * 60);
482 fmd_hdl_debug(hdl, "armed timer\n");
483 } else {
484 fmd_hdl_debug(hdl, "timer already armed\n");
485 }
486 }
487
488 /*
489 * savecore has now run and saved a crash dump to the filesystem. It is
490 * either a compressed dump (vmdump.n) or uncompressed {unix.n, vmcore.n}
491 * Savecore has raised an ireport to say the dump is there.
492 */
493
494 /*ARGSUSED*/
495 void
496 swde_panic_savecore_done(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
497 const char *class, void *arg)
498 {
499 boolean_t savecore_success = (arg != NULL);
500 boolean_t fm_panic;
501 nvlist_t *attr;
502 fmd_case_t *cp;
503 char *uuid;
504
505 fmd_hdl_debug(hdl, "savecore_done (%s)\n", savecore_success ?
506 "success" : "fail");
507
508 if (nvlist_lookup_nvlist(nvl, FM_IREPORT_ATTRIBUTES, &attr) != 0) {
509 BUMPSTAT(swde_panic_noattr);
510 return;
511 }
512
513 if (nvlist_lookup_boolean_value(attr, "fm-panic", &fm_panic) != 0 ||
514 fm_panic == B_TRUE) {
515 return; /* not expected, but just in case */
516 }
517
518 if (nvlist_lookup_string(attr, "os-instance-uuid", &uuid) != 0) {
519 BUMPSTAT(swde_panic_nouuid);
520 return;
521 }
522
523 /*
524 * Find the case related to the panicking kernel; our cases have
525 * the same uuid as the crashed OS image.
526 */
527 cp = fmd_case_uulookup(hdl, uuid);
528 if (!cp) {
529 /* Unable to find the case. */
530 fmd_hdl_debug(hdl, "savecore_done: can't find case for "
531 "image %s\n", uuid);
532 BUMPSTAT(swde_panic_nocase);
533 return;
534 }
535
536 fmd_hdl_debug(hdl, "savecore_done: solving case %s\n", uuid);
537 swde_panic_solve(hdl, cp, attr, ep, savecore_success);
538 }
539
540 const struct sw_disp swde_panic_disp[] = {
541 { SW_SUNOS_PANIC_DETECTED, swde_panic_detected, NULL },
542 { SW_SUNOS_PANIC_AVAIL, swde_panic_savecore_done, (void *)1 },
543 { SW_SUNOS_PANIC_FAILURE, swde_panic_savecore_done, NULL },
544 /*
545 * Something has to subscribe to every fault
546 * or defect diagnosed in fmd. We do that here, but throw it away.
547 */
548 { SW_SUNOS_PANIC_DEFECT, NULL, NULL },
549 { NULL, NULL, NULL }
550 };
551
552 /*ARGSUSED*/
553 int
554 swde_panic_init(fmd_hdl_t *hdl, id_t id, const struct sw_disp **dpp,
555 int *nelemp)
556 {
557 myid = id;
558
559 if (getzoneid() != GLOBAL_ZONEID)
560 return (SW_SUB_INIT_FAIL_VOLUNTARY);
561
562 (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC,
563 sizeof (swde_panic_stats) / sizeof (fmd_stat_t),
564 (fmd_stat_t *)&swde_panic_stats);
565
566 fmd_hdl_subscribe(hdl, SW_SUNOS_PANIC_DETECTED);
567 fmd_hdl_subscribe(hdl, SW_SUNOS_PANIC_FAILURE);
568 fmd_hdl_subscribe(hdl, SW_SUNOS_PANIC_AVAIL);
569
570 *dpp = &swde_panic_disp[0];
571 *nelemp = sizeof (swde_panic_disp) / sizeof (swde_panic_disp[0]);
572 return (SW_SUB_INIT_SUCCESS);
573 }
574
575 void
576 swde_panic_fini(fmd_hdl_t *hdl)
577 {
578 if (mytimerid)
579 sw_timer_remove(hdl, myid, mytimerid);
580 }
581
582 const struct sw_subinfo panic_diag_info = {
583 "panic diagnosis", /* swsub_name */
584 SW_CASE_PANIC, /* swsub_casetype */
585 swde_panic_init, /* swsub_init */
586 swde_panic_fini, /* swsub_fini */
587 swde_panic_timeout, /* swsub_timeout */
588 NULL, /* swsub_case_close */
589 swde_panic_vrfy, /* swsub_case_vrfy */
590 };