1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 #include <sys/sysmacros.h>
  26 #include <sys/types.h>
  27 #include <sys/kmem.h>
  28 #include <sys/modctl.h>
  29 #include <sys/ddi.h>
  30 #include <sys/sunddi.h>
  31 #include <sys/sunndi.h>
  32 #include <sys/fm/protocol.h>
  33 #include <sys/fm/util.h>
  34 #include <sys/fm/io/ddi.h>
  35 #include <sys/fm/io/pci.h>
  36 #include <sys/promif.h>
  37 #include <sys/disp.h>
  38 #include <sys/atomic.h>
  39 #include <sys/pcie.h>
  40 #include <sys/pci_cap.h>
  41 #include <sys/pcie_impl.h>
  42 
  43 #define PF_PCIE_BDG_ERR (PCIE_DEVSTS_FE_DETECTED | PCIE_DEVSTS_NFE_DETECTED | \
  44         PCIE_DEVSTS_CE_DETECTED)
  45 
  46 #define PF_PCI_BDG_ERR (PCI_STAT_S_SYSERR | PCI_STAT_S_TARG_AB | \
  47         PCI_STAT_R_MAST_AB | PCI_STAT_R_TARG_AB | PCI_STAT_S_PERROR)
  48 
  49 #define PF_AER_FATAL_ERR (PCIE_AER_UCE_DLP | PCIE_AER_UCE_SD |\
  50         PCIE_AER_UCE_FCP | PCIE_AER_UCE_RO | PCIE_AER_UCE_MTLP)
  51 #define PF_AER_NON_FATAL_ERR (PCIE_AER_UCE_PTLP | PCIE_AER_UCE_TO | \
  52         PCIE_AER_UCE_CA | PCIE_AER_UCE_ECRC | PCIE_AER_UCE_UR)
  53 
  54 #define PF_SAER_FATAL_ERR (PCIE_AER_SUCE_USC_MSG_DATA_ERR | \
  55         PCIE_AER_SUCE_UC_ATTR_ERR | PCIE_AER_SUCE_UC_ADDR_ERR | \
  56         PCIE_AER_SUCE_SERR_ASSERT)
  57 #define PF_SAER_NON_FATAL_ERR (PCIE_AER_SUCE_TA_ON_SC | \
  58         PCIE_AER_SUCE_MA_ON_SC | PCIE_AER_SUCE_RCVD_TA | \
  59         PCIE_AER_SUCE_RCVD_MA | PCIE_AER_SUCE_USC_ERR | \
  60         PCIE_AER_SUCE_UC_DATA_ERR | PCIE_AER_SUCE_TIMER_EXPIRED | \
  61         PCIE_AER_SUCE_PERR_ASSERT | PCIE_AER_SUCE_INTERNAL_ERR)
  62 
  63 #define PF_PCI_PARITY_ERR (PCI_STAT_S_PERROR | PCI_STAT_PERROR)
  64 
  65 #define PF_FIRST_AER_ERR(bit, adv) \
  66         (bit & (1 << (adv->pcie_adv_ctl & PCIE_AER_CTL_FST_ERR_PTR_MASK)))
  67 
  68 #define HAS_AER_LOGS(pfd_p, bit) \
  69         (PCIE_HAS_AER(pfd_p->pe_bus_p) && \
  70         PF_FIRST_AER_ERR(bit, PCIE_ADV_REG(pfd_p)))
  71 
  72 #define PF_FIRST_SAER_ERR(bit, adv) \
  73         (bit & (1 << (adv->pcie_sue_ctl & PCIE_AER_SCTL_FST_ERR_PTR_MASK)))
  74 
  75 #define HAS_SAER_LOGS(pfd_p, bit) \
  76         (PCIE_HAS_AER(pfd_p->pe_bus_p) && \
  77         PF_FIRST_SAER_ERR(bit, PCIE_ADV_BDG_REG(pfd_p)))
  78 
  79 #define GET_SAER_CMD(pfd_p) \
  80         ((PCIE_ADV_BDG_HDR(pfd_p, 1) >> \
  81         PCIE_AER_SUCE_HDR_CMD_LWR_SHIFT) & PCIE_AER_SUCE_HDR_CMD_LWR_MASK)
  82 
  83 #define CE_ADVISORY(pfd_p) \
  84         (PCIE_ADV_REG(pfd_p)->pcie_ce_status & PCIE_AER_CE_AD_NFE)
  85 
  86 /* PCIe Fault Fabric Error analysis table */
  87 typedef struct pf_fab_err_tbl {
  88         uint32_t        bit;            /* Error bit */
  89         int             (*handler)();   /* Error handling fuction */
  90         uint16_t        affected_flags; /* Primary affected flag */
  91         /*
  92          * Secondary affected flag, effective when the information
  93          * indicated by the primary flag is not available, eg.
  94          * PF_AFFECTED_AER/SAER/ADDR
  95          */
  96         uint16_t        sec_affected_flags;
  97 } pf_fab_err_tbl_t;
  98 
  99 static pcie_bus_t *pf_is_ready(dev_info_t *);
 100 /* Functions for scanning errors */
 101 static int pf_default_hdl(dev_info_t *, pf_impl_t *);
 102 static int pf_dispatch(dev_info_t *, pf_impl_t *, boolean_t);
 103 static boolean_t pf_in_addr_range(pcie_bus_t *, uint64_t);
 104 
 105 /* Functions for gathering errors */
 106 static void pf_pcix_ecc_regs_gather(pf_pcix_ecc_regs_t *pcix_ecc_regs,
 107     pcie_bus_t *bus_p, boolean_t bdg);
 108 static void pf_pcix_regs_gather(pf_data_t *pfd_p, pcie_bus_t *bus_p);
 109 static void pf_pcie_regs_gather(pf_data_t *pfd_p, pcie_bus_t *bus_p);
 110 static void pf_pci_regs_gather(pf_data_t *pfd_p, pcie_bus_t *bus_p);
 111 static int pf_dummy_cb(dev_info_t *, ddi_fm_error_t *, const void *);
 112 static void pf_en_dq(pf_data_t *pfd_p, pf_impl_t *impl_p);
 113 
 114 /* Functions for analysing errors */
 115 static int pf_analyse_error(ddi_fm_error_t *, pf_impl_t *);
 116 static void pf_adjust_for_no_aer(pf_data_t *);
 117 static void pf_adjust_for_no_saer(pf_data_t *);
 118 static pf_data_t *pf_get_pcie_bridge(pf_data_t *, pcie_req_id_t);
 119 static pf_data_t *pf_get_parent_pcie_bridge(pf_data_t *);
 120 static boolean_t pf_matched_in_rc(pf_data_t *, pf_data_t *,
 121     uint32_t);
 122 static int pf_analyse_error_tbl(ddi_fm_error_t *, pf_impl_t *,
 123     pf_data_t *, const pf_fab_err_tbl_t *, uint32_t);
 124 static int pf_analyse_ca_ur(ddi_fm_error_t *, uint32_t,
 125     pf_data_t *, pf_data_t *);
 126 static int pf_analyse_ma_ta(ddi_fm_error_t *, uint32_t,
 127     pf_data_t *, pf_data_t *);
 128 static int pf_analyse_pci(ddi_fm_error_t *, uint32_t,
 129     pf_data_t *, pf_data_t *);
 130 static int pf_analyse_perr_assert(ddi_fm_error_t *, uint32_t,
 131     pf_data_t *, pf_data_t *);
 132 static int pf_analyse_ptlp(ddi_fm_error_t *, uint32_t,
 133     pf_data_t *, pf_data_t *);
 134 static int pf_analyse_sc(ddi_fm_error_t *, uint32_t,
 135     pf_data_t *, pf_data_t *);
 136 static int pf_analyse_to(ddi_fm_error_t *, uint32_t,
 137     pf_data_t *, pf_data_t *);
 138 static int pf_analyse_uc(ddi_fm_error_t *, uint32_t,
 139     pf_data_t *, pf_data_t *);
 140 static int pf_analyse_uc_data(ddi_fm_error_t *, uint32_t,
 141     pf_data_t *, pf_data_t *);
 142 static int pf_no_panic(ddi_fm_error_t *, uint32_t,
 143     pf_data_t *, pf_data_t *);
 144 static int pf_panic(ddi_fm_error_t *, uint32_t,
 145     pf_data_t *, pf_data_t *);
 146 static void pf_send_ereport(ddi_fm_error_t *, pf_impl_t *);
 147 static int pf_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr);
 148 
 149 /* PCIe Fabric Handle Lookup Support Functions. */
 150 static int pf_hdl_child_lookup(dev_info_t *, ddi_fm_error_t *, uint32_t,
 151     uint64_t, pcie_req_id_t);
 152 static int pf_hdl_compare(dev_info_t *, ddi_fm_error_t *, uint32_t, uint64_t,
 153     pcie_req_id_t, ndi_fmc_t *);
 154 static int pf_log_hdl_lookup(dev_info_t *, ddi_fm_error_t *, pf_data_t *,
 155         boolean_t);
 156 
 157 static int pf_handler_enter(dev_info_t *, pf_impl_t *);
 158 static void pf_handler_exit(dev_info_t *);
 159 static void pf_reset_pfd(pf_data_t *);
 160 
 161 boolean_t pcie_full_scan = B_FALSE;     /* Force to always do a full scan */
 162 int pcie_disable_scan = 0;              /* Disable fabric scan */
 163 
 164 /* Inform interested parties that error handling is about to begin. */
 165 /* ARGSUSED */
 166 void
 167 pf_eh_enter(pcie_bus_t *bus_p) {
 168 }
 169 
 170 /* Inform interested parties that error handling has ended. */
 171 void
 172 pf_eh_exit(pcie_bus_t *bus_p)
 173 {
 174         pcie_bus_t *rbus_p = PCIE_DIP2BUS(bus_p->bus_rp_dip);
 175         pf_data_t *root_pfd_p = PCIE_BUS2PFD(rbus_p);
 176         pf_data_t *pfd_p;
 177         uint_t intr_type = PCIE_ROOT_EH_SRC(root_pfd_p)->intr_type;
 178 
 179         pciev_eh_exit(root_pfd_p, intr_type);
 180 
 181         /* Clear affected device info and INTR SRC */
 182         for (pfd_p = root_pfd_p; pfd_p; pfd_p = pfd_p->pe_next) {
 183                 PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags = 0;
 184                 PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = PCIE_INVALID_BDF;
 185                 if (PCIE_IS_ROOT(PCIE_PFD2BUS(pfd_p))) {
 186                         PCIE_ROOT_EH_SRC(pfd_p)->intr_type = PF_INTR_TYPE_NONE;
 187                         PCIE_ROOT_EH_SRC(pfd_p)->intr_data = NULL;
 188                 }
 189         }
 190 }
 191 
 192 /*
 193  * Scan Fabric is the entry point for PCI/PCIe IO fabric errors.  The
 194  * caller may create a local pf_data_t with the "root fault"
 195  * information populated to either do a precise or full scan.  More
 196  * than one pf_data_t maybe linked together if there are multiple
 197  * errors.  Only a PCIe compliant Root Port device may pass in NULL
 198  * for the root_pfd_p.
 199  *
 200  * "Root Complexes" such as NPE and PX should call scan_fabric using itself as
 201  * the rdip.  PCIe Root ports should call pf_scan_fabric using it's parent as
 202  * the rdip.
 203  *
 204  * Scan fabric initiated from RCs are likely due to a fabric message, traps or
 205  * any RC detected errors that propagated to/from the fabric.
 206  *
 207  * This code assumes that by the time pf_scan_fabric is
 208  * called, pf_handler_enter has NOT been called on the rdip.
 209  */
 210 int
 211 pf_scan_fabric(dev_info_t *rdip, ddi_fm_error_t *derr, pf_data_t *root_pfd_p)
 212 {
 213         pf_impl_t       impl;
 214         pf_data_t       *pfd_p, *pfd_head_p, *pfd_tail_p;
 215         int             scan_flag = PF_SCAN_SUCCESS;
 216         int             analyse_flag = PF_ERR_NO_ERROR;
 217         boolean_t       full_scan = pcie_full_scan;
 218 
 219         if (pcie_disable_scan)
 220                 return (analyse_flag);
 221 
 222         /* Find the head and tail of this link list */
 223         pfd_head_p = root_pfd_p;
 224         for (pfd_tail_p = root_pfd_p; pfd_tail_p && pfd_tail_p->pe_next;
 225             pfd_tail_p = pfd_tail_p->pe_next)
 226                 ;
 227 
 228         /* Save head/tail */
 229         impl.pf_total = 0;
 230         impl.pf_derr = derr;
 231         impl.pf_dq_head_p = pfd_head_p;
 232         impl.pf_dq_tail_p = pfd_tail_p;
 233 
 234         /* If scan is initiated from RP then RP itself must be scanned. */
 235         if (PCIE_IS_RP(PCIE_DIP2BUS(rdip)) && pf_is_ready(rdip) &&
 236             !root_pfd_p) {
 237                 scan_flag = pf_handler_enter(rdip, &impl);
 238                 if (scan_flag & PF_SCAN_DEADLOCK)
 239                         goto done;
 240 
 241                 scan_flag = pf_default_hdl(rdip, &impl);
 242                 if (scan_flag & PF_SCAN_NO_ERR_IN_CHILD)
 243                         goto done;
 244         }
 245 
 246         /*
 247          * Scan the fabric using the scan_bdf and scan_addr in error q.
 248          * scan_bdf will be valid in the following cases:
 249          *      - Fabric message
 250          *      - Poisoned TLP
 251          *      - Signaled UR/CA
 252          *      - Received UR/CA
 253          *      - PIO load failures
 254          */
 255         for (pfd_p = impl.pf_dq_head_p; pfd_p && PFD_IS_ROOT(pfd_p);
 256             pfd_p = pfd_p->pe_next) {
 257                 impl.pf_fault = PCIE_ROOT_FAULT(pfd_p);
 258 
 259                 if (PFD_IS_RC(pfd_p))
 260                         impl.pf_total++;
 261 
 262                 if (impl.pf_fault->full_scan)
 263                         full_scan = B_TRUE;
 264 
 265                 if (full_scan ||
 266                     PCIE_CHECK_VALID_BDF(impl.pf_fault->scan_bdf) ||
 267                     impl.pf_fault->scan_addr)
 268                         scan_flag |= pf_dispatch(rdip, &impl, full_scan);
 269 
 270                 if (full_scan)
 271                         break;
 272         }
 273 
 274 done:
 275         /*
 276          * If this is due to safe access, don't analyze the errors and return
 277          * success regardless of how scan fabric went.
 278          */
 279         if (derr->fme_flag != DDI_FM_ERR_UNEXPECTED) {
 280                 analyse_flag = PF_ERR_NO_PANIC;
 281         } else {
 282                 analyse_flag = pf_analyse_error(derr, &impl);
 283         }
 284 
 285         pf_send_ereport(derr, &impl);
 286 
 287         /*
 288          * Check if any hardened driver's callback reported a panic.
 289          * If so panic.
 290          */
 291         if (scan_flag & PF_SCAN_CB_FAILURE)
 292                 analyse_flag |= PF_ERR_PANIC;
 293 
 294         /*
 295          * If a deadlock was detected, panic the system as error analysis has
 296          * been compromised.
 297          */
 298         if (scan_flag & PF_SCAN_DEADLOCK)
 299                 analyse_flag |= PF_ERR_PANIC_DEADLOCK;
 300 
 301         derr->fme_status = PF_ERR2DDIFM_ERR(scan_flag);
 302 
 303         return (analyse_flag);
 304 }
 305 
 306 void
 307 pcie_force_fullscan() {
 308         pcie_full_scan = B_TRUE;
 309 }
 310 
 311 /*
 312  * pf_dispatch walks the device tree and calls the pf_default_hdl if the device
 313  * falls in the error path.
 314  *
 315  * Returns PF_SCAN_* flags
 316  */
 317 static int
 318 pf_dispatch(dev_info_t *pdip, pf_impl_t *impl, boolean_t full_scan)
 319 {
 320         dev_info_t      *dip;
 321         pcie_req_id_t   rid = impl->pf_fault->scan_bdf;
 322         pcie_bus_t      *bus_p;
 323         int             scan_flag = PF_SCAN_SUCCESS;
 324 
 325         for (dip = ddi_get_child(pdip); dip; dip = ddi_get_next_sibling(dip)) {
 326                 /* Make sure dip is attached and ready */
 327                 if (!(bus_p = pf_is_ready(dip)))
 328                         continue;
 329 
 330                 scan_flag |= pf_handler_enter(dip, impl);
 331                 if (scan_flag & PF_SCAN_DEADLOCK)
 332                         break;
 333 
 334                 /*
 335                  * Handle this device if it is a:
 336                  * o Full Scan
 337                  * o PCI/PCI-X Device
 338                  * o Fault BDF = Device BDF
 339                  * o BDF/ADDR is in range of the Bridge/Switch
 340                  */
 341                 if (full_scan ||
 342                     (bus_p->bus_bdf == rid) ||
 343                     pf_in_bus_range(bus_p, rid) ||
 344                     pf_in_addr_range(bus_p, impl->pf_fault->scan_addr)) {
 345                         int hdl_flag = pf_default_hdl(dip, impl);
 346                         scan_flag |= hdl_flag;
 347 
 348                         /*
 349                          * A bridge may have detected no errors in which case
 350                          * there is no need to scan further down.
 351                          */
 352                         if (hdl_flag & PF_SCAN_NO_ERR_IN_CHILD)
 353                                 continue;
 354                 } else {
 355                         pf_handler_exit(dip);
 356                         continue;
 357                 }
 358 
 359                 /* match or in bridge bus-range */
 360                 switch (bus_p->bus_dev_type) {
 361                 case PCIE_PCIECAP_DEV_TYPE_PCIE2PCI:
 362                 case PCIE_PCIECAP_DEV_TYPE_PCI2PCIE:
 363                         scan_flag |= pf_dispatch(dip, impl, B_TRUE);
 364                         break;
 365                 case PCIE_PCIECAP_DEV_TYPE_UP:
 366                 case PCIE_PCIECAP_DEV_TYPE_DOWN:
 367                 case PCIE_PCIECAP_DEV_TYPE_ROOT:
 368                 {
 369                         pf_data_t *pfd_p = PCIE_BUS2PFD(bus_p);
 370                         pf_pci_err_regs_t *err_p = PCI_ERR_REG(pfd_p);
 371                         pf_pci_bdg_err_regs_t *serr_p = PCI_BDG_ERR_REG(pfd_p);
 372                         /*
 373                          * Continue if the fault BDF != the switch or there is a
 374                          * parity error
 375                          */
 376                         if ((bus_p->bus_bdf != rid) ||
 377                             (err_p->pci_err_status & PF_PCI_PARITY_ERR) ||
 378                             (serr_p->pci_bdg_sec_stat & PF_PCI_PARITY_ERR))
 379                                 scan_flag |= pf_dispatch(dip, impl, full_scan);
 380                         break;
 381                 }
 382                 case PCIE_PCIECAP_DEV_TYPE_PCIE_DEV:
 383                 case PCIE_PCIECAP_DEV_TYPE_PCI_DEV:
 384                         /*
 385                          * Reached a PCIe end point so stop. Note dev_type
 386                          * PCI_DEV is just a PCIe device that requires IO Space
 387                          */
 388                         break;
 389                 case PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO:
 390                         if (PCIE_IS_BDG(bus_p))
 391                                 scan_flag |= pf_dispatch(dip, impl, B_TRUE);
 392                         break;
 393                 default:
 394                         ASSERT(B_FALSE);
 395                 }
 396         }
 397         return (scan_flag);
 398 }
 399 
 400 /* Returns whether the "bdf" is in the bus range of a switch/bridge */
 401 boolean_t
 402 pf_in_bus_range(pcie_bus_t *bus_p, pcie_req_id_t bdf)
 403 {
 404         pci_bus_range_t *br_p = &bus_p->bus_bus_range;
 405         uint8_t         bus_no = (bdf & PCIE_REQ_ID_BUS_MASK) >>
 406             PCIE_REQ_ID_BUS_SHIFT;
 407 
 408         /* check if given bdf falls within bridge's bus range */
 409         if (PCIE_IS_BDG(bus_p) &&
 410             ((bus_no >= br_p->lo) && (bus_no <= br_p->hi)))
 411                 return (B_TRUE);
 412         else
 413                 return (B_FALSE);
 414 }
 415 
 416 /*
 417  * Return whether the "addr" is in the assigned addr of a device.
 418  */
 419 boolean_t
 420 pf_in_assigned_addr(pcie_bus_t *bus_p, uint64_t addr)
 421 {
 422         uint_t          i;
 423         uint64_t        low, hi;
 424         pci_regspec_t   *assign_p = bus_p->bus_assigned_addr;
 425 
 426         for (i = 0; i < bus_p->bus_assigned_entries; i++, assign_p++) {
 427                 low = assign_p->pci_phys_low;
 428                 hi = low + assign_p->pci_size_low;
 429                 if ((addr < hi) && (addr >= low))
 430                         return (B_TRUE);
 431         }
 432         return (B_FALSE);
 433 }
 434 
 435 /*
 436  * Returns whether the "addr" is in the addr range of a switch/bridge, or if the
 437  * "addr" is in the assigned addr of a device.
 438  */
 439 static boolean_t
 440 pf_in_addr_range(pcie_bus_t *bus_p, uint64_t addr)
 441 {
 442         uint_t          i;
 443         uint64_t        low, hi;
 444         ppb_ranges_t    *ranges_p = bus_p->bus_addr_ranges;
 445 
 446         if (!addr)
 447                 return (B_FALSE);
 448 
 449         /* check if given address belongs to this device */
 450         if (pf_in_assigned_addr(bus_p, addr))
 451                 return (B_TRUE);
 452 
 453         /* check if given address belongs to a child below this device */
 454         if (!PCIE_IS_BDG(bus_p))
 455                 return (B_FALSE);
 456 
 457         for (i = 0; i < bus_p->bus_addr_entries; i++, ranges_p++) {
 458                 switch (ranges_p->child_high & PCI_ADDR_MASK) {
 459                 case PCI_ADDR_IO:
 460                 case PCI_ADDR_MEM32:
 461                         low = ranges_p->child_low;
 462                         hi = ranges_p->size_low + low;
 463                         if ((addr < hi) && (addr >= low))
 464                                 return (B_TRUE);
 465                         break;
 466                 case PCI_ADDR_MEM64:
 467                         low = ((uint64_t)ranges_p->child_mid << 32) |
 468                             (uint64_t)ranges_p->child_low;
 469                         hi = (((uint64_t)ranges_p->size_high << 32) |
 470                             (uint64_t)ranges_p->size_low) + low;
 471                         if ((addr < hi) && (addr >= low))
 472                                 return (B_TRUE);
 473                         break;
 474                 }
 475         }
 476         return (B_FALSE);
 477 }
 478 
 479 static pcie_bus_t *
 480 pf_is_ready(dev_info_t *dip)
 481 {
 482         pcie_bus_t      *bus_p = PCIE_DIP2BUS(dip);
 483         if (!bus_p)
 484                 return (NULL);
 485 
 486         if (!(bus_p->bus_fm_flags & PF_FM_READY))
 487                 return (NULL);
 488         return (bus_p);
 489 }
 490 
 491 static void
 492 pf_pcix_ecc_regs_gather(pf_pcix_ecc_regs_t *pcix_ecc_regs,
 493     pcie_bus_t *bus_p, boolean_t bdg)
 494 {
 495         if (bdg) {
 496                 pcix_ecc_regs->pcix_ecc_ctlstat = PCIX_CAP_GET(32, bus_p,
 497                     PCI_PCIX_BDG_ECC_STATUS);
 498                 pcix_ecc_regs->pcix_ecc_fstaddr = PCIX_CAP_GET(32, bus_p,
 499                     PCI_PCIX_BDG_ECC_FST_AD);
 500                 pcix_ecc_regs->pcix_ecc_secaddr = PCIX_CAP_GET(32, bus_p,
 501                     PCI_PCIX_BDG_ECC_SEC_AD);
 502                 pcix_ecc_regs->pcix_ecc_attr = PCIX_CAP_GET(32, bus_p,
 503                     PCI_PCIX_BDG_ECC_ATTR);
 504         } else {
 505                 pcix_ecc_regs->pcix_ecc_ctlstat = PCIX_CAP_GET(32, bus_p,
 506                     PCI_PCIX_ECC_STATUS);
 507                 pcix_ecc_regs->pcix_ecc_fstaddr = PCIX_CAP_GET(32, bus_p,
 508                     PCI_PCIX_ECC_FST_AD);
 509                 pcix_ecc_regs->pcix_ecc_secaddr = PCIX_CAP_GET(32, bus_p,
 510                     PCI_PCIX_ECC_SEC_AD);
 511                 pcix_ecc_regs->pcix_ecc_attr = PCIX_CAP_GET(32, bus_p,
 512                     PCI_PCIX_ECC_ATTR);
 513         }
 514 }
 515 
 516 
 517 static void
 518 pf_pcix_regs_gather(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 519 {
 520         /*
 521          * For PCI-X device PCI-X Capability only exists for Type 0 Headers.
 522          * PCI-X Bridge Capability only exists for Type 1 Headers.
 523          * Both capabilities do not exist at the same time.
 524          */
 525         if (PCIE_IS_BDG(bus_p)) {
 526                 pf_pcix_bdg_err_regs_t *pcix_bdg_regs;
 527 
 528                 pcix_bdg_regs = PCIX_BDG_ERR_REG(pfd_p);
 529 
 530                 pcix_bdg_regs->pcix_bdg_sec_stat = PCIX_CAP_GET(16, bus_p,
 531                     PCI_PCIX_SEC_STATUS);
 532                 pcix_bdg_regs->pcix_bdg_stat = PCIX_CAP_GET(32, bus_p,
 533                     PCI_PCIX_BDG_STATUS);
 534 
 535                 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
 536                         /*
 537                          * PCI Express to PCI-X bridges only implement the
 538                          * secondary side of the PCI-X ECC registers, bit one is
 539                          * read-only so we make sure we do not write to it.
 540                          */
 541                         if (!PCIE_IS_PCIE_BDG(bus_p)) {
 542                                 PCIX_CAP_PUT(32, bus_p, PCI_PCIX_BDG_ECC_STATUS,
 543                                     0);
 544                                 pf_pcix_ecc_regs_gather(
 545                                     PCIX_BDG_ECC_REG(pfd_p, 0), bus_p, B_TRUE);
 546                                 PCIX_CAP_PUT(32, bus_p, PCI_PCIX_BDG_ECC_STATUS,
 547                                     1);
 548                         }
 549                         pf_pcix_ecc_regs_gather(PCIX_BDG_ECC_REG(pfd_p, 0),
 550                             bus_p, B_TRUE);
 551                 }
 552         } else {
 553                 pf_pcix_err_regs_t *pcix_regs = PCIX_ERR_REG(pfd_p);
 554 
 555                 pcix_regs->pcix_command = PCIX_CAP_GET(16, bus_p,
 556                     PCI_PCIX_COMMAND);
 557                 pcix_regs->pcix_status = PCIX_CAP_GET(32, bus_p,
 558                     PCI_PCIX_STATUS);
 559                 if (PCIX_ECC_VERSION_CHECK(bus_p))
 560                         pf_pcix_ecc_regs_gather(PCIX_ECC_REG(pfd_p), bus_p,
 561                             B_TRUE);
 562         }
 563 }
 564 
 565 static void
 566 pf_pcie_regs_gather(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 567 {
 568         pf_pcie_err_regs_t *pcie_regs = PCIE_ERR_REG(pfd_p);
 569         pf_pcie_adv_err_regs_t *pcie_adv_regs = PCIE_ADV_REG(pfd_p);
 570 
 571         pcie_regs->pcie_err_status = PCIE_CAP_GET(16, bus_p, PCIE_DEVSTS);
 572         pcie_regs->pcie_err_ctl = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
 573         pcie_regs->pcie_dev_cap = PCIE_CAP_GET(32, bus_p, PCIE_DEVCAP);
 574 
 575         if (PCIE_IS_BDG(bus_p) && PCIE_IS_PCIX(bus_p))
 576                 pf_pcix_regs_gather(pfd_p, bus_p);
 577 
 578         if (PCIE_IS_ROOT(bus_p)) {
 579                 pf_pcie_rp_err_regs_t *pcie_rp_regs = PCIE_RP_REG(pfd_p);
 580 
 581                 pcie_rp_regs->pcie_rp_status = PCIE_CAP_GET(32, bus_p,
 582                     PCIE_ROOTSTS);
 583                 pcie_rp_regs->pcie_rp_ctl = PCIE_CAP_GET(16, bus_p,
 584                     PCIE_ROOTCTL);
 585         }
 586 
 587         if (!PCIE_HAS_AER(bus_p))
 588                 return;
 589 
 590         /* Gather UE AERs */
 591         pcie_adv_regs->pcie_adv_ctl = PCIE_AER_GET(32, bus_p,
 592             PCIE_AER_CTL);
 593         pcie_adv_regs->pcie_ue_status = PCIE_AER_GET(32, bus_p,
 594             PCIE_AER_UCE_STS);
 595         pcie_adv_regs->pcie_ue_mask = PCIE_AER_GET(32, bus_p,
 596             PCIE_AER_UCE_MASK);
 597         pcie_adv_regs->pcie_ue_sev = PCIE_AER_GET(32, bus_p,
 598             PCIE_AER_UCE_SERV);
 599         PCIE_ADV_HDR(pfd_p, 0) = PCIE_AER_GET(32, bus_p,
 600             PCIE_AER_HDR_LOG);
 601         PCIE_ADV_HDR(pfd_p, 1) = PCIE_AER_GET(32, bus_p,
 602             PCIE_AER_HDR_LOG + 0x4);
 603         PCIE_ADV_HDR(pfd_p, 2) = PCIE_AER_GET(32, bus_p,
 604             PCIE_AER_HDR_LOG + 0x8);
 605         PCIE_ADV_HDR(pfd_p, 3) = PCIE_AER_GET(32, bus_p,
 606             PCIE_AER_HDR_LOG + 0xc);
 607 
 608         /* Gather CE AERs */
 609         pcie_adv_regs->pcie_ce_status = PCIE_AER_GET(32, bus_p,
 610             PCIE_AER_CE_STS);
 611         pcie_adv_regs->pcie_ce_mask = PCIE_AER_GET(32, bus_p,
 612             PCIE_AER_CE_MASK);
 613 
 614         /*
 615          * If pci express to pci bridge then grab the bridge
 616          * error registers.
 617          */
 618         if (PCIE_IS_PCIE_BDG(bus_p)) {
 619                 pf_pcie_adv_bdg_err_regs_t *pcie_bdg_regs =
 620                     PCIE_ADV_BDG_REG(pfd_p);
 621 
 622                 pcie_bdg_regs->pcie_sue_ctl = PCIE_AER_GET(32, bus_p,
 623                     PCIE_AER_SCTL);
 624                 pcie_bdg_regs->pcie_sue_status = PCIE_AER_GET(32, bus_p,
 625                     PCIE_AER_SUCE_STS);
 626                 pcie_bdg_regs->pcie_sue_mask = PCIE_AER_GET(32, bus_p,
 627                     PCIE_AER_SUCE_MASK);
 628                 pcie_bdg_regs->pcie_sue_sev = PCIE_AER_GET(32, bus_p,
 629                     PCIE_AER_SUCE_SERV);
 630                 PCIE_ADV_BDG_HDR(pfd_p, 0) = PCIE_AER_GET(32, bus_p,
 631                     PCIE_AER_SHDR_LOG);
 632                 PCIE_ADV_BDG_HDR(pfd_p, 1) = PCIE_AER_GET(32, bus_p,
 633                     PCIE_AER_SHDR_LOG + 0x4);
 634                 PCIE_ADV_BDG_HDR(pfd_p, 2) = PCIE_AER_GET(32, bus_p,
 635                     PCIE_AER_SHDR_LOG + 0x8);
 636                 PCIE_ADV_BDG_HDR(pfd_p, 3) = PCIE_AER_GET(32, bus_p,
 637                     PCIE_AER_SHDR_LOG + 0xc);
 638         }
 639 
 640         /*
 641          * If PCI Express root port then grab the root port
 642          * error registers.
 643          */
 644         if (PCIE_IS_ROOT(bus_p)) {
 645                 pf_pcie_adv_rp_err_regs_t *pcie_rp_regs =
 646                     PCIE_ADV_RP_REG(pfd_p);
 647 
 648                 pcie_rp_regs->pcie_rp_err_cmd = PCIE_AER_GET(32, bus_p,
 649                     PCIE_AER_RE_CMD);
 650                 pcie_rp_regs->pcie_rp_err_status = PCIE_AER_GET(32, bus_p,
 651                     PCIE_AER_RE_STS);
 652                 pcie_rp_regs->pcie_rp_ce_src_id = PCIE_AER_GET(16, bus_p,
 653                     PCIE_AER_CE_SRC_ID);
 654                 pcie_rp_regs->pcie_rp_ue_src_id = PCIE_AER_GET(16, bus_p,
 655                     PCIE_AER_ERR_SRC_ID);
 656         }
 657 }
 658 
 659 static void
 660 pf_pci_regs_gather(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 661 {
 662         pf_pci_err_regs_t *pci_regs = PCI_ERR_REG(pfd_p);
 663 
 664         /*
 665          * Start by reading all the error registers that are available for
 666          * pci and pci express and for leaf devices and bridges/switches
 667          */
 668         pci_regs->pci_err_status = PCIE_GET(16, bus_p, PCI_CONF_STAT);
 669         pci_regs->pci_cfg_comm = PCIE_GET(16, bus_p, PCI_CONF_COMM);
 670 
 671         /*
 672          * If pci-pci bridge grab PCI bridge specific error registers.
 673          */
 674         if (PCIE_IS_BDG(bus_p)) {
 675                 pf_pci_bdg_err_regs_t *pci_bdg_regs = PCI_BDG_ERR_REG(pfd_p);
 676                 pci_bdg_regs->pci_bdg_sec_stat =
 677                     PCIE_GET(16, bus_p, PCI_BCNF_SEC_STATUS);
 678                 pci_bdg_regs->pci_bdg_ctrl =
 679                     PCIE_GET(16, bus_p, PCI_BCNF_BCNTRL);
 680         }
 681 
 682         /*
 683          * If pci express device grab pci express error registers and
 684          * check for advanced error reporting features and grab them if
 685          * available.
 686          */
 687         if (PCIE_IS_PCIE(bus_p))
 688                 pf_pcie_regs_gather(pfd_p, bus_p);
 689         else if (PCIE_IS_PCIX(bus_p))
 690                 pf_pcix_regs_gather(pfd_p, bus_p);
 691 
 692 }
 693 
 694 static void
 695 pf_pcix_regs_clear(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 696 {
 697         if (PCIE_IS_BDG(bus_p)) {
 698                 pf_pcix_bdg_err_regs_t *pcix_bdg_regs;
 699 
 700                 pcix_bdg_regs = PCIX_BDG_ERR_REG(pfd_p);
 701 
 702                 PCIX_CAP_PUT(16, bus_p, PCI_PCIX_SEC_STATUS,
 703                     pcix_bdg_regs->pcix_bdg_sec_stat);
 704 
 705                 PCIX_CAP_PUT(32, bus_p, PCI_PCIX_BDG_STATUS,
 706                     pcix_bdg_regs->pcix_bdg_stat);
 707 
 708                 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
 709                         pf_pcix_ecc_regs_t *pcix_bdg_ecc_regs;
 710                         /*
 711                          * PCI Express to PCI-X bridges only implement the
 712                          * secondary side of the PCI-X ECC registers.  For
 713                          * clearing, there is no need to "select" the ECC
 714                          * register, just write what was originally read.
 715                          */
 716                         if (!PCIE_IS_PCIE_BDG(bus_p)) {
 717                                 pcix_bdg_ecc_regs = PCIX_BDG_ECC_REG(pfd_p, 0);
 718                                 PCIX_CAP_PUT(32, bus_p, PCI_PCIX_BDG_ECC_STATUS,
 719                                     pcix_bdg_ecc_regs->pcix_ecc_ctlstat);
 720 
 721                         }
 722                         pcix_bdg_ecc_regs = PCIX_BDG_ECC_REG(pfd_p, 1);
 723                         PCIX_CAP_PUT(32, bus_p, PCI_PCIX_BDG_ECC_STATUS,
 724                             pcix_bdg_ecc_regs->pcix_ecc_ctlstat);
 725                 }
 726         } else {
 727                 pf_pcix_err_regs_t *pcix_regs = PCIX_ERR_REG(pfd_p);
 728 
 729                 PCIX_CAP_PUT(32, bus_p, PCI_PCIX_STATUS,
 730                     pcix_regs->pcix_status);
 731 
 732                 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
 733                         pf_pcix_ecc_regs_t *pcix_ecc_regs = PCIX_ECC_REG(pfd_p);
 734 
 735                         PCIX_CAP_PUT(32, bus_p, PCI_PCIX_ECC_STATUS,
 736                             pcix_ecc_regs->pcix_ecc_ctlstat);
 737                 }
 738         }
 739 }
 740 
 741 static void
 742 pf_pcie_regs_clear(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 743 {
 744         pf_pcie_err_regs_t *pcie_regs = PCIE_ERR_REG(pfd_p);
 745         pf_pcie_adv_err_regs_t *pcie_adv_regs = PCIE_ADV_REG(pfd_p);
 746 
 747         PCIE_CAP_PUT(16, bus_p, PCIE_DEVSTS, pcie_regs->pcie_err_status);
 748 
 749         if (PCIE_IS_BDG(bus_p) && PCIE_IS_PCIX(bus_p))
 750                 pf_pcix_regs_clear(pfd_p, bus_p);
 751 
 752         if (!PCIE_HAS_AER(bus_p))
 753                 return;
 754 
 755         PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_STS,
 756             pcie_adv_regs->pcie_ue_status);
 757 
 758         PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_STS,
 759             pcie_adv_regs->pcie_ce_status);
 760 
 761         if (PCIE_IS_PCIE_BDG(bus_p)) {
 762                 pf_pcie_adv_bdg_err_regs_t *pcie_bdg_regs =
 763                     PCIE_ADV_BDG_REG(pfd_p);
 764 
 765                 PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_STS,
 766                     pcie_bdg_regs->pcie_sue_status);
 767         }
 768 
 769         /*
 770          * If PCI Express root complex then clear the root complex
 771          * error registers.
 772          */
 773         if (PCIE_IS_ROOT(bus_p)) {
 774                 pf_pcie_adv_rp_err_regs_t *pcie_rp_regs;
 775 
 776                 pcie_rp_regs = PCIE_ADV_RP_REG(pfd_p);
 777 
 778                 PCIE_AER_PUT(32, bus_p, PCIE_AER_RE_STS,
 779                     pcie_rp_regs->pcie_rp_err_status);
 780         }
 781 }
 782 
 783 static void
 784 pf_pci_regs_clear(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 785 {
 786         if (PCIE_IS_PCIE(bus_p))
 787                 pf_pcie_regs_clear(pfd_p, bus_p);
 788         else if (PCIE_IS_PCIX(bus_p))
 789                 pf_pcix_regs_clear(pfd_p, bus_p);
 790 
 791         PCIE_PUT(16, bus_p, PCI_CONF_STAT, pfd_p->pe_pci_regs->pci_err_status);
 792 
 793         if (PCIE_IS_BDG(bus_p)) {
 794                 pf_pci_bdg_err_regs_t *pci_bdg_regs = PCI_BDG_ERR_REG(pfd_p);
 795                 PCIE_PUT(16, bus_p, PCI_BCNF_SEC_STATUS,
 796                     pci_bdg_regs->pci_bdg_sec_stat);
 797         }
 798 }
 799 
 800 /* ARGSUSED */
 801 void
 802 pcie_clear_errors(dev_info_t *dip)
 803 {
 804         pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
 805         pf_data_t *pfd_p = PCIE_DIP2PFD(dip);
 806 
 807         ASSERT(bus_p);
 808 
 809         pf_pci_regs_gather(pfd_p, bus_p);
 810         pf_pci_regs_clear(pfd_p, bus_p);
 811 }
 812 
 813 /* Find the fault BDF, fault Addr or full scan on a PCIe Root Port. */
 814 static void
 815 pf_pci_find_rp_fault(pf_data_t *pfd_p, pcie_bus_t *bus_p)
 816 {
 817         pf_root_fault_t *root_fault = PCIE_ROOT_FAULT(pfd_p);
 818         pf_pcie_adv_rp_err_regs_t *rp_regs = PCIE_ADV_RP_REG(pfd_p);
 819         uint32_t root_err = rp_regs->pcie_rp_err_status;
 820         uint32_t ue_err = PCIE_ADV_REG(pfd_p)->pcie_ue_status;
 821         int num_faults = 0;
 822 
 823         /* Since this data structure is reused, make sure to reset it */
 824         root_fault->full_scan = B_FALSE;
 825         root_fault->scan_bdf = PCIE_INVALID_BDF;
 826         root_fault->scan_addr = 0;
 827 
 828         if (!PCIE_HAS_AER(bus_p) &&
 829             (PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat & PF_PCI_BDG_ERR)) {
 830                 PCIE_ROOT_FAULT(pfd_p)->full_scan = B_TRUE;
 831                 return;
 832         }
 833 
 834         /*
 835          * Check to see if an error has been received that
 836          * requires a scan of the fabric.  Count the number of
 837          * faults seen.  If MUL CE/FE_NFE that counts for
 838          * atleast 2 faults, so just return with full_scan.
 839          */
 840         if ((root_err & PCIE_AER_RE_STS_MUL_CE_RCVD) ||
 841             (root_err & PCIE_AER_RE_STS_MUL_FE_NFE_RCVD)) {
 842                 PCIE_ROOT_FAULT(pfd_p)->full_scan = B_TRUE;
 843                 return;
 844         }
 845 
 846         if (root_err & PCIE_AER_RE_STS_CE_RCVD)
 847                 num_faults++;
 848 
 849         if (root_err & PCIE_AER_RE_STS_FE_NFE_RCVD)
 850                 num_faults++;
 851 
 852         if (ue_err & PCIE_AER_UCE_CA)
 853                 num_faults++;
 854 
 855         if (ue_err & PCIE_AER_UCE_UR)
 856                 num_faults++;
 857 
 858         /* If no faults just return */
 859         if (num_faults == 0)
 860                 return;
 861 
 862         /* If faults > 1 do full scan */
 863         if (num_faults > 1) {
 864                 PCIE_ROOT_FAULT(pfd_p)->full_scan = B_TRUE;
 865                 return;
 866         }
 867 
 868         /* By this point, there is only 1 fault detected */
 869         if (root_err & PCIE_AER_RE_STS_CE_RCVD) {
 870                 PCIE_ROOT_FAULT(pfd_p)->scan_bdf = rp_regs->pcie_rp_ce_src_id;
 871                 num_faults--;
 872         } else if (root_err & PCIE_AER_RE_STS_FE_NFE_RCVD) {
 873                 PCIE_ROOT_FAULT(pfd_p)->scan_bdf = rp_regs->pcie_rp_ue_src_id;
 874                 num_faults--;
 875         } else if ((HAS_AER_LOGS(pfd_p, PCIE_AER_UCE_CA) ||
 876             HAS_AER_LOGS(pfd_p, PCIE_AER_UCE_UR)) &&
 877             (pf_tlp_decode(PCIE_PFD2BUS(pfd_p), PCIE_ADV_REG(pfd_p)) ==
 878             DDI_SUCCESS)) {
 879                 PCIE_ROOT_FAULT(pfd_p)->scan_addr =
 880                     PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr;
 881                 num_faults--;
 882         }
 883 
 884         /*
 885          * This means an error did occur, but we couldn't extract the fault BDF
 886          */
 887         if (num_faults > 0)
 888                 PCIE_ROOT_FAULT(pfd_p)->full_scan = B_TRUE;
 889 
 890 }
 891 
 892 
 893 /*
 894  * Load PCIe Fault Data for PCI/PCIe devices into PCIe Fault Data Queue
 895  *
 896  * Returns a scan flag.
 897  * o PF_SCAN_SUCCESS - Error gathered and cleared sucessfuly, data added to
 898  *   Fault Q
 899  * o PF_SCAN_BAD_RESPONSE - Unable to talk to device, item added to fault Q
 900  * o PF_SCAN_CB_FAILURE - A hardened device deemed that the error was fatal.
 901  * o PF_SCAN_NO_ERR_IN_CHILD - Only applies to bridge to prevent further
 902  *   unnecessary scanning
 903  * o PF_SCAN_IN_DQ - This device has already been scanned; it was skipped this
 904  *   time.
 905  */
 906 static int
 907 pf_default_hdl(dev_info_t *dip, pf_impl_t *impl)
 908 {
 909         pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
 910         pf_data_t *pfd_p = PCIE_DIP2PFD(dip);
 911         int cb_sts, scan_flag = PF_SCAN_SUCCESS;
 912 
 913         /* Make sure this device hasn't already been snapshotted and cleared */
 914         if (pfd_p->pe_valid == B_TRUE) {
 915                 scan_flag |= PF_SCAN_IN_DQ;
 916                 goto done;
 917         }
 918 
 919         /*
 920          * Read vendor/device ID and check with cached data, if it doesn't match
 921          * could very well be a device that isn't responding anymore.  Just
 922          * stop.  Save the basic info in the error q for post mortem debugging
 923          * purposes.
 924          */
 925         if (PCIE_GET(32, bus_p, PCI_CONF_VENID) != bus_p->bus_dev_ven_id) {
 926                 char buf[FM_MAX_CLASS];
 927 
 928                 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s",
 929                     PCI_ERROR_SUBCLASS, PCI_NR);
 930                 ddi_fm_ereport_post(dip, buf, fm_ena_generate(0, FM_ENA_FMT1),
 931                     DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, 0, NULL);
 932 
 933                 /*
 934                  * For IOV/Hotplug purposes skip gathering info fo this device,
 935                  * but populate affected info and severity.  Clear out any data
 936                  * that maybe been saved in the last fabric scan.
 937                  */
 938                 pf_reset_pfd(pfd_p);
 939                 pfd_p->pe_severity_flags = PF_ERR_PANIC_BAD_RESPONSE;
 940                 PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags = PF_AFFECTED_SELF;
 941 
 942                 /* Add the snapshot to the error q */
 943                 pf_en_dq(pfd_p, impl);
 944                 pfd_p->pe_valid = B_TRUE;
 945 
 946                 return (PF_SCAN_BAD_RESPONSE);
 947         }
 948 
 949         pf_pci_regs_gather(pfd_p, bus_p);
 950         pf_pci_regs_clear(pfd_p, bus_p);
 951         if (PCIE_IS_RP(bus_p))
 952                 pf_pci_find_rp_fault(pfd_p, bus_p);
 953 
 954         cb_sts = pf_fm_callback(dip, impl->pf_derr);
 955 
 956         if (cb_sts == DDI_FM_FATAL || cb_sts == DDI_FM_UNKNOWN)
 957                 scan_flag |= PF_SCAN_CB_FAILURE;
 958 
 959         /* Add the snapshot to the error q */
 960         pf_en_dq(pfd_p, impl);
 961 
 962 done:
 963         /*
 964          * If a bridge does not have any error no need to scan any further down.
 965          * For PCIe devices, check the PCIe device status and PCI secondary
 966          * status.
 967          * - Some non-compliant PCIe devices do not utilize PCIe
 968          *   error registers.  If so rely on legacy PCI error registers.
 969          * For PCI devices, check the PCI secondary status.
 970          */
 971         if (PCIE_IS_PCIE_BDG(bus_p) &&
 972             !(PCIE_ERR_REG(pfd_p)->pcie_err_status & PF_PCIE_BDG_ERR) &&
 973             !(PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat & PF_PCI_BDG_ERR))
 974                 scan_flag |= PF_SCAN_NO_ERR_IN_CHILD;
 975 
 976         if (PCIE_IS_PCI_BDG(bus_p) &&
 977             !(PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat & PF_PCI_BDG_ERR))
 978                 scan_flag |= PF_SCAN_NO_ERR_IN_CHILD;
 979 
 980         pfd_p->pe_valid = B_TRUE;
 981         return (scan_flag);
 982 }
 983 
 984 /*
 985  * Called during postattach to initialize a device's error handling
 986  * capabilities.  If the devices has already been hardened, then there isn't
 987  * much needed.  Otherwise initialize the device's default FMA capabilities.
 988  *
 989  * In a future project where PCIe support is removed from pcifm, several
 990  * "properties" that are setup in ddi_fm_init and pci_ereport_setup need to be
 991  * created here so that the PCI/PCIe eversholt rules will work properly.
 992  */
 993 void
 994 pf_init(dev_info_t *dip, ddi_iblock_cookie_t ibc, ddi_attach_cmd_t cmd)
 995 {
 996         pcie_bus_t              *bus_p = PCIE_DIP2BUS(dip);
 997         struct i_ddi_fmhdl      *fmhdl = DEVI(dip)->devi_fmhdl;
 998         boolean_t               need_cb_register = B_FALSE;
 999 
1000         if (!bus_p) {
1001                 cmn_err(CE_WARN, "devi_bus information is not set for %s%d.\n",
1002                     ddi_driver_name(dip), ddi_get_instance(dip));
1003                 return;
1004         }
1005 
1006         if (fmhdl) {
1007                 /*
1008                  * If device is only ereport capable and not callback capable
1009                  * make it callback capable. The only downside is that the
1010                  * "fm-errcb-capable" property is not created for this device
1011                  * which should be ok since it's not used anywhere.
1012                  */
1013                 if (!(fmhdl->fh_cap & DDI_FM_ERRCB_CAPABLE))
1014                         need_cb_register = B_TRUE;
1015         } else {
1016                 int cap;
1017                 /*
1018                  * fm-capable in driver.conf can be used to set fm_capabilities.
1019                  * If fm-capable is not defined, set the default
1020                  * DDI_FM_EREPORT_CAPABLE and DDI_FM_ERRCB_CAPABLE.
1021                  */
1022                 cap = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
1023                     DDI_PROP_DONTPASS, "fm-capable",
1024                     DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE);
1025                 cap &= (DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE);
1026 
1027                 bus_p->bus_fm_flags |= PF_FM_IS_NH;
1028 
1029                 if (cmd == DDI_ATTACH) {
1030                         ddi_fm_init(dip, &cap, &ibc);
1031                         pci_ereport_setup(dip);
1032                 }
1033 
1034                 if (cap & DDI_FM_ERRCB_CAPABLE)
1035                         need_cb_register = B_TRUE;
1036 
1037                 fmhdl = DEVI(dip)->devi_fmhdl;
1038         }
1039 
1040         /* If ddi_fm_init fails for any reason RETURN */
1041         if (!fmhdl) {
1042                 bus_p->bus_fm_flags = 0;
1043                 return;
1044         }
1045 
1046         fmhdl->fh_cap |=  DDI_FM_ERRCB_CAPABLE;
1047         if (cmd == DDI_ATTACH) {
1048                 if (need_cb_register)
1049                         ddi_fm_handler_register(dip, pf_dummy_cb, NULL);
1050         }
1051 
1052         bus_p->bus_fm_flags |= PF_FM_READY;
1053 }
1054 
1055 /* undo FMA lock, called at predetach */
1056 void
1057 pf_fini(dev_info_t *dip, ddi_detach_cmd_t cmd)
1058 {
1059         pcie_bus_t      *bus_p = PCIE_DIP2BUS(dip);
1060 
1061         if (!bus_p)
1062                 return;
1063 
1064         /* Don't fini anything if device isn't FM Ready */
1065         if (!(bus_p->bus_fm_flags & PF_FM_READY))
1066                 return;
1067 
1068         /* no other code should set the flag to false */
1069         bus_p->bus_fm_flags &= ~PF_FM_READY;
1070 
1071         /*
1072          * Grab the mutex to make sure device isn't in the middle of
1073          * error handling.  Setting the bus_fm_flag to ~PF_FM_READY
1074          * should prevent this device from being error handled after
1075          * the mutex has been released.
1076          */
1077         (void) pf_handler_enter(dip, NULL);
1078         pf_handler_exit(dip);
1079 
1080         /* undo non-hardened drivers */
1081         if (bus_p->bus_fm_flags & PF_FM_IS_NH) {
1082                 if (cmd == DDI_DETACH) {
1083                         bus_p->bus_fm_flags &= ~PF_FM_IS_NH;
1084                         pci_ereport_teardown(dip);
1085                         /*
1086                          * ddi_fini itself calls ddi_handler_unregister,
1087                          * so no need to explicitly call unregister.
1088                          */
1089                         ddi_fm_fini(dip);
1090                 }
1091         }
1092 }
1093 
1094 /*ARGSUSED*/
1095 static int
1096 pf_dummy_cb(dev_info_t *dip, ddi_fm_error_t *derr, const void *not_used)
1097 {
1098         return (DDI_FM_OK);
1099 }
1100 
1101 /*
1102  * Add PFD to queue.  If it is an RC add it to the beginning,
1103  * otherwise add it to the end.
1104  */
1105 static void
1106 pf_en_dq(pf_data_t *pfd_p, pf_impl_t *impl)
1107 {
1108         pf_data_t *head_p = impl->pf_dq_head_p;
1109         pf_data_t *tail_p = impl->pf_dq_tail_p;
1110 
1111         impl->pf_total++;
1112 
1113         if (!head_p) {
1114                 ASSERT(PFD_IS_ROOT(pfd_p));
1115                 impl->pf_dq_head_p = pfd_p;
1116                 impl->pf_dq_tail_p = pfd_p;
1117                 pfd_p->pe_prev = NULL;
1118                 pfd_p->pe_next = NULL;
1119                 return;
1120         }
1121 
1122         /* Check if this is a Root Port eprt */
1123         if (PFD_IS_ROOT(pfd_p)) {
1124                 pf_data_t *root_p, *last_p = NULL;
1125 
1126                 /* The first item must be a RP */
1127                 root_p = head_p;
1128                 for (last_p = head_p; last_p && PFD_IS_ROOT(last_p);
1129                     last_p = last_p->pe_next)
1130                         root_p = last_p;
1131 
1132                 /* root_p is the last RP pfd. last_p is the first non-RP pfd. */
1133                 root_p->pe_next = pfd_p;
1134                 pfd_p->pe_prev = root_p;
1135                 pfd_p->pe_next = last_p;
1136 
1137                 if (last_p)
1138                         last_p->pe_prev = pfd_p;
1139                 else
1140                         tail_p = pfd_p;
1141         } else {
1142                 tail_p->pe_next = pfd_p;
1143                 pfd_p->pe_prev = tail_p;
1144                 pfd_p->pe_next = NULL;
1145                 tail_p = pfd_p;
1146         }
1147 
1148         impl->pf_dq_head_p = head_p;
1149         impl->pf_dq_tail_p = tail_p;
1150 }
1151 
1152 /*
1153  * Ignore:
1154  * - TRAINING: as leaves do not have children
1155  * - SD: as leaves do not have children
1156  */
1157 const pf_fab_err_tbl_t pcie_pcie_tbl[] = {
1158         {PCIE_AER_UCE_DLP,      pf_panic,
1159             PF_AFFECTED_PARENT, 0},
1160 
1161         {PCIE_AER_UCE_PTLP,     pf_analyse_ptlp,
1162             PF_AFFECTED_SELF, 0},
1163 
1164         {PCIE_AER_UCE_FCP,      pf_panic,
1165             PF_AFFECTED_PARENT, 0},
1166 
1167         {PCIE_AER_UCE_TO,       pf_analyse_to,
1168             PF_AFFECTED_SELF, 0},
1169 
1170         {PCIE_AER_UCE_CA,       pf_analyse_ca_ur,
1171             PF_AFFECTED_SELF, 0},
1172 
1173         {PCIE_AER_UCE_UC,       pf_analyse_uc,
1174             0, 0},
1175 
1176         {PCIE_AER_UCE_RO,       pf_panic,
1177             PF_AFFECTED_PARENT, 0},
1178 
1179         {PCIE_AER_UCE_MTLP,     pf_panic,
1180             PF_AFFECTED_PARENT, 0},
1181 
1182         {PCIE_AER_UCE_ECRC,     pf_panic,
1183             PF_AFFECTED_SELF, 0},
1184 
1185         {PCIE_AER_UCE_UR,       pf_analyse_ca_ur,
1186             PF_AFFECTED_SELF, 0},
1187 
1188         {NULL, NULL, NULL, NULL}
1189 };
1190 
1191 const pf_fab_err_tbl_t pcie_rp_tbl[] = {
1192         {PCIE_AER_UCE_TRAINING, pf_no_panic,
1193             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1194 
1195         {PCIE_AER_UCE_DLP,      pf_panic,
1196             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1197 
1198         {PCIE_AER_UCE_SD,       pf_no_panic,
1199             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1200 
1201         {PCIE_AER_UCE_PTLP,     pf_analyse_ptlp,
1202             PF_AFFECTED_AER, PF_AFFECTED_CHILDREN},
1203 
1204         {PCIE_AER_UCE_FCP,      pf_panic,
1205             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1206 
1207         {PCIE_AER_UCE_TO,       pf_panic,
1208             PF_AFFECTED_ADDR, PF_AFFECTED_CHILDREN},
1209 
1210         {PCIE_AER_UCE_CA,       pf_no_panic,
1211             PF_AFFECTED_AER, PF_AFFECTED_CHILDREN},
1212 
1213         {PCIE_AER_UCE_UC,       pf_analyse_uc,
1214             0, 0},
1215 
1216         {PCIE_AER_UCE_RO,       pf_panic,
1217             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1218 
1219         {PCIE_AER_UCE_MTLP,     pf_panic,
1220             PF_AFFECTED_SELF | PF_AFFECTED_AER,
1221             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN},
1222 
1223         {PCIE_AER_UCE_ECRC,     pf_panic,
1224             PF_AFFECTED_AER, PF_AFFECTED_CHILDREN},
1225 
1226         {PCIE_AER_UCE_UR,       pf_no_panic,
1227             PF_AFFECTED_AER, PF_AFFECTED_CHILDREN},
1228 
1229         {NULL, NULL, NULL, NULL}
1230 };
1231 
1232 const pf_fab_err_tbl_t pcie_sw_tbl[] = {
1233         {PCIE_AER_UCE_TRAINING, pf_no_panic,
1234             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1235 
1236         {PCIE_AER_UCE_DLP,      pf_panic,
1237             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1238 
1239         {PCIE_AER_UCE_SD,       pf_no_panic,
1240             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1241 
1242         {PCIE_AER_UCE_PTLP,     pf_analyse_ptlp,
1243             PF_AFFECTED_AER, PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN},
1244 
1245         {PCIE_AER_UCE_FCP,      pf_panic,
1246             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1247 
1248         {PCIE_AER_UCE_TO,       pf_analyse_to,
1249             PF_AFFECTED_CHILDREN, 0},
1250 
1251         {PCIE_AER_UCE_CA,       pf_analyse_ca_ur,
1252             PF_AFFECTED_AER, PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN},
1253 
1254         {PCIE_AER_UCE_UC,       pf_analyse_uc,
1255             0, 0},
1256 
1257         {PCIE_AER_UCE_RO,       pf_panic,
1258             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1259 
1260         {PCIE_AER_UCE_MTLP,     pf_panic,
1261             PF_AFFECTED_SELF | PF_AFFECTED_AER,
1262             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN},
1263 
1264         {PCIE_AER_UCE_ECRC,     pf_panic,
1265             PF_AFFECTED_AER, PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN},
1266 
1267         {PCIE_AER_UCE_UR,       pf_analyse_ca_ur,
1268             PF_AFFECTED_AER, PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN},
1269 
1270         {NULL, NULL, NULL, NULL}
1271 };
1272 
1273 const pf_fab_err_tbl_t pcie_pcie_bdg_tbl[] = {
1274         {PCIE_AER_SUCE_TA_ON_SC,        pf_analyse_sc,
1275             0, 0},
1276 
1277         {PCIE_AER_SUCE_MA_ON_SC,        pf_analyse_sc,
1278             0, 0},
1279 
1280         {PCIE_AER_SUCE_RCVD_TA,         pf_analyse_ma_ta,
1281             0, 0},
1282 
1283         {PCIE_AER_SUCE_RCVD_MA,         pf_analyse_ma_ta,
1284             0, 0},
1285 
1286         {PCIE_AER_SUCE_USC_ERR,         pf_panic,
1287             PF_AFFECTED_SAER, PF_AFFECTED_CHILDREN},
1288 
1289         {PCIE_AER_SUCE_USC_MSG_DATA_ERR, pf_analyse_ma_ta,
1290             PF_AFFECTED_SAER, PF_AFFECTED_CHILDREN},
1291 
1292         {PCIE_AER_SUCE_UC_DATA_ERR,     pf_analyse_uc_data,
1293             PF_AFFECTED_SAER, PF_AFFECTED_CHILDREN},
1294 
1295         {PCIE_AER_SUCE_UC_ATTR_ERR,     pf_panic,
1296             PF_AFFECTED_CHILDREN, 0},
1297 
1298         {PCIE_AER_SUCE_UC_ADDR_ERR,     pf_panic,
1299             PF_AFFECTED_CHILDREN, 0},
1300 
1301         {PCIE_AER_SUCE_TIMER_EXPIRED,   pf_panic,
1302             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1303 
1304         {PCIE_AER_SUCE_PERR_ASSERT,     pf_analyse_perr_assert,
1305             0, 0},
1306 
1307         {PCIE_AER_SUCE_SERR_ASSERT,     pf_no_panic,
1308             0, 0},
1309 
1310         {PCIE_AER_SUCE_INTERNAL_ERR,    pf_panic,
1311             PF_AFFECTED_SELF | PF_AFFECTED_CHILDREN, 0},
1312 
1313         {NULL, NULL, NULL, NULL}
1314 };
1315 
1316 const pf_fab_err_tbl_t pcie_pci_bdg_tbl[] = {
1317         {PCI_STAT_PERROR,       pf_analyse_pci,
1318             PF_AFFECTED_SELF, 0},
1319 
1320         {PCI_STAT_S_PERROR,     pf_analyse_pci,
1321             PF_AFFECTED_SELF, 0},
1322 
1323         {PCI_STAT_S_SYSERR,     pf_panic,
1324             PF_AFFECTED_SELF, 0},
1325 
1326         {PCI_STAT_R_MAST_AB,    pf_analyse_pci,
1327             PF_AFFECTED_SELF, 0},
1328 
1329         {PCI_STAT_R_TARG_AB,    pf_analyse_pci,
1330             PF_AFFECTED_SELF, 0},
1331 
1332         {PCI_STAT_S_TARG_AB,    pf_analyse_pci,
1333             PF_AFFECTED_SELF, 0},
1334 
1335         {NULL, NULL, NULL, NULL}
1336 };
1337 
1338 const pf_fab_err_tbl_t pcie_pci_tbl[] = {
1339         {PCI_STAT_PERROR,       pf_analyse_pci,
1340             PF_AFFECTED_SELF, 0},
1341 
1342         {PCI_STAT_S_PERROR,     pf_analyse_pci,
1343             PF_AFFECTED_SELF, 0},
1344 
1345         {PCI_STAT_S_SYSERR,     pf_panic,
1346             PF_AFFECTED_SELF, 0},
1347 
1348         {PCI_STAT_R_MAST_AB,    pf_analyse_pci,
1349             PF_AFFECTED_SELF, 0},
1350 
1351         {PCI_STAT_R_TARG_AB,    pf_analyse_pci,
1352             PF_AFFECTED_SELF, 0},
1353 
1354         {PCI_STAT_S_TARG_AB,    pf_analyse_pci,
1355             PF_AFFECTED_SELF, 0},
1356 
1357         {NULL, NULL, NULL, NULL}
1358 };
1359 
1360 #define PF_MASKED_AER_ERR(pfd_p) \
1361         (PCIE_ADV_REG(pfd_p)->pcie_ue_status & \
1362             ((PCIE_ADV_REG(pfd_p)->pcie_ue_mask) ^ 0xFFFFFFFF))
1363 #define PF_MASKED_SAER_ERR(pfd_p) \
1364         (PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status & \
1365             ((PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_mask) ^ 0xFFFFFFFF))
1366 /*
1367  * Analyse all the PCIe Fault Data (erpt) gathered during dispatch in the erpt
1368  * Queue.
1369  */
1370 static int
1371 pf_analyse_error(ddi_fm_error_t *derr, pf_impl_t *impl)
1372 {
1373         int             sts_flags, error_flags = 0;
1374         pf_data_t       *pfd_p;
1375 
1376         for (pfd_p = impl->pf_dq_head_p; pfd_p; pfd_p = pfd_p->pe_next) {
1377                 sts_flags = 0;
1378 
1379                 /* skip analysing error when no error info is gathered */
1380                 if (pfd_p->pe_severity_flags == PF_ERR_PANIC_BAD_RESPONSE)
1381                         goto done;
1382 
1383                 switch (PCIE_PFD2BUS(pfd_p)->bus_dev_type) {
1384                 case PCIE_PCIECAP_DEV_TYPE_PCIE_DEV:
1385                 case PCIE_PCIECAP_DEV_TYPE_PCI_DEV:
1386                         if (PCIE_DEVSTS_CE_DETECTED &
1387                             PCIE_ERR_REG(pfd_p)->pcie_err_status)
1388                                 sts_flags |= PF_ERR_CE;
1389 
1390                         pf_adjust_for_no_aer(pfd_p);
1391                         sts_flags |= pf_analyse_error_tbl(derr, impl,
1392                             pfd_p, pcie_pcie_tbl, PF_MASKED_AER_ERR(pfd_p));
1393                         break;
1394                 case PCIE_PCIECAP_DEV_TYPE_ROOT:
1395                         pf_adjust_for_no_aer(pfd_p);
1396                         sts_flags |= pf_analyse_error_tbl(derr, impl,
1397                             pfd_p, pcie_rp_tbl, PF_MASKED_AER_ERR(pfd_p));
1398                         break;
1399                 case PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO:
1400                         /* no adjust_for_aer for pseudo RC */
1401                         /* keep the severity passed on from RC if any */
1402                         sts_flags |= pfd_p->pe_severity_flags;
1403                         sts_flags |= pf_analyse_error_tbl(derr, impl, pfd_p,
1404                             pcie_rp_tbl, PF_MASKED_AER_ERR(pfd_p));
1405                         break;
1406                 case PCIE_PCIECAP_DEV_TYPE_UP:
1407                 case PCIE_PCIECAP_DEV_TYPE_DOWN:
1408                         if (PCIE_DEVSTS_CE_DETECTED &
1409                             PCIE_ERR_REG(pfd_p)->pcie_err_status)
1410                                 sts_flags |= PF_ERR_CE;
1411 
1412                         pf_adjust_for_no_aer(pfd_p);
1413                         sts_flags |= pf_analyse_error_tbl(derr, impl,
1414                             pfd_p, pcie_sw_tbl, PF_MASKED_AER_ERR(pfd_p));
1415                         break;
1416                 case PCIE_PCIECAP_DEV_TYPE_PCIE2PCI:
1417                         if (PCIE_DEVSTS_CE_DETECTED &
1418                             PCIE_ERR_REG(pfd_p)->pcie_err_status)
1419                                 sts_flags |= PF_ERR_CE;
1420 
1421                         pf_adjust_for_no_aer(pfd_p);
1422                         pf_adjust_for_no_saer(pfd_p);
1423                         sts_flags |= pf_analyse_error_tbl(derr,
1424                             impl, pfd_p, pcie_pcie_tbl,
1425                             PF_MASKED_AER_ERR(pfd_p));
1426                         sts_flags |= pf_analyse_error_tbl(derr,
1427                             impl, pfd_p, pcie_pcie_bdg_tbl,
1428                             PF_MASKED_SAER_ERR(pfd_p));
1429                         /*
1430                          * Some non-compliant PCIe devices do not utilize PCIe
1431                          * error registers.  So fallthrough and rely on legacy
1432                          * PCI error registers.
1433                          */
1434                         if ((PCIE_DEVSTS_NFE_DETECTED | PCIE_DEVSTS_FE_DETECTED)
1435                             & PCIE_ERR_REG(pfd_p)->pcie_err_status)
1436                                 break;
1437                         /* FALLTHROUGH */
1438                 case PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO:
1439                         sts_flags |= pf_analyse_error_tbl(derr, impl,
1440                             pfd_p, pcie_pci_tbl,
1441                             PCI_ERR_REG(pfd_p)->pci_err_status);
1442 
1443                         if (!PCIE_IS_BDG(PCIE_PFD2BUS(pfd_p)))
1444                                 break;
1445 
1446                         sts_flags |= pf_analyse_error_tbl(derr,
1447                             impl, pfd_p, pcie_pci_bdg_tbl,
1448                             PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat);
1449                 }
1450 
1451                 pfd_p->pe_severity_flags = sts_flags;
1452 
1453 done:
1454                 pfd_p->pe_orig_severity_flags = pfd_p->pe_severity_flags;
1455                 /* Have pciev_eh adjust the severity */
1456                 pfd_p->pe_severity_flags = pciev_eh(pfd_p, impl);
1457 
1458                 error_flags |= pfd_p->pe_severity_flags;
1459         }
1460 
1461         return (error_flags);
1462 }
1463 
1464 static int
1465 pf_analyse_error_tbl(ddi_fm_error_t *derr, pf_impl_t *impl,
1466     pf_data_t *pfd_p, const pf_fab_err_tbl_t *tbl, uint32_t err_reg)
1467 {
1468         const pf_fab_err_tbl_t *row;
1469         int err = 0;
1470         uint16_t flags;
1471         uint32_t bit;
1472 
1473         for (row = tbl; err_reg && (row->bit != NULL); row++) {
1474                 bit = row->bit;
1475                 if (!(err_reg & bit))
1476                         continue;
1477                 err |= row->handler(derr, bit, impl->pf_dq_head_p, pfd_p);
1478 
1479                 flags = row->affected_flags;
1480                 /*
1481                  * check if the primary flag is valid;
1482                  * if not, use the secondary flag
1483                  */
1484                 if (flags & PF_AFFECTED_AER) {
1485                         if (!HAS_AER_LOGS(pfd_p, bit)) {
1486                                 flags = row->sec_affected_flags;
1487                         }
1488                 } else if (flags & PF_AFFECTED_SAER) {
1489                         if (!HAS_SAER_LOGS(pfd_p, bit)) {
1490                                 flags = row->sec_affected_flags;
1491                         }
1492                 } else if (flags & PF_AFFECTED_ADDR) {
1493                         /* only Root has this flag */
1494                         if (PCIE_ROOT_FAULT(pfd_p)->scan_addr == 0) {
1495                                 flags = row->sec_affected_flags;
1496                         }
1497                 }
1498 
1499                 PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags |= flags;
1500         }
1501 
1502         if (!err)
1503                 err = PF_ERR_NO_ERROR;
1504 
1505         return (err);
1506 }
1507 
1508 /*
1509  * PCIe Completer Abort and Unsupport Request error analyser.  If a PCIe device
1510  * issues a CA/UR a corresponding Received CA/UR should have been seen in the
1511  * PCIe root complex.  Check to see if RC did indeed receive a CA/UR, if so then
1512  * this error may be safely ignored.  If not check the logs and see if an
1513  * associated handler for this transaction can be found.
1514  */
1515 /* ARGSUSED */
1516 static int
1517 pf_analyse_ca_ur(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1518     pf_data_t *pfd_p)
1519 {
1520         uint32_t        abort_type;
1521         dev_info_t      *rpdip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
1522 
1523         /* If UR's are masked forgive this error */
1524         if ((pcie_get_aer_uce_mask() & PCIE_AER_UCE_UR) &&
1525             (bit == PCIE_AER_UCE_UR))
1526                 return (PF_ERR_NO_PANIC);
1527 
1528         /*
1529          * If a RP has an CA/UR it means a leaf sent a bad request to the RP
1530          * such as a config read or a bad DMA address.
1531          */
1532         if (PCIE_IS_RP(PCIE_PFD2BUS(pfd_p)))
1533                 goto handle_lookup;
1534 
1535         if (bit == PCIE_AER_UCE_UR)
1536                 abort_type = PCI_STAT_R_MAST_AB;
1537         else
1538                 abort_type = PCI_STAT_R_TARG_AB;
1539 
1540         if (pf_matched_in_rc(dq_head_p, pfd_p, abort_type))
1541                 return (PF_ERR_MATCHED_RC);
1542 
1543 handle_lookup:
1544         if (HAS_AER_LOGS(pfd_p, bit) &&
1545             pf_log_hdl_lookup(rpdip, derr, pfd_p, B_TRUE) == PF_HDL_FOUND)
1546                         return (PF_ERR_MATCHED_DEVICE);
1547 
1548         return (PF_ERR_PANIC);
1549 }
1550 
1551 /*
1552  * PCIe-PCI Bridge Received Master Abort and Target error analyser.  If a PCIe
1553  * Bridge receives a MA/TA a corresponding sent CA/UR should have been seen in
1554  * the PCIe root complex.  Check to see if RC did indeed receive a CA/UR, if so
1555  * then this error may be safely ignored.  If not check the logs and see if an
1556  * associated handler for this transaction can be found.
1557  */
1558 /* ARGSUSED */
1559 static int
1560 pf_analyse_ma_ta(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1561     pf_data_t *pfd_p)
1562 {
1563         dev_info_t      *rpdip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
1564         uint32_t        abort_type;
1565 
1566         /* If UR's are masked forgive this error */
1567         if ((pcie_get_aer_uce_mask() & PCIE_AER_UCE_UR) &&
1568             (bit == PCIE_AER_SUCE_RCVD_MA))
1569                 return (PF_ERR_NO_PANIC);
1570 
1571         if (bit == PCIE_AER_SUCE_RCVD_MA)
1572                 abort_type = PCI_STAT_R_MAST_AB;
1573         else
1574                 abort_type = PCI_STAT_R_TARG_AB;
1575 
1576         if (pf_matched_in_rc(dq_head_p, pfd_p, abort_type))
1577                 return (PF_ERR_MATCHED_RC);
1578 
1579         if (!HAS_SAER_LOGS(pfd_p, bit))
1580                 return (PF_ERR_PANIC);
1581 
1582         if (pf_log_hdl_lookup(rpdip, derr, pfd_p, B_FALSE) == PF_HDL_FOUND)
1583                 return (PF_ERR_MATCHED_DEVICE);
1584 
1585         return (PF_ERR_PANIC);
1586 }
1587 
1588 /*
1589  * Generic PCI error analyser.  This function is used for Parity Errors,
1590  * Received Master Aborts, Received Target Aborts, and Signaled Target Aborts.
1591  * In general PCI devices do not have error logs, it is very difficult to figure
1592  * out what transaction caused the error.  Instead find the nearest PCIe-PCI
1593  * Bridge and check to see if it has logs and if it has an error associated with
1594  * this PCI Device.
1595  */
1596 /* ARGSUSED */
1597 static int
1598 pf_analyse_pci(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1599     pf_data_t *pfd_p)
1600 {
1601         pf_data_t       *parent_pfd_p;
1602         uint16_t        cmd;
1603         uint32_t        aer_ue_status;
1604         pcie_bus_t      *bus_p = PCIE_PFD2BUS(pfd_p);
1605         pf_pcie_adv_bdg_err_regs_t *parent_saer_p;
1606 
1607         if (PCI_ERR_REG(pfd_p)->pci_err_status & PCI_STAT_S_SYSERR)
1608                 return (PF_ERR_PANIC);
1609 
1610         /* If UR's are masked forgive this error */
1611         if ((pcie_get_aer_uce_mask() & PCIE_AER_UCE_UR) &&
1612             (bit == PCI_STAT_R_MAST_AB))
1613                 return (PF_ERR_NO_PANIC);
1614 
1615 
1616         if (bit & (PCI_STAT_PERROR | PCI_STAT_S_PERROR)) {
1617                 aer_ue_status = PCIE_AER_SUCE_PERR_ASSERT;
1618         } else {
1619                 aer_ue_status = (PCIE_AER_SUCE_TA_ON_SC |
1620                     PCIE_AER_SUCE_MA_ON_SC | PCIE_AER_SUCE_RCVD_TA |
1621                     PCIE_AER_SUCE_RCVD_MA);
1622         }
1623 
1624         parent_pfd_p = pf_get_parent_pcie_bridge(pfd_p);
1625         if (parent_pfd_p == NULL)
1626                 return (PF_ERR_PANIC);
1627 
1628         /* Check if parent bridge has seen this error */
1629         parent_saer_p = PCIE_ADV_BDG_REG(parent_pfd_p);
1630         if (!(parent_saer_p->pcie_sue_status & aer_ue_status) ||
1631             !HAS_SAER_LOGS(parent_pfd_p, aer_ue_status))
1632                 return (PF_ERR_PANIC);
1633 
1634         /*
1635          * If the addr or bdf from the parent PCIe bridge logs belong to this
1636          * PCI device, assume the PCIe bridge's error handling has already taken
1637          * care of this PCI device's error.
1638          */
1639         if (pf_pci_decode(parent_pfd_p, &cmd) != DDI_SUCCESS)
1640                 return (PF_ERR_PANIC);
1641 
1642         if ((parent_saer_p->pcie_sue_tgt_bdf == bus_p->bus_bdf) ||
1643             pf_in_addr_range(bus_p, parent_saer_p->pcie_sue_tgt_addr))
1644                 return (PF_ERR_MATCHED_PARENT);
1645 
1646         /*
1647          * If this device is a PCI-PCI bridge, check if the bdf in the parent
1648          * PCIe bridge logs is in the range of this PCI-PCI Bridge's bus ranges.
1649          * If they are, then assume the PCIe bridge's error handling has already
1650          * taken care of this PCI-PCI bridge device's error.
1651          */
1652         if (PCIE_IS_BDG(bus_p) &&
1653             pf_in_bus_range(bus_p, parent_saer_p->pcie_sue_tgt_bdf))
1654                 return (PF_ERR_MATCHED_PARENT);
1655 
1656         return (PF_ERR_PANIC);
1657 }
1658 
1659 /*
1660  * PCIe Bridge transactions associated with PERR.
1661  * o Bridge received a poisoned Non-Posted Write (CFG Writes) from PCIe
1662  * o Bridge received a poisoned Posted Write from (MEM Writes) from PCIe
1663  * o Bridge received a poisoned Completion on a Split Transction from PCIe
1664  * o Bridge received a poisoned Completion on a Delayed Transction from PCIe
1665  *
1666  * Check for non-poisoned PCIe transactions that got forwarded to the secondary
1667  * side and detects a PERR#.  Except for delayed read completions, a poisoned
1668  * TLP will be forwarded to the secondary bus and PERR# will be asserted.
1669  */
1670 /* ARGSUSED */
1671 static int
1672 pf_analyse_perr_assert(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1673     pf_data_t *pfd_p)
1674 {
1675         dev_info_t      *rpdip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
1676         uint16_t        cmd;
1677         int             hdl_sts = PF_HDL_NOTFOUND;
1678         int             err = PF_ERR_NO_ERROR;
1679         pf_pcie_adv_bdg_err_regs_t *saer_p;
1680 
1681 
1682         if (HAS_SAER_LOGS(pfd_p, bit)) {
1683                 saer_p = PCIE_ADV_BDG_REG(pfd_p);
1684                 if (pf_pci_decode(pfd_p, &cmd) != DDI_SUCCESS)
1685                         return (PF_ERR_PANIC);
1686 
1687 cmd_switch:
1688                 switch (cmd) {
1689                 case PCI_PCIX_CMD_IOWR:
1690                 case PCI_PCIX_CMD_MEMWR:
1691                 case PCI_PCIX_CMD_MEMWR_BL:
1692                 case PCI_PCIX_CMD_MEMWRBL:
1693                         /* Posted Writes Transactions */
1694                         if (saer_p->pcie_sue_tgt_trans == PF_ADDR_PIO)
1695                                 hdl_sts = pf_log_hdl_lookup(rpdip, derr, pfd_p,
1696                                     B_FALSE);
1697                         break;
1698                 case PCI_PCIX_CMD_CFWR:
1699                         /*
1700                          * Check to see if it is a non-posted write.  If so, a
1701                          * UR Completion would have been sent.
1702                          */
1703                         if (pf_matched_in_rc(dq_head_p, pfd_p,
1704                             PCI_STAT_R_MAST_AB)) {
1705                                 hdl_sts = PF_HDL_FOUND;
1706                                 err = PF_ERR_MATCHED_RC;
1707                                 goto done;
1708                         }
1709                         hdl_sts = pf_log_hdl_lookup(rpdip, derr, pfd_p,
1710                             B_FALSE);
1711                         break;
1712                 case PCI_PCIX_CMD_SPL:
1713                         hdl_sts = pf_log_hdl_lookup(rpdip, derr, pfd_p,
1714                             B_FALSE);
1715                         break;
1716                 case PCI_PCIX_CMD_DADR:
1717                         cmd = (PCIE_ADV_BDG_HDR(pfd_p, 1) >>
1718                             PCIE_AER_SUCE_HDR_CMD_UP_SHIFT) &
1719                             PCIE_AER_SUCE_HDR_CMD_UP_MASK;
1720                         if (cmd != PCI_PCIX_CMD_DADR)
1721                                 goto cmd_switch;
1722                         /* FALLTHROUGH */
1723                 default:
1724                         /* Unexpected situation, panic */
1725                         hdl_sts = PF_HDL_NOTFOUND;
1726                 }
1727 
1728                 if (hdl_sts == PF_HDL_FOUND)
1729                         err = PF_ERR_MATCHED_DEVICE;
1730                 else
1731                         err = PF_ERR_PANIC;
1732         } else {
1733                 /*
1734                  * Check to see if it is a non-posted write.  If so, a UR
1735                  * Completion would have been sent.
1736                  */
1737                 if ((PCIE_ERR_REG(pfd_p)->pcie_err_status &
1738                     PCIE_DEVSTS_UR_DETECTED) &&
1739                     pf_matched_in_rc(dq_head_p, pfd_p, PCI_STAT_R_MAST_AB))
1740                         err = PF_ERR_MATCHED_RC;
1741 
1742                 /* Check for posted writes.  Transaction is lost. */
1743                 if (PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat &
1744                     PCI_STAT_S_PERROR)
1745                         err = PF_ERR_PANIC;
1746 
1747                 /*
1748                  * All other scenarios are due to read completions.  Check for
1749                  * PERR on the primary side.  If found the primary side error
1750                  * handling will take care of this error.
1751                  */
1752                 if (err == PF_ERR_NO_ERROR) {
1753                         if (PCI_ERR_REG(pfd_p)->pci_err_status &
1754                             PCI_STAT_PERROR)
1755                                 err = PF_ERR_MATCHED_PARENT;
1756                         else
1757                                 err = PF_ERR_PANIC;
1758                 }
1759         }
1760 
1761 done:
1762         return (err);
1763 }
1764 
1765 /*
1766  * PCIe Poisoned TLP error analyser.  If a PCIe device receives a Poisoned TLP,
1767  * check the logs and see if an associated handler for this transaction can be
1768  * found.
1769  */
1770 /* ARGSUSED */
1771 static int
1772 pf_analyse_ptlp(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1773     pf_data_t *pfd_p)
1774 {
1775         dev_info_t      *rpdip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
1776 
1777         /*
1778          * If AERs are supported find the logs in this device, otherwise look in
1779          * it's parent's logs.
1780          */
1781         if (HAS_AER_LOGS(pfd_p, bit)) {
1782                 pcie_tlp_hdr_t *hdr = (pcie_tlp_hdr_t *)&PCIE_ADV_HDR(pfd_p, 0);
1783 
1784                 /*
1785                  * Double check that the log contains a poisoned TLP.
1786                  * Some devices like PLX switch do not log poison TLP headers.
1787                  */
1788                 if (hdr->ep) {
1789                         if (pf_log_hdl_lookup(rpdip, derr, pfd_p, B_TRUE) ==
1790                             PF_HDL_FOUND)
1791                                 return (PF_ERR_MATCHED_DEVICE);
1792                 }
1793 
1794                 /*
1795                  * If an address is found and hdl lookup failed panic.
1796                  * Otherwise check parents to see if there was enough
1797                  * information recover.
1798                  */
1799                 if (PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr)
1800                         return (PF_ERR_PANIC);
1801         }
1802 
1803         /*
1804          * Check to see if the rc has already handled this error or a parent has
1805          * already handled this error.
1806          *
1807          * If the error info in the RC wasn't enough to find the fault device,
1808          * such as if the faulting device lies behind a PCIe-PCI bridge from a
1809          * poisoned completion, check to see if the PCIe-PCI bridge has enough
1810          * info to recover.  For completion TLP's, the AER header logs only
1811          * contain the faulting BDF in the Root Port.  For PCIe device the fault
1812          * BDF is the fault device.  But if the fault device is behind a
1813          * PCIe-PCI bridge the fault BDF could turn out just to be a PCIe-PCI
1814          * bridge's secondary bus number.
1815          */
1816         if (!PFD_IS_ROOT(pfd_p)) {
1817                 dev_info_t *pdip = ddi_get_parent(PCIE_PFD2DIP(pfd_p));
1818                 pf_data_t *parent_pfd_p;
1819 
1820                 if (PCIE_PFD2BUS(pfd_p)->bus_rp_dip == pdip) {
1821                         if (pf_matched_in_rc(dq_head_p, pfd_p, PCI_STAT_PERROR))
1822                                 return (PF_ERR_MATCHED_RC);
1823                 }
1824 
1825                 parent_pfd_p = PCIE_DIP2PFD(pdip);
1826 
1827                 if (HAS_AER_LOGS(parent_pfd_p, bit))
1828                         return (PF_ERR_MATCHED_PARENT);
1829         } else {
1830                 pf_data_t *bdg_pfd_p;
1831                 pcie_req_id_t secbus;
1832 
1833                 /*
1834                  * Looking for a pcie bridge only makes sense if the BDF
1835                  * Dev/Func = 0/0
1836                  */
1837                 if (!PCIE_HAS_AER(PCIE_PFD2BUS(pfd_p)))
1838                         goto done;
1839 
1840                 secbus = PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf;
1841 
1842                 if (!PCIE_CHECK_VALID_BDF(secbus) || (secbus & 0xFF))
1843                         goto done;
1844 
1845                 bdg_pfd_p = pf_get_pcie_bridge(pfd_p, secbus);
1846 
1847                 if (bdg_pfd_p && HAS_SAER_LOGS(bdg_pfd_p,
1848                     PCIE_AER_SUCE_PERR_ASSERT)) {
1849                         return pf_analyse_perr_assert(derr,
1850                             PCIE_AER_SUCE_PERR_ASSERT, dq_head_p, pfd_p);
1851                 }
1852         }
1853 done:
1854         return (PF_ERR_PANIC);
1855 }
1856 
1857 /*
1858  * PCIe-PCI Bridge Received Master and Target abort error analyser on Split
1859  * Completions.  If a PCIe Bridge receives a MA/TA check logs and see if an
1860  * associated handler for this transaction can be found.
1861  */
1862 /* ARGSUSED */
1863 static int
1864 pf_analyse_sc(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1865     pf_data_t *pfd_p)
1866 {
1867         dev_info_t      *rpdip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
1868         uint16_t        cmd;
1869         int             sts = PF_HDL_NOTFOUND;
1870 
1871         if (!HAS_SAER_LOGS(pfd_p, bit))
1872                 return (PF_ERR_PANIC);
1873 
1874         if (pf_pci_decode(pfd_p, &cmd) != DDI_SUCCESS)
1875                 return (PF_ERR_PANIC);
1876 
1877         if (cmd == PCI_PCIX_CMD_SPL)
1878                 sts = pf_log_hdl_lookup(rpdip, derr, pfd_p, B_FALSE);
1879 
1880         if (sts == PF_HDL_FOUND)
1881                 return (PF_ERR_MATCHED_DEVICE);
1882 
1883         return (PF_ERR_PANIC);
1884 }
1885 
1886 /*
1887  * PCIe Timeout error analyser.  This error can be forgiven if it is marked as
1888  * CE Advisory.  If it is marked as advisory, this means the HW can recover
1889  * and/or retry the transaction automatically.
1890  */
1891 /* ARGSUSED */
1892 static int
1893 pf_analyse_to(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1894     pf_data_t *pfd_p)
1895 {
1896         if (HAS_AER_LOGS(pfd_p, bit) && CE_ADVISORY(pfd_p))
1897                 return (PF_ERR_NO_PANIC);
1898 
1899         return (PF_ERR_PANIC);
1900 }
1901 
1902 /*
1903  * PCIe Unexpected Completion.  Check to see if this TLP was misrouted by
1904  * matching the device BDF with the TLP Log.  If misrouting panic, otherwise
1905  * don't panic.
1906  */
1907 /* ARGSUSED */
1908 static int
1909 pf_analyse_uc(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1910     pf_data_t *pfd_p)
1911 {
1912         if (HAS_AER_LOGS(pfd_p, bit) &&
1913             (PCIE_PFD2BUS(pfd_p)->bus_bdf == (PCIE_ADV_HDR(pfd_p, 2) >> 16)))
1914                 return (PF_ERR_NO_PANIC);
1915 
1916         /*
1917          * This is a case of mis-routing. Any of the switches above this
1918          * device could be at fault.
1919          */
1920         PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags = PF_AFFECTED_ROOT;
1921 
1922         return (PF_ERR_PANIC);
1923 }
1924 
1925 /*
1926  * PCIe-PCI Bridge Uncorrectable Data error analyser.  All Uncorrectable Data
1927  * errors should have resulted in a PCIe Poisoned TLP to the RC, except for
1928  * Posted Writes.  Check the logs for Posted Writes and if the RC did not see a
1929  * Poisoned TLP.
1930  *
1931  * Non-Posted Writes will also generate a UR in the completion status, which the
1932  * RC should also see.
1933  */
1934 /* ARGSUSED */
1935 static int
1936 pf_analyse_uc_data(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1937     pf_data_t *pfd_p)
1938 {
1939         dev_info_t      *rpdip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
1940 
1941         if (!HAS_SAER_LOGS(pfd_p, bit))
1942                 return (PF_ERR_PANIC);
1943 
1944         if (pf_matched_in_rc(dq_head_p, pfd_p, PCI_STAT_PERROR))
1945                 return (PF_ERR_MATCHED_RC);
1946 
1947         if (pf_log_hdl_lookup(rpdip, derr, pfd_p, B_FALSE) == PF_HDL_FOUND)
1948                 return (PF_ERR_MATCHED_DEVICE);
1949 
1950         return (PF_ERR_PANIC);
1951 }
1952 
1953 /* ARGSUSED */
1954 static int
1955 pf_no_panic(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1956     pf_data_t *pfd_p)
1957 {
1958         return (PF_ERR_NO_PANIC);
1959 }
1960 
1961 /* ARGSUSED */
1962 static int
1963 pf_panic(ddi_fm_error_t *derr, uint32_t bit, pf_data_t *dq_head_p,
1964     pf_data_t *pfd_p)
1965 {
1966         return (PF_ERR_PANIC);
1967 }
1968 
1969 /*
1970  * If a PCIe device does not support AER, assume all AER statuses have been set,
1971  * unless other registers do not indicate a certain error occuring.
1972  */
1973 static void
1974 pf_adjust_for_no_aer(pf_data_t *pfd_p)
1975 {
1976         uint32_t        aer_ue = 0;
1977         uint16_t        status;
1978 
1979         if (PCIE_HAS_AER(PCIE_PFD2BUS(pfd_p)))
1980                 return;
1981 
1982         if (PCIE_ERR_REG(pfd_p)->pcie_err_status & PCIE_DEVSTS_FE_DETECTED)
1983                 aer_ue = PF_AER_FATAL_ERR;
1984 
1985         if (PCIE_ERR_REG(pfd_p)->pcie_err_status & PCIE_DEVSTS_NFE_DETECTED) {
1986                 aer_ue = PF_AER_NON_FATAL_ERR;
1987                 status = PCI_ERR_REG(pfd_p)->pci_err_status;
1988 
1989                 /* Check if the device received a PTLP */
1990                 if (!(status & PCI_STAT_PERROR))
1991                         aer_ue &= ~PCIE_AER_UCE_PTLP;
1992 
1993                 /* Check if the device signaled a CA */
1994                 if (!(status & PCI_STAT_S_TARG_AB))
1995                         aer_ue &= ~PCIE_AER_UCE_CA;
1996 
1997                 /* Check if the device sent a UR */
1998                 if (!(PCIE_ERR_REG(pfd_p)->pcie_err_status &
1999                     PCIE_DEVSTS_UR_DETECTED))
2000                         aer_ue &= ~PCIE_AER_UCE_UR;
2001 
2002                 /*
2003                  * Ignore ECRCs as it is optional and will manefest itself as
2004                  * another error like PTLP and MFP
2005                  */
2006                 aer_ue &= ~PCIE_AER_UCE_ECRC;
2007 
2008                 /*
2009                  * Generally if NFE is set, SERR should also be set. Exception:
2010                  * When certain non-fatal errors are masked, and some of them
2011                  * happened to be the cause of the NFE, SERR will not be set and
2012                  * they can not be the source of this interrupt.
2013                  *
2014                  * On x86, URs are masked (NFE + UR can be set), if any other
2015                  * non-fatal errors (i.e, PTLP, CTO, CA, UC, ECRC, ACS) did
2016                  * occur, SERR should be set since they are not masked. So if
2017                  * SERR is not set, none of them occurred.
2018                  */
2019                 if (!(status & PCI_STAT_S_SYSERR))
2020                         aer_ue &= ~PCIE_AER_UCE_TO;
2021         }
2022 
2023         if (!PCIE_IS_BDG(PCIE_PFD2BUS(pfd_p))) {
2024                 aer_ue &= ~PCIE_AER_UCE_TRAINING;
2025                 aer_ue &= ~PCIE_AER_UCE_SD;
2026         }
2027 
2028         PCIE_ADV_REG(pfd_p)->pcie_ue_status = aer_ue;
2029 }
2030 
2031 static void
2032 pf_adjust_for_no_saer(pf_data_t *pfd_p)
2033 {
2034         uint32_t        s_aer_ue = 0;
2035         uint16_t        status;
2036 
2037         if (PCIE_HAS_AER(PCIE_PFD2BUS(pfd_p)))
2038                 return;
2039 
2040         if (PCIE_ERR_REG(pfd_p)->pcie_err_status & PCIE_DEVSTS_FE_DETECTED)
2041                 s_aer_ue = PF_SAER_FATAL_ERR;
2042 
2043         if (PCIE_ERR_REG(pfd_p)->pcie_err_status & PCIE_DEVSTS_NFE_DETECTED) {
2044                 s_aer_ue = PF_SAER_NON_FATAL_ERR;
2045                 status = PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat;
2046 
2047                 /* Check if the device received a UC_DATA */
2048                 if (!(status & PCI_STAT_PERROR))
2049                         s_aer_ue &= ~PCIE_AER_SUCE_UC_DATA_ERR;
2050 
2051                 /* Check if the device received a RCVD_MA/MA_ON_SC */
2052                 if (!(status & (PCI_STAT_R_MAST_AB))) {
2053                         s_aer_ue &= ~PCIE_AER_SUCE_RCVD_MA;
2054                         s_aer_ue &= ~PCIE_AER_SUCE_MA_ON_SC;
2055                 }
2056 
2057                 /* Check if the device received a RCVD_TA/TA_ON_SC */
2058                 if (!(status & (PCI_STAT_R_TARG_AB))) {
2059                         s_aer_ue &= ~PCIE_AER_SUCE_RCVD_TA;
2060                         s_aer_ue &= ~PCIE_AER_SUCE_TA_ON_SC;
2061                 }
2062         }
2063 
2064         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status = s_aer_ue;
2065 }
2066 
2067 /* Find the PCIe-PCI bridge based on secondary bus number */
2068 static pf_data_t *
2069 pf_get_pcie_bridge(pf_data_t *pfd_p, pcie_req_id_t secbus)
2070 {
2071         pf_data_t *bdg_pfd_p;
2072 
2073         /* Search down for the PCIe-PCI device. */
2074         for (bdg_pfd_p = pfd_p->pe_next; bdg_pfd_p;
2075             bdg_pfd_p = bdg_pfd_p->pe_next) {
2076                 if (PCIE_IS_PCIE_BDG(PCIE_PFD2BUS(bdg_pfd_p)) &&
2077                     PCIE_PFD2BUS(bdg_pfd_p)->bus_bdg_secbus == secbus)
2078                         return (bdg_pfd_p);
2079         }
2080 
2081         return (NULL);
2082 }
2083 
2084 /* Find the PCIe-PCI bridge of a PCI device */
2085 static pf_data_t *
2086 pf_get_parent_pcie_bridge(pf_data_t *pfd_p)
2087 {
2088         dev_info_t      *dip, *rp_dip = PCIE_PFD2BUS(pfd_p)->bus_rp_dip;
2089 
2090         /* This only makes sense if the device is a PCI device */
2091         if (!PCIE_IS_PCI(PCIE_PFD2BUS(pfd_p)))
2092                 return (NULL);
2093 
2094         /*
2095          * Search up for the PCIe-PCI device.  Watchout for x86 where pci
2096          * devices hang directly off of NPE.
2097          */
2098         for (dip = PCIE_PFD2DIP(pfd_p); dip; dip = ddi_get_parent(dip)) {
2099                 if (dip == rp_dip)
2100                         dip = NULL;
2101 
2102                 if (PCIE_IS_PCIE_BDG(PCIE_DIP2BUS(dip)))
2103                         return (PCIE_DIP2PFD(dip));
2104         }
2105 
2106         return (NULL);
2107 }
2108 
2109 /*
2110  * See if a leaf error was bubbled up to the Root Complex (RC) and handled.
2111  * As of right now only RC's have enough information to have errors found in the
2112  * fabric to be matched to the RC.  Note that Root Port's (RP) do not carry
2113  * enough information.  Currently known RC's are SPARC Fire architecture and
2114  * it's equivalents, and x86's NPE.
2115  * SPARC Fire architectures have a plethora of error registers, while currently
2116  * NPE only have the address of a failed load.
2117  *
2118  * Check if the RC logged an error with the appropriate status type/abort type.
2119  * Ex: Parity Error, Received Master/Target Abort
2120  * Check if either the fault address found in the rc matches the device's
2121  * assigned address range (PIO's only) or the fault BDF in the rc matches the
2122  * device's BDF or Secondary Bus/Bus Range.
2123  */
2124 static boolean_t
2125 pf_matched_in_rc(pf_data_t *dq_head_p, pf_data_t *pfd_p,
2126     uint32_t abort_type)
2127 {
2128         pcie_bus_t      *bus_p = PCIE_PFD2BUS(pfd_p);
2129         pf_data_t       *rc_pfd_p;
2130         pcie_req_id_t   fault_bdf;
2131 
2132         for (rc_pfd_p = dq_head_p; PFD_IS_ROOT(rc_pfd_p);
2133             rc_pfd_p = rc_pfd_p->pe_next) {
2134                 /* Only root complex's have enough information to match */
2135                 if (!PCIE_IS_RC(PCIE_PFD2BUS(rc_pfd_p)))
2136                         continue;
2137 
2138                 /* If device and rc abort type does not match continue */
2139                 if (!(PCI_BDG_ERR_REG(rc_pfd_p)->pci_bdg_sec_stat & abort_type))
2140                         continue;
2141 
2142                 fault_bdf = PCIE_ROOT_FAULT(rc_pfd_p)->scan_bdf;
2143 
2144                 /* The Fault BDF = Device's BDF */
2145                 if (fault_bdf == bus_p->bus_bdf)
2146                         return (B_TRUE);
2147 
2148                 /* The Fault Addr is in device's address range */
2149                 if (pf_in_addr_range(bus_p,
2150                     PCIE_ROOT_FAULT(rc_pfd_p)->scan_addr))
2151                         return (B_TRUE);
2152 
2153                 /* The Fault BDF is from PCIe-PCI Bridge's secondary bus */
2154                 if (PCIE_IS_PCIE_BDG(bus_p) &&
2155                     pf_in_bus_range(bus_p, fault_bdf))
2156                         return (B_TRUE);
2157         }
2158 
2159         return (B_FALSE);
2160 }
2161 
2162 /*
2163  * Check the RP and see if the error is PIO/DMA.  If the RP also has a PERR then
2164  * it is a DMA, otherwise it's a PIO
2165  */
2166 static void
2167 pf_pci_find_trans_type(pf_data_t *pfd_p, uint64_t *addr, uint32_t *trans_type,
2168     pcie_req_id_t *bdf) {
2169         pf_data_t *rc_pfd_p;
2170 
2171         /* Could be DMA or PIO.  Find out by look at error type. */
2172         switch (PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status) {
2173         case PCIE_AER_SUCE_TA_ON_SC:
2174         case PCIE_AER_SUCE_MA_ON_SC:
2175                 *trans_type = PF_ADDR_DMA;
2176                 return;
2177         case PCIE_AER_SUCE_RCVD_TA:
2178         case PCIE_AER_SUCE_RCVD_MA:
2179                 *bdf = PCIE_INVALID_BDF;
2180                 *trans_type = PF_ADDR_PIO;
2181                 return;
2182         case PCIE_AER_SUCE_USC_ERR:
2183         case PCIE_AER_SUCE_UC_DATA_ERR:
2184         case PCIE_AER_SUCE_PERR_ASSERT:
2185                 break;
2186         default:
2187                 *addr = 0;
2188                 *bdf = PCIE_INVALID_BDF;
2189                 *trans_type = 0;
2190                 return;
2191         }
2192 
2193         *bdf = PCIE_INVALID_BDF;
2194         *trans_type = PF_ADDR_PIO;
2195         for (rc_pfd_p = pfd_p->pe_prev; rc_pfd_p;
2196             rc_pfd_p = rc_pfd_p->pe_prev) {
2197                 if (PFD_IS_ROOT(rc_pfd_p) &&
2198                     (PCI_BDG_ERR_REG(rc_pfd_p)->pci_bdg_sec_stat &
2199                     PCI_STAT_PERROR)) {
2200                         *trans_type = PF_ADDR_DMA;
2201                         return;
2202                 }
2203         }
2204 }
2205 
2206 /*
2207  * pf_pci_decode function decodes the secondary aer transaction logs in
2208  * PCIe-PCI bridges.
2209  *
2210  * The log is 128 bits long and arranged in this manner.
2211  * [0:35]   Transaction Attribute       (s_aer_h0-saer_h1)
2212  * [36:39]  Transaction lower command   (saer_h1)
2213  * [40:43]  Transaction upper command   (saer_h1)
2214  * [44:63]  Reserved
2215  * [64:127] Address                     (saer_h2-saer_h3)
2216  */
2217 /* ARGSUSED */
2218 int
2219 pf_pci_decode(pf_data_t *pfd_p, uint16_t *cmd) {
2220         pcix_attr_t     *attr;
2221         uint64_t        addr;
2222         uint32_t        trans_type;
2223         pcie_req_id_t   bdf = PCIE_INVALID_BDF;
2224 
2225         attr = (pcix_attr_t *)&PCIE_ADV_BDG_HDR(pfd_p, 0);
2226         *cmd = GET_SAER_CMD(pfd_p);
2227 
2228 cmd_switch:
2229         switch (*cmd) {
2230         case PCI_PCIX_CMD_IORD:
2231         case PCI_PCIX_CMD_IOWR:
2232                 /* IO Access should always be down stream */
2233                 addr = PCIE_ADV_BDG_HDR(pfd_p, 2);
2234                 bdf = attr->rid;
2235                 trans_type = PF_ADDR_PIO;
2236                 break;
2237         case PCI_PCIX_CMD_MEMRD_DW:
2238         case PCI_PCIX_CMD_MEMRD_BL:
2239         case PCI_PCIX_CMD_MEMRDBL:
2240         case PCI_PCIX_CMD_MEMWR:
2241         case PCI_PCIX_CMD_MEMWR_BL:
2242         case PCI_PCIX_CMD_MEMWRBL:
2243                 addr = ((uint64_t)PCIE_ADV_BDG_HDR(pfd_p, 3) <<
2244                     PCIE_AER_SUCE_HDR_ADDR_SHIFT) | PCIE_ADV_BDG_HDR(pfd_p, 2);
2245                 bdf = attr->rid;
2246 
2247                 pf_pci_find_trans_type(pfd_p, &addr, &trans_type, &bdf);
2248                 break;
2249         case PCI_PCIX_CMD_CFRD:
2250         case PCI_PCIX_CMD_CFWR:
2251                 /*
2252                  * CFG Access should always be down stream.  Match the BDF in
2253                  * the address phase.
2254                  */
2255                 addr = 0;
2256                 bdf = attr->rid;
2257                 trans_type = PF_ADDR_CFG;
2258                 break;
2259         case PCI_PCIX_CMD_SPL:
2260                 /*
2261                  * Check for DMA read completions.  The requesting BDF is in the
2262                  * Address phase.
2263                  */
2264                 addr = 0;
2265                 bdf = attr->rid;
2266                 trans_type = PF_ADDR_DMA;
2267                 break;
2268         case PCI_PCIX_CMD_DADR:
2269                 /*
2270                  * For Dual Address Cycles the transaction command is in the 2nd
2271                  * address phase.
2272                  */
2273                 *cmd = (PCIE_ADV_BDG_HDR(pfd_p, 1) >>
2274                     PCIE_AER_SUCE_HDR_CMD_UP_SHIFT) &
2275                     PCIE_AER_SUCE_HDR_CMD_UP_MASK;
2276                 if (*cmd != PCI_PCIX_CMD_DADR)
2277                         goto cmd_switch;
2278                 /* FALLTHROUGH */
2279         default:
2280                 PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_trans = 0;
2281                 PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf = PCIE_INVALID_BDF;
2282                 PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_addr = 0;
2283                 return (DDI_FAILURE);
2284         }
2285         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_trans = trans_type;
2286         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf = bdf;
2287         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_addr = addr;
2288         return (DDI_SUCCESS);
2289 }
2290 
2291 /*
2292  * Based on either the BDF/ADDR find and mark the faulting DMA/ACC handler.
2293  * Returns either PF_HDL_NOTFOUND or PF_HDL_FOUND.
2294  */
2295 int
2296 pf_hdl_lookup(dev_info_t *dip, uint64_t ena, uint32_t flag, uint64_t addr,
2297     pcie_req_id_t bdf)
2298 {
2299         ddi_fm_error_t          derr;
2300 
2301         /* If we don't know the addr or rid just return with NOTFOUND */
2302         if ((addr == NULL) && !PCIE_CHECK_VALID_BDF(bdf))
2303                 return (PF_HDL_NOTFOUND);
2304 
2305         /*
2306          * Disable DMA handle lookup until DMA errors can be handled and
2307          * reported synchronously.  When enabled again, check for the
2308          * PF_ADDR_DMA flag
2309          */
2310         if (!(flag & (PF_ADDR_PIO | PF_ADDR_CFG))) {
2311                 return (PF_HDL_NOTFOUND);
2312         }
2313 
2314         bzero(&derr, sizeof (ddi_fm_error_t));
2315         derr.fme_version = DDI_FME_VERSION;
2316         derr.fme_flag = DDI_FM_ERR_UNEXPECTED;
2317         derr.fme_ena = ena;
2318 
2319         return (pf_hdl_child_lookup(dip, &derr, flag, addr, bdf));
2320 }
2321 
2322 static int
2323 pf_hdl_child_lookup(dev_info_t *dip, ddi_fm_error_t *derr, uint32_t flag,
2324     uint64_t addr, pcie_req_id_t bdf)
2325 {
2326         int                     status = PF_HDL_NOTFOUND;
2327         ndi_fmc_t               *fcp = NULL;
2328         struct i_ddi_fmhdl      *fmhdl = DEVI(dip)->devi_fmhdl;
2329         pcie_req_id_t           dip_bdf;
2330         boolean_t               have_lock = B_FALSE;
2331         pcie_bus_t              *bus_p;
2332         dev_info_t              *cdip;
2333 
2334         if (!(bus_p = pf_is_ready(dip))) {
2335                 return (status);
2336         }
2337 
2338         ASSERT(fmhdl);
2339         if (!i_ddi_fm_handler_owned(dip)) {
2340                 /*
2341                  * pf_handler_enter always returns SUCCESS if the 'impl' arg is
2342                  * NULL.
2343                  */
2344                 (void) pf_handler_enter(dip, NULL);
2345                 have_lock = B_TRUE;
2346         }
2347 
2348         dip_bdf = PCI_GET_BDF(dip);
2349 
2350         /* Check if dip and BDF match, if not recurse to it's children. */
2351         if (!PCIE_IS_RC(bus_p) && (!PCIE_CHECK_VALID_BDF(bdf) ||
2352             dip_bdf == bdf)) {
2353                 if ((flag & PF_ADDR_DMA) && DDI_FM_DMA_ERR_CAP(fmhdl->fh_cap))
2354                         fcp = fmhdl->fh_dma_cache;
2355                 else
2356                         fcp = NULL;
2357 
2358                 if (fcp)
2359                         status = pf_hdl_compare(dip, derr, DMA_HANDLE, addr,
2360                             bdf, fcp);
2361 
2362 
2363                 if (((flag & PF_ADDR_PIO) || (flag & PF_ADDR_CFG)) &&
2364                     DDI_FM_ACC_ERR_CAP(fmhdl->fh_cap))
2365                         fcp = fmhdl->fh_acc_cache;
2366                 else
2367                         fcp = NULL;
2368 
2369                 if (fcp)
2370                         status = pf_hdl_compare(dip, derr, ACC_HANDLE, addr,
2371                             bdf, fcp);
2372         }
2373 
2374         /* If we found the handler or know it's this device, we're done */
2375         if (!PCIE_IS_RC(bus_p) && ((dip_bdf == bdf) ||
2376             (status == PF_HDL_FOUND)))
2377                 goto done;
2378 
2379         /*
2380          * If the current devuce us a PCIe-PCI bridge need to check for special
2381          * cases:
2382          *
2383          * If it is a PIO and we don't have an address or this is a DMA, check
2384          * to see if the BDF = secondary bus.  If so stop.  The BDF isn't a real
2385          * BDF and the fault device could have come from any device in the PCI
2386          * bus.
2387          */
2388         if (PCIE_IS_PCIE_BDG(bus_p) &&
2389             ((flag & PF_ADDR_DMA || flag & PF_ADDR_PIO)) &&
2390             ((bus_p->bus_bdg_secbus << PCIE_REQ_ID_BUS_SHIFT) == bdf))
2391                 goto done;
2392 
2393 
2394         /* If we can't find the handler check it's children */
2395         for (cdip = ddi_get_child(dip); cdip;
2396             cdip = ddi_get_next_sibling(cdip)) {
2397                 if ((bus_p = PCIE_DIP2BUS(cdip)) == NULL)
2398                         continue;
2399 
2400                 if (pf_in_bus_range(bus_p, bdf) ||
2401                     pf_in_addr_range(bus_p, addr))
2402                         status = pf_hdl_child_lookup(cdip, derr, flag, addr,
2403                             bdf);
2404 
2405                 if (status == PF_HDL_FOUND)
2406                         goto done;
2407         }
2408 
2409 done:
2410         if (have_lock == B_TRUE)
2411                 pf_handler_exit(dip);
2412 
2413         return (status);
2414 }
2415 
2416 static int
2417 pf_hdl_compare(dev_info_t *dip, ddi_fm_error_t *derr, uint32_t flag,
2418     uint64_t addr, pcie_req_id_t bdf, ndi_fmc_t *fcp) {
2419         ndi_fmcentry_t  *fep;
2420         int             found = 0;
2421         int             status;
2422 
2423         mutex_enter(&fcp->fc_lock);
2424         for (fep = fcp->fc_head; fep != NULL; fep = fep->fce_next) {
2425                 ddi_fmcompare_t compare_func;
2426 
2427                 /*
2428                  * Compare captured error state with handle
2429                  * resources.  During the comparison and
2430                  * subsequent error handling, we block
2431                  * attempts to free the cache entry.
2432                  */
2433                 compare_func = (flag == ACC_HANDLE) ?
2434                     i_ddi_fm_acc_err_cf_get((ddi_acc_handle_t)
2435                         fep->fce_resource) :
2436                     i_ddi_fm_dma_err_cf_get((ddi_dma_handle_t)
2437                         fep->fce_resource);
2438 
2439                 if (compare_func == NULL) /* unbound or not FLAGERR */
2440                         continue;
2441 
2442                 status = compare_func(dip, fep->fce_resource,
2443                             (void *)&addr, (void *)&bdf);
2444 
2445                 if (status == DDI_FM_NONFATAL) {
2446                         found++;
2447 
2448                         /* Set the error for this resource handle */
2449                         if (flag == ACC_HANDLE) {
2450                                 ddi_acc_handle_t ap = fep->fce_resource;
2451 
2452                                 i_ddi_fm_acc_err_set(ap, derr->fme_ena, status,
2453                                     DDI_FM_ERR_UNEXPECTED);
2454                                 ddi_fm_acc_err_get(ap, derr, DDI_FME_VERSION);
2455                                 derr->fme_acc_handle = ap;
2456                         } else {
2457                                 ddi_dma_handle_t dp = fep->fce_resource;
2458 
2459                                 i_ddi_fm_dma_err_set(dp, derr->fme_ena, status,
2460                                     DDI_FM_ERR_UNEXPECTED);
2461                                 ddi_fm_dma_err_get(dp, derr, DDI_FME_VERSION);
2462                                 derr->fme_dma_handle = dp;
2463                         }
2464                 }
2465         }
2466         mutex_exit(&fcp->fc_lock);
2467 
2468         /*
2469          * If a handler isn't found and we know this is the right device mark
2470          * them all failed.
2471          */
2472         if ((addr != NULL) && PCIE_CHECK_VALID_BDF(bdf) && (found == 0)) {
2473                 status = pf_hdl_compare(dip, derr, flag, addr, bdf, fcp);
2474                 if (status == PF_HDL_FOUND)
2475                         found++;
2476         }
2477 
2478         return ((found) ? PF_HDL_FOUND : PF_HDL_NOTFOUND);
2479 }
2480 
2481 /*
2482  * Automatically decode AER header logs and does a handling look up based on the
2483  * AER header decoding.
2484  *
2485  * For this function only the Primary/Secondary AER Header Logs need to be valid
2486  * in the pfd (PCIe Fault Data) arg.
2487  *
2488  * Returns either PF_HDL_NOTFOUND or PF_HDL_FOUND.
2489  */
2490 /* ARGSUSED */
2491 static int
2492 pf_log_hdl_lookup(dev_info_t *rpdip, ddi_fm_error_t *derr, pf_data_t *pfd_p,
2493         boolean_t is_primary)
2494 {
2495         /*
2496          * Disabling this function temporarily until errors can be handled
2497          * synchronously.
2498          *
2499          * This function is currently only called during the middle of a fabric
2500          * scan.  If the fabric scan is called synchronously with an error seen
2501          * in the RP/RC, then the related errors in the fabric will have a
2502          * PF_ERR_MATCHED_RC error severity.  pf_log_hdl_lookup code will be by
2503          * passed when the severity is PF_ERR_MATCHED_RC.  Handle lookup would
2504          * have already happened in RP/RC error handling in a synchronous
2505          * manner.  Errors unrelated should panic, because they are being
2506          * handled asynchronously.
2507          *
2508          * If fabric scan is called asynchronously from any RP/RC error, then
2509          * DMA/PIO UE errors seen in the fabric should panic.  pf_lop_hdl_lookup
2510          * will return PF_HDL_NOTFOUND to ensure that the system panics.
2511          */
2512         return (PF_HDL_NOTFOUND);
2513 }
2514 
2515 /*
2516  * Decodes the TLP and returns the BDF of the handler, address and transaction
2517  * type if known.
2518  *
2519  * Types of TLP logs seen in RC, and what to extract:
2520  *
2521  * Memory(DMA) - Requester BDF, address, PF_DMA_ADDR
2522  * Memory(PIO) - address, PF_PIO_ADDR
2523  * CFG - Should not occur and result in UR
2524  * Completion(DMA) - Requester BDF, PF_DMA_ADDR
2525  * Completion(PIO) - Requester BDF, PF_PIO_ADDR
2526  *
2527  * Types of TLP logs seen in SW/Leaf, and what to extract:
2528  *
2529  * Memory(DMA) - Requester BDF, address, PF_DMA_ADDR
2530  * Memory(PIO) - address, PF_PIO_ADDR
2531  * CFG - Destined BDF, address, PF_CFG_ADDR
2532  * Completion(DMA) - Requester BDF, PF_DMA_ADDR
2533  * Completion(PIO) - Requester BDF, PF_PIO_ADDR
2534  *
2535  * The adv_reg_p must be passed in separately for use with SPARC RPs.  A
2536  * SPARC RP could have multiple AER header logs which cannot be directly
2537  * accessed via the bus_p.
2538  */
2539 int
2540 pf_tlp_decode(pcie_bus_t *bus_p, pf_pcie_adv_err_regs_t *adv_reg_p) {
2541         pcie_tlp_hdr_t  *tlp_hdr = (pcie_tlp_hdr_t *)adv_reg_p->pcie_ue_hdr;
2542         pcie_req_id_t   my_bdf, tlp_bdf, flt_bdf = PCIE_INVALID_BDF;
2543         uint64_t        flt_addr = 0;
2544         uint32_t        flt_trans_type = 0;
2545 
2546         adv_reg_p->pcie_ue_tgt_addr = 0;
2547         adv_reg_p->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
2548         adv_reg_p->pcie_ue_tgt_trans = 0;
2549 
2550         my_bdf = bus_p->bus_bdf;
2551         switch (tlp_hdr->type) {
2552         case PCIE_TLP_TYPE_IO:
2553         case PCIE_TLP_TYPE_MEM:
2554         case PCIE_TLP_TYPE_MEMLK:
2555                 /* Grab the 32/64bit fault address */
2556                 if (tlp_hdr->fmt & 0x1) {
2557                         flt_addr = ((uint64_t)adv_reg_p->pcie_ue_hdr[2] << 32);
2558                         flt_addr |= adv_reg_p->pcie_ue_hdr[3];
2559                 } else {
2560                         flt_addr = adv_reg_p->pcie_ue_hdr[2];
2561                 }
2562 
2563                 tlp_bdf = (pcie_req_id_t)(adv_reg_p->pcie_ue_hdr[1] >> 16);
2564 
2565                 /*
2566                  * If the req bdf >= this.bdf, then it means the request is this
2567                  * device or came from a device below it.  Unless this device is
2568                  * a PCIe root port then it means is a DMA, otherwise PIO.
2569                  */
2570                 if ((tlp_bdf >= my_bdf) && !PCIE_IS_ROOT(bus_p)) {
2571                         flt_trans_type = PF_ADDR_DMA;
2572                         flt_bdf = tlp_bdf;
2573                 } else if (PCIE_IS_ROOT(bus_p) &&
2574                     (PF_FIRST_AER_ERR(PCIE_AER_UCE_PTLP, adv_reg_p) ||
2575                         (PF_FIRST_AER_ERR(PCIE_AER_UCE_CA, adv_reg_p)))) {
2576                         flt_trans_type = PF_ADDR_DMA;
2577                         flt_bdf = tlp_bdf;
2578                 } else {
2579                         flt_trans_type = PF_ADDR_PIO;
2580                         flt_bdf = PCIE_INVALID_BDF;
2581                 }
2582                 break;
2583         case PCIE_TLP_TYPE_CFG0:
2584         case PCIE_TLP_TYPE_CFG1:
2585                 flt_addr = 0;
2586                 flt_bdf = (pcie_req_id_t)(adv_reg_p->pcie_ue_hdr[2] >> 16);
2587                 flt_trans_type = PF_ADDR_CFG;
2588                 break;
2589         case PCIE_TLP_TYPE_CPL:
2590         case PCIE_TLP_TYPE_CPLLK:
2591         {
2592                 pcie_cpl_t *cpl_tlp = (pcie_cpl_t *)&adv_reg_p->pcie_ue_hdr[1];
2593 
2594                 flt_addr = NULL;
2595                 flt_bdf = (cpl_tlp->rid > cpl_tlp->cid) ? cpl_tlp->rid :
2596                     cpl_tlp->cid;
2597 
2598                 /*
2599                  * If the cpl bdf < this.bdf, then it means the request is this
2600                  * device or came from a device below it.  Unless this device is
2601                  * a PCIe root port then it means is a DMA, otherwise PIO.
2602                  */
2603                 if (cpl_tlp->rid > cpl_tlp->cid) {
2604                         flt_trans_type = PF_ADDR_DMA;
2605                 } else {
2606                         flt_trans_type = PF_ADDR_PIO | PF_ADDR_CFG;
2607                 }
2608                 break;
2609         }
2610         default:
2611                 return (DDI_FAILURE);
2612         }
2613 
2614         adv_reg_p->pcie_ue_tgt_addr = flt_addr;
2615         adv_reg_p->pcie_ue_tgt_bdf = flt_bdf;
2616         adv_reg_p->pcie_ue_tgt_trans = flt_trans_type;
2617 
2618         return (DDI_SUCCESS);
2619 }
2620 
2621 #define PCIE_EREPORT    DDI_IO_CLASS "." PCI_ERROR_SUBCLASS "." PCIEX_FABRIC
2622 static int
2623 pf_ereport_setup(dev_info_t *dip, uint64_t ena, nvlist_t **ereport,
2624     nvlist_t **detector, errorq_elem_t **eqep)
2625 {
2626         struct i_ddi_fmhdl *fmhdl = DEVI(dip)->devi_fmhdl;
2627         char device_path[MAXPATHLEN];
2628         nv_alloc_t *nva;
2629 
2630         *eqep = errorq_reserve(fmhdl->fh_errorq);
2631         if (*eqep == NULL) {
2632                 atomic_add_64(&fmhdl->fh_kstat.fek_erpt_dropped.value.ui64, 1);
2633                 return (DDI_FAILURE);
2634         }
2635 
2636         *ereport = errorq_elem_nvl(fmhdl->fh_errorq, *eqep);
2637         nva = errorq_elem_nva(fmhdl->fh_errorq, *eqep);
2638 
2639         ASSERT(*ereport);
2640         ASSERT(nva);
2641 
2642         /*
2643          * Use the dev_path/devid for this device instance.
2644          */
2645         *detector = fm_nvlist_create(nva);
2646         if (dip == ddi_root_node()) {
2647                 device_path[0] = '/';
2648                 device_path[1] = '\0';
2649         } else {
2650                 (void) ddi_pathname(dip, device_path);
2651         }
2652 
2653         fm_fmri_dev_set(*detector, FM_DEV_SCHEME_VERSION, NULL,
2654             device_path, NULL, NULL);
2655 
2656         if (ena == 0)
2657                 ena = fm_ena_generate(0, FM_ENA_FMT1);
2658 
2659         fm_ereport_set(*ereport, 0, PCIE_EREPORT, ena, *detector, NULL);
2660 
2661         return (DDI_SUCCESS);
2662 }
2663 
2664 /* ARGSUSED */
2665 static void
2666 pf_ereport_post(dev_info_t *dip, nvlist_t **ereport, nvlist_t **detector,
2667     errorq_elem_t **eqep)
2668 {
2669         struct i_ddi_fmhdl *fmhdl = DEVI(dip)->devi_fmhdl;
2670 
2671         errorq_commit(fmhdl->fh_errorq, *eqep, ERRORQ_ASYNC);
2672 }
2673 
2674 static void
2675 pf_send_ereport(ddi_fm_error_t *derr, pf_impl_t *impl)
2676 {
2677         nvlist_t        *ereport;
2678         nvlist_t        *detector;
2679         errorq_elem_t   *eqep;
2680         pcie_bus_t      *bus_p;
2681         pf_data_t       *pfd_p;
2682         uint32_t        total = impl->pf_total;
2683 
2684         /*
2685          * Ereports need to be sent in a top down fashion. The fabric translator
2686          * expects the ereports from the Root first. This is needed to tell if
2687          * the system contains a PCIe complaint RC/RP.
2688          */
2689         for (pfd_p = impl->pf_dq_head_p; pfd_p; pfd_p = pfd_p->pe_next) {
2690                 bus_p = PCIE_PFD2BUS(pfd_p);
2691                 pfd_p->pe_valid = B_FALSE;
2692 
2693                 if (derr->fme_flag != DDI_FM_ERR_UNEXPECTED ||
2694                     !DDI_FM_EREPORT_CAP(ddi_fm_capable(PCIE_PFD2DIP(pfd_p))))
2695                         continue;
2696 
2697                 if (pf_ereport_setup(PCIE_BUS2DIP(bus_p), derr->fme_ena,
2698                     &ereport, &detector, &eqep) != DDI_SUCCESS)
2699                         continue;
2700 
2701                 if (PFD_IS_RC(pfd_p)) {
2702                         fm_payload_set(ereport,
2703                             "scan_bdf", DATA_TYPE_UINT16,
2704                             PCIE_ROOT_FAULT(pfd_p)->scan_bdf,
2705                             "scan_addr", DATA_TYPE_UINT64,
2706                             PCIE_ROOT_FAULT(pfd_p)->scan_addr,
2707                             "intr_src", DATA_TYPE_UINT16,
2708                             PCIE_ROOT_EH_SRC(pfd_p)->intr_type,
2709                             NULL);
2710                         goto generic;
2711                 }
2712 
2713                 /* Generic PCI device information */
2714                 fm_payload_set(ereport,
2715                     "bdf", DATA_TYPE_UINT16, bus_p->bus_bdf,
2716                     "device_id", DATA_TYPE_UINT16,
2717                     (bus_p->bus_dev_ven_id >> 16),
2718                     "vendor_id", DATA_TYPE_UINT16,
2719                     (bus_p->bus_dev_ven_id & 0xFFFF),
2720                     "rev_id", DATA_TYPE_UINT8, bus_p->bus_rev_id,
2721                     "dev_type", DATA_TYPE_UINT16, bus_p->bus_dev_type,
2722                     "pcie_off", DATA_TYPE_UINT16, bus_p->bus_pcie_off,
2723                     "pcix_off", DATA_TYPE_UINT16, bus_p->bus_pcix_off,
2724                     "aer_off", DATA_TYPE_UINT16, bus_p->bus_aer_off,
2725                     "ecc_ver", DATA_TYPE_UINT16, bus_p->bus_ecc_ver,
2726                     NULL);
2727 
2728                 /* PCI registers */
2729                 fm_payload_set(ereport,
2730                     "pci_status", DATA_TYPE_UINT16,
2731                     PCI_ERR_REG(pfd_p)->pci_err_status,
2732                     "pci_command", DATA_TYPE_UINT16,
2733                     PCI_ERR_REG(pfd_p)->pci_cfg_comm,
2734                     NULL);
2735 
2736                 /* PCI bridge registers */
2737                 if (PCIE_IS_BDG(bus_p)) {
2738                         fm_payload_set(ereport,
2739                             "pci_bdg_sec_status", DATA_TYPE_UINT16,
2740                             PCI_BDG_ERR_REG(pfd_p)->pci_bdg_sec_stat,
2741                             "pci_bdg_ctrl", DATA_TYPE_UINT16,
2742                             PCI_BDG_ERR_REG(pfd_p)->pci_bdg_ctrl,
2743                             NULL);
2744                 }
2745 
2746                 /* PCIx registers */
2747                 if (PCIE_IS_PCIX(bus_p) && !PCIE_IS_BDG(bus_p)) {
2748                         fm_payload_set(ereport,
2749                             "pcix_status", DATA_TYPE_UINT32,
2750                             PCIX_ERR_REG(pfd_p)->pcix_status,
2751                             "pcix_command", DATA_TYPE_UINT16,
2752                             PCIX_ERR_REG(pfd_p)->pcix_command,
2753                             NULL);
2754                 }
2755 
2756                 /* PCIx ECC Registers */
2757                 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
2758                         pf_pcix_ecc_regs_t *ecc_bdg_reg;
2759                         pf_pcix_ecc_regs_t *ecc_reg;
2760 
2761                         if (PCIE_IS_BDG(bus_p))
2762                                 ecc_bdg_reg = PCIX_BDG_ECC_REG(pfd_p, 0);
2763                         ecc_reg = PCIX_ECC_REG(pfd_p);
2764                         fm_payload_set(ereport,
2765                             "pcix_ecc_control_0", DATA_TYPE_UINT16,
2766                             PCIE_IS_BDG(bus_p) ?
2767                             (ecc_bdg_reg->pcix_ecc_ctlstat >> 16) :
2768                             (ecc_reg->pcix_ecc_ctlstat >> 16),
2769                             "pcix_ecc_status_0", DATA_TYPE_UINT16,
2770                             PCIE_IS_BDG(bus_p) ?
2771                             (ecc_bdg_reg->pcix_ecc_ctlstat & 0xFFFF) :
2772                             (ecc_reg->pcix_ecc_ctlstat & 0xFFFF),
2773                             "pcix_ecc_fst_addr_0", DATA_TYPE_UINT32,
2774                             PCIE_IS_BDG(bus_p) ?
2775                             ecc_bdg_reg->pcix_ecc_fstaddr :
2776                             ecc_reg->pcix_ecc_fstaddr,
2777                             "pcix_ecc_sec_addr_0", DATA_TYPE_UINT32,
2778                             PCIE_IS_BDG(bus_p) ?
2779                             ecc_bdg_reg->pcix_ecc_secaddr :
2780                             ecc_reg->pcix_ecc_secaddr,
2781                             "pcix_ecc_attr_0", DATA_TYPE_UINT32,
2782                             PCIE_IS_BDG(bus_p) ?
2783                             ecc_bdg_reg->pcix_ecc_attr :
2784                             ecc_reg->pcix_ecc_attr,
2785                             NULL);
2786                 }
2787 
2788                 /* PCIx ECC Bridge Registers */
2789                 if (PCIX_ECC_VERSION_CHECK(bus_p) && PCIE_IS_BDG(bus_p)) {
2790                         pf_pcix_ecc_regs_t *ecc_bdg_reg;
2791 
2792                         ecc_bdg_reg = PCIX_BDG_ECC_REG(pfd_p, 1);
2793                         fm_payload_set(ereport,
2794                             "pcix_ecc_control_1", DATA_TYPE_UINT16,
2795                             (ecc_bdg_reg->pcix_ecc_ctlstat >> 16),
2796                             "pcix_ecc_status_1", DATA_TYPE_UINT16,
2797                             (ecc_bdg_reg->pcix_ecc_ctlstat & 0xFFFF),
2798                             "pcix_ecc_fst_addr_1", DATA_TYPE_UINT32,
2799                             ecc_bdg_reg->pcix_ecc_fstaddr,
2800                             "pcix_ecc_sec_addr_1", DATA_TYPE_UINT32,
2801                             ecc_bdg_reg->pcix_ecc_secaddr,
2802                             "pcix_ecc_attr_1", DATA_TYPE_UINT32,
2803                             ecc_bdg_reg->pcix_ecc_attr,
2804                             NULL);
2805                 }
2806 
2807                 /* PCIx Bridge */
2808                 if (PCIE_IS_PCIX(bus_p) && PCIE_IS_BDG(bus_p)) {
2809                         fm_payload_set(ereport,
2810                             "pcix_bdg_status", DATA_TYPE_UINT32,
2811                             PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_stat,
2812                             "pcix_bdg_sec_status", DATA_TYPE_UINT16,
2813                             PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_sec_stat,
2814                             NULL);
2815                 }
2816 
2817                 /* PCIe registers */
2818                 if (PCIE_IS_PCIE(bus_p)) {
2819                         fm_payload_set(ereport,
2820                             "pcie_status", DATA_TYPE_UINT16,
2821                             PCIE_ERR_REG(pfd_p)->pcie_err_status,
2822                             "pcie_command", DATA_TYPE_UINT16,
2823                             PCIE_ERR_REG(pfd_p)->pcie_err_ctl,
2824                             "pcie_dev_cap", DATA_TYPE_UINT32,
2825                             PCIE_ERR_REG(pfd_p)->pcie_dev_cap,
2826                             NULL);
2827                 }
2828 
2829                 /* PCIe AER registers */
2830                 if (PCIE_HAS_AER(bus_p)) {
2831                         fm_payload_set(ereport,
2832                             "pcie_adv_ctl", DATA_TYPE_UINT32,
2833                             PCIE_ADV_REG(pfd_p)->pcie_adv_ctl,
2834                             "pcie_ue_status", DATA_TYPE_UINT32,
2835                             PCIE_ADV_REG(pfd_p)->pcie_ue_status,
2836                             "pcie_ue_mask", DATA_TYPE_UINT32,
2837                             PCIE_ADV_REG(pfd_p)->pcie_ue_mask,
2838                             "pcie_ue_sev", DATA_TYPE_UINT32,
2839                             PCIE_ADV_REG(pfd_p)->pcie_ue_sev,
2840                             "pcie_ue_hdr0", DATA_TYPE_UINT32,
2841                             PCIE_ADV_REG(pfd_p)->pcie_ue_hdr[0],
2842                             "pcie_ue_hdr1", DATA_TYPE_UINT32,
2843                             PCIE_ADV_REG(pfd_p)->pcie_ue_hdr[1],
2844                             "pcie_ue_hdr2", DATA_TYPE_UINT32,
2845                             PCIE_ADV_REG(pfd_p)->pcie_ue_hdr[2],
2846                             "pcie_ue_hdr3", DATA_TYPE_UINT32,
2847                             PCIE_ADV_REG(pfd_p)->pcie_ue_hdr[3],
2848                             "pcie_ce_status", DATA_TYPE_UINT32,
2849                             PCIE_ADV_REG(pfd_p)->pcie_ce_status,
2850                             "pcie_ce_mask", DATA_TYPE_UINT32,
2851                             PCIE_ADV_REG(pfd_p)->pcie_ce_mask,
2852                             NULL);
2853                 }
2854 
2855                 /* PCIe AER decoded header */
2856                 if (HAS_AER_LOGS(pfd_p, PCIE_ADV_REG(pfd_p)->pcie_ue_status)) {
2857                         fm_payload_set(ereport,
2858                             "pcie_ue_tgt_trans", DATA_TYPE_UINT32,
2859                             PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_trans,
2860                             "pcie_ue_tgt_addr", DATA_TYPE_UINT64,
2861                             PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr,
2862                             "pcie_ue_tgt_bdf", DATA_TYPE_UINT16,
2863                             PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf,
2864                             NULL);
2865                         /* Clear these values as they no longer valid */
2866                         PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_trans = 0;
2867                         PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr = 0;
2868                         PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
2869                 }
2870 
2871                 /* PCIe BDG AER registers */
2872                 if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_HAS_AER(bus_p)) {
2873                         fm_payload_set(ereport,
2874                             "pcie_sue_adv_ctl", DATA_TYPE_UINT32,
2875                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_ctl,
2876                             "pcie_sue_status", DATA_TYPE_UINT32,
2877                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status,
2878                             "pcie_sue_mask", DATA_TYPE_UINT32,
2879                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_mask,
2880                             "pcie_sue_sev", DATA_TYPE_UINT32,
2881                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_sev,
2882                             "pcie_sue_hdr0", DATA_TYPE_UINT32,
2883                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_hdr[0],
2884                             "pcie_sue_hdr1", DATA_TYPE_UINT32,
2885                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_hdr[1],
2886                             "pcie_sue_hdr2", DATA_TYPE_UINT32,
2887                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_hdr[2],
2888                             "pcie_sue_hdr3", DATA_TYPE_UINT32,
2889                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_hdr[3],
2890                             NULL);
2891                 }
2892 
2893                 /* PCIe BDG AER decoded header */
2894                 if (PCIE_IS_PCIE_BDG(bus_p) && HAS_SAER_LOGS(pfd_p,
2895                     PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_status)) {
2896                         fm_payload_set(ereport,
2897                             "pcie_sue_tgt_trans", DATA_TYPE_UINT32,
2898                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_trans,
2899                             "pcie_sue_tgt_addr", DATA_TYPE_UINT64,
2900                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_addr,
2901                             "pcie_sue_tgt_bdf", DATA_TYPE_UINT16,
2902                             PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf,
2903                             NULL);
2904                         /* Clear these values as they no longer valid */
2905                         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_trans = 0;
2906                         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_addr = 0;
2907                         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf =
2908                             PCIE_INVALID_BDF;
2909                 }
2910 
2911                 /* PCIe RP registers */
2912                 if (PCIE_IS_RP(bus_p)) {
2913                         fm_payload_set(ereport,
2914                             "pcie_rp_status", DATA_TYPE_UINT32,
2915                             PCIE_RP_REG(pfd_p)->pcie_rp_status,
2916                             "pcie_rp_control", DATA_TYPE_UINT16,
2917                             PCIE_RP_REG(pfd_p)->pcie_rp_ctl,
2918                             NULL);
2919                 }
2920 
2921                 /* PCIe RP AER registers */
2922                 if (PCIE_IS_RP(bus_p) && PCIE_HAS_AER(bus_p)) {
2923                         fm_payload_set(ereport,
2924                             "pcie_adv_rp_status", DATA_TYPE_UINT32,
2925                             PCIE_ADV_RP_REG(pfd_p)->pcie_rp_err_status,
2926                             "pcie_adv_rp_command", DATA_TYPE_UINT32,
2927                             PCIE_ADV_RP_REG(pfd_p)->pcie_rp_err_cmd,
2928                             "pcie_adv_rp_ce_src_id", DATA_TYPE_UINT16,
2929                             PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id,
2930                             "pcie_adv_rp_ue_src_id", DATA_TYPE_UINT16,
2931                             PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id,
2932                             NULL);
2933                 }
2934 
2935 generic:
2936                 /* IOV related information */
2937                 if (!PCIE_BDG_IS_UNASSIGNED(PCIE_PFD2BUS(impl->pf_dq_head_p))) {
2938                         fm_payload_set(ereport,
2939                             "pcie_aff_flags", DATA_TYPE_UINT16,
2940                             PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags,
2941                             "pcie_aff_bdf", DATA_TYPE_UINT16,
2942                             PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf,
2943                             "orig_sev", DATA_TYPE_UINT32,
2944                             pfd_p->pe_orig_severity_flags,
2945                             NULL);
2946                 }
2947 
2948                 /* Misc ereport information */
2949                 fm_payload_set(ereport,
2950                     "remainder", DATA_TYPE_UINT32, --total,
2951                     "severity", DATA_TYPE_UINT32, pfd_p->pe_severity_flags,
2952                     NULL);
2953 
2954                 pf_ereport_post(PCIE_BUS2DIP(bus_p), &ereport, &detector,
2955                     &eqep);
2956         }
2957 
2958         /* Unlock all the devices in the queue */
2959         for (pfd_p = impl->pf_dq_tail_p; pfd_p; pfd_p = pfd_p->pe_prev) {
2960                 if (pfd_p->pe_lock) {
2961                         pf_handler_exit(PCIE_PFD2DIP(pfd_p));
2962                 }
2963         }
2964 }
2965 
2966 /*
2967  * pf_handler_enter must be called to serial access to each device's pf_data_t.
2968  * Once error handling is finished with the device call pf_handler_exit to allow
2969  * other threads to access it.  The same thread may call pf_handler_enter
2970  * several times without any consequences.
2971  *
2972  * The "impl" variable is passed in during scan fabric to double check that
2973  * there is not a recursive algorithm and to ensure only one thread is doing a
2974  * fabric scan at all times.
2975  *
2976  * In some cases "impl" is not available, such as "child lookup" being called
2977  * from outside of scan fabric, just pass in NULL for this variable and this
2978  * extra check will be skipped.
2979  */
2980 static int
2981 pf_handler_enter(dev_info_t *dip, pf_impl_t *impl)
2982 {
2983         pf_data_t *pfd_p = PCIE_DIP2PFD(dip);
2984 
2985         ASSERT(pfd_p);
2986 
2987         /*
2988          * Check to see if the lock has already been taken by this
2989          * thread.  If so just return and don't take lock again.
2990          */
2991         if (!pfd_p->pe_lock || !impl) {
2992                 i_ddi_fm_handler_enter(dip);
2993                 pfd_p->pe_lock = B_TRUE;
2994                 return (PF_SCAN_SUCCESS);
2995         }
2996 
2997         /* Check to see that this dip is already in the "impl" error queue */
2998         for (pfd_p = impl->pf_dq_head_p; pfd_p; pfd_p = pfd_p->pe_next) {
2999                 if (PCIE_PFD2DIP(pfd_p) == dip) {
3000                         return (PF_SCAN_SUCCESS);
3001                 }
3002         }
3003 
3004         return (PF_SCAN_DEADLOCK);
3005 }
3006 
3007 static void
3008 pf_handler_exit(dev_info_t *dip)
3009 {
3010         pf_data_t *pfd_p = PCIE_DIP2PFD(dip);
3011 
3012         ASSERT(pfd_p);
3013 
3014         ASSERT(pfd_p->pe_lock == B_TRUE);
3015         i_ddi_fm_handler_exit(dip);
3016         pfd_p->pe_lock = B_FALSE;
3017 }
3018 
3019 /*
3020  * This function calls the driver's callback function (if it's FMA hardened
3021  * and callback capable). This function relies on the current thread already
3022  * owning the driver's fmhdl lock.
3023  */
3024 static int
3025 pf_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr)
3026 {
3027         int cb_sts = DDI_FM_OK;
3028 
3029         if (DDI_FM_ERRCB_CAP(ddi_fm_capable(dip))) {
3030                 dev_info_t *pdip = ddi_get_parent(dip);
3031                 struct i_ddi_fmhdl *hdl = DEVI(pdip)->devi_fmhdl;
3032                 struct i_ddi_fmtgt *tgt = hdl->fh_tgts;
3033                 struct i_ddi_errhdl *errhdl;
3034                 while (tgt != NULL) {
3035                         if (dip == tgt->ft_dip) {
3036                                 errhdl = tgt->ft_errhdl;
3037                                 cb_sts = errhdl->eh_func(dip, derr,
3038                                     errhdl->eh_impl);
3039                                 break;
3040                         }
3041                         tgt = tgt->ft_next;
3042                 }
3043         }
3044         return (cb_sts);
3045 }
3046 
3047 static void
3048 pf_reset_pfd(pf_data_t *pfd_p)
3049 {
3050         pcie_bus_t      *bus_p = PCIE_PFD2BUS(pfd_p);
3051 
3052         pfd_p->pe_severity_flags = 0;
3053         pfd_p->pe_orig_severity_flags = 0;
3054         /* pe_lock and pe_valid were reset in pf_send_ereport */
3055 
3056         PFD_AFFECTED_DEV(pfd_p)->pe_affected_flags = 0;
3057         PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = PCIE_INVALID_BDF;
3058 
3059         if (PCIE_IS_ROOT(bus_p)) {
3060                 PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
3061                 PCIE_ROOT_FAULT(pfd_p)->scan_addr = 0;
3062                 PCIE_ROOT_FAULT(pfd_p)->full_scan = B_FALSE;
3063                 PCIE_ROOT_EH_SRC(pfd_p)->intr_type = PF_INTR_TYPE_NONE;
3064                 PCIE_ROOT_EH_SRC(pfd_p)->intr_data = NULL;
3065         }
3066 
3067         if (PCIE_IS_BDG(bus_p)) {
3068                 bzero(PCI_BDG_ERR_REG(pfd_p), sizeof (pf_pci_bdg_err_regs_t));
3069         }
3070 
3071         PCI_ERR_REG(pfd_p)->pci_err_status = 0;
3072         PCI_ERR_REG(pfd_p)->pci_cfg_comm = 0;
3073 
3074         if (PCIE_IS_PCIE(bus_p)) {
3075                 if (PCIE_IS_ROOT(bus_p)) {
3076                         bzero(PCIE_RP_REG(pfd_p),
3077                             sizeof (pf_pcie_rp_err_regs_t));
3078                         bzero(PCIE_ADV_RP_REG(pfd_p),
3079                             sizeof (pf_pcie_adv_rp_err_regs_t));
3080                         PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id =
3081                             PCIE_INVALID_BDF;
3082                         PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id =
3083                             PCIE_INVALID_BDF;
3084                 } else if (PCIE_IS_PCIE_BDG(bus_p)) {
3085                         bzero(PCIE_ADV_BDG_REG(pfd_p),
3086                             sizeof (pf_pcie_adv_bdg_err_regs_t));
3087                         PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf =
3088                             PCIE_INVALID_BDF;
3089                 }
3090 
3091                 if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_IS_PCIX(bus_p)) {
3092                         if (PCIX_ECC_VERSION_CHECK(bus_p)) {
3093                                 bzero(PCIX_BDG_ECC_REG(pfd_p, 0),
3094                                     sizeof (pf_pcix_ecc_regs_t));
3095                                 bzero(PCIX_BDG_ECC_REG(pfd_p, 1),
3096                                     sizeof (pf_pcix_ecc_regs_t));
3097                         }
3098                         PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_sec_stat = 0;
3099                         PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_stat = 0;
3100                 }
3101 
3102                 PCIE_ADV_REG(pfd_p)->pcie_adv_ctl = 0;
3103                 PCIE_ADV_REG(pfd_p)->pcie_ue_status = 0;
3104                 PCIE_ADV_REG(pfd_p)->pcie_ue_mask = 0;
3105                 PCIE_ADV_REG(pfd_p)->pcie_ue_sev = 0;
3106                 PCIE_ADV_HDR(pfd_p, 0) = 0;
3107                 PCIE_ADV_HDR(pfd_p, 1) = 0;
3108                 PCIE_ADV_HDR(pfd_p, 2) = 0;
3109                 PCIE_ADV_HDR(pfd_p, 3) = 0;
3110                 PCIE_ADV_REG(pfd_p)->pcie_ce_status = 0;
3111                 PCIE_ADV_REG(pfd_p)->pcie_ce_mask = 0;
3112                 PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_trans = 0;
3113                 PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_addr = 0;
3114                 PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
3115 
3116                 PCIE_ERR_REG(pfd_p)->pcie_err_status = 0;
3117                 PCIE_ERR_REG(pfd_p)->pcie_err_ctl = 0;
3118                 PCIE_ERR_REG(pfd_p)->pcie_dev_cap = 0;
3119 
3120         } else if (PCIE_IS_PCIX(bus_p)) {
3121                 if (PCIE_IS_BDG(bus_p)) {
3122                         if (PCIX_ECC_VERSION_CHECK(bus_p)) {
3123                                 bzero(PCIX_BDG_ECC_REG(pfd_p, 0),
3124                                     sizeof (pf_pcix_ecc_regs_t));
3125                                 bzero(PCIX_BDG_ECC_REG(pfd_p, 1),
3126                                     sizeof (pf_pcix_ecc_regs_t));
3127                         }
3128                         PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_sec_stat = 0;
3129                         PCIX_BDG_ERR_REG(pfd_p)->pcix_bdg_stat = 0;
3130                 } else {
3131                         if (PCIX_ECC_VERSION_CHECK(bus_p)) {
3132                                 bzero(PCIX_ECC_REG(pfd_p),
3133                                     sizeof (pf_pcix_ecc_regs_t));
3134                         }
3135                         PCIX_ERR_REG(pfd_p)->pcix_command = 0;
3136                         PCIX_ERR_REG(pfd_p)->pcix_status = 0;
3137                 }
3138         }
3139 
3140         pfd_p->pe_prev = NULL;
3141         pfd_p->pe_next = NULL;
3142         pfd_p->pe_rber_fatal = B_FALSE;
3143 }
3144 
3145 pcie_bus_t *
3146 pf_find_busp_by_bdf(pf_impl_t *impl, pcie_req_id_t bdf)
3147 {
3148         pcie_bus_t *temp_bus_p;
3149         pf_data_t *temp_pfd_p;
3150 
3151         for (temp_pfd_p = impl->pf_dq_head_p;
3152             temp_pfd_p;
3153             temp_pfd_p = temp_pfd_p->pe_next) {
3154                 temp_bus_p = PCIE_PFD2BUS(temp_pfd_p);
3155 
3156                 if (bdf == temp_bus_p->bus_bdf) {
3157                         return (temp_bus_p);
3158                 }
3159         }
3160 
3161         return (NULL);
3162 }
3163 
3164 pcie_bus_t *
3165 pf_find_busp_by_addr(pf_impl_t *impl, uint64_t addr)
3166 {
3167         pcie_bus_t *temp_bus_p;
3168         pf_data_t *temp_pfd_p;
3169 
3170         for (temp_pfd_p = impl->pf_dq_head_p;
3171             temp_pfd_p;
3172             temp_pfd_p = temp_pfd_p->pe_next) {
3173                 temp_bus_p = PCIE_PFD2BUS(temp_pfd_p);
3174 
3175                 if (pf_in_assigned_addr(temp_bus_p, addr)) {
3176                         return (temp_bus_p);
3177                 }
3178         }
3179 
3180         return (NULL);
3181 }
3182 
3183 pcie_bus_t *
3184 pf_find_busp_by_aer(pf_impl_t *impl, pf_data_t *pfd_p)
3185 {
3186         pf_pcie_adv_err_regs_t *reg_p = PCIE_ADV_REG(pfd_p);
3187         pcie_bus_t *temp_bus_p = NULL;
3188         pcie_req_id_t bdf;
3189         uint64_t addr;
3190         pcie_tlp_hdr_t *tlp_hdr = (pcie_tlp_hdr_t *)reg_p->pcie_ue_hdr;
3191         uint32_t trans_type = reg_p->pcie_ue_tgt_trans;
3192 
3193         if ((tlp_hdr->type == PCIE_TLP_TYPE_CPL) ||
3194             (tlp_hdr->type == PCIE_TLP_TYPE_CPLLK)) {
3195                 pcie_cpl_t *cpl_tlp = (pcie_cpl_t *)&reg_p->pcie_ue_hdr[1];
3196 
3197                 bdf = (cpl_tlp->rid > cpl_tlp->cid) ? cpl_tlp->rid :
3198                     cpl_tlp->cid;
3199                 temp_bus_p = pf_find_busp_by_bdf(impl, bdf);
3200         } else if (trans_type == PF_ADDR_PIO) {
3201                 addr = reg_p->pcie_ue_tgt_addr;
3202                 temp_bus_p = pf_find_busp_by_addr(impl, addr);
3203         } else {
3204                 /* PF_ADDR_DMA type */
3205                 bdf = reg_p->pcie_ue_tgt_bdf;
3206                 temp_bus_p = pf_find_busp_by_bdf(impl, bdf);
3207         }
3208 
3209         return (temp_bus_p);
3210 }
3211 
3212 pcie_bus_t *
3213 pf_find_busp_by_saer(pf_impl_t *impl, pf_data_t *pfd_p)
3214 {
3215         pf_pcie_adv_bdg_err_regs_t *reg_p = PCIE_ADV_BDG_REG(pfd_p);
3216         pcie_bus_t *temp_bus_p = NULL;
3217         pcie_req_id_t bdf;
3218         uint64_t addr;
3219 
3220         addr = reg_p->pcie_sue_tgt_addr;
3221         bdf = reg_p->pcie_sue_tgt_bdf;
3222 
3223         if (addr != NULL) {
3224                 temp_bus_p = pf_find_busp_by_addr(impl, addr);
3225         } else if (PCIE_CHECK_VALID_BDF(bdf)) {
3226                 temp_bus_p = pf_find_busp_by_bdf(impl, bdf);
3227         }
3228 
3229         return (temp_bus_p);
3230 }