1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  25  */
  26 
  27 /*
  28  * AHCI (Advanced Host Controller Interface) SATA HBA Driver
  29  *
  30  * Power Management Support
  31  * ------------------------
  32  *
  33  * At the moment, the ahci driver only implements suspend/resume to
  34  * support Suspend to RAM on X86 feature. Device power management isn't
  35  * implemented, link power management is disabled, and hot plug isn't
  36  * allowed during the period from suspend to resume.
  37  *
  38  * For s/r support, the ahci driver only need to implement DDI_SUSPEND
  39  * and DDI_RESUME entries, and don't need to take care of new requests
  40  * sent down after suspend because the target driver (sd) has already
  41  * handled these conditions, and blocked these requests. For the detailed
  42  * information, please check with sdopen, sdclose and sdioctl routines.
  43  *
  44  */
  45 
  46 #include <sys/note.h>
  47 #include <sys/scsi/scsi.h>
  48 #include <sys/pci.h>
  49 #include <sys/disp.h>
  50 #include <sys/sata/sata_hba.h>
  51 #include <sys/sata/adapters/ahci/ahcireg.h>
  52 #include <sys/sata/adapters/ahci/ahcivar.h>
  53 
  54 /*
  55  * FMA header files
  56  */
  57 #include <sys/ddifm.h>
  58 #include <sys/fm/protocol.h>
  59 #include <sys/fm/util.h>
  60 #include <sys/fm/io/ddi.h>
  61 
  62 /*
  63  * This is the string displayed by modinfo, etc.
  64  */
  65 static char ahci_ident[] = "ahci driver";
  66 
  67 /*
  68  * Function prototypes for driver entry points
  69  */
  70 static  int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
  71 static  int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
  72 static  int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
  73 static  int ahci_quiesce(dev_info_t *);
  74 
  75 /*
  76  * Function prototypes for SATA Framework interfaces
  77  */
  78 static  int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
  79 static  int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
  80 
  81 static  int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
  82 static  int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
  83 static  int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
  84 static  int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
  85 static  int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
  86 static  int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
  87 #if defined(__lock_lint)
  88 static  int ahci_selftest(dev_info_t *, sata_device_t *);
  89 #endif
  90 
  91 /*
  92  * FMA Prototypes
  93  */
  94 static  void ahci_fm_init(ahci_ctl_t *);
  95 static  void ahci_fm_fini(ahci_ctl_t *);
  96 static  int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
  97 int     ahci_check_acc_handle(ddi_acc_handle_t);
  98 int     ahci_check_dma_handle(ddi_dma_handle_t);
  99 void    ahci_fm_ereport(ahci_ctl_t *, char *);
 100 static  int ahci_check_all_handle(ahci_ctl_t *);
 101 static  int ahci_check_ctl_handle(ahci_ctl_t *);
 102 static  int ahci_check_port_handle(ahci_ctl_t *, int);
 103 static  int ahci_check_slot_handle(ahci_port_t *, int);
 104 
 105 /*
 106  * Local function prototypes
 107  */
 108 static  int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
 109 static  int ahci_alloc_ports_state(ahci_ctl_t *);
 110 static  void ahci_dealloc_ports_state(ahci_ctl_t *);
 111 static  int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
 112 static  void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
 113 static  int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
 114 static  void ahci_dealloc_rcvd_fis(ahci_port_t *);
 115 static  int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
 116 static  void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
 117 static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
 118 static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
 119 static  void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
 120 static  void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
 121 
 122 static  int ahci_initialize_controller(ahci_ctl_t *);
 123 static  void ahci_uninitialize_controller(ahci_ctl_t *);
 124 static  int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 125 static  int ahci_config_space_init(ahci_ctl_t *);
 126 static  void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
 127 
 128 static  void ahci_drain_ports_taskq(ahci_ctl_t *);
 129 static  int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
 130     uint8_t);
 131 static  int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
 132 static  int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
 133 static  int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
 134     sata_device_t *);
 135 static  int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
 136     sata_pmult_gscr_t *);
 137 static  int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
 138     sata_device_t *);
 139 static  int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 140 static  int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 141 static  int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
 142     sata_device_t *);
 143 
 144 static  void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
 145 static  int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
 146 static  void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
 147     ahci_addr_t *);
 148 static  void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
 149 static  int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
 150     ahci_addr_t *, sata_pkt_t *);
 151 static  int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
 152     ahci_addr_t *, sata_pkt_t *);
 153 static  int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
 154     ahci_addr_t *, int);
 155 static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
 156 static  void ahci_copy_ncq_err_page(sata_cmd_t *,
 157     struct sata_ncq_error_recovery_page *);
 158 static  void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
 159 static  void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
 160 static  void ahci_flush_doneq(ahci_port_t *);
 161 
 162 static  int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 163 static  int ahci_hba_reset(ahci_ctl_t *);
 164 static  int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 165 static  int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 166 static  void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
 167 static  int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 168     ahci_addr_t *);
 169 static  int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 170     ahci_addr_t *);
 171 static  int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
 172     ahci_addr_t *);
 173 static  int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
 174 static  int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
 175     uint8_t);
 176 static  int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
 177     uint8_t, int, int *);
 178 static  void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
 179     uint32_t, uint32_t, uint32_t, uint32_t);
 180 static  uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
 181 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
 182     uint8_t, sata_pkt_t *);
 183 static  void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
 184     ahci_addr_t *, uint32_t);
 185 static  void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
 186     uint8_t, uint32_t);
 187 static  void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
 188     uint8_t, uint32_t);
 189 static  void ahci_events_handler(void *);
 190 static  void ahci_watchdog_handler(ahci_ctl_t *);
 191 
 192 static  uint_t ahci_intr(caddr_t, caddr_t);
 193 static  void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
 194 static  int ahci_add_intrs(ahci_ctl_t *, int);
 195 static  void ahci_rem_intrs(ahci_ctl_t *);
 196 static  void ahci_enable_all_intrs(ahci_ctl_t *);
 197 static  void ahci_disable_all_intrs(ahci_ctl_t *);
 198 static  void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
 199 static  void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
 200 
 201 static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
 202 static  int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
 203 static  int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
 204 static  int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
 205 static  int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
 206 static  int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
 207     ahci_port_t *, uint8_t);
 208 static  int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
 209 static  int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
 210     uint8_t, uint32_t);
 211 static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
 212     uint8_t, uint32_t);
 213 static  int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
 214 
 215 static  void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
 216 static  int ahci_get_num_implemented_ports(uint32_t);
 217 static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
 218 static  void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
 219 static  void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
 220 #if AHCI_DEBUG
 221 static  void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
 222 #endif
 223 
 224 
 225 /*
 226  * DMA attributes for the data buffer
 227  *
 228  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 229  * does not support 64-bit addressing
 230  */
 231 static ddi_dma_attr_t buffer_dma_attr = {
 232         DMA_ATTR_V0,            /* dma_attr_version */
 233         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 234         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 235         0x3fffffull,            /* dma_attr_count_max i.e. for one cookie */
 236         0x2ull,                 /* dma_attr_align: word aligned */
 237         1,                      /* dma_attr_burstsizes */
 238         1,                      /* dma_attr_minxfer */
 239         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 240         0xffffffffull,          /* dma_attr_seg */
 241         AHCI_PRDT_NUMBER,       /* dma_attr_sgllen */
 242         512,                    /* dma_attr_granular */
 243         0,                      /* dma_attr_flags */
 244 };
 245 
 246 /*
 247  * DMA attributes for the rcvd FIS
 248  *
 249  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 250  * does not support 64-bit addressing
 251  */
 252 static ddi_dma_attr_t rcvd_fis_dma_attr = {
 253         DMA_ATTR_V0,            /* dma_attr_version */
 254         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 255         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 256         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 257         0x100ull,               /* dma_attr_align: 256-byte aligned */
 258         1,                      /* dma_attr_burstsizes */
 259         1,                      /* dma_attr_minxfer */
 260         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 261         0xffffffffull,          /* dma_attr_seg */
 262         1,                      /* dma_attr_sgllen */
 263         1,                      /* dma_attr_granular */
 264         0,                      /* dma_attr_flags */
 265 };
 266 
 267 /*
 268  * DMA attributes for the command list
 269  *
 270  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 271  * does not support 64-bit addressing
 272  */
 273 static ddi_dma_attr_t cmd_list_dma_attr = {
 274         DMA_ATTR_V0,            /* dma_attr_version */
 275         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 276         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 277         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 278         0x400ull,               /* dma_attr_align: 1K-byte aligned */
 279         1,                      /* dma_attr_burstsizes */
 280         1,                      /* dma_attr_minxfer */
 281         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 282         0xffffffffull,          /* dma_attr_seg */
 283         1,                      /* dma_attr_sgllen */
 284         1,                      /* dma_attr_granular */
 285         0,                      /* dma_attr_flags */
 286 };
 287 
 288 /*
 289  * DMA attributes for cmd tables
 290  *
 291  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 292  * does not support 64-bit addressing
 293  */
 294 static ddi_dma_attr_t cmd_table_dma_attr = {
 295         DMA_ATTR_V0,            /* dma_attr_version */
 296         0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
 297         0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
 298         0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
 299         0x80ull,                /* dma_attr_align: 128-byte aligned */
 300         1,                      /* dma_attr_burstsizes */
 301         1,                      /* dma_attr_minxfer */
 302         0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
 303         0xffffffffull,          /* dma_attr_seg */
 304         1,                      /* dma_attr_sgllen */
 305         1,                      /* dma_attr_granular */
 306         0,                      /* dma_attr_flags */
 307 };
 308 
 309 
 310 /* Device access attributes */
 311 static ddi_device_acc_attr_t accattr = {
 312         DDI_DEVICE_ATTR_V1,
 313         DDI_STRUCTURE_LE_ACC,
 314         DDI_STRICTORDER_ACC,
 315         DDI_DEFAULT_ACC
 316 };
 317 
 318 
 319 static struct dev_ops ahcictl_dev_ops = {
 320         DEVO_REV,               /* devo_rev */
 321         0,                      /* refcnt  */
 322         ahci_getinfo,           /* info */
 323         nulldev,                /* identify */
 324         nulldev,                /* probe */
 325         ahci_attach,            /* attach */
 326         ahci_detach,            /* detach */
 327         nodev,                  /* no reset */
 328         (struct cb_ops *)0,     /* driver operations */
 329         NULL,                   /* bus operations */
 330         NULL,                   /* power */
 331         ahci_quiesce,           /* quiesce */
 332 };
 333 
 334 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
 335         SATA_TRAN_HOTPLUG_OPS_REV_1,
 336         ahci_tran_hotplug_port_activate,
 337         ahci_tran_hotplug_port_deactivate
 338 };
 339 
 340 extern struct mod_ops mod_driverops;
 341 
 342 static  struct modldrv modldrv = {
 343         &mod_driverops,             /* driverops */
 344         ahci_ident,             /* short description */
 345         &ahcictl_dev_ops,   /* driver ops */
 346 };
 347 
 348 static  struct modlinkage modlinkage = {
 349         MODREV_1,
 350         &modldrv,
 351         NULL
 352 };
 353 
 354 /* The following variables are watchdog handler related */
 355 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
 356 static clock_t ahci_watchdog_tick;
 357 
 358 /*
 359  * This static variable indicates the size of command table,
 360  * and it's changeable with prdt number, which ahci_dma_prdt_number
 361  * indicates.
 362  */
 363 static size_t ahci_cmd_table_size;
 364 
 365 /*
 366  * The below global variables are tunable via /etc/system
 367  *
 368  *      ahci_dma_prdt_number
 369  *      ahci_msi_enabled
 370  *      ahci_buf_64bit_dma
 371  *      ahci_commu_64bit_dma
 372  */
 373 
 374 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
 375 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
 376 
 377 /* AHCI MSI is tunable */
 378 boolean_t ahci_msi_enabled = B_TRUE;
 379 
 380 /*
 381  * 64-bit dma addressing for data buffer is tunable
 382  *
 383  * The variable controls only the below value:
 384  *      DBAU (upper 32-bits physical address of data block)
 385  */
 386 boolean_t ahci_buf_64bit_dma = B_TRUE;
 387 
 388 /*
 389  * 64-bit dma addressing for communication system descriptors is tunable
 390  *
 391  * The variable controls the below three values:
 392  *
 393  *      PxCLBU (upper 32-bits for the command list base physical address)
 394  *      PxFBU (upper 32-bits for the received FIS base physical address)
 395  *      CTBAU (upper 32-bits of command table base)
 396  */
 397 boolean_t ahci_commu_64bit_dma = B_TRUE;
 398 
 399 /*
 400  * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
 401  * chipset. If the users want to have a try with 64-bit dma, please change
 402  * the below variable value to enable it.
 403  */
 404 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
 405 
 406 /*
 407  * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
 408  * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
 409  * please change the below value to enable it.
 410  */
 411 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
 412 
 413 
 414 /*
 415  * End of global tunable variable definition
 416  */
 417 
 418 #if AHCI_DEBUG
 419 uint32_t ahci_debug_flags = 0;
 420 #else
 421 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
 422 #endif
 423 
 424 
 425 #if AHCI_DEBUG
 426 /* The following is needed for ahci_log() */
 427 static kmutex_t ahci_log_mutex;
 428 static char ahci_log_buf[512];
 429 #endif
 430 
 431 /* Opaque state pointer initialized by ddi_soft_state_init() */
 432 static void *ahci_statep = NULL;
 433 
 434 /*
 435  *  ahci module initialization.
 436  */
 437 int
 438 _init(void)
 439 {
 440         int     ret;
 441 
 442         ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
 443         if (ret != 0) {
 444                 goto err_out;
 445         }
 446 
 447 #if AHCI_DEBUG
 448         mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
 449 #endif
 450 
 451         if ((ret = sata_hba_init(&modlinkage)) != 0) {
 452 #if AHCI_DEBUG
 453                 mutex_destroy(&ahci_log_mutex);
 454 #endif
 455                 ddi_soft_state_fini(&ahci_statep);
 456                 goto err_out;
 457         }
 458 
 459         /* watchdog tick */
 460         ahci_watchdog_tick = drv_sectohz(ahci_watchdog_timeout);
 461 
 462         ret = mod_install(&modlinkage);
 463         if (ret != 0) {
 464                 sata_hba_fini(&modlinkage);
 465 #if AHCI_DEBUG
 466                 mutex_destroy(&ahci_log_mutex);
 467 #endif
 468                 ddi_soft_state_fini(&ahci_statep);
 469                 goto err_out;
 470         }
 471 
 472         return (ret);
 473 
 474 err_out:
 475         cmn_err(CE_WARN, "!ahci: Module init failed");
 476         return (ret);
 477 }
 478 
 479 /*
 480  * ahci module uninitialize.
 481  */
 482 int
 483 _fini(void)
 484 {
 485         int     ret;
 486 
 487         ret = mod_remove(&modlinkage);
 488         if (ret != 0) {
 489                 return (ret);
 490         }
 491 
 492         /* Remove the resources allocated in _init(). */
 493         sata_hba_fini(&modlinkage);
 494 #if AHCI_DEBUG
 495         mutex_destroy(&ahci_log_mutex);
 496 #endif
 497         ddi_soft_state_fini(&ahci_statep);
 498 
 499         return (ret);
 500 }
 501 
 502 /*
 503  * _info entry point
 504  */
 505 int
 506 _info(struct modinfo *modinfop)
 507 {
 508         return (mod_info(&modlinkage, modinfop));
 509 }
 510 
 511 /*
 512  * The attach entry point for dev_ops.
 513  */
 514 static int
 515 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 516 {
 517         ahci_ctl_t *ahci_ctlp = NULL;
 518         int instance = ddi_get_instance(dip);
 519         int status;
 520         int attach_state;
 521         uint32_t cap_status, ahci_version;
 522         uint32_t ghc_control;
 523         int intr_types;
 524         int i;
 525         pci_regspec_t *regs;
 526         int regs_length;
 527         int rnumber;
 528 #if AHCI_DEBUG
 529         int speed;
 530 #endif
 531 
 532         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
 533             NULL);
 534 
 535         switch (cmd) {
 536         case DDI_ATTACH:
 537                 break;
 538 
 539         case DDI_RESUME:
 540 
 541                 /*
 542                  * During DDI_RESUME, the hardware state of the device
 543                  * (power may have been removed from the device) must be
 544                  * restored, allow pending requests to continue, and
 545                  * service new requests.
 546                  */
 547                 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
 548                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 549 
 550                 /*
 551                  * GHC.AE must be set to 1 before any other AHCI register
 552                  * is accessed
 553                  */
 554                 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 555                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
 556                 ghc_control |= AHCI_HBA_GHC_AE;
 557                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
 558                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
 559 
 560                 /* Restart watch thread */
 561                 if (ahci_ctlp->ahcictl_timeout_id == 0)
 562                         ahci_ctlp->ahcictl_timeout_id = timeout(
 563                             (void (*)(void *))ahci_watchdog_handler,
 564                             (caddr_t)ahci_ctlp, ahci_watchdog_tick);
 565 
 566                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 567 
 568                 /*
 569                  * Re-initialize the controller and enable the interrupts and
 570                  * restart all the ports.
 571                  *
 572                  * Note that so far we don't support hot-plug during
 573                  * suspend/resume.
 574                  */
 575                 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
 576                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
 577                             "Failed to initialize the controller "
 578                             "during DDI_RESUME", NULL);
 579                         return (DDI_FAILURE);
 580                 }
 581 
 582                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 583                 ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
 584                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 585 
 586                 return (DDI_SUCCESS);
 587 
 588         default:
 589                 return (DDI_FAILURE);
 590         }
 591 
 592         attach_state = AHCI_ATTACH_STATE_NONE;
 593 
 594         /* Allocate soft state */
 595         status = ddi_soft_state_zalloc(ahci_statep, instance);
 596         if (status != DDI_SUCCESS) {
 597                 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
 598                     instance);
 599                 goto err_out;
 600         }
 601 
 602         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
 603         ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
 604         ahci_ctlp->ahcictl_dip = dip;
 605 
 606         /* Initialize the cport/port mapping */
 607         for (i = 0; i < AHCI_MAX_PORTS; i++) {
 608                 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
 609                 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
 610         }
 611 
 612         attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
 613 
 614         /* Initialize FMA properties */
 615         ahci_fm_init(ahci_ctlp);
 616 
 617         attach_state |= AHCI_ATTACH_STATE_FMA;
 618 
 619         /*
 620          * Now map the AHCI base address; which includes global
 621          * registers and port control registers
 622          *
 623          * According to the spec, the AHCI Base Address is BAR5,
 624          * but BAR0-BAR4 are optional, so we need to check which
 625          * rnumber is used for BAR5.
 626          */
 627 
 628         /*
 629          * search through DDI "reg" property for the AHCI register set
 630          */
 631         if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
 632             DDI_PROP_DONTPASS, "reg", (int **)&regs,
 633             (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
 634                 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
 635                     instance);
 636                 goto err_out;
 637         }
 638 
 639         /* AHCI Base Address is located at 0x24 offset */
 640         for (rnumber = 0; rnumber < regs_length; ++rnumber) {
 641                 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
 642                     == AHCI_PCI_RNUM)
 643                         break;
 644         }
 645 
 646         ddi_prop_free(regs);
 647 
 648         if (rnumber == regs_length) {
 649                 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
 650                     instance);
 651                 goto err_out;
 652         }
 653 
 654         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
 655 
 656         status = ddi_regs_map_setup(dip,
 657             rnumber,
 658             (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
 659             0,
 660             0,
 661             &accattr,
 662             &ahci_ctlp->ahcictl_ahci_acc_handle);
 663         if (status != DDI_SUCCESS) {
 664                 cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
 665                     instance);
 666                 goto err_out;
 667         }
 668 
 669         attach_state |= AHCI_ATTACH_STATE_REG_MAP;
 670 
 671         /*
 672          * GHC.AE must be set to 1 before any other AHCI register
 673          * is accessed
 674          */
 675         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 676             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
 677         ghc_control |= AHCI_HBA_GHC_AE;
 678         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
 679             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
 680 
 681         /* Get the AHCI version information */
 682         ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 683             (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
 684 
 685         cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
 686             (ahci_version & 0xffff0000) >> 16,
 687             ((ahci_version & 0x0000ff00) >> 4 |
 688             (ahci_version & 0x000000ff)));
 689 
 690         /* We don't support controllers whose versions are lower than 1.0 */
 691         if (!(ahci_version & 0xffff0000)) {
 692                 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
 693                     "than version 1.0", instance);
 694                 goto err_out;
 695         }
 696 
 697         /* Get the HBA capabilities information */
 698         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 699             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
 700 
 701         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
 702             cap_status);
 703 
 704         /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
 705         if (ahci_version >= 0x00010200) {
 706                 uint32_t cap2_status;
 707 
 708                 /* Get the HBA capabilities extended information */
 709                 cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 710                     (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
 711 
 712                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 713                     "hba capabilities extended = 0x%x", cap2_status);
 714         }
 715 
 716 #if AHCI_DEBUG
 717         /* Get the interface speed supported by the HBA */
 718         speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
 719         if (speed == 0x01) {
 720                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 721                     "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
 722         } else if (speed == 0x10) {
 723                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 724                     "hba interface speed support: Gen 2 (3 Gbps)", NULL);
 725         } else if (speed == 0x11) {
 726                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 727                     "hba interface speed support: Gen 3 (6 Gbps)", NULL);
 728         }
 729 #endif
 730 
 731         /* Get the number of command slots supported by the HBA */
 732         ahci_ctlp->ahcictl_num_cmd_slots =
 733             ((cap_status & AHCI_HBA_CAP_NCS) >>
 734             AHCI_HBA_CAP_NCS_SHIFT) + 1;
 735 
 736         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
 737             ahci_ctlp->ahcictl_num_cmd_slots);
 738 
 739         /* Get the bit map which indicates ports implemented by the HBA */
 740         ahci_ctlp->ahcictl_ports_implemented =
 741             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
 742             (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
 743 
 744         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
 745             ahci_ctlp->ahcictl_ports_implemented);
 746 
 747         /* Max port number implemented */
 748         ahci_ctlp->ahcictl_num_ports =
 749             ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
 750 
 751         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
 752             (cap_status & AHCI_HBA_CAP_NP) + 1);
 753 
 754         /* Get the number of implemented ports by the HBA */
 755         ahci_ctlp->ahcictl_num_implemented_ports =
 756             ahci_get_num_implemented_ports(
 757             ahci_ctlp->ahcictl_ports_implemented);
 758 
 759         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 760             "hba number of implemented ports: %d",
 761             ahci_ctlp->ahcictl_num_implemented_ports);
 762 
 763         /* Check whether HBA supports 64bit DMA addressing */
 764         if (!(cap_status & AHCI_HBA_CAP_S64A)) {
 765                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
 766                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
 767                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 768                     "hba does not support 64-bit addressing", NULL);
 769         }
 770 
 771         /* Checking for the support of Port Multiplier */
 772         if (cap_status & AHCI_HBA_CAP_SPM) {
 773                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
 774                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 775                     "hba supports port multiplier (CBSS)", NULL);
 776 
 777                 /* Support FIS-based switching ? */
 778                 if (cap_status & AHCI_HBA_CAP_FBSS) {
 779                         ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
 780                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 781                             "hba supports FIS-based switching (FBSS)", NULL);
 782                 }
 783         }
 784 
 785         /* Checking for Support Command List Override */
 786         if (cap_status & AHCI_HBA_CAP_SCLO) {
 787                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
 788                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 789                     "hba supports command list override.", NULL);
 790         }
 791 
 792         /* Checking for Asynchronous Notification */
 793         if (cap_status & AHCI_HBA_CAP_SSNTF) {
 794                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
 795                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
 796                     "hba supports asynchronous notification.", NULL);
 797         }
 798 
 799         if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
 800             != DDI_SUCCESS) {
 801                 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
 802                     instance);
 803                 goto err_out;
 804         }
 805 
 806         attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
 807 
 808         /*
 809          * Check the pci configuration space, and set caps. We also
 810          * handle the hardware defect in this function.
 811          *
 812          * For example, force ATI SB600 to use 32-bit dma addressing
 813          * since it doesn't support 64-bit dma though its CAP register
 814          * declares it support.
 815          */
 816         if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
 817                 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
 818                     instance);
 819                 goto err_out;
 820         }
 821 
 822         /*
 823          * Disable the whole controller interrupts before adding
 824          * interrupt handlers(s).
 825          */
 826         ahci_disable_all_intrs(ahci_ctlp);
 827 
 828         /* Get supported interrupt types */
 829         if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
 830                 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
 831                     instance);
 832                 goto err_out;
 833         }
 834 
 835         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 836             "ddi_intr_get_supported_types() returned: 0x%x",
 837             intr_types);
 838 
 839         if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
 840                 /*
 841                  * Try MSI first, but fall back to FIXED if failed
 842                  */
 843                 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
 844                     DDI_SUCCESS) {
 845                         ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
 846                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 847                             "Using MSI interrupt type", NULL);
 848                         goto intr_done;
 849                 }
 850 
 851                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 852                     "MSI registration failed, "
 853                     "trying FIXED interrupts", NULL);
 854         }
 855 
 856         if (intr_types & DDI_INTR_TYPE_FIXED) {
 857                 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
 858                     DDI_SUCCESS) {
 859                         ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
 860                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 861                             "Using FIXED interrupt type", NULL);
 862                         goto intr_done;
 863                 }
 864 
 865                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
 866                     "FIXED interrupt registration failed", NULL);
 867         }
 868 
 869         cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
 870 
 871         goto err_out;
 872 
 873 intr_done:
 874 
 875         attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
 876 
 877         /* Initialize the controller mutex */
 878         mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
 879             (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
 880 
 881         attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
 882 
 883         if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
 884                 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
 885         } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
 886                 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
 887         }
 888 
 889         ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
 890             (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
 891             sizeof (ahci_prdt_item_t));
 892 
 893         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
 894             "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
 895             " ahci_cmd_table_size is 0x%x",
 896             ahci_dma_prdt_number, ahci_cmd_table_size);
 897 
 898         if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
 899                 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
 900                     ahci_dma_prdt_number;
 901 
 902         ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
 903         ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
 904         ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
 905         ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
 906 
 907         /*
 908          * enable 64bit dma for data buffer for SB600 if
 909          * sb600_buf_64bit_dma_disable is B_FALSE
 910          */
 911         if ((ahci_buf_64bit_dma == B_FALSE) ||
 912             ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
 913             !(sb600_buf_64bit_dma_disable == B_FALSE &&
 914             ahci_ctlp->ahcictl_venid == 0x1002 &&
 915             ahci_ctlp->ahcictl_devid == 0x4380))) {
 916                 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
 917                     0xffffffffull;
 918         }
 919 
 920         /*
 921          * enable 64bit dma for command buffer for SB600/700/710/800
 922          * if sbxxx_commu_64bit_dma_disable is B_FALSE
 923          */
 924         if ((ahci_commu_64bit_dma == B_FALSE) ||
 925             ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
 926             !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
 927             ahci_ctlp->ahcictl_venid == 0x1002 &&
 928             (ahci_ctlp->ahcictl_devid == 0x4380 ||
 929             ahci_ctlp->ahcictl_devid == 0x4391)))) {
 930                 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
 931                     0xffffffffull;
 932                 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
 933                     0xffffffffull;
 934                 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
 935                     0xffffffffull;
 936         }
 937 
 938         /* Allocate the ports structure */
 939         status = ahci_alloc_ports_state(ahci_ctlp);
 940         if (status != AHCI_SUCCESS) {
 941                 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
 942                     instance);
 943                 goto err_out;
 944         }
 945 
 946         attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
 947 
 948         /*
 949          * Initialize the controller and ports.
 950          */
 951         status = ahci_initialize_controller(ahci_ctlp);
 952         if (status != AHCI_SUCCESS) {
 953                 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
 954                     instance);
 955                 goto err_out;
 956         }
 957 
 958         attach_state |= AHCI_ATTACH_STATE_HW_INIT;
 959 
 960         /* Start one thread to check packet timeouts */
 961         ahci_ctlp->ahcictl_timeout_id = timeout(
 962             (void (*)(void *))ahci_watchdog_handler,
 963             (caddr_t)ahci_ctlp, ahci_watchdog_tick);
 964 
 965         attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
 966 
 967         if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
 968                 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
 969                     instance);
 970                 goto err_out;
 971         }
 972 
 973         /* Check all handles at the end of the attach operation. */
 974         if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
 975                 cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
 976                     instance);
 977                 goto err_out;
 978         }
 979 
 980         ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
 981 
 982         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
 983 
 984         return (DDI_SUCCESS);
 985 
 986 err_out:
 987         /* FMA message */
 988         ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
 989         ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
 990 
 991         if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
 992                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
 993                 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
 994                 ahci_ctlp->ahcictl_timeout_id = 0;
 995                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
 996         }
 997 
 998         if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
 999                 ahci_uninitialize_controller(ahci_ctlp);
1000         }
1001 
1002         if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1003                 ahci_dealloc_ports_state(ahci_ctlp);
1004         }
1005 
1006         if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1007                 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1008         }
1009 
1010         if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1011                 ahci_rem_intrs(ahci_ctlp);
1012         }
1013 
1014         if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1015                 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1016         }
1017 
1018         if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1019                 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1020         }
1021 
1022         if (attach_state & AHCI_ATTACH_STATE_FMA) {
1023                 ahci_fm_fini(ahci_ctlp);
1024         }
1025 
1026         if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1027                 ddi_soft_state_free(ahci_statep, instance);
1028         }
1029 
1030         return (DDI_FAILURE);
1031 }
1032 
1033 /*
1034  * The detach entry point for dev_ops.
1035  */
1036 static int
1037 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1038 {
1039         ahci_ctl_t *ahci_ctlp;
1040         int instance;
1041         int ret;
1042 
1043         instance = ddi_get_instance(dip);
1044         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1045 
1046         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1047 
1048         switch (cmd) {
1049         case DDI_DETACH:
1050 
1051                 /* disable the interrupts for an uninterrupted detach */
1052                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1053                 ahci_disable_all_intrs(ahci_ctlp);
1054                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1055 
1056                 /* unregister from the sata framework. */
1057                 ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1058                 if (ret != AHCI_SUCCESS) {
1059                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1060                         ahci_enable_all_intrs(ahci_ctlp);
1061                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1062                         return (DDI_FAILURE);
1063                 }
1064 
1065                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1066 
1067                 /* stop the watchdog handler */
1068                 (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1069                 ahci_ctlp->ahcictl_timeout_id = 0;
1070 
1071                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1072 
1073                 /* uninitialize the controller */
1074                 ahci_uninitialize_controller(ahci_ctlp);
1075 
1076                 /* remove the interrupts */
1077                 ahci_rem_intrs(ahci_ctlp);
1078 
1079                 /* deallocate the ports structures */
1080                 ahci_dealloc_ports_state(ahci_ctlp);
1081 
1082                 /* destroy mutex */
1083                 mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1084 
1085                 /* teardown the pci config */
1086                 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1087 
1088                 /* remove the reg maps. */
1089                 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1090 
1091                 /* release fma resource */
1092                 ahci_fm_fini(ahci_ctlp);
1093 
1094                 /* free the soft state. */
1095                 ddi_soft_state_free(ahci_statep, instance);
1096 
1097                 return (DDI_SUCCESS);
1098 
1099         case DDI_SUSPEND:
1100 
1101                 /*
1102                  * The steps associated with suspension must include putting
1103                  * the underlying device into a quiescent state so that it
1104                  * will not generate interrupts or modify or access memory.
1105                  */
1106                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1107                 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1108                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1109                         return (DDI_SUCCESS);
1110                 }
1111 
1112                 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1113 
1114                 /* stop the watchdog handler */
1115                 if (ahci_ctlp->ahcictl_timeout_id) {
1116                         (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1117                         ahci_ctlp->ahcictl_timeout_id = 0;
1118                 }
1119 
1120                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1121 
1122                 /*
1123                  * drain the taskq
1124                  */
1125                 ahci_drain_ports_taskq(ahci_ctlp);
1126 
1127                 /*
1128                  * Disable the interrupts and stop all the ports.
1129                  */
1130                 ahci_uninitialize_controller(ahci_ctlp);
1131 
1132                 return (DDI_SUCCESS);
1133 
1134         default:
1135                 return (DDI_FAILURE);
1136         }
1137 }
1138 
1139 /*
1140  * The info entry point for dev_ops.
1141  *
1142  */
1143 static int
1144 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1145                     void *arg, void **result)
1146 {
1147 #ifndef __lock_lint
1148         _NOTE(ARGUNUSED(dip))
1149 #endif /* __lock_lint */
1150 
1151         ahci_ctl_t *ahci_ctlp;
1152         int instance;
1153         dev_t dev;
1154 
1155         dev = (dev_t)arg;
1156         instance = getminor(dev);
1157 
1158         switch (infocmd) {
1159                 case DDI_INFO_DEVT2DEVINFO:
1160                         ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
1161                         if (ahci_ctlp != NULL) {
1162                                 *result = ahci_ctlp->ahcictl_dip;
1163                                 return (DDI_SUCCESS);
1164                         } else {
1165                                 *result = NULL;
1166                                 return (DDI_FAILURE);
1167                         }
1168                 case DDI_INFO_DEVT2INSTANCE:
1169                         *(int *)result = instance;
1170                         break;
1171                 default:
1172                         break;
1173         }
1174 
1175         return (DDI_SUCCESS);
1176 }
1177 
1178 /*
1179  * Registers the ahci with sata framework.
1180  */
1181 static int
1182 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1183 {
1184         struct  sata_hba_tran   *sata_hba_tran;
1185 
1186         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1187             "ahci_register_sata_hba_tran enter", NULL);
1188 
1189         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1190 
1191         /* Allocate memory for the sata_hba_tran  */
1192         sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1193 
1194         sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1195         sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1196         sata_hba_tran->sata_tran_hba_dma_attr =
1197             &ahci_ctlp->ahcictl_buffer_dma_attr;
1198 
1199         /* Report the number of implemented ports */
1200         sata_hba_tran->sata_tran_hba_num_cports =
1201             ahci_ctlp->ahcictl_num_implemented_ports;
1202 
1203         /* Support ATAPI device */
1204         sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1205 
1206         /* Get the data transfer capability for PIO command by the HBA */
1207         if (cap_status & AHCI_HBA_CAP_PMD) {
1208                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1209                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1210                     "DRQ block data transfer for PIO command protocol", NULL);
1211         }
1212 
1213         /*
1214          * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1215          * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1216          * QUEUED (EXT), and SERVICE commands). Queued operations are
1217          * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1218          * QUEUED commands when the HBA and device support native command
1219          * queuing(NCQ).
1220          *
1221          * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1222          * CAP register of the HBA indicates NCQ is supported.
1223          *
1224          * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1225          * set because the previous register content of PxCI can be re-written
1226          * in the register write.
1227          */
1228         if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1229             !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1230                 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1231                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1232                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1233                     "Command Queuing", NULL);
1234         }
1235 
1236         /* Support port multiplier? */
1237         if (cap_status & AHCI_HBA_CAP_SPM) {
1238                 sata_hba_tran->sata_tran_hba_features_support |=
1239                     SATA_CTLF_PORT_MULTIPLIER;
1240 
1241                 /* Support FIS-based switching for port multiplier? */
1242                 if (cap_status & AHCI_HBA_CAP_FBSS) {
1243                         sata_hba_tran->sata_tran_hba_features_support |=
1244                             SATA_CTLF_PMULT_FBS;
1245                 }
1246         }
1247 
1248         /* Report the number of command slots */
1249         sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1250 
1251         sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1252         sata_hba_tran->sata_tran_start = ahci_tran_start;
1253         sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1254         sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1255         sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1256 #ifdef __lock_lint
1257         sata_hba_tran->sata_tran_selftest = ahci_selftest;
1258 #endif
1259         /*
1260          * When SATA framework adds support for pwrmgt the
1261          * pwrmgt_ops needs to be updated
1262          */
1263         sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1264         sata_hba_tran->sata_tran_ioctl = NULL;
1265 
1266         ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1267 
1268         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1269 
1270         /* Attach it to SATA framework */
1271         if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1272             != DDI_SUCCESS) {
1273                 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1274                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
1275                 ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1276                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
1277                 return (AHCI_FAILURE);
1278         }
1279 
1280         return (AHCI_SUCCESS);
1281 }
1282 
1283 /*
1284  * Unregisters the ahci with sata framework.
1285  */
1286 static int
1287 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1288 {
1289         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1290             "ahci_unregister_sata_hba_tran enter", NULL);
1291 
1292         /* Detach from the SATA framework. */
1293         if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1294             DDI_SUCCESS) {
1295                 return (AHCI_FAILURE);
1296         }
1297 
1298         /* Deallocate sata_hba_tran. */
1299         kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1300             sizeof (sata_hba_tran_t));
1301 
1302         mutex_enter(&ahci_ctlp->ahcictl_mutex);
1303         ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1304         mutex_exit(&ahci_ctlp->ahcictl_mutex);
1305 
1306         return (AHCI_SUCCESS);
1307 }
1308 
1309 #define SET_PORTSTR(str, addrp)                                         \
1310         if (AHCI_ADDR_IS_PORT(addrp))                                   \
1311                 (void) sprintf((str), "%d", (addrp)->aa_port);               \
1312         else if (AHCI_ADDR_IS_PMULT(addrp))                             \
1313                 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port);       \
1314         else                                                            \
1315                 (void) sprintf((str), "%d:%d", (addrp)->aa_port,     \
1316                     (addrp)->aa_pmport);
1317 
1318 /*
1319  * ahci_tran_probe_port is called by SATA framework. It returns port state,
1320  * port status registers and an attached device type via sata_device
1321  * structure.
1322  *
1323  * We return the cached information from a previous hardware probe. The
1324  * actual hardware probing itself was done either from within
1325  * ahci_initialize_controller() during the driver attach or from a phy
1326  * ready change interrupt handler.
1327  */
1328 static int
1329 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1330 {
1331         ahci_ctl_t *ahci_ctlp;
1332         ahci_port_t *ahci_portp;
1333         ahci_addr_t addr, pmult_addr;
1334         uint8_t cport = sd->satadev_addr.cport;
1335         char portstr[10];
1336         uint8_t device_type;
1337         uint32_t port_state;
1338         uint8_t port;
1339         int rval = SATA_SUCCESS, rval_init;
1340 
1341         ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1342         port = ahci_ctlp->ahcictl_cport_to_port[cport];
1343 
1344         ahci_portp = ahci_ctlp->ahcictl_ports[port];
1345 
1346         mutex_enter(&ahci_portp->ahciport_mutex);
1347 
1348         ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1349         ASSERT(AHCI_ADDR_IS_VALID(&addr));
1350         SET_PORTSTR(portstr, &addr);
1351 
1352         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1353             "ahci_tran_probe_port enter: port %s", portstr);
1354 
1355         if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1356             (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1357             ahci_portp->ahciport_pmult_info == NULL)) {
1358                 /* port mutliplier is removed. */
1359                 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1360                     "ahci_tran_probe_port: "
1361                     "pmult is removed from port %s", portstr);
1362                 mutex_exit(&ahci_portp->ahciport_mutex);
1363                 return (SATA_FAILURE);
1364         }
1365 
1366         /*
1367          * The sata_device may refer to
1368          * 1. A controller port.
1369          *    A controller port should be ready here.
1370          * 2. A port multiplier.
1371          *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1372          *    it and register the port multiplier to the framework.
1373          *    SATA_ADDR_PMULT - check the status of all its device ports.
1374          * 3. A port multiplier port.
1375          *    If it has not been initialized, initialized it.
1376          *
1377          * A port multiplier or a port multiplier port may require some
1378          * initialization because we cannot do these time-consuming jobs in an
1379          * interrupt context.
1380          */
1381         if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1382                 AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1383                 /* Initialize registers on a port multiplier */
1384                 rval_init = ahci_initialize_pmult(ahci_ctlp,
1385                     ahci_portp, &pmult_addr, sd);
1386                 if (rval_init != AHCI_SUCCESS) {
1387                         AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1388                             "ahci_tran_probe_port: "
1389                             "pmult initialization failed.", NULL);
1390                         mutex_exit(&ahci_portp->ahciport_mutex);
1391                         return (SATA_FAILURE);
1392                 }
1393         } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1394                 /* Check pmports hotplug events */
1395                 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1396         } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1397             SATA_ADDR_DPMPORT)) {
1398                 if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1399                     &addr, sd) != AHCI_SUCCESS) {
1400                         rval = SATA_FAILURE;
1401                         goto out;
1402                 }
1403         }
1404 
1405         /* Update port state and device type */
1406         port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1407 
1408         switch (port_state) {
1409 
1410         case SATA_PSTATE_FAILED:
1411                 sd->satadev_state = SATA_PSTATE_FAILED;
1412                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1413                     "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1414                 goto out;
1415 
1416         case SATA_PSTATE_SHUTDOWN:
1417                 sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1418                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1419                     "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1420                 goto out;
1421 
1422         case SATA_PSTATE_PWROFF:
1423                 sd->satadev_state = SATA_PSTATE_PWROFF;
1424                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1425                     "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1426                 goto out;
1427 
1428         case SATA_PSTATE_PWRON:
1429                 sd->satadev_state = SATA_PSTATE_PWRON;
1430                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1431                     "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1432                 break;
1433 
1434         default:
1435                 sd->satadev_state = port_state;
1436                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1437                     "ahci_tran_probe_port: port %s PORT NORMAL %x",
1438                     portstr, port_state);
1439                 break;
1440         }
1441 
1442         device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1443 
1444         switch (device_type) {
1445 
1446         case SATA_DTYPE_ATADISK:
1447                 sd->satadev_type = SATA_DTYPE_ATADISK;
1448                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1449                     "ahci_tran_probe_port: port %s DISK found", portstr);
1450                 break;
1451 
1452         case SATA_DTYPE_ATAPI:
1453                 /*
1454                  * HBA driver only knows it's an ATAPI device, and don't know
1455                  * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1456                  * type need to be determined by checking IDENTIFY PACKET
1457                  * DEVICE data
1458                  */
1459                 sd->satadev_type = SATA_DTYPE_ATAPI;
1460                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1461                     "ahci_tran_probe_port: port %s ATAPI found", portstr);
1462                 break;
1463 
1464         case SATA_DTYPE_PMULT:
1465                 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1466                 sd->satadev_type = SATA_DTYPE_PMULT;
1467 
1468                 /* Update the number of pmports. */
1469                 ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1470                 sd->satadev_add_info = ahci_portp->
1471                     ahciport_pmult_info->ahcipmi_num_dev_ports;
1472 
1473                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1474                     "ahci_tran_probe_port: port %s Port Multiplier found",
1475                     portstr);
1476                 break;
1477 
1478         case SATA_DTYPE_UNKNOWN:
1479                 sd->satadev_type = SATA_DTYPE_UNKNOWN;
1480                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1481                     "ahci_tran_probe_port: port %s Unknown device found",
1482                     portstr);
1483                 break;
1484 
1485         default:
1486                 /* we don't support any other device types */
1487                 sd->satadev_type = SATA_DTYPE_NONE;
1488                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1489                     "ahci_tran_probe_port: port %s No device found", portstr);
1490                 break;
1491         }
1492 
1493 out:
1494         /* Register update only fails while probing a pmult/pmport */
1495         if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1496                 ahci_update_sata_registers(ahci_ctlp, port, sd);
1497         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1498                 if (port_state & SATA_STATE_READY)
1499                         if (ahci_update_pmult_pscr(ahci_ctlp,
1500                             &addr, sd) != AHCI_SUCCESS)
1501                                 rval = SATA_FAILURE;
1502         }
1503 
1504         /* Check handles for the sata registers access */
1505         if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1506             (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1507                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1508                     DDI_SERVICE_UNAFFECTED);
1509                 rval = SATA_FAILURE;
1510         }
1511 
1512         mutex_exit(&ahci_portp->ahciport_mutex);
1513         return (rval);
1514 }
1515 
1516 /*
1517  * There are four operation modes in sata framework:
1518  * SATA_OPMODE_INTERRUPTS
1519  * SATA_OPMODE_POLLING
1520  * SATA_OPMODE_ASYNCH
1521  * SATA_OPMODE_SYNCH
1522  *
1523  * Their combined meanings as following:
1524  *
1525  * SATA_OPMODE_SYNCH
1526  * The command has to be completed before sata_tran_start functions returns.
1527  * Either interrupts or polling could be used - it's up to the driver.
1528  * Mode used currently for internal, sata-module initiated operations.
1529  *
1530  * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1531  * It is the same as the one above.
1532  *
1533  * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1534  * The command has to be completed before sata_tran_start function returns.
1535  * No interrupt used, polling only. This should be the mode used for scsi
1536  * packets with FLAG_NOINTR.
1537  *
1538  * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1539  * The command may be queued (callback function specified). Interrupts could
1540  * be used. It's normal operation mode.
1541  */
1542 /*
1543  * Called by sata framework to transport a sata packet down stream.
1544  */
1545 static int
1546 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1547 {
1548         ahci_ctl_t *ahci_ctlp;
1549         ahci_port_t *ahci_portp;
1550         ahci_addr_t addr;
1551         uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
1552         uint8_t port;
1553         char portstr[10];
1554 
1555         ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1556         port = ahci_ctlp->ahcictl_cport_to_port[cport];
1557 
1558         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1559             "ahci_tran_start enter: cport %d satapkt 0x%p",
1560             cport, (void *)spkt);
1561 
1562         ahci_portp = ahci_ctlp->ahcictl_ports[port];
1563 
1564         mutex_enter(&ahci_portp->ahciport_mutex);
1565         ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1566         SET_PORTSTR(portstr, &addr);
1567 
1568         /* Sanity check */
1569         if (AHCI_ADDR_IS_PMPORT(&addr)) {
1570                 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1571                     ahci_portp->ahciport_pmult_info == NULL) {
1572 
1573                         spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1574                         spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1575                         spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1576                         ahci_update_sata_registers(ahci_ctlp, port,
1577                             &spkt->satapkt_device);
1578                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1579                             "ahci_tran_start returning PORT_ERROR while "
1580                             "pmult removed: port: %s", portstr);
1581                         mutex_exit(&ahci_portp->ahciport_mutex);
1582                         return (SATA_TRAN_PORT_ERROR);
1583                 }
1584 
1585                 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1586                     SATA_STATE_READY)) {
1587                         if (!ddi_in_panic() ||
1588                             ahci_initialize_pmport(ahci_ctlp,
1589                             ahci_portp, &addr) != AHCI_SUCCESS) {
1590                                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1591                                 spkt->satapkt_device.satadev_type =
1592                                     AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1593                                 spkt->satapkt_device.satadev_state =
1594                                     AHCIPORT_GET_STATE(ahci_portp, &addr);
1595                                 ahci_update_sata_registers(ahci_ctlp, port,
1596                                     &spkt->satapkt_device);
1597                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1598                                     "ahci_tran_start returning PORT_ERROR "
1599                                     "while sub-link is not initialized "
1600                                     "at port: %s", portstr);
1601                                 mutex_exit(&ahci_portp->ahciport_mutex);
1602                                 return (SATA_TRAN_PORT_ERROR);
1603                         }
1604                 }
1605         }
1606 
1607         if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1608             AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1609             AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1610                 /*
1611                  * In case the target driver would send the packet before
1612                  * sata framework can have the opportunity to process those
1613                  * event reports.
1614                  */
1615                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1616                 spkt->satapkt_device.satadev_state =
1617                     ahci_portp->ahciport_port_state;
1618                 ahci_update_sata_registers(ahci_ctlp, port,
1619                     &spkt->satapkt_device);
1620                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1621                     "ahci_tran_start returning PORT_ERROR while "
1622                     "port in FAILED/SHUTDOWN/PWROFF state: "
1623                     "port: %s", portstr);
1624                 mutex_exit(&ahci_portp->ahciport_mutex);
1625                 return (SATA_TRAN_PORT_ERROR);
1626         }
1627 
1628         if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1629                 /*
1630                  * ahci_intr_phyrdy_change() may have rendered it to
1631                  * SATA_DTYPE_NONE.
1632                  */
1633                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1634                 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1635                 spkt->satapkt_device.satadev_state =
1636                     ahci_portp->ahciport_port_state;
1637                 ahci_update_sata_registers(ahci_ctlp, port,
1638                     &spkt->satapkt_device);
1639                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1640                     "ahci_tran_start returning PORT_ERROR while "
1641                     "no device attached: port: %s", portstr);
1642                 mutex_exit(&ahci_portp->ahciport_mutex);
1643                 return (SATA_TRAN_PORT_ERROR);
1644         }
1645 
1646         /* R/W PMULT command will occupy the whole HBA port */
1647         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1648                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1649                     "ahci_tran_start returning BUSY while "
1650                     "executing READ/WRITE PORT-MULT command: "
1651                     "port: %s", portstr);
1652                 spkt->satapkt_reason = SATA_PKT_BUSY;
1653                 mutex_exit(&ahci_portp->ahciport_mutex);
1654                 return (SATA_TRAN_BUSY);
1655         }
1656 
1657         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1658                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1659                     "ahci_tran_start returning BUSY while "
1660                     "hot-plug in progress: port: %s", portstr);
1661                 spkt->satapkt_reason = SATA_PKT_BUSY;
1662                 mutex_exit(&ahci_portp->ahciport_mutex);
1663                 return (SATA_TRAN_BUSY);
1664         }
1665 
1666         /*
1667          * SATA HBA driver should remember that a device was reset and it
1668          * is supposed to reject any packets which do not specify either
1669          * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1670          *
1671          * This is to prevent a race condition when a device was arbitrarily
1672          * reset by the HBA driver (and lost it's setting) and a target
1673          * driver sending some commands to a device before the sata framework
1674          * has a chance to restore the device setting (such as cache enable/
1675          * disable or other resettable stuff).
1676          */
1677         /*
1678          * It is unnecessary to use specific flags to indicate
1679          * reset_in_progress for a pmport. While mopping, all command will be
1680          * mopped so that the entire HBA port is being dealt as a single
1681          * object.
1682          */
1683         if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1684                 ahci_portp->ahciport_reset_in_progress = 0;
1685                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1686                     "ahci_tran_start [CLEAR] the "
1687                     "reset_in_progress for port: %d", port);
1688         }
1689 
1690         if (ahci_portp->ahciport_reset_in_progress &&
1691             ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1692             ! ddi_in_panic()) {
1693                 spkt->satapkt_reason = SATA_PKT_BUSY;
1694                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1695                     "ahci_tran_start returning BUSY while "
1696                     "reset in progress: port: %d", port);
1697                 mutex_exit(&ahci_portp->ahciport_mutex);
1698                 return (SATA_TRAN_BUSY);
1699         }
1700 
1701 #ifdef AHCI_DEBUG
1702         if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1703                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1704                     "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1705                     spkt, port);
1706         }
1707 #endif
1708 
1709         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1710                 spkt->satapkt_reason = SATA_PKT_BUSY;
1711                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1712                     "ahci_tran_start returning BUSY while "
1713                     "mopping in progress: port: %d", port);
1714                 mutex_exit(&ahci_portp->ahciport_mutex);
1715                 return (SATA_TRAN_BUSY);
1716         }
1717 
1718         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1719                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1720                     DDI_SERVICE_UNAFFECTED);
1721                 mutex_exit(&ahci_portp->ahciport_mutex);
1722                 return (SATA_TRAN_BUSY);
1723         }
1724 
1725         if (spkt->satapkt_op_mode &
1726             (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1727                 /*
1728                  * If a SYNC command to be executed in interrupt context,
1729                  * bounce it back to sata module.
1730                  */
1731                 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1732                     servicing_interrupt()) {
1733                         spkt->satapkt_reason = SATA_PKT_BUSY;
1734                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1735                             "ahci_tran_start returning BUSY while "
1736                             "sending SYNC mode under interrupt context: "
1737                             "port : %d", port);
1738                         mutex_exit(&ahci_portp->ahciport_mutex);
1739                         return (SATA_TRAN_BUSY);
1740                 }
1741 
1742                 /* We need to do the sync start now */
1743                 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1744                     spkt) == AHCI_FAILURE) {
1745                         goto fail_out;
1746                 }
1747         } else {
1748                 /* Async start, using interrupt */
1749                 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1750                     == AHCI_FAILURE) {
1751                         spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1752                         goto fail_out;
1753                 }
1754         }
1755 
1756         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1757             "sata tran accepted: port %s", portstr);
1758 
1759         mutex_exit(&ahci_portp->ahciport_mutex);
1760         return (SATA_TRAN_ACCEPTED);
1761 
1762 fail_out:
1763         /*
1764          * Failed to deliver packet to the controller.
1765          * Check if it's caused by invalid handles.
1766          */
1767         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1768             ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1769                 spkt->satapkt_device.satadev_type =
1770                     AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1771                 spkt->satapkt_device.satadev_state =
1772                     AHCIPORT_GET_STATE(ahci_portp, &addr);
1773                 spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1774                 mutex_exit(&ahci_portp->ahciport_mutex);
1775                 return (SATA_TRAN_PORT_ERROR);
1776         }
1777 
1778         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1779             "return QUEUE_FULL: port %d", port);
1780         mutex_exit(&ahci_portp->ahciport_mutex);
1781         return (SATA_TRAN_QUEUE_FULL);
1782 }
1783 
1784 /*
1785  * SATA_OPMODE_SYNCH flag is set
1786  *
1787  * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1788  * without interrupt, otherwise we can still use the interrupt.
1789  */
1790 static int
1791 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1792     ahci_addr_t *addrp, sata_pkt_t *spkt)
1793 {
1794         int pkt_timeout_ticks;
1795         uint32_t timeout_tags;
1796         int rval;
1797         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1798         uint8_t port = addrp->aa_port;
1799 
1800         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1801 
1802         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1803             "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1804 
1805         if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1806                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1807                 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1808                     addrp, spkt)) == AHCI_FAILURE) {
1809                         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1810                         return (rval);
1811                 }
1812 
1813                 pkt_timeout_ticks =
1814                     drv_sectohz((clock_t)spkt->satapkt_time);
1815 
1816                 while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1817                         mutex_exit(&ahci_portp->ahciport_mutex);
1818 
1819                         /* Simulate the interrupt */
1820                         ahci_port_intr(ahci_ctlp, ahci_portp, port);
1821 
1822                         drv_usecwait(AHCI_10MS_USECS);
1823 
1824                         mutex_enter(&ahci_portp->ahciport_mutex);
1825                         pkt_timeout_ticks -= AHCI_10MS_TICKS;
1826                         if (pkt_timeout_ticks < 0) {
1827                                 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1828                                     "port %d satapkt 0x%p timed out\n",
1829                                     instance, port, (void *)spkt);
1830                                 timeout_tags = (0x1 << rval);
1831                                 mutex_exit(&ahci_portp->ahciport_mutex);
1832                                 ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1833                                     port, timeout_tags);
1834                                 mutex_enter(&ahci_portp->ahciport_mutex);
1835                         }
1836                 }
1837                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1838                 return (AHCI_SUCCESS);
1839 
1840         } else {
1841                 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1842                     addrp, spkt)) == AHCI_FAILURE)
1843                         return (rval);
1844 
1845 #if AHCI_DEBUG
1846                 /*
1847                  * Note that the driver always uses the slot 0 to deliver
1848                  * REQUEST SENSE or READ LOG EXT command
1849                  */
1850                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1851                         ASSERT(rval == 0);
1852 #endif
1853 
1854                 while (spkt->satapkt_reason == SATA_PKT_BUSY)
1855                         cv_wait(&ahci_portp->ahciport_cv,
1856                             &ahci_portp->ahciport_mutex);
1857 
1858                 return (AHCI_SUCCESS);
1859         }
1860 }
1861 
1862 /*
1863  * Searches for and claims a free command slot.
1864  *
1865  * Returns value:
1866  *
1867  * AHCI_FAILURE returned only if
1868  *      1. No empty slot left
1869  *      2. Non-queued command requested while queued command(s) is outstanding
1870  *      3. Queued command requested while non-queued command(s) is outstanding
1871  *      4. HBA doesn't support multiple-use of command list while already a
1872  *         non-queued command is oustanding
1873  *      5. Queued command requested while some queued command(s) has been
1874  *         outstanding on a different port multiplier port. (AHCI spec 1.2,
1875  *         9.1.2)
1876  *
1877  * claimed slot number returned if succeeded
1878  *
1879  * NOTE: it will always return slot 0 for following commands to simplify the
1880  * algorithm.
1881  *      1. REQUEST SENSE or READ LOG EXT command during error recovery process
1882  *      2. READ/WRITE PORTMULT command
1883  */
1884 static int
1885 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1886     ahci_addr_t *addrp, int command_type)
1887 {
1888         uint32_t port_cmd_issue;
1889         uint32_t free_slots;
1890         int slot;
1891 
1892         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1893 
1894         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1895             "ahciport_pending_tags = 0x%x "
1896             "ahciport_pending_ncq_tags = 0x%x",
1897             ahci_portp->ahciport_pending_tags,
1898             ahci_portp->ahciport_pending_ncq_tags);
1899 
1900         /*
1901          * According to the AHCI spec, system software is responsible to
1902          * ensure that queued and non-queued commands are not mixed in
1903          * the command list.
1904          */
1905         if (command_type == AHCI_NON_NCQ_CMD) {
1906                 /* Non-NCQ command request */
1907                 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1908                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1909                             "ahci_claim_free_slot: there is still pending "
1910                             "queued command(s) in the command list, "
1911                             "so no available slot for the non-queued "
1912                             "command", NULL);
1913                         return (AHCI_FAILURE);
1914                 }
1915                 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1916                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1917                             "ahci_claim_free_slot: there is still pending "
1918                             "read/write port-mult command(s) in command list, "
1919                             "so no available slot for the non-queued command",
1920                             NULL);
1921                         return (AHCI_FAILURE);
1922                 }
1923                 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
1924                     NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1925                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1926                             "ahci_claim_free_slot: HBA cannot support multiple-"
1927                             "use of the command list for non-queued commands",
1928                             NULL);
1929                         return (AHCI_FAILURE);
1930                 }
1931                 free_slots = (~ahci_portp->ahciport_pending_tags) &
1932                     AHCI_SLOT_MASK(ahci_ctlp);
1933         } else if (command_type == AHCI_NCQ_CMD) {
1934                 /* NCQ command request */
1935                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
1936                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
1937                             "ahci_claim_free_slot: there is still pending "
1938                             "non-queued command(s) in the command list, "
1939                             "so no available slot for the queued command",
1940                             NULL);
1941                         return (AHCI_FAILURE);
1942                 }
1943 
1944                 /*
1945                  * NCQ commands cannot be sent to different port multiplier
1946                  * ports in Command-Based Switching mode
1947                  */
1948                 /*
1949                  * NOTE: In Command-Based Switching mode, AHCI controller
1950                  * usually reports a 'Handshake Error' when multiple NCQ
1951                  * commands are outstanding simultaneously.
1952                  */
1953                 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
1954                         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1955                         if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
1956                             NCQ_CMD_IN_PROGRESS(ahci_portp) &&
1957                             AHCIPORT_NCQ_PMPORT(ahci_portp) !=
1958                             addrp->aa_pmport) {
1959                                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1960                                     "ahci_claim_free_slot: there is still "
1961                                     "pending queued command(s) in the "
1962                                     "command list for another Port Multiplier "
1963                                     "port, so no available slot.", NULL);
1964                                 return (AHCI_FAILURE);
1965                         }
1966                 }
1967 
1968                 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
1969                     AHCI_NCQ_SLOT_MASK(ahci_portp);
1970         } else if (command_type == AHCI_ERR_RETRI_CMD) {
1971                 /* Error retrieval command request */
1972                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1973                     "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
1974                     "SENSE or READ LOG EXT command", NULL);
1975                 slot = 0;
1976                 goto out;
1977         } else if (command_type == AHCI_RDWR_PMULT_CMD) {
1978                 /*
1979                  * An extra check on PxCI. Sometimes PxCI bits may not be
1980                  * cleared during hot-plug or error recovery process.
1981                  */
1982                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
1983                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
1984 
1985                 if (port_cmd_issue != 0) {
1986                         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
1987                             "ahci_claim_free_slot: there is still pending "
1988                             "command(s) in command list (0x%x/0x%x, PxCI %x),"
1989                             "so no available slot for R/W PMULT command.",
1990                             NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
1991                             NCQ_CMD_IN_PROGRESS(ahci_portp),
1992                             port_cmd_issue);
1993                         return (AHCI_FAILURE);
1994                 }
1995 
1996                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1997                     "ahci_claim_free_slot: slot 0 is allocated for "
1998                     "READ/WRITE PORTMULT command", NULL);
1999                 slot = 0;
2000                 goto out;
2001         }
2002 
2003         slot = ddi_ffs(free_slots) - 1;
2004         if (slot == -1) {
2005                 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2006                     "ahci_claim_free_slot: no empty slots", NULL);
2007                 return (AHCI_FAILURE);
2008         }
2009 
2010         /*
2011          * According to the AHCI spec, to allow a simple mechanism for the
2012          * HBA to map command list slots to queue entries, software must
2013          * match the tag number it uses to the slot it is placing the command
2014          * in. For example, if a queued command is placed in slot 5, the tag
2015          * for that command must be 5.
2016          */
2017         if (command_type == AHCI_NCQ_CMD) {
2018                 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2019                 if (AHCI_ADDR_IS_PMPORT(addrp)) {
2020                         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2021                         AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2022                 }
2023         }
2024 
2025         ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2026 
2027 out:
2028         AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2029             "ahci_claim_free_slot: found slot: 0x%x", slot);
2030 
2031         return (slot);
2032 }
2033 
2034 /*
2035  * Builds the Command Table for the sata packet and delivers it to controller.
2036  *
2037  * Returns:
2038  *      slot number if we can obtain a slot successfully
2039  *      otherwise, return AHCI_FAILURE
2040  */
2041 static int
2042 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2043     ahci_addr_t *addrp, sata_pkt_t *spkt)
2044 {
2045         int cmd_slot;
2046         sata_cmd_t *scmd;
2047         ahci_fis_h2d_register_t *h2d_register_fisp;
2048         ahci_cmd_table_t *cmd_table;
2049         ahci_cmd_header_t *cmd_header;
2050         int ncookies;
2051         int i;
2052         int command_type = AHCI_NON_NCQ_CMD;
2053         int ncq_qdepth;
2054         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2055         uint8_t port, pmport;
2056 #if AHCI_DEBUG
2057         uint32_t *ptr;
2058         uint8_t *ptr2;
2059 #endif
2060 
2061         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2062 
2063         port = addrp->aa_port;
2064         pmport = addrp->aa_pmport;
2065 
2066         spkt->satapkt_reason = SATA_PKT_BUSY;
2067 
2068         scmd = &spkt->satapkt_cmd;
2069 
2070         /* Check if the command is a NCQ command */
2071         if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2072             scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2073                 command_type = AHCI_NCQ_CMD;
2074 
2075                 /*
2076                  * When NCQ is support, system software must determine the
2077                  * maximum tag allowed by the device and the HBA, and it
2078                  * must use a value not beyond of the lower bound of the two.
2079                  *
2080                  * Sata module is going to calculate the qdepth and send
2081                  * down to HBA driver via sata_cmd.
2082                  */
2083                 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2084 
2085                 /*
2086                  * At the moment, the driver doesn't support the dynamic
2087                  * setting of the maximum ncq depth, and the value can be
2088                  * set either during the attach or after hot-plug insertion.
2089                  */
2090                 if (ahci_portp->ahciport_max_ncq_tags == 0) {
2091                         ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2092                         AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2093                             "ahci_deliver_satapkt: port %d the max tags for "
2094                             "NCQ command is %d", port, ncq_qdepth);
2095                 } else {
2096                         if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2097                                 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2098                                     " port %d the max tag for NCQ command is "
2099                                     "requested to change from %d to %d, at the"
2100                                     " moment the driver doesn't support the "
2101                                     "dynamic change so it's going to "
2102                                     "still use the previous tag value",
2103                                     instance, port,
2104                                     ahci_portp->ahciport_max_ncq_tags,
2105                                     ncq_qdepth);
2106                         }
2107                 }
2108         }
2109 
2110         /* Check if the command is an error retrieval command */
2111         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2112                 command_type = AHCI_ERR_RETRI_CMD;
2113 
2114         /* Check if the command is an read/write pmult command */
2115         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2116                 command_type = AHCI_RDWR_PMULT_CMD;
2117 
2118         /* Check if there is an empty command slot */
2119         cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2120             addrp, command_type);
2121         if (cmd_slot == AHCI_FAILURE) {
2122                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2123                 return (AHCI_FAILURE);
2124         }
2125 
2126         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2127             "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2128             "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2129             cmd_slot, port, (void *)spkt);
2130 
2131         cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2132         bzero((void *)cmd_table, ahci_cmd_table_size);
2133 
2134         /* For data transfer operations, it is the H2D Register FIS */
2135         h2d_register_fisp =
2136             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2137 
2138         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2139 
2140         /*
2141          * PMP field only make sense when target is a port multiplier or a
2142          * device behind a port multiplier. Otherwise should set it to 0.
2143          */
2144         if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2145                 SET_FIS_PMP(h2d_register_fisp, pmport);
2146 
2147         SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2148         SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2149         SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2150         SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2151 
2152         switch (scmd->satacmd_addr_type) {
2153 
2154         case 0:
2155                 /*
2156                  * satacmd_addr_type will be 0 for the commands below:
2157                  *      ATAPI command
2158                  *      SATAC_IDLE_IM
2159                  *      SATAC_STANDBY_IM
2160                  *      SATAC_DOWNLOAD_MICROCODE
2161                  *      SATAC_FLUSH_CACHE
2162                  *      SATAC_SET_FEATURES
2163                  *      SATAC_SMART
2164                  *      SATAC_ID_PACKET_DEVICE
2165                  *      SATAC_ID_DEVICE
2166                  *      SATAC_READ_PORTMULT
2167                  *      SATAC_WRITE_PORTMULT
2168                  */
2169                 /* FALLTHRU */
2170 
2171         case ATA_ADDR_LBA:
2172                 /* FALLTHRU */
2173 
2174         case ATA_ADDR_LBA28:
2175                 /* LBA[7:0] */
2176                 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2177 
2178                 /* LBA[15:8] */
2179                 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2180 
2181                 /* LBA[23:16] */
2182                 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2183 
2184                 /* LBA [27:24] (also called dev_head) */
2185                 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2186 
2187                 break;
2188 
2189         case ATA_ADDR_LBA48:
2190                 /* LBA[7:0] */
2191                 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2192 
2193                 /* LBA[15:8] */
2194                 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2195 
2196                 /* LBA[23:16] */
2197                 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2198 
2199                 /* LBA [31:24] */
2200                 SET_FIS_SECTOR_EXP(h2d_register_fisp,
2201                     scmd->satacmd_lba_low_msb);
2202 
2203                 /* LBA [39:32] */
2204                 SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2205                     scmd->satacmd_lba_mid_msb);
2206 
2207                 /* LBA [47:40] */
2208                 SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2209                     scmd->satacmd_lba_high_msb);
2210 
2211                 /* Set dev_head */
2212                 SET_FIS_DEV_HEAD(h2d_register_fisp,
2213                     scmd->satacmd_device_reg);
2214 
2215                 /* Set the extended sector count and features */
2216                 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2217                     scmd->satacmd_sec_count_msb);
2218                 SET_FIS_FEATURES_EXP(h2d_register_fisp,
2219                     scmd->satacmd_features_reg_ext);
2220                 break;
2221         }
2222 
2223         /*
2224          * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2225          * filled into features field, and sector count 8:15 is filled into
2226          * features (exp) field. The hba driver doesn't need to anything
2227          * special with regard to this, since sata framework has already
2228          * done so.
2229          *
2230          * However the driver needs to make sure TAG is filled into sector
2231          * field.
2232          */
2233         if (command_type == AHCI_NCQ_CMD) {
2234                 SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2235                     (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2236         }
2237 
2238         ncookies = scmd->satacmd_num_dma_cookies;
2239         AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2240             "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2241             ncookies, ahci_dma_prdt_number);
2242 
2243         ASSERT(ncookies <= ahci_dma_prdt_number);
2244         ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2245 
2246         /* *** now fill the scatter gather list ******* */
2247         for (i = 0; i < ncookies; i++) {
2248                 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2249                     scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2250                 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2251                     scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2252                 cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2253                     scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2254                 ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2255                     scmd->satacmd_dma_cookie_list[i].dmac_size;
2256         }
2257 
2258         AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2259             "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2260             ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2261 
2262         /* The ACMD field is filled in for ATAPI command */
2263         if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2264                 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2265                     SATA_ATAPI_MAX_CDB_LEN);
2266         }
2267 
2268         /* Set Command Header in Command List */
2269         cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2270         BZERO_DESCR_INFO(cmd_header);
2271         BZERO_PRD_BYTE_COUNT(cmd_header);
2272 
2273         /* Set the number of entries in the PRD table */
2274         SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2275 
2276         /* Set the length of the command in the CFIS area */
2277         SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2278 
2279         /*
2280          * PMP field only make sense when target is a port multiplier or a
2281          * device behind a port multiplier. Otherwise should set it to 0.
2282          */
2283         if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2284                 SET_PORT_MULTI_PORT(cmd_header, pmport);
2285 
2286         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2287             "sata_data_direction = 0x%x",
2288             scmd->satacmd_flags.sata_data_direction);
2289 
2290         /* Set A bit if it is an ATAPI command */
2291         if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2292                 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2293 
2294         /* Set W bit if data is going to the device */
2295         if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2296                 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2297 
2298         /*
2299          * Set the prefetchable bit - this bit is only valid if the PRDTL
2300          * field is non-zero or the ATAPI 'A' bit is set in the command
2301          * header. This bit cannot be set when using native command
2302          * queuing commands or when using FIS-based switching with a Port
2303          * multiplier.
2304          */
2305         if (command_type != AHCI_NCQ_CMD)
2306                 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2307 
2308         /*
2309          * Now remember the sata packet in ahciport_slot_pkts[].
2310          * Error retrieval command and r/w port multiplier command will
2311          * be stored specifically for each port.
2312          */
2313         if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2314             !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2315                 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2316 
2317         /*
2318          * Keep the timeout value
2319          */
2320         ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2321 
2322         /*
2323          * If the intial timout is less than 1 tick, then make it longer by
2324          * 1 tick to avoid immediate timeout
2325          */
2326         if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2327             ahci_watchdog_timeout)
2328                 ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2329                     ahci_watchdog_timeout;
2330 
2331 #if AHCI_DEBUG
2332         if (ahci_debug_flags & AHCIDBG_ATACMD &&
2333             scmd->satacmd_cmd_reg != SATAC_PACKET ||
2334             ahci_debug_flags & AHCIDBG_ATAPICMD &&
2335             scmd->satacmd_cmd_reg == SATAC_PACKET) {
2336 
2337                 /* Dump the command header and table */
2338                 ahci_log(ahci_ctlp, CE_WARN, "\n");
2339                 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2340                     "0x%p cmd_reg 0x%x port %d", spkt,
2341                     scmd->satacmd_cmd_reg, port);
2342                 ptr = (uint32_t *)cmd_header;
2343                 ahci_log(ahci_ctlp, CE_WARN,
2344                     "  Command Header:%8x %8x %8x %8x",
2345                     ptr[0], ptr[1], ptr[2], ptr[3]);
2346 
2347                 /* Dump the H2D register FIS */
2348                 ptr = (uint32_t *)h2d_register_fisp;
2349                 ahci_log(ahci_ctlp, CE_WARN,
2350                     "  Command FIS:   %8x %8x %8x %8x",
2351                     ptr[0], ptr[1], ptr[2], ptr[3]);
2352 
2353                 /* Dump the ACMD register FIS */
2354                 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2355                 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2356                         if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2357                                 ahci_log(ahci_ctlp, CE_WARN,
2358                                     "  ATAPI command: %2x %2x %2x %2x "
2359                                     "%2x %2x %2x %2x",
2360                                     ptr2[8 * i], ptr2[8 * i + 1],
2361                                     ptr2[8 * i + 2], ptr2[8 * i + 3],
2362                                     ptr2[8 * i + 4], ptr2[8 * i + 5],
2363                                     ptr2[8 * i + 6], ptr2[8 * i + 7]);
2364 
2365                 /* Dump the PRDT */
2366                 for (i = 0; i < ncookies; i++) {
2367                         ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2368                         ahci_log(ahci_ctlp, CE_WARN,
2369                             "  Cookie %d:      %8x %8x %8x %8x",
2370                             i, ptr[0], ptr[1], ptr[2], ptr[3]);
2371                 }
2372         }
2373 #endif
2374 
2375         (void) ddi_dma_sync(
2376             ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2377             0,
2378             ahci_cmd_table_size,
2379             DDI_DMA_SYNC_FORDEV);
2380 
2381         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2382             cmd_slot * sizeof (ahci_cmd_header_t),
2383             sizeof (ahci_cmd_header_t),
2384             DDI_DMA_SYNC_FORDEV);
2385 
2386         if ((ahci_check_dma_handle(ahci_portp->
2387             ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2388             ahci_check_dma_handle(ahci_portp->
2389             ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2390                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2391                     DDI_SERVICE_UNAFFECTED);
2392                 return (AHCI_FAILURE);
2393         }
2394 
2395         /* Set the corresponding bit in the PxSACT.DS for queued command */
2396         if (command_type == AHCI_NCQ_CMD) {
2397                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2398                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2399                     (0x1 << cmd_slot));
2400         }
2401 
2402         /* Indicate to the HBA that a command is active. */
2403         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2404             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2405             (0x1 << cmd_slot));
2406 
2407         AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2408             "exit: port %d", port);
2409 
2410         /* Make sure the command is started by the PxSACT/PxCI */
2411         if (ahci_check_acc_handle(ahci_ctlp->
2412             ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2413                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2414                     DDI_SERVICE_UNAFFECTED);
2415                 return (AHCI_FAILURE);
2416         }
2417 
2418         return (cmd_slot);
2419 }
2420 
2421 /*
2422  * Called by the sata framework to abort the previously sent packet(s).
2423  *
2424  * Reset device to abort commands.
2425  */
2426 static int
2427 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2428 {
2429         ahci_ctl_t *ahci_ctlp;
2430         ahci_port_t *ahci_portp;
2431         uint32_t slot_status = 0;
2432         uint32_t aborted_tags = 0;
2433         uint32_t finished_tags = 0;
2434         uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2435         uint8_t port;
2436         int tmp_slot;
2437         int instance = ddi_get_instance(dip);
2438 
2439         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2440         port = ahci_ctlp->ahcictl_cport_to_port[cport];
2441 
2442         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2443             "ahci_tran_abort enter: port %d", port);
2444 
2445         ahci_portp = ahci_ctlp->ahcictl_ports[port];
2446         mutex_enter(&ahci_portp->ahciport_mutex);
2447 
2448         /*
2449          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2450          * commands are being mopped, therefore there is nothing else to do
2451          */
2452         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2453                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2454                     "ahci_tran_abort: port %d is in "
2455                     "mopping process, so just return directly ", port);
2456                 mutex_exit(&ahci_portp->ahciport_mutex);
2457                 return (SATA_SUCCESS);
2458         }
2459 
2460         /*
2461          * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2462          * command is being executed so no other commands is outstanding,
2463          * nothing to do.
2464          */
2465         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2466                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2467                     "ahci_tran_abort: port %d is reading/writing "
2468                     "port multiplier, so just return directly ", port);
2469                 mutex_exit(&ahci_portp->ahciport_mutex);
2470                 return (SATA_SUCCESS);
2471         }
2472 
2473         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2474             ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2475             ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2476                 /*
2477                  * In case the targer driver would send the request before
2478                  * sata framework can have the opportunity to process those
2479                  * event reports.
2480                  */
2481                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2482                 spkt->satapkt_device.satadev_state =
2483                     ahci_portp->ahciport_port_state;
2484                 ahci_update_sata_registers(ahci_ctlp, port,
2485                     &spkt->satapkt_device);
2486                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2487                     "ahci_tran_abort returning SATA_FAILURE while "
2488                     "port in FAILED/SHUTDOWN/PWROFF state: "
2489                     "port: %d", port);
2490                 mutex_exit(&ahci_portp->ahciport_mutex);
2491                 return (SATA_FAILURE);
2492         }
2493 
2494         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2495                 /*
2496                  * ahci_intr_phyrdy_change() may have rendered it to
2497                  * AHCI_PORT_TYPE_NODEV.
2498                  */
2499                 spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2500                 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2501                 spkt->satapkt_device.satadev_state =
2502                     ahci_portp->ahciport_port_state;
2503                 ahci_update_sata_registers(ahci_ctlp, port,
2504                     &spkt->satapkt_device);
2505                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2506                     "ahci_tran_abort returning SATA_FAILURE while "
2507                     "no device attached: port: %d", port);
2508                 mutex_exit(&ahci_portp->ahciport_mutex);
2509                 return (SATA_FAILURE);
2510         }
2511 
2512         if (flag == SATA_ABORT_ALL_PACKETS) {
2513                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2514                         aborted_tags = ahci_portp->ahciport_pending_tags;
2515                 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2516                         aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2517 
2518                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2519                     instance, port);
2520         } else {
2521                 aborted_tags = 0xffffffff;
2522                 /*
2523                  * Aborting one specific packet, first search the
2524                  * ahciport_slot_pkts[] list for matching spkt.
2525                  */
2526                 for (tmp_slot = 0;
2527                     tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2528                         if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2529                                 aborted_tags = (0x1 << tmp_slot);
2530                                 break;
2531                         }
2532                 }
2533 
2534                 if (aborted_tags == 0xffffffff) {
2535                         /* request packet is not on the pending list */
2536                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2537                             "Cannot find the aborting pkt 0x%p on the "
2538                             "pending list", (void *)spkt);
2539                         ahci_update_sata_registers(ahci_ctlp, port,
2540                             &spkt->satapkt_device);
2541                         mutex_exit(&ahci_portp->ahciport_mutex);
2542                         return (SATA_FAILURE);
2543                 }
2544                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2545                     instance, port, (void *)spkt);
2546         }
2547 
2548         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2549                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2550                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2551         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2552                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2553                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2554 
2555         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2556         ahci_portp->ahciport_mop_in_progress++;
2557 
2558         /*
2559          * To abort the packet(s), first we are trying to clear PxCMD.ST
2560          * to stop the port, and if the port can be stopped
2561          * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2562          * then we just send back the aborted packet(s) with ABORTED flag
2563          * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2564          * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2565          * perform a COMRESET.
2566          */
2567         (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2568             ahci_portp, port, NULL, NULL);
2569 
2570         /*
2571          * Compute which have finished and which need to be retried.
2572          *
2573          * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2574          * minus the slot_status. The aborted_tags has to be deducted by
2575          * finished_tags since we can't possibly abort a tag which had finished
2576          * already.
2577          */
2578         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2579                 finished_tags = ahci_portp->ahciport_pending_tags &
2580                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2581         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2582                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2583                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2584 
2585         aborted_tags &= ~finished_tags;
2586 
2587         ahci_mop_commands(ahci_ctlp,
2588             ahci_portp,
2589             slot_status,
2590             0, /* failed tags */
2591             0, /* timeout tags */
2592             aborted_tags,
2593             0); /* reset tags */
2594 
2595         ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2596         mutex_exit(&ahci_portp->ahciport_mutex);
2597 
2598         return (SATA_SUCCESS);
2599 }
2600 
2601 /*
2602  * Used to do device reset and reject all the pending packets on a device
2603  * during the reset operation.
2604  *
2605  * NOTE: ONLY called by ahci_tran_reset_dport
2606  */
2607 static int
2608 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2609     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2610 {
2611         uint32_t slot_status = 0;
2612         uint32_t reset_tags = 0;
2613         uint32_t finished_tags = 0;
2614         uint8_t port = addrp->aa_port;
2615         sata_device_t sdevice;
2616         int ret;
2617 
2618         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2619 
2620         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2621             "ahci_reset_device_reject_pkts on port: %d", port);
2622 
2623         /*
2624          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2625          * commands are being mopped, therefore there is nothing else to do
2626          */
2627         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2628                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2629                     "ahci_reset_device_reject_pkts: port %d is in "
2630                     "mopping process, so return directly ", port);
2631                 return (SATA_SUCCESS);
2632         }
2633 
2634         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2635                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2636                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2637                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2638         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2639                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2640                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2641                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2642         }
2643 
2644         if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2645             != AHCI_SUCCESS) {
2646                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2647                     "Try to do a port reset after software "
2648                     "reset failed", port);
2649                 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2650                 if (ret != AHCI_SUCCESS) {
2651                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2652                             "ahci_reset_device_reject_pkts: port %d "
2653                             "failed", port);
2654                         return (SATA_FAILURE);
2655                 }
2656         }
2657         /* Set the reset in progress flag */
2658         ahci_portp->ahciport_reset_in_progress = 1;
2659 
2660         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2661         ahci_portp->ahciport_mop_in_progress++;
2662 
2663         /* Indicate to the framework that a reset has happened */
2664         bzero((void *)&sdevice, sizeof (sata_device_t));
2665         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2666         sdevice.satadev_addr.pmport = 0;
2667         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2668         sdevice.satadev_state = SATA_DSTATE_RESET |
2669             SATA_DSTATE_PWR_ACTIVE;
2670         mutex_exit(&ahci_portp->ahciport_mutex);
2671         sata_hba_event_notify(
2672             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2673             &sdevice,
2674             SATA_EVNT_DEVICE_RESET);
2675         mutex_enter(&ahci_portp->ahciport_mutex);
2676 
2677         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2678             "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2679 
2680         /* Next try to mop the pending commands */
2681         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2682                 finished_tags = ahci_portp->ahciport_pending_tags &
2683                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2684         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2685                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2686                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2687 
2688         reset_tags &= ~finished_tags;
2689 
2690         ahci_mop_commands(ahci_ctlp,
2691             ahci_portp,
2692             slot_status,
2693             0, /* failed tags */
2694             0, /* timeout tags */
2695             0, /* aborted tags */
2696             reset_tags); /* reset tags */
2697 
2698         return (SATA_SUCCESS);
2699 }
2700 
2701 /*
2702  * Used to do device reset and reject all the pending packets on a device
2703  * during the reset operation.
2704  *
2705  * NOTE: ONLY called by ahci_tran_reset_dport
2706  */
2707 static int
2708 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2709     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2710 {
2711         uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2712         uint8_t port = addrp->aa_port;
2713         uint8_t pmport = addrp->aa_pmport;
2714         sata_device_t sdevice;
2715 
2716         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2717 
2718         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2719             "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2720 
2721         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2722                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2723                     "ahci_reset_pmdevice_reject_pkts: port %d is in "
2724                     "mopping process, so return directly ", port);
2725                 return (SATA_SUCCESS);
2726         }
2727 
2728         /* Checking for outstanding commands */
2729         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2730                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2731                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2732                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2733         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2734                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2735                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2736                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2737         }
2738 
2739         /* Issue SOFTWARE reset command. */
2740         if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2741             != AHCI_SUCCESS) {
2742                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2743                     "Try to do a port reset after software "
2744                     "reset failed", port);
2745                 return (SATA_FAILURE);
2746         }
2747 
2748         /* Set the reset in progress flag */
2749         ahci_portp->ahciport_reset_in_progress = 1;
2750 
2751         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2752         ahci_portp->ahciport_mop_in_progress++;
2753 
2754         /* Indicate to the framework that a reset has happened */
2755         bzero((void *)&sdevice, sizeof (sata_device_t));
2756         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2757         sdevice.satadev_addr.pmport = pmport;
2758         if (AHCI_ADDR_IS_PMULT(addrp))
2759                 sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2760         else
2761                 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2762         sdevice.satadev_state = SATA_DSTATE_RESET |
2763             SATA_DSTATE_PWR_ACTIVE;
2764         mutex_exit(&ahci_portp->ahciport_mutex);
2765         sata_hba_event_notify(
2766             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2767             &sdevice,
2768             SATA_EVNT_DEVICE_RESET);
2769         mutex_enter(&ahci_portp->ahciport_mutex);
2770 
2771         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2772             "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2773             port, pmport);
2774 
2775         /* Next try to mop the pending commands */
2776         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2777                 finished_tags = ahci_portp->ahciport_pending_tags &
2778                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2779         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2780                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2781                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2782         reset_tags &= ~finished_tags;
2783 
2784         AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2785             "reset_tags = %x, finished_tags = %x, slot_status = %x",
2786             reset_tags, finished_tags, slot_status);
2787 
2788         /*
2789          * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2790          * try to reset a single device behind a port multiplier will
2791          * terminate all the commands on that HBA port. We need mop these
2792          * commands as well.
2793          */
2794         ahci_mop_commands(ahci_ctlp,
2795             ahci_portp,
2796             slot_status,
2797             0, /* failed tags */
2798             0, /* timeout tags */
2799             0, /* aborted tags */
2800             reset_tags); /* reset tags */
2801 
2802         return (SATA_SUCCESS);
2803 }
2804 
2805 /*
2806  * Used to do port reset and reject all the pending packets on a port during
2807  * the reset operation.
2808  */
2809 static int
2810 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2811     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2812 {
2813         uint32_t slot_status = 0;
2814         uint32_t reset_tags = 0;
2815         uint32_t finished_tags = 0;
2816         uint8_t port = addrp->aa_port;
2817 
2818         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2819 
2820         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2821             "ahci_reset_port_reject_pkts at port: %d", port);
2822 
2823         /*
2824          * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2825          * commands are being mopped, therefore there is nothing else to do
2826          */
2827         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2828                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2829                     "ahci_reset_port_reject_pkts: port %d is in "
2830                     "mopping process, so return directly ", port);
2831                 return (SATA_SUCCESS);
2832         }
2833 
2834         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2835         ahci_portp->ahciport_mop_in_progress++;
2836 
2837         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2838                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2839                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2840                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2841         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2842                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2843                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2844                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2845         }
2846 
2847         if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2848             ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2849             NULL) != AHCI_SUCCESS) {
2850 
2851                 /* Clear mop flag */
2852                 ahci_portp->ahciport_mop_in_progress--;
2853                 if (ahci_portp->ahciport_mop_in_progress == 0)
2854                         ahci_portp->ahciport_flags &=
2855                             ~AHCI_PORT_FLAG_MOPPING;
2856                 return (SATA_FAILURE);
2857         }
2858 
2859         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2860                 finished_tags = ahci_portp->ahciport_pending_tags &
2861                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2862         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2863                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2864                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2865 
2866         reset_tags &= ~finished_tags;
2867 
2868         ahci_mop_commands(ahci_ctlp,
2869             ahci_portp,
2870             slot_status,
2871             0, /* failed tags */
2872             0, /* timeout tags */
2873             0, /* aborted tags */
2874             reset_tags); /* reset tags */
2875 
2876         return (SATA_SUCCESS);
2877 }
2878 
2879 /*
2880  * Used to do hba reset and reject all the pending packets on all ports
2881  * during the reset operation.
2882  */
2883 static int
2884 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2885 {
2886         ahci_port_t *ahci_portp;
2887         uint32_t slot_status[AHCI_MAX_PORTS];
2888         uint32_t reset_tags[AHCI_MAX_PORTS];
2889         uint32_t finished_tags[AHCI_MAX_PORTS];
2890         int port;
2891         int ret = SATA_SUCCESS;
2892 
2893         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2894             "ahci_reset_hba_reject_pkts enter", NULL);
2895 
2896         bzero(slot_status, sizeof (slot_status));
2897         bzero(reset_tags, sizeof (reset_tags));
2898         bzero(finished_tags, sizeof (finished_tags));
2899 
2900         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2901                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2902                         continue;
2903                 }
2904 
2905                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2906 
2907                 mutex_enter(&ahci_portp->ahciport_mutex);
2908                 ahci_portp->ahciport_reset_in_progress = 1;
2909                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2910                         slot_status[port] = ddi_get32(
2911                             ahci_ctlp->ahcictl_ahci_acc_handle,
2912                             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2913                         reset_tags[port] = slot_status[port] &
2914                             AHCI_SLOT_MASK(ahci_ctlp);
2915                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2916                             "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2917                             port, reset_tags[port],
2918                             ahci_portp->ahciport_pending_tags);
2919                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2920                         slot_status[port] = ddi_get32(
2921                             ahci_ctlp->ahcictl_ahci_acc_handle,
2922                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2923                         reset_tags[port] = slot_status[port] &
2924                             AHCI_NCQ_SLOT_MASK(ahci_portp);
2925                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
2926                             "port %d: reset_tags = 0x%x pending_tags = 0x%x",
2927                             port, reset_tags[port],
2928                             ahci_portp->ahciport_pending_tags);
2929                 }
2930                 mutex_exit(&ahci_portp->ahciport_mutex);
2931         }
2932 
2933         if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
2934                 ret = SATA_FAILURE;
2935         }
2936 
2937         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2938                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2939                         continue;
2940                 }
2941 
2942                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
2943 
2944                 mutex_enter(&ahci_portp->ahciport_mutex);
2945                 /*
2946                  * To prevent recursive enter to ahci_mop_commands, we need
2947                  * check AHCI_PORT_FLAG_MOPPING flag.
2948                  */
2949                 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2950                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2951                             "ahci_reset_hba_reject_pkts: port %d is in "
2952                             "mopping process, so return directly ", port);
2953                         mutex_exit(&ahci_portp->ahciport_mutex);
2954                         continue;
2955                 }
2956 
2957                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2958                 ahci_portp->ahciport_mop_in_progress++;
2959 
2960                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2961                         finished_tags[port]  =
2962                             ahci_portp->ahciport_pending_tags &
2963                             ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
2964                 else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2965                         finished_tags[port] =
2966                             ahci_portp->ahciport_pending_ncq_tags &
2967                             ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
2968 
2969                 reset_tags[port] &= ~finished_tags[port];
2970 
2971                 ahci_mop_commands(ahci_ctlp,
2972                     ahci_portp,
2973                     slot_status[port],
2974                     0, /* failed tags */
2975                     0, /* timeout tags */
2976                     0, /* aborted tags */
2977                     reset_tags[port]); /* reset tags */
2978                 mutex_exit(&ahci_portp->ahciport_mutex);
2979         }
2980 out:
2981         return (ret);
2982 }
2983 
2984 /*
2985  * Called by sata framework to reset a port(s) or device.
2986  */
2987 static int
2988 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
2989 {
2990         ahci_ctl_t *ahci_ctlp;
2991         ahci_port_t *ahci_portp;
2992         ahci_addr_t addr;
2993         uint8_t cport = sd->satadev_addr.cport;
2994         uint8_t pmport = sd->satadev_addr.pmport;
2995         uint8_t port;
2996         int ret = SATA_SUCCESS;
2997         int instance = ddi_get_instance(dip);
2998 
2999         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3000         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3001         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3002 
3003         ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3004 
3005         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3006             "ahci_tran_reset_dport enter: cport %d", cport);
3007 
3008         switch (sd->satadev_addr.qual) {
3009         case SATA_ADDR_PMPORT:
3010                 /*
3011                  * If we want to issue a COMRESET on a pmport, we need to
3012                  * reject the outstanding commands on that pmport. According
3013                  * to AHCI spec, PxCI register could only be cleared by
3014                  * clearing PxCMD.ST, which will halt the controller port - as
3015                  * well as other pmports.
3016                  *
3017                  * Therefore we directly reset the controller port for
3018                  * simplicity. ahci_tran_probe_port() will handle reset stuff
3019                  * like initializing the given pmport.
3020                  */
3021                 /* FALLTHRU */
3022         case SATA_ADDR_CPORT:
3023                 /* Port reset */
3024                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3025                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3026                     "port %d reset port", instance, port);
3027 
3028                 mutex_enter(&ahci_portp->ahciport_mutex);
3029                 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3030                 mutex_exit(&ahci_portp->ahciport_mutex);
3031 
3032                 break;
3033 
3034         case SATA_ADDR_DPMPORT:
3035                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3036                     "port %d:%d reset device", instance, port, pmport);
3037                 /* FALLTHRU */
3038         case SATA_ADDR_DCPORT:
3039                 /* Device reset */
3040                 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3041                         cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3042                             "port %d reset device", instance, port);
3043 
3044                 mutex_enter(&ahci_portp->ahciport_mutex);
3045                 /*
3046                  * software reset request must be sent to SATA_PMULT_HOSTPORT
3047                  * if target is a port multiplier:
3048                  */
3049                 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3050                     ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3051                         AHCI_ADDR_SET_PMULT(&addr, port);
3052 
3053                 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3054                     ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3055                     ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3056                         /*
3057                          * In case the targer driver would send the request
3058                          * before sata framework can have the opportunity to
3059                          * process those event reports.
3060                          */
3061                         sd->satadev_state = ahci_portp->ahciport_port_state;
3062                         ahci_update_sata_registers(ahci_ctlp, port, sd);
3063                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3064                             "ahci_tran_reset_dport returning SATA_FAILURE "
3065                             "while port in FAILED/SHUTDOWN/PWROFF state: "
3066                             "port: %d", port);
3067                         mutex_exit(&ahci_portp->ahciport_mutex);
3068                         ret = SATA_FAILURE;
3069                         break;
3070                 }
3071 
3072                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3073                     SATA_DTYPE_NONE) {
3074                         /*
3075                          * ahci_intr_phyrdy_change() may have rendered it to
3076                          * AHCI_PORT_TYPE_NODEV.
3077                          */
3078                         sd->satadev_type = SATA_DTYPE_NONE;
3079                         sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3080                             &addr);
3081                         ahci_update_sata_registers(ahci_ctlp, port, sd);
3082                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3083                             "ahci_tran_reset_dport returning SATA_FAILURE "
3084                             "while no device attached: port: %d", port);
3085                         mutex_exit(&ahci_portp->ahciport_mutex);
3086                         ret = SATA_FAILURE;
3087                         break;
3088                 }
3089 
3090                 if (AHCI_ADDR_IS_PORT(&addr)) {
3091                         ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3092                             ahci_portp, &addr);
3093                 } else {
3094                         ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3095                             ahci_portp, &addr);
3096                 }
3097 
3098                 mutex_exit(&ahci_portp->ahciport_mutex);
3099                 break;
3100 
3101         case SATA_ADDR_CNTRL:
3102                 /* Reset the whole controller */
3103                 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3104                     "reset the whole hba", instance);
3105                 ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3106                 break;
3107 
3108         default:
3109                 ret = SATA_FAILURE;
3110         }
3111 
3112         return (ret);
3113 }
3114 
3115 /*
3116  * Called by sata framework to activate a port as part of hotplug.
3117  * (cfgadm -c connect satax/y)
3118  * Support port multiplier.
3119  */
3120 static int
3121 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3122 {
3123         ahci_ctl_t *ahci_ctlp;
3124         ahci_port_t *ahci_portp;
3125         ahci_addr_t addr;
3126         uint8_t cport = satadev->satadev_addr.cport;
3127         uint8_t pmport = satadev->satadev_addr.pmport;
3128         uint8_t port;
3129         int instance = ddi_get_instance(dip);
3130 
3131         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3132         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3133 
3134         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3135             "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3136 
3137         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3138 
3139         mutex_enter(&ahci_portp->ahciport_mutex);
3140         ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3141         ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3142 
3143         if (AHCI_ADDR_IS_PORT(&addr)) {
3144                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3145                     instance, port);
3146 
3147                 /* Enable the interrupts on the port */
3148                 ahci_enable_port_intrs(ahci_ctlp, port);
3149 
3150                 /*
3151                  * Reset the port so that the PHY communication would be
3152                  * re-established.  But this reset is an internal operation
3153                  * and the sata module doesn't need to know about it.
3154                  * Moreover, the port with a device attached will be started
3155                  * too.
3156                  */
3157                 (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3158                     ahci_portp, port,
3159                     AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3160                     NULL);
3161 
3162                 /*
3163                  * Need to check the link status and device status of the port
3164                  * and consider raising power if the port was in D3 state
3165                  */
3166                 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3167                 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3168                 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3169         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3170                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3171                     instance, port, pmport);
3172                 /* AHCI_ADDR_PMPORT */
3173                 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3174                 AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3175                     ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3176         }
3177 
3178         satadev->satadev_state = ahci_portp->ahciport_port_state;
3179 
3180         ahci_update_sata_registers(ahci_ctlp, port, satadev);
3181 
3182         mutex_exit(&ahci_portp->ahciport_mutex);
3183         return (SATA_SUCCESS);
3184 }
3185 
3186 /*
3187  * Called by sata framework to deactivate a port as part of hotplug.
3188  * (cfgadm -c disconnect satax/y)
3189  * Support port multiplier.
3190  */
3191 static int
3192 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3193 {
3194         ahci_ctl_t *ahci_ctlp;
3195         ahci_port_t *ahci_portp;
3196         ahci_addr_t addr;
3197         uint8_t cport = satadev->satadev_addr.cport;
3198         uint8_t pmport = satadev->satadev_addr.pmport;
3199         uint8_t port;
3200         uint32_t port_scontrol;
3201         int instance = ddi_get_instance(dip);
3202 
3203         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3204         port = ahci_ctlp->ahcictl_cport_to_port[cport];
3205 
3206         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3207             "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3208 
3209         ahci_portp = ahci_ctlp->ahcictl_ports[port];
3210         mutex_enter(&ahci_portp->ahciport_mutex);
3211         ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3212         ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3213 
3214         if (AHCI_ADDR_IS_PORT(&addr)) {
3215                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3216                     instance, port);
3217 
3218                 /* Disable the interrupts on the port */
3219                 ahci_disable_port_intrs(ahci_ctlp, port);
3220 
3221                 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3222 
3223                         /* First to abort all the pending commands */
3224                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3225 
3226                         /* Then stop the port */
3227                         (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3228                             ahci_portp, port);
3229                 }
3230 
3231                 /* Next put the PHY offline */
3232                 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3233                     (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3234                 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3235                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3236                     AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3237         } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3238                 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3239                     instance, port, pmport);
3240 
3241                 ahci_disable_port_intrs(ahci_ctlp, port);
3242                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3243                     != SATA_DTYPE_NONE)
3244                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3245 
3246                 /* Re-enable the interrupts for the other pmports */
3247                 ahci_enable_port_intrs(ahci_ctlp, port);
3248         }
3249 
3250         /* Update port state */
3251         AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3252         satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3253 
3254         ahci_update_sata_registers(ahci_ctlp, port, satadev);
3255 
3256         mutex_exit(&ahci_portp->ahciport_mutex);
3257         return (SATA_SUCCESS);
3258 }
3259 
3260 /*
3261  * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3262  * when a device is unplugged or a port is deactivated.
3263  */
3264 static void
3265 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3266     ahci_port_t *ahci_portp, uint8_t port)
3267 {
3268         uint32_t slot_status = 0;
3269         uint32_t abort_tags = 0;
3270 
3271         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3272 
3273         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3274             "ahci_reject_all_abort_pkts at port: %d", port);
3275 
3276         /* Read/write port multiplier command takes highest priority */
3277         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3278                 slot_status = 0x1;
3279                 abort_tags = 0x1;
3280                 goto out;
3281         }
3282 
3283         /*
3284          * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3285          * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3286          * to get the error data, if yes when the device is removed, the
3287          * command needs to be aborted too.
3288          */
3289         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3290                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3291                         slot_status = 0x1;
3292                         abort_tags = 0x1;
3293                         goto out;
3294                 } else {
3295                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3296                             "ahci_reject_all_abort_pkts return directly "
3297                             "port %d no needs to reject any outstanding "
3298                             "commands", port);
3299                         return;
3300                 }
3301         }
3302 
3303         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3304                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3305                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3306                 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3307         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3308                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3309                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3310                 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3311         }
3312 
3313 out:
3314         /* No need to do mop when there is no outstanding commands */
3315         if (slot_status != 0) {
3316                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3317                 ahci_portp->ahciport_mop_in_progress++;
3318 
3319                 ahci_mop_commands(ahci_ctlp,
3320                     ahci_portp,
3321                     slot_status,
3322                     0, /* failed tags */
3323                     0, /* timeout tags */
3324                     abort_tags, /* aborting tags */
3325                     0); /* reset tags */
3326         }
3327 }
3328 
3329 #if defined(__lock_lint)
3330 static int
3331 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3332 {
3333         return (SATA_SUCCESS);
3334 }
3335 #endif
3336 
3337 /*
3338  * Initialize fma capabilities and register with IO fault services.
3339  */
3340 static void
3341 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3342 {
3343         /*
3344          * Need to change iblock to priority for new MSI intr
3345          */
3346         ddi_iblock_cookie_t fm_ibc;
3347 
3348         ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3349             ahci_ctlp->ahcictl_dip,
3350             DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3351             DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3352             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3353 
3354         /* Only register with IO Fault Services if we have some capability */
3355         if (ahci_ctlp->ahcictl_fm_cap) {
3356                 /* Adjust access and dma attributes for FMA */
3357                 accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3358                 buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3359                 rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3360                 cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3361                 cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3362 
3363                 /*
3364                  * Register capabilities with IO Fault Services.
3365                  * ahcictl_fm_cap will be updated to indicate
3366                  * capabilities actually supported (not requested.)
3367                  */
3368                 ddi_fm_init(ahci_ctlp->ahcictl_dip,
3369                     &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3370 
3371                 if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3372                         cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3373                             ddi_get_instance(ahci_ctlp->ahcictl_dip));
3374                         return;
3375                 }
3376                 /*
3377                  * Initialize pci ereport capabilities if ereport
3378                  * capable (should always be.)
3379                  */
3380                 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3381                     DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3382                         pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3383                 }
3384 
3385                 /*
3386                  * Register error callback if error callback capable.
3387                  */
3388                 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3389                         ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3390                             ahci_fm_error_cb, (void *) ahci_ctlp);
3391                 }
3392 
3393                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3394                     "ahci_fm_fini: fma enabled.", NULL);
3395         }
3396 }
3397 
3398 /*
3399  * Releases fma capabilities and un-registers with IO fault services.
3400  */
3401 static void
3402 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3403 {
3404         /* Only unregister FMA capabilities if registered */
3405         if (ahci_ctlp->ahcictl_fm_cap) {
3406                 /*
3407                  * Un-register error callback if error callback capable.
3408                  */
3409                 if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3410                         ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3411                 }
3412 
3413                 /*
3414                  * Release any resources allocated by pci_ereport_setup()
3415                  */
3416                 if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3417                     DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3418                         pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3419                 }
3420 
3421                 /* Unregister from IO Fault Services */
3422                 ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3423 
3424                 /* Adjust access and dma attributes for FMA */
3425                 accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3426                 buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3427                 rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3428                 cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3429                 cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3430 
3431                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3432                     "ahci_fm_fini: fma disabled.", NULL);
3433         }
3434 }
3435 
3436 /*ARGSUSED*/
3437 static int
3438 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
3439 {
3440         /*
3441          * as the driver can always deal with an error in any dma or
3442          * access handle, we can just return the fme_status value.
3443          */
3444         pci_ereport_post(dip, err, NULL);
3445         return (err->fme_status);
3446 }
3447 
3448 int
3449 ahci_check_acc_handle(ddi_acc_handle_t handle)
3450 {
3451         ddi_fm_error_t de;
3452 
3453         ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
3454         return (de.fme_status);
3455 }
3456 
3457 int
3458 ahci_check_dma_handle(ddi_dma_handle_t handle)
3459 {
3460         ddi_fm_error_t de;
3461 
3462         ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
3463         return (de.fme_status);
3464 }
3465 
3466 /*
3467  * Generate an ereport
3468  */
3469 void
3470 ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
3471 {
3472         uint64_t ena;
3473         char buf[FM_MAX_CLASS];
3474 
3475         (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
3476         ena = fm_ena_generate(0, FM_ENA_FMT1);
3477         if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3478                 ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
3479                     DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
3480                     FM_EREPORT_VERSION, NULL);
3481         }
3482 }
3483 
3484 /*
3485  * Check if all handles are correctly allocated.
3486  */
3487 static int
3488 ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
3489 {
3490         int port;
3491 
3492         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
3493                 return (DDI_FAILURE);
3494         }
3495 
3496         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3497                 ahci_port_t *ahci_portp;
3498 
3499                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3500                         continue;
3501 
3502                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3503 
3504                 mutex_enter(&ahci_portp->ahciport_mutex);
3505                 if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
3506                         mutex_exit(&ahci_portp->ahciport_mutex);
3507                         return (DDI_FAILURE);
3508                 }
3509                 mutex_exit(&ahci_portp->ahciport_mutex);
3510         }
3511 
3512         return (DDI_SUCCESS);
3513 }
3514 
3515 /*
3516  * Check the access handles for the controller. Note that
3517  * ahcictl_pci_conf_handle is only used in attach process.
3518  */
3519 static int
3520 ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
3521 {
3522         if ((ahci_check_acc_handle(ahci_ctlp->
3523             ahcictl_pci_conf_handle) != DDI_FM_OK) ||
3524             (ahci_check_acc_handle(ahci_ctlp->
3525             ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
3526                 return (DDI_FAILURE);
3527         }
3528         return (DDI_SUCCESS);
3529 }
3530 
3531 /*
3532  * Check the DMA handles and the access handles of a controller port.
3533  */
3534 static int
3535 ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
3536 {
3537         ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
3538         int slot;
3539 
3540         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3541 
3542         if ((ahci_check_dma_handle(ahci_portp->
3543             ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
3544             (ahci_check_dma_handle(ahci_portp->
3545             ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
3546             (ahci_check_acc_handle(ahci_portp->
3547             ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
3548             (ahci_check_acc_handle(ahci_portp->
3549             ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
3550                 return (DDI_FAILURE);
3551         }
3552         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
3553                 if (ahci_check_slot_handle(ahci_portp, slot)
3554                     != DDI_SUCCESS) {
3555                         return (DDI_FAILURE);
3556                 }
3557         }
3558         return (DDI_SUCCESS);
3559 }
3560 
3561 /*
3562  * Check the DMA handles and the access handles of a cmd table slot.
3563  */
3564 static int
3565 ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
3566 {
3567         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3568 
3569         if ((ahci_check_acc_handle(ahci_portp->
3570             ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
3571             (ahci_check_dma_handle(ahci_portp->
3572             ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
3573                 return (DDI_FAILURE);
3574         }
3575         return (DDI_SUCCESS);
3576 }
3577 
3578 /*
3579  * Allocate the ports structure, only called by ahci_attach
3580  */
3581 static int
3582 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
3583 {
3584         int port, cport = 0;
3585 
3586         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3587             "ahci_alloc_ports_state enter", NULL);
3588 
3589         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3590 
3591         /* Allocate structures only for the implemented ports */
3592         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3593                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3594                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3595                             "hba port %d not implemented", port);
3596                         continue;
3597                 }
3598 
3599                 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
3600                 ahci_ctlp->ahcictl_port_to_cport[port] =
3601                     (uint8_t)cport++;
3602 
3603                 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
3604                         goto err_out;
3605                 }
3606         }
3607 
3608         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3609         return (AHCI_SUCCESS);
3610 
3611 err_out:
3612         for (port--; port >= 0; port--) {
3613                 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3614                         ahci_dealloc_port_state(ahci_ctlp, port);
3615                 }
3616         }
3617 
3618         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3619         return (AHCI_FAILURE);
3620 }
3621 
3622 /*
3623  * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
3624  */
3625 static void
3626 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
3627 {
3628         int port;
3629 
3630         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3631         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3632                 /* if this port is implemented by the HBA */
3633                 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
3634                         ahci_dealloc_port_state(ahci_ctlp, port);
3635         }
3636         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3637 }
3638 
3639 /*
3640  * Drain the taskq.
3641  */
3642 static void
3643 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
3644 {
3645         ahci_port_t *ahci_portp;
3646         int port;
3647 
3648         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3649                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3650                         continue;
3651                 }
3652 
3653                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3654 
3655                 mutex_enter(&ahci_portp->ahciport_mutex);
3656                 ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
3657                 mutex_exit(&ahci_portp->ahciport_mutex);
3658         }
3659 }
3660 
3661 /*
3662  * Initialize the controller and all ports. And then try to start the ports
3663  * if there are devices attached.
3664  *
3665  * This routine can be called from three seperate cases: DDI_ATTACH,
3666  * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
3667  * other two cases; device signature probing are attempted only during
3668  * DDI_ATTACH case.
3669  */
3670 static int
3671 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
3672 {
3673         ahci_port_t *ahci_portp;
3674         ahci_addr_t addr;
3675         int port;
3676 
3677         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3678             "ahci_initialize_controller enter", NULL);
3679 
3680         /* Disable the whole controller interrupts */
3681         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3682         ahci_disable_all_intrs(ahci_ctlp);
3683         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3684 
3685         /* Initialize the implemented ports and structures */
3686         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3687                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3688                         continue;
3689                 }
3690 
3691                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3692                 mutex_enter(&ahci_portp->ahciport_mutex);
3693 
3694                 /*
3695                  * Ensure that the controller is not in the running state
3696                  * by checking every implemented port's PxCMD register
3697                  */
3698                 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);
3699 
3700                 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
3701                     != AHCI_SUCCESS) {
3702                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3703                             "ahci_initialize_controller: failed to "
3704                             "initialize port %d", port);
3705                         /*
3706                          * Set the port state to SATA_PSTATE_FAILED if
3707                          * failed to initialize it.
3708                          */
3709                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
3710                 }
3711 
3712                 mutex_exit(&ahci_portp->ahciport_mutex);
3713         }
3714 
3715         /* Enable the whole controller interrupts */
3716         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3717         ahci_enable_all_intrs(ahci_ctlp);
3718         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3719 
3720         return (AHCI_SUCCESS);
3721 }
3722 
3723 /*
3724  * Reverse of ahci_initialize_controller()
3725  *
3726  * We only need to stop the ports and disable the interrupt.
3727  */
3728 static void
3729 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
3730 {
3731         ahci_port_t *ahci_portp;
3732         int port;
3733 
3734         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3735             "ahci_uninitialize_controller enter", NULL);
3736 
3737         /* disable all the interrupts. */
3738         mutex_enter(&ahci_ctlp->ahcictl_mutex);
3739         ahci_disable_all_intrs(ahci_ctlp);
3740         mutex_exit(&ahci_ctlp->ahcictl_mutex);
3741 
3742         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3743                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3744                         continue;
3745                 }
3746 
3747                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
3748 
3749                 /* Stop the port by clearing PxCMD.ST */
3750                 mutex_enter(&ahci_portp->ahciport_mutex);
3751 
3752                 /*
3753                  * Here we must disable the port interrupt because
3754                  * ahci_disable_all_intrs only clear GHC.IE, and IS
3755                  * register will be still set if PxIE is enabled.
3756                  * When ahci shares one IRQ with other drivers, the
3757                  * intr handler may claim the intr mistakenly.
3758                  */
3759                 ahci_disable_port_intrs(ahci_ctlp, port);
3760                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3761                     ahci_portp, port);
3762                 mutex_exit(&ahci_portp->ahciport_mutex);
3763         }
3764 }
3765 
3766 /*
3767  * ahci_alloc_pmult()
3768  * 1. Setting HBA port registers which are necessary for a port multiplier.
3769  *    (Set PxCMD.PMA while PxCMD.ST is '0')
3770  * 2. Allocate ahci_pmult_info structure.
3771  *
3772  * NOTE: Must stop port before the function is called.
3773  */
3774 static void
3775 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3776 {
3777         uint32_t port_cmd_status;
3778         uint8_t port = ahci_portp->ahciport_port_num;
3779 
3780         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3781 
3782         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3783             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3784 
3785         /* The port must have been stopped before. */
3786         ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));
3787 
3788         if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
3789                 /* set PMA bit */
3790                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3791                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3792                     port_cmd_status|AHCI_CMD_STATUS_PMA);
3793 
3794                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3795                     "ahci_alloc_pmult: "
3796                     "PxCMD.PMA bit set at port %d.", port);
3797         }
3798 
3799         /* Allocate port multiplier information structure */
3800         if (ahci_portp->ahciport_pmult_info == NULL) {
3801                 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
3802                     kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
3803         }
3804 
3805         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
3806 }
3807 
3808 /*
3809  * ahci_dealloc_pmult()
3810  * 1. Clearing related registers when a port multiplier is detached.
3811  *    (Clear PxCMD.PMA while PxCMD.ST is '0')
3812  * 2. Deallocate ahci_pmult_info structure.
3813  *
3814  * NOTE: Must stop port before the function is called.
3815  */
3816 static void
3817 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
3818 {
3819         uint32_t port_cmd_status;
3820         uint8_t port = ahci_portp->ahciport_port_num;
3821 
3822         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3823 
3824         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3825             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3826 
3827         if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
3828                 /* Clear PMA bit */
3829                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3830                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3831                     (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));
3832 
3833                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
3834                     "ahci_dealloc_pmult: "
3835                     "PxCMD.PMA bit cleared at port %d.", port);
3836         }
3837 
3838         /* Release port multiplier information structure */
3839         if (ahci_portp->ahciport_pmult_info != NULL) {
3840                 kmem_free(ahci_portp->ahciport_pmult_info,
3841                     sizeof (ahci_pmult_info_t));
3842                 ahci_portp->ahciport_pmult_info = NULL;
3843         }
3844 }
3845 
3846 /*
3847  * Staggered Spin-up.
3848  */
3849 static void
3850 ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
3851 {
3852         uint32_t cap_status;
3853         uint32_t port_cmd_status;
3854 
3855         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
3856 
3857         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3858             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
3859 
3860         /* Check for staggered spin-up support */
3861         if (!(cap_status & AHCI_HBA_CAP_SSS))
3862                 return;
3863 
3864         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3865             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3866 
3867         /* If PxCMD.SUD == 1, no staggered spin-up is needed */
3868         if (port_cmd_status & AHCI_CMD_STATUS_SUD)
3869                 return;
3870 
3871         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);
3872 
3873         /* Set PxCMD.SUD */
3874         port_cmd_status |= AHCI_CMD_STATUS_SUD;
3875         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3876             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3877             port_cmd_status);
3878 }
3879 
3880 /*
3881  * The routine is to initialize a port. First put the port in NotRunning
3882  * state, then enable port interrupt and clear Serror register. And under
3883  * AHCI_ATTACH case, find device signature and then try to start the port.
3884  *
3885  * Called by
3886  *    1. ahci_initialize_controller
3887  *    2. ahci_intr_phyrdy_change (hotplug)
3888  */
3889 static int
3890 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3891     ahci_addr_t *addrp)
3892 {
3893         uint32_t port_sstatus, port_task_file, port_cmd_status;
3894         uint8_t port = addrp->aa_port;
3895         boolean_t resuming = B_TRUE;    /*  processing DDI_RESUME */
3896         int ret;
3897 
3898         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3899 
3900         /* AHCI_ADDR_PORT: We've no idea of the attached device here.  */
3901         ASSERT(AHCI_ADDR_IS_PORT(addrp));
3902 
3903         /*
3904          * At the time being, only probe ports/devices and get the types of
3905          * attached devices during DDI_ATTACH. In fact, the device can be
3906          * changed during power state changes, but at the time being, we
3907          * don't support the situation.
3908          */
3909         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
3910                 resuming = B_FALSE;
3911         } else {
3912                 /* check for DDI_RESUME case */
3913                 mutex_exit(&ahci_portp->ahciport_mutex);
3914                 mutex_enter(&ahci_ctlp->ahcictl_mutex);
3915                 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
3916                         resuming = B_FALSE;
3917                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
3918                 mutex_enter(&ahci_portp->ahciport_mutex);
3919         }
3920 
3921         if (resuming) {
3922                 /*
3923                  * During the resume, we need to set the PxCLB, PxCLBU, PxFB
3924                  * and PxFBU registers in case these registers were cleared
3925                  * during the suspend.
3926                  */
3927                 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
3928                     "ahci_initialize_port: port %d "
3929                     "set PxCLB, PxCLBU, PxFB and PxFBU "
3930                     "during resume", port);
3931 
3932                 if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
3933                     AHCI_SUCCESS)
3934                         return (AHCI_FAILURE);
3935         }
3936 
3937         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3938             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3939 
3940         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3941             "ahci_initialize_port: port %d ", port);
3942 
3943         /*
3944          * Check whether the port is in NotRunning state, if not,
3945          * put the port in NotRunning state
3946          */
3947         if (port_cmd_status &
3948             (AHCI_CMD_STATUS_ST |
3949             AHCI_CMD_STATUS_CR |
3950             AHCI_CMD_STATUS_FRE |
3951             AHCI_CMD_STATUS_FR)) {
3952                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3953                     ahci_portp, port);
3954         }
3955 
3956         /* Make sure the drive is spun-up */
3957         ahci_staggered_spin_up(ahci_ctlp, port);
3958 
3959         /* Disable interrupt */
3960         ahci_disable_port_intrs(ahci_ctlp, port);
3961 
3962         /* Device is unknown at first */
3963         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
3964 
3965         /* Disable the interface power management */
3966         ahci_disable_interface_pm(ahci_ctlp, port);
3967 
3968         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3969             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
3970         port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3971             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
3972 
3973         /* Check physcial link status */
3974         if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
3975             SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||
3976 
3977             /* Check interface status */
3978             port_task_file & AHCI_TFD_STS_BSY ||
3979             port_task_file & AHCI_TFD_STS_DRQ ||
3980 
3981             /* Check whether port reset must be executed */
3982             ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||
3983 
3984             /* Always reset port on RESUME */
3985             resuming != B_FALSE) {
3986 
3987                 /* Something went wrong, we need do some reset things */
3988                 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
3989 
3990                 /* Does port reset succeed on HBA port? */
3991                 if (ret != AHCI_SUCCESS) {
3992                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
3993                             "ahci_initialize_port:"
3994                             "port reset failed at port %d", port);
3995                         return (AHCI_FAILURE);
3996                 }
3997 
3998                 /* Is port failed? */
3999                 if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4000                     SATA_PSTATE_FAILED) {
4001                         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4002                             "ahci_initialize_port: port %d state 0x%x",
4003                             port, ahci_portp->ahciport_port_state);
4004                         return (AHCI_FAILURE);
4005                 }
4006         }
4007 
4008         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4009         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);
4010 
4011         /*
4012          * Try to get the device signature if the port is not empty.
4013          */
4014         if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
4015             SATA_DTYPE_NONE)
4016                 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4017 
4018         /* Return directly if no device connected */
4019         if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
4020                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4021                     "No device connected to port %d", port);
4022                 goto out;
4023         }
4024 
4025         /* If this is a port multiplier, we need do some initialization */
4026         if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
4027                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4028                     "Port multiplier found at port %d", port);
4029                 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
4030         }
4031 
4032         /* Try to start the port */
4033         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4034             != AHCI_SUCCESS) {
4035                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4036                     "failed to start port %d", port);
4037                 return (AHCI_FAILURE);
4038         }
4039 out:
4040         /* Enable port interrupts */
4041         ahci_enable_port_intrs(ahci_ctlp, port);
4042 
4043         return (AHCI_SUCCESS);
4044 }
4045 
4046 /*
4047  *  Handle hardware defect, and check the capabilities. For example,
4048  *  power management capabilty and MSI capability.
4049  */
4050 static int
4051 ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
4052 {
4053         ushort_t caps_ptr, cap_count, cap;
4054 #if AHCI_DEBUG
4055         ushort_t pmcap, pmcsr;
4056         ushort_t msimc;
4057 #endif
4058         uint8_t revision;
4059 
4060         ahci_ctlp->ahcictl_venid =
4061             pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4062             PCI_CONF_VENID);
4063 
4064         ahci_ctlp->ahcictl_devid =
4065             pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4066             PCI_CONF_DEVID);
4067 
4068         /*
4069          * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
4070          * controllers with 0x00 revision id work on 4-byte aligned buffer,
4071          * which is a bug and was fixed after 0x00 revision id controllers.
4072          *
4073          * Moreover, VT8251 cannot use multiple command slots in the command
4074          * list for non-queued commands because the previous register content
4075          * of PxCI can be re-written in the register write, so a flag will be
4076          * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
4077          *
4078          * For VT8251, software reset also has the same defect as the below
4079          * AMD/ATI chipset. That is, software reset will get failed if 0xf
4080          * is filled in pmport field. Therefore, another software reset need
4081          * to be done with 0 filled in pmport field.
4082          */
4083         if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
4084                 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4085                     PCI_CONF_REVID);
4086                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4087                     "revision id = 0x%x", revision);
4088                 if (revision == 0x00) {
4089                         ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
4090                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4091                             "change ddi_attr_align to 0x4", NULL);
4092                 }
4093 
4094                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
4095                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4096                     "VT8251 cannot use multiple command lists for "
4097                     "non-queued commands", NULL);
4098 
4099                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4100         }
4101 
4102         /*
4103          * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
4104          * DMA addressing for communication memory descriptors though S64A bit
4105          * of CAP register declares it supports. Even though 64-bit DMA for
4106          * data buffer works on ASUS M2A-VM with newer BIOS, three other
4107          * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
4108          * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
4109          *
4110          * Due to certain hardware issue, the chipset must do port reset during
4111          * initialization, otherwise, when retrieving device signature,
4112          * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
4113          * need to set.
4114          *
4115          * For this chipset software reset will get failure if the pmport of
4116          * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
4117          * multiplier is connected to the port. In order to fix the issue,
4118          * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
4119          * reset got failure, the driver will try to do another software reset
4120          * with pmport 0.
4121          */
4122         if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4123             ahci_ctlp->ahcictl_devid == 0x4380) {
4124                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
4125                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4126                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4127                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4128 
4129                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4130                     "ATI SB600 cannot do 64-bit DMA for both data buffer and "
4131                     "communication memory descriptors though CAP indicates "
4132                     "support, so force it to use 32-bit DMA", NULL);
4133                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4134                     "ATI SB600 need to do a port reset during initialization",
4135                     NULL);
4136                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4137                     "ATI SB600 will get software reset failure if pmport "
4138                     "is set 0xf and no port multiplier is attached", NULL);
4139         }
4140 
4141         /*
4142          * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
4143          * vendor ID and device ID (0x1002,0x4391).
4144          *
4145          * SB700/750 AHCI chipset on some boards doesn't support 64-bit
4146          * DMA addressing for communication memory descriptors though S64A bit
4147          * of CAP register declares the support. However, it does support
4148          * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
4149          * set for this controller.
4150          *
4151          * SB710 has the same initialization issue as SB600, so it also need
4152          * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
4153          *
4154          * SB700 also has the same issue about software reset, and thus
4155          * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
4156          */
4157         if (ahci_ctlp->ahcictl_venid == 0x1002 &&
4158             ahci_ctlp->ahcictl_devid == 0x4391) {
4159                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
4160                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
4161                 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
4162 
4163                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4164                     "ATI SB700/750 cannot do 64-bit DMA for communication "
4165                     "memory descriptors though CAP indicates support, "
4166                     "so force it to use 32-bit DMA", NULL);
4167                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4168                     "ATI SB710 need to do a port reset during initialization",
4169                     NULL);
4170                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4171                     "ATI SB700 will get software reset failure if pmport "
4172                     "is set 0xf and no port multiplier is attached", NULL);
4173         }
4174 
4175         /*
4176          * Check if capabilities list is supported and if so,
4177          * get initial capabilities pointer and clear bits 0,1.
4178          */
4179         if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
4180             PCI_CONF_STAT) & PCI_STAT_CAP) {
4181                 caps_ptr = P2ALIGN(pci_config_get8(
4182                     ahci_ctlp->ahcictl_pci_conf_handle,
4183                     PCI_CONF_CAP_PTR), 4);
4184         } else {
4185                 caps_ptr = PCI_CAP_NEXT_PTR_NULL;
4186         }
4187 
4188         /*
4189          * Walk capabilities if supported.
4190          */
4191         for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {
4192 
4193                 /*
4194                  * Check that we haven't exceeded the maximum number of
4195                  * capabilities and that the pointer is in a valid range.
4196                  */
4197                 if (++cap_count > PCI_CAP_MAX_PTR) {
4198                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4199                             "too many device capabilities", NULL);
4200                         return (AHCI_FAILURE);
4201                 }
4202                 if (caps_ptr < PCI_CAP_PTR_OFF) {
4203                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4204                             "capabilities pointer 0x%x out of range",
4205                             caps_ptr);
4206                         return (AHCI_FAILURE);
4207                 }
4208 
4209                 /*
4210                  * Get next capability and check that it is valid.
4211                  * For now, we only support power management.
4212                  */
4213                 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
4214                     caps_ptr);
4215                 switch (cap) {
4216                 case PCI_CAP_ID_PM:
4217 
4218                         /* power management supported */
4219                         ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;
4220 
4221                         /* Save PMCSR offset */
4222                         ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;
4223 
4224 #if AHCI_DEBUG
4225                         pmcap = pci_config_get16(
4226                             ahci_ctlp->ahcictl_pci_conf_handle,
4227                             caps_ptr + PCI_PMCAP);
4228                         pmcsr = pci_config_get16(
4229                             ahci_ctlp->ahcictl_pci_conf_handle,
4230                             ahci_ctlp->ahcictl_pmcsr_offset);
4231                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4232                             "Power Management capability found PCI_PMCAP "
4233                             "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
4234                         if ((pmcap & 0x3) == 0x3)
4235                                 AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4236                                     "PCI Power Management Interface "
4237                                     "spec 1.2 compliant", NULL);
4238 #endif
4239                         break;
4240 
4241                 case PCI_CAP_ID_MSI:
4242 #if AHCI_DEBUG
4243                         msimc = pci_config_get16(
4244                             ahci_ctlp->ahcictl_pci_conf_handle,
4245                             caps_ptr + PCI_MSI_CTRL);
4246                         AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4247                             "Message Signaled Interrupt capability found "
4248                             "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
4249 #endif
4250                         AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
4251                             "MSI capability found", NULL);
4252                         break;
4253 
4254                 case PCI_CAP_ID_PCIX:
4255                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4256                             "PCI-X capability found", NULL);
4257                         break;
4258 
4259                 case PCI_CAP_ID_PCI_E:
4260                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4261                             "PCI Express capability found", NULL);
4262                         break;
4263 
4264                 case PCI_CAP_ID_MSI_X:
4265                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4266                             "MSI-X capability found", NULL);
4267                         break;
4268 
4269                 case PCI_CAP_ID_SATA:
4270                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4271                             "SATA capability found", NULL);
4272                         break;
4273 
4274                 case PCI_CAP_ID_VS:
4275                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4276                             "Vendor Specific capability found", NULL);
4277                         break;
4278 
4279                 default:
4280                         AHCIDBG(AHCIDBG_PM, ahci_ctlp,
4281                             "unrecognized capability 0x%x", cap);
4282                         break;
4283                 }
4284 
4285                 /*
4286                  * Get next capabilities pointer and clear bits 0,1.
4287                  */
4288                 caps_ptr = P2ALIGN(pci_config_get8(
4289                     ahci_ctlp->ahcictl_pci_conf_handle,
4290                     (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
4291         }
4292 
4293         return (AHCI_SUCCESS);
4294 }
4295 
4296 /*
4297  * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
4298  * PORTMULT command. SYNC & POLLING mode is used.
4299  */
4300 static int
4301 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4302     uint8_t regn, uint32_t *pregv, uint8_t type)
4303 {
4304         ahci_port_t *ahci_portp;
4305         ahci_addr_t pmult_addr;
4306         sata_pkt_t *spkt;
4307         sata_cmd_t *scmd;
4308         sata_device_t sata_device;
4309         uint8_t port = addrp->aa_port;
4310         uint8_t pmport = addrp->aa_pmport;
4311         uint8_t cport;
4312         uint32_t intr_mask;
4313         int rval;
4314         char portstr[10];
4315 
4316         SET_PORTSTR(portstr, addrp);
4317         cport = ahci_ctlp->ahcictl_port_to_cport[port];
4318         ahci_portp = ahci_ctlp->ahcictl_ports[port];
4319 
4320         ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
4321         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4322 
4323         /* Check the existence of the port multiplier */
4324         if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
4325                 return (AHCI_FAILURE);
4326 
4327         /* Request a READ/WRITE PORTMULT sata packet. */
4328         bzero(&sata_device, sizeof (sata_device_t));
4329         sata_device.satadev_addr.cport = cport;
4330         sata_device.satadev_addr.pmport = pmport;
4331         sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
4332         sata_device.satadev_rev = SATA_DEVICE_REV;
4333 
4334         /*
4335          * Make sure no command is outstanding here. All R/W PMULT requests
4336          * come from
4337          *
4338          * 1. ahci_attach()
4339          *    The port should be empty.
4340          *
4341          * 2. ahci_tran_probe_port()
4342          *    Any request from SATA framework (via ahci_tran_start) should be
4343          *    rejected if R/W PMULT command is outstanding.
4344          *
4345          *    If we are doing mopping, do not check those flags because no
4346          *    command will be actually outstanding.
4347          *
4348          *    If the port has been occupied by any other commands, the probe
4349          *    function will return a SATA_RETRY. SATA framework will retry
4350          *    later.
4351          */
4352         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
4353                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4354                     "R/W PMULT failed: R/W PMULT in progress at port %d.",
4355                     port, ahci_portp->ahciport_flags);
4356                 return (AHCI_FAILURE);
4357         }
4358 
4359         if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
4360             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
4361             NCQ_CMD_IN_PROGRESS(ahci_portp) ||
4362             NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
4363                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4364                     "R/W PMULT failed: port %d is occupied (flags 0x%x).",
4365                     port, ahci_portp->ahciport_flags);
4366                 return (AHCI_FAILURE);
4367         }
4368 
4369         /*
4370          * The port multiplier is gone. This may happen when
4371          * 1. Cutting off the power of an enclosure. The device lose the power
4372          *    before port multiplier.
4373          * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
4374          *
4375          * The issued command should be aborted and the following command
4376          * should not be continued.
4377          */
4378         if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
4379                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4380                     "READ/WRITE PMULT failed: "
4381                     "port-mult is removed from port %d", port);
4382                 return (AHCI_FAILURE);
4383         }
4384 
4385         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;
4386 
4387         spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
4388             &sata_device, regn, *pregv, type);
4389 
4390         /*
4391          * READ/WRITE PORTMULT command is intended to sent to the control port
4392          * of the port multiplier.
4393          */
4394         AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);
4395 
4396         ahci_portp->ahciport_rdwr_pmult_pkt = spkt;
4397 
4398         /* No interrupt here. Store the interrupt enable mask. */
4399         intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4400             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
4401         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4402             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
4403 
4404         rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);
4405 
4406         if (rval == AHCI_SUCCESS &&
4407             spkt->satapkt_reason == SATA_PKT_COMPLETED) {
4408                 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
4409                         scmd = &spkt->satapkt_cmd;
4410                         *pregv = scmd->satacmd_lba_high_lsb << 24 |
4411                             scmd->satacmd_lba_mid_lsb << 16 |
4412                             scmd->satacmd_lba_low_lsb << 8 |
4413                             scmd->satacmd_sec_count_lsb;
4414                 }
4415         } else {
4416                 /* Failed or not completed. */
4417                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4418                     "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
4419                     type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
4420                     AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
4421                 rval = AHCI_FAILURE;
4422         }
4423 out:
4424         /* Restore the interrupt mask */
4425         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4426             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);
4427 
4428         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
4429         ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
4430         sata_free_rdwr_pmult_pkt(spkt);
4431         return (rval);
4432 }
4433 
4434 static int
4435 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4436     uint8_t regn, uint32_t *pregv)
4437 {
4438         return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
4439             SATA_RDWR_PMULT_PKT_TYPE_READ);
4440 }
4441 
4442 static int
4443 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4444     uint8_t regn, uint32_t regv)
4445 {
4446         return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, &regv,
4447             SATA_RDWR_PMULT_PKT_TYPE_WRITE);
4448 }
4449 
4450 #define READ_PMULT(addrp, r, pv, out)                                   \
4451         if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS)   \
4452                 goto out;
4453 
4454 #define WRITE_PMULT(addrp, r, v, out)                                   \
4455         if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS)   \
4456                 goto out;
4457 
4458 /*
4459  * Update sata registers on port multiplier, including GSCR/PSCR registers.
4460  * ahci_update_pmult_gscr()
4461  * ahci_update_pmult_pscr()
4462  */
4463 static int
4464 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4465     sata_pmult_gscr_t *sg)
4466 {
4467         ASSERT(MUTEX_HELD(
4468             &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4469 
4470         READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
4471         READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
4472         READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
4473         READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);
4474 
4475         return (AHCI_SUCCESS);
4476 
4477 err:    /* R/W PMULT error */
4478         return (AHCI_FAILURE);
4479 }
4480 
4481 static int
4482 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
4483     sata_device_t *sd)
4484 {
4485         ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4486         ASSERT(MUTEX_HELD(
4487             &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));
4488 
4489         READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
4490         READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
4491         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
4492         READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);
4493 
4494         return (AHCI_SUCCESS);
4495 
4496 err:    /* R/W PMULT error */
4497         return (AHCI_FAILURE);
4498 }
4499 
4500 /*
4501  * ahci_initialize_pmult()
4502  *
4503  * Initialize a port multiplier, including
4504  * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
4505  * 2. Redefine MASK register. (SATA Chap 16.?)
4506  */
4507 static int
4508 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4509     ahci_addr_t *addrp, sata_device_t *sd)
4510 {
4511         sata_pmult_gscr_t sg;
4512         uint32_t gscr64;
4513         uint8_t port = addrp->aa_port;
4514 
4515         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4516 
4517         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4518             "[Initialize] Port-multiplier at port %d.", port);
4519 
4520         /*
4521          * Enable features of port multiplier. Currently only
4522          * Asynchronous Notification is enabled.
4523          */
4524         /* Check gscr64 for supported features. */
4525         READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);
4526 
4527         if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
4528                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4529                     "port %d: Port Multiplier supports "
4530                     "Asynchronous Notification.", port);
4531 
4532                 /* Write to gscr96 to enabled features */
4533                 WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
4534                     SATA_PMULT_CAP_SNOTIF, err);
4535 
4536                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4537                     (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4538                     AHCI_SNOTIF_CLEAR_ALL);
4539                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4540                     "port %d: PMult PxSNTF cleared.", port);
4541 
4542         }
4543 
4544         /*
4545          * Now we need to update gscr33 register to enable hot-plug interrupt
4546          * for sub devices behind port multiplier.
4547          */
4548         WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
4549         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4550             "port %d: gscr33 mask set to %x.", port, (0x1ffff));
4551 
4552         /*
4553          * Fetch the number of device ports of the port multiplier
4554          */
4555         if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
4556                 return (AHCI_FAILURE);
4557 
4558         /* Register the port multiplier to SATA Framework. */
4559         mutex_exit(&ahci_portp->ahciport_mutex);
4560         sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
4561         mutex_enter(&ahci_portp->ahciport_mutex);
4562 
4563         ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
4564             sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;
4565 
4566         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
4567             "port %d: pmult sub-port number updated to %x.", port,
4568             ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);
4569 
4570         /* Till now port-mult is successfully initialized */
4571         ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
4572         return (AHCI_SUCCESS);
4573 
4574 err:    /* R/W PMULT error */
4575         return (AHCI_FAILURE);
4576 }
4577 
4578 /*
4579  * Initialize a port multiplier port. According to spec, firstly we need
4580  * issue a COMRESET, then a software reset to get its signature.
4581  *
4582  * NOTE: This function should only be called in ahci_probe_pmport()
4583  */
4584 static int
4585 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4586     ahci_addr_t *addrp)
4587 {
4588         uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
4589         uint8_t port = addrp->aa_port;
4590         uint8_t pmport = addrp->aa_pmport;
4591         int ret = AHCI_FAILURE;
4592 
4593         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4594         ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
4595 
4596         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
4597             "ahci_initialize_pmport: port %d:%d", port, pmport);
4598 
4599         /* Check HBA port state */
4600         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
4601                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4602                     "ahci_initialize_pmport:"
4603                     "port %d:%d Port Multiplier is failed.",
4604                     port, pmport);
4605                 return (AHCI_FAILURE);
4606         }
4607 
4608         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
4609                 return (AHCI_FAILURE);
4610         }
4611         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
4612 
4613         /* Checking for outstanding commands */
4614         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4615                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4616                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
4617                 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4618         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
4619                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4620                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
4621                 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4622         }
4623 
4624         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
4625         ahci_portp->ahciport_mop_in_progress++;
4626 
4627         /* Clear status */
4628         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);
4629 
4630         /* Firstly assume an unknown device */
4631         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
4632 
4633         ahci_disable_port_intrs(ahci_ctlp, port);
4634 
4635         /* port reset is necessary for port multiplier port */
4636         if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
4637                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
4638                     "ahci_initialize_pmport:"
4639                     "port reset failed at port %d:%d",
4640                     port, pmport);
4641                 goto out;
4642         }
4643 
4644         /* Is port failed? */
4645         if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
4646             SATA_PSTATE_FAILED) {
4647                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4648                     "ahci_initialize_pmport: port %d:%d failed. "
4649                     "state = 0x%x", port, pmport,
4650                     ahci_portp->ahciport_port_state);
4651                 goto out;
4652         }
4653 
4654         /* Is there any device attached? */
4655         if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
4656             == SATA_DTYPE_NONE) {
4657                 /* Do not waste time on an empty port */
4658                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
4659                     "ahci_initialize_pmport: No device is found "
4660                     "at port %d:%d", port, pmport);
4661                 ret = AHCI_SUCCESS;
4662                 goto out;
4663         }
4664 
4665         AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
4666         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
4667             "port %d:%d is ready now.", port, pmport);
4668 
4669         /*
4670          * Till now we can assure a device attached to that HBA port and work
4671          * correctly. Now try to get the device signature. This is an optional
4672          * step. If failed, unknown device is assumed, then SATA module will
4673          * continue to use IDENTIFY DEVICE to get the information of the
4674          * device.
4675          */
4676         ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);
4677 
4678         ret = AHCI_SUCCESS;
4679 
4680 out:
4681         /* Next try to mop the pending commands */
4682         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
4683                 finished_tags = ahci_portp->ahciport_pending_tags &
4684                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
4685         else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
4686                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
4687                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
4688         reset_tags &= ~finished_tags;
4689 
4690         ahci_mop_commands(ahci_ctlp,
4691             ahci_portp,
4692             slot_status,
4693             0, /* failed tags */
4694             0, /* timeout tags */
4695             0, /* aborted tags */
4696             reset_tags); /* reset tags */
4697 
4698         /* Clear PxSNTF register if supported. */
4699         if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
4700                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4701                     (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
4702                     AHCI_SNOTIF_CLEAR_ALL);
4703         }
4704 
4705         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
4706         ahci_enable_port_intrs(ahci_ctlp, port);
4707         return (ret);
4708 }
4709 
4710 /*
4711  * ahci_probe_pmult()
4712  *
4713  * This function will be called to probe a port multiplier, which will
4714  * handle hotplug events on port multiplier ports.
4715  *
4716  * NOTE: Only called from ahci_tran_probe_port()
4717  */
4718 static int
4719 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4720     ahci_addr_t *addrp)
4721 {
4722         sata_device_t sdevice;
4723         ahci_addr_t pmport_addr;
4724         uint32_t gscr32, port_hotplug_tags;
4725         uint32_t pmport_sstatus;
4726         int dev_exists_now = 0, dev_existed_previously = 0;
4727         uint8_t port = addrp->aa_port;
4728         int npmport;
4729 
4730         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4731 
4732         /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
4733         READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
4734         port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);
4735 
4736         do {
4737                 npmport = ddi_ffs(port_hotplug_tags) - 1;
4738                 if (npmport == -1)
4739                         /* no pending hot plug events. */
4740                         return (AHCI_SUCCESS);
4741 
4742                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4743                     "hot-plug event at port %d:%d", port, npmport);
4744 
4745                 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);
4746 
4747                 /* Check previous device at that port */
4748                 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
4749                     != SATA_DTYPE_NONE)
4750                         dev_existed_previously = 1;
4751 
4752                 /* PxSStatus tells the presence of device. */
4753                 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
4754                     &pmport_sstatus, err);
4755 
4756                 if (SSTATUS_GET_DET(pmport_sstatus) ==
4757                     SSTATUS_DET_DEVPRE_PHYCOM)
4758                         dev_exists_now = 1;
4759 
4760                 /*
4761                  * Clear PxSERR is critical. The transition from 0 to 1 will
4762                  * emit a FIS which generates an asynchronous notification
4763                  * event at controller. If we fail to clear the PxSERR, the
4764                  * Async Notif events will no longer be activated on this
4765                  * pmport.
4766                  */
4767                 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
4768                     AHCI_SERROR_CLEAR_ALL, err);
4769 
4770                 bzero((void *)&sdevice, sizeof (sata_device_t));
4771                 sdevice.satadev_addr.cport = ahci_ctlp->
4772                     ahcictl_port_to_cport[port];
4773                 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
4774                 sdevice.satadev_addr.pmport = (uint8_t)npmport;
4775                 sdevice.satadev_state = SATA_PSTATE_PWRON;
4776 
4777                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4778                     "[Existence] %d -> %d", dev_existed_previously,
4779                     dev_exists_now);
4780 
4781                 if (dev_exists_now) {
4782                         if (dev_existed_previously) {
4783                                 /* Link (may) not change: Exist -> Exist * */
4784                                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
4785                                     "ahci_probe_pmult: port %d:%d "
4786                                     "device link lost/established",
4787                                     port, npmport);
4788 
4789                                 mutex_exit(&ahci_portp->ahciport_mutex);
4790                                 sata_hba_event_notify(
4791                                     ahci_ctlp->ahcictl_sata_hba_tran->
4792                                     sata_tran_hba_dip,
4793                                     &sdevice,
4794                                     SATA_EVNT_LINK_LOST|
4795                                     SATA_EVNT_LINK_ESTABLISHED);
4796                                 mutex_enter(&ahci_portp->ahciport_mutex);
4797                         } else {
4798                                 /* Link change: None -> Exist */
4799                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4800                                     "ahci_probe_pmult: port %d:%d "
4801                                     "device link established", port, npmport);
4802 
4803                                 /* Clear port state */
4804                                 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4805                                     SATA_STATE_UNKNOWN);
4806                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4807                                     "ahci_probe_pmult: port %d "
4808                                     "ahciport_port_state [Cleared].", port);
4809 
4810                                 mutex_exit(&ahci_portp->ahciport_mutex);
4811                                 sata_hba_event_notify(
4812                                     ahci_ctlp->ahcictl_sata_hba_tran->
4813                                     sata_tran_hba_dip,
4814                                     &sdevice,
4815                                     SATA_EVNT_LINK_ESTABLISHED);
4816                                 mutex_enter(&ahci_portp->ahciport_mutex);
4817                         }
4818                 } else { /* No device exists now */
4819                         if (dev_existed_previously) {
4820 
4821                                 /* Link change: Exist -> None */
4822                                 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
4823                                     "ahci_probe_pmult: port %d:%d "
4824                                     "device link lost", port, npmport);
4825 
4826                                 /* An existing device is lost. */
4827                                 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
4828                                     SATA_STATE_UNKNOWN);
4829                                 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
4830                                     SATA_DTYPE_NONE);
4831 
4832                                 mutex_exit(&ahci_portp->ahciport_mutex);
4833                                 sata_hba_event_notify(
4834                                     ahci_ctlp->ahcictl_sata_hba_tran->
4835                                     sata_tran_hba_dip,
4836                                     &sdevice,
4837                                     SATA_EVNT_LINK_LOST);
4838                                 mutex_enter(&ahci_portp->ahciport_mutex);
4839                         }
4840                 }
4841 
4842                 CLEAR_BIT(port_hotplug_tags, npmport);
4843         } while (port_hotplug_tags != 0);
4844 
4845         return (AHCI_SUCCESS);
4846 
4847 err:    /* R/W PMULT error */
4848         return (AHCI_FAILURE);
4849 }
4850 
4851 /*
4852  * Probe and initialize a port multiplier port.
4853  * A port multiplier port could only be initilaizer here.
4854  */
4855 static int
4856 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4857     ahci_addr_t *addrp, sata_device_t *sd)
4858 {
4859         uint32_t port_state;
4860         uint8_t port = addrp->aa_port;
4861         ahci_addr_t addr_pmult;
4862 
4863         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4864 
4865         /*
4866          * Check the parent - port multiplier first.
4867          */
4868 
4869         /*
4870          * Parent port multiplier might have been removed. This event will be
4871          * ignored and failure.
4872          */
4873         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
4874             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
4875                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4876                     "ahci_tran_probe_port: "
4877                     "parent device removed, ignore event.", NULL);
4878 
4879                 return (AHCI_FAILURE);
4880         }
4881 
4882         /* The port is ready? */
4883         port_state = ahci_portp->ahciport_port_state;
4884         if (!(port_state & SATA_STATE_READY)) {
4885                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4886                     "ahci_tran_probe_port: "
4887                     "parent port-mult is NOT ready.", NULL);
4888 
4889                 if (ahci_restart_port_wait_till_ready(ahci_ctlp,
4890                     ahci_portp, port, AHCI_PORT_RESET, NULL) !=
4891                     AHCI_SUCCESS) {
4892                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
4893                             "ahci_tran_probe_port: "
4894                             "restart port-mult failed.", NULL);
4895                         return (AHCI_FAILURE);
4896                 }
4897         }
4898 
4899         /*
4900          * If port-mult is restarted due to some reason, we need
4901          * re-initialized the PMult.
4902          */
4903         if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
4904                 /* Initialize registers on a port multiplier */
4905                 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
4906                 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
4907                     &addr_pmult, sd) != AHCI_SUCCESS)
4908                         return (AHCI_FAILURE);
4909         }
4910 
4911         /*
4912          * Then we check the port-mult port
4913          */
4914         /* Is this pmport initialized? */
4915         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
4916         if (!(port_state & SATA_STATE_READY)) {
4917 
4918                 /* ahci_initialize_pmport() will set READY state */
4919                 if (ahci_initialize_pmport(ahci_ctlp,
4920                     ahci_portp, addrp) != AHCI_SUCCESS)
4921                         return (AHCI_FAILURE);
4922         }
4923 
4924         return (AHCI_SUCCESS);
4925 }
4926 
4927 /*
4928  * AHCI device reset ...; a single device on one of the ports is reset,
4929  * but the HBA and physical communication remain intact. This is the
4930  * least intrusive.
4931  *
4932  * When issuing a software reset sequence, there should not be other
4933  * commands in the command list, so we will first clear and then re-set
4934  * PxCMD.ST to clear PxCI. And before issuing the software reset,
4935  * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
4936  * cleared unless command list override (PxCMD.CLO) is supported.
4937  */
4938 static int
4939 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4940     ahci_addr_t *addrp)
4941 {
4942         ahci_fis_h2d_register_t *h2d_register_fisp;
4943         ahci_cmd_table_t *cmd_table;
4944         ahci_cmd_header_t *cmd_header;
4945         uint32_t port_cmd_status, port_cmd_issue, port_task_file;
4946         int slot, loop_count;
4947         uint8_t port = addrp->aa_port;
4948         uint8_t pmport = addrp->aa_pmport;
4949         int rval = AHCI_FAILURE;
4950 
4951         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
4952 
4953         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
4954             "port %d:%d device software resetting (FIS)", port, pmport);
4955 
4956         /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
4957         if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
4958             port) != AHCI_SUCCESS) {
4959                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4960                     "ahci_software_reset: cannot stop HBA port %d.", port);
4961                 goto out;
4962         }
4963 
4964         /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
4965         port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4966             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4967 
4968         if (port_task_file & AHCI_TFD_STS_BSY ||
4969             port_task_file & AHCI_TFD_STS_DRQ) {
4970                 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
4971                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4972                             "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
4973                             "cannot issue a software reset.", port_task_file);
4974                         goto out;
4975                 }
4976 
4977                 /*
4978                  * If HBA Support CLO, as Command List Override (CAP.SCLO is
4979                  * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
4980                  * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
4981                  */
4982                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
4983                     "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)
4984 
4985                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4986                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4987                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4988                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
4989                     port_cmd_status|AHCI_CMD_STATUS_CLO);
4990 
4991                 /* Waiting till PxCMD.SCLO bit is cleared */
4992                 loop_count = 0;
4993                 do {
4994                         /* Wait for 10 millisec */
4995                         drv_usecwait(AHCI_10MS_USECS);
4996 
4997                         /* We are effectively timing out after 1 sec. */
4998                         if (loop_count++ > 100) {
4999                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5000                                     "SCLO time out. port %d is busy.", port);
5001                                 goto out;
5002                         }
5003 
5004                         port_cmd_status =
5005                             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5006                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5007                 } while (port_cmd_status & AHCI_CMD_STATUS_CLO);
5008 
5009                 /* Re-check */
5010                 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5011                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5012                 if (port_task_file & AHCI_TFD_STS_BSY ||
5013                     port_task_file & AHCI_TFD_STS_DRQ) {
5014                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5015                             "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
5016                             port_task_file);
5017                         goto out;
5018                 }
5019         }
5020 
5021         /* Then start port */
5022         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
5023             != AHCI_SUCCESS) {
5024                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5025                     "ahci_software_reset: cannot start AHCI port %d.", port);
5026                 goto out;
5027         }
5028 
5029         /*
5030          * When ahci_port.ahciport_mop_in_progress is set, A non-zero
5031          * ahci_port.ahciport_pending_ncq_tags may fail
5032          * ahci_claim_free_slot(). Actually according to spec, by clearing
5033          * PxCMD.ST there is no command outstanding while executing software
5034          * reseting. Hence we directly use slot 0 instead of
5035          * ahci_claim_free_slot().
5036          */
5037         slot = 0;
5038 
5039         /* Now send the first H2D Register FIS with SRST set to 1 */
5040         cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5041         bzero((void *)cmd_table, ahci_cmd_table_size);
5042 
5043         h2d_register_fisp =
5044             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5045 
5046         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5047         SET_FIS_PMP(h2d_register_fisp, pmport);
5048         SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
5049 
5050         /* Set Command Header in Command List */
5051         cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5052         BZERO_DESCR_INFO(cmd_header);
5053         BZERO_PRD_BYTE_COUNT(cmd_header);
5054         SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5055         SET_PORT_MULTI_PORT(cmd_header, pmport);
5056 
5057         SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
5058         SET_RESET(cmd_header, 1);
5059         SET_WRITE(cmd_header, 1);
5060 
5061         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5062             0,
5063             ahci_cmd_table_size,
5064             DDI_DMA_SYNC_FORDEV);
5065 
5066         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5067             slot * sizeof (ahci_cmd_header_t),
5068             sizeof (ahci_cmd_header_t),
5069             DDI_DMA_SYNC_FORDEV);
5070 
5071         /* Indicate to the HBA that a command is active. */
5072         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5073             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5074             (0x1 << slot));
5075 
5076         loop_count = 0;
5077 
5078         /* Loop till the first command is finished */
5079         do {
5080                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5081                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5082 
5083                 /* We are effectively timing out after 1 sec. */
5084                 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5085                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5086                             "the first SRST FIS is timed out, "
5087                             "loop_count = %d", loop_count);
5088                         goto out;
5089                 }
5090                 /* Wait for 10 millisec */
5091                 drv_usecwait(AHCI_10MS_USECS);
5092         } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5093 
5094         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5095             "ahci_software_reset: 1st loop count: %d, "
5096             "port_cmd_issue = 0x%x, slot = 0x%x",
5097             loop_count, port_cmd_issue, slot);
5098 
5099         /* According to ATA spec, we need wait at least 5 microsecs here. */
5100         drv_usecwait(AHCI_1MS_USECS);
5101 
5102         /* Now send the second H2D Register FIS with SRST cleard to zero */
5103         cmd_table = ahci_portp->ahciport_cmd_tables[slot];
5104         bzero((void *)cmd_table, ahci_cmd_table_size);
5105 
5106         h2d_register_fisp =
5107             &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
5108 
5109         SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
5110         SET_FIS_PMP(h2d_register_fisp, pmport);
5111 
5112         /* Set Command Header in Command List */
5113         cmd_header = &ahci_portp->ahciport_cmd_list[slot];
5114         BZERO_DESCR_INFO(cmd_header);
5115         BZERO_PRD_BYTE_COUNT(cmd_header);
5116         SET_COMMAND_FIS_LENGTH(cmd_header, 5);
5117         SET_PORT_MULTI_PORT(cmd_header, pmport);
5118 
5119         SET_WRITE(cmd_header, 1);
5120 
5121         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
5122             0,
5123             ahci_cmd_table_size,
5124             DDI_DMA_SYNC_FORDEV);
5125 
5126         (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
5127             slot * sizeof (ahci_cmd_header_t),
5128             sizeof (ahci_cmd_header_t),
5129             DDI_DMA_SYNC_FORDEV);
5130 
5131         /* Indicate to the HBA that a command is active. */
5132         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5133             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
5134             (0x1 << slot));
5135 
5136         loop_count = 0;
5137 
5138         /* Loop till the second command is finished */
5139         do {
5140                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5141                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5142 
5143                 /* We are effectively timing out after 1 sec. */
5144                 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
5145                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5146                             "the second SRST FIS is timed out, "
5147                             "loop_count = %d", loop_count);
5148                         goto out;
5149                 }
5150 
5151                 /* Wait for 10 millisec */
5152                 drv_usecwait(AHCI_10MS_USECS);
5153         } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
5154 
5155         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5156             "ahci_software_reset: 2nd loop count: %d, "
5157             "port_cmd_issue = 0x%x, slot = 0x%x",
5158             loop_count, port_cmd_issue, slot);
5159 
5160         if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
5161             (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
5162                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
5163                     DDI_SERVICE_UNAFFECTED);
5164                 goto out;
5165         }
5166 
5167         rval = AHCI_SUCCESS;
5168 out:
5169         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5170             "ahci_software_reset: %s at port %d:%d",
5171             rval == AHCI_SUCCESS ? "succeed" : "failed",
5172             port, pmport);
5173 
5174         return (rval);
5175 }
5176 
5177 /*
5178  * AHCI port reset ...; the physical communication between the HBA and device
5179  * on a port are disabled. This is more intrusive.
5180  *
5181  * When an HBA or port reset occurs, Phy communication is going to
5182  * be re-established with the device through a COMRESET followed by the
5183  * normal out-of-band communication sequence defined in Serial ATA. At
5184  * the end of reset, the device, if working properly, will send a D2H
5185  * Register FIS, which contains the device signature. When the HBA receives
5186  * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
5187  * the PxSIG register with the signature.
5188  *
5189  * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
5190  * function is called. If not, it is assumed the interface is in hung
5191  * condition.
5192  */
5193 static int
5194 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5195     ahci_addr_t *addrp)
5196 {
5197         ahci_addr_t pmult_addr;
5198         uint32_t port_cmd_status;
5199         uint32_t port_scontrol, port_sstatus;
5200         uint32_t port_task_file;
5201         uint32_t port_state;
5202         uint8_t port = addrp->aa_port;
5203 
5204         int loop_count;
5205         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5206 
5207         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5208 
5209         /* Target is a port multiplier port? */
5210         if (AHCI_ADDR_IS_PMPORT(addrp))
5211                 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));
5212 
5213         /* Otherwise it must be an HBA port. */
5214         ASSERT(AHCI_ADDR_IS_PORT(addrp));
5215 
5216         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5217             "Port %d port resetting...", port);
5218         ahci_portp->ahciport_port_state = 0;
5219 
5220         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5221             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5222 
5223         /*
5224          * According to the spec, SUD bit should be set here,
5225          * but JMicron JMB363 doesn't follow it, so print
5226          * a debug message.
5227          */
5228         if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
5229                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5230                     "ahci_port_reset: port %d SUD bit not set", port);
5231 
5232         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5233             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5234         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5235 
5236         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5237             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5238             port_scontrol);
5239 
5240         /* Enable PxCMD.FRE to read device */
5241         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5242             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5243             port_cmd_status|AHCI_CMD_STATUS_FRE);
5244 
5245         /*
5246          * The port enters P:StartComm state, and the HBA tells the link layer
5247          * to start communication, which involves sending COMRESET to the
5248          * device. And the HBA resets PxTFD.STS to 7Fh.
5249          *
5250          * Give time for COMRESET to percolate, according to the AHCI
5251          * spec, software shall wait at least 1 millisecond before
5252          * clearing PxSCTL.DET
5253          */
5254         drv_usecwait(AHCI_1MS_USECS * 2);
5255 
5256         /* Fetch the SCONTROL again and rewrite the DET part with 0 */
5257         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5258             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5259         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5260         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5261             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
5262             port_scontrol);
5263 
5264         /*
5265          * When a COMINIT is received from the device, then the port enters
5266          * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5267          * PxSSTS.DET to 1h to indicate a device is detected but communication
5268          * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5269          * a COMINIT has been received.
5270          */
5271         /*
5272          * The DET field is valid only if IPM field indicates
5273          * that the interface is in active state.
5274          */
5275         loop_count = 0;
5276         for (;;) {
5277                 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5278                     (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
5279 
5280                 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5281                         /*
5282                          * If the interface is not active, the DET field
5283                          * is considered not accurate. So we want to
5284                          * continue looping.
5285                          */
5286                         SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5287                 }
5288 
5289                 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
5290                         break;
5291 
5292                 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5293                         /*
5294                          * We are effectively timing out after 0.1 sec.
5295                          */
5296                         break;
5297                 }
5298 
5299                 /* Wait for 10 millisec */
5300                 drv_usecwait(AHCI_10MS_USECS);
5301         }
5302 
5303         AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5304             "ahci_port_reset: 1st loop count: %d, "
5305             "port_sstatus = 0x%x port %d",
5306             loop_count, port_sstatus, port);
5307 
5308         if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
5309                 /*
5310                  * Either the port is not active or there
5311                  * is no device present.
5312                  */
5313                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5314                 return (AHCI_SUCCESS);
5315         }
5316 
5317         /* Clear port serror register for the port */
5318         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5319             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5320             AHCI_SERROR_CLEAR_ALL);
5321 
5322         /*
5323          * Devices should return a FIS contains its signature to HBA after
5324          * COMINIT signal. Check whether a D2H Register FIS is received by
5325          * polling PxTFD.STS.
5326          */
5327         loop_count = 0;
5328         for (;;) {
5329                 port_task_file =
5330                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5331                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
5332 
5333                 if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
5334                     AHCI_TFD_STS_ERR)) == 0)
5335                         break;
5336 
5337                 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5338                         /*
5339                          * We are effectively timing out after 11 sec.
5340                          */
5341                         cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
5342                             "the device hardware has been initialized and "
5343                             "the power-up diagnostics failed",
5344                             instance, port);
5345 
5346                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
5347                             "port %d: some or all of BSY, DRQ and ERR in "
5348                             "PxTFD.STS are not clear. We need another "
5349                             "software reset.", port);
5350 
5351                         /* Clear port serror register for the port */
5352                         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5353                             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5354                             AHCI_SERROR_CLEAR_ALL);
5355 
5356                         AHCI_ADDR_SET_PMULT(&pmult_addr, port);
5357 
5358                         /* Try another software reset. */
5359                         if (ahci_software_reset(ahci_ctlp, ahci_portp,
5360                             &pmult_addr) != AHCI_SUCCESS) {
5361                                 AHCIPORT_SET_STATE(ahci_portp, addrp,
5362                                     SATA_PSTATE_FAILED);
5363                                 return (AHCI_FAILURE);
5364                         }
5365                         break;
5366                 }
5367 
5368                 /* Wait for 10 millisec */
5369                 drv_usecwait(AHCI_10MS_USECS);
5370         }
5371 
5372         AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5373             "ahci_port_reset: 2nd loop count: %d, "
5374             "port_task_file = 0x%x port %d",
5375             loop_count, port_task_file, port);
5376 
5377         /* Clear port serror register for the port */
5378         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5379             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5380             AHCI_SERROR_CLEAR_ALL);
5381 
5382         /* Set port as ready */
5383         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5384         AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5385 
5386         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5387             "ahci_port_reset: succeed at port %d.", port);
5388         return (AHCI_SUCCESS);
5389 }
5390 
5391 /*
5392  * COMRESET on a port multiplier port.
5393  *
5394  * NOTE: Only called in ahci_port_reset()
5395  */
5396 static int
5397 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5398     ahci_addr_t *addrp)
5399 {
5400         uint32_t port_scontrol, port_sstatus, port_serror;
5401         uint32_t port_cmd_status, port_intr_status;
5402         uint32_t port_state;
5403         uint8_t port = addrp->aa_port;
5404         uint8_t pmport = addrp->aa_pmport;
5405         int loop_count;
5406         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
5407 
5408         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
5409             "port %d:%d: pmport resetting", port, pmport);
5410 
5411         /* Initialize pmport state */
5412         AHCIPORT_SET_STATE(ahci_portp, addrp, 0);
5413 
5414         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5415         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
5416         WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5417 
5418         /* PxCMD.FRE should be set before. */
5419         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5420             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5421         ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
5422         if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
5423                 return (AHCI_FAILURE);
5424 
5425         /*
5426          * Give time for COMRESET to percolate, according to the AHCI
5427          * spec, software shall wait at least 1 millisecond before
5428          * clearing PxSCTL.DET
5429          */
5430         drv_usecwait(AHCI_1MS_USECS*2);
5431 
5432         /*
5433          * Fetch the SCONTROL again and rewrite the DET part with 0
5434          * This will generate an Asychronous Notification events.
5435          */
5436         READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
5437         SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
5438         WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);
5439 
5440         /*
5441          * The port enters P:StartComm state, and HBA tells link layer to
5442          * start communication, which involves sending COMRESET to device.
5443          * And the HBA resets PxTFD.STS to 7Fh.
5444          *
5445          * When a COMINIT is received from the device, then the port enters
5446          * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
5447          * PxSSTS.DET to 1h to indicate a device is detected but communication
5448          * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
5449          * a COMINIT has been received.
5450          */
5451         /*
5452          * The DET field is valid only if IPM field indicates
5453          * that the interface is in active state.
5454          */
5455         loop_count = 0;
5456         do {
5457                 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);
5458 
5459                 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
5460                         /*
5461                          * If the interface is not active, the DET field
5462                          * is considered not accurate. So we want to
5463                          * continue looping.
5464                          */
5465                         SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
5466                 }
5467 
5468                 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
5469                         /*
5470                          * We are effectively timing out after 0.1 sec.
5471                          */
5472                         break;
5473                 }
5474 
5475                 /* Wait for 10 millisec */
5476                 drv_usecwait(AHCI_10MS_USECS);
5477 
5478         } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);
5479 
5480         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5481             "ahci_pmport_reset: 1st loop count: %d, "
5482             "port_sstatus = 0x%x port %d:%d",
5483             loop_count, port_sstatus, port, pmport);
5484 
5485         if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
5486             (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
5487                 /*
5488                  * Either the port is not active or there
5489                  * is no device present.
5490                  */
5491                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5492                     "ahci_pmport_reset: "
5493                     "no device attached to port %d:%d",
5494                     port, pmport);
5495                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
5496                 return (AHCI_SUCCESS);
5497         }
5498 
5499         /* Now we can make sure there is a device connected to the port */
5500         /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
5501         READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);
5502 
5503         if (!(port_serror & (1 << 26))) {
5504                 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
5505                     "COMINIT signal from the device not received port %d:%d",
5506                     instance, port, pmport);
5507 
5508                 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
5509                 return (AHCI_FAILURE);
5510         }
5511 
5512         /*
5513          * After clear PxSERR register, we will receive a D2H FIS.
5514          * Normally this FIS will cause a IPMS error according to AHCI spec
5515          * v1.2 because there is no command outstanding for it. So we need
5516          * to ignore this error.
5517          */
5518         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
5519         WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);
5520 
5521         /* Now we need to check the D2H FIS by checking IPMS error. */
5522         loop_count = 0;
5523         do {
5524                 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5525                     (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
5526 
5527                 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
5528                         /*
5529                          * No D2H FIS received. This is possible according
5530                          * to SATA 2.6 spec.
5531                          */
5532                         cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
5533                             "PxIS.IPMS is not set, we need another "
5534                             "software reset.", port, pmport);
5535 
5536                         break;
5537                 }
5538 
5539                 /* Wait for 10 millisec */
5540                 mutex_exit(&ahci_portp->ahciport_mutex);
5541                 delay(AHCI_10MS_TICKS);
5542                 mutex_enter(&ahci_portp->ahciport_mutex);
5543 
5544         } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));
5545 
5546         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5547             "ahci_pmport_reset: 2st loop count: %d, "
5548             "port_sstatus = 0x%x port %d:%d",
5549             loop_count, port_sstatus, port, pmport);
5550 
5551         /* Clear IPMS */
5552         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5553             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
5554             AHCI_INTR_STATUS_IPMS);
5555         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5556 
5557         /* This pmport is now ready for ahci_tran_start() */
5558         port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
5559         AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);
5560 
5561         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5562             "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
5563         return (AHCI_SUCCESS);
5564 
5565 err:    /* R/W PMULT error */
5566         /* IPMS flags might be set before. */
5567         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
5568         AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
5569             "ahci_pmport_reset: failed at port %d:%d", port, pmport);
5570 
5571         return (AHCI_FAILURE);
5572 }
5573 
5574 /*
5575  * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
5576  * This is the most intrusive.
5577  *
5578  * When an HBA reset occurs, Phy communication will be re-established with
5579  * the device through a COMRESET followed by the normal out-of-band
5580  * communication sequence defined in Serial ATA. At the end of reset, the
5581  * device, if working properly, will send a D2H Register FIS, which contains
5582  * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
5583  * and PxTFD.ERR register fields, and updates the PxSIG register with the
5584  * signature.
5585  *
5586  * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
5587  */
5588 static int
5589 ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
5590 {
5591         ahci_port_t *ahci_portp;
5592         uint32_t ghc_control;
5593         uint8_t port;
5594         int loop_count;
5595         int rval = AHCI_SUCCESS;
5596 
5597         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
5598             NULL);
5599 
5600         mutex_enter(&ahci_ctlp->ahcictl_mutex);
5601 
5602         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5603             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5604 
5605         /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
5606         ghc_control |= AHCI_HBA_GHC_HR;
5607         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5608             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5609 
5610         /*
5611          * Wait until HBA Reset complete or timeout
5612          */
5613         loop_count = 0;
5614         do {
5615                 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5616                     (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5617 
5618                 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
5619                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5620                             "ahci hba reset is timing out, "
5621                             "ghc_control = 0x%x", ghc_control);
5622                         /* We are effectively timing out after 1 sec. */
5623                         break;
5624                 }
5625 
5626                 /* Wait for 10 millisec */
5627                 drv_usecwait(AHCI_10MS_USECS);
5628         } while (ghc_control & AHCI_HBA_GHC_HR);
5629 
5630         AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
5631             "ahci_hba_reset: 1st loop count: %d, "
5632             "ghc_control = 0x%x", loop_count, ghc_control);
5633 
5634         if (ghc_control & AHCI_HBA_GHC_HR) {
5635                 /* The hba is not reset for some reasons */
5636                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
5637                     "hba reset failed: HBA in a hung or locked state", NULL);
5638                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
5639                 return (AHCI_FAILURE);
5640         }
5641 
5642         /*
5643          * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
5644          */
5645         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5646             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
5647         ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
5648         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5649             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
5650 
5651         mutex_exit(&ahci_ctlp->ahcictl_mutex);
5652 
5653         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
5654                 /* Only check implemented ports */
5655                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
5656                         continue;
5657                 }
5658 
5659                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
5660                 mutex_enter(&ahci_portp->ahciport_mutex);
5661 
5662                 /* Make sure the drive is spun-up */
5663                 ahci_staggered_spin_up(ahci_ctlp, port);
5664 
5665                 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5666                     port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
5667                     AHCI_SUCCESS) {
5668                         rval = AHCI_FAILURE;
5669                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5670                             "ahci_hba_reset: port %d failed", port);
5671                         /*
5672                          * Set the port state to SATA_PSTATE_FAILED if
5673                          * failed to initialize it.
5674                          */
5675                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5676                 }
5677 
5678                 mutex_exit(&ahci_portp->ahciport_mutex);
5679         }
5680 
5681         return (rval);
5682 }
5683 
5684 /*
5685  * This routine is only called from AHCI_ATTACH or phyrdy change
5686  * case. It first calls software reset, then stop the port and try to
5687  * read PxSIG register to find the type of device attached to the port.
5688  *
5689  * The caller should make sure a valid device exists on specified port and
5690  * physical communication has been established so that the signature could
5691  * be retrieved by software reset.
5692  *
5693  * NOTE: The port interrupts should be disabled before the function is called.
5694  */
5695 static void
5696 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5697     ahci_addr_t *addrp)
5698 {
5699         ahci_addr_t dev_addr;
5700         uint32_t signature;
5701         uint8_t port = addrp->aa_port;
5702         uint8_t pmport = addrp->aa_pmport;
5703         int rval;
5704 
5705         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5706         ASSERT(AHCI_ADDR_IS_VALID(addrp));
5707 
5708         /*
5709          * If the HBA doesn't support port multiplier, then the driver
5710          * doesn't need to bother to check port multiplier device.
5711          *
5712          * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
5713          * connected to Silicon Image 4723, to which the two sata drives
5714          * attached can be set with RAID1, RAID0 or Spanning mode.
5715          *
5716          * We found software reset will get failure if port multiplier address
5717          * 0xf is used by software reset, so just ignore the check since
5718          * ICH7 doesn't support port multiplier device at all.
5719          */
5720         if (AHCI_ADDR_IS_PORT(addrp) &&
5721             (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
5722                 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5723                     "ahci_find_dev_signature enter: port %d", port);
5724 
5725                 /*
5726                  * NOTE: when the ahci address is a HBA port, we do not know
5727                  * it is a device or a port multiplier that attached. we need
5728                  * try a software reset at port multiplier address (0xf
5729                  * pmport)
5730                  */
5731                 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
5732         } else {
5733                 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
5734                     "ahci_find_dev_signature enter: port %d:%d",
5735                     port, pmport);
5736                 dev_addr = *addrp;
5737         }
5738 
5739         /* Assume it is unknown. */
5740         AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5741 
5742         /* Issue a software reset to get the signature */
5743         rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
5744         if (rval != AHCI_SUCCESS) {
5745 
5746                 /*
5747                  * Try to do software reset again with pmport set with 0 if
5748                  * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
5749                  * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
5750                  */
5751                 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
5752                     (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
5753                         dev_addr.aa_pmport = 0;
5754                         rval = ahci_software_reset(ahci_ctlp, ahci_portp,
5755                             &dev_addr);
5756                 }
5757 
5758                 if (rval != AHCI_SUCCESS) {
5759                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
5760                             "ahci_find_dev_signature: software reset failed "
5761                             "at port %d:%d, cannot get signature.",
5762                             port, pmport);
5763 
5764                         AHCIPORT_SET_STATE(ahci_portp, addrp,
5765                             SATA_PSTATE_FAILED);
5766                         return;
5767                 }
5768         }
5769 
5770         /*
5771          * ahci_software_reset has started the port, so we need manually stop
5772          * the port again.
5773          */
5774         if (AHCI_ADDR_IS_PORT(addrp)) {
5775                 if (ahci_put_port_into_notrunning_state(ahci_ctlp,
5776                     ahci_portp, port) != AHCI_SUCCESS) {
5777                         AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5778                             "ahci_find_dev_signature: cannot stop port %d.",
5779                             port);
5780                         ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
5781                         return;
5782                 }
5783         }
5784 
5785         /* Now we can make sure that a valid signature is received. */
5786         signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5787             (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
5788 
5789         if (AHCI_ADDR_IS_PMPORT(addrp)) {
5790                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
5791                     "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
5792                     signature, port, pmport);
5793         } else {
5794                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
5795                     "ahci_find_dev_signature: signature = 0x%x at port %d",
5796                     signature, port);
5797         }
5798 
5799         /* NOTE: Only support ATAPI device at controller port. */
5800         if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
5801                 signature = SATA_DTYPE_UNKNOWN;
5802 
5803         switch (signature) {
5804 
5805         case AHCI_SIGNATURE_DISK:
5806                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
5807                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5808                     "Disk is found at port: %d", port);
5809                 break;
5810 
5811         case AHCI_SIGNATURE_ATAPI:
5812                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
5813                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5814                     "ATAPI device is found at port: %d", port);
5815                 break;
5816 
5817         case AHCI_SIGNATURE_PORT_MULTIPLIER:
5818                 /* Port Multiplier cannot recursively attached. */
5819                 ASSERT(AHCI_ADDR_IS_PORT(addrp));
5820                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
5821                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5822                     "Port Multiplier is found at port: %d", port);
5823                 break;
5824 
5825         default:
5826                 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
5827                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
5828                     "Unknown device is found at port: %d", port);
5829         }
5830 }
5831 
5832 /*
5833  * According to the spec, to reliably detect hot plug removals, software
5834  * must disable interface power management. Software should perform the
5835  * following initialization on a port after a device is attached:
5836  *   Set PxSCTL.IPM to 3h to disable interface state transitions
5837  *   Set PxCMD.ALPE to '0' to disable aggressive power management
5838  *   Disable device initiated interface power management by SET FEATURE
5839  *
5840  * We can ignore the last item because by default the feature is disabled
5841  */
5842 static void
5843 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
5844 {
5845         uint32_t port_scontrol, port_cmd_status;
5846 
5847         port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5848             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
5849         SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH);
5850         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5851             (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
5852 
5853         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5854             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5855         port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
5856         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5857             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
5858 }
5859 
5860 /*
5861  * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
5862  * to 1, then set it firstly.
5863  *
5864  * Each port contains two major DMA engines. One DMA engine walks through
5865  * the command list, and is controlled by PxCMD.ST. The second DMA engine
5866  * copies received FISes into system memory, and is controlled by PxCMD.FRE.
5867  *
5868  * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
5869  * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
5870  * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
5871  *
5872  * Software shall not set PxCMD.ST to '1' unless a functional device is
5873  * present on the port(as determined by PxTFD.STS.BSY = '0',
5874  * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
5875  */
5876 static int
5877 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
5878 {
5879         uint32_t port_cmd_status;
5880 
5881         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5882 
5883         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
5884 
5885         if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
5886                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5887                     "the state for port %d is 0x%x",
5888                     port, ahci_portp->ahciport_port_state);
5889                 return (AHCI_FAILURE);
5890         }
5891 
5892         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5893                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
5894                     "no device is attached at port %d", port);
5895                 return (AHCI_FAILURE);
5896         }
5897 
5898         /* First to set PxCMD.FRE before setting PxCMD.ST. */
5899         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5900             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5901 
5902         if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
5903                 port_cmd_status |= AHCI_CMD_STATUS_FRE;
5904                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5905                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5906                     port_cmd_status);
5907         }
5908 
5909         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5910             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5911 
5912         port_cmd_status |= AHCI_CMD_STATUS_ST;
5913 
5914         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5915             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5916             port_cmd_status);
5917 
5918         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
5919 
5920         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
5921             "PxCMD.ST set to '1' at port %d", port);
5922 
5923         return (AHCI_SUCCESS);
5924 }
5925 
5926 /*
5927  * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
5928  * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
5929  * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
5930  */
5931 static int
5932 ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
5933 {
5934         uint8_t port = ahci_portp->ahciport_port_num;
5935         uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5936             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5937 
5938         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
5939 
5940         /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
5941         if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
5942                 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5943                     port) != AHCI_SUCCESS)
5944                         return (AHCI_FAILURE);
5945 
5946                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5947                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5948         }
5949 
5950         /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
5951         if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
5952                 int loop_count = 0;
5953 
5954                 /* Clear PxCMD.FRE */
5955                 port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
5956                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5957                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
5958                     port_cmd_status);
5959 
5960                 /* Wait until PxCMD.FR is cleared */
5961                 for (;;) {
5962                         port_cmd_status =
5963                             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5964                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5965 
5966                         if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
5967                                 break;
5968 
5969                         if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
5970                                 AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
5971                                     "ahci_setup_port_base_addresses: cannot "
5972                                     "clear PxCMD.FR for port %d.", port);
5973 
5974                                 /*
5975                                  * We are effectively timing out after 0.5 sec.
5976                                  * This value is specified in AHCI spec.
5977                                  */
5978                                 return (AHCI_FAILURE);
5979                         }
5980 
5981                         /* Wait for 1 millisec */
5982                         drv_usecwait(AHCI_1MS_USECS);
5983                 }
5984         }
5985 
5986         /* Step 3: Config Port Command List Base Address */
5987         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5988             (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
5989             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
5990 
5991         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5992             (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
5993             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);
5994 
5995         /* Step 4: Config Port Received FIS Base Address */
5996         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5997             (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
5998             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
5999 
6000         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6001             (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
6002             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);
6003 
6004         return (AHCI_SUCCESS);
6005 }
6006 
6007 /*
6008  * Allocate the ahci_port_t including Received FIS and Command List.
6009  * The argument - port is the physical port number, and not logical
6010  * port number seen by the SATA framework.
6011  */
6012 static int
6013 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6014 {
6015         dev_info_t *dip = ahci_ctlp->ahcictl_dip;
6016         ahci_port_t *ahci_portp;
6017         char taskq_name[64] = "event_handle_taskq";
6018 
6019         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6020 
6021         ahci_portp =
6022             (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
6023 
6024         ahci_ctlp->ahcictl_ports[port] = ahci_portp;
6025         ahci_portp->ahciport_port_num = port;
6026 
6027         /* Initialize the port condition variable */
6028         cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
6029 
6030         /* Initialize the port mutex */
6031         mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
6032             (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6033 
6034         mutex_enter(&ahci_portp->ahciport_mutex);
6035 
6036         /*
6037          * Allocate memory for received FIS structure and
6038          * command list for this port
6039          */
6040         if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6041                 goto err_case1;
6042         }
6043 
6044         if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6045                 goto err_case2;
6046         }
6047 
6048         /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
6049         if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
6050             AHCI_SUCCESS) {
6051                 goto err_case3;
6052         }
6053 
6054         (void) snprintf(taskq_name + strlen(taskq_name),
6055             sizeof (taskq_name) - strlen(taskq_name),
6056             "_port%d", port);
6057 
6058         /* Create the taskq for the port */
6059         if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
6060             taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
6061                 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
6062                     "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
6063                 goto err_case3;
6064         }
6065 
6066         /* Allocate the argument for the taskq */
6067         ahci_portp->ahciport_event_args =
6068             kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
6069 
6070         ahci_portp->ahciport_event_args->ahciea_addrp =
6071             kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);
6072 
6073         if (ahci_portp->ahciport_event_args == NULL)
6074                 goto err_case4;
6075 
6076         /* Initialize the done queue */
6077         ahci_portp->ahciport_doneq = NULL;
6078         ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
6079         ahci_portp->ahciport_doneq_len = 0;
6080 
6081         mutex_exit(&ahci_portp->ahciport_mutex);
6082 
6083         return (AHCI_SUCCESS);
6084 
6085 err_case4:
6086         ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6087 
6088 err_case3:
6089         ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6090 
6091 err_case2:
6092         ahci_dealloc_rcvd_fis(ahci_portp);
6093 
6094 err_case1:
6095         mutex_exit(&ahci_portp->ahciport_mutex);
6096         mutex_destroy(&ahci_portp->ahciport_mutex);
6097         cv_destroy(&ahci_portp->ahciport_cv);
6098 
6099         kmem_free(ahci_portp, sizeof (ahci_port_t));
6100 
6101         return (AHCI_FAILURE);
6102 }
6103 
6104 /*
6105  * Reverse of ahci_alloc_port_state().
6106  */
6107 static void
6108 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
6109 {
6110         ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
6111 
6112         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
6113         ASSERT(ahci_portp != NULL);
6114 
6115         mutex_enter(&ahci_portp->ahciport_mutex);
6116         kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
6117             sizeof (ahci_addr_t));
6118         ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
6119         kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
6120         ahci_portp->ahciport_event_args = NULL;
6121         ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
6122         ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
6123         ahci_dealloc_rcvd_fis(ahci_portp);
6124         ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
6125         mutex_exit(&ahci_portp->ahciport_mutex);
6126 
6127         mutex_destroy(&ahci_portp->ahciport_mutex);
6128         cv_destroy(&ahci_portp->ahciport_cv);
6129 
6130         kmem_free(ahci_portp, sizeof (ahci_port_t));
6131 
6132         ahci_ctlp->ahcictl_ports[port] = NULL;
6133 }
6134 
6135 /*
6136  * Allocates memory for the Received FIS Structure
6137  */
6138 static int
6139 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6140 {
6141         size_t rcvd_fis_size;
6142         size_t ret_len;
6143         uint_t cookie_count;
6144 
6145         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6146 
6147         rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
6148 
6149         /* allocate rcvd FIS dma handle. */
6150         if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6151             &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
6152             DDI_DMA_SLEEP,
6153             NULL,
6154             &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
6155             DDI_SUCCESS) {
6156                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6157                     "rcvd FIS dma handle alloc failed", NULL);
6158 
6159                 return (AHCI_FAILURE);
6160         }
6161 
6162         if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
6163             rcvd_fis_size,
6164             &accattr,
6165             DDI_DMA_CONSISTENT,
6166             DDI_DMA_SLEEP,
6167             NULL,
6168             (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
6169             &ret_len,
6170             &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
6171 
6172                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6173                     "rcvd FIS dma mem alloc fail", NULL);
6174                 /* error.. free the dma handle. */
6175                 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6176                 return (AHCI_FAILURE);
6177         }
6178 
6179         if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
6180             NULL,
6181             (caddr_t)ahci_portp->ahciport_rcvd_fis,
6182             rcvd_fis_size,
6183             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6184             DDI_DMA_SLEEP,
6185             NULL,
6186             &ahci_portp->ahciport_rcvd_fis_dma_cookie,
6187             &cookie_count) !=  DDI_DMA_MAPPED) {
6188 
6189                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6190                     "rcvd FIS dma handle bind fail", NULL);
6191                 /*  error.. free the dma handle & free the memory. */
6192                 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6193                 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6194                 return (AHCI_FAILURE);
6195         }
6196 
6197         bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
6198 
6199         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6200             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
6201         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6202             ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);
6203 
6204         return (AHCI_SUCCESS);
6205 }
6206 
6207 /*
6208  * Deallocates the Received FIS Structure
6209  */
6210 static void
6211 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
6212 {
6213         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6214 
6215         /* Unbind the cmd list dma handle first. */
6216         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
6217 
6218         /* Then free the underlying memory. */
6219         ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
6220 
6221         /* Now free the handle itself. */
6222         ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
6223 }
6224 
6225 /*
6226  * Allocates memory for the Command List, which contains up to 32 entries.
6227  * Each entry contains a command header, which is a 32-byte structure that
6228  * includes the pointer to the command table.
6229  */
6230 static int
6231 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6232 {
6233         size_t cmd_list_size;
6234         size_t ret_len;
6235         uint_t cookie_count;
6236 
6237         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6238 
6239         cmd_list_size =
6240             ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
6241 
6242         /* allocate cmd list dma handle. */
6243         if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6244             &ahci_ctlp->ahcictl_cmd_list_dma_attr,
6245             DDI_DMA_SLEEP,
6246             NULL,
6247             &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
6248 
6249                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6250                     "cmd list dma handle alloc failed", NULL);
6251                 return (AHCI_FAILURE);
6252         }
6253 
6254         if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
6255             cmd_list_size,
6256             &accattr,
6257             DDI_DMA_CONSISTENT,
6258             DDI_DMA_SLEEP,
6259             NULL,
6260             (caddr_t *)&ahci_portp->ahciport_cmd_list,
6261             &ret_len,
6262             &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
6263 
6264                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6265                     "cmd list dma mem alloc fail", NULL);
6266                 /* error.. free the dma handle. */
6267                 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6268                 return (AHCI_FAILURE);
6269         }
6270 
6271         if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
6272             NULL,
6273             (caddr_t)ahci_portp->ahciport_cmd_list,
6274             cmd_list_size,
6275             DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6276             DDI_DMA_SLEEP,
6277             NULL,
6278             &ahci_portp->ahciport_cmd_list_dma_cookie,
6279             &cookie_count) !=  DDI_DMA_MAPPED) {
6280 
6281                 AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6282                     "cmd list dma handle bind fail", NULL);
6283                 /*  error.. free the dma handle & free the memory. */
6284                 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6285                 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6286                 return (AHCI_FAILURE);
6287         }
6288 
6289         bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
6290 
6291         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
6292             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);
6293 
6294         AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
6295             ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);
6296 
6297         if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
6298                 goto err_out;
6299         }
6300 
6301         return (AHCI_SUCCESS);
6302 
6303 err_out:
6304         /* Unbind the cmd list dma handle first. */
6305         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6306 
6307         /* Then free the underlying memory. */
6308         ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6309 
6310         /* Now free the handle itself. */
6311         ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6312 
6313         return (AHCI_FAILURE);
6314 }
6315 
6316 /*
6317  * Deallocates the Command List
6318  */
6319 static void
6320 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6321 {
6322         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6323 
6324         /* First dealloc command table */
6325         ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
6326 
6327         /* Unbind the cmd list dma handle first. */
6328         (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
6329 
6330         /* Then free the underlying memory. */
6331         ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
6332 
6333         /* Now free the handle itself. */
6334         ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
6335 }
6336 
6337 /*
6338  * Allocates memory for all Command Tables, which contains Command FIS,
6339  * ATAPI Command and Physical Region Descriptor Table.
6340  */
6341 static int
6342 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6343 {
6344         size_t ret_len;
6345         ddi_dma_cookie_t cmd_table_dma_cookie;
6346         uint_t cookie_count;
6347         int slot;
6348 
6349         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6350 
6351         AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
6352             "ahci_alloc_cmd_tables: port %d enter",
6353             ahci_portp->ahciport_port_num);
6354 
6355         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6356                 /* Allocate cmd table dma handle. */
6357                 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
6358                     &ahci_ctlp->ahcictl_cmd_table_dma_attr,
6359                     DDI_DMA_SLEEP,
6360                     NULL,
6361                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
6362                     DDI_SUCCESS) {
6363 
6364                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6365                             "cmd table dma handle alloc failed", NULL);
6366 
6367                         goto err_out;
6368                 }
6369 
6370                 if (ddi_dma_mem_alloc(
6371                     ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6372                     ahci_cmd_table_size,
6373                     &accattr,
6374                     DDI_DMA_CONSISTENT,
6375                     DDI_DMA_SLEEP,
6376                     NULL,
6377                     (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
6378                     &ret_len,
6379                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
6380                     NULL) {
6381 
6382                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6383                             "cmd table dma mem alloc fail", NULL);
6384 
6385                         /* error.. free the dma handle. */
6386                         ddi_dma_free_handle(
6387                             &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6388                         goto err_out;
6389                 }
6390 
6391                 if (ddi_dma_addr_bind_handle(
6392                     ahci_portp->ahciport_cmd_tables_dma_handle[slot],
6393                     NULL,
6394                     (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
6395                     ahci_cmd_table_size,
6396                     DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
6397                     DDI_DMA_SLEEP,
6398                     NULL,
6399                     &cmd_table_dma_cookie,
6400                     &cookie_count) !=  DDI_DMA_MAPPED) {
6401 
6402                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
6403                             "cmd table dma handle bind fail", NULL);
6404                         /*  error.. free the dma handle & free the memory. */
6405                         ddi_dma_mem_free(
6406                             &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6407                         ddi_dma_free_handle(
6408                             &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6409                         goto err_out;
6410                 }
6411 
6412                 bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
6413                     ahci_cmd_table_size);
6414 
6415                 /* Config Port Command Table Base Address */
6416                 SET_COMMAND_TABLE_BASE_ADDR(
6417                     (&ahci_portp->ahciport_cmd_list[slot]),
6418                     cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
6419 
6420 #ifndef __lock_lint
6421                 SET_COMMAND_TABLE_BASE_ADDR_UPPER(
6422                     (&ahci_portp->ahciport_cmd_list[slot]),
6423                     cmd_table_dma_cookie.dmac_laddress >> 32);
6424 #endif
6425         }
6426 
6427         return (AHCI_SUCCESS);
6428 err_out:
6429 
6430         for (slot--; slot >= 0; slot--) {
6431                 /* Unbind the cmd table dma handle first */
6432                 (void) ddi_dma_unbind_handle(
6433                     ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6434 
6435                 /* Then free the underlying memory */
6436                 ddi_dma_mem_free(
6437                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6438 
6439                 /* Now free the handle itself */
6440                 ddi_dma_free_handle(
6441                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6442         }
6443 
6444         return (AHCI_FAILURE);
6445 }
6446 
6447 /*
6448  * Deallocates memory for all Command Tables.
6449  */
6450 static void
6451 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
6452 {
6453         int slot;
6454 
6455         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
6456 
6457         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
6458             "ahci_dealloc_cmd_tables: %d enter",
6459             ahci_portp->ahciport_port_num);
6460 
6461         for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
6462                 /* Unbind the cmd table dma handle first. */
6463                 (void) ddi_dma_unbind_handle(
6464                     ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6465 
6466                 /* Then free the underlying memory. */
6467                 ddi_dma_mem_free(
6468                     &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
6469 
6470                 /* Now free the handle itself. */
6471                 ddi_dma_free_handle(
6472                     &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
6473         }
6474 }
6475 
6476 /*
6477  * Update SATA registers at controller ports
6478  */
6479 static void
6480 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
6481     sata_device_t *sd)
6482 {
6483         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
6484 
6485         sd->satadev_scr.sstatus =
6486             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6487             (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
6488         sd->satadev_scr.serror =
6489             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6490             (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
6491         sd->satadev_scr.scontrol =
6492             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6493             (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
6494         sd->satadev_scr.sactive =
6495             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6496             (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
6497 }
6498 
6499 /*
6500  * For poll mode, ahci_port_intr will be called to emulate the interrupt
6501  */
6502 static void
6503 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
6504 {
6505         uint32_t port_intr_status;
6506         uint32_t port_intr_enable;
6507 
6508         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
6509             "ahci_port_intr enter: port %d", port);
6510 
6511         mutex_enter(&ahci_portp->ahciport_mutex);
6512         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
6513                 /* For SATA_OPMODE_POLLING commands */
6514                 port_intr_enable =
6515                     (AHCI_INTR_STATUS_DHRS |
6516                     AHCI_INTR_STATUS_PSS |
6517                     AHCI_INTR_STATUS_SDBS |
6518                     AHCI_INTR_STATUS_UFS |
6519                     AHCI_INTR_STATUS_PCS |
6520                     AHCI_INTR_STATUS_PRCS |
6521                     AHCI_INTR_STATUS_OFS |
6522                     AHCI_INTR_STATUS_INFS |
6523                     AHCI_INTR_STATUS_IFS |
6524                     AHCI_INTR_STATUS_HBDS |
6525                     AHCI_INTR_STATUS_HBFS |
6526                     AHCI_INTR_STATUS_TFES);
6527         } else {
6528                 /*
6529                  * port_intr_enable indicates that the corresponding interrrupt
6530                  * reporting is enabled.
6531                  */
6532                 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6533                     (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
6534         }
6535 
6536         /* IPMS error in port reset should be ignored according AHCI spec. */
6537         if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
6538                 port_intr_enable |= AHCI_INTR_STATUS_IPMS;
6539         mutex_exit(&ahci_portp->ahciport_mutex);
6540 
6541         /*
6542          * port_intr_stats indicates that the corresponding interrupt
6543          * condition is active.
6544          */
6545         port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6546             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
6547 
6548         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6549             "ahci_port_intr: port %d, port_intr_status = 0x%x, "
6550             "port_intr_enable = 0x%x",
6551             port, port_intr_status, port_intr_enable);
6552 
6553         port_intr_status &= port_intr_enable;
6554 
6555         /*
6556          * Pending interrupt events are indicated by the PxIS register.
6557          * Make sure we don't miss any event.
6558          */
6559         if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
6560                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6561                     DDI_SERVICE_UNAFFECTED);
6562                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6563                     DDI_FME_VERSION);
6564                 return;
6565         }
6566 
6567         /* First clear the port interrupts status */
6568         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6569             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
6570             port_intr_status);
6571 
6572         /* Check the completed non-queued commands */
6573         if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
6574             AHCI_INTR_STATUS_PSS)) {
6575                 (void) ahci_intr_cmd_cmplt(ahci_ctlp,
6576                     ahci_portp, port);
6577         }
6578 
6579         /* Check the completed queued commands */
6580         if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
6581                 (void) ahci_intr_set_device_bits(ahci_ctlp,
6582                     ahci_portp, port);
6583         }
6584 
6585         /* Check the port connect change status interrupt bit */
6586         if (port_intr_status & AHCI_INTR_STATUS_PCS) {
6587                 (void) ahci_intr_port_connect_change(ahci_ctlp,
6588                     ahci_portp, port);
6589         }
6590 
6591         /* Check the device mechanical presence status interrupt bit */
6592         if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
6593                 (void) ahci_intr_device_mechanical_presence_status(
6594                     ahci_ctlp, ahci_portp, port);
6595         }
6596 
6597         /* Check the PhyRdy change status interrupt bit */
6598         if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
6599                 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
6600                     port);
6601         }
6602 
6603         /*
6604          * Check the non-fatal error interrupt bits, there are four
6605          * kinds of non-fatal errors at the time being:
6606          *
6607          *    PxIS.UFS - Unknown FIS Error
6608          *    PxIS.OFS - Overflow Error
6609          *    PxIS.INFS - Interface Non-Fatal Error
6610          *    PxIS.IPMS - Incorrect Port Multiplier Status Error
6611          *
6612          * For these non-fatal errors, the HBA can continue to operate,
6613          * so the driver just log the error messages.
6614          */
6615         if (port_intr_status & (AHCI_INTR_STATUS_UFS |
6616             AHCI_INTR_STATUS_OFS |
6617             AHCI_INTR_STATUS_IPMS |
6618             AHCI_INTR_STATUS_INFS)) {
6619                 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
6620                     port, port_intr_status);
6621         }
6622 
6623         /*
6624          * Check the fatal error interrupt bits, there are four kinds
6625          * of fatal errors for AHCI controllers:
6626          *
6627          *    PxIS.HBFS - Host Bus Fatal Error
6628          *    PxIS.HBDS - Host Bus Data Error
6629          *    PxIS.IFS - Interface Fatal Error
6630          *    PxIS.TFES - Task File Error
6631          *
6632          * The fatal error means the HBA can not recover from it by
6633          * itself, and it will try to abort the transfer, and the software
6634          * must intervene to restart the port.
6635          */
6636         if (port_intr_status & (AHCI_INTR_STATUS_IFS |
6637             AHCI_INTR_STATUS_HBDS |
6638             AHCI_INTR_STATUS_HBFS |
6639             AHCI_INTR_STATUS_TFES))
6640                 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
6641                     port, port_intr_status);
6642 
6643         /* Check the cold port detect interrupt bit */
6644         if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
6645                 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
6646         }
6647 
6648         /* Second clear the corresponding bit in IS.IPS */
6649         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
6650             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
6651 
6652         /* Try to recover at the end of the interrupt handler. */
6653         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6654             DDI_FM_OK) {
6655                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6656                     DDI_SERVICE_UNAFFECTED);
6657                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6658                     DDI_FME_VERSION);
6659         }
6660 }
6661 
6662 /*
6663  * Interrupt service handler
6664  */
6665 static uint_t
6666 ahci_intr(caddr_t arg1, caddr_t arg2)
6667 {
6668 #ifndef __lock_lint
6669         _NOTE(ARGUNUSED(arg2))
6670 #endif
6671         /* LINTED */
6672         ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
6673         ahci_port_t *ahci_portp;
6674         int32_t global_intr_status;
6675         uint8_t port;
6676 
6677         /*
6678          * global_intr_status indicates that the corresponding port has
6679          * an interrupt pending.
6680          */
6681         global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6682             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
6683 
6684         if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
6685                 /* The interrupt is not ours */
6686                 return (DDI_INTR_UNCLAIMED);
6687         }
6688 
6689         /*
6690          * Check the handle after reading global_intr_status - we don't want
6691          * to miss any port with pending interrupts.
6692          */
6693         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
6694             DDI_FM_OK) {
6695                 ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
6696                     DDI_SERVICE_UNAFFECTED);
6697                 ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
6698                     DDI_FME_VERSION);
6699                 return (DDI_INTR_UNCLAIMED);
6700         }
6701 
6702         /* Loop for all the ports */
6703         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
6704                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
6705                         continue;
6706                 }
6707                 if (!((0x1 << port) & global_intr_status)) {
6708                         continue;
6709                 }
6710 
6711                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
6712 
6713                 /* Call ahci_port_intr */
6714                 ahci_port_intr(ahci_ctlp, ahci_portp, port);
6715         }
6716 
6717         return (DDI_INTR_CLAIMED);
6718 }
6719 
6720 /*
6721  * For non-queued commands, when the corresponding bit in the PxCI register
6722  * is cleared, it means the command is completed successfully. And according
6723  * to the HBA state machine, there are three conditions which possibly will
6724  * try to clear the PxCI register bit.
6725  *      1. Receive one D2H Register FIS which is with 'I' bit set
6726  *      2. Update PIO Setup FIS
6727  *      3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
6728  *
6729  * Process completed non-queued commands when the interrupt status bit -
6730  * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
6731  *
6732  * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
6733  * with the 'I' bit set. And the following commands will send thus
6734  * FIS with 'I' bit set upon the successful completion:
6735  *      1. Non-data commands
6736  *      2. DMA data-in command
6737  *      3. DMA data-out command
6738  *      4. PIO data-out command
6739  *      5. PACKET non-data commands
6740  *      6. PACKET PIO data-in command
6741  *      7. PACKET PIO data-out command
6742  *      8. PACKET DMA data-in command
6743  *      9. PACKET DMA data-out command
6744  *
6745  * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
6746  * with the 'I' bit set. And the following commands will send this
6747  * FIS upon the successful completion:
6748  *      1. PIO data-in command
6749  */
6750 static int
6751 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
6752     ahci_port_t *ahci_portp, uint8_t port)
6753 {
6754         uint32_t port_cmd_issue = 0;
6755         uint32_t finished_tags;
6756         int finished_slot;
6757         sata_pkt_t *satapkt;
6758         ahci_fis_d2h_register_t *rcvd_fisp;
6759 #if AHCI_DEBUG
6760         ahci_cmd_header_t *cmd_header;
6761         uint32_t cmd_dmacount;
6762 #endif
6763 
6764         mutex_enter(&ahci_portp->ahciport_mutex);
6765 
6766         if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6767             !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6768             !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
6769                 /*
6770                  * Spurious interrupt. Nothing to be done.
6771                  */
6772                 mutex_exit(&ahci_portp->ahciport_mutex);
6773                 return (AHCI_SUCCESS);
6774         }
6775 
6776         port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
6777             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
6778 
6779         /* If the PxCI corrupts, don't complete the commmands. */
6780         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
6781             != DDI_FM_OK) {
6782                 mutex_exit(&ahci_portp->ahciport_mutex);
6783                 return (AHCI_FAILURE);
6784         }
6785 
6786         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
6787                 /* Slot 0 is always used during error recovery */
6788                 finished_tags = 0x1 & ~port_cmd_issue;
6789                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
6790                     "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
6791                     "retrieval is finished, and finished_tags = 0x%x",
6792                     port, finished_tags);
6793         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
6794                 finished_tags = 0x1 & ~port_cmd_issue;
6795                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
6796                     "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
6797                     "port multiplier is finished, and finished_tags = 0x%x",
6798                     port, finished_tags);
6799 
6800         } else {
6801 
6802                 finished_tags = ahci_portp->ahciport_pending_tags &
6803                     ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
6804         }
6805 
6806         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6807             "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
6808             "port_cmd_issue = 0x%x finished_tags = 0x%x",
6809             ahci_portp->ahciport_pending_tags, port_cmd_issue,
6810             finished_tags);
6811 
6812         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
6813             (finished_tags == 0x1)) {
6814                 satapkt = ahci_portp->ahciport_err_retri_pkt;
6815                 ASSERT(satapkt != NULL);
6816 
6817                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6818                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6819                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6820 
6821                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6822                 goto out;
6823         }
6824 
6825         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
6826             (finished_tags == 0x1)) {
6827                 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
6828                 ASSERT(satapkt != NULL);
6829 
6830                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6831                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6832                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6833 
6834                 /* READ PORTMULT need copy out FIS content. */
6835                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6836                         rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6837                             ahcirf_d2h_register_fis);
6838                         satapkt->satapkt_cmd.satacmd_status_reg =
6839                             GET_RFIS_STATUS(rcvd_fisp);
6840                         ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6841                 }
6842 
6843                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6844                 goto out;
6845         }
6846 
6847         while (finished_tags) {
6848                 finished_slot = ddi_ffs(finished_tags) - 1;
6849                 if (finished_slot == -1) {
6850                         goto out;
6851                 }
6852 
6853                 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
6854                 ASSERT(satapkt != NULL);
6855 #if AHCI_DEBUG
6856                 /*
6857                  * For non-native queued commands, the PRD byte count field
6858                  * shall contain an accurate count of the number of bytes
6859                  * transferred for the command before the PxCI bit is cleared
6860                  * to '0' for the command.
6861                  *
6862                  * The purpose of this field is to let software know how many
6863                  * bytes transferred for a given operation in order to
6864                  * determine if underflow occurred. When issuing native command
6865                  * queuing commands, this field should not be used and is not
6866                  * required to be valid since in this case underflow is always
6867                  * illegal.
6868                  *
6869                  * For data reads, the HBA will update its PRD byte count with
6870                  * the total number of bytes received from the last FIS, and
6871                  * may be able to continue normally. For data writes, the
6872                  * device will detect an error, and HBA most likely will get
6873                  * a fatal error.
6874                  *
6875                  * Therefore, here just put code to debug part. And please
6876                  * refer to the comment above ahci_intr_fatal_error for the
6877                  * definition of underflow error.
6878                  */
6879                 cmd_dmacount =
6880                     ahci_portp->ahciport_prd_bytecounts[finished_slot];
6881                 if (cmd_dmacount) {
6882                         cmd_header =
6883                             &ahci_portp->ahciport_cmd_list[finished_slot];
6884                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
6885                             "ahci_intr_cmd_cmplt: port %d, "
6886                             "PRD Byte Count = 0x%x, "
6887                             "ahciport_prd_bytecounts = 0x%x", port,
6888                             cmd_header->ahcich_prd_byte_count,
6889                             cmd_dmacount);
6890 
6891                         if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
6892                                 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
6893                                     "ahci_intr_cmd_cmplt: port %d, "
6894                                     "an underflow occurred", port);
6895                         }
6896                 }
6897 #endif
6898 
6899                 /*
6900                  * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
6901                  * feature, sata_special_regs flag will be set, and the
6902                  * driver should copy the status and the other corresponding
6903                  * register values in the D2H Register FIS received (It's
6904                  * working on Non-data protocol) from the device back to
6905                  * the sata_cmd.
6906                  *
6907                  * For every AHCI port, there is only one Received FIS
6908                  * structure, which contains the FISes received from the
6909                  * device, So we're trying to copy the content of D2H
6910                  * Register FIS in the Received FIS structure back to
6911                  * the sata_cmd.
6912                  */
6913                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
6914                         rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
6915                             ahcirf_d2h_register_fis);
6916                         satapkt->satapkt_cmd.satacmd_status_reg =
6917                             GET_RFIS_STATUS(rcvd_fisp);
6918                         ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
6919                 }
6920 
6921                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
6922                     "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
6923                     "with SATA_PKT_COMPLETED", (void *)satapkt);
6924 
6925                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
6926                 CLEAR_BIT(finished_tags, finished_slot);
6927                 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
6928 
6929                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
6930         }
6931 out:
6932         AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
6933             "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
6934             ahci_portp->ahciport_pending_tags);
6935 
6936         ahci_flush_doneq(ahci_portp);
6937 
6938         mutex_exit(&ahci_portp->ahciport_mutex);
6939 
6940         return (AHCI_SUCCESS);
6941 }
6942 
6943 /*
6944  * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
6945  * with the 'I' bit set and has been copied into system memory. It will
6946  * be sent under the following situations:
6947  *
6948  * 1. NCQ command is completed
6949  *
6950  * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
6951  * via the Set Device Bits FIS. When such event is generated, the software
6952  * needs to read PxSACT register and compares the current value to the
6953  * list of commands previously issue by software. ahciport_pending_ncq_tags
6954  * keeps the tags of previously issued commands.
6955  *
6956  * 2. Asynchronous Notification
6957  *
6958  * Asynchronous Notification is a feature in SATA spec 2.6.
6959  *
6960  * 1) ATAPI device will send a signal to the host when media is inserted or
6961  * removed and avoids polling the device for media changes. The signal
6962  * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
6963  * set to '1'. At the moment, it's not supported yet.
6964  *
6965  * 2) Port multiplier will send a signal to the host when a hot plug event
6966  * has occured on a port multiplier port. It is used when command based
6967  * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
6968  */
6969 static int
6970 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
6971     ahci_port_t *ahci_portp, uint8_t port)
6972 {
6973         ahci_addr_t addr;
6974 
6975         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
6976             "ahci_intr_set_device_bits enter: port %d", port);
6977 
6978         /* Initialize HBA port address */
6979         AHCI_ADDR_SET_PORT(&addr, port);
6980 
6981         /* NCQ plug handler */
6982         (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);
6983 
6984         /* Check port multiplier's asynchronous notification events */
6985         if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
6986                 (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
6987                     ahci_portp, port);
6988         }
6989 
6990         /* ATAPI events is not supported yet */
6991 
6992         return (AHCI_SUCCESS);
6993 }
6994 /*
6995  * NCQ interrupt handler. Called upon a NCQ command is completed.
6996  * Only be called from ahci_intr_set_device_bits().
6997  */
6998 static int
6999 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
7000     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
7001 {
7002         uint32_t port_sactive;
7003         uint32_t port_cmd_issue;
7004         uint32_t issued_tags;
7005         int issued_slot;
7006         uint32_t finished_tags;
7007         int finished_slot;
7008         uint8_t port = addrp->aa_port;
7009         sata_pkt_t *satapkt;
7010 
7011         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7012             "ahci_intr_set_device_bits enter: port %d", port);
7013 
7014         mutex_enter(&ahci_portp->ahciport_mutex);
7015         if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7016                 mutex_exit(&ahci_portp->ahciport_mutex);
7017                 return (AHCI_SUCCESS);
7018         }
7019 
7020         /*
7021          * First the handler got which commands are finished by checking
7022          * PxSACT register
7023          */
7024         port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7025             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7026 
7027         finished_tags = ahci_portp->ahciport_pending_ncq_tags &
7028             ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);
7029 
7030         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7031             "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
7032             "port_sactive = 0x%x", port,
7033             ahci_portp->ahciport_pending_ncq_tags, port_sactive);
7034 
7035         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7036             "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);
7037 
7038         /*
7039          * For NCQ commands, the software can determine which command has
7040          * already been transmitted to the device by checking PxCI register.
7041          */
7042         port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7043             (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7044 
7045         issued_tags = ahci_portp->ahciport_pending_tags &
7046             ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
7047 
7048         /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
7049         if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
7050             != DDI_FM_OK) {
7051                 mutex_exit(&ahci_portp->ahciport_mutex);
7052                 return (AHCI_FAILURE);
7053         }
7054 
7055         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7056             "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
7057             "port_cmd_issue = 0x%x", port,
7058             ahci_portp->ahciport_pending_tags, port_cmd_issue);
7059 
7060         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7061             "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);
7062 
7063         /*
7064          * Clear ahciport_pending_tags bit when the corresponding command
7065          * is already sent down to the device.
7066          */
7067         while (issued_tags) {
7068                 issued_slot = ddi_ffs(issued_tags) - 1;
7069                 if (issued_slot == -1) {
7070                         goto next;
7071                 }
7072                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
7073                 CLEAR_BIT(issued_tags, issued_slot);
7074         }
7075 
7076 next:
7077         while (finished_tags) {
7078                 finished_slot = ddi_ffs(finished_tags) - 1;
7079                 if (finished_slot == -1) {
7080                         goto out;
7081                 }
7082 
7083                 /* The command is certainly transmitted to the device */
7084                 ASSERT(!(ahci_portp->ahciport_pending_tags &
7085                     (0x1 << finished_slot)));
7086 
7087                 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
7088                 ASSERT(satapkt != NULL);
7089 
7090                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
7091                     "ahci_intr_set_device_bits: sending up pkt 0x%p "
7092                     "with SATA_PKT_COMPLETED", (void *)satapkt);
7093 
7094                 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
7095                 CLEAR_BIT(finished_tags, finished_slot);
7096                 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
7097 
7098                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
7099         }
7100 out:
7101         AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
7102             "ahci_intr_set_device_bits: port %d "
7103             "pending_ncq_tags = 0x%x pending_tags = 0x%x",
7104             port, ahci_portp->ahciport_pending_ncq_tags,
7105             ahci_portp->ahciport_pending_tags);
7106 
7107         ahci_flush_doneq(ahci_portp);
7108 
7109         mutex_exit(&ahci_portp->ahciport_mutex);
7110 
7111         return (AHCI_SUCCESS);
7112 }
7113 
7114 /*
7115  * Port multiplier asynchronous notification event handler. Called upon a
7116  * device is hot plugged/pulled.
7117  *
7118  * The async-notification event will only be recorded by ahcipmi_snotif_tags
7119  * here and will be handled by ahci_probe_pmult().
7120  *
7121  * NOTE: called only from ahci_port_intr().
7122  */
7123 static int
7124 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7125     uint8_t port)
7126 {
7127         sata_device_t sdevice;
7128 
7129         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7130             "ahci_intr_pmult_sntf_events enter: port %d ", port);
7131 
7132         /* no hot-plug while attaching process */
7133         mutex_enter(&ahci_ctlp->ahcictl_mutex);
7134         if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
7135                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7136                 return (AHCI_SUCCESS);
7137         }
7138         mutex_exit(&ahci_ctlp->ahcictl_mutex);
7139 
7140         mutex_enter(&ahci_portp->ahciport_mutex);
7141         if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
7142                 mutex_exit(&ahci_portp->ahciport_mutex);
7143                 return (AHCI_SUCCESS);
7144         }
7145 
7146         ASSERT(ahci_portp->ahciport_pmult_info != NULL);
7147 
7148         ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
7149             ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7150             (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
7151         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7152             (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
7153             AHCI_SNOTIF_CLEAR_ALL);
7154 
7155         if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
7156                 mutex_exit(&ahci_portp->ahciport_mutex);
7157                 return (AHCI_SUCCESS);
7158         }
7159 
7160         /* Port Multiplier sub-device hot-plug handler */
7161         if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
7162                 mutex_exit(&ahci_portp->ahciport_mutex);
7163                 return (AHCI_SUCCESS);
7164         }
7165 
7166         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
7167                 /* Not allowed to re-enter. */
7168                 mutex_exit(&ahci_portp->ahciport_mutex);
7169                 return (AHCI_SUCCESS);
7170         }
7171 
7172         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;
7173 
7174         /*
7175          * NOTE:
7176          * Even if Asynchronous Notification is supported (and enabled) by
7177          * both controller and the port multiplier, the content of PxSNTF
7178          * register is always set to 0x8000 by async notification event. We
7179          * need to check GSCR[32] on the port multiplier to find out the
7180          * owner of this event.
7181          * This is not accord with SATA spec 2.6 and needs further
7182          * clarification.
7183          */
7184         /* hot-plug will not reported while reseting. */
7185         if (ahci_portp->ahciport_reset_in_progress == 1) {
7186                 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7187                     "port %d snotif event ignored", port);
7188                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7189                 mutex_exit(&ahci_portp->ahciport_mutex);
7190                 return (AHCI_SUCCESS);
7191         }
7192 
7193         AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
7194             "PxSNTF is set to 0x%x by port multiplier",
7195             ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);
7196 
7197         /*
7198          * Now we need do some necessary operation and inform SATA framework
7199          * that link/device events has happened.
7200          */
7201         bzero((void *)&sdevice, sizeof (sata_device_t));
7202         sdevice.satadev_addr.cport = ahci_ctlp->
7203             ahcictl_port_to_cport[port];
7204         sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
7205         sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
7206         sdevice.satadev_state = SATA_PSTATE_PWRON;
7207 
7208         /* Just reject packets, do not stop that port. */
7209         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7210 
7211         mutex_exit(&ahci_portp->ahciport_mutex);
7212         sata_hba_event_notify(
7213             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7214             &sdevice,
7215             SATA_EVNT_PMULT_LINK_CHANGED);
7216         mutex_enter(&ahci_portp->ahciport_mutex);
7217 
7218         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
7219         mutex_exit(&ahci_portp->ahciport_mutex);
7220 
7221         return (AHCI_SUCCESS);
7222 }
7223 
7224 /*
7225  * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
7226  * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
7227  * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
7228  * indicates a COMINIT signal was received.
7229  *
7230  * Hot plug insertion is detected by reception of a COMINIT signal from the
7231  * device. On reception of unsolicited COMINIT, the HBA shall generate a
7232  * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
7233  * begin the normal communication negotiation sequence as outlined in the
7234  * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
7235  * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
7236  * PxSSTS.DET field shall be set to 1h. When the communication negotiation
7237  * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
7238  * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
7239  * to handle hot plug insertion. In this interrupt handler, just do nothing
7240  * but print some log message and clear the bit.
7241  */
7242 static int
7243 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
7244     ahci_port_t *ahci_portp, uint8_t port)
7245 {
7246 #if AHCI_DEBUG
7247         uint32_t port_serror;
7248 #endif
7249 
7250         mutex_enter(&ahci_portp->ahciport_mutex);
7251 
7252 #if AHCI_DEBUG
7253         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7254             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7255 
7256         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7257             "ahci_intr_port_connect_change: port %d, "
7258             "port_serror = 0x%x", port, port_serror);
7259 #endif
7260 
7261         /* Clear PxSERR.DIAG.X to clear the interrupt bit */
7262         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7263             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7264             SERROR_EXCHANGED_ERR);
7265 
7266         mutex_exit(&ahci_portp->ahciport_mutex);
7267 
7268         return (AHCI_SUCCESS);
7269 }
7270 
7271 /*
7272  * Hot Plug Operation for platforms that support Mechanical Presence
7273  * Switches.
7274  *
7275  * When set, it indicates that a mechanical presence switch attached to this
7276  * port has been opened or closed, which may lead to a change in the connection
7277  * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
7278  * are set to '1'.
7279  *
7280  * At the moment, this interrupt is not needed and disabled and we just log
7281  * the debug message.
7282  */
7283 static int
7284 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
7285     ahci_port_t *ahci_portp, uint8_t port)
7286 {
7287         uint32_t cap_status, port_cmd_status;
7288 
7289         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7290             "ahci_intr_device_mechanical_presence_status enter, "
7291             "port %d", port);
7292 
7293         cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7294             (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
7295 
7296         mutex_enter(&ahci_portp->ahciport_mutex);
7297         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7298             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7299 
7300         if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
7301             !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
7302                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7303                     "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
7304                     "the interrupt: cap_status = 0x%x, "
7305                     "port_cmd_status = 0x%x", cap_status, port_cmd_status);
7306                 mutex_exit(&ahci_portp->ahciport_mutex);
7307 
7308                 return (AHCI_SUCCESS);
7309         }
7310 
7311 #if AHCI_DEBUG
7312         if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
7313                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7314                     "The mechanical presence switch is open: "
7315                     "port %d, port_cmd_status = 0x%x",
7316                     port, port_cmd_status);
7317         } else {
7318                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7319                     "The mechanical presence switch is close: "
7320                     "port %d, port_cmd_status = 0x%x",
7321                     port, port_cmd_status);
7322         }
7323 #endif
7324 
7325         mutex_exit(&ahci_portp->ahciport_mutex);
7326 
7327         return (AHCI_SUCCESS);
7328 }
7329 
7330 /*
7331  * Native Hot Plug Support.
7332  *
7333  * When set, it indicates that the internal PHYRDY signal changed state.
7334  * This bit reflects the state of PxSERR.DIAG.N.
7335  *
7336  * There are three kinds of conditions to generate this interrupt event:
7337  * 1. a device is inserted
7338  * 2. a device is disconnected
7339  * 3. when the link enters/exits a Partial or Slumber interface power
7340  *    management state
7341  *
7342  * If inteface power management is enabled for a port, the PxSERR.DIAG.N
7343  * bit may be set due to the link entering the Partial or Slumber power
7344  * management state, rather than due to a hot plug insertion or removal
7345  * event. So far, the interface power management is disabled, so the
7346  * driver can reliably get removal detection notification via the
7347  * PxSERR.DIAG.N bit.
7348  */
7349 static int
7350 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
7351     ahci_port_t *ahci_portp, uint8_t port)
7352 {
7353         uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
7354         sata_device_t sdevice;
7355         int dev_exists_now = 0;
7356         int dev_existed_previously = 0;
7357         ahci_addr_t port_addr;
7358 
7359         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
7360             "ahci_intr_phyrdy_change enter, port %d", port);
7361 
7362         /* Clear PxSERR.DIAG.N to clear the interrupt bit */
7363         mutex_enter(&ahci_portp->ahciport_mutex);
7364         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7365             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7366             SERROR_PHY_RDY_CHG);
7367         mutex_exit(&ahci_portp->ahciport_mutex);
7368 
7369         mutex_enter(&ahci_ctlp->ahcictl_mutex);
7370         if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
7371             (ahci_portp == NULL)) {
7372                 /* The whole controller setup is not yet done. */
7373                 mutex_exit(&ahci_ctlp->ahcictl_mutex);
7374                 return (AHCI_SUCCESS);
7375         }
7376         mutex_exit(&ahci_ctlp->ahcictl_mutex);
7377 
7378         mutex_enter(&ahci_portp->ahciport_mutex);
7379 
7380         /* SStatus tells the presence of device. */
7381         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7382             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
7383 
7384         if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
7385                 dev_exists_now = 1;
7386         }
7387 
7388         if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
7389                 dev_existed_previously = 1;
7390         }
7391 
7392         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
7393                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
7394                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7395                     "ahci_intr_phyrdy_change: port %d "
7396                     "AHCI_PORT_FLAG_NODEV is cleared", port);
7397                 if (dev_exists_now == 0)
7398                         dev_existed_previously = 1;
7399         }
7400 
7401         bzero((void *)&sdevice, sizeof (sata_device_t));
7402         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7403         sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7404         sdevice.satadev_addr.pmport = 0;
7405         sdevice.satadev_state = SATA_PSTATE_PWRON;
7406         ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
7407 
7408         AHCI_ADDR_SET_PORT(&port_addr, port);
7409 
7410         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
7411         if (dev_exists_now) {
7412                 if (dev_existed_previously) { /* 1 -> 1 */
7413                         /* Things are fine now. The loss was temporary. */
7414                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7415                             "ahci_intr_phyrdy_change  port %d "
7416                             "device link lost/established", port);
7417 
7418                         mutex_exit(&ahci_portp->ahciport_mutex);
7419                         sata_hba_event_notify(
7420                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7421                             &sdevice,
7422                             SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
7423                         mutex_enter(&ahci_portp->ahciport_mutex);
7424 
7425                 } else { /* 0 -> 1 */
7426                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7427                             "ahci_intr_phyrdy_change: port %d "
7428                             "device link established", port);
7429 
7430                         /*
7431                          * A new device has been detected. The new device
7432                          * might be a port multiplier instead of a drive, so
7433                          * we cannot update the signature directly.
7434                          */
7435                         (void) ahci_initialize_port(ahci_ctlp,
7436                             ahci_portp, &port_addr);
7437 
7438                         /* Try to start the port */
7439                         if (ahci_start_port(ahci_ctlp, ahci_portp, port)
7440                             != AHCI_SUCCESS) {
7441                                 sdevice.satadev_state |= SATA_PSTATE_FAILED;
7442                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7443                                     "ahci_intr_phyrdy_change: port %d failed "
7444                                     "at start port", port);
7445                         }
7446 
7447                         /* Clear the max queue depth for inserted device */
7448                         ahci_portp->ahciport_max_ncq_tags = 0;
7449 
7450                         mutex_exit(&ahci_portp->ahciport_mutex);
7451                         sata_hba_event_notify(
7452                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7453                             &sdevice,
7454                             SATA_EVNT_LINK_ESTABLISHED);
7455                         mutex_enter(&ahci_portp->ahciport_mutex);
7456 
7457                 }
7458         } else { /* No device exists now */
7459 
7460                 if (dev_existed_previously) { /* 1 -> 0 */
7461                         AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
7462                             "ahci_intr_phyrdy_change: port %d "
7463                             "device link lost", port);
7464 
7465                         ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
7466                         (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
7467                             ahci_portp, port);
7468 
7469                         if (ahci_portp->ahciport_device_type ==
7470                             SATA_DTYPE_PMULT) {
7471                                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
7472                         }
7473 
7474                         /* An existing device is lost. */
7475                         ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
7476                         ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;
7477 
7478                         mutex_exit(&ahci_portp->ahciport_mutex);
7479                         sata_hba_event_notify(
7480                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7481                             &sdevice,
7482                             SATA_EVNT_LINK_LOST);
7483                         mutex_enter(&ahci_portp->ahciport_mutex);
7484                 }
7485         }
7486         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
7487 
7488         mutex_exit(&ahci_portp->ahciport_mutex);
7489 
7490         return (AHCI_SUCCESS);
7491 }
7492 
7493 /*
7494  * PxIS.UFS - Unknown FIS Error
7495  *
7496  * This interrupt event means an unknown FIS was received and has been
7497  * copied into system memory. An unknown FIS is not considered an illegal
7498  * FIS, unless the length received is more than 64 bytes. If an unknown
7499  * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
7500  * normal operation. If the unknown FIS is more than 64 bytes, then it
7501  * won't be posted to memory and PxSERR.ERR.P will be set, which is then
7502  * a fatal error.
7503  *
7504  * PxIS.IPMS - Incorrect Port Multiplier Status
7505  *
7506  * IPMS Indicates that the HBA received a FIS from a device that did not
7507  * have a command outstanding. The IPMS bit may be set during enumeration
7508  * of devices on a Port Multiplier due to the normal Port Multiplier
7509  * enumeration process. It is recommended that IPMS only be used after
7510  * enumeration is complete on the Port Multiplier (copied from spec).
7511  *
7512  * PxIS.OFS - Overflow Error
7513  *
7514  * Command list overflow is defined as software building a command table
7515  * that has fewer total bytes than the transaction given to the device.
7516  * On device writes, the HBA will run out of data, and on reads, there
7517  * will be no room to put the data.
7518  *
7519  * For an overflow on data read, either PIO or DMA, the HBA will set
7520  * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
7521  * non-fatal error when the HBA can continues. Sometimes, it will cause
7522  * a fatal error and need the software to do something.
7523  *
7524  * For an overflow on data write, setting PxIS.OFS is optional for both
7525  * DMA and PIO, and it's a fatal error, and a COMRESET is required by
7526  * software to clean up from this serious error.
7527  *
7528  * PxIS.INFS - Interface Non-Fatal Error
7529  *
7530  * This interrupt event indicates that the HBA encountered an error on
7531  * the Serial ATA interface but was able to continue operation. The kind
7532  * of error usually occurred during a non-Data FIS, and under this condition
7533  * the FIS will be re-transmitted by HBA automatically.
7534  *
7535  * When the FMA is implemented, there should be a stat structure to
7536  * record how many every kind of error happens.
7537  */
7538 static int
7539 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
7540     uint8_t port, uint32_t intr_status)
7541 {
7542         uint32_t port_serror;
7543 #if AHCI_DEBUG
7544         uint32_t port_cmd_status;
7545         uint32_t port_cmd_issue;
7546         uint32_t port_sactive;
7547         int current_slot;
7548         uint32_t current_tags;
7549         sata_pkt_t *satapkt;
7550         ahci_cmd_header_t *cmd_header;
7551         uint32_t cmd_dmacount;
7552 #endif
7553 
7554         mutex_enter(&ahci_portp->ahciport_mutex);
7555 
7556         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7557             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7558 
7559         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
7560             "ahci_intr_non_fatal_error: port %d, "
7561             "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);
7562 
7563         ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);
7564 
7565         if (intr_status & AHCI_INTR_STATUS_UFS) {
7566                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
7567                     "ahci port %d has unknown FIS error", port);
7568 
7569                 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
7570                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7571                     (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
7572                     SERROR_FIS_TYPE);
7573         }
7574 
7575 #if AHCI_DEBUG
7576         if (intr_status & AHCI_INTR_STATUS_IPMS) {
7577                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
7578                     "has Incorrect Port Multiplier Status error", port);
7579         }
7580 
7581         if (intr_status & AHCI_INTR_STATUS_OFS) {
7582                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7583                     "ahci port %d has overflow error", port);
7584         }
7585 
7586         if (intr_status & AHCI_INTR_STATUS_INFS) {
7587                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7588                     "ahci port %d has interface non fatal error", port);
7589         }
7590 
7591         /*
7592          * Record the error occurred command's slot.
7593          */
7594         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
7595             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7596                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7597                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7598 
7599                 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7600                     AHCI_CMD_STATUS_CCS_SHIFT;
7601 
7602                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
7603                         satapkt = ahci_portp->ahciport_err_retri_pkt;
7604                         ASSERT(satapkt != NULL);
7605                         ASSERT(current_slot == 0);
7606                 } else {
7607                         satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7608                 }
7609 
7610                 if (satapkt != NULL) {
7611                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7612                             "ahci_intr_non_fatal_error: pending_tags = 0x%x "
7613                             "cmd 0x%x", ahci_portp->ahciport_pending_tags,
7614                             satapkt->satapkt_cmd.satacmd_cmd_reg);
7615 
7616                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7617                             "ahci_intr_non_fatal_error: port %d, "
7618                             "satapkt 0x%p is being processed when error occurs",
7619                             port, (void *)satapkt);
7620 
7621                         /*
7622                          * PRD Byte Count field of command header is not
7623                          * required to reflect the total number of bytes
7624                          * transferred when an overflow occurs, so here
7625                          * just log the value.
7626                          */
7627                         cmd_dmacount =
7628                             ahci_portp->ahciport_prd_bytecounts[current_slot];
7629                         if (cmd_dmacount) {
7630                                 cmd_header = &ahci_portp->
7631                                     ahciport_cmd_list[current_slot];
7632                                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7633                                     "ahci_intr_non_fatal_error: port %d, "
7634                                     "PRD Byte Count = 0x%x, "
7635                                     "ahciport_prd_bytecounts = 0x%x", port,
7636                                     cmd_header->ahcich_prd_byte_count,
7637                                     cmd_dmacount);
7638                         }
7639                 }
7640         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7641                 /*
7642                  * For queued command, list those command which have already
7643                  * been transmitted to the device and still not completed.
7644                  */
7645                 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7646                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7647 
7648                 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7649                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
7650 
7651                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
7652                     "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
7653                     "port_sactive = 0x%x port_cmd_issue = 0x%x",
7654                     ahci_portp->ahciport_pending_ncq_tags,
7655                     port_sactive, port_cmd_issue);
7656 
7657                 current_tags = ahci_portp->ahciport_pending_ncq_tags &
7658                     port_sactive & ~port_cmd_issue &
7659                     AHCI_NCQ_SLOT_MASK(ahci_portp);
7660 
7661                 while (current_tags) {
7662                         current_slot = ddi_ffs(current_tags) - 1;
7663                         if (current_slot == -1) {
7664                                 goto out;
7665                         }
7666 
7667                         satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
7668                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
7669                             ahci_ctlp, "ahci_intr_non_fatal_error: "
7670                             "port %d, satapkt 0x%p is outstanding when "
7671                             "error occurs", port, (void *)satapkt);
7672 
7673                         CLEAR_BIT(current_tags, current_slot);
7674                 }
7675         }
7676 out:
7677 #endif
7678         mutex_exit(&ahci_portp->ahciport_mutex);
7679 
7680         return (AHCI_SUCCESS);
7681 }
7682 
7683 /*
7684  * According to the AHCI spec, the error types include system memory
7685  * errors, interface errors, port multiplier errors, device errors,
7686  * command list overflow, command list underflow, native command
7687  * queuing tag errors and pio data transfer errors.
7688  *
7689  * System memory errors such as target abort, master abort, and parity
7690  * may cause the host to stop, and they are serious errors and needed
7691  * to be recovered with software intervention. When system software
7692  * has given a pointer to the HBA that doesn't exist in physical memory,
7693  * a master/target abort error occurs, and PxIS.HBFS will be set. A
7694  * data error such as CRC or parity occurs, the HBA aborts the transfer
7695  * (if necessary) and PxIS.HBDS will be set.
7696  *
7697  * Interface errors are errors that occur due to electrical issues on
7698  * the interface, or protocol miscommunication between the device and
7699  * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
7700  * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
7701  * causes PxIS.IFS/PxIS.INFS to be set are
7702  *      1. in PxSERR.ERR, P bit is set to '1'
7703  *      2. in PxSERR.DIAG, C or H bit is set to '1'
7704  *      3. PhyRdy drop unexpectly, N bit is set to '1'
7705  * If the error occurred during a non-data FIS, the FIS must be
7706  * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
7707  * the error occurred during a data FIS, the transfer will stop, so
7708  * the error is fatal and PxIS.IFS is set.
7709  *
7710  * When a FIS arrives that updates the taskfile, the HBA checks to see
7711  * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
7712  * stops processing any more commands.
7713  *
7714  * Command list overflow is defined as software building a command table
7715  * that has fewer total bytes than the transaction given to the device.
7716  * On device writes, the HBA will run out of data, and on reads, there
7717  * will be no room to put the data. For an overflow on data read, either
7718  * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
7719  * For an overflow on data write, setting PxIS.OFS is optional for both
7720  * DMA and PIO, and a COMRESET is required by software to clean up from
7721  * this serious error.
7722  *
7723  * Command list underflow is defined as software building a command
7724  * table that has more total bytes than the transaction given to the
7725  * device. For data writes, both PIO and DMA, the device will detect
7726  * an error and end the transfer. And these errors are most likely going
7727  * to be fatal errors that will cause the port to be restarted. For
7728  * data reads, the HBA updates its PRD byte count, and may be
7729  * able to continue normally, but is not required to. And The HBA is
7730  * not required to detect underflow conditions for native command
7731  * queuing command.
7732  *
7733  * The HBA does not actively check incoming DMA Setup FISes to ensure
7734  * that the PxSACT register bit for that slot is set. Existing error
7735  * mechanisms, such as host bus failure, or bad protocol, are used to
7736  * recover from this case.
7737  *
7738  * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
7739  * DATA FIS must be an integral number of Dwords. If the HBA receives
7740  * a request which is not an integral number of Dwords, the HBA
7741  * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
7742  * software restarts the port. And the HBA ensures that the size
7743  * of the DATA FIS received during a PIO command matches the size in
7744  * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
7745  * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
7746  * stop running until software restarts the port.
7747  */
7748 /*
7749  * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
7750  *
7751  * PxIS.IFS indicates that the hba encountered an error on the serial ata
7752  * interface which caused the transfer to stop.
7753  *
7754  * PxIS.HBDS indicates that the hba encountered a data error
7755  * (uncorrectable ecc/parity) when reading from or writing to system memory.
7756  *
7757  * PxIS.HBFS indicates that the hba encountered a host bus error that it
7758  * cannot recover from, such as a bad software pointer.
7759  *
7760  * PxIS.TFES is set whenever the status register is updated by the device
7761  * and the error bit (bit 0) is set.
7762  */
7763 static int
7764 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
7765     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
7766 {
7767         uint32_t port_cmd_status;
7768         uint32_t port_serror;
7769         uint32_t task_file_status;
7770         int failed_slot;
7771         sata_pkt_t *spkt = NULL;
7772         uint8_t err_byte;
7773         ahci_event_arg_t *args;
7774         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
7775         uint32_t failed_tags = 0;
7776         int task_fail_flag = 0, task_abort_flag = 0;
7777         uint32_t slot_status;
7778 
7779         mutex_enter(&ahci_portp->ahciport_mutex);
7780 
7781         /*
7782          * ahci_intr_phyrdy_change() may have rendered it to
7783          * SATA_DTYPE_NONE.
7784          */
7785         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
7786                 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
7787                     "ahci_intr_fatal_error: port %d no device attached, "
7788                     "and just return without doing anything", port);
7789                 goto out0;
7790         }
7791 
7792         if (intr_status & AHCI_INTR_STATUS_TFES) {
7793                 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7794                     (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
7795                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7796                     "ahci_intr_fatal_error: port %d "
7797                     "task_file_status = 0x%x", port, task_file_status);
7798                 task_fail_flag = 1;
7799 
7800                 err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
7801                     >> AHCI_TFD_ERR_SHIFT;
7802                 if (err_byte == SATA_ERROR_ABORT)
7803                         task_abort_flag = 1;
7804         }
7805 
7806         /*
7807          * Here we just log the fatal error info in interrupt context.
7808          * Misc recovery processing will be handled in task queue.
7809          */
7810         if (task_fail_flag  == 1) {
7811                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7812                         /*
7813                          * Read PxCMD.CCS to determine the slot that the HBA
7814                          * was processing when the error occurred.
7815                          */
7816                         port_cmd_status = ddi_get32(
7817                             ahci_ctlp->ahcictl_ahci_acc_handle,
7818                             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7819                         failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
7820                             AHCI_CMD_STATUS_CCS_SHIFT;
7821                         failed_tags = 0x1 << failed_slot;
7822 
7823                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
7824                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
7825                             "ahci_intr_fatal_error: spkt 0x%p is being "
7826                             "processed when fatal error occurred for port %d",
7827                             spkt, port);
7828 
7829                         /*
7830                          * Won't emit the error message if it is an IDENTIFY
7831                          * DEVICE command sent to an ATAPI device.
7832                          */
7833                         if ((spkt != NULL) &&
7834                             (spkt->satapkt_cmd.satacmd_cmd_reg ==
7835                             SATAC_ID_DEVICE) &&
7836                             (task_abort_flag == 1))
7837                         goto out1;
7838 
7839                         /*
7840                          * Won't emit the error message if it is an ATAPI PACKET
7841                          * command
7842                          */
7843                         if ((spkt != NULL) &&
7844                             (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
7845                                 goto out1;
7846 
7847                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
7848                         slot_status = ddi_get32(
7849                             ahci_ctlp->ahcictl_ahci_acc_handle,
7850                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
7851                         failed_tags = slot_status &
7852                             AHCI_NCQ_SLOT_MASK(ahci_portp);
7853                 }
7854         }
7855 
7856         /* print the fatal error type */
7857         ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
7858         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;
7859 
7860         port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7861             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
7862 
7863         /* print PxSERR related error message */
7864         ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);
7865 
7866         /* print task file register value */
7867         if (task_fail_flag == 1) {
7868                 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
7869                     "= 0x%x", instance, port, task_file_status);
7870                 if (task_abort_flag == 1) {
7871                         cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
7872                             "port %d are aborted", instance, port);
7873                         ahci_dump_commands(ahci_ctlp, port, failed_tags);
7874                 }
7875         }
7876 
7877 out1:
7878         /* Prepare the argument for the taskq */
7879         args = ahci_portp->ahciport_event_args;
7880         args->ahciea_ctlp = (void *)ahci_ctlp;
7881         args->ahciea_portp = (void *)ahci_portp;
7882         args->ahciea_event = intr_status;
7883         AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);
7884 
7885         /* Start the taskq to handle error recovery */
7886         if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
7887             ahci_events_handler,
7888             (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
7889                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
7890                 cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
7891                     "port %d failed", instance, port);
7892         }
7893 out0:
7894         mutex_exit(&ahci_portp->ahciport_mutex);
7895 
7896         return (AHCI_SUCCESS);
7897 }
7898 
7899 /*
7900  * Hot Plug Operation for platforms that support Cold Presence Detect.
7901  *
7902  * When set, a device status has changed as detected by the cold presence
7903  * detect logic. This bit can either be set due to a non-connected port
7904  * receiving a device, or a connected port having its device removed.
7905  * This bit is only valid if the port supports cold presence detect as
7906  * indicated by PxCMD.CPD set to '1'.
7907  *
7908  * At the moment, this interrupt is not needed and disabled and we just
7909  * log the debug message.
7910  */
7911 static int
7912 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
7913     ahci_port_t *ahci_portp, uint8_t port)
7914 {
7915         uint32_t port_cmd_status;
7916         sata_device_t sdevice;
7917 
7918         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7919             "ahci_intr_cold_port_detect enter, port %d", port);
7920 
7921         mutex_enter(&ahci_portp->ahciport_mutex);
7922 
7923         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
7924             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
7925         if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
7926                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7927                     "port %d does not support cold presence detect, so "
7928                     "we just ignore this interrupt", port);
7929                 mutex_exit(&ahci_portp->ahciport_mutex);
7930                 return (AHCI_SUCCESS);
7931         }
7932 
7933         AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7934             "port %d device status has changed", port);
7935 
7936         bzero((void *)&sdevice, sizeof (sata_device_t));
7937         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
7938         sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
7939         sdevice.satadev_addr.pmport = 0;
7940         sdevice.satadev_state = SATA_PSTATE_PWRON;
7941 
7942         if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
7943                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7944                     "port %d: a device is hot plugged", port);
7945                 mutex_exit(&ahci_portp->ahciport_mutex);
7946                 sata_hba_event_notify(
7947                     ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7948                     &sdevice,
7949                     SATA_EVNT_DEVICE_ATTACHED);
7950                 mutex_enter(&ahci_portp->ahciport_mutex);
7951 
7952         } else {
7953                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
7954                     "port %d: a device is hot unplugged", port);
7955                 mutex_exit(&ahci_portp->ahciport_mutex);
7956                 sata_hba_event_notify(
7957                     ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
7958                     &sdevice,
7959                     SATA_EVNT_DEVICE_DETACHED);
7960                 mutex_enter(&ahci_portp->ahciport_mutex);
7961         }
7962 
7963         mutex_exit(&ahci_portp->ahciport_mutex);
7964 
7965         return (AHCI_SUCCESS);
7966 }
7967 
7968 /*
7969  * Enable the interrupts for a particular port.
7970  */
7971 static void
7972 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
7973 {
7974         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
7975 
7976         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
7977             "ahci_enable_port_intrs enter, port %d", port);
7978 
7979         /*
7980          * Clear port interrupt status before enabling interrupt
7981          */
7982         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7983             (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
7984             AHCI_PORT_INTR_MASK);
7985 
7986         /*
7987          * Clear the pending bit from IS.IPS
7988          */
7989         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
7990             (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
7991 
7992         /*
7993          * Enable the following interrupts:
7994          *      Device to Host Register FIS Interrupt (DHRS)
7995          *      PIO Setup FIS Interrupt (PSS)
7996          *      Set Device Bits Interrupt (SDBS)
7997          *      Unknown FIS Interrupt (UFS)
7998          *      Port Connect Change Status (PCS)
7999          *      PhyRdy Change Status (PRCS)
8000          *      Overflow Status (OFS)
8001          *      Interface Non-fatal Error Status (INFS)
8002          *      Interface Fatal Error Status (IFS)
8003          *      Host Bus Data Error Status (HBDS)
8004          *      Host Bus Fatal Error Status (HBFS)
8005          *      Task File Error Status (TFES)
8006          */
8007         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8008             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
8009             (AHCI_INTR_STATUS_DHRS |
8010             AHCI_INTR_STATUS_PSS |
8011             AHCI_INTR_STATUS_SDBS |
8012             AHCI_INTR_STATUS_UFS |
8013             AHCI_INTR_STATUS_DPS |
8014             AHCI_INTR_STATUS_PCS |
8015             AHCI_INTR_STATUS_PRCS |
8016             AHCI_INTR_STATUS_OFS |
8017             AHCI_INTR_STATUS_INFS |
8018             AHCI_INTR_STATUS_IFS |
8019             AHCI_INTR_STATUS_HBDS |
8020             AHCI_INTR_STATUS_HBFS |
8021             AHCI_INTR_STATUS_TFES));
8022 }
8023 
8024 /*
8025  * Enable interrupts for all the ports.
8026  */
8027 static void
8028 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
8029 {
8030         uint32_t ghc_control;
8031 
8032         ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8033 
8034         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);
8035 
8036         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8037             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8038 
8039         ghc_control |= AHCI_HBA_GHC_IE;
8040 
8041         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8042             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8043 }
8044 
8045 /*
8046  * Disable interrupts for a particular port.
8047  */
8048 static void
8049 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
8050 {
8051         ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8052             MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8053 
8054         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8055             "ahci_disable_port_intrs enter, port %d", port);
8056 
8057         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8058             (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
8059 }
8060 
8061 /*
8062  * Disable interrupts for the whole HBA.
8063  *
8064  * The global bit is cleared, then all interrupt sources from all
8065  * ports are disabled.
8066  */
8067 static void
8068 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
8069 {
8070         uint32_t ghc_control;
8071 
8072         ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
8073             MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
8074 
8075         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
8076             NULL);
8077 
8078         ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8079             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
8080 
8081         ghc_control &= ~AHCI_HBA_GHC_IE;
8082 
8083         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8084             (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
8085 }
8086 
8087 /*
8088  * Handle FIXED or MSI interrupts.
8089  */
8090 /*
8091  * According to AHCI spec, the HBA may support several interrupt modes:
8092  *      * pin based interrupts (FIXED)
8093  *      * single MSI message interrupts
8094  *      * multiple MSI based message interrupts
8095  *
8096  * For pin based interrupts, the software interrupt handler need to check IS
8097  * register to find out which port has pending interrupts. And then check
8098  * PxIS register to find out which interrupt events happened on that port.
8099  *
8100  * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
8101  * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
8102  * in that software interrupt handler need to check IS register to determine
8103  * which port triggered the interrupts since it uses a single message for all
8104  * port interrupts.
8105  *
8106  * HBA may optionally support multiple MSI message for better performance. In
8107  * this mode, each port may have its own interrupt message, and thus generation
8108  * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
8109  * represents a power-of-2 wrapper on the number of implemented ports, and
8110  * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
8111  * maximum number of assigned interrupts. When the number of MSI messages
8112  * allocated is less than the number requested, then hardware may have two
8113  * implementation behaviors:
8114  *      * assign each ports its own interrupt and then force all additional
8115  *        ports to share the last interrupt message, and this condition is
8116  *        indicated by clearing GHC.MRSM to '0'
8117  *      * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
8118  * When multiple-message MSI is enabled, hardware will still set IS register
8119  * as single message case. And this IS register may be used by software when
8120  * fewer than the requested number of messages is granted in order to determine
8121  * which port had the interrupt.
8122  *
8123  * Note: The current ahci driver only supports the first two interrupt modes:
8124  * pin based interrupts and single MSI message interrupts, and the reason
8125  * is indicated in below code.
8126  */
8127 static int
8128 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
8129 {
8130         dev_info_t *dip = ahci_ctlp->ahcictl_dip;
8131         int             count, avail, actual;
8132         int             i, rc;
8133 
8134         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
8135             "ahci_add_intrs enter interrupt type 0x%x", intr_type);
8136 
8137         /* get number of interrupts. */
8138         rc = ddi_intr_get_nintrs(dip, intr_type, &count);
8139         if ((rc != DDI_SUCCESS) || (count == 0)) {
8140                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8141                     "ddi_intr_get_nintrs() failed, "
8142                     "rc %d count %d\n", rc, count);
8143                 return (DDI_FAILURE);
8144         }
8145 
8146         /* get number of available interrupts. */
8147         rc = ddi_intr_get_navail(dip, intr_type, &avail);
8148         if ((rc != DDI_SUCCESS) || (avail == 0)) {
8149                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8150                     "ddi_intr_get_navail() failed, "
8151                     "rc %d avail %d\n", rc, avail);
8152                 return (DDI_FAILURE);
8153         }
8154 
8155 #if AHCI_DEBUG
8156         if (avail < count) {
8157                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8158                     "ddi_intr_get_nintrs returned %d, navail() returned %d",
8159                     count, avail);
8160         }
8161 #endif
8162 
8163         /*
8164          * Note: So far Solaris restricts the maximum number of messages for
8165          * x86 to 2, that is avail is 2, so here we set the count with 1 to
8166          * force the driver to use single MSI message interrupt. In future if
8167          * Solaris remove the restriction, then we need to delete the below
8168          * code and try to use multiple interrupt routine to gain better
8169          * performance.
8170          */
8171         if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
8172                 AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
8173                     "force to use one interrupt routine though the "
8174                     "HBA supports %d interrupt", count);
8175                 count = 1;
8176         }
8177 
8178         /* Allocate an array of interrupt handles. */
8179         ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
8180         ahci_ctlp->ahcictl_intr_htable =
8181             kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
8182 
8183         /* call ddi_intr_alloc(). */
8184         rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
8185             intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);
8186 
8187         if ((rc != DDI_SUCCESS) || (actual == 0)) {
8188                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8189                     "ddi_intr_alloc() failed, rc %d count %d actual %d "
8190                     "avail %d\n", rc, count, actual, avail);
8191                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8192                     ahci_ctlp->ahcictl_intr_size);
8193                 return (DDI_FAILURE);
8194         }
8195 
8196         /* use interrupt count returned */
8197 #if AHCI_DEBUG
8198         if (actual < count) {
8199                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8200                     "Requested: %d, Received: %d", count, actual);
8201         }
8202 #endif
8203 
8204         ahci_ctlp->ahcictl_intr_cnt = actual;
8205 
8206         /*
8207          * Get priority for first, assume remaining are all the same.
8208          */
8209         if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
8210             &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
8211                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8212                     "ddi_intr_get_pri() failed", NULL);
8213 
8214                 /* Free already allocated intr. */
8215                 for (i = 0; i < actual; i++) {
8216                         (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8217                 }
8218 
8219                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8220                     ahci_ctlp->ahcictl_intr_size);
8221                 return (DDI_FAILURE);
8222         }
8223 
8224         /* Test for high level interrupt. */
8225         if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
8226                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8227                     "ahci_add_intrs: Hi level intr not supported", NULL);
8228 
8229                 /* Free already allocated intr. */
8230                 for (i = 0; i < actual; i++) {
8231                         (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
8232                 }
8233 
8234                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8235                     sizeof (ddi_intr_handle_t));
8236 
8237                 return (DDI_FAILURE);
8238         }
8239 
8240         /* Call ddi_intr_add_handler(). */
8241         for (i = 0; i < actual; i++) {
8242                 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
8243                     ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
8244                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8245                             "ddi_intr_add_handler() failed", NULL);
8246 
8247                         /* Free already allocated intr. */
8248                         for (i = 0; i < actual; i++) {
8249                                 (void) ddi_intr_free(
8250                                     ahci_ctlp->ahcictl_intr_htable[i]);
8251                         }
8252 
8253                         kmem_free(ahci_ctlp->ahcictl_intr_htable,
8254                             ahci_ctlp->ahcictl_intr_size);
8255                         return (DDI_FAILURE);
8256                 }
8257         }
8258 
8259         if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
8260             &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
8261                 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
8262                     "ddi_intr_get_cap() failed", NULL);
8263 
8264                 /* Free already allocated intr. */
8265                 for (i = 0; i < actual; i++) {
8266                         (void) ddi_intr_free(
8267                             ahci_ctlp->ahcictl_intr_htable[i]);
8268                 }
8269 
8270                 kmem_free(ahci_ctlp->ahcictl_intr_htable,
8271                     ahci_ctlp->ahcictl_intr_size);
8272                 return (DDI_FAILURE);
8273         }
8274 
8275         if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
8276                 /* Call ddi_intr_block_enable() for MSI. */
8277                 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
8278                     ahci_ctlp->ahcictl_intr_cnt);
8279         } else {
8280                 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
8281                 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
8282                         (void) ddi_intr_enable(
8283                             ahci_ctlp->ahcictl_intr_htable[i]);
8284                 }
8285         }
8286 
8287         return (DDI_SUCCESS);
8288 }
8289 
8290 /*
8291  * Removes the registered interrupts irrespective of whether they
8292  * were legacy or MSI.
8293  *
8294  * NOTE: The controller interrupts must be disabled before calling
8295  * this routine.
8296  */
8297 static void
8298 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
8299 {
8300         int x;
8301 
8302         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);
8303 
8304         /* Disable all interrupts. */
8305         if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
8306             (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
8307                 /* Call ddi_intr_block_disable(). */
8308                 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
8309                     ahci_ctlp->ahcictl_intr_cnt);
8310         } else {
8311                 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8312                         (void) ddi_intr_disable(
8313                             ahci_ctlp->ahcictl_intr_htable[x]);
8314                 }
8315         }
8316 
8317         /* Call ddi_intr_remove_handler(). */
8318         for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
8319                 (void) ddi_intr_remove_handler(
8320                     ahci_ctlp->ahcictl_intr_htable[x]);
8321                 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
8322         }
8323 
8324         kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
8325 }
8326 
8327 /*
8328  * This routine tries to put port into P:NotRunning state by clearing
8329  * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
8330  * and PxCMD.CR to '0'.
8331  */
8332 static int
8333 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
8334     ahci_port_t *ahci_portp, uint8_t port)
8335 {
8336         uint32_t port_cmd_status;
8337         int loop_count;
8338 
8339         ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
8340             MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));
8341 
8342         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8343             "ahci_put_port_into_notrunning_state enter: port %d", port);
8344 
8345         port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8346             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8347 
8348         port_cmd_status &= ~AHCI_CMD_STATUS_ST;
8349         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8350             (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
8351 
8352         /* Wait until PxCMD.CR is cleared */
8353         loop_count = 0;
8354         do {
8355                 port_cmd_status =
8356                     ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8357                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
8358 
8359                 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
8360                         AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
8361                             "clearing port %d CMD.CR timeout, "
8362                             "port_cmd_status = 0x%x", port,
8363                             port_cmd_status);
8364                         /*
8365                          * We are effectively timing out after 0.5 sec.
8366                          * This value is specified in AHCI spec.
8367                          */
8368                         break;
8369                 }
8370 
8371                 /* Wait for 10 millisec */
8372                 drv_usecwait(AHCI_10MS_USECS);
8373         } while (port_cmd_status & AHCI_CMD_STATUS_CR);
8374 
8375         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
8376 
8377         if (port_cmd_status & AHCI_CMD_STATUS_CR) {
8378                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8379                     "ahci_put_port_into_notrunning_state: failed to clear "
8380                     "PxCMD.CR to '0' after loop count: %d, and "
8381                     "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8382                 return (AHCI_FAILURE);
8383         } else {
8384                 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
8385                     "ahci_put_port_into_notrunning_state: succeeded to clear "
8386                     "PxCMD.CR to '0' after loop count: %d, and "
8387                     "port_cmd_status = 0x%x", loop_count, port_cmd_status);
8388                 return (AHCI_SUCCESS);
8389         }
8390 }
8391 
8392 /*
8393  * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
8394  * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
8395  * stable state, then set PxCMD.ST to '1' to start the port directly.
8396  * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
8397  * COMRESET to the device to put it in an idle state.
8398  *
8399  * The fifth argument returns whether the port reset is involved during
8400  * the process.
8401  *
8402  * The routine will be called under following scenarios:
8403  *      + To reset the HBA
8404  *      + To abort the packet(s)
8405  *      + To reset the port
8406  *      + To activate the port
8407  *      + Fatal error recovery
8408  *      + To abort the timeout packet(s)
8409  *
8410  * NOTES!!! During this procedure, PxSERR register will be cleared, and
8411  * according to the spec, the clearance of three bits will also clear
8412  * three interrupt status bits.
8413  *      1. PxSERR.DIAG.F will clear PxIS.UFS
8414  *      2. PxSERR.DIAG.X will clear PxIS.PCS
8415  *      3. PxSERR.DIAG.N will clear PxIS.PRCS
8416  *
8417  * Among these three interrupt events, the driver needs to take care of
8418  * PxIS.PRCS, which is the hot plug event. When the driver found out
8419  * a device was unplugged, it will call the interrupt handler.
8420  */
8421 static int
8422 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
8423     ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
8424 {
8425         uint32_t port_sstatus;
8426         uint32_t task_file_status;
8427         sata_device_t sdevice;
8428         int rval;
8429         ahci_addr_t addr_port;
8430         ahci_pmult_info_t *pminfo = NULL;
8431         int dev_exists_begin = 0;
8432         int dev_exists_end = 0;
8433         uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
8434         int npmport = 0;
8435         uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];
8436 
8437         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8438 
8439         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
8440             "ahci_restart_port_wait_till_ready: port %d enter", port);
8441 
8442         AHCI_ADDR_SET_PORT(&addr_port, port);
8443 
8444         if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
8445                 dev_exists_begin = 1;
8446 
8447         /* First clear PxCMD.ST */
8448         rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
8449             port);
8450         if (rval != AHCI_SUCCESS)
8451                 /*
8452                  * If PxCMD.CR does not clear within a reasonable time, it
8453                  * may assume the interface is in a hung condition and may
8454                  * continue with issuing the port reset.
8455                  */
8456                 goto reset;
8457 
8458         /* Then clear PxSERR */
8459         ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
8460             (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
8461             AHCI_SERROR_CLEAR_ALL);
8462 
8463         /* Then get PxTFD */
8464         task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8465             (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
8466 
8467         /*
8468          * Check whether the device is in a stable status, if yes,
8469          * then start the port directly. However for ahci_tran_reset_dport,
8470          * we may have to perform a port reset.
8471          */
8472         if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
8473             !(flag & AHCI_PORT_RESET))
8474                 goto out;
8475 
8476 reset:
8477         /*
8478          * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
8479          * a COMRESET to the device
8480          */
8481         ahci_disable_port_intrs(ahci_ctlp, port);
8482         rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
8483         ahci_enable_port_intrs(ahci_ctlp, port);
8484 
8485 #ifdef AHCI_DEBUG
8486         if (rval != AHCI_SUCCESS)
8487                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8488                     "ahci_restart_port_wait_till_ready: port %d failed",
8489                     port);
8490 #endif
8491 
8492         if (reset_flag != NULL)
8493                 *reset_flag = 1;
8494 
8495         /* Indicate to the framework that a reset has happened. */
8496         if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
8497             (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
8498             !(flag & AHCI_RESET_NO_EVENTS_UP)) {
8499                 /* Set the reset in progress flag */
8500                 ahci_portp->ahciport_reset_in_progress = 1;
8501 
8502                 bzero((void *)&sdevice, sizeof (sata_device_t));
8503                 sdevice.satadev_addr.cport =
8504                     ahci_ctlp->ahcictl_port_to_cport[port];
8505                 sdevice.satadev_addr.pmport = 0;
8506                 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8507 
8508                 sdevice.satadev_state = SATA_DSTATE_RESET |
8509                     SATA_DSTATE_PWR_ACTIVE;
8510                 if (ahci_ctlp->ahcictl_sata_hba_tran) {
8511                         mutex_exit(&ahci_portp->ahciport_mutex);
8512                         sata_hba_event_notify(
8513                             ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
8514                             &sdevice,
8515                             SATA_EVNT_DEVICE_RESET);
8516                         mutex_enter(&ahci_portp->ahciport_mutex);
8517                 }
8518 
8519                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8520                     "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8521         } else {
8522                 ahci_portp->ahciport_reset_in_progress = 0;
8523         }
8524 
8525 out:
8526         (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8527 
8528         /* SStatus tells the presence of device. */
8529         port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
8530             (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
8531 
8532         if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
8533                 dev_exists_end = 1;
8534         }
8535 
8536         if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
8537                 return (rval);
8538 
8539         /* Check whether a hot plug event happened */
8540         if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
8541                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8542                     "ahci_restart_port_wait_till_ready: port %d "
8543                     "device is removed", port);
8544                 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
8545                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8546                     "ahci_restart_port_wait_till_ready: port %d "
8547                     "AHCI_PORT_FLAG_NODEV flag is set", port);
8548                 mutex_exit(&ahci_portp->ahciport_mutex);
8549                 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
8550                 mutex_enter(&ahci_portp->ahciport_mutex);
8551 
8552                 return (rval);
8553         }
8554 
8555 
8556         /* 0/1 -> 1 : device may change */
8557         /*
8558          * May be called by ahci_fatal_error_recovery_handler, so
8559          * don't issue software if the previous device is ATAPI.
8560          */
8561         if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
8562                 return (rval);
8563 
8564         /*
8565          * The COMRESET will make port multiplier enter legacy mode.
8566          * Issue a software reset to make it work again.
8567          */
8568         ahci_disable_port_intrs(ahci_ctlp, port);
8569         ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
8570         ahci_enable_port_intrs(ahci_ctlp, port);
8571 
8572         /*
8573          * Following codes are specific for the port multiplier
8574          */
8575         if (previous_dev_type != SATA_DTYPE_PMULT &&
8576             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8577                 /* in case previous_dev_type is corrupt */
8578                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8579                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8580                 return (rval);
8581         }
8582 
8583         /* Device change: PMult -> Non-PMult */
8584         if (previous_dev_type == SATA_DTYPE_PMULT &&
8585             ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
8586                 /*
8587                  * This might happen because
8588                  * 1. Software reset failed. Port multiplier is not correctly
8589                  *    enumerated.
8590                  * 2. Another non-port-multiplier device is attached. Perhaps
8591                  *    the port multiplier was replaced by another device by
8592                  *    whatever reason, but AHCI driver missed hot-plug event.
8593                  *
8594                  * Now that the port has been initialized, we just need to
8595                  * update the port structure according new device, then report
8596                  * and wait SATA framework to probe new device.
8597                  */
8598 
8599                 /* Force to release pmult resource */
8600                 ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
8601                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8602 
8603                 bzero((void *)&sdevice, sizeof (sata_device_t));
8604                 sdevice.satadev_addr.cport =
8605                     ahci_ctlp->ahcictl_port_to_cport[port];
8606                 sdevice.satadev_addr.pmport = 0;
8607                 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
8608 
8609                 sdevice.satadev_state = SATA_DSTATE_RESET |
8610                     SATA_DSTATE_PWR_ACTIVE;
8611 
8612                 mutex_exit(&ahci_portp->ahciport_mutex);
8613                 sata_hba_event_notify(
8614                     ahci_ctlp->ahcictl_dip,
8615                     &sdevice,
8616                     SATA_EVNT_DEVICE_RESET);
8617                 mutex_enter(&ahci_portp->ahciport_mutex);
8618 
8619                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8620                     "Port multiplier is [Gone] at port %d ", port);
8621                 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
8622                     "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
8623 
8624                 return (AHCI_SUCCESS);
8625         }
8626 
8627         /* Device change: Non-PMult -> PMult */
8628         if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
8629 
8630                 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
8631                 ahci_alloc_pmult(ahci_ctlp, ahci_portp);
8632 
8633                 (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
8634         }
8635         pminfo = ahci_portp->ahciport_pmult_info;
8636         ASSERT(pminfo != NULL);
8637 
8638         /* Device (may) change: PMult -> PMult */
8639         /*
8640          * First initialize port multiplier. Set state to READY and wait for
8641          * probe entry point to initialize it
8642          */
8643         ahci_portp->ahciport_port_state = SATA_STATE_READY;
8644 
8645         /*
8646          * It's a little complicated while target is a port multiplier. we
8647          * need to COMRESET all pmports behind that PMult otherwise those
8648          * sub-links between the PMult and the sub-devices will be in an
8649          * inactive state (indicated by PSCR0/PxSSTS) and the following access
8650          * to those sub-devices will be rejected by Link-Fatal-Error.
8651          */
8652         /*
8653          * The PxSNTF will be set soon after the pmult is plugged. While the
8654          * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
8655          * simply mark every sub-port as 'unknown', then ahci_probe_pmport
8656          * will initialized it.
8657          */
8658         for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
8659                 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;
8660 
8661         /* Report reset event. */
8662         ahci_portp->ahciport_reset_in_progress = 1;
8663 
8664         bzero((void *)&sdevice, sizeof (sata_device_t));
8665         sdevice.satadev_addr.cport = cport;
8666         sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
8667         sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
8668         sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
8669         sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
8670             SATA_EVNT_DEVICE_RESET);
8671 
8672         return (rval);
8673 }
8674 
8675 /*
8676  * This routine may be called under four scenarios:
8677  *      a) do the recovery from fatal error
8678  *      b) or we need to timeout some commands
8679  *      c) or we need to abort some commands
8680  *      d) or we need reset device/port/controller
8681  *
8682  * In all these scenarios, we need to send any pending unfinished
8683  * commands up to sata framework.
8684  */
8685 static void
8686 ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
8687     ahci_port_t *ahci_portp,
8688     uint32_t slot_status,
8689     uint32_t failed_tags,
8690     uint32_t timeout_tags,
8691     uint32_t aborted_tags,
8692     uint32_t reset_tags)
8693 {
8694         uint32_t finished_tags = 0;
8695         uint32_t unfinished_tags = 0;
8696         int tmp_slot;
8697         sata_pkt_t *satapkt;
8698         int ncq_cmd_in_progress = 0;
8699         int err_retri_cmd_in_progress = 0;
8700         int rdwr_pmult_cmd_in_progress = 0;
8701 
8702         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
8703 
8704         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8705             "ahci_mop_commands entered: port: %d slot_status: 0x%x",
8706             ahci_portp->ahciport_port_num, slot_status);
8707 
8708         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8709             "ahci_mop_commands: failed_tags: 0x%x, "
8710             "timeout_tags: 0x%x aborted_tags: 0x%x, "
8711             "reset_tags: 0x%x", failed_tags,
8712             timeout_tags, aborted_tags, reset_tags);
8713 
8714 #ifdef AHCI_DEBUG
8715         if (ahci_debug_flags & AHCIDBG_ERRS) {
8716                 int i;
8717                 char msg_buf[200] = {0, };
8718                 for (i = 0x1f; i >= 0; i--) {
8719                         if (ahci_portp->ahciport_slot_pkts[i] != NULL)
8720                                 msg_buf[i] = 'X';
8721                         else
8722                                 msg_buf[i] = '.';
8723                 }
8724                 msg_buf[0x20] = '\0';
8725                 cmn_err(CE_NOTE, "port[%d] slots: %s",
8726                     ahci_portp->ahciport_port_num, msg_buf);
8727                 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
8728                     (void *)ahci_portp->ahciport_err_retri_pkt,
8729                     (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
8730         }
8731 #endif
8732 
8733         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8734                 finished_tags = ahci_portp->ahciport_pending_tags &
8735                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
8736 
8737                 unfinished_tags = slot_status &
8738                     AHCI_SLOT_MASK(ahci_ctlp) &
8739                     ~failed_tags &
8740                     ~aborted_tags &
8741                     ~reset_tags &
8742                     ~timeout_tags;
8743         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
8744                 ncq_cmd_in_progress = 1;
8745                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
8746                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
8747 
8748                 unfinished_tags = slot_status &
8749                     AHCI_NCQ_SLOT_MASK(ahci_portp) &
8750                     ~failed_tags &
8751                     ~aborted_tags &
8752                     ~reset_tags &
8753                     ~timeout_tags;
8754         } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
8755 
8756                 /*
8757                  * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
8758                  * set, it means REQUEST SENSE or READ LOG EXT command doesn't
8759                  * complete successfully due to one of the following three
8760                  * conditions:
8761                  *
8762                  *      1. Fatal error - failed_tags includes its slot
8763                  *      2. Timed out - timeout_tags includes its slot
8764                  *      3. Aborted when hot unplug - aborted_tags includes its
8765                  *         slot
8766                  *
8767                  * Please note that the command is always sent down in Slot 0
8768                  */
8769                 err_retri_cmd_in_progress = 1;
8770                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
8771                     "ahci_mop_commands is called for port %d while "
8772                     "REQUEST SENSE or READ LOG EXT for error retrieval "
8773                     "is being executed slot_status = 0x%x",
8774                     ahci_portp->ahciport_port_num, slot_status);
8775                 ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
8776                 ASSERT(slot_status == 0x1);
8777         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
8778                 rdwr_pmult_cmd_in_progress = 1;
8779                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
8780                     "ahci_mop_commands is called for port %d while "
8781                     "READ/WRITE PORTMULT command is being executed",
8782                     ahci_portp->ahciport_port_num);
8783 
8784                 ASSERT(slot_status == 0x1);
8785         }
8786 
8787 #ifdef AHCI_DEBUG
8788         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
8789             "ahci_mop_commands: finished_tags: 0x%x, "
8790             "unfinished_tags 0x%x", finished_tags, unfinished_tags);
8791 #endif
8792 
8793         /* Send up finished packets with SATA_PKT_COMPLETED */
8794         while (finished_tags) {
8795                 tmp_slot = ddi_ffs(finished_tags) - 1;
8796                 if (tmp_slot == -1) {
8797                         break;
8798                 }
8799 
8800                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8801                 ASSERT(satapkt != NULL);
8802 
8803                 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
8804                     "sending up pkt 0x%p with SATA_PKT_COMPLETED",
8805                     (void *)satapkt);
8806 
8807                 /*
8808                  * Cannot fetch the return register content since the port
8809                  * was restarted, so the corresponding tag will be set to
8810                  * aborted tags.
8811                  */
8812                 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
8813                         CLEAR_BIT(finished_tags, tmp_slot);
8814                         aborted_tags |= tmp_slot;
8815                         continue;
8816                 }
8817 
8818                 if (ncq_cmd_in_progress)
8819                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8820                             tmp_slot);
8821                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8822                 CLEAR_BIT(finished_tags, tmp_slot);
8823                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8824 
8825                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
8826         }
8827 
8828         /* Send up failed packets with SATA_PKT_DEV_ERROR. */
8829         while (failed_tags) {
8830                 if (err_retri_cmd_in_progress) {
8831                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8832                         ASSERT(satapkt != NULL);
8833                         ASSERT(failed_tags == 0x1);
8834 
8835                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8836                             "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8837                             (void *)satapkt);
8838                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8839                         break;
8840                 }
8841                 if (rdwr_pmult_cmd_in_progress) {
8842                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8843                         ASSERT(satapkt != NULL);
8844                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8845                             "ahci_mop_commands: sending up "
8846                             "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
8847                             (void *)satapkt);
8848                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8849                         break;
8850                 }
8851 
8852                 tmp_slot = ddi_ffs(failed_tags) - 1;
8853                 if (tmp_slot == -1) {
8854                         break;
8855                 }
8856 
8857                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8858                 ASSERT(satapkt != NULL);
8859 
8860                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8861                     "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
8862                     (void *)satapkt);
8863 
8864                 if (ncq_cmd_in_progress)
8865                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8866                             tmp_slot);
8867                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8868                 CLEAR_BIT(failed_tags, tmp_slot);
8869                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8870 
8871                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
8872         }
8873 
8874         /* Send up timeout packets with SATA_PKT_TIMEOUT. */
8875         while (timeout_tags) {
8876                 if (err_retri_cmd_in_progress) {
8877                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8878                         ASSERT(satapkt != NULL);
8879                         ASSERT(timeout_tags == 0x1);
8880 
8881                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8882                             "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8883                             (void *)satapkt);
8884                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8885                         break;
8886                 }
8887                 if (rdwr_pmult_cmd_in_progress) {
8888                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8889                         ASSERT(satapkt != NULL);
8890                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8891                             "ahci_mop_commands: sending up "
8892                             "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
8893                             (void *)satapkt);
8894                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8895                         break;
8896                 }
8897 
8898                 tmp_slot = ddi_ffs(timeout_tags) - 1;
8899                 if (tmp_slot == -1) {
8900                         break;
8901                 }
8902 
8903                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8904                 ASSERT(satapkt != NULL);
8905 
8906                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8907                     "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
8908                     (void *)satapkt);
8909 
8910                 if (ncq_cmd_in_progress)
8911                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8912                             tmp_slot);
8913                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8914                 CLEAR_BIT(timeout_tags, tmp_slot);
8915                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8916 
8917                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
8918         }
8919 
8920         /* Send up aborted packets with SATA_PKT_ABORTED */
8921         while (aborted_tags) {
8922                 if (err_retri_cmd_in_progress) {
8923                         satapkt = ahci_portp->ahciport_err_retri_pkt;
8924                         ASSERT(satapkt != NULL);
8925                         ASSERT(aborted_tags == 0x1);
8926 
8927                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8928                             "sending up pkt 0x%p with SATA_PKT_ABORTED",
8929                             (void *)satapkt);
8930                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8931                         break;
8932                 }
8933                 if (rdwr_pmult_cmd_in_progress) {
8934                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8935                         ASSERT(satapkt != NULL);
8936                         ASSERT(aborted_tags == 0x1);
8937                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8938                             "ahci_mop_commands: sending up "
8939                             "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
8940                             (void *)satapkt);
8941                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8942                         break;
8943                 }
8944 
8945                 tmp_slot = ddi_ffs(aborted_tags) - 1;
8946                 if (tmp_slot == -1) {
8947                         break;
8948                 }
8949 
8950                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8951                 ASSERT(satapkt != NULL);
8952 
8953                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8954                     "sending up pkt 0x%p with SATA_PKT_ABORTED",
8955                     (void *)satapkt);
8956 
8957                 if (ncq_cmd_in_progress)
8958                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8959                             tmp_slot);
8960                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8961                 CLEAR_BIT(aborted_tags, tmp_slot);
8962                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8963 
8964                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
8965         }
8966 
8967         /* Send up reset packets with SATA_PKT_RESET. */
8968         while (reset_tags) {
8969                 if (rdwr_pmult_cmd_in_progress) {
8970                         satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
8971                         ASSERT(satapkt != NULL);
8972                         ASSERT(aborted_tags == 0x1);
8973                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
8974                             "ahci_mop_commands: sending up "
8975                             "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
8976                             (void *)satapkt);
8977                         ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
8978                         break;
8979                 }
8980 
8981                 tmp_slot = ddi_ffs(reset_tags) - 1;
8982                 if (tmp_slot == -1) {
8983                         break;
8984                 }
8985 
8986                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
8987                 ASSERT(satapkt != NULL);
8988 
8989                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
8990                     "sending up pkt 0x%p with SATA_PKT_RESET",
8991                     (void *)satapkt);
8992 
8993                 if (ncq_cmd_in_progress)
8994                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
8995                             tmp_slot);
8996                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
8997                 CLEAR_BIT(reset_tags, tmp_slot);
8998                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
8999 
9000                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9001         }
9002 
9003         /* Send up unfinished packets with SATA_PKT_RESET */
9004         while (unfinished_tags) {
9005                 tmp_slot = ddi_ffs(unfinished_tags) - 1;
9006                 if (tmp_slot == -1) {
9007                         break;
9008                 }
9009 
9010                 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9011                 ASSERT(satapkt != NULL);
9012 
9013                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
9014                     "sending up pkt 0x%p with SATA_PKT_RESET",
9015                     (void *)satapkt);
9016 
9017                 if (ncq_cmd_in_progress)
9018                         CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
9019                             tmp_slot);
9020                 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
9021                 CLEAR_BIT(unfinished_tags, tmp_slot);
9022                 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
9023 
9024                 ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
9025         }
9026 
9027         ahci_portp->ahciport_mop_in_progress--;
9028         ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
9029 
9030         if (ahci_portp->ahciport_mop_in_progress == 0)
9031                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
9032 
9033         ahci_flush_doneq(ahci_portp);
9034 }
9035 
9036 /*
9037  * This routine is going to first request a READ LOG EXT sata pkt from sata
9038  * module, and then deliver it to the HBA to get the ncq failure context.
9039  * The return value is the exactly failed tags.
9040  */
9041 static uint32_t
9042 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9043     uint8_t port)
9044 {
9045         sata_device_t   sdevice;
9046         sata_pkt_t      *rdlog_spkt, *spkt;
9047         ddi_dma_handle_t buf_dma_handle;
9048         ahci_addr_t     addr;
9049         int             loop_count;
9050         int             rval;
9051         int             failed_slot;
9052         uint32_t        failed_tags = 0;
9053         struct sata_ncq_error_recovery_page *ncq_err_page;
9054 
9055         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
9056             "ahci_get_rdlogext_data enter: port %d", port);
9057 
9058         /* Prepare the sdevice data */
9059         bzero((void *)&sdevice, sizeof (sata_device_t));
9060         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9061 
9062         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9063         sdevice.satadev_addr.pmport = 0;
9064 
9065         /* Translate sata_device.satadev_addr -> ahci_addr */
9066         ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9067 
9068         /*
9069          * Call the sata hba interface to get a rdlog spkt
9070          */
9071         loop_count = 0;
9072 loop:
9073         rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9074             &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
9075         if (rdlog_spkt == NULL) {
9076                 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9077                         /* Sleep for a while */
9078                         drv_usecwait(AHCI_10MS_USECS);
9079                         goto loop;
9080                 }
9081                 /* Timed out after 1s */
9082                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9083                     "failed to get rdlog spkt for port %d", port);
9084                 return (failed_tags);
9085         }
9086 
9087         ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9088 
9089         /*
9090          * This flag is used to handle the specific error recovery when the
9091          * READ LOG EXT command gets a failure (fatal error or time-out).
9092          */
9093         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;
9094 
9095         /*
9096          * This start is not supposed to fail because after port is restarted,
9097          * the whole command list is empty.
9098          */
9099         ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
9100         (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
9101         ahci_portp->ahciport_err_retri_pkt = NULL;
9102 
9103         /* Remove the flag after READ LOG EXT command is completed */
9104         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;
9105 
9106         if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9107                 /* Update the request log data */
9108                 buf_dma_handle = *(ddi_dma_handle_t *)
9109                     (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9110                 rval = ddi_dma_sync(buf_dma_handle, 0, 0,
9111                     DDI_DMA_SYNC_FORKERNEL);
9112                 if (rval == DDI_SUCCESS) {
9113                         ncq_err_page =
9114                             (struct sata_ncq_error_recovery_page *)rdlog_spkt->
9115                             satapkt_cmd.satacmd_bp->b_un.b_addr;
9116 
9117                         /* Get the failed tag */
9118                         failed_slot = ncq_err_page->ncq_tag;
9119                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9120                             "ahci_get_rdlogext_data: port %d "
9121                             "failed slot %d", port, failed_slot);
9122                         if (failed_slot & NQ) {
9123                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9124                                     "the failed slot is not a valid tag", NULL);
9125                                 goto out;
9126                         }
9127 
9128                         failed_slot &= NCQ_TAG_MASK;
9129                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9130                         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9131                             "ahci_get_rdlogext_data: failed spkt 0x%p",
9132                             (void *)spkt);
9133                         if (spkt == NULL) {
9134                                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9135                                     "the failed slot spkt is NULL", NULL);
9136                                 goto out;
9137                         }
9138 
9139                         failed_tags = 0x1 << failed_slot;
9140 
9141                         /* Fill out the error context */
9142                         ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
9143                             ncq_err_page);
9144                         ahci_update_sata_registers(ahci_ctlp, port,
9145                             &spkt->satapkt_device);
9146                 }
9147         }
9148 out:
9149         sata_free_error_retrieval_pkt(rdlog_spkt);
9150 
9151         return (failed_tags);
9152 }
9153 
9154 /*
9155  * This routine is going to first request a REQUEST SENSE sata pkt from sata
9156  * module, and then deliver it to the HBA to get the sense data and copy
9157  * the sense data back to the orignal failed sata pkt, and free the REQUEST
9158  * SENSE sata pkt later.
9159  */
9160 static void
9161 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9162     uint8_t port, sata_pkt_t *spkt)
9163 {
9164         sata_device_t   sdevice;
9165         sata_pkt_t      *rs_spkt;
9166         sata_cmd_t      *sata_cmd;
9167         ddi_dma_handle_t buf_dma_handle;
9168         ahci_addr_t     addr;
9169         int             loop_count;
9170 #if AHCI_DEBUG
9171         struct scsi_extended_sense *rqsense;
9172 #endif
9173 
9174         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9175             "ahci_get_rqsense_data enter: port %d", port);
9176 
9177         /* Prepare the sdevice data */
9178         bzero((void *)&sdevice, sizeof (sata_device_t));
9179         sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
9180 
9181         sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
9182         sdevice.satadev_addr.pmport = 0;
9183 
9184         /* Translate sata_device.satadev_addr -> ahci_addr */
9185         ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);
9186 
9187         sata_cmd = &spkt->satapkt_cmd;
9188 
9189         /*
9190          * Call the sata hba interface to get a rs spkt
9191          */
9192         loop_count = 0;
9193 loop:
9194         rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
9195             &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
9196         if (rs_spkt == NULL) {
9197                 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
9198                         /* Sleep for a while */
9199                         drv_usecwait(AHCI_10MS_USECS);
9200                         goto loop;
9201 
9202                 }
9203                 /* Timed out after 1s */
9204                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9205                     "failed to get rs spkt for port %d", port);
9206                 return;
9207         }
9208 
9209         ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
9210 
9211         /*
9212          * This flag is used to handle the specific error recovery when the
9213          * REQUEST SENSE command gets a faiure (fatal error or time-out).
9214          */
9215         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
9216 
9217         /*
9218          * This start is not supposed to fail because after port is restarted,
9219          * the whole command list is empty.
9220          */
9221         ahci_portp->ahciport_err_retri_pkt = rs_spkt;
9222         (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
9223         ahci_portp->ahciport_err_retri_pkt = NULL;
9224 
9225         /* Remove the flag after REQUEST SENSE command is completed */
9226         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;
9227 
9228         if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
9229                 /* Update the request sense data */
9230                 buf_dma_handle = *(ddi_dma_handle_t *)
9231                     (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
9232                 (void) ddi_dma_sync(buf_dma_handle, 0, 0,
9233                     DDI_DMA_SYNC_FORKERNEL);
9234                 /* Copy the request sense data */
9235                 bcopy(rs_spkt->
9236                     satapkt_cmd.satacmd_bp->b_un.b_addr,
9237                     &sata_cmd->satacmd_rqsense,
9238                     SATA_ATAPI_MIN_RQSENSE_LEN);
9239 #if AHCI_DEBUG
9240                 rqsense = (struct scsi_extended_sense *)
9241                     sata_cmd->satacmd_rqsense;
9242 
9243                 /* Dump the sense data */
9244                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
9245                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9246                     "Sense data for satapkt %p ATAPI cmd 0x%x",
9247                     spkt, sata_cmd->satacmd_acdb[0]);
9248                 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
9249                     "  es_code 0x%x es_class 0x%x "
9250                     "es_key 0x%x es_add_code 0x%x "
9251                     "es_qual_code 0x%x",
9252                     rqsense->es_code, rqsense->es_class,
9253                     rqsense->es_key, rqsense->es_add_code,
9254                     rqsense->es_qual_code);
9255 #endif
9256         }
9257 
9258         sata_free_error_retrieval_pkt(rs_spkt);
9259 }
9260 
9261 /*
9262  * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
9263  * the port must be restarted. When the HBA detects thus error, it may try
9264  * to abort a transfer. And if the transfer was aborted, the device is
9265  * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
9266  * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
9267  * that the device is in a stable status and transfers may be restarted without
9268  * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
9269  * then the software will send the COMRESET to do the port reset.
9270  *
9271  * Software should perform the appropriate error recovery actions based on
9272  * whether non-queued commands were being issued or natived command queuing
9273  * commands were being issued.
9274  *
9275  * And software will complete the command that had the error with error mark
9276  * to higher level software.
9277  *
9278  * Fatal errors include the following:
9279  *      PxIS.IFS - Interface Fatal Error Status
9280  *      PxIS.HBDS - Host Bus Data Error Status
9281  *      PxIS.HBFS - Host Bus Fatal Error Status
9282  *      PxIS.TFES - Task File Error Status
9283  */
9284 static void
9285 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9286     ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
9287 {
9288         uint32_t        port_cmd_status;
9289         uint32_t        slot_status = 0;
9290         uint32_t        failed_tags = 0;
9291         int             failed_slot;
9292         int             reset_flag = 0, flag = 0;
9293         ahci_fis_d2h_register_t *ahci_rcvd_fisp;
9294         sata_cmd_t      *sata_cmd = NULL;
9295         sata_pkt_t      *spkt = NULL;
9296 #if AHCI_DEBUG
9297         ahci_cmd_header_t *cmd_header;
9298 #endif
9299         uint8_t         port = addrp->aa_port;
9300         int             instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9301         int             rval;
9302 
9303         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9304 
9305         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9306             "ahci_fatal_error_recovery_handler enter: port %d", port);
9307 
9308         /* Port multiplier error */
9309         if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
9310                 /* FBS code is neither completed nor tested. */
9311                 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
9312                     port, intr_status);
9313 
9314                 /* Force a port reset */
9315                 flag = AHCI_PORT_RESET;
9316         }
9317 
9318         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9319             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9320 
9321                 /* Read PxCI to see which commands are still outstanding */
9322                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9323                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9324 
9325                 /*
9326                  * Read PxCMD.CCS to determine the slot that the HBA
9327                  * was processing when the error occurred.
9328                  */
9329                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9330                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9331                 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9332                     AHCI_CMD_STATUS_CCS_SHIFT;
9333 
9334                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9335                         spkt = ahci_portp->ahciport_err_retri_pkt;
9336                         ASSERT(spkt != NULL);
9337                 } else {
9338                         spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
9339                         if (spkt == NULL) {
9340                                 /* May happen when interface errors occur? */
9341                                 goto next;
9342                         }
9343                 }
9344 
9345 #if AHCI_DEBUG
9346                 /*
9347                  * Debugging purpose...
9348                  */
9349                 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
9350                         cmd_header =
9351                             &ahci_portp->ahciport_cmd_list[failed_slot];
9352                         AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9353                             "ahci_fatal_error_recovery_handler: port %d, "
9354                             "PRD Byte Count = 0x%x, "
9355                             "ahciport_prd_bytecounts = 0x%x", port,
9356                             cmd_header->ahcich_prd_byte_count,
9357                             ahci_portp->ahciport_prd_bytecounts[failed_slot]);
9358                 }
9359 #endif
9360 
9361                 sata_cmd = &spkt->satapkt_cmd;
9362 
9363                 /* Fill out the status and error registers for PxIS.TFES */
9364                 if (intr_status & AHCI_INTR_STATUS_TFES) {
9365                         ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
9366                             ahcirf_d2h_register_fis);
9367 
9368                         /* Copy the error context back to the sata_cmd */
9369                         ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
9370                 }
9371 
9372                 /* The failed command must be one of the outstanding commands */
9373                 failed_tags = 0x1 << failed_slot;
9374                 ASSERT(failed_tags & slot_status);
9375 
9376                 /* Update the sata registers, especially PxSERR register */
9377                 ahci_update_sata_registers(ahci_ctlp, port,
9378                     &spkt->satapkt_device);
9379 
9380         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9381                 /* Read PxSACT to see which commands are still outstanding */
9382                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9383                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9384         }
9385 next:
9386 
9387 #if AHCI_DEBUG
9388         /*
9389          * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9390          * set, it means a fatal error happened after REQUEST SENSE command
9391          * or READ LOG EXT command is delivered to the HBA during the error
9392          * recovery process. At this time, the only outstanding command is
9393          * supposed to be REQUEST SENSE command or READ LOG EXT command.
9394          */
9395         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9396                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9397                     "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
9398                     "command or READ LOG EXT command for error data retrieval "
9399                     "failed", port);
9400                 ASSERT(slot_status == 0x1);
9401                 ASSERT(failed_slot == 0);
9402                 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
9403                     SCMD_REQUEST_SENSE ||
9404                     spkt->satapkt_cmd.satacmd_cmd_reg ==
9405                     SATAC_READ_LOG_EXT);
9406         }
9407 #endif
9408 
9409         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9410         ahci_portp->ahciport_mop_in_progress++;
9411 
9412         rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9413             port, flag, &reset_flag);
9414 
9415         if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9416                 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9417                 if (rval == AHCI_SUCCESS)
9418                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9419                             "succeed", instance, port);
9420                 else
9421                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9422                             "failed", instance, port);
9423         }
9424 
9425         /*
9426          * Won't retrieve error information:
9427          * 1. Port reset was involved to recover
9428          * 2. Device is gone
9429          * 3. IDENTIFY DEVICE command sent to ATAPI device
9430          * 4. REQUEST SENSE or READ LOG EXT command during error recovery
9431          */
9432         if (reset_flag ||
9433             ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
9434             spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
9435             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9436                 goto out;
9437 
9438         /*
9439          * Deliver READ LOG EXT to gather information about the error when
9440          * a COMRESET has not been performed as part of the error recovery
9441          * during NCQ command processing.
9442          */
9443         if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9444                 failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
9445                     ahci_portp, port);
9446                 goto out;
9447         }
9448 
9449         /*
9450          * Deliver REQUEST SENSE for ATAPI command to gather information about
9451          * the error when a COMRESET has not been performed as part of the
9452          * error recovery.
9453          */
9454         if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
9455                 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
9456 out:
9457         AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9458             "ahci_fatal_error_recovery_handler: port %d fatal error "
9459             "occurred slot_status = 0x%x, pending_tags = 0x%x, "
9460             "pending_ncq_tags = 0x%x failed_tags = 0x%x",
9461             port, slot_status, ahci_portp->ahciport_pending_tags,
9462             ahci_portp->ahciport_pending_ncq_tags, failed_tags);
9463 
9464         ahci_mop_commands(ahci_ctlp,
9465             ahci_portp,
9466             slot_status,
9467             failed_tags, /* failed tags */
9468             0, /* timeout tags */
9469             0, /* aborted tags */
9470             0); /* reset tags */
9471 }
9472 
9473 /*
9474  * Used to recovery a PMULT pmport fatal error under FIS-based switching.
9475  *      1. device specific.PxFBS.SDE=1
9476  *      2. Non-Deivce specific.
9477  * Nothing will be done when Command-based switching is employed.
9478  *
9479  * Currently code is neither completed nor tested.
9480  */
9481 static void
9482 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
9483     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
9484 {
9485 #ifndef __lock_lint
9486         _NOTE(ARGUNUSED(intr_status))
9487 #endif
9488         uint32_t        port_fbs_ctrl;
9489         int loop_count = 0;
9490         ahci_addr_t     addr;
9491 
9492         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
9493 
9494         /* Nothing will be done under Command-based switching. */
9495         if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
9496                 return;
9497 
9498         port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9499             (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9500 
9501         if (!(port_fbs_ctrl & AHCI_FBS_EN))
9502                 /* FBS is not enabled. */
9503                 return;
9504 
9505         /* Problem's getting complicated now. */
9506         /*
9507          * If FIS-based switching is used, we need to check
9508          * the PxFBS to see the error type.
9509          */
9510         port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9511             (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));
9512 
9513         /* Refer to spec(v1.2) 9.3.6.1 */
9514         if (port_fbs_ctrl & AHCI_FBS_SDE) {
9515                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
9516                     "A Device Sepcific Error: port %d", port);
9517                 /*
9518                  * Controller has paused commands for all other
9519                  * sub-devices until PxFBS.DEC is set.
9520                  */
9521                 ahci_reject_all_abort_pkts(ahci_ctlp,
9522                     ahci_portp, 0);
9523 
9524                 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
9525                     (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
9526                     port_fbs_ctrl | AHCI_FBS_DEC);
9527 
9528                 /*
9529                  * Wait controller clear PxFBS.DEC,
9530                  * then we can continue.
9531                  */
9532                 loop_count = 0;
9533                 do {
9534                         port_fbs_ctrl = ddi_get32(ahci_ctlp->
9535                             ahcictl_ahci_acc_handle, (uint32_t *)
9536                             AHCI_PORT_PxFBS(ahci_ctlp, port));
9537 
9538                         if (loop_count++ > 1000)
9539                                 /*
9540                                  * Esclate the error. Follow
9541                                  * non-device specific error
9542                                  * procedure.
9543                                  */
9544                                 return;
9545 
9546                         drv_usecwait(AHCI_100US_USECS);
9547                 } while (port_fbs_ctrl & AHCI_FBS_DEC);
9548 
9549                 /*
9550                  * Issue a software reset to ensure drive is in
9551                  * a known state.
9552                  */
9553                 (void) ahci_software_reset(ahci_ctlp,
9554                     ahci_portp, &addr);
9555 
9556         } else {
9557 
9558                 /* Process Non-Device Specific Error. */
9559                 /* This will be handled later on. */
9560                 cmn_err(CE_NOTE, "!FBS is not supported now.");
9561         }
9562 }
9563 /*
9564  * Handle events - fatal error recovery
9565  */
9566 static void
9567 ahci_events_handler(void *args)
9568 {
9569         ahci_event_arg_t *ahci_event_arg;
9570         ahci_ctl_t *ahci_ctlp;
9571         ahci_port_t *ahci_portp;
9572         ahci_addr_t *addrp;
9573         uint32_t event;
9574         int instance;
9575 
9576         ahci_event_arg = (ahci_event_arg_t *)args;
9577 
9578         ahci_ctlp = ahci_event_arg->ahciea_ctlp;
9579         ahci_portp = ahci_event_arg->ahciea_portp;
9580         addrp = ahci_event_arg->ahciea_addrp;
9581         event = ahci_event_arg->ahciea_event;
9582         instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9583 
9584         AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
9585             "ahci_events_handler enter: port %d intr_status = 0x%x",
9586             ahci_portp->ahciport_port_num, event);
9587 
9588         mutex_enter(&ahci_portp->ahciport_mutex);
9589 
9590         /*
9591          * ahci_intr_phyrdy_change() may have rendered it to
9592          * SATA_DTYPE_NONE.
9593          */
9594         if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9595                 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
9596                     "ahci_events_handler: port %d no device attached, "
9597                     "and just return without doing anything",
9598                     ahci_portp->ahciport_port_num);
9599 
9600                 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
9601                         ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
9602                         cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
9603                             "succeed", instance, ahci_portp->ahciport_port_num);
9604                 }
9605 
9606                 goto out;
9607         }
9608 
9609         if (event & (AHCI_INTR_STATUS_IFS |
9610             AHCI_INTR_STATUS_HBDS |
9611             AHCI_INTR_STATUS_HBFS |
9612             AHCI_INTR_STATUS_TFES))
9613                 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
9614                     addrp, event);
9615 
9616 out:
9617         mutex_exit(&ahci_portp->ahciport_mutex);
9618 }
9619 
9620 /*
9621  * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
9622  * detect there are some commands which are timed out.
9623  */
9624 static void
9625 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
9626     uint8_t port, uint32_t tmp_timeout_tags)
9627 {
9628         uint32_t slot_status = 0;
9629         uint32_t finished_tags = 0;
9630         uint32_t timeout_tags = 0;
9631 
9632         AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
9633             "ahci_timeout_pkts enter: port %d", port);
9634 
9635         mutex_enter(&ahci_portp->ahciport_mutex);
9636 
9637         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
9638             RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
9639             ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9640                 /* Read PxCI to see which commands are still outstanding */
9641                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9642                     (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
9643         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9644                 /* Read PxSACT to see which commands are still outstanding */
9645                 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9646                     (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9647         }
9648 
9649 #if AHCI_DEBUG
9650         /*
9651          * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
9652          * set, it means a fatal error happened after REQUEST SENSE command
9653          * or READ LOG EXT command is delivered to the HBA during the error
9654          * recovery process. At this time, the only outstanding command is
9655          * supposed to be REQUEST SENSE command or READ LOG EXT command.
9656          */
9657         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
9658                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9659                     "ahci_timeout_pkts called while REQUEST SENSE "
9660                     "command or READ LOG EXT command for error recovery "
9661                     "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
9662                     "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
9663                     tmp_timeout_tags, slot_status,
9664                     ahci_portp->ahciport_pending_tags,
9665                     ahci_portp->ahciport_pending_ncq_tags);
9666                 ASSERT(slot_status == 0x1);
9667         } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9668                 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
9669                     "ahci_timeout_pkts called while executing R/W PMULT "
9670                     "command timeout_tags = 0x%x, slot_status = 0x%x",
9671                     tmp_timeout_tags, slot_status);
9672                 ASSERT(slot_status == 0x1);
9673         }
9674 #endif
9675 
9676         ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
9677         ahci_portp->ahciport_mop_in_progress++;
9678 
9679         (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
9680             port, AHCI_PORT_RESET, NULL);
9681 
9682         /*
9683          * Re-identify timeout tags because some previously checked commands
9684          * could already complete.
9685          */
9686         if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9687                 finished_tags = ahci_portp->ahciport_pending_tags &
9688                     ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
9689                 timeout_tags = tmp_timeout_tags & ~finished_tags;
9690 
9691                 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
9692                     "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9693                     "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
9694                     "pending_tags = 0x%x ",
9695                     port, finished_tags, timeout_tags,
9696                     slot_status, ahci_portp->ahciport_pending_tags);
9697         } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9698                 finished_tags = ahci_portp->ahciport_pending_ncq_tags &
9699                     ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
9700                 timeout_tags = tmp_timeout_tags & ~finished_tags;
9701 
9702                 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
9703                     "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
9704                     "timeout_tags = 0x%x, port_sactive = 0x%x, "
9705                     "pending_ncq_tags = 0x%x ",
9706                     port, finished_tags, timeout_tags,
9707                     slot_status, ahci_portp->ahciport_pending_ncq_tags);
9708         } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9709             RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9710                 timeout_tags = tmp_timeout_tags;
9711         }
9712 
9713         ahci_mop_commands(ahci_ctlp,
9714             ahci_portp,
9715             slot_status,
9716             0,                  /* failed tags */
9717             timeout_tags,       /* timeout tags */
9718             0,                  /* aborted tags */
9719             0);                 /* reset tags */
9720 
9721         mutex_exit(&ahci_portp->ahciport_mutex);
9722 }
9723 
9724 /*
9725  * Watchdog handler kicks in every 5 seconds to timeout any commands pending
9726  * for long time.
9727  */
9728 static void
9729 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
9730 {
9731         ahci_port_t *ahci_portp;
9732         sata_pkt_t *spkt;
9733         uint32_t pending_tags;
9734         uint32_t timeout_tags;
9735         uint32_t port_cmd_status;
9736         uint32_t port_sactive;
9737         uint8_t port;
9738         int tmp_slot;
9739         int current_slot;
9740         uint32_t current_tags;
9741         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9742 
9743         mutex_enter(&ahci_ctlp->ahcictl_mutex);
9744 
9745         AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
9746             "ahci_watchdog_handler entered", NULL);
9747 
9748         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
9749                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
9750                         continue;
9751                 }
9752 
9753                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
9754 
9755                 mutex_enter(&ahci_portp->ahciport_mutex);
9756                 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
9757                         mutex_exit(&ahci_portp->ahciport_mutex);
9758                         continue;
9759                 }
9760 
9761                 /* Skip the check for those ports in error recovery */
9762                 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
9763                     !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
9764                         mutex_exit(&ahci_portp->ahciport_mutex);
9765                         continue;
9766                 }
9767 
9768                 pending_tags = 0;
9769                 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
9770                     (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
9771 
9772                 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
9773                     RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
9774                         current_slot = 0;
9775                         pending_tags = 0x1;
9776                 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9777                         current_slot =
9778                             (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
9779                             AHCI_CMD_STATUS_CCS_SHIFT;
9780                         pending_tags = ahci_portp->ahciport_pending_tags;
9781                 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9782                         port_sactive = ddi_get32(
9783                             ahci_ctlp->ahcictl_ahci_acc_handle,
9784                             (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
9785                         current_tags = port_sactive &
9786                             ~port_cmd_status &
9787                             AHCI_NCQ_SLOT_MASK(ahci_portp);
9788                         pending_tags = ahci_portp->ahciport_pending_ncq_tags;
9789                 }
9790 
9791                 timeout_tags = 0;
9792                 while (pending_tags) {
9793                         tmp_slot = ddi_ffs(pending_tags) - 1;
9794                         if (tmp_slot == -1) {
9795                                 break;
9796                         }
9797 
9798                         if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
9799                                 spkt = ahci_portp->ahciport_err_retri_pkt;
9800                         else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
9801                                 spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
9802                         else
9803                                 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9804 
9805                         if ((spkt != NULL) && spkt->satapkt_time &&
9806                             !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
9807                                 /*
9808                                  * If a packet has survived for more than it's
9809                                  * max life cycles, it is a candidate for time
9810                                  * out.
9811                                  */
9812                                 ahci_portp->ahciport_slot_timeout[tmp_slot] -=
9813                                     ahci_watchdog_timeout;
9814 
9815                                 if (ahci_portp->ahciport_slot_timeout[tmp_slot]
9816                                     > 0)
9817                                         goto next;
9818 
9819 #if AHCI_DEBUG
9820                                 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
9821                                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9822                                             ahci_ctlp, "watchdog: the current "
9823                                             "tags is 0x%x", current_tags);
9824                                 } else {
9825                                         AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
9826                                             ahci_ctlp, "watchdog: the current "
9827                                             "slot is %d", current_slot);
9828                                 }
9829 #endif
9830 
9831                                 /*
9832                                  * We need to check whether the HBA has
9833                                  * begun to execute the command, if not,
9834                                  * then re-set the timer of the command.
9835                                  */
9836                                 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9837                                     (tmp_slot != current_slot) ||
9838                                     NCQ_CMD_IN_PROGRESS(ahci_portp) &&
9839                                     ((0x1 << tmp_slot) & current_tags)) {
9840                                         ahci_portp->ahciport_slot_timeout \
9841                                             [tmp_slot] = spkt->satapkt_time;
9842                                 } else {
9843                                         timeout_tags |= (0x1 << tmp_slot);
9844                                         cmn_err(CE_WARN, "!ahci%d: watchdog "
9845                                             "port %d satapkt 0x%p timed out\n",
9846                                             instance, port, (void *)spkt);
9847                                 }
9848                         }
9849 next:
9850                         CLEAR_BIT(pending_tags, tmp_slot);
9851                 }
9852 
9853                 if (timeout_tags) {
9854                         mutex_exit(&ahci_portp->ahciport_mutex);
9855                         mutex_exit(&ahci_ctlp->ahcictl_mutex);
9856                         ahci_timeout_pkts(ahci_ctlp, ahci_portp,
9857                             port, timeout_tags);
9858                         mutex_enter(&ahci_ctlp->ahcictl_mutex);
9859                         mutex_enter(&ahci_portp->ahciport_mutex);
9860                 }
9861 
9862                 mutex_exit(&ahci_portp->ahciport_mutex);
9863         }
9864 
9865         /* Re-install the watchdog timeout handler */
9866         if (ahci_ctlp->ahcictl_timeout_id != 0) {
9867                 ahci_ctlp->ahcictl_timeout_id =
9868                     timeout((void (*)(void *))ahci_watchdog_handler,
9869                     (caddr_t)ahci_ctlp, ahci_watchdog_tick);
9870         }
9871 
9872         mutex_exit(&ahci_ctlp->ahcictl_mutex);
9873 }
9874 
9875 /*
9876  * Fill the error context into sata_cmd for non-queued command error.
9877  */
9878 static void
9879 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9880 {
9881         scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
9882         scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9883         scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9884         scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9885         scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9886         scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9887         scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9888 
9889         if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
9890                 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9891                 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9892                 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9893                 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9894         }
9895 }
9896 
9897 /*
9898  * Fill the ncq error page into sata_cmd for queued command error.
9899  */
9900 static void
9901 ahci_copy_ncq_err_page(sata_cmd_t *scmd,
9902     struct sata_ncq_error_recovery_page *ncq_err_page)
9903 {
9904         scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
9905         scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
9906         scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
9907         scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
9908         scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
9909         scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
9910         scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
9911         scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
9912         scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
9913         scmd->satacmd_status_reg = ncq_err_page->ncq_status;
9914         scmd->satacmd_error_reg = ncq_err_page->ncq_error;
9915 }
9916 
9917 /*
9918  * Put the respective register value to sata_cmd_t for satacmd_flags.
9919  */
9920 static void
9921 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
9922 {
9923         if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
9924                 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
9925         if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
9926                 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
9927         if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
9928                 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
9929         if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
9930                 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
9931         if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
9932                 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
9933         if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
9934                 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
9935         if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
9936                 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
9937         if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
9938                 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
9939         if (scmd->satacmd_flags.sata_copy_out_device_reg)
9940                 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
9941         if (scmd->satacmd_flags.sata_copy_out_error_reg)
9942                 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
9943 }
9944 
9945 static void
9946 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
9947     uint32_t intr_status)
9948 {
9949         int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
9950 
9951         if (intr_status & AHCI_INTR_STATUS_IFS)
9952                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
9953                     "error", instance, port);
9954 
9955         if (intr_status & AHCI_INTR_STATUS_HBDS)
9956                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
9957                     instance, port);
9958 
9959         if (intr_status & AHCI_INTR_STATUS_HBFS)
9960                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
9961                     instance, port);
9962 
9963         if (intr_status & AHCI_INTR_STATUS_TFES)
9964                 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
9965                     instance, port);
9966 
9967         cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
9968             "recovery", instance, port);
9969 }
9970 
9971 static void
9972 ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
9973     uint32_t slot_tags)
9974 {
9975         ahci_port_t *ahci_portp;
9976         int tmp_slot;
9977         sata_pkt_t *spkt;
9978         sata_cmd_t cmd;
9979 
9980         ahci_portp = ahci_ctlp->ahcictl_ports[port];
9981         ASSERT(ahci_portp != NULL);
9982 
9983         while (slot_tags) {
9984                 tmp_slot = ddi_ffs(slot_tags) - 1;
9985                 if (tmp_slot == -1) {
9986                         break;
9987                 }
9988 
9989                 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
9990                 ASSERT(spkt != NULL);
9991                 cmd = spkt->satapkt_cmd;
9992 
9993                 cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
9994                     "features_reg = 0x%x sec_count_msb = 0x%x "
9995                     "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
9996                     "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
9997                     "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
9998                     "lba_high_lsb = 0x%x device_reg = 0x%x "
9999                     "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
10000                     cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
10001                     cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
10002                     cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
10003                     cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
10004                     cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
10005                     cmd.satacmd_device_reg, cmd.satacmd_addr_type,
10006                     *((uint32_t *)&(cmd.satacmd_flags)));
10007 
10008                 CLEAR_BIT(slot_tags, tmp_slot);
10009         }
10010 }
10011 
10012 /*
10013  * Dump the serror message to the log.
10014  */
10015 static void
10016 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
10017     uint32_t port_serror, int debug_only)
10018 {
10019         static char err_buf[512];
10020         static char err_msg_header[16];
10021         char *err_msg = err_buf;
10022 
10023         *err_buf = '\0';
10024         *err_msg_header = '\0';
10025 
10026         if (port_serror & SERROR_DATA_ERR_FIXED) {
10027                 err_msg = strcat(err_msg,
10028                     "\tRecovered Data Integrity Error (I)\n");
10029         }
10030 
10031         if (port_serror & SERROR_COMM_ERR_FIXED) {
10032                 err_msg = strcat(err_msg,
10033                     "\tRecovered Communication Error (M)\n");
10034         }
10035 
10036         if (port_serror & SERROR_DATA_ERR) {
10037                 err_msg = strcat(err_msg,
10038                     "\tTransient Data Integrity Error (T)\n");
10039         }
10040 
10041         if (port_serror & SERROR_PERSISTENT_ERR) {
10042                 err_msg = strcat(err_msg,
10043                     "\tPersistent Communication or Data Integrity Error (C)\n");
10044         }
10045 
10046         if (port_serror & SERROR_PROTOCOL_ERR) {
10047                 err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
10048         }
10049 
10050         if (port_serror & SERROR_INT_ERR) {
10051                 err_msg = strcat(err_msg, "\tInternal Error (E)\n");
10052         }
10053 
10054         if (port_serror & SERROR_PHY_RDY_CHG) {
10055                 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
10056         }
10057 
10058         if (port_serror & SERROR_PHY_INT_ERR) {
10059                 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
10060         }
10061 
10062         if (port_serror & SERROR_COMM_WAKE) {
10063                 err_msg = strcat(err_msg, "\tComm Wake (W)\n");
10064         }
10065 
10066         if (port_serror & SERROR_10B_TO_8B_ERR) {
10067                 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
10068         }
10069 
10070         if (port_serror & SERROR_DISPARITY_ERR) {
10071                 err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
10072         }
10073 
10074         if (port_serror & SERROR_CRC_ERR) {
10075                 err_msg = strcat(err_msg, "\tCRC Error (C)\n");
10076         }
10077 
10078         if (port_serror & SERROR_HANDSHAKE_ERR) {
10079                 err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
10080         }
10081 
10082         if (port_serror & SERROR_LINK_SEQ_ERR) {
10083                 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
10084         }
10085 
10086         if (port_serror & SERROR_TRANS_ERR) {
10087                 err_msg = strcat(err_msg,
10088                     "\tTransport state transition error (T)\n");
10089         }
10090 
10091         if (port_serror & SERROR_FIS_TYPE) {
10092                 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
10093         }
10094 
10095         if (port_serror & SERROR_EXCHANGED_ERR) {
10096                 err_msg = strcat(err_msg, "\tExchanged (X)\n");
10097         }
10098 
10099         if (*err_msg == '\0')
10100                 return;
10101 
10102         if (debug_only) {
10103                 (void) sprintf(err_msg_header, "port %d", port);
10104                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
10105                 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
10106         } else if (ahci_ctlp) {
10107                 cmn_err(CE_WARN, "!ahci%d: %s %s",
10108                     ddi_get_instance(ahci_ctlp->ahcictl_dip),
10109                     err_msg_header, err_msg);
10110 
10111                 /* sata trace debug */
10112                 sata_trace_debug(ahci_ctlp->ahcictl_dip,
10113                     "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
10114                     err_msg_header, err_msg);
10115         } else {
10116                 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);
10117 
10118                 /* sata trace debug */
10119                 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
10120         }
10121 }
10122 
10123 /*
10124  * Translate the sata_address_t type into the ahci_addr_t type.
10125  * sata_device.satadev_addr structure is used as source.
10126  */
10127 static void
10128 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
10129     ahci_addr_t *ahci_addrp)
10130 {
10131         sata_address_t *sata_addrp = &sd->satadev_addr;
10132         ahci_addrp->aa_port =
10133             ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
10134         ahci_addrp->aa_pmport = sata_addrp->pmport;
10135 
10136         switch (sata_addrp->qual) {
10137         case SATA_ADDR_DCPORT:
10138         case SATA_ADDR_CPORT:
10139                 ahci_addrp->aa_qual = AHCI_ADDR_PORT;
10140                 break;
10141         case SATA_ADDR_PMULT:
10142         case SATA_ADDR_PMULT_SPEC:
10143                 ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
10144                 break;
10145         case SATA_ADDR_DPMPORT:
10146         case SATA_ADDR_PMPORT:
10147                 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
10148                 break;
10149         case SATA_ADDR_NULL:
10150         default:
10151                 /* something went wrong */
10152                 ahci_addrp->aa_qual = AHCI_ADDR_NULL;
10153                 break;
10154         }
10155 }
10156 
10157 /*
10158  * This routine is to calculate the total number of ports implemented
10159  * by the HBA.
10160  */
10161 static int
10162 ahci_get_num_implemented_ports(uint32_t ports_implemented)
10163 {
10164         uint8_t i;
10165         int num = 0;
10166 
10167         for (i = 0; i < AHCI_MAX_PORTS; i++) {
10168                 if (((uint32_t)0x1 << i) & ports_implemented)
10169                         num++;
10170         }
10171 
10172         return (num);
10173 }
10174 
10175 #if AHCI_DEBUG
10176 static void
10177 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
10178 {
10179         static char name[16];
10180         va_list ap;
10181 
10182         mutex_enter(&ahci_log_mutex);
10183 
10184         va_start(ap, fmt);
10185         if (ahci_ctlp) {
10186                 (void) sprintf(name, "ahci%d: ",
10187                     ddi_get_instance(ahci_ctlp->ahcictl_dip));
10188         } else {
10189                 (void) sprintf(name, "ahci: ");
10190         }
10191 
10192         (void) vsprintf(ahci_log_buf, fmt, ap);
10193         va_end(ap);
10194 
10195         cmn_err(level, "%s%s", name, ahci_log_buf);
10196 
10197         mutex_exit(&ahci_log_mutex);
10198 }
10199 #endif
10200 
10201 /*
10202  * quiesce(9E) entry point.
10203  *
10204  * This function is called when the system is single-threaded at high
10205  * PIL with preemption disabled. Therefore, this function must not be
10206  * blocked.
10207  *
10208  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
10209  * DDI_FAILURE indicates an error condition and should almost never happen.
10210  */
10211 static int
10212 ahci_quiesce(dev_info_t *dip)
10213 {
10214         ahci_ctl_t *ahci_ctlp;
10215         ahci_port_t *ahci_portp;
10216         int instance, port;
10217 
10218         instance = ddi_get_instance(dip);
10219         ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
10220 
10221         if (ahci_ctlp == NULL)
10222                 return (DDI_FAILURE);
10223 
10224 #if AHCI_DEBUG
10225         ahci_debug_flags = 0;
10226 #endif
10227 
10228         ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;
10229 
10230         /* disable all the interrupts. */
10231         ahci_disable_all_intrs(ahci_ctlp);
10232 
10233         for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
10234                 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
10235                         continue;
10236                 }
10237 
10238                 ahci_portp = ahci_ctlp->ahcictl_ports[port];
10239 
10240                 /*
10241                  * Stop the port by clearing PxCMD.ST
10242                  *
10243                  * Here we must disable the port interrupt because
10244                  * ahci_disable_all_intrs only clear GHC.IE, and IS
10245                  * register will be still set if PxIE is enabled.
10246                  * When ahci shares one IRQ with other drivers, the
10247                  * intr handler may claim the intr mistakenly.
10248                  */
10249                 ahci_disable_port_intrs(ahci_ctlp, port);
10250                 (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
10251                     ahci_portp, port);
10252         }
10253 
10254         ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;
10255 
10256         return (DDI_SUCCESS);
10257 }
10258 
10259 /*
10260  * The function will add a sata packet to the done queue.
10261  */
10262 static void
10263 ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
10264 {
10265         ASSERT(satapkt != NULL);
10266         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10267 
10268         /* set the reason for all packets */
10269         satapkt->satapkt_reason = reason;
10270         satapkt->satapkt_hba_driver_private = NULL;
10271 
10272         if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10273             satapkt->satapkt_comp) {
10274                 /*
10275                  * only add to queue when mode is not synch and there is
10276                  * completion callback
10277                  */
10278                 *ahci_portp->ahciport_doneqtail = satapkt;
10279                 ahci_portp->ahciport_doneqtail =
10280                     (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
10281                 ahci_portp->ahciport_doneq_len++;
10282 
10283         } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
10284             ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
10285                 /*
10286                  * for sync/non-poll mode, just call cv_broadcast
10287                  */
10288                 cv_broadcast(&ahci_portp->ahciport_cv);
10289 }
10290 
10291 /*
10292  * The function will call completion callback of sata packet on the
10293  * completed queue
10294  */
10295 static void
10296 ahci_flush_doneq(ahci_port_t *ahci_portp)
10297 {
10298         sata_pkt_t *satapkt, *next;
10299 
10300         ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
10301 
10302         if (ahci_portp->ahciport_doneq) {
10303                 satapkt = ahci_portp->ahciport_doneq;
10304 
10305                 ahci_portp->ahciport_doneq = NULL;
10306                 ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
10307                 ahci_portp->ahciport_doneq_len = 0;
10308 
10309                 mutex_exit(&ahci_portp->ahciport_mutex);
10310 
10311                 while (satapkt != NULL) {
10312                         next = satapkt->satapkt_hba_driver_private;
10313                         satapkt->satapkt_hba_driver_private = NULL;
10314 
10315                         /* Call the callback */
10316                         (*satapkt->satapkt_comp)(satapkt);
10317 
10318                         satapkt = next;
10319                 }
10320 
10321                 mutex_enter(&ahci_portp->ahciport_mutex);
10322         }
10323 }