1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 /*
  27  * Copyright 2014 Garrett D'Amore <garrett@damore.org>
  28  */
  29 
  30 /*
  31  * sun4 specific DDI implementation
  32  */
  33 #include <sys/cpuvar.h>
  34 #include <sys/ddi_subrdefs.h>
  35 #include <sys/machsystm.h>
  36 #include <sys/sunndi.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/ontrap.h>
  39 #include <vm/seg_kmem.h>
  40 #include <sys/membar.h>
  41 #include <sys/dditypes.h>
  42 #include <sys/ndifm.h>
  43 #include <sys/fm/io/ddi.h>
  44 #include <sys/ivintr.h>
  45 #include <sys/bootconf.h>
  46 #include <sys/conf.h>
  47 #include <sys/ethernet.h>
  48 #include <sys/idprom.h>
  49 #include <sys/promif.h>
  50 #include <sys/prom_plat.h>
  51 #include <sys/systeminfo.h>
  52 #include <sys/fpu/fpusystm.h>
  53 #include <sys/vm.h>
  54 #include <sys/ddi_isa.h>
  55 #include <sys/modctl.h>
  56 
  57 dev_info_t *get_intr_parent(dev_info_t *, dev_info_t *,
  58     ddi_intr_handle_impl_t *);
  59 #pragma weak get_intr_parent
  60 
  61 int process_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
  62     ddi_intr_handle_impl_t *, void *);
  63 #pragma weak process_intr_ops
  64 
  65 void cells_1275_copy(prop_1275_cell_t *, prop_1275_cell_t *, int32_t);
  66     prop_1275_cell_t *cells_1275_cmp(prop_1275_cell_t *, prop_1275_cell_t *,
  67     int32_t len);
  68 #pragma weak cells_1275_copy
  69 
  70 /*
  71  * Wrapper for ddi_prop_lookup_int_array().
  72  * This is handy because it returns the prop length in
  73  * bytes which is what most of the callers require.
  74  */
  75 
  76 static int
  77 get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen)
  78 {
  79         int ret;
  80 
  81         if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di,
  82             DDI_PROP_DONTPASS, pname, pval, plen)) == DDI_PROP_SUCCESS) {
  83                 *plen = (*plen) * (uint_t)sizeof (int);
  84         }
  85         return (ret);
  86 }
  87 
  88 /*
  89  * SECTION: DDI Node Configuration
  90  */
  91 
  92 /*
  93  * init_regspec_64:
  94  *
  95  * If the parent #size-cells is 2, convert the upa-style or
  96  * safari-style reg property from 2-size cells to 1 size cell
  97  * format, ignoring the size_hi, which must be zero for devices.
  98  * (It won't be zero in the memory list properties in the memory
  99  * nodes, but that doesn't matter here.)
 100  */
 101 struct ddi_parent_private_data *
 102 init_regspec_64(dev_info_t *dip)
 103 {
 104         struct ddi_parent_private_data *pd;
 105         dev_info_t *parent;
 106         int size_cells;
 107 
 108         /*
 109          * If there are no "reg"s in the child node, return.
 110          */
 111         pd = ddi_get_parent_data(dip);
 112         if ((pd == NULL) || (pd->par_nreg == 0)) {
 113                 return (pd);
 114         }
 115         parent = ddi_get_parent(dip);
 116 
 117         size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
 118             DDI_PROP_DONTPASS, "#size-cells", 1);
 119 
 120         if (size_cells != 1)  {
 121 
 122                 int n, j;
 123                 struct regspec *irp;
 124                 struct reg_64 {
 125                         uint_t addr_hi, addr_lo, size_hi, size_lo;
 126                 };
 127                 struct reg_64 *r64_rp;
 128                 struct regspec *rp;
 129                 uint_t len = 0;
 130                 int *reg_prop;
 131 
 132                 ASSERT(size_cells == 2);
 133 
 134                 /*
 135                  * We already looked the property up once before if
 136                  * pd is non-NULL.
 137                  */
 138                 (void) ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
 139                     DDI_PROP_DONTPASS, OBP_REG, &reg_prop, &len);
 140                 ASSERT(len != 0);
 141 
 142                 n = sizeof (struct reg_64) / sizeof (int);
 143                 n = len / n;
 144 
 145                 /*
 146                  * We're allocating a buffer the size of the PROM's property,
 147                  * but we're only using a smaller portion when we assign it
 148                  * to a regspec.  We do this so that in the
 149                  * impl_ddi_sunbus_removechild function, we will
 150                  * always free the right amount of memory.
 151                  */
 152                 irp = rp = (struct regspec *)reg_prop;
 153                 r64_rp = (struct reg_64 *)pd->par_reg;
 154 
 155                 for (j = 0; j < n; ++j, ++rp, ++r64_rp) {
 156                         ASSERT(r64_rp->size_hi == 0);
 157                         rp->regspec_bustype = r64_rp->addr_hi;
 158                         rp->regspec_addr = r64_rp->addr_lo;
 159                         rp->regspec_size = r64_rp->size_lo;
 160                 }
 161 
 162                 ddi_prop_free((void *)pd->par_reg);
 163                 pd->par_nreg = n;
 164                 pd->par_reg = irp;
 165         }
 166         return (pd);
 167 }
 168 
 169 /*
 170  * Create a ddi_parent_private_data structure from the ddi properties of
 171  * the dev_info node.
 172  *
 173  * The "reg" is required if the driver wishes to create mappings on behalf
 174  * of the device. The "reg" property is assumed to be a list of at least
 175  * one triplet
 176  *
 177  *      <bustype, address, size>*1
 178  *
 179  * The "interrupt" property is no longer part of parent private data on
 180  * sun4u. The interrupt parent is may not be the device tree parent.
 181  *
 182  * The "ranges" property describes the mapping of child addresses to parent
 183  * addresses.
 184  *
 185  * N.B. struct rangespec is defined for the following default values:
 186  *                      parent  child
 187  *      #address-cells  2       2
 188  *      #size-cells     1       1
 189  * This function doesn't deal with non-default cells and will not create
 190  * ranges in such cases.
 191  */
 192 void
 193 make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd)
 194 {
 195         struct ddi_parent_private_data *pdptr;
 196         int *reg_prop, *rng_prop;
 197         uint_t reg_len = 0, rng_len = 0;
 198         dev_info_t *parent;
 199         int parent_addr_cells, parent_size_cells;
 200         int child_addr_cells, child_size_cells;
 201 
 202         *ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP);
 203 
 204         /*
 205          * root node has no parent private data, so *ppd should
 206          * be initialized for naming to work properly.
 207          */
 208         if ((parent = ddi_get_parent(child)) == NULL)
 209                 return;
 210 
 211         /*
 212          * Set reg field of parent data from "reg" property
 213          */
 214         if ((get_prop_int_array(child, OBP_REG, &reg_prop, &reg_len)
 215             == DDI_PROP_SUCCESS) && (reg_len != 0)) {
 216                 pdptr->par_nreg = (int)(reg_len / sizeof (struct regspec));
 217                 pdptr->par_reg = (struct regspec *)reg_prop;
 218         }
 219 
 220         /*
 221          * "ranges" property ...
 222          *
 223          * This function does not handle cases where #address-cells != 2
 224          * and * min(parent, child) #size-cells != 1 (see bugid 4211124).
 225          *
 226          * Nexus drivers with such exceptions (e.g. pci ranges)
 227          * should either create a separate function for handling
 228          * ranges or not use parent private data to store ranges.
 229          */
 230 
 231         /* root node has no ranges */
 232         if ((parent = ddi_get_parent(child)) == NULL)
 233                 return;
 234 
 235         child_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
 236             DDI_PROP_DONTPASS, "#address-cells", 2);
 237         child_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
 238             DDI_PROP_DONTPASS, "#size-cells", 1);
 239         parent_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
 240             DDI_PROP_DONTPASS, "#address-cells", 2);
 241         parent_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
 242             DDI_PROP_DONTPASS, "#size-cells", 1);
 243         if (child_addr_cells != 2 || parent_addr_cells != 2 ||
 244             (child_size_cells != 1 && parent_size_cells != 1)) {
 245                 NDI_CONFIG_DEBUG((CE_NOTE, "!ranges not made in parent data; "
 246                     "#address-cells or #size-cells have non-default value"));
 247                 return;
 248         }
 249 
 250         if (get_prop_int_array(child, OBP_RANGES, &rng_prop, &rng_len)
 251             == DDI_PROP_SUCCESS) {
 252                 pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec));
 253                 pdptr->par_rng = (struct rangespec *)rng_prop;
 254         }
 255 }
 256 
 257 /*
 258  * Free ddi_parent_private_data structure
 259  */
 260 void
 261 impl_free_ddi_ppd(dev_info_t *dip)
 262 {
 263         struct ddi_parent_private_data *pdptr = ddi_get_parent_data(dip);
 264 
 265         if (pdptr == NULL)
 266                 return;
 267 
 268         if (pdptr->par_nrng != 0)
 269                 ddi_prop_free((void *)pdptr->par_rng);
 270 
 271         if (pdptr->par_nreg != 0)
 272                 ddi_prop_free((void *)pdptr->par_reg);
 273 
 274         kmem_free(pdptr, sizeof (*pdptr));
 275         ddi_set_parent_data(dip, NULL);
 276 }
 277 
 278 /*
 279  * Name a child of sun busses based on the reg spec.
 280  * Handles the following properties:
 281  *
 282  *      Property        value
 283  *      Name            type
 284  *
 285  *      reg             register spec
 286  *      interrupts      new (bus-oriented) interrupt spec
 287  *      ranges          range spec
 288  *
 289  * This may be called multiple times, independent of
 290  * initchild calls.
 291  */
 292 static int
 293 impl_sunbus_name_child(dev_info_t *child, char *name, int namelen)
 294 {
 295         struct ddi_parent_private_data *pdptr;
 296         struct regspec *rp;
 297 
 298         /*
 299          * Fill in parent-private data and this function returns to us
 300          * an indication if it used "registers" to fill in the data.
 301          */
 302         if (ddi_get_parent_data(child) == NULL) {
 303                 make_ddi_ppd(child, &pdptr);
 304                 ddi_set_parent_data(child, pdptr);
 305         }
 306 
 307         /*
 308          * No reg property, return null string as address
 309          * (e.g. root node)
 310          */
 311         name[0] = '\0';
 312         if (sparc_pd_getnreg(child) == 0) {
 313                 return (DDI_SUCCESS);
 314         }
 315 
 316         rp = sparc_pd_getreg(child, 0);
 317         (void) snprintf(name, namelen, "%x,%x",
 318             rp->regspec_bustype, rp->regspec_addr);
 319         return (DDI_SUCCESS);
 320 }
 321 
 322 
 323 /*
 324  * Called from the bus_ctl op of some drivers.
 325  * to implement the DDI_CTLOPS_INITCHILD operation.
 326  *
 327  * NEW drivers should NOT use this function, but should declare
 328  * there own initchild/uninitchild handlers. (This function assumes
 329  * the layout of the parent private data and the format of "reg",
 330  * "ranges", "interrupts" properties and that #address-cells and
 331  * #size-cells of the parent bus are defined to be default values.)
 332  */
 333 int
 334 impl_ddi_sunbus_initchild(dev_info_t *child)
 335 {
 336         char name[MAXNAMELEN];
 337 
 338         (void) impl_sunbus_name_child(child, name, MAXNAMELEN);
 339         ddi_set_name_addr(child, name);
 340 
 341         /*
 342          * Try to merge .conf node. If successful, return failure to
 343          * remove this child.
 344          */
 345         if ((ndi_dev_is_persistent_node(child) == 0) &&
 346             (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) {
 347                 impl_ddi_sunbus_removechild(child);
 348                 return (DDI_FAILURE);
 349         }
 350         return (DDI_SUCCESS);
 351 }
 352 
 353 /*
 354  * A better name for this function would be impl_ddi_sunbus_uninitchild()
 355  * It does not remove the child, it uninitializes it, reclaiming the
 356  * resources taken by impl_ddi_sunbus_initchild.
 357  */
 358 void
 359 impl_ddi_sunbus_removechild(dev_info_t *dip)
 360 {
 361         impl_free_ddi_ppd(dip);
 362         ddi_set_name_addr(dip, NULL);
 363         /*
 364          * Strip the node to properly convert it back to prototype form
 365          */
 366         impl_rem_dev_props(dip);
 367 }
 368 
 369 /*
 370  * SECTION: DDI Interrupt
 371  */
 372 
 373 void
 374 cells_1275_copy(prop_1275_cell_t *from, prop_1275_cell_t *to, int32_t len)
 375 {
 376         int i;
 377         for (i = 0; i < len; i++)
 378                 *to = *from;
 379 }
 380 
 381 prop_1275_cell_t *
 382 cells_1275_cmp(prop_1275_cell_t *cell1, prop_1275_cell_t *cell2, int32_t len)
 383 {
 384         prop_1275_cell_t *match_cell = 0;
 385         int32_t i;
 386 
 387         for (i = 0; i < len; i++)
 388                 if (cell1[i] != cell2[i]) {
 389                         match_cell = &cell1[i];
 390                         break;
 391                 }
 392 
 393         return (match_cell);
 394 }
 395 
 396 /*
 397  * get_intr_parent() is a generic routine that process a 1275 interrupt
 398  * map (imap) property.  This function returns a dev_info_t structure
 399  * which claims ownership of the interrupt domain.
 400  * It also returns the new interrupt translation within this new domain.
 401  * If an interrupt-parent or interrupt-map property are not found,
 402  * then we fallback to using the device tree's parent.
 403  *
 404  * imap entry format:
 405  * <reg>,<interrupt>,<phandle>,<translated interrupt>
 406  * reg - The register specification in the interrupts domain
 407  * interrupt - The interrupt specification
 408  * phandle - PROM handle of the device that owns the xlated interrupt domain
 409  * translated interrupt - interrupt specifier in the parents domain
 410  * note: <reg>,<interrupt> - The reg and interrupt can be combined to create
 411  *      a unique entry called a unit interrupt specifier.
 412  *
 413  * Here's the processing steps:
 414  * step1 - If the interrupt-parent property exists, create the ispec and
 415  *      return the dip of the interrupt parent.
 416  * step2 - Extract the interrupt-map property and the interrupt-map-mask
 417  *      If these don't exist, just return the device tree parent.
 418  * step3 - build up the unit interrupt specifier to match against the
 419  *      interrupt map property
 420  * step4 - Scan the interrupt-map property until a match is found
 421  * step4a - Extract the interrupt parent
 422  * step4b - Compare the unit interrupt specifier
 423  */
 424 dev_info_t *
 425 get_intr_parent(dev_info_t *pdip, dev_info_t *dip, ddi_intr_handle_impl_t *hdlp)
 426 {
 427         prop_1275_cell_t *imap, *imap_mask, *scan, *reg_p, *match_req;
 428         int32_t imap_sz, imap_cells, imap_scan_cells, imap_mask_sz,
 429             addr_cells, intr_cells, reg_len, i, j;
 430         int32_t match_found = 0;
 431         dev_info_t *intr_parent_dip = NULL;
 432         uint32_t *intr = &hdlp->ih_vector;
 433         uint32_t nodeid;
 434 #ifdef DEBUG
 435         static int debug = 0;
 436 #endif
 437 
 438         /*
 439          * step1
 440          * If we have an interrupt-parent property, this property represents
 441          * the nodeid of our interrupt parent.
 442          */
 443         if ((nodeid = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
 444             "interrupt-parent", -1)) != -1) {
 445                 intr_parent_dip = e_ddi_nodeid_to_dip(nodeid);
 446                 ASSERT(intr_parent_dip);
 447 
 448                 /*
 449                  * Attach the interrupt parent.
 450                  *
 451                  * N.B. e_ddi_nodeid_to_dip() isn't safe under DR.
 452                  *      Also, interrupt parent isn't held. This needs
 453                  *      to be revisited if DR-capable platforms implement
 454                  *      interrupt redirection.
 455                  */
 456                 if (i_ddi_attach_node_hierarchy(intr_parent_dip)
 457                     != DDI_SUCCESS) {
 458                         ndi_rele_devi(intr_parent_dip);
 459                         return (NULL);
 460                 }
 461 
 462                 return (intr_parent_dip);
 463         }
 464 
 465         /*
 466          * step2
 467          * Get interrupt map structure from PROM property
 468          */
 469         if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
 470             "interrupt-map", (caddr_t)&imap, &imap_sz)
 471             != DDI_PROP_SUCCESS) {
 472                 /*
 473                  * If we don't have an imap property, default to using the
 474                  * device tree.
 475                  */
 476 
 477                 ndi_hold_devi(pdip);
 478                 return (pdip);
 479         }
 480 
 481         /* Get the interrupt mask property */
 482         if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
 483             "interrupt-map-mask", (caddr_t)&imap_mask, &imap_mask_sz)
 484             != DDI_PROP_SUCCESS) {
 485                 /*
 486                  * If we don't find this property, we have to fail the request
 487                  * because the 1275 imap property wasn't defined correctly.
 488                  */
 489                 ASSERT(intr_parent_dip == NULL);
 490                 goto exit2;
 491         }
 492 
 493         /* Get the address cell size */
 494         addr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
 495             "#address-cells", 2);
 496 
 497         /* Get the interrupts cell size */
 498         intr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
 499             "#interrupt-cells", 1);
 500 
 501         /*
 502          * step3
 503          * Now lets build up the unit interrupt specifier e.g. reg,intr
 504          * and apply the imap mask.  match_req will hold this when we're
 505          * through.
 506          */
 507         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
 508             (caddr_t)&reg_p, &reg_len) != DDI_SUCCESS) {
 509                 ASSERT(intr_parent_dip == NULL);
 510                 goto exit3;
 511         }
 512 
 513         match_req = kmem_alloc(CELLS_1275_TO_BYTES(addr_cells) +
 514             CELLS_1275_TO_BYTES(intr_cells), KM_SLEEP);
 515 
 516         for (i = 0; i < addr_cells; i++)
 517                 match_req[i] = (reg_p[i] & imap_mask[i]);
 518 
 519         for (j = 0; j < intr_cells; i++, j++)
 520                 match_req[i] = (intr[j] & imap_mask[i]);
 521 
 522         /* Calculate the imap size in cells */
 523         imap_cells = BYTES_TO_1275_CELLS(imap_sz);
 524 
 525 #ifdef DEBUG
 526         if (debug)
 527                 prom_printf("reg cell size 0x%x, intr cell size 0x%x, "
 528                     "match_request 0x%p, imap 0x%p\n", addr_cells, intr_cells,
 529                     (void *)match_req, (void *)imap);
 530 #endif
 531 
 532         /*
 533          * Scan the imap property looking for a match of the interrupt unit
 534          * specifier.  This loop is rather complex since the data within the
 535          * imap property may vary in size.
 536          */
 537         for (scan = imap, imap_scan_cells = i = 0;
 538             imap_scan_cells < imap_cells; scan += i, imap_scan_cells += i) {
 539                 int new_intr_cells;
 540 
 541                 /* Set the index to the nodeid field */
 542                 i = addr_cells + intr_cells;
 543 
 544                 /*
 545                  * step4a
 546                  * Translate the nodeid field to a dip
 547                  */
 548                 ASSERT(intr_parent_dip == NULL);
 549                 intr_parent_dip = e_ddi_nodeid_to_dip((uint_t)scan[i++]);
 550 
 551                 ASSERT(intr_parent_dip != 0);
 552 #ifdef DEBUG
 553                 if (debug)
 554                         prom_printf("scan 0x%p\n", (void *)scan);
 555 #endif
 556                 /*
 557                  * The tmp_dip describes the new domain, get it's interrupt
 558                  * cell size
 559                  */
 560                 new_intr_cells = ddi_getprop(DDI_DEV_T_ANY, intr_parent_dip, 0,
 561                     "#interrupts-cells", 1);
 562 
 563                 /*
 564                  * step4b
 565                  * See if we have a match on the interrupt unit specifier
 566                  */
 567                 if (cells_1275_cmp(match_req, scan, addr_cells + intr_cells)
 568                     == 0) {
 569                         uint32_t *intr;
 570 
 571                         match_found = 1;
 572 
 573                         /*
 574                          * If we have an imap parent whose not in our device
 575                          * tree path, we need to hold and install that driver.
 576                          */
 577                         if (i_ddi_attach_node_hierarchy(intr_parent_dip)
 578                             != DDI_SUCCESS) {
 579                                 ndi_rele_devi(intr_parent_dip);
 580                                 intr_parent_dip = (dev_info_t *)NULL;
 581                                 goto exit4;
 582                         }
 583 
 584                         /*
 585                          * We need to handcraft an ispec along with a bus
 586                          * interrupt value, so we can dup it into our
 587                          * standard ispec structure.
 588                          */
 589                         /* Extract the translated interrupt information */
 590                         intr = kmem_alloc(
 591                             CELLS_1275_TO_BYTES(new_intr_cells), KM_SLEEP);
 592 
 593                         for (j = 0; j < new_intr_cells; j++, i++)
 594                                 intr[j] = scan[i];
 595 
 596                         cells_1275_copy(intr, &hdlp->ih_vector, new_intr_cells);
 597 
 598                         kmem_free(intr, CELLS_1275_TO_BYTES(new_intr_cells));
 599 
 600 #ifdef DEBUG
 601                         if (debug)
 602                                 prom_printf("dip 0x%p\n",
 603                                     (void *)intr_parent_dip);
 604 #endif
 605                         break;
 606                 } else {
 607 #ifdef DEBUG
 608                         if (debug)
 609                                 prom_printf("dip 0x%p\n",
 610                                     (void *)intr_parent_dip);
 611 #endif
 612                         ndi_rele_devi(intr_parent_dip);
 613                         intr_parent_dip = NULL;
 614                         i += new_intr_cells;
 615                 }
 616         }
 617 
 618         /*
 619          * If we haven't found our interrupt parent at this point, fallback
 620          * to using the device tree.
 621          */
 622         if (!match_found) {
 623                 ndi_hold_devi(pdip);
 624                 ASSERT(intr_parent_dip == NULL);
 625                 intr_parent_dip = pdip;
 626         }
 627 
 628         ASSERT(intr_parent_dip != NULL);
 629 
 630 exit4:
 631         kmem_free(reg_p, reg_len);
 632         kmem_free(match_req, CELLS_1275_TO_BYTES(addr_cells) +
 633             CELLS_1275_TO_BYTES(intr_cells));
 634 
 635 exit3:
 636         kmem_free(imap_mask, imap_mask_sz);
 637 
 638 exit2:
 639         kmem_free(imap, imap_sz);
 640 
 641         return (intr_parent_dip);
 642 }
 643 
 644 /*
 645  * process_intr_ops:
 646  *
 647  * Process the interrupt op via the interrupt parent.
 648  */
 649 int
 650 process_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t op,
 651     ddi_intr_handle_impl_t *hdlp, void *result)
 652 {
 653         int             ret = DDI_FAILURE;
 654 
 655         if (NEXUS_HAS_INTR_OP(pdip)) {
 656                 ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->
 657                     bus_intr_op)) (pdip, rdip, op, hdlp, result);
 658         } else {
 659                 cmn_err(CE_WARN, "Failed to process interrupt "
 660                     "for %s%d due to down-rev nexus driver %s%d",
 661                     ddi_get_name(rdip), ddi_get_instance(rdip),
 662                     ddi_get_name(pdip), ddi_get_instance(pdip));
 663         }
 664 
 665         return (ret);
 666 }
 667 
 668 /*ARGSUSED*/
 669 uint_t
 670 softlevel1(caddr_t arg)
 671 {
 672         softint();
 673         return (1);
 674 }
 675 
 676 /*
 677  * indirection table, to save us some large switch statements
 678  * NOTE: This must agree with "INTLEVEL_foo" constants in
 679  *      <sys/avintr.h>
 680  */
 681 struct autovec *const vectorlist[] = { 0 };
 682 
 683 /*
 684  * This value is exported here for the functions in avintr.c
 685  */
 686 const uint_t maxautovec = (sizeof (vectorlist) / sizeof (vectorlist[0]));
 687 
 688 /*
 689  * Check for machine specific interrupt levels which cannot be reassigned by
 690  * settrap(), sun4u version.
 691  *
 692  * sun4u does not support V8 SPARC "fast trap" handlers.
 693  */
 694 /*ARGSUSED*/
 695 int
 696 exclude_settrap(int lvl)
 697 {
 698         return (1);
 699 }
 700 
 701 /*
 702  * Check for machine specific interrupt levels which cannot have interrupt
 703  * handlers added. We allow levels 1 through 15; level 0 is nonsense.
 704  */
 705 /*ARGSUSED*/
 706 int
 707 exclude_level(int lvl)
 708 {
 709         return ((lvl < 1) || (lvl > 15));
 710 }
 711 
 712 /*
 713  * Wrapper functions used by New DDI interrupt framework.
 714  */
 715 
 716 /*
 717  * i_ddi_intr_ops:
 718  */
 719 int
 720 i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op,
 721     ddi_intr_handle_impl_t *hdlp, void *result)
 722 {
 723         dev_info_t      *pdip = ddi_get_parent(dip);
 724         int             ret = DDI_FAILURE;
 725 
 726         /*
 727          * The following check is required to address
 728          * one of the test case of ADDI test suite.
 729          */
 730         if (pdip == NULL)
 731                 return (DDI_FAILURE);
 732 
 733         if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
 734                 return (process_intr_ops(pdip, rdip, op, hdlp, result));
 735 
 736         if (hdlp->ih_vector == 0)
 737                 hdlp->ih_vector = i_ddi_get_inum(rdip, hdlp->ih_inum);
 738 
 739         if (hdlp->ih_pri == 0)
 740                 hdlp->ih_pri = i_ddi_get_intr_pri(rdip, hdlp->ih_inum);
 741 
 742         switch (op) {
 743         case DDI_INTROP_ADDISR:
 744         case DDI_INTROP_REMISR:
 745         case DDI_INTROP_GETTARGET:
 746         case DDI_INTROP_SETTARGET:
 747         case DDI_INTROP_ENABLE:
 748         case DDI_INTROP_DISABLE:
 749         case DDI_INTROP_BLOCKENABLE:
 750         case DDI_INTROP_BLOCKDISABLE:
 751                 /*
 752                  * Try and determine our parent and possibly an interrupt
 753                  * translation. intr parent dip returned held
 754                  */
 755                 if ((pdip = get_intr_parent(pdip, dip, hdlp)) == NULL)
 756                         goto done;
 757         }
 758 
 759         ret = process_intr_ops(pdip, rdip, op, hdlp, result);
 760 
 761 done:
 762         switch (op) {
 763         case DDI_INTROP_ADDISR:
 764         case DDI_INTROP_REMISR:
 765         case DDI_INTROP_ENABLE:
 766         case DDI_INTROP_DISABLE:
 767         case DDI_INTROP_BLOCKENABLE:
 768         case DDI_INTROP_BLOCKDISABLE:
 769                 /* Release hold acquired in get_intr_parent() */
 770                 if (pdip)
 771                         ndi_rele_devi(pdip);
 772         }
 773 
 774         hdlp->ih_vector = 0;
 775 
 776         return (ret);
 777 }
 778 
 779 /*
 780  * i_ddi_add_ivintr:
 781  */
 782 /*ARGSUSED*/
 783 int
 784 i_ddi_add_ivintr(ddi_intr_handle_impl_t *hdlp)
 785 {
 786         /*
 787          * If the PIL was set and is valid use it, otherwise
 788          * default it to 1
 789          */
 790         if ((hdlp->ih_pri < 1) || (hdlp->ih_pri > PIL_MAX))
 791                 hdlp->ih_pri = 1;
 792 
 793         VERIFY(add_ivintr(hdlp->ih_vector, hdlp->ih_pri,
 794             (intrfunc)hdlp->ih_cb_func, hdlp->ih_cb_arg1,
 795             hdlp->ih_cb_arg2, NULL) == 0);
 796 
 797         return (DDI_SUCCESS);
 798 }
 799 
 800 /*
 801  * i_ddi_rem_ivintr:
 802  */
 803 /*ARGSUSED*/
 804 void
 805 i_ddi_rem_ivintr(ddi_intr_handle_impl_t *hdlp)
 806 {
 807         VERIFY(rem_ivintr(hdlp->ih_vector, hdlp->ih_pri) == 0);
 808 }
 809 
 810 /*
 811  * i_ddi_get_inum - Get the interrupt number property from the
 812  * specified device. Note that this function is called only for
 813  * the FIXED interrupt type.
 814  */
 815 uint32_t
 816 i_ddi_get_inum(dev_info_t *dip, uint_t inumber)
 817 {
 818         int32_t                 intrlen, intr_cells, max_intrs;
 819         prop_1275_cell_t        *ip, intr_sz;
 820         uint32_t                intr = 0;
 821 
 822         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
 823             DDI_PROP_CANSLEEP,
 824             "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
 825 
 826                 intr_cells = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
 827                     "#interrupt-cells", 1);
 828 
 829                 /* adjust for number of bytes */
 830                 intr_sz = CELLS_1275_TO_BYTES(intr_cells);
 831 
 832                 /* Calculate the number of interrupts */
 833                 max_intrs = intrlen / intr_sz;
 834 
 835                 if (inumber < max_intrs) {
 836                         prop_1275_cell_t *intrp = ip;
 837 
 838                         /* Index into interrupt property */
 839                         intrp += (inumber * intr_cells);
 840 
 841                         cells_1275_copy(intrp, &intr, intr_cells);
 842                 }
 843 
 844                 kmem_free(ip, intrlen);
 845         }
 846 
 847         return (intr);
 848 }
 849 
 850 /*
 851  * i_ddi_get_intr_pri - Get the interrupt-priorities property from
 852  * the specified device. Note that this function is called only for
 853  * the FIXED interrupt type.
 854  */
 855 uint32_t
 856 i_ddi_get_intr_pri(dev_info_t *dip, uint_t inumber)
 857 {
 858         uint32_t        *intr_prio_p;
 859         uint32_t        pri = 0;
 860         int32_t         i;
 861 
 862         /*
 863          * Use the "interrupt-priorities" property to determine the
 864          * the pil/ipl for the interrupt handler.
 865          */
 866         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
 867             "interrupt-priorities", (caddr_t)&intr_prio_p,
 868             &i) == DDI_SUCCESS) {
 869                 if (inumber < (i / sizeof (int32_t)))
 870                         pri = intr_prio_p[inumber];
 871                 kmem_free(intr_prio_p, i);
 872         }
 873 
 874         return (pri);
 875 }
 876 
 877 int
 878 i_ddi_get_intx_nintrs(dev_info_t *dip)
 879 {
 880         int32_t intrlen;
 881         prop_1275_cell_t intr_sz;
 882         prop_1275_cell_t *ip;
 883         int32_t ret = 0;
 884 
 885         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
 886             DDI_PROP_CANSLEEP,
 887             "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
 888 
 889                 intr_sz = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
 890                     "#interrupt-cells", 1);
 891                 /* adjust for number of bytes */
 892                 intr_sz = CELLS_1275_TO_BYTES(intr_sz);
 893 
 894                 ret = intrlen / intr_sz;
 895 
 896                 kmem_free(ip, intrlen);
 897         }
 898 
 899         return (ret);
 900 }
 901 
 902 /*
 903  * i_ddi_add_softint - allocate and add a software interrupt.
 904  *
 905  * NOTE: All software interrupts that are registered through DDI
 906  *       should be triggered only on a single target or CPU.
 907  */
 908 int
 909 i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp)
 910 {
 911         if ((hdlp->ih_private = (void *)add_softintr(hdlp->ih_pri,
 912             hdlp->ih_cb_func, hdlp->ih_cb_arg1, SOFTINT_ST)) == NULL)
 913                 return (DDI_FAILURE);
 914 
 915         return (DDI_SUCCESS);
 916 }
 917 
 918 /*
 919  * i_ddi_remove_softint - remove and free a software interrupt.
 920  */
 921 void
 922 i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp)
 923 {
 924         ASSERT(hdlp->ih_private != NULL);
 925 
 926         if (rem_softintr((uint64_t)hdlp->ih_private) == 0)
 927                 hdlp->ih_private = NULL;
 928 }
 929 
 930 /*
 931  * i_ddi_trigger_softint - trigger a software interrupt.
 932  */
 933 int
 934 i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2)
 935 {
 936         int     ret;
 937 
 938         ASSERT(hdlp->ih_private != NULL);
 939 
 940         /* Update the second argument for the software interrupt */
 941         if ((ret = update_softint_arg2((uint64_t)hdlp->ih_private, arg2)) == 0)
 942                 setsoftint((uint64_t)hdlp->ih_private);
 943 
 944         return (ret ? DDI_EPENDING : DDI_SUCCESS);
 945 }
 946 
 947 /*
 948  * i_ddi_set_softint_pri - change software interrupt priority.
 949  */
 950 /* ARGSUSED */
 951 int
 952 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri)
 953 {
 954         int     ret;
 955 
 956         ASSERT(hdlp->ih_private != NULL);
 957 
 958         /* Update the interrupt priority for the software interrupt */
 959         ret = update_softint_pri((uint64_t)hdlp->ih_private, hdlp->ih_pri);
 960 
 961         return (ret ? DDI_FAILURE : DDI_SUCCESS);
 962 }
 963 
 964 /*ARGSUSED*/
 965 void
 966 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp)
 967 {
 968 }
 969 
 970 /*ARGSUSED*/
 971 void
 972 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp)
 973 {
 974 }
 975 
 976 /*
 977  * SECTION: DDI Memory/DMA
 978  */
 979 
 980 /* set HAT endianess attributes from ddi_device_acc_attr */
 981 void
 982 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t *devaccp, uint_t *hataccp)
 983 {
 984         if (devaccp != NULL) {
 985                 if (devaccp->devacc_attr_endian_flags == DDI_STRUCTURE_LE_ACC) {
 986                         *hataccp &= ~HAT_ENDIAN_MASK;
 987                         *hataccp |= HAT_STRUCTURE_LE;
 988                 }
 989         }
 990 }
 991 
 992 /*
 993  * Check if the specified cache attribute is supported on the platform.
 994  * This function must be called before i_ddi_cacheattr_to_hatacc().
 995  */
 996 boolean_t
 997 i_ddi_check_cache_attr(uint_t flags)
 998 {
 999         /*
1000          * The cache attributes are mutually exclusive. Any combination of
1001          * the attributes leads to a failure.
1002          */
1003         uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1004         if ((cache_attr != 0) && ((cache_attr & (cache_attr - 1)) != 0))
1005                 return (B_FALSE);
1006 
1007         /*
1008          * On the sparc architecture, only IOMEM_DATA_CACHED is meaningful,
1009          * but others lead to a failure.
1010          */
1011         if (cache_attr & IOMEM_DATA_CACHED)
1012                 return (B_TRUE);
1013         else
1014                 return (B_FALSE);
1015 }
1016 
1017 /* set HAT cache attributes from the cache attributes */
1018 void
1019 i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp)
1020 {
1021         uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1022         static char *fname = "i_ddi_cacheattr_to_hatacc";
1023 #if defined(lint)
1024         *hataccp = *hataccp;
1025 #endif
1026         /*
1027          * set HAT attrs according to the cache attrs.
1028          */
1029         switch (cache_attr) {
1030         /*
1031          * The cache coherency is always maintained on SPARC, and
1032          * nothing is required.
1033          */
1034         case IOMEM_DATA_CACHED:
1035                 break;
1036         /*
1037          * Both IOMEM_DATA_UC_WRITE_COMBINED and IOMEM_DATA_UNCACHED are
1038          * not supported on SPARC -- this case must not occur because the
1039          * cache attribute is scrutinized before this function is called.
1040          */
1041         case IOMEM_DATA_UNCACHED:
1042         case IOMEM_DATA_UC_WR_COMBINE:
1043         default:
1044                 cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.",
1045                     fname, cache_attr);
1046         }
1047 }
1048 
1049 static vmem_t *little_endian_arena;
1050 static vmem_t *big_endian_arena;
1051 
1052 static void *
1053 segkmem_alloc_le(vmem_t *vmp, size_t size, int flag)
1054 {
1055         return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_LE,
1056             segkmem_page_create, NULL));
1057 }
1058 
1059 static void *
1060 segkmem_alloc_be(vmem_t *vmp, size_t size, int flag)
1061 {
1062         return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_BE,
1063             segkmem_page_create, NULL));
1064 }
1065 
1066 void
1067 ka_init(void)
1068 {
1069         little_endian_arena = vmem_create("little_endian", NULL, 0, 1,
1070             segkmem_alloc_le, segkmem_free, heap_arena, 0, VM_SLEEP);
1071         big_endian_arena = vmem_create("big_endian", NULL, 0, 1,
1072             segkmem_alloc_be, segkmem_free, heap_arena, 0, VM_SLEEP);
1073 }
1074 
1075 /*
1076  * Allocate from the system, aligned on a specific boundary.
1077  * The alignment, if non-zero, must be a power of 2.
1078  */
1079 static void *
1080 kalloca(size_t size, size_t align, int cansleep, uint_t endian_flags)
1081 {
1082         size_t *addr, *raddr, rsize;
1083         size_t hdrsize = 4 * sizeof (size_t);   /* must be power of 2 */
1084 
1085         align = MAX(align, hdrsize);
1086         ASSERT((align & (align - 1)) == 0);
1087 
1088         /*
1089          * We need to allocate
1090          *    rsize = size + hdrsize + align - MIN(hdrsize, buffer_alignment)
1091          * bytes to be sure we have enough freedom to satisfy the request.
1092          * Since the buffer alignment depends on the request size, this is
1093          * not straightforward to use directly.
1094          *
1095          * kmem guarantees that any allocation of a 64-byte multiple will be
1096          * 64-byte aligned.  Since rounding up the request could add more
1097          * than we save, we compute the size with and without alignment, and
1098          * use the smaller of the two.
1099          */
1100         rsize = size + hdrsize + align;
1101 
1102         if (endian_flags == DDI_STRUCTURE_LE_ACC) {
1103                 raddr = vmem_alloc(little_endian_arena, rsize,
1104                     cansleep ? VM_SLEEP : VM_NOSLEEP);
1105         } else {
1106                 raddr = vmem_alloc(big_endian_arena, rsize,
1107                     cansleep ? VM_SLEEP : VM_NOSLEEP);
1108         }
1109 
1110         if (raddr == NULL)
1111                 return (NULL);
1112 
1113         addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align);
1114         ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize);
1115 
1116         addr[-3] = (size_t)endian_flags;
1117         addr[-2] = (size_t)raddr;
1118         addr[-1] = rsize;
1119 
1120         return (addr);
1121 }
1122 
1123 static void
1124 kfreea(void *addr)
1125 {
1126         size_t *saddr = addr;
1127 
1128         if (saddr[-3] == DDI_STRUCTURE_LE_ACC)
1129                 vmem_free(little_endian_arena, (void *)saddr[-2], saddr[-1]);
1130         else
1131                 vmem_free(big_endian_arena, (void *)saddr[-2], saddr[-1]);
1132 }
1133 
1134 /*
1135  * This used to be ddi_iomin, but we were the only remaining caller, so
1136  * we've made it private and moved it here.
1137  */
1138 static int
1139 i_ddi_iomin(dev_info_t *a, int i, int stream)
1140 {
1141         int r;
1142 
1143         /*
1144          * Make sure that the initial value is sane
1145          */
1146         if (i & (i - 1))
1147                 return (0);
1148         if (i == 0)
1149                 i = (stream) ? 4 : 1;
1150 
1151         r = ddi_ctlops(a, a,
1152             DDI_CTLOPS_IOMIN, (void *)(uintptr_t)stream, (void *)&i);
1153         if (r != DDI_SUCCESS || (i & (i - 1)))
1154                 return (0);
1155         return (i);
1156 }
1157 
1158 int
1159 i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr,
1160     size_t length, int cansleep, int flags,
1161     ddi_device_acc_attr_t *accattrp,
1162     caddr_t *kaddrp, size_t *real_length, ddi_acc_hdl_t *handlep)
1163 {
1164         caddr_t a;
1165         int iomin, align, streaming;
1166         uint_t endian_flags = DDI_NEVERSWAP_ACC;
1167 
1168 #if defined(lint)
1169         *handlep = *handlep;
1170 #endif
1171 
1172         /*
1173          * Check legality of arguments
1174          */
1175         if (length == 0 || kaddrp == NULL || attr == NULL) {
1176                 return (DDI_FAILURE);
1177         }
1178 
1179         if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 ||
1180             (attr->dma_attr_align & (attr->dma_attr_align - 1)) ||
1181             (attr->dma_attr_minxfer & (attr->dma_attr_minxfer - 1))) {
1182                 return (DDI_FAILURE);
1183         }
1184 
1185         /*
1186          * check if a streaming sequential xfer is requested.
1187          */
1188         streaming = (flags & DDI_DMA_STREAMING) ? 1 : 0;
1189 
1190         /*
1191          * Drivers for 64-bit capable SBus devices will encode
1192          * the burtsizes for 64-bit xfers in the upper 16-bits.
1193          * For DMA alignment, we use the most restrictive
1194          * alignment of 32-bit and 64-bit xfers.
1195          */
1196         iomin = (attr->dma_attr_burstsizes & 0xffff) |
1197             ((attr->dma_attr_burstsizes >> 16) & 0xffff);
1198         /*
1199          * If a driver set burtsizes to 0, we give him byte alignment.
1200          * Otherwise align at the burtsizes boundary.
1201          */
1202         if (iomin == 0)
1203                 iomin = 1;
1204         else
1205                 iomin = 1 << (ddi_fls(iomin) - 1);
1206         iomin = maxbit(iomin, attr->dma_attr_minxfer);
1207         iomin = maxbit(iomin, attr->dma_attr_align);
1208         iomin = i_ddi_iomin(dip, iomin, streaming);
1209         if (iomin == 0)
1210                 return (DDI_FAILURE);
1211 
1212         ASSERT((iomin & (iomin - 1)) == 0);
1213         ASSERT(iomin >= attr->dma_attr_minxfer);
1214         ASSERT(iomin >= attr->dma_attr_align);
1215 
1216         length = P2ROUNDUP(length, iomin);
1217         align = iomin;
1218 
1219         if (accattrp != NULL)
1220                 endian_flags = accattrp->devacc_attr_endian_flags;
1221 
1222         a = kalloca(length, align, cansleep, endian_flags);
1223         if ((*kaddrp = a) == 0) {
1224                 return (DDI_FAILURE);
1225         } else {
1226                 if (real_length) {
1227                         *real_length = length;
1228                 }
1229                 if (handlep) {
1230                         /*
1231                          * assign handle information
1232                          */
1233                         impl_acc_hdl_init(handlep);
1234                 }
1235                 return (DDI_SUCCESS);
1236         }
1237 }
1238 
1239 /* ARGSUSED */
1240 void
1241 i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap)
1242 {
1243         kfreea(kaddr);
1244 }
1245 
1246 /*
1247  * SECTION: DDI Data Access
1248  */
1249 
1250 static uintptr_t impl_acc_hdl_id = 0;
1251 
1252 /*
1253  * access handle allocator
1254  */
1255 ddi_acc_hdl_t *
1256 impl_acc_hdl_get(ddi_acc_handle_t hdl)
1257 {
1258         /*
1259          * Extract the access handle address from the DDI implemented
1260          * access handle
1261          */
1262         return (&((ddi_acc_impl_t *)hdl)->ahi_common);
1263 }
1264 
1265 ddi_acc_handle_t
1266 impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
1267 {
1268         ddi_acc_impl_t *hp;
1269         on_trap_data_t *otp;
1270         int sleepflag;
1271 
1272         sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1273 
1274         /*
1275          * Allocate and initialize the data access handle and error status.
1276          */
1277         if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
1278                 goto fail;
1279         if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
1280             sizeof (ndi_err_t), sleepflag)) == NULL) {
1281                 kmem_free(hp, sizeof (ddi_acc_impl_t));
1282                 goto fail;
1283         }
1284         if ((otp = (on_trap_data_t *)kmem_zalloc(
1285             sizeof (on_trap_data_t), sleepflag)) == NULL) {
1286                 kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1287                 kmem_free(hp, sizeof (ddi_acc_impl_t));
1288                 goto fail;
1289         }
1290         hp->ahi_err->err_ontrap = otp;
1291         hp->ahi_common.ah_platform_private = (void *)hp;
1292 
1293         return ((ddi_acc_handle_t)hp);
1294 fail:
1295         if ((waitfp != (int (*)())KM_SLEEP) &&
1296             (waitfp != (int (*)())KM_NOSLEEP))
1297                 ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
1298         return (NULL);
1299 }
1300 
1301 void
1302 impl_acc_hdl_free(ddi_acc_handle_t handle)
1303 {
1304         ddi_acc_impl_t *hp;
1305 
1306         /*
1307          * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
1308          * because that's what we allocated in impl_acc_hdl_alloc() above.
1309          */
1310         hp = (ddi_acc_impl_t *)handle;
1311         if (hp) {
1312                 kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
1313                 kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1314                 kmem_free(hp, sizeof (ddi_acc_impl_t));
1315                 if (impl_acc_hdl_id)
1316                         ddi_run_callback(&impl_acc_hdl_id);
1317         }
1318 }
1319 
1320 #define PCI_GET_MP_PFN(mp, page_no)     ((mp)->dmai_ndvmapages == 1 ? \
1321         (pfn_t)(mp)->dmai_iopte:(((pfn_t *)(mp)->dmai_iopte)[page_no]))
1322 
1323 /*
1324  * Function called after a dma fault occurred to find out whether the
1325  * fault address is associated with a driver that is able to handle faults
1326  * and recover from faults.
1327  */
1328 /* ARGSUSED */
1329 int
1330 impl_dma_check(dev_info_t *dip, const void *handle, const void *addr,
1331     const void *not_used)
1332 {
1333         ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1334         pfn_t fault_pfn = mmu_btop(*(uint64_t *)addr);
1335         pfn_t comp_pfn;
1336 
1337         /*
1338          * The driver has to set DDI_DMA_FLAGERR to recover from dma faults.
1339          */
1340         int page;
1341 
1342         ASSERT(mp);
1343         for (page = 0; page < mp->dmai_ndvmapages; page++) {
1344                 comp_pfn = PCI_GET_MP_PFN(mp, page);
1345                 if (fault_pfn == comp_pfn)
1346                         return (DDI_FM_NONFATAL);
1347         }
1348         return (DDI_FM_UNKNOWN);
1349 }
1350 
1351 /*
1352  * Function used to check if a given access handle owns the failing address.
1353  * Called by ndi_fmc_error, when we detect a PIO error.
1354  */
1355 /* ARGSUSED */
1356 static int
1357 impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
1358     const void *not_used)
1359 {
1360         pfn_t pfn, fault_pfn;
1361         ddi_acc_hdl_t *hp;
1362 
1363         hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);
1364 
1365         ASSERT(hp);
1366 
1367         if (addr != NULL) {
1368                 pfn = hp->ah_pfn;
1369                 fault_pfn = mmu_btop(*(uint64_t *)addr);
1370                 if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
1371                         return (DDI_FM_NONFATAL);
1372         }
1373         return (DDI_FM_UNKNOWN);
1374 }
1375 
1376 void
1377 impl_acc_err_init(ddi_acc_hdl_t *handlep)
1378 {
1379         int fmcap;
1380         ndi_err_t *errp;
1381         on_trap_data_t *otp;
1382         ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;
1383 
1384         fmcap = ddi_fm_capable(handlep->ah_dip);
1385 
1386         if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
1387             !DDI_FM_ACC_ERR_CAP(fmcap)) {
1388                 handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
1389         } else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
1390                 if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
1391                         if (handlep->ah_xfermodes)
1392                                 return;
1393                         i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
1394                             NULL, DDI_NOSLEEP);
1395                 } else {
1396                         errp = hp->ahi_err;
1397                         otp = (on_trap_data_t *)errp->err_ontrap;
1398                         otp->ot_handle = (void *)(hp);
1399                         otp->ot_prot = OT_DATA_ACCESS;
1400                         if (handlep->ah_acc.devacc_attr_access ==
1401                             DDI_CAUTIOUS_ACC)
1402                                 otp->ot_trampoline =
1403                                     (uintptr_t)&i_ddi_caut_trampoline;
1404                         else
1405                                 otp->ot_trampoline =
1406                                     (uintptr_t)&i_ddi_prot_trampoline;
1407                         errp->err_status = DDI_FM_OK;
1408                         errp->err_expected = DDI_FM_ERR_UNEXPECTED;
1409                         errp->err_cf = impl_acc_check;
1410                 }
1411         }
1412 }
1413 
1414 void
1415 impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
1416 {
1417         ddi_acc_impl_t *hp;
1418 
1419         ASSERT(handlep);
1420 
1421         hp = (ddi_acc_impl_t *)handlep;
1422 
1423         /*
1424          * check for SW byte-swapping
1425          */
1426         hp->ahi_get8 = i_ddi_get8;
1427         hp->ahi_put8 = i_ddi_put8;
1428         hp->ahi_rep_get8 = i_ddi_rep_get8;
1429         hp->ahi_rep_put8 = i_ddi_rep_put8;
1430         if (handlep->ah_acc.devacc_attr_endian_flags & DDI_STRUCTURE_LE_ACC) {
1431                 hp->ahi_get16 = i_ddi_swap_get16;
1432                 hp->ahi_get32 = i_ddi_swap_get32;
1433                 hp->ahi_get64 = i_ddi_swap_get64;
1434                 hp->ahi_put16 = i_ddi_swap_put16;
1435                 hp->ahi_put32 = i_ddi_swap_put32;
1436                 hp->ahi_put64 = i_ddi_swap_put64;
1437                 hp->ahi_rep_get16 = i_ddi_swap_rep_get16;
1438                 hp->ahi_rep_get32 = i_ddi_swap_rep_get32;
1439                 hp->ahi_rep_get64 = i_ddi_swap_rep_get64;
1440                 hp->ahi_rep_put16 = i_ddi_swap_rep_put16;
1441                 hp->ahi_rep_put32 = i_ddi_swap_rep_put32;
1442                 hp->ahi_rep_put64 = i_ddi_swap_rep_put64;
1443         } else {
1444                 hp->ahi_get16 = i_ddi_get16;
1445                 hp->ahi_get32 = i_ddi_get32;
1446                 hp->ahi_get64 = i_ddi_get64;
1447                 hp->ahi_put16 = i_ddi_put16;
1448                 hp->ahi_put32 = i_ddi_put32;
1449                 hp->ahi_put64 = i_ddi_put64;
1450                 hp->ahi_rep_get16 = i_ddi_rep_get16;
1451                 hp->ahi_rep_get32 = i_ddi_rep_get32;
1452                 hp->ahi_rep_get64 = i_ddi_rep_get64;
1453                 hp->ahi_rep_put16 = i_ddi_rep_put16;
1454                 hp->ahi_rep_put32 = i_ddi_rep_put32;
1455                 hp->ahi_rep_put64 = i_ddi_rep_put64;
1456         }
1457 
1458         /* Legacy fault flags and support */
1459         hp->ahi_fault_check = i_ddi_acc_fault_check;
1460         hp->ahi_fault_notify = i_ddi_acc_fault_notify;
1461         hp->ahi_fault = 0;
1462         impl_acc_err_init(handlep);
1463 }
1464 
1465 void
1466 i_ddi_acc_set_fault(ddi_acc_handle_t handle)
1467 {
1468         ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1469 
1470         if (!hp->ahi_fault) {
1471                 hp->ahi_fault = 1;
1472                         (*hp->ahi_fault_notify)(hp);
1473         }
1474 }
1475 
1476 void
1477 i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
1478 {
1479         ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1480 
1481         if (hp->ahi_fault) {
1482                 hp->ahi_fault = 0;
1483                         (*hp->ahi_fault_notify)(hp);
1484         }
1485 }
1486 
1487 /* ARGSUSED */
1488 void
1489 i_ddi_acc_fault_notify(ddi_acc_impl_t *hp)
1490 {
1491         /* Default version, does nothing */
1492 }
1493 
1494 /*
1495  * SECTION: Misc functions
1496  */
1497 
1498 /*
1499  * instance wrappers
1500  */
1501 /*ARGSUSED*/
1502 uint_t
1503 impl_assign_instance(dev_info_t *dip)
1504 {
1505         return ((uint_t)-1);
1506 }
1507 
1508 /*ARGSUSED*/
1509 int
1510 impl_keep_instance(dev_info_t *dip)
1511 {
1512         return (DDI_FAILURE);
1513 }
1514 
1515 /*ARGSUSED*/
1516 int
1517 impl_free_instance(dev_info_t *dip)
1518 {
1519         return (DDI_FAILURE);
1520 }
1521 
1522 /*ARGSUSED*/
1523 int
1524 impl_check_cpu(dev_info_t *devi)
1525 {
1526         return (DDI_SUCCESS);
1527 }
1528 
1529 
1530 static const char *nocopydevs[] = {
1531         "SUNW,ffb",
1532         "SUNW,afb",
1533         NULL
1534 };
1535 
1536 /*
1537  * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
1538  * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
1539  */
1540 /*ARGSUSED*/
1541 int
1542 e_ddi_copyfromdev(dev_info_t *devi,
1543     off_t off, const void *devaddr, void *kaddr, size_t len)
1544 {
1545         const char **argv;
1546 
1547         for (argv = nocopydevs; *argv; argv++)
1548                 if (strcmp(ddi_binding_name(devi), *argv) == 0) {
1549                         bzero(kaddr, len);
1550                         return (0);
1551                 }
1552 
1553         bcopy(devaddr, kaddr, len);
1554         return (0);
1555 }
1556 
1557 /*
1558  * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
1559  * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
1560  */
1561 /*ARGSUSED*/
1562 int
1563 e_ddi_copytodev(dev_info_t *devi,
1564     off_t off, const void *kaddr, void *devaddr, size_t len)
1565 {
1566         const char **argv;
1567 
1568         for (argv = nocopydevs; *argv; argv++)
1569                 if (strcmp(ddi_binding_name(devi), *argv) == 0)
1570                         return (1);
1571 
1572         bcopy(kaddr, devaddr, len);
1573         return (0);
1574 }
1575 
1576 /*
1577  * Boot Configuration
1578  */
1579 idprom_t idprom;
1580 
1581 /*
1582  * Configure the hardware on the system.
1583  * Called before the rootfs is mounted
1584  */
1585 void
1586 configure(void)
1587 {
1588         extern void i_ddi_init_root();
1589 
1590         /* We better have released boot by this time! */
1591         ASSERT(!bootops);
1592 
1593         /*
1594          * Determine whether or not to use the fpu, V9 SPARC cpus
1595          * always have one. Could check for existence of a fp queue,
1596          * Ultra I, II and IIa do not have a fp queue.
1597          */
1598         if (fpu_exists)
1599                 fpu_probe();
1600         else
1601                 cmn_err(CE_CONT, "FPU not in use\n");
1602 
1603 #if 0 /* XXXQ - not necessary for sun4u */
1604         /*
1605          * This following line fixes bugid 1041296; we need to do a
1606          * prom_nextnode(0) because this call ALSO patches the DMA+
1607          * bug in Campus-B and Phoenix. The prom uncaches the traptable
1608          * page as a side-effect of devr_next(0) (which prom_nextnode calls),
1609          * so this *must* be executed early on. (XXX This is untrue for sun4u)
1610          */
1611         (void) prom_nextnode((pnode_t)0);
1612 #endif
1613 
1614         /*
1615          * Initialize devices on the machine.
1616          * Uses configuration tree built by the PROMs to determine what
1617          * is present, and builds a tree of prototype dev_info nodes
1618          * corresponding to the hardware which identified itself.
1619          */
1620         i_ddi_init_root();
1621 
1622 #ifdef  DDI_PROP_DEBUG
1623         (void) ddi_prop_debug(1);       /* Enable property debugging */
1624 #endif  /* DDI_PROP_DEBUG */
1625 }
1626 
1627 /*
1628  * The "status" property indicates the operational status of a device.
1629  * If this property is present, the value is a string indicating the
1630  * status of the device as follows:
1631  *
1632  *      "okay"          operational.
1633  *      "disabled"      not operational, but might become operational.
1634  *      "fail"          not operational because a fault has been detected,
1635  *                      and it is unlikely that the device will become
1636  *                      operational without repair. no additional details
1637  *                      are available.
1638  *      "fail-xxx"      not operational because a fault has been detected,
1639  *                      and it is unlikely that the device will become
1640  *                      operational without repair. "xxx" is additional
1641  *                      human-readable information about the particular
1642  *                      fault condition that was detected.
1643  *
1644  * The absence of this property means that the operational status is
1645  * unknown or okay.
1646  *
1647  * This routine checks the status property of the specified device node
1648  * and returns 0 if the operational status indicates failure, and 1 otherwise.
1649  *
1650  * The property may exist on plug-in cards the existed before IEEE 1275-1994.
1651  * And, in that case, the property may not even be a string. So we carefully
1652  * check for the value "fail", in the beginning of the string, noting
1653  * the property length.
1654  */
1655 int
1656 status_okay(int id, char *buf, int buflen)
1657 {
1658         char status_buf[OBP_MAXPROPNAME];
1659         char *bufp = buf;
1660         int len = buflen;
1661         int proplen;
1662         static const char *status = "status";
1663         static const char *fail = "fail";
1664         size_t fail_len = strlen(fail);
1665 
1666         /*
1667          * Get the proplen ... if it's smaller than "fail",
1668          * or doesn't exist ... then we don't care, since
1669          * the value can't begin with the char string "fail".
1670          *
1671          * NB: proplen, if it's a string, includes the NULL in the
1672          * the size of the property, and fail_len does not.
1673          */
1674         proplen = prom_getproplen((pnode_t)id, (caddr_t)status);
1675         if (proplen <= fail_len)     /* nonexistent or uninteresting len */
1676                 return (1);
1677 
1678         /*
1679          * if a buffer was provided, use it
1680          */
1681         if ((buf == (char *)NULL) || (buflen <= 0)) {
1682                 bufp = status_buf;
1683                 len = sizeof (status_buf);
1684         }
1685         *bufp = (char)0;
1686 
1687         /*
1688          * Get the property into the buffer, to the extent of the buffer,
1689          * and in case the buffer is smaller than the property size,
1690          * NULL terminate the buffer. (This handles the case where
1691          * a buffer was passed in and the caller wants to print the
1692          * value, but the buffer was too small).
1693          */
1694         (void) prom_bounded_getprop((pnode_t)id, (caddr_t)status,
1695             (caddr_t)bufp, len);
1696         *(bufp + len - 1) = (char)0;
1697 
1698         /*
1699          * If the value begins with the char string "fail",
1700          * then it means the node is failed. We don't care
1701          * about any other values. We assume the node is ok
1702          * although it might be 'disabled'.
1703          */
1704         if (strncmp(bufp, fail, fail_len) == 0)
1705                 return (0);
1706 
1707         return (1);
1708 }
1709 
1710 
1711 /*
1712  * We set the cpu type from the idprom, if we can.
1713  * Note that we just read out the contents of it, for the most part.
1714  */
1715 void
1716 setcputype(void)
1717 {
1718         /*
1719          * We cache the idprom info early on so that we don't
1720          * rummage through the NVRAM unnecessarily later.
1721          */
1722         (void) prom_getidprom((caddr_t)&idprom, sizeof (idprom));
1723 }
1724 
1725 /*
1726  *  Here is where we actually infer meanings to the members of idprom_t
1727  */
1728 void
1729 parse_idprom(void)
1730 {
1731         if (idprom.id_format == IDFORM_1) {
1732                 (void) localetheraddr((struct ether_addr *)idprom.id_ether,
1733                     (struct ether_addr *)NULL);
1734                 (void) snprintf(hw_serial, HW_HOSTID_LEN, "%u",
1735                     (idprom.id_machine << 24) + idprom.id_serial);
1736         } else
1737                 prom_printf("Invalid format code in IDprom.\n");
1738 }
1739 
1740 /*
1741  * Allow for implementation specific correction of PROM property values.
1742  */
1743 /*ARGSUSED*/
1744 void
1745 impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len,
1746     caddr_t buffer)
1747 {
1748         /*
1749          * There are no adjustments needed in this implementation.
1750          */
1751 }
1752 
1753 /*
1754  * The following functions ready a cautious request to go up to the nexus
1755  * driver.  It is up to the nexus driver to decide how to process the request.
1756  * It may choose to call i_ddi_do_caut_get/put in this file, or do it
1757  * differently.
1758  */
1759 
1760 static void
1761 i_ddi_caut_getput_ctlops(
1762     ddi_acc_impl_t *hp, uint64_t host_addr, uint64_t dev_addr, size_t size,
1763     size_t repcount, uint_t flags, ddi_ctl_enum_t cmd)
1764 {
1765         peekpoke_ctlops_t       cautacc_ctlops_arg;
1766 
1767         cautacc_ctlops_arg.size = size;
1768         cautacc_ctlops_arg.dev_addr = dev_addr;
1769         cautacc_ctlops_arg.host_addr = host_addr;
1770         cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp;
1771         cautacc_ctlops_arg.repcount = repcount;
1772         cautacc_ctlops_arg.flags = flags;
1773 
1774         (void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd,
1775             &cautacc_ctlops_arg, NULL);
1776 }
1777 
1778 uint8_t
1779 i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr)
1780 {
1781         uint8_t value;
1782         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1783             sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK);
1784 
1785         return (value);
1786 }
1787 
1788 uint16_t
1789 i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr)
1790 {
1791         uint16_t value;
1792         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1793             sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK);
1794 
1795         return (value);
1796 }
1797 
1798 uint32_t
1799 i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr)
1800 {
1801         uint32_t value;
1802         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1803             sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK);
1804 
1805         return (value);
1806 }
1807 
1808 uint64_t
1809 i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr)
1810 {
1811         uint64_t value;
1812         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1813             sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK);
1814 
1815         return (value);
1816 }
1817 
1818 void
1819 i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value)
1820 {
1821         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1822             sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE);
1823 }
1824 
1825 void
1826 i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value)
1827 {
1828         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1829             sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE);
1830 }
1831 
1832 void
1833 i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value)
1834 {
1835         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1836             sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE);
1837 }
1838 
1839 void
1840 i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value)
1841 {
1842         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1843             sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE);
1844 }
1845 
1846 void
1847 i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1848         size_t repcount, uint_t flags)
1849 {
1850         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1851             sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK);
1852 }
1853 
1854 void
1855 i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1856     uint16_t *dev_addr, size_t repcount, uint_t flags)
1857 {
1858         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1859             sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK);
1860 }
1861 
1862 void
1863 i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1864     uint32_t *dev_addr, size_t repcount, uint_t flags)
1865 {
1866         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1867             sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK);
1868 }
1869 
1870 void
1871 i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1872     uint64_t *dev_addr, size_t repcount, uint_t flags)
1873 {
1874         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1875             sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK);
1876 }
1877 
1878 void
1879 i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1880         size_t repcount, uint_t flags)
1881 {
1882         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1883             sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE);
1884 }
1885 
1886 void
1887 i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1888     uint16_t *dev_addr, size_t repcount, uint_t flags)
1889 {
1890         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1891             sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE);
1892 }
1893 
1894 void
1895 i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1896     uint32_t *dev_addr, size_t repcount, uint_t flags)
1897 {
1898         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1899             sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE);
1900 }
1901 
1902 void
1903 i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1904     uint64_t *dev_addr, size_t repcount, uint_t flags)
1905 {
1906         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1907             sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE);
1908 }
1909 
1910 /*
1911  * This is called only to process peek/poke when the DIP is NULL.
1912  * Assume that this is for memory, as nexi take care of device safe accesses.
1913  */
1914 int
1915 peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args)
1916 {
1917         int err = DDI_SUCCESS;
1918         on_trap_data_t otd;
1919 
1920         /* Set up protected environment. */
1921         if (!on_trap(&otd, OT_DATA_ACCESS)) {
1922                 uintptr_t tramp = otd.ot_trampoline;
1923 
1924                 if (cmd == DDI_CTLOPS_POKE) {
1925                         otd.ot_trampoline = (uintptr_t)&poke_fault;
1926                         err = do_poke(in_args->size, (void *)in_args->dev_addr,
1927                             (void *)in_args->host_addr);
1928                 } else {
1929                         otd.ot_trampoline = (uintptr_t)&peek_fault;
1930                         err = do_peek(in_args->size, (void *)in_args->dev_addr,
1931                             (void *)in_args->host_addr);
1932                 }
1933                 otd.ot_trampoline = tramp;
1934         } else
1935                 err = DDI_FAILURE;
1936 
1937         /* Take down protected environment. */
1938         no_trap();
1939 
1940         return (err);
1941 }