5255 uts shouldn't open-code ISP2

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 /*
  27  * Copyright 2014 Garrett D'Amore <garrett@damore.org>
  28  */
  29 
  30 /*
  31  * sun4 specific DDI implementation
  32  */
  33 #include <sys/cpuvar.h>
  34 #include <sys/ddi_subrdefs.h>
  35 #include <sys/machsystm.h>
  36 #include <sys/sunndi.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/ontrap.h>
  39 #include <vm/seg_kmem.h>
  40 #include <sys/membar.h>
  41 #include <sys/dditypes.h>
  42 #include <sys/ndifm.h>
  43 #include <sys/fm/io/ddi.h>
  44 #include <sys/ivintr.h>
  45 #include <sys/bootconf.h>
  46 #include <sys/conf.h>
  47 #include <sys/ethernet.h>
  48 #include <sys/idprom.h>
  49 #include <sys/promif.h>
  50 #include <sys/prom_plat.h>
  51 #include <sys/systeminfo.h>
  52 #include <sys/fpu/fpusystm.h>
  53 #include <sys/vm.h>
  54 #include <sys/ddi_isa.h>
  55 #include <sys/modctl.h>
  56 
  57 dev_info_t *get_intr_parent(dev_info_t *, dev_info_t *,
  58     ddi_intr_handle_impl_t *);
  59 #pragma weak get_intr_parent
  60 
  61 int process_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
  62     ddi_intr_handle_impl_t *, void *);
  63 #pragma weak process_intr_ops
  64 
  65 void cells_1275_copy(prop_1275_cell_t *, prop_1275_cell_t *, int32_t);
  66     prop_1275_cell_t *cells_1275_cmp(prop_1275_cell_t *, prop_1275_cell_t *,
  67     int32_t len);
  68 #pragma weak cells_1275_copy
  69 
  70 /*
  71  * Wrapper for ddi_prop_lookup_int_array().
  72  * This is handy because it returns the prop length in
  73  * bytes which is what most of the callers require.
  74  */
  75 
  76 static int
  77 get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen)
  78 {
  79         int ret;
  80 
  81         if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di,
  82             DDI_PROP_DONTPASS, pname, pval, plen)) == DDI_PROP_SUCCESS) {
  83                 *plen = (*plen) * (uint_t)sizeof (int);
  84         }
  85         return (ret);
  86 }
  87 
  88 /*
  89  * SECTION: DDI Node Configuration
  90  */
  91 
  92 /*
  93  * init_regspec_64:
  94  *
  95  * If the parent #size-cells is 2, convert the upa-style or
  96  * safari-style reg property from 2-size cells to 1 size cell
  97  * format, ignoring the size_hi, which must be zero for devices.
  98  * (It won't be zero in the memory list properties in the memory
  99  * nodes, but that doesn't matter here.)
 100  */
 101 struct ddi_parent_private_data *
 102 init_regspec_64(dev_info_t *dip)
 103 {
 104         struct ddi_parent_private_data *pd;
 105         dev_info_t *parent;
 106         int size_cells;
 107 
 108         /*
 109          * If there are no "reg"s in the child node, return.
 110          */
 111         pd = ddi_get_parent_data(dip);
 112         if ((pd == NULL) || (pd->par_nreg == 0)) {
 113                 return (pd);
 114         }
 115         parent = ddi_get_parent(dip);
 116 
 117         size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
 118             DDI_PROP_DONTPASS, "#size-cells", 1);
 119 
 120         if (size_cells != 1)  {
 121 
 122                 int n, j;
 123                 struct regspec *irp;
 124                 struct reg_64 {
 125                         uint_t addr_hi, addr_lo, size_hi, size_lo;
 126                 };
 127                 struct reg_64 *r64_rp;
 128                 struct regspec *rp;
 129                 uint_t len = 0;
 130                 int *reg_prop;
 131 
 132                 ASSERT(size_cells == 2);
 133 
 134                 /*
 135                  * We already looked the property up once before if
 136                  * pd is non-NULL.
 137                  */
 138                 (void) ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
 139                     DDI_PROP_DONTPASS, OBP_REG, &reg_prop, &len);
 140                 ASSERT(len != 0);
 141 
 142                 n = sizeof (struct reg_64) / sizeof (int);
 143                 n = len / n;
 144 
 145                 /*
 146                  * We're allocating a buffer the size of the PROM's property,
 147                  * but we're only using a smaller portion when we assign it
 148                  * to a regspec.  We do this so that in the
 149                  * impl_ddi_sunbus_removechild function, we will
 150                  * always free the right amount of memory.
 151                  */
 152                 irp = rp = (struct regspec *)reg_prop;
 153                 r64_rp = (struct reg_64 *)pd->par_reg;
 154 
 155                 for (j = 0; j < n; ++j, ++rp, ++r64_rp) {
 156                         ASSERT(r64_rp->size_hi == 0);
 157                         rp->regspec_bustype = r64_rp->addr_hi;
 158                         rp->regspec_addr = r64_rp->addr_lo;
 159                         rp->regspec_size = r64_rp->size_lo;
 160                 }
 161 
 162                 ddi_prop_free((void *)pd->par_reg);
 163                 pd->par_nreg = n;
 164                 pd->par_reg = irp;
 165         }
 166         return (pd);
 167 }
 168 
 169 /*
 170  * Create a ddi_parent_private_data structure from the ddi properties of
 171  * the dev_info node.
 172  *
 173  * The "reg" is required if the driver wishes to create mappings on behalf
 174  * of the device. The "reg" property is assumed to be a list of at least
 175  * one triplet
 176  *
 177  *      <bustype, address, size>*1
 178  *
 179  * The "interrupt" property is no longer part of parent private data on
 180  * sun4u. The interrupt parent is may not be the device tree parent.
 181  *
 182  * The "ranges" property describes the mapping of child addresses to parent
 183  * addresses.
 184  *
 185  * N.B. struct rangespec is defined for the following default values:
 186  *                      parent  child
 187  *      #address-cells  2       2
 188  *      #size-cells     1       1
 189  * This function doesn't deal with non-default cells and will not create
 190  * ranges in such cases.
 191  */
 192 void
 193 make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd)
 194 {
 195         struct ddi_parent_private_data *pdptr;
 196         int *reg_prop, *rng_prop;
 197         uint_t reg_len = 0, rng_len = 0;
 198         dev_info_t *parent;
 199         int parent_addr_cells, parent_size_cells;
 200         int child_addr_cells, child_size_cells;
 201 
 202         *ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP);
 203 
 204         /*
 205          * root node has no parent private data, so *ppd should
 206          * be initialized for naming to work properly.
 207          */
 208         if ((parent = ddi_get_parent(child)) == NULL)
 209                 return;
 210 
 211         /*
 212          * Set reg field of parent data from "reg" property
 213          */
 214         if ((get_prop_int_array(child, OBP_REG, &reg_prop, &reg_len)
 215             == DDI_PROP_SUCCESS) && (reg_len != 0)) {
 216                 pdptr->par_nreg = (int)(reg_len / sizeof (struct regspec));
 217                 pdptr->par_reg = (struct regspec *)reg_prop;
 218         }
 219 
 220         /*
 221          * "ranges" property ...
 222          *
 223          * This function does not handle cases where #address-cells != 2
 224          * and * min(parent, child) #size-cells != 1 (see bugid 4211124).
 225          *
 226          * Nexus drivers with such exceptions (e.g. pci ranges)
 227          * should either create a separate function for handling
 228          * ranges or not use parent private data to store ranges.
 229          */
 230 
 231         /* root node has no ranges */
 232         if ((parent = ddi_get_parent(child)) == NULL)
 233                 return;
 234 
 235         child_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
 236             DDI_PROP_DONTPASS, "#address-cells", 2);
 237         child_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
 238             DDI_PROP_DONTPASS, "#size-cells", 1);
 239         parent_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
 240             DDI_PROP_DONTPASS, "#address-cells", 2);
 241         parent_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
 242             DDI_PROP_DONTPASS, "#size-cells", 1);
 243         if (child_addr_cells != 2 || parent_addr_cells != 2 ||
 244             (child_size_cells != 1 && parent_size_cells != 1)) {
 245                 NDI_CONFIG_DEBUG((CE_NOTE, "!ranges not made in parent data; "
 246                     "#address-cells or #size-cells have non-default value"));
 247                 return;
 248         }
 249 
 250         if (get_prop_int_array(child, OBP_RANGES, &rng_prop, &rng_len)
 251             == DDI_PROP_SUCCESS) {
 252                 pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec));
 253                 pdptr->par_rng = (struct rangespec *)rng_prop;
 254         }
 255 }
 256 
 257 /*
 258  * Free ddi_parent_private_data structure
 259  */
 260 void
 261 impl_free_ddi_ppd(dev_info_t *dip)
 262 {
 263         struct ddi_parent_private_data *pdptr = ddi_get_parent_data(dip);
 264 
 265         if (pdptr == NULL)
 266                 return;
 267 
 268         if (pdptr->par_nrng != 0)
 269                 ddi_prop_free((void *)pdptr->par_rng);
 270 
 271         if (pdptr->par_nreg != 0)
 272                 ddi_prop_free((void *)pdptr->par_reg);
 273 
 274         kmem_free(pdptr, sizeof (*pdptr));
 275         ddi_set_parent_data(dip, NULL);
 276 }
 277 
 278 /*
 279  * Name a child of sun busses based on the reg spec.
 280  * Handles the following properties:
 281  *
 282  *      Property        value
 283  *      Name            type
 284  *
 285  *      reg             register spec
 286  *      interrupts      new (bus-oriented) interrupt spec
 287  *      ranges          range spec
 288  *
 289  * This may be called multiple times, independent of
 290  * initchild calls.
 291  */
 292 static int
 293 impl_sunbus_name_child(dev_info_t *child, char *name, int namelen)
 294 {
 295         struct ddi_parent_private_data *pdptr;
 296         struct regspec *rp;
 297 
 298         /*
 299          * Fill in parent-private data and this function returns to us
 300          * an indication if it used "registers" to fill in the data.
 301          */
 302         if (ddi_get_parent_data(child) == NULL) {
 303                 make_ddi_ppd(child, &pdptr);
 304                 ddi_set_parent_data(child, pdptr);
 305         }
 306 
 307         /*
 308          * No reg property, return null string as address
 309          * (e.g. root node)
 310          */
 311         name[0] = '\0';
 312         if (sparc_pd_getnreg(child) == 0) {
 313                 return (DDI_SUCCESS);
 314         }
 315 
 316         rp = sparc_pd_getreg(child, 0);
 317         (void) snprintf(name, namelen, "%x,%x",
 318             rp->regspec_bustype, rp->regspec_addr);
 319         return (DDI_SUCCESS);
 320 }
 321 
 322 
 323 /*
 324  * Called from the bus_ctl op of some drivers.
 325  * to implement the DDI_CTLOPS_INITCHILD operation.
 326  *
 327  * NEW drivers should NOT use this function, but should declare
 328  * there own initchild/uninitchild handlers. (This function assumes
 329  * the layout of the parent private data and the format of "reg",
 330  * "ranges", "interrupts" properties and that #address-cells and
 331  * #size-cells of the parent bus are defined to be default values.)
 332  */
 333 int
 334 impl_ddi_sunbus_initchild(dev_info_t *child)
 335 {
 336         char name[MAXNAMELEN];
 337 
 338         (void) impl_sunbus_name_child(child, name, MAXNAMELEN);
 339         ddi_set_name_addr(child, name);
 340 
 341         /*
 342          * Try to merge .conf node. If successful, return failure to
 343          * remove this child.
 344          */
 345         if ((ndi_dev_is_persistent_node(child) == 0) &&
 346             (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) {
 347                 impl_ddi_sunbus_removechild(child);
 348                 return (DDI_FAILURE);
 349         }
 350         return (DDI_SUCCESS);
 351 }
 352 
 353 /*
 354  * A better name for this function would be impl_ddi_sunbus_uninitchild()
 355  * It does not remove the child, it uninitializes it, reclaiming the
 356  * resources taken by impl_ddi_sunbus_initchild.
 357  */
 358 void
 359 impl_ddi_sunbus_removechild(dev_info_t *dip)
 360 {
 361         impl_free_ddi_ppd(dip);
 362         ddi_set_name_addr(dip, NULL);
 363         /*
 364          * Strip the node to properly convert it back to prototype form
 365          */
 366         impl_rem_dev_props(dip);
 367 }
 368 
 369 /*
 370  * SECTION: DDI Interrupt
 371  */
 372 
 373 void
 374 cells_1275_copy(prop_1275_cell_t *from, prop_1275_cell_t *to, int32_t len)
 375 {
 376         int i;
 377         for (i = 0; i < len; i++)
 378                 *to = *from;
 379 }
 380 
 381 prop_1275_cell_t *
 382 cells_1275_cmp(prop_1275_cell_t *cell1, prop_1275_cell_t *cell2, int32_t len)
 383 {
 384         prop_1275_cell_t *match_cell = 0;
 385         int32_t i;
 386 
 387         for (i = 0; i < len; i++)
 388                 if (cell1[i] != cell2[i]) {
 389                         match_cell = &cell1[i];
 390                         break;
 391                 }
 392 
 393         return (match_cell);
 394 }
 395 
 396 /*
 397  * get_intr_parent() is a generic routine that process a 1275 interrupt
 398  * map (imap) property.  This function returns a dev_info_t structure
 399  * which claims ownership of the interrupt domain.
 400  * It also returns the new interrupt translation within this new domain.
 401  * If an interrupt-parent or interrupt-map property are not found,
 402  * then we fallback to using the device tree's parent.
 403  *
 404  * imap entry format:
 405  * <reg>,<interrupt>,<phandle>,<translated interrupt>
 406  * reg - The register specification in the interrupts domain
 407  * interrupt - The interrupt specification
 408  * phandle - PROM handle of the device that owns the xlated interrupt domain
 409  * translated interrupt - interrupt specifier in the parents domain
 410  * note: <reg>,<interrupt> - The reg and interrupt can be combined to create
 411  *      a unique entry called a unit interrupt specifier.
 412  *
 413  * Here's the processing steps:
 414  * step1 - If the interrupt-parent property exists, create the ispec and
 415  *      return the dip of the interrupt parent.
 416  * step2 - Extract the interrupt-map property and the interrupt-map-mask
 417  *      If these don't exist, just return the device tree parent.
 418  * step3 - build up the unit interrupt specifier to match against the
 419  *      interrupt map property
 420  * step4 - Scan the interrupt-map property until a match is found
 421  * step4a - Extract the interrupt parent
 422  * step4b - Compare the unit interrupt specifier
 423  */
 424 dev_info_t *
 425 get_intr_parent(dev_info_t *pdip, dev_info_t *dip, ddi_intr_handle_impl_t *hdlp)
 426 {
 427         prop_1275_cell_t *imap, *imap_mask, *scan, *reg_p, *match_req;
 428         int32_t imap_sz, imap_cells, imap_scan_cells, imap_mask_sz,
 429             addr_cells, intr_cells, reg_len, i, j;
 430         int32_t match_found = 0;
 431         dev_info_t *intr_parent_dip = NULL;
 432         uint32_t *intr = &hdlp->ih_vector;
 433         uint32_t nodeid;
 434 #ifdef DEBUG
 435         static int debug = 0;
 436 #endif
 437 
 438         /*
 439          * step1
 440          * If we have an interrupt-parent property, this property represents
 441          * the nodeid of our interrupt parent.
 442          */
 443         if ((nodeid = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
 444             "interrupt-parent", -1)) != -1) {
 445                 intr_parent_dip = e_ddi_nodeid_to_dip(nodeid);
 446                 ASSERT(intr_parent_dip);
 447 
 448                 /*
 449                  * Attach the interrupt parent.
 450                  *
 451                  * N.B. e_ddi_nodeid_to_dip() isn't safe under DR.
 452                  *      Also, interrupt parent isn't held. This needs
 453                  *      to be revisited if DR-capable platforms implement
 454                  *      interrupt redirection.
 455                  */
 456                 if (i_ddi_attach_node_hierarchy(intr_parent_dip)
 457                     != DDI_SUCCESS) {
 458                         ndi_rele_devi(intr_parent_dip);
 459                         return (NULL);
 460                 }
 461 
 462                 return (intr_parent_dip);
 463         }
 464 
 465         /*
 466          * step2
 467          * Get interrupt map structure from PROM property
 468          */
 469         if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
 470             "interrupt-map", (caddr_t)&imap, &imap_sz)
 471             != DDI_PROP_SUCCESS) {
 472                 /*
 473                  * If we don't have an imap property, default to using the
 474                  * device tree.
 475                  */
 476 
 477                 ndi_hold_devi(pdip);
 478                 return (pdip);
 479         }
 480 
 481         /* Get the interrupt mask property */
 482         if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
 483             "interrupt-map-mask", (caddr_t)&imap_mask, &imap_mask_sz)
 484             != DDI_PROP_SUCCESS) {
 485                 /*
 486                  * If we don't find this property, we have to fail the request
 487                  * because the 1275 imap property wasn't defined correctly.
 488                  */
 489                 ASSERT(intr_parent_dip == NULL);
 490                 goto exit2;
 491         }
 492 
 493         /* Get the address cell size */
 494         addr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
 495             "#address-cells", 2);
 496 
 497         /* Get the interrupts cell size */
 498         intr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
 499             "#interrupt-cells", 1);
 500 
 501         /*
 502          * step3
 503          * Now lets build up the unit interrupt specifier e.g. reg,intr
 504          * and apply the imap mask.  match_req will hold this when we're
 505          * through.
 506          */
 507         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
 508             (caddr_t)&reg_p, &reg_len) != DDI_SUCCESS) {
 509                 ASSERT(intr_parent_dip == NULL);
 510                 goto exit3;
 511         }
 512 
 513         match_req = kmem_alloc(CELLS_1275_TO_BYTES(addr_cells) +
 514             CELLS_1275_TO_BYTES(intr_cells), KM_SLEEP);
 515 
 516         for (i = 0; i < addr_cells; i++)
 517                 match_req[i] = (reg_p[i] & imap_mask[i]);
 518 
 519         for (j = 0; j < intr_cells; i++, j++)
 520                 match_req[i] = (intr[j] & imap_mask[i]);
 521 
 522         /* Calculate the imap size in cells */
 523         imap_cells = BYTES_TO_1275_CELLS(imap_sz);
 524 
 525 #ifdef DEBUG
 526         if (debug)
 527                 prom_printf("reg cell size 0x%x, intr cell size 0x%x, "
 528                     "match_request 0x%p, imap 0x%p\n", addr_cells, intr_cells,
 529                     (void *)match_req, (void *)imap);
 530 #endif
 531 
 532         /*
 533          * Scan the imap property looking for a match of the interrupt unit
 534          * specifier.  This loop is rather complex since the data within the
 535          * imap property may vary in size.
 536          */
 537         for (scan = imap, imap_scan_cells = i = 0;
 538             imap_scan_cells < imap_cells; scan += i, imap_scan_cells += i) {
 539                 int new_intr_cells;
 540 
 541                 /* Set the index to the nodeid field */
 542                 i = addr_cells + intr_cells;
 543 
 544                 /*
 545                  * step4a
 546                  * Translate the nodeid field to a dip
 547                  */
 548                 ASSERT(intr_parent_dip == NULL);
 549                 intr_parent_dip = e_ddi_nodeid_to_dip((uint_t)scan[i++]);
 550 
 551                 ASSERT(intr_parent_dip != 0);
 552 #ifdef DEBUG
 553                 if (debug)
 554                         prom_printf("scan 0x%p\n", (void *)scan);
 555 #endif
 556                 /*
 557                  * The tmp_dip describes the new domain, get it's interrupt
 558                  * cell size
 559                  */
 560                 new_intr_cells = ddi_getprop(DDI_DEV_T_ANY, intr_parent_dip, 0,
 561                     "#interrupts-cells", 1);
 562 
 563                 /*
 564                  * step4b
 565                  * See if we have a match on the interrupt unit specifier
 566                  */
 567                 if (cells_1275_cmp(match_req, scan, addr_cells + intr_cells)
 568                     == 0) {
 569                         uint32_t *intr;
 570 
 571                         match_found = 1;
 572 
 573                         /*
 574                          * If we have an imap parent whose not in our device
 575                          * tree path, we need to hold and install that driver.
 576                          */
 577                         if (i_ddi_attach_node_hierarchy(intr_parent_dip)
 578                             != DDI_SUCCESS) {
 579                                 ndi_rele_devi(intr_parent_dip);
 580                                 intr_parent_dip = (dev_info_t *)NULL;
 581                                 goto exit4;
 582                         }
 583 
 584                         /*
 585                          * We need to handcraft an ispec along with a bus
 586                          * interrupt value, so we can dup it into our
 587                          * standard ispec structure.
 588                          */
 589                         /* Extract the translated interrupt information */
 590                         intr = kmem_alloc(
 591                             CELLS_1275_TO_BYTES(new_intr_cells), KM_SLEEP);
 592 
 593                         for (j = 0; j < new_intr_cells; j++, i++)
 594                                 intr[j] = scan[i];
 595 
 596                         cells_1275_copy(intr, &hdlp->ih_vector, new_intr_cells);
 597 
 598                         kmem_free(intr, CELLS_1275_TO_BYTES(new_intr_cells));
 599 
 600 #ifdef DEBUG
 601                         if (debug)
 602                                 prom_printf("dip 0x%p\n",
 603                                     (void *)intr_parent_dip);
 604 #endif
 605                         break;
 606                 } else {
 607 #ifdef DEBUG
 608                         if (debug)
 609                                 prom_printf("dip 0x%p\n",
 610                                     (void *)intr_parent_dip);
 611 #endif
 612                         ndi_rele_devi(intr_parent_dip);
 613                         intr_parent_dip = NULL;
 614                         i += new_intr_cells;
 615                 }
 616         }
 617 
 618         /*
 619          * If we haven't found our interrupt parent at this point, fallback
 620          * to using the device tree.
 621          */
 622         if (!match_found) {
 623                 ndi_hold_devi(pdip);
 624                 ASSERT(intr_parent_dip == NULL);
 625                 intr_parent_dip = pdip;
 626         }
 627 
 628         ASSERT(intr_parent_dip != NULL);
 629 
 630 exit4:
 631         kmem_free(reg_p, reg_len);
 632         kmem_free(match_req, CELLS_1275_TO_BYTES(addr_cells) +
 633             CELLS_1275_TO_BYTES(intr_cells));
 634 
 635 exit3:
 636         kmem_free(imap_mask, imap_mask_sz);
 637 
 638 exit2:
 639         kmem_free(imap, imap_sz);
 640 
 641         return (intr_parent_dip);
 642 }
 643 
 644 /*
 645  * process_intr_ops:
 646  *
 647  * Process the interrupt op via the interrupt parent.
 648  */
 649 int
 650 process_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t op,
 651     ddi_intr_handle_impl_t *hdlp, void *result)
 652 {
 653         int             ret = DDI_FAILURE;
 654 
 655         if (NEXUS_HAS_INTR_OP(pdip)) {
 656                 ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->
 657                     bus_intr_op)) (pdip, rdip, op, hdlp, result);
 658         } else {
 659                 cmn_err(CE_WARN, "Failed to process interrupt "
 660                     "for %s%d due to down-rev nexus driver %s%d",
 661                     ddi_get_name(rdip), ddi_get_instance(rdip),
 662                     ddi_get_name(pdip), ddi_get_instance(pdip));
 663         }
 664 
 665         return (ret);
 666 }
 667 
 668 /*ARGSUSED*/
 669 uint_t
 670 softlevel1(caddr_t arg)
 671 {
 672         softint();
 673         return (1);
 674 }
 675 
 676 /*
 677  * indirection table, to save us some large switch statements
 678  * NOTE: This must agree with "INTLEVEL_foo" constants in
 679  *      <sys/avintr.h>
 680  */
 681 struct autovec *const vectorlist[] = { 0 };
 682 
 683 /*
 684  * This value is exported here for the functions in avintr.c
 685  */
 686 const uint_t maxautovec = (sizeof (vectorlist) / sizeof (vectorlist[0]));
 687 
 688 /*
 689  * Check for machine specific interrupt levels which cannot be reassigned by
 690  * settrap(), sun4u version.
 691  *
 692  * sun4u does not support V8 SPARC "fast trap" handlers.
 693  */
 694 /*ARGSUSED*/
 695 int
 696 exclude_settrap(int lvl)
 697 {
 698         return (1);
 699 }
 700 
 701 /*
 702  * Check for machine specific interrupt levels which cannot have interrupt
 703  * handlers added. We allow levels 1 through 15; level 0 is nonsense.
 704  */
 705 /*ARGSUSED*/
 706 int
 707 exclude_level(int lvl)
 708 {
 709         return ((lvl < 1) || (lvl > 15));
 710 }
 711 
 712 /*
 713  * Wrapper functions used by New DDI interrupt framework.
 714  */
 715 
 716 /*
 717  * i_ddi_intr_ops:
 718  */
 719 int
 720 i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op,
 721     ddi_intr_handle_impl_t *hdlp, void *result)
 722 {
 723         dev_info_t      *pdip = ddi_get_parent(dip);
 724         int             ret = DDI_FAILURE;
 725 
 726         /*
 727          * The following check is required to address
 728          * one of the test case of ADDI test suite.
 729          */
 730         if (pdip == NULL)
 731                 return (DDI_FAILURE);
 732 
 733         if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
 734                 return (process_intr_ops(pdip, rdip, op, hdlp, result));
 735 
 736         if (hdlp->ih_vector == 0)
 737                 hdlp->ih_vector = i_ddi_get_inum(rdip, hdlp->ih_inum);
 738 
 739         if (hdlp->ih_pri == 0)
 740                 hdlp->ih_pri = i_ddi_get_intr_pri(rdip, hdlp->ih_inum);
 741 
 742         switch (op) {
 743         case DDI_INTROP_ADDISR:
 744         case DDI_INTROP_REMISR:
 745         case DDI_INTROP_GETTARGET:
 746         case DDI_INTROP_SETTARGET:
 747         case DDI_INTROP_ENABLE:
 748         case DDI_INTROP_DISABLE:
 749         case DDI_INTROP_BLOCKENABLE:
 750         case DDI_INTROP_BLOCKDISABLE:
 751                 /*
 752                  * Try and determine our parent and possibly an interrupt
 753                  * translation. intr parent dip returned held
 754                  */
 755                 if ((pdip = get_intr_parent(pdip, dip, hdlp)) == NULL)
 756                         goto done;
 757         }
 758 
 759         ret = process_intr_ops(pdip, rdip, op, hdlp, result);
 760 
 761 done:
 762         switch (op) {
 763         case DDI_INTROP_ADDISR:
 764         case DDI_INTROP_REMISR:
 765         case DDI_INTROP_ENABLE:
 766         case DDI_INTROP_DISABLE:
 767         case DDI_INTROP_BLOCKENABLE:
 768         case DDI_INTROP_BLOCKDISABLE:
 769                 /* Release hold acquired in get_intr_parent() */
 770                 if (pdip)
 771                         ndi_rele_devi(pdip);
 772         }
 773 
 774         hdlp->ih_vector = 0;
 775 
 776         return (ret);
 777 }
 778 
 779 /*
 780  * i_ddi_add_ivintr:
 781  */
 782 /*ARGSUSED*/
 783 int
 784 i_ddi_add_ivintr(ddi_intr_handle_impl_t *hdlp)
 785 {
 786         /*
 787          * If the PIL was set and is valid use it, otherwise
 788          * default it to 1
 789          */
 790         if ((hdlp->ih_pri < 1) || (hdlp->ih_pri > PIL_MAX))
 791                 hdlp->ih_pri = 1;
 792 
 793         VERIFY(add_ivintr(hdlp->ih_vector, hdlp->ih_pri,
 794             (intrfunc)hdlp->ih_cb_func, hdlp->ih_cb_arg1,
 795             hdlp->ih_cb_arg2, NULL) == 0);
 796 
 797         return (DDI_SUCCESS);
 798 }
 799 
 800 /*
 801  * i_ddi_rem_ivintr:
 802  */
 803 /*ARGSUSED*/
 804 void
 805 i_ddi_rem_ivintr(ddi_intr_handle_impl_t *hdlp)
 806 {
 807         VERIFY(rem_ivintr(hdlp->ih_vector, hdlp->ih_pri) == 0);
 808 }
 809 
 810 /*
 811  * i_ddi_get_inum - Get the interrupt number property from the
 812  * specified device. Note that this function is called only for
 813  * the FIXED interrupt type.
 814  */
 815 uint32_t
 816 i_ddi_get_inum(dev_info_t *dip, uint_t inumber)
 817 {
 818         int32_t                 intrlen, intr_cells, max_intrs;
 819         prop_1275_cell_t        *ip, intr_sz;
 820         uint32_t                intr = 0;
 821 
 822         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
 823             DDI_PROP_CANSLEEP,
 824             "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
 825 
 826                 intr_cells = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
 827                     "#interrupt-cells", 1);
 828 
 829                 /* adjust for number of bytes */
 830                 intr_sz = CELLS_1275_TO_BYTES(intr_cells);
 831 
 832                 /* Calculate the number of interrupts */
 833                 max_intrs = intrlen / intr_sz;
 834 
 835                 if (inumber < max_intrs) {
 836                         prop_1275_cell_t *intrp = ip;
 837 
 838                         /* Index into interrupt property */
 839                         intrp += (inumber * intr_cells);
 840 
 841                         cells_1275_copy(intrp, &intr, intr_cells);
 842                 }
 843 
 844                 kmem_free(ip, intrlen);
 845         }
 846 
 847         return (intr);
 848 }
 849 
 850 /*
 851  * i_ddi_get_intr_pri - Get the interrupt-priorities property from
 852  * the specified device. Note that this function is called only for
 853  * the FIXED interrupt type.
 854  */
 855 uint32_t
 856 i_ddi_get_intr_pri(dev_info_t *dip, uint_t inumber)
 857 {
 858         uint32_t        *intr_prio_p;
 859         uint32_t        pri = 0;
 860         int32_t         i;
 861 
 862         /*
 863          * Use the "interrupt-priorities" property to determine the
 864          * the pil/ipl for the interrupt handler.
 865          */
 866         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
 867             "interrupt-priorities", (caddr_t)&intr_prio_p,
 868             &i) == DDI_SUCCESS) {
 869                 if (inumber < (i / sizeof (int32_t)))
 870                         pri = intr_prio_p[inumber];
 871                 kmem_free(intr_prio_p, i);
 872         }
 873 
 874         return (pri);
 875 }
 876 
 877 int
 878 i_ddi_get_intx_nintrs(dev_info_t *dip)
 879 {
 880         int32_t intrlen;
 881         prop_1275_cell_t intr_sz;
 882         prop_1275_cell_t *ip;
 883         int32_t ret = 0;
 884 
 885         if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
 886             DDI_PROP_CANSLEEP,
 887             "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
 888 
 889                 intr_sz = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
 890                     "#interrupt-cells", 1);
 891                 /* adjust for number of bytes */
 892                 intr_sz = CELLS_1275_TO_BYTES(intr_sz);
 893 
 894                 ret = intrlen / intr_sz;
 895 
 896                 kmem_free(ip, intrlen);
 897         }
 898 
 899         return (ret);
 900 }
 901 
 902 /*
 903  * i_ddi_add_softint - allocate and add a software interrupt.
 904  *
 905  * NOTE: All software interrupts that are registered through DDI
 906  *       should be triggered only on a single target or CPU.
 907  */
 908 int
 909 i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp)
 910 {
 911         if ((hdlp->ih_private = (void *)add_softintr(hdlp->ih_pri,
 912             hdlp->ih_cb_func, hdlp->ih_cb_arg1, SOFTINT_ST)) == NULL)
 913                 return (DDI_FAILURE);
 914 
 915         return (DDI_SUCCESS);
 916 }
 917 
 918 /*
 919  * i_ddi_remove_softint - remove and free a software interrupt.
 920  */
 921 void
 922 i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp)
 923 {
 924         ASSERT(hdlp->ih_private != NULL);
 925 
 926         if (rem_softintr((uint64_t)hdlp->ih_private) == 0)
 927                 hdlp->ih_private = NULL;
 928 }
 929 
 930 /*
 931  * i_ddi_trigger_softint - trigger a software interrupt.
 932  */
 933 int
 934 i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2)
 935 {
 936         int     ret;
 937 
 938         ASSERT(hdlp->ih_private != NULL);
 939 
 940         /* Update the second argument for the software interrupt */
 941         if ((ret = update_softint_arg2((uint64_t)hdlp->ih_private, arg2)) == 0)
 942                 setsoftint((uint64_t)hdlp->ih_private);
 943 
 944         return (ret ? DDI_EPENDING : DDI_SUCCESS);
 945 }
 946 
 947 /*
 948  * i_ddi_set_softint_pri - change software interrupt priority.
 949  */
 950 /* ARGSUSED */
 951 int
 952 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri)
 953 {
 954         int     ret;
 955 
 956         ASSERT(hdlp->ih_private != NULL);
 957 
 958         /* Update the interrupt priority for the software interrupt */
 959         ret = update_softint_pri((uint64_t)hdlp->ih_private, hdlp->ih_pri);
 960 
 961         return (ret ? DDI_FAILURE : DDI_SUCCESS);
 962 }
 963 
 964 /*ARGSUSED*/
 965 void
 966 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp)
 967 {
 968 }
 969 
 970 /*ARGSUSED*/
 971 void
 972 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp)
 973 {
 974 }
 975 
 976 /*
 977  * SECTION: DDI Memory/DMA
 978  */
 979 
 980 /* set HAT endianess attributes from ddi_device_acc_attr */
 981 void
 982 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t *devaccp, uint_t *hataccp)
 983 {
 984         if (devaccp != NULL) {
 985                 if (devaccp->devacc_attr_endian_flags == DDI_STRUCTURE_LE_ACC) {
 986                         *hataccp &= ~HAT_ENDIAN_MASK;
 987                         *hataccp |= HAT_STRUCTURE_LE;
 988                 }
 989         }
 990 }
 991 
 992 /*
 993  * Check if the specified cache attribute is supported on the platform.
 994  * This function must be called before i_ddi_cacheattr_to_hatacc().
 995  */
 996 boolean_t
 997 i_ddi_check_cache_attr(uint_t flags)
 998 {
 999         /*
1000          * The cache attributes are mutually exclusive. Any combination of
1001          * the attributes leads to a failure.
1002          */
1003         uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1004         if ((cache_attr != 0) && !ISP2(cache_attr))
1005                 return (B_FALSE);
1006 
1007         /*
1008          * On the sparc architecture, only IOMEM_DATA_CACHED is meaningful,
1009          * but others lead to a failure.
1010          */
1011         if (cache_attr & IOMEM_DATA_CACHED)
1012                 return (B_TRUE);
1013         else
1014                 return (B_FALSE);
1015 }
1016 
1017 /* set HAT cache attributes from the cache attributes */
1018 void
1019 i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp)
1020 {
1021         uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1022         static char *fname = "i_ddi_cacheattr_to_hatacc";
1023 #if defined(lint)
1024         *hataccp = *hataccp;
1025 #endif
1026         /*
1027          * set HAT attrs according to the cache attrs.
1028          */
1029         switch (cache_attr) {
1030         /*
1031          * The cache coherency is always maintained on SPARC, and
1032          * nothing is required.
1033          */
1034         case IOMEM_DATA_CACHED:
1035                 break;
1036         /*
1037          * Both IOMEM_DATA_UC_WRITE_COMBINED and IOMEM_DATA_UNCACHED are
1038          * not supported on SPARC -- this case must not occur because the
1039          * cache attribute is scrutinized before this function is called.
1040          */
1041         case IOMEM_DATA_UNCACHED:
1042         case IOMEM_DATA_UC_WR_COMBINE:
1043         default:
1044                 cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.",
1045                     fname, cache_attr);
1046         }
1047 }
1048 
1049 static vmem_t *little_endian_arena;
1050 static vmem_t *big_endian_arena;
1051 
1052 static void *
1053 segkmem_alloc_le(vmem_t *vmp, size_t size, int flag)
1054 {
1055         return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_LE,
1056             segkmem_page_create, NULL));
1057 }
1058 
1059 static void *
1060 segkmem_alloc_be(vmem_t *vmp, size_t size, int flag)
1061 {
1062         return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_BE,
1063             segkmem_page_create, NULL));
1064 }
1065 
1066 void
1067 ka_init(void)
1068 {
1069         little_endian_arena = vmem_create("little_endian", NULL, 0, 1,
1070             segkmem_alloc_le, segkmem_free, heap_arena, 0, VM_SLEEP);
1071         big_endian_arena = vmem_create("big_endian", NULL, 0, 1,
1072             segkmem_alloc_be, segkmem_free, heap_arena, 0, VM_SLEEP);
1073 }
1074 
1075 /*
1076  * Allocate from the system, aligned on a specific boundary.
1077  * The alignment, if non-zero, must be a power of 2.
1078  */
1079 static void *
1080 kalloca(size_t size, size_t align, int cansleep, uint_t endian_flags)
1081 {
1082         size_t *addr, *raddr, rsize;
1083         size_t hdrsize = 4 * sizeof (size_t);   /* must be power of 2 */
1084 
1085         align = MAX(align, hdrsize);
1086         ASSERT((align & (align - 1)) == 0);
1087 
1088         /*
1089          * We need to allocate
1090          *    rsize = size + hdrsize + align - MIN(hdrsize, buffer_alignment)
1091          * bytes to be sure we have enough freedom to satisfy the request.
1092          * Since the buffer alignment depends on the request size, this is
1093          * not straightforward to use directly.
1094          *
1095          * kmem guarantees that any allocation of a 64-byte multiple will be
1096          * 64-byte aligned.  Since rounding up the request could add more
1097          * than we save, we compute the size with and without alignment, and
1098          * use the smaller of the two.
1099          */
1100         rsize = size + hdrsize + align;
1101 
1102         if (endian_flags == DDI_STRUCTURE_LE_ACC) {
1103                 raddr = vmem_alloc(little_endian_arena, rsize,
1104                     cansleep ? VM_SLEEP : VM_NOSLEEP);
1105         } else {
1106                 raddr = vmem_alloc(big_endian_arena, rsize,
1107                     cansleep ? VM_SLEEP : VM_NOSLEEP);
1108         }
1109 
1110         if (raddr == NULL)
1111                 return (NULL);
1112 
1113         addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align);
1114         ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize);
1115 
1116         addr[-3] = (size_t)endian_flags;
1117         addr[-2] = (size_t)raddr;
1118         addr[-1] = rsize;
1119 
1120         return (addr);
1121 }
1122 
1123 static void
1124 kfreea(void *addr)
1125 {
1126         size_t *saddr = addr;
1127 
1128         if (saddr[-3] == DDI_STRUCTURE_LE_ACC)
1129                 vmem_free(little_endian_arena, (void *)saddr[-2], saddr[-1]);
1130         else
1131                 vmem_free(big_endian_arena, (void *)saddr[-2], saddr[-1]);
1132 }
1133 
1134 /*
1135  * This used to be ddi_iomin, but we were the only remaining caller, so
1136  * we've made it private and moved it here.
1137  */
1138 static int
1139 i_ddi_iomin(dev_info_t *a, int i, int stream)
1140 {
1141         int r;
1142 
1143         /*
1144          * Make sure that the initial value is sane
1145          */
1146         if (!ISP2(i))
1147                 return (0);
1148         if (i == 0)
1149                 i = (stream) ? 4 : 1;
1150 
1151         r = ddi_ctlops(a, a,
1152             DDI_CTLOPS_IOMIN, (void *)(uintptr_t)stream, (void *)&i);
1153         if (r != DDI_SUCCESS || !ISP2(i))
1154                 return (0);
1155         return (i);
1156 }
1157 
1158 int
1159 i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr,
1160     size_t length, int cansleep, int flags,
1161     ddi_device_acc_attr_t *accattrp,
1162     caddr_t *kaddrp, size_t *real_length, ddi_acc_hdl_t *handlep)
1163 {
1164         caddr_t a;
1165         int iomin, align, streaming;
1166         uint_t endian_flags = DDI_NEVERSWAP_ACC;
1167 
1168 #if defined(lint)
1169         *handlep = *handlep;
1170 #endif
1171 
1172         /*
1173          * Check legality of arguments
1174          */
1175         if (length == 0 || kaddrp == NULL || attr == NULL) {
1176                 return (DDI_FAILURE);
1177         }
1178 
1179         if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 ||
1180             !ISP2(attr->dma_attr_align) || !ISP2(attr->dma_attr_minxfer)) {

1181                 return (DDI_FAILURE);
1182         }
1183 
1184         /*
1185          * check if a streaming sequential xfer is requested.
1186          */
1187         streaming = (flags & DDI_DMA_STREAMING) ? 1 : 0;
1188 
1189         /*
1190          * Drivers for 64-bit capable SBus devices will encode
1191          * the burtsizes for 64-bit xfers in the upper 16-bits.
1192          * For DMA alignment, we use the most restrictive
1193          * alignment of 32-bit and 64-bit xfers.
1194          */
1195         iomin = (attr->dma_attr_burstsizes & 0xffff) |
1196             ((attr->dma_attr_burstsizes >> 16) & 0xffff);
1197         /*
1198          * If a driver set burtsizes to 0, we give him byte alignment.
1199          * Otherwise align at the burtsizes boundary.
1200          */
1201         if (iomin == 0)
1202                 iomin = 1;
1203         else
1204                 iomin = 1 << (ddi_fls(iomin) - 1);
1205         iomin = maxbit(iomin, attr->dma_attr_minxfer);
1206         iomin = maxbit(iomin, attr->dma_attr_align);
1207         iomin = i_ddi_iomin(dip, iomin, streaming);
1208         if (iomin == 0)
1209                 return (DDI_FAILURE);
1210 
1211         ASSERT((iomin & (iomin - 1)) == 0);
1212         ASSERT(iomin >= attr->dma_attr_minxfer);
1213         ASSERT(iomin >= attr->dma_attr_align);
1214 
1215         length = P2ROUNDUP(length, iomin);
1216         align = iomin;
1217 
1218         if (accattrp != NULL)
1219                 endian_flags = accattrp->devacc_attr_endian_flags;
1220 
1221         a = kalloca(length, align, cansleep, endian_flags);
1222         if ((*kaddrp = a) == 0) {
1223                 return (DDI_FAILURE);
1224         } else {
1225                 if (real_length) {
1226                         *real_length = length;
1227                 }
1228                 if (handlep) {
1229                         /*
1230                          * assign handle information
1231                          */
1232                         impl_acc_hdl_init(handlep);
1233                 }
1234                 return (DDI_SUCCESS);
1235         }
1236 }
1237 
1238 /* ARGSUSED */
1239 void
1240 i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap)
1241 {
1242         kfreea(kaddr);
1243 }
1244 
1245 /*
1246  * SECTION: DDI Data Access
1247  */
1248 
1249 static uintptr_t impl_acc_hdl_id = 0;
1250 
1251 /*
1252  * access handle allocator
1253  */
1254 ddi_acc_hdl_t *
1255 impl_acc_hdl_get(ddi_acc_handle_t hdl)
1256 {
1257         /*
1258          * Extract the access handle address from the DDI implemented
1259          * access handle
1260          */
1261         return (&((ddi_acc_impl_t *)hdl)->ahi_common);
1262 }
1263 
1264 ddi_acc_handle_t
1265 impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
1266 {
1267         ddi_acc_impl_t *hp;
1268         on_trap_data_t *otp;
1269         int sleepflag;
1270 
1271         sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1272 
1273         /*
1274          * Allocate and initialize the data access handle and error status.
1275          */
1276         if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
1277                 goto fail;
1278         if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
1279             sizeof (ndi_err_t), sleepflag)) == NULL) {
1280                 kmem_free(hp, sizeof (ddi_acc_impl_t));
1281                 goto fail;
1282         }
1283         if ((otp = (on_trap_data_t *)kmem_zalloc(
1284             sizeof (on_trap_data_t), sleepflag)) == NULL) {
1285                 kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1286                 kmem_free(hp, sizeof (ddi_acc_impl_t));
1287                 goto fail;
1288         }
1289         hp->ahi_err->err_ontrap = otp;
1290         hp->ahi_common.ah_platform_private = (void *)hp;
1291 
1292         return ((ddi_acc_handle_t)hp);
1293 fail:
1294         if ((waitfp != (int (*)())KM_SLEEP) &&
1295             (waitfp != (int (*)())KM_NOSLEEP))
1296                 ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
1297         return (NULL);
1298 }
1299 
1300 void
1301 impl_acc_hdl_free(ddi_acc_handle_t handle)
1302 {
1303         ddi_acc_impl_t *hp;
1304 
1305         /*
1306          * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
1307          * because that's what we allocated in impl_acc_hdl_alloc() above.
1308          */
1309         hp = (ddi_acc_impl_t *)handle;
1310         if (hp) {
1311                 kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
1312                 kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1313                 kmem_free(hp, sizeof (ddi_acc_impl_t));
1314                 if (impl_acc_hdl_id)
1315                         ddi_run_callback(&impl_acc_hdl_id);
1316         }
1317 }
1318 
1319 #define PCI_GET_MP_PFN(mp, page_no)     ((mp)->dmai_ndvmapages == 1 ? \
1320         (pfn_t)(mp)->dmai_iopte:(((pfn_t *)(mp)->dmai_iopte)[page_no]))
1321 
1322 /*
1323  * Function called after a dma fault occurred to find out whether the
1324  * fault address is associated with a driver that is able to handle faults
1325  * and recover from faults.
1326  */
1327 /* ARGSUSED */
1328 int
1329 impl_dma_check(dev_info_t *dip, const void *handle, const void *addr,
1330     const void *not_used)
1331 {
1332         ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1333         pfn_t fault_pfn = mmu_btop(*(uint64_t *)addr);
1334         pfn_t comp_pfn;
1335 
1336         /*
1337          * The driver has to set DDI_DMA_FLAGERR to recover from dma faults.
1338          */
1339         int page;
1340 
1341         ASSERT(mp);
1342         for (page = 0; page < mp->dmai_ndvmapages; page++) {
1343                 comp_pfn = PCI_GET_MP_PFN(mp, page);
1344                 if (fault_pfn == comp_pfn)
1345                         return (DDI_FM_NONFATAL);
1346         }
1347         return (DDI_FM_UNKNOWN);
1348 }
1349 
1350 /*
1351  * Function used to check if a given access handle owns the failing address.
1352  * Called by ndi_fmc_error, when we detect a PIO error.
1353  */
1354 /* ARGSUSED */
1355 static int
1356 impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
1357     const void *not_used)
1358 {
1359         pfn_t pfn, fault_pfn;
1360         ddi_acc_hdl_t *hp;
1361 
1362         hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);
1363 
1364         ASSERT(hp);
1365 
1366         if (addr != NULL) {
1367                 pfn = hp->ah_pfn;
1368                 fault_pfn = mmu_btop(*(uint64_t *)addr);
1369                 if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
1370                         return (DDI_FM_NONFATAL);
1371         }
1372         return (DDI_FM_UNKNOWN);
1373 }
1374 
1375 void
1376 impl_acc_err_init(ddi_acc_hdl_t *handlep)
1377 {
1378         int fmcap;
1379         ndi_err_t *errp;
1380         on_trap_data_t *otp;
1381         ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;
1382 
1383         fmcap = ddi_fm_capable(handlep->ah_dip);
1384 
1385         if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
1386             !DDI_FM_ACC_ERR_CAP(fmcap)) {
1387                 handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
1388         } else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
1389                 if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
1390                         if (handlep->ah_xfermodes)
1391                                 return;
1392                         i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
1393                             NULL, DDI_NOSLEEP);
1394                 } else {
1395                         errp = hp->ahi_err;
1396                         otp = (on_trap_data_t *)errp->err_ontrap;
1397                         otp->ot_handle = (void *)(hp);
1398                         otp->ot_prot = OT_DATA_ACCESS;
1399                         if (handlep->ah_acc.devacc_attr_access ==
1400                             DDI_CAUTIOUS_ACC)
1401                                 otp->ot_trampoline =
1402                                     (uintptr_t)&i_ddi_caut_trampoline;
1403                         else
1404                                 otp->ot_trampoline =
1405                                     (uintptr_t)&i_ddi_prot_trampoline;
1406                         errp->err_status = DDI_FM_OK;
1407                         errp->err_expected = DDI_FM_ERR_UNEXPECTED;
1408                         errp->err_cf = impl_acc_check;
1409                 }
1410         }
1411 }
1412 
1413 void
1414 impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
1415 {
1416         ddi_acc_impl_t *hp;
1417 
1418         ASSERT(handlep);
1419 
1420         hp = (ddi_acc_impl_t *)handlep;
1421 
1422         /*
1423          * check for SW byte-swapping
1424          */
1425         hp->ahi_get8 = i_ddi_get8;
1426         hp->ahi_put8 = i_ddi_put8;
1427         hp->ahi_rep_get8 = i_ddi_rep_get8;
1428         hp->ahi_rep_put8 = i_ddi_rep_put8;
1429         if (handlep->ah_acc.devacc_attr_endian_flags & DDI_STRUCTURE_LE_ACC) {
1430                 hp->ahi_get16 = i_ddi_swap_get16;
1431                 hp->ahi_get32 = i_ddi_swap_get32;
1432                 hp->ahi_get64 = i_ddi_swap_get64;
1433                 hp->ahi_put16 = i_ddi_swap_put16;
1434                 hp->ahi_put32 = i_ddi_swap_put32;
1435                 hp->ahi_put64 = i_ddi_swap_put64;
1436                 hp->ahi_rep_get16 = i_ddi_swap_rep_get16;
1437                 hp->ahi_rep_get32 = i_ddi_swap_rep_get32;
1438                 hp->ahi_rep_get64 = i_ddi_swap_rep_get64;
1439                 hp->ahi_rep_put16 = i_ddi_swap_rep_put16;
1440                 hp->ahi_rep_put32 = i_ddi_swap_rep_put32;
1441                 hp->ahi_rep_put64 = i_ddi_swap_rep_put64;
1442         } else {
1443                 hp->ahi_get16 = i_ddi_get16;
1444                 hp->ahi_get32 = i_ddi_get32;
1445                 hp->ahi_get64 = i_ddi_get64;
1446                 hp->ahi_put16 = i_ddi_put16;
1447                 hp->ahi_put32 = i_ddi_put32;
1448                 hp->ahi_put64 = i_ddi_put64;
1449                 hp->ahi_rep_get16 = i_ddi_rep_get16;
1450                 hp->ahi_rep_get32 = i_ddi_rep_get32;
1451                 hp->ahi_rep_get64 = i_ddi_rep_get64;
1452                 hp->ahi_rep_put16 = i_ddi_rep_put16;
1453                 hp->ahi_rep_put32 = i_ddi_rep_put32;
1454                 hp->ahi_rep_put64 = i_ddi_rep_put64;
1455         }
1456 
1457         /* Legacy fault flags and support */
1458         hp->ahi_fault_check = i_ddi_acc_fault_check;
1459         hp->ahi_fault_notify = i_ddi_acc_fault_notify;
1460         hp->ahi_fault = 0;
1461         impl_acc_err_init(handlep);
1462 }
1463 
1464 void
1465 i_ddi_acc_set_fault(ddi_acc_handle_t handle)
1466 {
1467         ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1468 
1469         if (!hp->ahi_fault) {
1470                 hp->ahi_fault = 1;
1471                         (*hp->ahi_fault_notify)(hp);
1472         }
1473 }
1474 
1475 void
1476 i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
1477 {
1478         ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1479 
1480         if (hp->ahi_fault) {
1481                 hp->ahi_fault = 0;
1482                         (*hp->ahi_fault_notify)(hp);
1483         }
1484 }
1485 
1486 /* ARGSUSED */
1487 void
1488 i_ddi_acc_fault_notify(ddi_acc_impl_t *hp)
1489 {
1490         /* Default version, does nothing */
1491 }
1492 
1493 /*
1494  * SECTION: Misc functions
1495  */
1496 
1497 /*
1498  * instance wrappers
1499  */
1500 /*ARGSUSED*/
1501 uint_t
1502 impl_assign_instance(dev_info_t *dip)
1503 {
1504         return ((uint_t)-1);
1505 }
1506 
1507 /*ARGSUSED*/
1508 int
1509 impl_keep_instance(dev_info_t *dip)
1510 {
1511         return (DDI_FAILURE);
1512 }
1513 
1514 /*ARGSUSED*/
1515 int
1516 impl_free_instance(dev_info_t *dip)
1517 {
1518         return (DDI_FAILURE);
1519 }
1520 
1521 /*ARGSUSED*/
1522 int
1523 impl_check_cpu(dev_info_t *devi)
1524 {
1525         return (DDI_SUCCESS);
1526 }
1527 
1528 
1529 static const char *nocopydevs[] = {
1530         "SUNW,ffb",
1531         "SUNW,afb",
1532         NULL
1533 };
1534 
1535 /*
1536  * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
1537  * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
1538  */
1539 /*ARGSUSED*/
1540 int
1541 e_ddi_copyfromdev(dev_info_t *devi,
1542     off_t off, const void *devaddr, void *kaddr, size_t len)
1543 {
1544         const char **argv;
1545 
1546         for (argv = nocopydevs; *argv; argv++)
1547                 if (strcmp(ddi_binding_name(devi), *argv) == 0) {
1548                         bzero(kaddr, len);
1549                         return (0);
1550                 }
1551 
1552         bcopy(devaddr, kaddr, len);
1553         return (0);
1554 }
1555 
1556 /*
1557  * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
1558  * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
1559  */
1560 /*ARGSUSED*/
1561 int
1562 e_ddi_copytodev(dev_info_t *devi,
1563     off_t off, const void *kaddr, void *devaddr, size_t len)
1564 {
1565         const char **argv;
1566 
1567         for (argv = nocopydevs; *argv; argv++)
1568                 if (strcmp(ddi_binding_name(devi), *argv) == 0)
1569                         return (1);
1570 
1571         bcopy(kaddr, devaddr, len);
1572         return (0);
1573 }
1574 
1575 /*
1576  * Boot Configuration
1577  */
1578 idprom_t idprom;
1579 
1580 /*
1581  * Configure the hardware on the system.
1582  * Called before the rootfs is mounted
1583  */
1584 void
1585 configure(void)
1586 {
1587         extern void i_ddi_init_root();
1588 
1589         /* We better have released boot by this time! */
1590         ASSERT(!bootops);
1591 
1592         /*
1593          * Determine whether or not to use the fpu, V9 SPARC cpus
1594          * always have one. Could check for existence of a fp queue,
1595          * Ultra I, II and IIa do not have a fp queue.
1596          */
1597         if (fpu_exists)
1598                 fpu_probe();
1599         else
1600                 cmn_err(CE_CONT, "FPU not in use\n");
1601 
1602 #if 0 /* XXXQ - not necessary for sun4u */
1603         /*
1604          * This following line fixes bugid 1041296; we need to do a
1605          * prom_nextnode(0) because this call ALSO patches the DMA+
1606          * bug in Campus-B and Phoenix. The prom uncaches the traptable
1607          * page as a side-effect of devr_next(0) (which prom_nextnode calls),
1608          * so this *must* be executed early on. (XXX This is untrue for sun4u)
1609          */
1610         (void) prom_nextnode((pnode_t)0);
1611 #endif
1612 
1613         /*
1614          * Initialize devices on the machine.
1615          * Uses configuration tree built by the PROMs to determine what
1616          * is present, and builds a tree of prototype dev_info nodes
1617          * corresponding to the hardware which identified itself.
1618          */
1619         i_ddi_init_root();
1620 
1621 #ifdef  DDI_PROP_DEBUG
1622         (void) ddi_prop_debug(1);       /* Enable property debugging */
1623 #endif  /* DDI_PROP_DEBUG */
1624 }
1625 
1626 /*
1627  * The "status" property indicates the operational status of a device.
1628  * If this property is present, the value is a string indicating the
1629  * status of the device as follows:
1630  *
1631  *      "okay"          operational.
1632  *      "disabled"      not operational, but might become operational.
1633  *      "fail"          not operational because a fault has been detected,
1634  *                      and it is unlikely that the device will become
1635  *                      operational without repair. no additional details
1636  *                      are available.
1637  *      "fail-xxx"      not operational because a fault has been detected,
1638  *                      and it is unlikely that the device will become
1639  *                      operational without repair. "xxx" is additional
1640  *                      human-readable information about the particular
1641  *                      fault condition that was detected.
1642  *
1643  * The absence of this property means that the operational status is
1644  * unknown or okay.
1645  *
1646  * This routine checks the status property of the specified device node
1647  * and returns 0 if the operational status indicates failure, and 1 otherwise.
1648  *
1649  * The property may exist on plug-in cards the existed before IEEE 1275-1994.
1650  * And, in that case, the property may not even be a string. So we carefully
1651  * check for the value "fail", in the beginning of the string, noting
1652  * the property length.
1653  */
1654 int
1655 status_okay(int id, char *buf, int buflen)
1656 {
1657         char status_buf[OBP_MAXPROPNAME];
1658         char *bufp = buf;
1659         int len = buflen;
1660         int proplen;
1661         static const char *status = "status";
1662         static const char *fail = "fail";
1663         size_t fail_len = strlen(fail);
1664 
1665         /*
1666          * Get the proplen ... if it's smaller than "fail",
1667          * or doesn't exist ... then we don't care, since
1668          * the value can't begin with the char string "fail".
1669          *
1670          * NB: proplen, if it's a string, includes the NULL in the
1671          * the size of the property, and fail_len does not.
1672          */
1673         proplen = prom_getproplen((pnode_t)id, (caddr_t)status);
1674         if (proplen <= fail_len)     /* nonexistent or uninteresting len */
1675                 return (1);
1676 
1677         /*
1678          * if a buffer was provided, use it
1679          */
1680         if ((buf == (char *)NULL) || (buflen <= 0)) {
1681                 bufp = status_buf;
1682                 len = sizeof (status_buf);
1683         }
1684         *bufp = (char)0;
1685 
1686         /*
1687          * Get the property into the buffer, to the extent of the buffer,
1688          * and in case the buffer is smaller than the property size,
1689          * NULL terminate the buffer. (This handles the case where
1690          * a buffer was passed in and the caller wants to print the
1691          * value, but the buffer was too small).
1692          */
1693         (void) prom_bounded_getprop((pnode_t)id, (caddr_t)status,
1694             (caddr_t)bufp, len);
1695         *(bufp + len - 1) = (char)0;
1696 
1697         /*
1698          * If the value begins with the char string "fail",
1699          * then it means the node is failed. We don't care
1700          * about any other values. We assume the node is ok
1701          * although it might be 'disabled'.
1702          */
1703         if (strncmp(bufp, fail, fail_len) == 0)
1704                 return (0);
1705 
1706         return (1);
1707 }
1708 
1709 
1710 /*
1711  * We set the cpu type from the idprom, if we can.
1712  * Note that we just read out the contents of it, for the most part.
1713  */
1714 void
1715 setcputype(void)
1716 {
1717         /*
1718          * We cache the idprom info early on so that we don't
1719          * rummage through the NVRAM unnecessarily later.
1720          */
1721         (void) prom_getidprom((caddr_t)&idprom, sizeof (idprom));
1722 }
1723 
1724 /*
1725  *  Here is where we actually infer meanings to the members of idprom_t
1726  */
1727 void
1728 parse_idprom(void)
1729 {
1730         if (idprom.id_format == IDFORM_1) {
1731                 (void) localetheraddr((struct ether_addr *)idprom.id_ether,
1732                     (struct ether_addr *)NULL);
1733                 (void) snprintf(hw_serial, HW_HOSTID_LEN, "%u",
1734                     (idprom.id_machine << 24) + idprom.id_serial);
1735         } else
1736                 prom_printf("Invalid format code in IDprom.\n");
1737 }
1738 
1739 /*
1740  * Allow for implementation specific correction of PROM property values.
1741  */
1742 /*ARGSUSED*/
1743 void
1744 impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len,
1745     caddr_t buffer)
1746 {
1747         /*
1748          * There are no adjustments needed in this implementation.
1749          */
1750 }
1751 
1752 /*
1753  * The following functions ready a cautious request to go up to the nexus
1754  * driver.  It is up to the nexus driver to decide how to process the request.
1755  * It may choose to call i_ddi_do_caut_get/put in this file, or do it
1756  * differently.
1757  */
1758 
1759 static void
1760 i_ddi_caut_getput_ctlops(
1761     ddi_acc_impl_t *hp, uint64_t host_addr, uint64_t dev_addr, size_t size,
1762     size_t repcount, uint_t flags, ddi_ctl_enum_t cmd)
1763 {
1764         peekpoke_ctlops_t       cautacc_ctlops_arg;
1765 
1766         cautacc_ctlops_arg.size = size;
1767         cautacc_ctlops_arg.dev_addr = dev_addr;
1768         cautacc_ctlops_arg.host_addr = host_addr;
1769         cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp;
1770         cautacc_ctlops_arg.repcount = repcount;
1771         cautacc_ctlops_arg.flags = flags;
1772 
1773         (void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd,
1774             &cautacc_ctlops_arg, NULL);
1775 }
1776 
1777 uint8_t
1778 i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr)
1779 {
1780         uint8_t value;
1781         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1782             sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK);
1783 
1784         return (value);
1785 }
1786 
1787 uint16_t
1788 i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr)
1789 {
1790         uint16_t value;
1791         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1792             sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK);
1793 
1794         return (value);
1795 }
1796 
1797 uint32_t
1798 i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr)
1799 {
1800         uint32_t value;
1801         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1802             sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK);
1803 
1804         return (value);
1805 }
1806 
1807 uint64_t
1808 i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr)
1809 {
1810         uint64_t value;
1811         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1812             sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK);
1813 
1814         return (value);
1815 }
1816 
1817 void
1818 i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value)
1819 {
1820         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1821             sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE);
1822 }
1823 
1824 void
1825 i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value)
1826 {
1827         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1828             sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE);
1829 }
1830 
1831 void
1832 i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value)
1833 {
1834         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1835             sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE);
1836 }
1837 
1838 void
1839 i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value)
1840 {
1841         i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1842             sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE);
1843 }
1844 
1845 void
1846 i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1847         size_t repcount, uint_t flags)
1848 {
1849         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1850             sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK);
1851 }
1852 
1853 void
1854 i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1855     uint16_t *dev_addr, size_t repcount, uint_t flags)
1856 {
1857         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1858             sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK);
1859 }
1860 
1861 void
1862 i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1863     uint32_t *dev_addr, size_t repcount, uint_t flags)
1864 {
1865         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1866             sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK);
1867 }
1868 
1869 void
1870 i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1871     uint64_t *dev_addr, size_t repcount, uint_t flags)
1872 {
1873         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1874             sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK);
1875 }
1876 
1877 void
1878 i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1879         size_t repcount, uint_t flags)
1880 {
1881         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1882             sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE);
1883 }
1884 
1885 void
1886 i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1887     uint16_t *dev_addr, size_t repcount, uint_t flags)
1888 {
1889         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1890             sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE);
1891 }
1892 
1893 void
1894 i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1895     uint32_t *dev_addr, size_t repcount, uint_t flags)
1896 {
1897         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1898             sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE);
1899 }
1900 
1901 void
1902 i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1903     uint64_t *dev_addr, size_t repcount, uint_t flags)
1904 {
1905         i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1906             sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE);
1907 }
1908 
1909 /*
1910  * This is called only to process peek/poke when the DIP is NULL.
1911  * Assume that this is for memory, as nexi take care of device safe accesses.
1912  */
1913 int
1914 peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args)
1915 {
1916         int err = DDI_SUCCESS;
1917         on_trap_data_t otd;
1918 
1919         /* Set up protected environment. */
1920         if (!on_trap(&otd, OT_DATA_ACCESS)) {
1921                 uintptr_t tramp = otd.ot_trampoline;
1922 
1923                 if (cmd == DDI_CTLOPS_POKE) {
1924                         otd.ot_trampoline = (uintptr_t)&poke_fault;
1925                         err = do_poke(in_args->size, (void *)in_args->dev_addr,
1926                             (void *)in_args->host_addr);
1927                 } else {
1928                         otd.ot_trampoline = (uintptr_t)&peek_fault;
1929                         err = do_peek(in_args->size, (void *)in_args->dev_addr,
1930                             (void *)in_args->host_addr);
1931                 }
1932                 otd.ot_trampoline = tramp;
1933         } else
1934                 err = DDI_FAILURE;
1935 
1936         /* Take down protected environment. */
1937         no_trap();
1938 
1939         return (err);
1940 }
--- EOF ---