1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2014 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 */ 26 27 #include <sys/sysmacros.h> 28 #include <sys/zfs_context.h> 29 #include <sys/fm/fs/zfs.h> 30 #include <sys/spa.h> 31 #include <sys/txg.h> 32 #include <sys/spa_impl.h> 33 #include <sys/vdev_impl.h> 34 #include <sys/zio_impl.h> 35 #include <sys/zio_compress.h> 36 #include <sys/zio_checksum.h> 37 #include <sys/dmu_objset.h> 38 #include <sys/arc.h> 39 #include <sys/ddt.h> 40 #include <sys/blkptr.h> 41 #include <sys/zfeature.h> 42 43 /* 44 * ========================================================================== 45 * I/O type descriptions 46 * ========================================================================== 47 */ 48 const char *zio_type_name[ZIO_TYPES] = { 49 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 50 "zio_ioctl" 51 }; 52 53 /* 54 * ========================================================================== 55 * I/O kmem caches 56 * ========================================================================== 57 */ 58 kmem_cache_t *zio_cache; 59 kmem_cache_t *zio_link_cache; 60 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 61 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 62 63 #ifdef _KERNEL 64 extern vmem_t *zio_alloc_arena; 65 #endif 66 67 /* 68 * The following actions directly effect the spa's sync-to-convergence logic. 69 * The values below define the sync pass when we start performing the action. 70 * Care should be taken when changing these values as they directly impact 71 * spa_sync() performance. Tuning these values may introduce subtle performance 72 * pathologies and should only be done in the context of performance analysis. 73 * These tunables will eventually be removed and replaced with #defines once 74 * enough analysis has been done to determine optimal values. 75 * 76 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 77 * regular blocks are not deferred. 78 */ 79 int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */ 80 int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */ 81 int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */ 82 83 /* 84 * An allocating zio is one that either currently has the DVA allocate 85 * stage set or will have it later in its lifetime. 86 */ 87 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 88 89 boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 90 91 #ifdef ZFS_DEBUG 92 int zio_buf_debug_limit = 16384; 93 #else 94 int zio_buf_debug_limit = 0; 95 #endif 96 97 void 98 zio_init(void) 99 { 100 size_t c; 101 vmem_t *data_alloc_arena = NULL; 102 103 #ifdef _KERNEL 104 data_alloc_arena = zio_alloc_arena; 105 #endif 106 zio_cache = kmem_cache_create("zio_cache", 107 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 108 zio_link_cache = kmem_cache_create("zio_link_cache", 109 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 110 111 /* 112 * For small buffers, we want a cache for each multiple of 113 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 114 * for each quarter-power of 2. For large buffers, we want 115 * a cache for each multiple of PAGESIZE. 116 */ 117 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 118 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 119 size_t p2 = size; 120 size_t align = 0; 121 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 122 123 while (!ISP2(p2)) 124 p2 &= p2 - 1; 125 126 #ifndef _KERNEL 127 /* 128 * If we are using watchpoints, put each buffer on its own page, 129 * to eliminate the performance overhead of trapping to the 130 * kernel when modifying a non-watched buffer that shares the 131 * page with a watched buffer. 132 */ 133 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 134 continue; 135 #endif 136 if (size <= 4 * SPA_MINBLOCKSIZE) { 137 align = SPA_MINBLOCKSIZE; 138 } else if (IS_P2ALIGNED(size, PAGESIZE)) { 139 align = PAGESIZE; 140 } else if (IS_P2ALIGNED(size, p2 >> 2)) { 141 align = p2 >> 2; 142 } 143 144 if (align != 0) { 145 char name[36]; 146 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 147 zio_buf_cache[c] = kmem_cache_create(name, size, 148 align, NULL, NULL, NULL, NULL, NULL, cflags); 149 150 /* 151 * Since zio_data bufs do not appear in crash dumps, we 152 * pass KMC_NOTOUCH so that no allocator metadata is 153 * stored with the buffers. 154 */ 155 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 156 zio_data_buf_cache[c] = kmem_cache_create(name, size, 157 align, NULL, NULL, NULL, NULL, data_alloc_arena, 158 cflags | KMC_NOTOUCH); 159 } 160 } 161 162 while (--c != 0) { 163 ASSERT(zio_buf_cache[c] != NULL); 164 if (zio_buf_cache[c - 1] == NULL) 165 zio_buf_cache[c - 1] = zio_buf_cache[c]; 166 167 ASSERT(zio_data_buf_cache[c] != NULL); 168 if (zio_data_buf_cache[c - 1] == NULL) 169 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 170 } 171 172 zio_inject_init(); 173 } 174 175 void 176 zio_fini(void) 177 { 178 size_t c; 179 kmem_cache_t *last_cache = NULL; 180 kmem_cache_t *last_data_cache = NULL; 181 182 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 183 if (zio_buf_cache[c] != last_cache) { 184 last_cache = zio_buf_cache[c]; 185 kmem_cache_destroy(zio_buf_cache[c]); 186 } 187 zio_buf_cache[c] = NULL; 188 189 if (zio_data_buf_cache[c] != last_data_cache) { 190 last_data_cache = zio_data_buf_cache[c]; 191 kmem_cache_destroy(zio_data_buf_cache[c]); 192 } 193 zio_data_buf_cache[c] = NULL; 194 } 195 196 kmem_cache_destroy(zio_link_cache); 197 kmem_cache_destroy(zio_cache); 198 199 zio_inject_fini(); 200 } 201 202 /* 203 * ========================================================================== 204 * Allocate and free I/O buffers 205 * ========================================================================== 206 */ 207 208 /* 209 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 210 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 211 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 212 * excess / transient data in-core during a crashdump. 213 */ 214 void * 215 zio_buf_alloc(size_t size) 216 { 217 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 218 219 ASSERT3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 220 221 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 222 } 223 224 /* 225 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 226 * crashdump if the kernel panics. This exists so that we will limit the amount 227 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 228 * of kernel heap dumped to disk when the kernel panics) 229 */ 230 void * 231 zio_data_buf_alloc(size_t size) 232 { 233 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 234 235 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 236 237 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 238 } 239 240 void 241 zio_buf_free(void *buf, size_t size) 242 { 243 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 244 245 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 246 247 kmem_cache_free(zio_buf_cache[c], buf); 248 } 249 250 void 251 zio_data_buf_free(void *buf, size_t size) 252 { 253 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 254 255 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 256 257 kmem_cache_free(zio_data_buf_cache[c], buf); 258 } 259 260 /* 261 * ========================================================================== 262 * Push and pop I/O transform buffers 263 * ========================================================================== 264 */ 265 static void 266 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize, 267 zio_transform_func_t *transform) 268 { 269 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 270 271 zt->zt_orig_data = zio->io_data; 272 zt->zt_orig_size = zio->io_size; 273 zt->zt_bufsize = bufsize; 274 zt->zt_transform = transform; 275 276 zt->zt_next = zio->io_transform_stack; 277 zio->io_transform_stack = zt; 278 279 zio->io_data = data; 280 zio->io_size = size; 281 } 282 283 static void 284 zio_pop_transforms(zio_t *zio) 285 { 286 zio_transform_t *zt; 287 288 while ((zt = zio->io_transform_stack) != NULL) { 289 if (zt->zt_transform != NULL) 290 zt->zt_transform(zio, 291 zt->zt_orig_data, zt->zt_orig_size); 292 293 if (zt->zt_bufsize != 0) 294 zio_buf_free(zio->io_data, zt->zt_bufsize); 295 296 zio->io_data = zt->zt_orig_data; 297 zio->io_size = zt->zt_orig_size; 298 zio->io_transform_stack = zt->zt_next; 299 300 kmem_free(zt, sizeof (zio_transform_t)); 301 } 302 } 303 304 /* 305 * ========================================================================== 306 * I/O transform callbacks for subblocks and decompression 307 * ========================================================================== 308 */ 309 static void 310 zio_subblock(zio_t *zio, void *data, uint64_t size) 311 { 312 ASSERT(zio->io_size > size); 313 314 if (zio->io_type == ZIO_TYPE_READ) 315 bcopy(zio->io_data, data, size); 316 } 317 318 static void 319 zio_decompress(zio_t *zio, void *data, uint64_t size) 320 { 321 if (zio->io_error == 0 && 322 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 323 zio->io_data, data, zio->io_size, size) != 0) 324 zio->io_error = SET_ERROR(EIO); 325 } 326 327 /* 328 * ========================================================================== 329 * I/O parent/child relationships and pipeline interlocks 330 * ========================================================================== 331 */ 332 /* 333 * NOTE - Callers to zio_walk_parents() and zio_walk_children must 334 * continue calling these functions until they return NULL. 335 * Otherwise, the next caller will pick up the list walk in 336 * some indeterminate state. (Otherwise every caller would 337 * have to pass in a cookie to keep the state represented by 338 * io_walk_link, which gets annoying.) 339 */ 340 zio_t * 341 zio_walk_parents(zio_t *cio) 342 { 343 zio_link_t *zl = cio->io_walk_link; 344 list_t *pl = &cio->io_parent_list; 345 346 zl = (zl == NULL) ? list_head(pl) : list_next(pl, zl); 347 cio->io_walk_link = zl; 348 349 if (zl == NULL) 350 return (NULL); 351 352 ASSERT(zl->zl_child == cio); 353 return (zl->zl_parent); 354 } 355 356 zio_t * 357 zio_walk_children(zio_t *pio) 358 { 359 zio_link_t *zl = pio->io_walk_link; 360 list_t *cl = &pio->io_child_list; 361 362 zl = (zl == NULL) ? list_head(cl) : list_next(cl, zl); 363 pio->io_walk_link = zl; 364 365 if (zl == NULL) 366 return (NULL); 367 368 ASSERT(zl->zl_parent == pio); 369 return (zl->zl_child); 370 } 371 372 zio_t * 373 zio_unique_parent(zio_t *cio) 374 { 375 zio_t *pio = zio_walk_parents(cio); 376 377 VERIFY(zio_walk_parents(cio) == NULL); 378 return (pio); 379 } 380 381 void 382 zio_add_child(zio_t *pio, zio_t *cio) 383 { 384 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 385 386 /* 387 * Logical I/Os can have logical, gang, or vdev children. 388 * Gang I/Os can have gang or vdev children. 389 * Vdev I/Os can only have vdev children. 390 * The following ASSERT captures all of these constraints. 391 */ 392 ASSERT(cio->io_child_type <= pio->io_child_type); 393 394 zl->zl_parent = pio; 395 zl->zl_child = cio; 396 397 mutex_enter(&cio->io_lock); 398 mutex_enter(&pio->io_lock); 399 400 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 401 402 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 403 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 404 405 list_insert_head(&pio->io_child_list, zl); 406 list_insert_head(&cio->io_parent_list, zl); 407 408 pio->io_child_count++; 409 cio->io_parent_count++; 410 411 mutex_exit(&pio->io_lock); 412 mutex_exit(&cio->io_lock); 413 } 414 415 static void 416 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 417 { 418 ASSERT(zl->zl_parent == pio); 419 ASSERT(zl->zl_child == cio); 420 421 mutex_enter(&cio->io_lock); 422 mutex_enter(&pio->io_lock); 423 424 list_remove(&pio->io_child_list, zl); 425 list_remove(&cio->io_parent_list, zl); 426 427 pio->io_child_count--; 428 cio->io_parent_count--; 429 430 mutex_exit(&pio->io_lock); 431 mutex_exit(&cio->io_lock); 432 433 kmem_cache_free(zio_link_cache, zl); 434 } 435 436 static boolean_t 437 zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) 438 { 439 uint64_t *countp = &zio->io_children[child][wait]; 440 boolean_t waiting = B_FALSE; 441 442 mutex_enter(&zio->io_lock); 443 ASSERT(zio->io_stall == NULL); 444 if (*countp != 0) { 445 zio->io_stage >>= 1; 446 zio->io_stall = countp; 447 waiting = B_TRUE; 448 } 449 mutex_exit(&zio->io_lock); 450 451 return (waiting); 452 } 453 454 static void 455 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 456 { 457 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 458 int *errorp = &pio->io_child_error[zio->io_child_type]; 459 460 mutex_enter(&pio->io_lock); 461 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 462 *errorp = zio_worst_error(*errorp, zio->io_error); 463 pio->io_reexecute |= zio->io_reexecute; 464 ASSERT3U(*countp, >, 0); 465 466 (*countp)--; 467 468 if (*countp == 0 && pio->io_stall == countp) { 469 pio->io_stall = NULL; 470 mutex_exit(&pio->io_lock); 471 zio_execute(pio); 472 } else { 473 mutex_exit(&pio->io_lock); 474 } 475 } 476 477 static void 478 zio_inherit_child_errors(zio_t *zio, enum zio_child c) 479 { 480 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 481 zio->io_error = zio->io_child_error[c]; 482 } 483 484 /* 485 * ========================================================================== 486 * Create the various types of I/O (read, write, free, etc) 487 * ========================================================================== 488 */ 489 static zio_t * 490 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 491 void *data, uint64_t size, zio_done_func_t *done, void *private, 492 zio_type_t type, zio_priority_t priority, enum zio_flag flags, 493 vdev_t *vd, uint64_t offset, const zbookmark_phys_t *zb, 494 enum zio_stage stage, enum zio_stage pipeline) 495 { 496 zio_t *zio; 497 498 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 499 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 500 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 501 502 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 503 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 504 ASSERT(vd || stage == ZIO_STAGE_OPEN); 505 506 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 507 bzero(zio, sizeof (zio_t)); 508 509 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 510 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 511 512 list_create(&zio->io_parent_list, sizeof (zio_link_t), 513 offsetof(zio_link_t, zl_parent_node)); 514 list_create(&zio->io_child_list, sizeof (zio_link_t), 515 offsetof(zio_link_t, zl_child_node)); 516 517 if (vd != NULL) 518 zio->io_child_type = ZIO_CHILD_VDEV; 519 else if (flags & ZIO_FLAG_GANG_CHILD) 520 zio->io_child_type = ZIO_CHILD_GANG; 521 else if (flags & ZIO_FLAG_DDT_CHILD) 522 zio->io_child_type = ZIO_CHILD_DDT; 523 else 524 zio->io_child_type = ZIO_CHILD_LOGICAL; 525 526 if (bp != NULL) { 527 zio->io_bp = (blkptr_t *)bp; 528 zio->io_bp_copy = *bp; 529 zio->io_bp_orig = *bp; 530 if (type != ZIO_TYPE_WRITE || 531 zio->io_child_type == ZIO_CHILD_DDT) 532 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 533 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 534 zio->io_logical = zio; 535 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 536 pipeline |= ZIO_GANG_STAGES; 537 } 538 539 zio->io_spa = spa; 540 zio->io_txg = txg; 541 zio->io_done = done; 542 zio->io_private = private; 543 zio->io_type = type; 544 zio->io_priority = priority; 545 zio->io_vd = vd; 546 zio->io_offset = offset; 547 zio->io_orig_data = zio->io_data = data; 548 zio->io_orig_size = zio->io_size = size; 549 zio->io_orig_flags = zio->io_flags = flags; 550 zio->io_orig_stage = zio->io_stage = stage; 551 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 552 553 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 554 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 555 556 if (zb != NULL) 557 zio->io_bookmark = *zb; 558 559 if (pio != NULL) { 560 if (zio->io_logical == NULL) 561 zio->io_logical = pio->io_logical; 562 if (zio->io_child_type == ZIO_CHILD_GANG) 563 zio->io_gang_leader = pio->io_gang_leader; 564 zio_add_child(pio, zio); 565 } 566 567 return (zio); 568 } 569 570 static void 571 zio_destroy(zio_t *zio) 572 { 573 list_destroy(&zio->io_parent_list); 574 list_destroy(&zio->io_child_list); 575 mutex_destroy(&zio->io_lock); 576 cv_destroy(&zio->io_cv); 577 kmem_cache_free(zio_cache, zio); 578 } 579 580 zio_t * 581 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 582 void *private, enum zio_flag flags) 583 { 584 zio_t *zio; 585 586 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 587 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 588 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 589 590 return (zio); 591 } 592 593 zio_t * 594 zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 595 { 596 return (zio_null(NULL, spa, NULL, done, private, flags)); 597 } 598 599 zio_t * 600 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 601 void *data, uint64_t size, zio_done_func_t *done, void *private, 602 zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb) 603 { 604 zio_t *zio; 605 606 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 607 data, size, done, private, 608 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 609 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 610 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 611 612 return (zio); 613 } 614 615 zio_t * 616 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 617 void *data, uint64_t size, const zio_prop_t *zp, 618 zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done, 619 void *private, 620 zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb) 621 { 622 zio_t *zio; 623 624 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 625 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 626 zp->zp_compress >= ZIO_COMPRESS_OFF && 627 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 628 DMU_OT_IS_VALID(zp->zp_type) && 629 zp->zp_level < 32 && 630 zp->zp_copies > 0 && 631 zp->zp_copies <= spa_max_replication(spa)); 632 633 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 634 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 635 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 636 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 637 638 zio->io_ready = ready; 639 zio->io_physdone = physdone; 640 zio->io_prop = *zp; 641 642 /* 643 * Data can be NULL if we are going to call zio_write_override() to 644 * provide the already-allocated BP. But we may need the data to 645 * verify a dedup hit (if requested). In this case, don't try to 646 * dedup (just take the already-allocated BP verbatim). 647 */ 648 if (data == NULL && zio->io_prop.zp_dedup_verify) { 649 zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE; 650 } 651 652 return (zio); 653 } 654 655 zio_t * 656 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, 657 uint64_t size, zio_done_func_t *done, void *private, 658 zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb) 659 { 660 zio_t *zio; 661 662 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 663 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 664 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 665 666 return (zio); 667 } 668 669 void 670 zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite) 671 { 672 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 673 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 674 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 675 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 676 677 /* 678 * We must reset the io_prop to match the values that existed 679 * when the bp was first written by dmu_sync() keeping in mind 680 * that nopwrite and dedup are mutually exclusive. 681 */ 682 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 683 zio->io_prop.zp_nopwrite = nopwrite; 684 zio->io_prop.zp_copies = copies; 685 zio->io_bp_override = bp; 686 } 687 688 void 689 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 690 { 691 692 /* 693 * The check for EMBEDDED is a performance optimization. We 694 * process the free here (by ignoring it) rather than 695 * putting it on the list and then processing it in zio_free_sync(). 696 */ 697 if (BP_IS_EMBEDDED(bp)) 698 return; 699 metaslab_check_free(spa, bp); 700 701 /* 702 * Frees that are for the currently-syncing txg, are not going to be 703 * deferred, and which will not need to do a read (i.e. not GANG or 704 * DEDUP), can be processed immediately. Otherwise, put them on the 705 * in-memory list for later processing. 706 */ 707 if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp) || 708 txg != spa->spa_syncing_txg || 709 spa_sync_pass(spa) >= zfs_sync_pass_deferred_free) { 710 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 711 } else { 712 VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 0))); 713 } 714 } 715 716 zio_t * 717 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 718 enum zio_flag flags) 719 { 720 zio_t *zio; 721 enum zio_stage stage = ZIO_FREE_PIPELINE; 722 723 ASSERT(!BP_IS_HOLE(bp)); 724 ASSERT(spa_syncing_txg(spa) == txg); 725 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free); 726 727 if (BP_IS_EMBEDDED(bp)) 728 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 729 730 metaslab_check_free(spa, bp); 731 arc_freed(spa, bp); 732 733 /* 734 * GANG and DEDUP blocks can induce a read (for the gang block header, 735 * or the DDT), so issue them asynchronously so that this thread is 736 * not tied up. 737 */ 738 if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp)) 739 stage |= ZIO_STAGE_ISSUE_ASYNC; 740 741 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 742 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW, flags, 743 NULL, 0, NULL, ZIO_STAGE_OPEN, stage); 744 745 return (zio); 746 } 747 748 zio_t * 749 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 750 zio_done_func_t *done, void *private, enum zio_flag flags) 751 { 752 zio_t *zio; 753 754 dprintf_bp(bp, "claiming in txg %llu", txg); 755 756 if (BP_IS_EMBEDDED(bp)) 757 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 758 759 /* 760 * A claim is an allocation of a specific block. Claims are needed 761 * to support immediate writes in the intent log. The issue is that 762 * immediate writes contain committed data, but in a txg that was 763 * *not* committed. Upon opening the pool after an unclean shutdown, 764 * the intent log claims all blocks that contain immediate write data 765 * so that the SPA knows they're in use. 766 * 767 * All claims *must* be resolved in the first txg -- before the SPA 768 * starts allocating blocks -- so that nothing is allocated twice. 769 * If txg == 0 we just verify that the block is claimable. 770 */ 771 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 772 ASSERT(txg == spa_first_txg(spa) || txg == 0); 773 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 774 775 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 776 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags, 777 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 778 779 return (zio); 780 } 781 782 zio_t * 783 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 784 zio_done_func_t *done, void *private, enum zio_flag flags) 785 { 786 zio_t *zio; 787 int c; 788 789 if (vd->vdev_children == 0) { 790 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 791 ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 792 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 793 794 zio->io_cmd = cmd; 795 } else { 796 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 797 798 for (c = 0; c < vd->vdev_children; c++) 799 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 800 done, private, flags)); 801 } 802 803 return (zio); 804 } 805 806 zio_t * 807 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 808 void *data, int checksum, zio_done_func_t *done, void *private, 809 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 810 { 811 zio_t *zio; 812 813 ASSERT(vd->vdev_children == 0); 814 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 815 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 816 ASSERT3U(offset + size, <=, vd->vdev_psize); 817 818 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 819 ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd, offset, 820 NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 821 822 zio->io_prop.zp_checksum = checksum; 823 824 return (zio); 825 } 826 827 zio_t * 828 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 829 void *data, int checksum, zio_done_func_t *done, void *private, 830 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 831 { 832 zio_t *zio; 833 834 ASSERT(vd->vdev_children == 0); 835 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 836 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 837 ASSERT3U(offset + size, <=, vd->vdev_psize); 838 839 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 840 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd, offset, 841 NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 842 843 zio->io_prop.zp_checksum = checksum; 844 845 if (zio_checksum_table[checksum].ci_eck) { 846 /* 847 * zec checksums are necessarily destructive -- they modify 848 * the end of the write buffer to hold the verifier/checksum. 849 * Therefore, we must make a local copy in case the data is 850 * being written to multiple places in parallel. 851 */ 852 void *wbuf = zio_buf_alloc(size); 853 bcopy(data, wbuf, size); 854 zio_push_transform(zio, wbuf, size, size, NULL); 855 } 856 857 return (zio); 858 } 859 860 /* 861 * Create a child I/O to do some work for us. 862 */ 863 zio_t * 864 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 865 void *data, uint64_t size, int type, zio_priority_t priority, 866 enum zio_flag flags, zio_done_func_t *done, void *private) 867 { 868 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 869 zio_t *zio; 870 871 ASSERT(vd->vdev_parent == 872 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev)); 873 874 if (type == ZIO_TYPE_READ && bp != NULL) { 875 /* 876 * If we have the bp, then the child should perform the 877 * checksum and the parent need not. This pushes error 878 * detection as close to the leaves as possible and 879 * eliminates redundant checksums in the interior nodes. 880 */ 881 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 882 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 883 } 884 885 if (vd->vdev_children == 0) 886 offset += VDEV_LABEL_START_SIZE; 887 888 flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE; 889 890 /* 891 * If we've decided to do a repair, the write is not speculative -- 892 * even if the original read was. 893 */ 894 if (flags & ZIO_FLAG_IO_REPAIR) 895 flags &= ~ZIO_FLAG_SPECULATIVE; 896 897 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, 898 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 899 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 900 901 zio->io_physdone = pio->io_physdone; 902 if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL) 903 zio->io_logical->io_phys_children++; 904 905 return (zio); 906 } 907 908 zio_t * 909 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size, 910 int type, zio_priority_t priority, enum zio_flag flags, 911 zio_done_func_t *done, void *private) 912 { 913 zio_t *zio; 914 915 ASSERT(vd->vdev_ops->vdev_op_leaf); 916 917 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 918 data, size, done, private, type, priority, 919 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED, 920 vd, offset, NULL, 921 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 922 923 return (zio); 924 } 925 926 void 927 zio_flush(zio_t *zio, vdev_t *vd) 928 { 929 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 930 NULL, NULL, 931 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 932 } 933 934 void 935 zio_shrink(zio_t *zio, uint64_t size) 936 { 937 ASSERT(zio->io_executor == NULL); 938 ASSERT(zio->io_orig_size == zio->io_size); 939 ASSERT(size <= zio->io_size); 940 941 /* 942 * We don't shrink for raidz because of problems with the 943 * reconstruction when reading back less than the block size. 944 * Note, BP_IS_RAIDZ() assumes no compression. 945 */ 946 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 947 if (!BP_IS_RAIDZ(zio->io_bp)) 948 zio->io_orig_size = zio->io_size = size; 949 } 950 951 /* 952 * ========================================================================== 953 * Prepare to read and write logical blocks 954 * ========================================================================== 955 */ 956 957 static int 958 zio_read_bp_init(zio_t *zio) 959 { 960 blkptr_t *bp = zio->io_bp; 961 962 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 963 zio->io_child_type == ZIO_CHILD_LOGICAL && 964 !(zio->io_flags & ZIO_FLAG_RAW)) { 965 uint64_t psize = 966 BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp); 967 void *cbuf = zio_buf_alloc(psize); 968 969 zio_push_transform(zio, cbuf, psize, psize, zio_decompress); 970 } 971 972 if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) { 973 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 974 decode_embedded_bp_compressed(bp, zio->io_data); 975 } else { 976 ASSERT(!BP_IS_EMBEDDED(bp)); 977 } 978 979 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0) 980 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 981 982 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 983 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 984 985 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 986 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 987 988 return (ZIO_PIPELINE_CONTINUE); 989 } 990 991 static int 992 zio_write_bp_init(zio_t *zio) 993 { 994 spa_t *spa = zio->io_spa; 995 zio_prop_t *zp = &zio->io_prop; 996 enum zio_compress compress = zp->zp_compress; 997 blkptr_t *bp = zio->io_bp; 998 uint64_t lsize = zio->io_size; 999 uint64_t psize = lsize; 1000 int pass = 1; 1001 1002 /* 1003 * If our children haven't all reached the ready stage, 1004 * wait for them and then repeat this pipeline stage. 1005 */ 1006 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 1007 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) 1008 return (ZIO_PIPELINE_STOP); 1009 1010 if (!IO_IS_ALLOCATING(zio)) 1011 return (ZIO_PIPELINE_CONTINUE); 1012 1013 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1014 1015 if (zio->io_bp_override) { 1016 ASSERT(bp->blk_birth != zio->io_txg); 1017 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 1018 1019 *bp = *zio->io_bp_override; 1020 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1021 1022 if (BP_IS_EMBEDDED(bp)) 1023 return (ZIO_PIPELINE_CONTINUE); 1024 1025 /* 1026 * If we've been overridden and nopwrite is set then 1027 * set the flag accordingly to indicate that a nopwrite 1028 * has already occurred. 1029 */ 1030 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 1031 ASSERT(!zp->zp_dedup); 1032 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1033 return (ZIO_PIPELINE_CONTINUE); 1034 } 1035 1036 ASSERT(!zp->zp_nopwrite); 1037 1038 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 1039 return (ZIO_PIPELINE_CONTINUE); 1040 1041 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup || 1042 zp->zp_dedup_verify); 1043 1044 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { 1045 BP_SET_DEDUP(bp, 1); 1046 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1047 return (ZIO_PIPELINE_CONTINUE); 1048 } 1049 zio->io_bp_override = NULL; 1050 BP_ZERO(bp); 1051 } 1052 1053 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) { 1054 /* 1055 * We're rewriting an existing block, which means we're 1056 * working on behalf of spa_sync(). For spa_sync() to 1057 * converge, it must eventually be the case that we don't 1058 * have to allocate new blocks. But compression changes 1059 * the blocksize, which forces a reallocate, and makes 1060 * convergence take longer. Therefore, after the first 1061 * few passes, stop compressing to ensure convergence. 1062 */ 1063 pass = spa_sync_pass(spa); 1064 1065 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1066 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1067 ASSERT(!BP_GET_DEDUP(bp)); 1068 1069 if (pass >= zfs_sync_pass_dont_compress) 1070 compress = ZIO_COMPRESS_OFF; 1071 1072 /* Make sure someone doesn't change their mind on overwrites */ 1073 ASSERT(BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp), 1074 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 1075 } 1076 1077 if (compress != ZIO_COMPRESS_OFF) { 1078 void *cbuf = zio_buf_alloc(lsize); 1079 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize); 1080 if (psize == 0 || psize == lsize) { 1081 compress = ZIO_COMPRESS_OFF; 1082 zio_buf_free(cbuf, lsize); 1083 } else if (!zp->zp_dedup && psize <= BPE_PAYLOAD_SIZE && 1084 zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && 1085 spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { 1086 encode_embedded_bp_compressed(bp, 1087 cbuf, compress, lsize, psize); 1088 BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); 1089 BP_SET_TYPE(bp, zio->io_prop.zp_type); 1090 BP_SET_LEVEL(bp, zio->io_prop.zp_level); 1091 zio_buf_free(cbuf, lsize); 1092 bp->blk_birth = zio->io_txg; 1093 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1094 ASSERT(spa_feature_is_active(spa, 1095 SPA_FEATURE_EMBEDDED_DATA)); 1096 return (ZIO_PIPELINE_CONTINUE); 1097 } else { 1098 /* 1099 * Round up compressed size to MINBLOCKSIZE and 1100 * zero the tail. 1101 */ 1102 size_t rounded = 1103 P2ROUNDUP(psize, (size_t)SPA_MINBLOCKSIZE); 1104 if (rounded > psize) { 1105 bzero((char *)cbuf + psize, rounded - psize); 1106 psize = rounded; 1107 } 1108 if (psize == lsize) { 1109 compress = ZIO_COMPRESS_OFF; 1110 zio_buf_free(cbuf, lsize); 1111 } else { 1112 zio_push_transform(zio, cbuf, 1113 psize, lsize, NULL); 1114 } 1115 } 1116 } 1117 1118 /* 1119 * The final pass of spa_sync() must be all rewrites, but the first 1120 * few passes offer a trade-off: allocating blocks defers convergence, 1121 * but newly allocated blocks are sequential, so they can be written 1122 * to disk faster. Therefore, we allow the first few passes of 1123 * spa_sync() to allocate new blocks, but force rewrites after that. 1124 * There should only be a handful of blocks after pass 1 in any case. 1125 */ 1126 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg && 1127 BP_GET_PSIZE(bp) == psize && 1128 pass >= zfs_sync_pass_rewrite) { 1129 ASSERT(psize != 0); 1130 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1131 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1132 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1133 } else { 1134 BP_ZERO(bp); 1135 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1136 } 1137 1138 if (psize == 0) { 1139 if (zio->io_bp_orig.blk_birth != 0 && 1140 spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) { 1141 BP_SET_LSIZE(bp, lsize); 1142 BP_SET_TYPE(bp, zp->zp_type); 1143 BP_SET_LEVEL(bp, zp->zp_level); 1144 BP_SET_BIRTH(bp, zio->io_txg, 0); 1145 } 1146 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1147 } else { 1148 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1149 BP_SET_LSIZE(bp, lsize); 1150 BP_SET_TYPE(bp, zp->zp_type); 1151 BP_SET_LEVEL(bp, zp->zp_level); 1152 BP_SET_PSIZE(bp, psize); 1153 BP_SET_COMPRESS(bp, compress); 1154 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1155 BP_SET_DEDUP(bp, zp->zp_dedup); 1156 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1157 if (zp->zp_dedup) { 1158 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1159 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1160 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1161 } 1162 if (zp->zp_nopwrite) { 1163 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1164 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1165 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1166 } 1167 } 1168 1169 return (ZIO_PIPELINE_CONTINUE); 1170 } 1171 1172 static int 1173 zio_free_bp_init(zio_t *zio) 1174 { 1175 blkptr_t *bp = zio->io_bp; 1176 1177 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1178 if (BP_GET_DEDUP(bp)) 1179 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1180 } 1181 1182 return (ZIO_PIPELINE_CONTINUE); 1183 } 1184 1185 /* 1186 * ========================================================================== 1187 * Execute the I/O pipeline 1188 * ========================================================================== 1189 */ 1190 1191 static void 1192 zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline) 1193 { 1194 spa_t *spa = zio->io_spa; 1195 zio_type_t t = zio->io_type; 1196 int flags = (cutinline ? TQ_FRONT : 0); 1197 1198 /* 1199 * If we're a config writer or a probe, the normal issue and 1200 * interrupt threads may all be blocked waiting for the config lock. 1201 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1202 */ 1203 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1204 t = ZIO_TYPE_NULL; 1205 1206 /* 1207 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1208 */ 1209 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1210 t = ZIO_TYPE_NULL; 1211 1212 /* 1213 * If this is a high priority I/O, then use the high priority taskq if 1214 * available. 1215 */ 1216 if (zio->io_priority == ZIO_PRIORITY_NOW && 1217 spa->spa_zio_taskq[t][q + 1].stqs_count != 0) 1218 q++; 1219 1220 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1221 1222 /* 1223 * NB: We are assuming that the zio can only be dispatched 1224 * to a single taskq at a time. It would be a grievous error 1225 * to dispatch the zio to another taskq at the same time. 1226 */ 1227 ASSERT(zio->io_tqent.tqent_next == NULL); 1228 spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio, 1229 flags, &zio->io_tqent); 1230 } 1231 1232 static boolean_t 1233 zio_taskq_member(zio_t *zio, zio_taskq_type_t q) 1234 { 1235 kthread_t *executor = zio->io_executor; 1236 spa_t *spa = zio->io_spa; 1237 1238 for (zio_type_t t = 0; t < ZIO_TYPES; t++) { 1239 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1240 uint_t i; 1241 for (i = 0; i < tqs->stqs_count; i++) { 1242 if (taskq_member(tqs->stqs_taskq[i], executor)) 1243 return (B_TRUE); 1244 } 1245 } 1246 1247 return (B_FALSE); 1248 } 1249 1250 static int 1251 zio_issue_async(zio_t *zio) 1252 { 1253 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1254 1255 return (ZIO_PIPELINE_STOP); 1256 } 1257 1258 void 1259 zio_interrupt(zio_t *zio) 1260 { 1261 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1262 } 1263 1264 /* 1265 * Execute the I/O pipeline until one of the following occurs: 1266 * 1267 * (1) the I/O completes 1268 * (2) the pipeline stalls waiting for dependent child I/Os 1269 * (3) the I/O issues, so we're waiting for an I/O completion interrupt 1270 * (4) the I/O is delegated by vdev-level caching or aggregation 1271 * (5) the I/O is deferred due to vdev-level queueing 1272 * (6) the I/O is handed off to another thread. 1273 * 1274 * In all cases, the pipeline stops whenever there's no CPU work; it never 1275 * burns a thread in cv_wait(). 1276 * 1277 * There's no locking on io_stage because there's no legitimate way 1278 * for multiple threads to be attempting to process the same I/O. 1279 */ 1280 static zio_pipe_stage_t *zio_pipeline[]; 1281 1282 void 1283 zio_execute(zio_t *zio) 1284 { 1285 zio->io_executor = curthread; 1286 1287 while (zio->io_stage < ZIO_STAGE_DONE) { 1288 enum zio_stage pipeline = zio->io_pipeline; 1289 enum zio_stage stage = zio->io_stage; 1290 int rv; 1291 1292 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1293 ASSERT(ISP2(stage)); 1294 ASSERT(zio->io_stall == NULL); 1295 1296 do { 1297 stage <<= 1; 1298 } while ((stage & pipeline) == 0); 1299 1300 ASSERT(stage <= ZIO_STAGE_DONE); 1301 1302 /* 1303 * If we are in interrupt context and this pipeline stage 1304 * will grab a config lock that is held across I/O, 1305 * or may wait for an I/O that needs an interrupt thread 1306 * to complete, issue async to avoid deadlock. 1307 * 1308 * For VDEV_IO_START, we cut in line so that the io will 1309 * be sent to disk promptly. 1310 */ 1311 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1312 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1313 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1314 zio_requeue_io_start_cut_in_line : B_FALSE; 1315 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1316 return; 1317 } 1318 1319 zio->io_stage = stage; 1320 rv = zio_pipeline[highbit64(stage) - 1](zio); 1321 1322 if (rv == ZIO_PIPELINE_STOP) 1323 return; 1324 1325 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1326 } 1327 } 1328 1329 /* 1330 * ========================================================================== 1331 * Initiate I/O, either sync or async 1332 * ========================================================================== 1333 */ 1334 int 1335 zio_wait(zio_t *zio) 1336 { 1337 int error; 1338 1339 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1340 ASSERT(zio->io_executor == NULL); 1341 1342 zio->io_waiter = curthread; 1343 1344 zio_execute(zio); 1345 1346 mutex_enter(&zio->io_lock); 1347 while (zio->io_executor != NULL) 1348 cv_wait(&zio->io_cv, &zio->io_lock); 1349 mutex_exit(&zio->io_lock); 1350 1351 error = zio->io_error; 1352 zio_destroy(zio); 1353 1354 return (error); 1355 } 1356 1357 void 1358 zio_nowait(zio_t *zio) 1359 { 1360 ASSERT(zio->io_executor == NULL); 1361 1362 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1363 zio_unique_parent(zio) == NULL) { 1364 /* 1365 * This is a logical async I/O with no parent to wait for it. 1366 * We add it to the spa_async_root_zio "Godfather" I/O which 1367 * will ensure they complete prior to unloading the pool. 1368 */ 1369 spa_t *spa = zio->io_spa; 1370 1371 zio_add_child(spa->spa_async_zio_root[CPU_SEQID], zio); 1372 } 1373 1374 zio_execute(zio); 1375 } 1376 1377 /* 1378 * ========================================================================== 1379 * Reexecute or suspend/resume failed I/O 1380 * ========================================================================== 1381 */ 1382 1383 static void 1384 zio_reexecute(zio_t *pio) 1385 { 1386 zio_t *cio, *cio_next; 1387 1388 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1389 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1390 ASSERT(pio->io_gang_leader == NULL); 1391 ASSERT(pio->io_gang_tree == NULL); 1392 1393 pio->io_flags = pio->io_orig_flags; 1394 pio->io_stage = pio->io_orig_stage; 1395 pio->io_pipeline = pio->io_orig_pipeline; 1396 pio->io_reexecute = 0; 1397 pio->io_flags |= ZIO_FLAG_REEXECUTED; 1398 pio->io_error = 0; 1399 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1400 pio->io_state[w] = 0; 1401 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1402 pio->io_child_error[c] = 0; 1403 1404 if (IO_IS_ALLOCATING(pio)) 1405 BP_ZERO(pio->io_bp); 1406 1407 /* 1408 * As we reexecute pio's children, new children could be created. 1409 * New children go to the head of pio's io_child_list, however, 1410 * so we will (correctly) not reexecute them. The key is that 1411 * the remainder of pio's io_child_list, from 'cio_next' onward, 1412 * cannot be affected by any side effects of reexecuting 'cio'. 1413 */ 1414 for (cio = zio_walk_children(pio); cio != NULL; cio = cio_next) { 1415 cio_next = zio_walk_children(pio); 1416 mutex_enter(&pio->io_lock); 1417 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1418 pio->io_children[cio->io_child_type][w]++; 1419 mutex_exit(&pio->io_lock); 1420 zio_reexecute(cio); 1421 } 1422 1423 /* 1424 * Now that all children have been reexecuted, execute the parent. 1425 * We don't reexecute "The Godfather" I/O here as it's the 1426 * responsibility of the caller to wait on him. 1427 */ 1428 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) 1429 zio_execute(pio); 1430 } 1431 1432 void 1433 zio_suspend(spa_t *spa, zio_t *zio) 1434 { 1435 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1436 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1437 "failure and the failure mode property for this pool " 1438 "is set to panic.", spa_name(spa)); 1439 1440 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); 1441 1442 mutex_enter(&spa->spa_suspend_lock); 1443 1444 if (spa->spa_suspend_zio_root == NULL) 1445 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1446 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1447 ZIO_FLAG_GODFATHER); 1448 1449 spa->spa_suspended = B_TRUE; 1450 1451 if (zio != NULL) { 1452 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1453 ASSERT(zio != spa->spa_suspend_zio_root); 1454 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1455 ASSERT(zio_unique_parent(zio) == NULL); 1456 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1457 zio_add_child(spa->spa_suspend_zio_root, zio); 1458 } 1459 1460 mutex_exit(&spa->spa_suspend_lock); 1461 } 1462 1463 int 1464 zio_resume(spa_t *spa) 1465 { 1466 zio_t *pio; 1467 1468 /* 1469 * Reexecute all previously suspended i/o. 1470 */ 1471 mutex_enter(&spa->spa_suspend_lock); 1472 spa->spa_suspended = B_FALSE; 1473 cv_broadcast(&spa->spa_suspend_cv); 1474 pio = spa->spa_suspend_zio_root; 1475 spa->spa_suspend_zio_root = NULL; 1476 mutex_exit(&spa->spa_suspend_lock); 1477 1478 if (pio == NULL) 1479 return (0); 1480 1481 zio_reexecute(pio); 1482 return (zio_wait(pio)); 1483 } 1484 1485 void 1486 zio_resume_wait(spa_t *spa) 1487 { 1488 mutex_enter(&spa->spa_suspend_lock); 1489 while (spa_suspended(spa)) 1490 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 1491 mutex_exit(&spa->spa_suspend_lock); 1492 } 1493 1494 /* 1495 * ========================================================================== 1496 * Gang blocks. 1497 * 1498 * A gang block is a collection of small blocks that looks to the DMU 1499 * like one large block. When zio_dva_allocate() cannot find a block 1500 * of the requested size, due to either severe fragmentation or the pool 1501 * being nearly full, it calls zio_write_gang_block() to construct the 1502 * block from smaller fragments. 1503 * 1504 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 1505 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 1506 * an indirect block: it's an array of block pointers. It consumes 1507 * only one sector and hence is allocatable regardless of fragmentation. 1508 * The gang header's bps point to its gang members, which hold the data. 1509 * 1510 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 1511 * as the verifier to ensure uniqueness of the SHA256 checksum. 1512 * Critically, the gang block bp's blk_cksum is the checksum of the data, 1513 * not the gang header. This ensures that data block signatures (needed for 1514 * deduplication) are independent of how the block is physically stored. 1515 * 1516 * Gang blocks can be nested: a gang member may itself be a gang block. 1517 * Thus every gang block is a tree in which root and all interior nodes are 1518 * gang headers, and the leaves are normal blocks that contain user data. 1519 * The root of the gang tree is called the gang leader. 1520 * 1521 * To perform any operation (read, rewrite, free, claim) on a gang block, 1522 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 1523 * in the io_gang_tree field of the original logical i/o by recursively 1524 * reading the gang leader and all gang headers below it. This yields 1525 * an in-core tree containing the contents of every gang header and the 1526 * bps for every constituent of the gang block. 1527 * 1528 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 1529 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 1530 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 1531 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 1532 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 1533 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 1534 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 1535 * of the gang header plus zio_checksum_compute() of the data to update the 1536 * gang header's blk_cksum as described above. 1537 * 1538 * The two-phase assemble/issue model solves the problem of partial failure -- 1539 * what if you'd freed part of a gang block but then couldn't read the 1540 * gang header for another part? Assembling the entire gang tree first 1541 * ensures that all the necessary gang header I/O has succeeded before 1542 * starting the actual work of free, claim, or write. Once the gang tree 1543 * is assembled, free and claim are in-memory operations that cannot fail. 1544 * 1545 * In the event that a gang write fails, zio_dva_unallocate() walks the 1546 * gang tree to immediately free (i.e. insert back into the space map) 1547 * everything we've allocated. This ensures that we don't get ENOSPC 1548 * errors during repeated suspend/resume cycles due to a flaky device. 1549 * 1550 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 1551 * the gang tree, we won't modify the block, so we can safely defer the free 1552 * (knowing that the block is still intact). If we *can* assemble the gang 1553 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 1554 * each constituent bp and we can allocate a new block on the next sync pass. 1555 * 1556 * In all cases, the gang tree allows complete recovery from partial failure. 1557 * ========================================================================== 1558 */ 1559 1560 static zio_t * 1561 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1562 { 1563 if (gn != NULL) 1564 return (pio); 1565 1566 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp), 1567 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1568 &pio->io_bookmark)); 1569 } 1570 1571 zio_t * 1572 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1573 { 1574 zio_t *zio; 1575 1576 if (gn != NULL) { 1577 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1578 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority, 1579 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1580 /* 1581 * As we rewrite each gang header, the pipeline will compute 1582 * a new gang block header checksum for it; but no one will 1583 * compute a new data checksum, so we do that here. The one 1584 * exception is the gang leader: the pipeline already computed 1585 * its data checksum because that stage precedes gang assembly. 1586 * (Presently, nothing actually uses interior data checksums; 1587 * this is just good hygiene.) 1588 */ 1589 if (gn != pio->io_gang_leader->io_gang_tree) { 1590 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 1591 data, BP_GET_PSIZE(bp)); 1592 } 1593 /* 1594 * If we are here to damage data for testing purposes, 1595 * leave the GBH alone so that we can detect the damage. 1596 */ 1597 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 1598 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 1599 } else { 1600 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1601 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority, 1602 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1603 } 1604 1605 return (zio); 1606 } 1607 1608 /* ARGSUSED */ 1609 zio_t * 1610 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1611 { 1612 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 1613 ZIO_GANG_CHILD_FLAGS(pio))); 1614 } 1615 1616 /* ARGSUSED */ 1617 zio_t * 1618 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1619 { 1620 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 1621 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 1622 } 1623 1624 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 1625 NULL, 1626 zio_read_gang, 1627 zio_rewrite_gang, 1628 zio_free_gang, 1629 zio_claim_gang, 1630 NULL 1631 }; 1632 1633 static void zio_gang_tree_assemble_done(zio_t *zio); 1634 1635 static zio_gang_node_t * 1636 zio_gang_node_alloc(zio_gang_node_t **gnpp) 1637 { 1638 zio_gang_node_t *gn; 1639 1640 ASSERT(*gnpp == NULL); 1641 1642 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 1643 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 1644 *gnpp = gn; 1645 1646 return (gn); 1647 } 1648 1649 static void 1650 zio_gang_node_free(zio_gang_node_t **gnpp) 1651 { 1652 zio_gang_node_t *gn = *gnpp; 1653 1654 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1655 ASSERT(gn->gn_child[g] == NULL); 1656 1657 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 1658 kmem_free(gn, sizeof (*gn)); 1659 *gnpp = NULL; 1660 } 1661 1662 static void 1663 zio_gang_tree_free(zio_gang_node_t **gnpp) 1664 { 1665 zio_gang_node_t *gn = *gnpp; 1666 1667 if (gn == NULL) 1668 return; 1669 1670 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1671 zio_gang_tree_free(&gn->gn_child[g]); 1672 1673 zio_gang_node_free(gnpp); 1674 } 1675 1676 static void 1677 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 1678 { 1679 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 1680 1681 ASSERT(gio->io_gang_leader == gio); 1682 ASSERT(BP_IS_GANG(bp)); 1683 1684 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh, 1685 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn, 1686 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 1687 } 1688 1689 static void 1690 zio_gang_tree_assemble_done(zio_t *zio) 1691 { 1692 zio_t *gio = zio->io_gang_leader; 1693 zio_gang_node_t *gn = zio->io_private; 1694 blkptr_t *bp = zio->io_bp; 1695 1696 ASSERT(gio == zio_unique_parent(zio)); 1697 ASSERT(zio->io_child_count == 0); 1698 1699 if (zio->io_error) 1700 return; 1701 1702 if (BP_SHOULD_BYTESWAP(bp)) 1703 byteswap_uint64_array(zio->io_data, zio->io_size); 1704 1705 ASSERT(zio->io_data == gn->gn_gbh); 1706 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1707 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1708 1709 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1710 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1711 if (!BP_IS_GANG(gbp)) 1712 continue; 1713 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 1714 } 1715 } 1716 1717 static void 1718 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data) 1719 { 1720 zio_t *gio = pio->io_gang_leader; 1721 zio_t *zio; 1722 1723 ASSERT(BP_IS_GANG(bp) == !!gn); 1724 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 1725 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 1726 1727 /* 1728 * If you're a gang header, your data is in gn->gn_gbh. 1729 * If you're a gang member, your data is in 'data' and gn == NULL. 1730 */ 1731 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data); 1732 1733 if (gn != NULL) { 1734 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1735 1736 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1737 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1738 if (BP_IS_HOLE(gbp)) 1739 continue; 1740 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data); 1741 data = (char *)data + BP_GET_PSIZE(gbp); 1742 } 1743 } 1744 1745 if (gn == gio->io_gang_tree) 1746 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data); 1747 1748 if (zio != pio) 1749 zio_nowait(zio); 1750 } 1751 1752 static int 1753 zio_gang_assemble(zio_t *zio) 1754 { 1755 blkptr_t *bp = zio->io_bp; 1756 1757 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 1758 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1759 1760 zio->io_gang_leader = zio; 1761 1762 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 1763 1764 return (ZIO_PIPELINE_CONTINUE); 1765 } 1766 1767 static int 1768 zio_gang_issue(zio_t *zio) 1769 { 1770 blkptr_t *bp = zio->io_bp; 1771 1772 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) 1773 return (ZIO_PIPELINE_STOP); 1774 1775 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 1776 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1777 1778 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 1779 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data); 1780 else 1781 zio_gang_tree_free(&zio->io_gang_tree); 1782 1783 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1784 1785 return (ZIO_PIPELINE_CONTINUE); 1786 } 1787 1788 static void 1789 zio_write_gang_member_ready(zio_t *zio) 1790 { 1791 zio_t *pio = zio_unique_parent(zio); 1792 zio_t *gio = zio->io_gang_leader; 1793 dva_t *cdva = zio->io_bp->blk_dva; 1794 dva_t *pdva = pio->io_bp->blk_dva; 1795 uint64_t asize; 1796 1797 if (BP_IS_HOLE(zio->io_bp)) 1798 return; 1799 1800 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 1801 1802 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 1803 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 1804 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 1805 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 1806 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 1807 1808 mutex_enter(&pio->io_lock); 1809 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 1810 ASSERT(DVA_GET_GANG(&pdva[d])); 1811 asize = DVA_GET_ASIZE(&pdva[d]); 1812 asize += DVA_GET_ASIZE(&cdva[d]); 1813 DVA_SET_ASIZE(&pdva[d], asize); 1814 } 1815 mutex_exit(&pio->io_lock); 1816 } 1817 1818 static int 1819 zio_write_gang_block(zio_t *pio) 1820 { 1821 spa_t *spa = pio->io_spa; 1822 blkptr_t *bp = pio->io_bp; 1823 zio_t *gio = pio->io_gang_leader; 1824 zio_t *zio; 1825 zio_gang_node_t *gn, **gnpp; 1826 zio_gbh_phys_t *gbh; 1827 uint64_t txg = pio->io_txg; 1828 uint64_t resid = pio->io_size; 1829 uint64_t lsize; 1830 int copies = gio->io_prop.zp_copies; 1831 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 1832 zio_prop_t zp; 1833 int error; 1834 1835 error = metaslab_alloc(spa, spa_normal_class(spa), SPA_GANGBLOCKSIZE, 1836 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, 1837 METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER); 1838 if (error) { 1839 pio->io_error = error; 1840 return (ZIO_PIPELINE_CONTINUE); 1841 } 1842 1843 if (pio == gio) { 1844 gnpp = &gio->io_gang_tree; 1845 } else { 1846 gnpp = pio->io_private; 1847 ASSERT(pio->io_ready == zio_write_gang_member_ready); 1848 } 1849 1850 gn = zio_gang_node_alloc(gnpp); 1851 gbh = gn->gn_gbh; 1852 bzero(gbh, SPA_GANGBLOCKSIZE); 1853 1854 /* 1855 * Create the gang header. 1856 */ 1857 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL, 1858 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1859 1860 /* 1861 * Create and nowait the gang children. 1862 */ 1863 for (int g = 0; resid != 0; resid -= lsize, g++) { 1864 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 1865 SPA_MINBLOCKSIZE); 1866 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 1867 1868 zp.zp_checksum = gio->io_prop.zp_checksum; 1869 zp.zp_compress = ZIO_COMPRESS_OFF; 1870 zp.zp_type = DMU_OT_NONE; 1871 zp.zp_level = 0; 1872 zp.zp_copies = gio->io_prop.zp_copies; 1873 zp.zp_dedup = B_FALSE; 1874 zp.zp_dedup_verify = B_FALSE; 1875 zp.zp_nopwrite = B_FALSE; 1876 1877 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 1878 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, 1879 zio_write_gang_member_ready, NULL, NULL, &gn->gn_child[g], 1880 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1881 &pio->io_bookmark)); 1882 } 1883 1884 /* 1885 * Set pio's pipeline to just wait for zio to finish. 1886 */ 1887 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1888 1889 zio_nowait(zio); 1890 1891 return (ZIO_PIPELINE_CONTINUE); 1892 } 1893 1894 /* 1895 * The zio_nop_write stage in the pipeline determines if allocating 1896 * a new bp is necessary. By leveraging a cryptographically secure checksum, 1897 * such as SHA256, we can compare the checksums of the new data and the old 1898 * to determine if allocating a new block is required. The nopwrite 1899 * feature can handle writes in either syncing or open context (i.e. zil 1900 * writes) and as a result is mutually exclusive with dedup. 1901 */ 1902 static int 1903 zio_nop_write(zio_t *zio) 1904 { 1905 blkptr_t *bp = zio->io_bp; 1906 blkptr_t *bp_orig = &zio->io_bp_orig; 1907 zio_prop_t *zp = &zio->io_prop; 1908 1909 ASSERT(BP_GET_LEVEL(bp) == 0); 1910 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1911 ASSERT(zp->zp_nopwrite); 1912 ASSERT(!zp->zp_dedup); 1913 ASSERT(zio->io_bp_override == NULL); 1914 ASSERT(IO_IS_ALLOCATING(zio)); 1915 1916 /* 1917 * Check to see if the original bp and the new bp have matching 1918 * characteristics (i.e. same checksum, compression algorithms, etc). 1919 * If they don't then just continue with the pipeline which will 1920 * allocate a new bp. 1921 */ 1922 if (BP_IS_HOLE(bp_orig) || 1923 !zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_dedup || 1924 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 1925 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 1926 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 1927 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 1928 return (ZIO_PIPELINE_CONTINUE); 1929 1930 /* 1931 * If the checksums match then reset the pipeline so that we 1932 * avoid allocating a new bp and issuing any I/O. 1933 */ 1934 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 1935 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup); 1936 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 1937 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 1938 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 1939 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop, 1940 sizeof (uint64_t)) == 0); 1941 1942 *bp = *bp_orig; 1943 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1944 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1945 } 1946 1947 return (ZIO_PIPELINE_CONTINUE); 1948 } 1949 1950 /* 1951 * ========================================================================== 1952 * Dedup 1953 * ========================================================================== 1954 */ 1955 static void 1956 zio_ddt_child_read_done(zio_t *zio) 1957 { 1958 blkptr_t *bp = zio->io_bp; 1959 ddt_entry_t *dde = zio->io_private; 1960 ddt_phys_t *ddp; 1961 zio_t *pio = zio_unique_parent(zio); 1962 1963 mutex_enter(&pio->io_lock); 1964 ddp = ddt_phys_select(dde, bp); 1965 if (zio->io_error == 0) 1966 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 1967 if (zio->io_error == 0 && dde->dde_repair_data == NULL) 1968 dde->dde_repair_data = zio->io_data; 1969 else 1970 zio_buf_free(zio->io_data, zio->io_size); 1971 mutex_exit(&pio->io_lock); 1972 } 1973 1974 static int 1975 zio_ddt_read_start(zio_t *zio) 1976 { 1977 blkptr_t *bp = zio->io_bp; 1978 1979 ASSERT(BP_GET_DEDUP(bp)); 1980 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 1981 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1982 1983 if (zio->io_child_error[ZIO_CHILD_DDT]) { 1984 ddt_t *ddt = ddt_select(zio->io_spa, bp); 1985 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 1986 ddt_phys_t *ddp = dde->dde_phys; 1987 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 1988 blkptr_t blk; 1989 1990 ASSERT(zio->io_vsd == NULL); 1991 zio->io_vsd = dde; 1992 1993 if (ddp_self == NULL) 1994 return (ZIO_PIPELINE_CONTINUE); 1995 1996 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1997 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 1998 continue; 1999 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 2000 &blk); 2001 zio_nowait(zio_read(zio, zio->io_spa, &blk, 2002 zio_buf_alloc(zio->io_size), zio->io_size, 2003 zio_ddt_child_read_done, dde, zio->io_priority, 2004 ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE, 2005 &zio->io_bookmark)); 2006 } 2007 return (ZIO_PIPELINE_CONTINUE); 2008 } 2009 2010 zio_nowait(zio_read(zio, zio->io_spa, bp, 2011 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority, 2012 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 2013 2014 return (ZIO_PIPELINE_CONTINUE); 2015 } 2016 2017 static int 2018 zio_ddt_read_done(zio_t *zio) 2019 { 2020 blkptr_t *bp = zio->io_bp; 2021 2022 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) 2023 return (ZIO_PIPELINE_STOP); 2024 2025 ASSERT(BP_GET_DEDUP(bp)); 2026 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2027 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2028 2029 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2030 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2031 ddt_entry_t *dde = zio->io_vsd; 2032 if (ddt == NULL) { 2033 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 2034 return (ZIO_PIPELINE_CONTINUE); 2035 } 2036 if (dde == NULL) { 2037 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 2038 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 2039 return (ZIO_PIPELINE_STOP); 2040 } 2041 if (dde->dde_repair_data != NULL) { 2042 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size); 2043 zio->io_child_error[ZIO_CHILD_DDT] = 0; 2044 } 2045 ddt_repair_done(ddt, dde); 2046 zio->io_vsd = NULL; 2047 } 2048 2049 ASSERT(zio->io_vsd == NULL); 2050 2051 return (ZIO_PIPELINE_CONTINUE); 2052 } 2053 2054 static boolean_t 2055 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 2056 { 2057 spa_t *spa = zio->io_spa; 2058 2059 /* 2060 * Note: we compare the original data, not the transformed data, 2061 * because when zio->io_bp is an override bp, we will not have 2062 * pushed the I/O transforms. That's an important optimization 2063 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 2064 */ 2065 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2066 zio_t *lio = dde->dde_lead_zio[p]; 2067 2068 if (lio != NULL) { 2069 return (lio->io_orig_size != zio->io_orig_size || 2070 bcmp(zio->io_orig_data, lio->io_orig_data, 2071 zio->io_orig_size) != 0); 2072 } 2073 } 2074 2075 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2076 ddt_phys_t *ddp = &dde->dde_phys[p]; 2077 2078 if (ddp->ddp_phys_birth != 0) { 2079 arc_buf_t *abuf = NULL; 2080 uint32_t aflags = ARC_WAIT; 2081 blkptr_t blk = *zio->io_bp; 2082 int error; 2083 2084 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 2085 2086 ddt_exit(ddt); 2087 2088 error = arc_read(NULL, spa, &blk, 2089 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 2090 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2091 &aflags, &zio->io_bookmark); 2092 2093 if (error == 0) { 2094 if (arc_buf_size(abuf) != zio->io_orig_size || 2095 bcmp(abuf->b_data, zio->io_orig_data, 2096 zio->io_orig_size) != 0) 2097 error = SET_ERROR(EEXIST); 2098 VERIFY(arc_buf_remove_ref(abuf, &abuf)); 2099 } 2100 2101 ddt_enter(ddt); 2102 return (error != 0); 2103 } 2104 } 2105 2106 return (B_FALSE); 2107 } 2108 2109 static void 2110 zio_ddt_child_write_ready(zio_t *zio) 2111 { 2112 int p = zio->io_prop.zp_copies; 2113 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2114 ddt_entry_t *dde = zio->io_private; 2115 ddt_phys_t *ddp = &dde->dde_phys[p]; 2116 zio_t *pio; 2117 2118 if (zio->io_error) 2119 return; 2120 2121 ddt_enter(ddt); 2122 2123 ASSERT(dde->dde_lead_zio[p] == zio); 2124 2125 ddt_phys_fill(ddp, zio->io_bp); 2126 2127 while ((pio = zio_walk_parents(zio)) != NULL) 2128 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 2129 2130 ddt_exit(ddt); 2131 } 2132 2133 static void 2134 zio_ddt_child_write_done(zio_t *zio) 2135 { 2136 int p = zio->io_prop.zp_copies; 2137 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2138 ddt_entry_t *dde = zio->io_private; 2139 ddt_phys_t *ddp = &dde->dde_phys[p]; 2140 2141 ddt_enter(ddt); 2142 2143 ASSERT(ddp->ddp_refcnt == 0); 2144 ASSERT(dde->dde_lead_zio[p] == zio); 2145 dde->dde_lead_zio[p] = NULL; 2146 2147 if (zio->io_error == 0) { 2148 while (zio_walk_parents(zio) != NULL) 2149 ddt_phys_addref(ddp); 2150 } else { 2151 ddt_phys_clear(ddp); 2152 } 2153 2154 ddt_exit(ddt); 2155 } 2156 2157 static void 2158 zio_ddt_ditto_write_done(zio_t *zio) 2159 { 2160 int p = DDT_PHYS_DITTO; 2161 zio_prop_t *zp = &zio->io_prop; 2162 blkptr_t *bp = zio->io_bp; 2163 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2164 ddt_entry_t *dde = zio->io_private; 2165 ddt_phys_t *ddp = &dde->dde_phys[p]; 2166 ddt_key_t *ddk = &dde->dde_key; 2167 2168 ddt_enter(ddt); 2169 2170 ASSERT(ddp->ddp_refcnt == 0); 2171 ASSERT(dde->dde_lead_zio[p] == zio); 2172 dde->dde_lead_zio[p] = NULL; 2173 2174 if (zio->io_error == 0) { 2175 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 2176 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 2177 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 2178 if (ddp->ddp_phys_birth != 0) 2179 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 2180 ddt_phys_fill(ddp, bp); 2181 } 2182 2183 ddt_exit(ddt); 2184 } 2185 2186 static int 2187 zio_ddt_write(zio_t *zio) 2188 { 2189 spa_t *spa = zio->io_spa; 2190 blkptr_t *bp = zio->io_bp; 2191 uint64_t txg = zio->io_txg; 2192 zio_prop_t *zp = &zio->io_prop; 2193 int p = zp->zp_copies; 2194 int ditto_copies; 2195 zio_t *cio = NULL; 2196 zio_t *dio = NULL; 2197 ddt_t *ddt = ddt_select(spa, bp); 2198 ddt_entry_t *dde; 2199 ddt_phys_t *ddp; 2200 2201 ASSERT(BP_GET_DEDUP(bp)); 2202 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 2203 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 2204 2205 ddt_enter(ddt); 2206 dde = ddt_lookup(ddt, bp, B_TRUE); 2207 ddp = &dde->dde_phys[p]; 2208 2209 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2210 /* 2211 * If we're using a weak checksum, upgrade to a strong checksum 2212 * and try again. If we're already using a strong checksum, 2213 * we can't resolve it, so just convert to an ordinary write. 2214 * (And automatically e-mail a paper to Nature?) 2215 */ 2216 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) { 2217 zp->zp_checksum = spa_dedup_checksum(spa); 2218 zio_pop_transforms(zio); 2219 zio->io_stage = ZIO_STAGE_OPEN; 2220 BP_ZERO(bp); 2221 } else { 2222 zp->zp_dedup = B_FALSE; 2223 } 2224 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2225 ddt_exit(ddt); 2226 return (ZIO_PIPELINE_CONTINUE); 2227 } 2228 2229 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2230 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2231 2232 if (ditto_copies > ddt_ditto_copies_present(dde) && 2233 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2234 zio_prop_t czp = *zp; 2235 2236 czp.zp_copies = ditto_copies; 2237 2238 /* 2239 * If we arrived here with an override bp, we won't have run 2240 * the transform stack, so we won't have the data we need to 2241 * generate a child i/o. So, toss the override bp and restart. 2242 * This is safe, because using the override bp is just an 2243 * optimization; and it's rare, so the cost doesn't matter. 2244 */ 2245 if (zio->io_bp_override) { 2246 zio_pop_transforms(zio); 2247 zio->io_stage = ZIO_STAGE_OPEN; 2248 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2249 zio->io_bp_override = NULL; 2250 BP_ZERO(bp); 2251 ddt_exit(ddt); 2252 return (ZIO_PIPELINE_CONTINUE); 2253 } 2254 2255 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2256 zio->io_orig_size, &czp, NULL, NULL, 2257 zio_ddt_ditto_write_done, dde, zio->io_priority, 2258 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2259 2260 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); 2261 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2262 } 2263 2264 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2265 if (ddp->ddp_phys_birth != 0) 2266 ddt_bp_fill(ddp, bp, txg); 2267 if (dde->dde_lead_zio[p] != NULL) 2268 zio_add_child(zio, dde->dde_lead_zio[p]); 2269 else 2270 ddt_phys_addref(ddp); 2271 } else if (zio->io_bp_override) { 2272 ASSERT(bp->blk_birth == txg); 2273 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 2274 ddt_phys_fill(ddp, bp); 2275 ddt_phys_addref(ddp); 2276 } else { 2277 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2278 zio->io_orig_size, zp, zio_ddt_child_write_ready, NULL, 2279 zio_ddt_child_write_done, dde, zio->io_priority, 2280 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2281 2282 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL); 2283 dde->dde_lead_zio[p] = cio; 2284 } 2285 2286 ddt_exit(ddt); 2287 2288 if (cio) 2289 zio_nowait(cio); 2290 if (dio) 2291 zio_nowait(dio); 2292 2293 return (ZIO_PIPELINE_CONTINUE); 2294 } 2295 2296 ddt_entry_t *freedde; /* for debugging */ 2297 2298 static int 2299 zio_ddt_free(zio_t *zio) 2300 { 2301 spa_t *spa = zio->io_spa; 2302 blkptr_t *bp = zio->io_bp; 2303 ddt_t *ddt = ddt_select(spa, bp); 2304 ddt_entry_t *dde; 2305 ddt_phys_t *ddp; 2306 2307 ASSERT(BP_GET_DEDUP(bp)); 2308 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2309 2310 ddt_enter(ddt); 2311 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 2312 ddp = ddt_phys_select(dde, bp); 2313 ddt_phys_decref(ddp); 2314 ddt_exit(ddt); 2315 2316 return (ZIO_PIPELINE_CONTINUE); 2317 } 2318 2319 /* 2320 * ========================================================================== 2321 * Allocate and free blocks 2322 * ========================================================================== 2323 */ 2324 static int 2325 zio_dva_allocate(zio_t *zio) 2326 { 2327 spa_t *spa = zio->io_spa; 2328 metaslab_class_t *mc = spa_normal_class(spa); 2329 blkptr_t *bp = zio->io_bp; 2330 int error; 2331 int flags = 0; 2332 2333 if (zio->io_gang_leader == NULL) { 2334 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2335 zio->io_gang_leader = zio; 2336 } 2337 2338 ASSERT(BP_IS_HOLE(bp)); 2339 ASSERT0(BP_GET_NDVAS(bp)); 2340 ASSERT3U(zio->io_prop.zp_copies, >, 0); 2341 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 2342 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 2343 2344 /* 2345 * The dump device does not support gang blocks so allocation on 2346 * behalf of the dump device (i.e. ZIO_FLAG_NODATA) must avoid 2347 * the "fast" gang feature. 2348 */ 2349 flags |= (zio->io_flags & ZIO_FLAG_NODATA) ? METASLAB_GANG_AVOID : 0; 2350 flags |= (zio->io_flags & ZIO_FLAG_GANG_CHILD) ? 2351 METASLAB_GANG_CHILD : 0; 2352 error = metaslab_alloc(spa, mc, zio->io_size, bp, 2353 zio->io_prop.zp_copies, zio->io_txg, NULL, flags); 2354 2355 if (error) { 2356 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, " 2357 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 2358 error); 2359 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 2360 return (zio_write_gang_block(zio)); 2361 zio->io_error = error; 2362 } 2363 2364 return (ZIO_PIPELINE_CONTINUE); 2365 } 2366 2367 static int 2368 zio_dva_free(zio_t *zio) 2369 { 2370 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 2371 2372 return (ZIO_PIPELINE_CONTINUE); 2373 } 2374 2375 static int 2376 zio_dva_claim(zio_t *zio) 2377 { 2378 int error; 2379 2380 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 2381 if (error) 2382 zio->io_error = error; 2383 2384 return (ZIO_PIPELINE_CONTINUE); 2385 } 2386 2387 /* 2388 * Undo an allocation. This is used by zio_done() when an I/O fails 2389 * and we want to give back the block we just allocated. 2390 * This handles both normal blocks and gang blocks. 2391 */ 2392 static void 2393 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 2394 { 2395 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2396 ASSERT(zio->io_bp_override == NULL); 2397 2398 if (!BP_IS_HOLE(bp)) 2399 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 2400 2401 if (gn != NULL) { 2402 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2403 zio_dva_unallocate(zio, gn->gn_child[g], 2404 &gn->gn_gbh->zg_blkptr[g]); 2405 } 2406 } 2407 } 2408 2409 /* 2410 * Try to allocate an intent log block. Return 0 on success, errno on failure. 2411 */ 2412 int 2413 zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp, 2414 uint64_t size, boolean_t use_slog) 2415 { 2416 int error = 1; 2417 2418 ASSERT(txg > spa_syncing_txg(spa)); 2419 2420 /* 2421 * ZIL blocks are always contiguous (i.e. not gang blocks) so we 2422 * set the METASLAB_GANG_AVOID flag so that they don't "fast gang" 2423 * when allocating them. 2424 */ 2425 if (use_slog) { 2426 error = metaslab_alloc(spa, spa_log_class(spa), size, 2427 new_bp, 1, txg, old_bp, 2428 METASLAB_HINTBP_AVOID | METASLAB_GANG_AVOID); 2429 } 2430 2431 if (error) { 2432 error = metaslab_alloc(spa, spa_normal_class(spa), size, 2433 new_bp, 1, txg, old_bp, 2434 METASLAB_HINTBP_AVOID); 2435 } 2436 2437 if (error == 0) { 2438 BP_SET_LSIZE(new_bp, size); 2439 BP_SET_PSIZE(new_bp, size); 2440 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 2441 BP_SET_CHECKSUM(new_bp, 2442 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 2443 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 2444 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 2445 BP_SET_LEVEL(new_bp, 0); 2446 BP_SET_DEDUP(new_bp, 0); 2447 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 2448 } 2449 2450 return (error); 2451 } 2452 2453 /* 2454 * Free an intent log block. 2455 */ 2456 void 2457 zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) 2458 { 2459 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG); 2460 ASSERT(!BP_IS_GANG(bp)); 2461 2462 zio_free(spa, txg, bp); 2463 } 2464 2465 /* 2466 * ========================================================================== 2467 * Read and write to physical devices 2468 * ========================================================================== 2469 */ 2470 static int 2471 zio_vdev_io_start(zio_t *zio) 2472 { 2473 vdev_t *vd = zio->io_vd; 2474 uint64_t align; 2475 spa_t *spa = zio->io_spa; 2476 2477 ASSERT(zio->io_error == 0); 2478 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 2479 2480 if (vd == NULL) { 2481 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2482 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 2483 2484 /* 2485 * The mirror_ops handle multiple DVAs in a single BP. 2486 */ 2487 return (vdev_mirror_ops.vdev_op_io_start(zio)); 2488 } 2489 2490 /* 2491 * We keep track of time-sensitive I/Os so that the scan thread 2492 * can quickly react to certain workloads. In particular, we care 2493 * about non-scrubbing, top-level reads and writes with the following 2494 * characteristics: 2495 * - synchronous writes of user data to non-slog devices 2496 * - any reads of user data 2497 * When these conditions are met, adjust the timestamp of spa_last_io 2498 * which allows the scan thread to adjust its workload accordingly. 2499 */ 2500 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL && 2501 vd == vd->vdev_top && !vd->vdev_islog && 2502 zio->io_bookmark.zb_objset != DMU_META_OBJSET && 2503 zio->io_txg != spa_syncing_txg(spa)) { 2504 uint64_t old = spa->spa_last_io; 2505 uint64_t new = ddi_get_lbolt64(); 2506 if (old != new) 2507 (void) atomic_cas_64(&spa->spa_last_io, old, new); 2508 } 2509 2510 align = 1ULL << vd->vdev_top->vdev_ashift; 2511 2512 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && 2513 P2PHASE(zio->io_size, align) != 0) { 2514 /* Transform logical writes to be a full physical block size. */ 2515 uint64_t asize = P2ROUNDUP(zio->io_size, align); 2516 char *abuf = zio_buf_alloc(asize); 2517 ASSERT(vd == vd->vdev_top); 2518 if (zio->io_type == ZIO_TYPE_WRITE) { 2519 bcopy(zio->io_data, abuf, zio->io_size); 2520 bzero(abuf + zio->io_size, asize - zio->io_size); 2521 } 2522 zio_push_transform(zio, abuf, asize, asize, zio_subblock); 2523 } 2524 2525 /* 2526 * If this is not a physical io, make sure that it is properly aligned 2527 * before proceeding. 2528 */ 2529 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) { 2530 ASSERT0(P2PHASE(zio->io_offset, align)); 2531 ASSERT0(P2PHASE(zio->io_size, align)); 2532 } else { 2533 /* 2534 * For physical writes, we allow 512b aligned writes and assume 2535 * the device will perform a read-modify-write as necessary. 2536 */ 2537 ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE)); 2538 ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE)); 2539 } 2540 2541 VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa)); 2542 2543 /* 2544 * If this is a repair I/O, and there's no self-healing involved -- 2545 * that is, we're just resilvering what we expect to resilver -- 2546 * then don't do the I/O unless zio's txg is actually in vd's DTL. 2547 * This prevents spurious resilvering with nested replication. 2548 * For example, given a mirror of mirrors, (A+B)+(C+D), if only 2549 * A is out of date, we'll read from C+D, then use the data to 2550 * resilver A+B -- but we don't actually want to resilver B, just A. 2551 * The top-level mirror has no way to know this, so instead we just 2552 * discard unnecessary repairs as we work our way down the vdev tree. 2553 * The same logic applies to any form of nested replication: 2554 * ditto + mirror, RAID-Z + replacing, etc. This covers them all. 2555 */ 2556 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 2557 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 2558 zio->io_txg != 0 && /* not a delegated i/o */ 2559 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 2560 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2561 zio_vdev_io_bypass(zio); 2562 return (ZIO_PIPELINE_CONTINUE); 2563 } 2564 2565 if (vd->vdev_ops->vdev_op_leaf && 2566 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2567 2568 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio)) 2569 return (ZIO_PIPELINE_CONTINUE); 2570 2571 if ((zio = vdev_queue_io(zio)) == NULL) 2572 return (ZIO_PIPELINE_STOP); 2573 2574 if (!vdev_accessible(vd, zio)) { 2575 zio->io_error = SET_ERROR(ENXIO); 2576 zio_interrupt(zio); 2577 return (ZIO_PIPELINE_STOP); 2578 } 2579 } 2580 2581 return (vd->vdev_ops->vdev_op_io_start(zio)); 2582 } 2583 2584 static int 2585 zio_vdev_io_done(zio_t *zio) 2586 { 2587 vdev_t *vd = zio->io_vd; 2588 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 2589 boolean_t unexpected_error = B_FALSE; 2590 2591 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2592 return (ZIO_PIPELINE_STOP); 2593 2594 ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); 2595 2596 if (vd != NULL && vd->vdev_ops->vdev_op_leaf) { 2597 2598 vdev_queue_io_done(zio); 2599 2600 if (zio->io_type == ZIO_TYPE_WRITE) 2601 vdev_cache_write(zio); 2602 2603 if (zio_injection_enabled && zio->io_error == 0) 2604 zio->io_error = zio_handle_device_injection(vd, 2605 zio, EIO); 2606 2607 if (zio_injection_enabled && zio->io_error == 0) 2608 zio->io_error = zio_handle_label_injection(zio, EIO); 2609 2610 if (zio->io_error) { 2611 if (!vdev_accessible(vd, zio)) { 2612 zio->io_error = SET_ERROR(ENXIO); 2613 } else { 2614 unexpected_error = B_TRUE; 2615 } 2616 } 2617 } 2618 2619 ops->vdev_op_io_done(zio); 2620 2621 if (unexpected_error) 2622 VERIFY(vdev_probe(vd, zio) == NULL); 2623 2624 return (ZIO_PIPELINE_CONTINUE); 2625 } 2626 2627 /* 2628 * For non-raidz ZIOs, we can just copy aside the bad data read from the 2629 * disk, and use that to finish the checksum ereport later. 2630 */ 2631 static void 2632 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 2633 const void *good_buf) 2634 { 2635 /* no processing needed */ 2636 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 2637 } 2638 2639 /*ARGSUSED*/ 2640 void 2641 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 2642 { 2643 void *buf = zio_buf_alloc(zio->io_size); 2644 2645 bcopy(zio->io_data, buf, zio->io_size); 2646 2647 zcr->zcr_cbinfo = zio->io_size; 2648 zcr->zcr_cbdata = buf; 2649 zcr->zcr_finish = zio_vsd_default_cksum_finish; 2650 zcr->zcr_free = zio_buf_free; 2651 } 2652 2653 static int 2654 zio_vdev_io_assess(zio_t *zio) 2655 { 2656 vdev_t *vd = zio->io_vd; 2657 2658 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2659 return (ZIO_PIPELINE_STOP); 2660 2661 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2662 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 2663 2664 if (zio->io_vsd != NULL) { 2665 zio->io_vsd_ops->vsd_free(zio); 2666 zio->io_vsd = NULL; 2667 } 2668 2669 if (zio_injection_enabled && zio->io_error == 0) 2670 zio->io_error = zio_handle_fault_injection(zio, EIO); 2671 2672 /* 2673 * If the I/O failed, determine whether we should attempt to retry it. 2674 * 2675 * On retry, we cut in line in the issue queue, since we don't want 2676 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 2677 */ 2678 if (zio->io_error && vd == NULL && 2679 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 2680 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 2681 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 2682 zio->io_error = 0; 2683 zio->io_flags |= ZIO_FLAG_IO_RETRY | 2684 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 2685 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 2686 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 2687 zio_requeue_io_start_cut_in_line); 2688 return (ZIO_PIPELINE_STOP); 2689 } 2690 2691 /* 2692 * If we got an error on a leaf device, convert it to ENXIO 2693 * if the device is not accessible at all. 2694 */ 2695 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 2696 !vdev_accessible(vd, zio)) 2697 zio->io_error = SET_ERROR(ENXIO); 2698 2699 /* 2700 * If we can't write to an interior vdev (mirror or RAID-Z), 2701 * set vdev_cant_write so that we stop trying to allocate from it. 2702 */ 2703 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 2704 vd != NULL && !vd->vdev_ops->vdev_op_leaf) { 2705 vd->vdev_cant_write = B_TRUE; 2706 } 2707 2708 if (zio->io_error) 2709 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2710 2711 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 2712 zio->io_physdone != NULL) { 2713 ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED)); 2714 ASSERT(zio->io_child_type == ZIO_CHILD_VDEV); 2715 zio->io_physdone(zio->io_logical); 2716 } 2717 2718 return (ZIO_PIPELINE_CONTINUE); 2719 } 2720 2721 void 2722 zio_vdev_io_reissue(zio_t *zio) 2723 { 2724 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2725 ASSERT(zio->io_error == 0); 2726 2727 zio->io_stage >>= 1; 2728 } 2729 2730 void 2731 zio_vdev_io_redone(zio_t *zio) 2732 { 2733 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 2734 2735 zio->io_stage >>= 1; 2736 } 2737 2738 void 2739 zio_vdev_io_bypass(zio_t *zio) 2740 { 2741 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2742 ASSERT(zio->io_error == 0); 2743 2744 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 2745 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 2746 } 2747 2748 /* 2749 * ========================================================================== 2750 * Generate and verify checksums 2751 * ========================================================================== 2752 */ 2753 static int 2754 zio_checksum_generate(zio_t *zio) 2755 { 2756 blkptr_t *bp = zio->io_bp; 2757 enum zio_checksum checksum; 2758 2759 if (bp == NULL) { 2760 /* 2761 * This is zio_write_phys(). 2762 * We're either generating a label checksum, or none at all. 2763 */ 2764 checksum = zio->io_prop.zp_checksum; 2765 2766 if (checksum == ZIO_CHECKSUM_OFF) 2767 return (ZIO_PIPELINE_CONTINUE); 2768 2769 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 2770 } else { 2771 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 2772 ASSERT(!IO_IS_ALLOCATING(zio)); 2773 checksum = ZIO_CHECKSUM_GANG_HEADER; 2774 } else { 2775 checksum = BP_GET_CHECKSUM(bp); 2776 } 2777 } 2778 2779 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size); 2780 2781 return (ZIO_PIPELINE_CONTINUE); 2782 } 2783 2784 static int 2785 zio_checksum_verify(zio_t *zio) 2786 { 2787 zio_bad_cksum_t info; 2788 blkptr_t *bp = zio->io_bp; 2789 int error; 2790 2791 ASSERT(zio->io_vd != NULL); 2792 2793 if (bp == NULL) { 2794 /* 2795 * This is zio_read_phys(). 2796 * We're either verifying a label checksum, or nothing at all. 2797 */ 2798 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 2799 return (ZIO_PIPELINE_CONTINUE); 2800 2801 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 2802 } 2803 2804 if ((error = zio_checksum_error(zio, &info)) != 0) { 2805 zio->io_error = error; 2806 if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 2807 zfs_ereport_start_checksum(zio->io_spa, 2808 zio->io_vd, zio, zio->io_offset, 2809 zio->io_size, NULL, &info); 2810 } 2811 } 2812 2813 return (ZIO_PIPELINE_CONTINUE); 2814 } 2815 2816 /* 2817 * Called by RAID-Z to ensure we don't compute the checksum twice. 2818 */ 2819 void 2820 zio_checksum_verified(zio_t *zio) 2821 { 2822 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 2823 } 2824 2825 /* 2826 * ========================================================================== 2827 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 2828 * An error of 0 indicates success. ENXIO indicates whole-device failure, 2829 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 2830 * indicate errors that are specific to one I/O, and most likely permanent. 2831 * Any other error is presumed to be worse because we weren't expecting it. 2832 * ========================================================================== 2833 */ 2834 int 2835 zio_worst_error(int e1, int e2) 2836 { 2837 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 2838 int r1, r2; 2839 2840 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 2841 if (e1 == zio_error_rank[r1]) 2842 break; 2843 2844 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 2845 if (e2 == zio_error_rank[r2]) 2846 break; 2847 2848 return (r1 > r2 ? e1 : e2); 2849 } 2850 2851 /* 2852 * ========================================================================== 2853 * I/O completion 2854 * ========================================================================== 2855 */ 2856 static int 2857 zio_ready(zio_t *zio) 2858 { 2859 blkptr_t *bp = zio->io_bp; 2860 zio_t *pio, *pio_next; 2861 2862 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 2863 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) 2864 return (ZIO_PIPELINE_STOP); 2865 2866 if (zio->io_ready) { 2867 ASSERT(IO_IS_ALLOCATING(zio)); 2868 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 2869 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 2870 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 2871 2872 zio->io_ready(zio); 2873 } 2874 2875 if (bp != NULL && bp != &zio->io_bp_copy) 2876 zio->io_bp_copy = *bp; 2877 2878 if (zio->io_error) 2879 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2880 2881 mutex_enter(&zio->io_lock); 2882 zio->io_state[ZIO_WAIT_READY] = 1; 2883 pio = zio_walk_parents(zio); 2884 mutex_exit(&zio->io_lock); 2885 2886 /* 2887 * As we notify zio's parents, new parents could be added. 2888 * New parents go to the head of zio's io_parent_list, however, 2889 * so we will (correctly) not notify them. The remainder of zio's 2890 * io_parent_list, from 'pio_next' onward, cannot change because 2891 * all parents must wait for us to be done before they can be done. 2892 */ 2893 for (; pio != NULL; pio = pio_next) { 2894 pio_next = zio_walk_parents(zio); 2895 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 2896 } 2897 2898 if (zio->io_flags & ZIO_FLAG_NODATA) { 2899 if (BP_IS_GANG(bp)) { 2900 zio->io_flags &= ~ZIO_FLAG_NODATA; 2901 } else { 2902 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE); 2903 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 2904 } 2905 } 2906 2907 if (zio_injection_enabled && 2908 zio->io_spa->spa_syncing_txg == zio->io_txg) 2909 zio_handle_ignored_writes(zio); 2910 2911 return (ZIO_PIPELINE_CONTINUE); 2912 } 2913 2914 static int 2915 zio_done(zio_t *zio) 2916 { 2917 spa_t *spa = zio->io_spa; 2918 zio_t *lio = zio->io_logical; 2919 blkptr_t *bp = zio->io_bp; 2920 vdev_t *vd = zio->io_vd; 2921 uint64_t psize = zio->io_size; 2922 zio_t *pio, *pio_next; 2923 2924 /* 2925 * If our children haven't all completed, 2926 * wait for them and then repeat this pipeline stage. 2927 */ 2928 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || 2929 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || 2930 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || 2931 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) 2932 return (ZIO_PIPELINE_STOP); 2933 2934 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 2935 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 2936 ASSERT(zio->io_children[c][w] == 0); 2937 2938 if (bp != NULL && !BP_IS_EMBEDDED(bp)) { 2939 ASSERT(bp->blk_pad[0] == 0); 2940 ASSERT(bp->blk_pad[1] == 0); 2941 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 2942 (bp == zio_unique_parent(zio)->io_bp)); 2943 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 2944 zio->io_bp_override == NULL && 2945 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 2946 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 2947 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 2948 ASSERT(BP_COUNT_GANG(bp) == 0 || 2949 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 2950 } 2951 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 2952 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig)); 2953 } 2954 2955 /* 2956 * If there were child vdev/gang/ddt errors, they apply to us now. 2957 */ 2958 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 2959 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 2960 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 2961 2962 /* 2963 * If the I/O on the transformed data was successful, generate any 2964 * checksum reports now while we still have the transformed data. 2965 */ 2966 if (zio->io_error == 0) { 2967 while (zio->io_cksum_report != NULL) { 2968 zio_cksum_report_t *zcr = zio->io_cksum_report; 2969 uint64_t align = zcr->zcr_align; 2970 uint64_t asize = P2ROUNDUP(psize, align); 2971 char *abuf = zio->io_data; 2972 2973 if (asize != psize) { 2974 abuf = zio_buf_alloc(asize); 2975 bcopy(zio->io_data, abuf, psize); 2976 bzero(abuf + psize, asize - psize); 2977 } 2978 2979 zio->io_cksum_report = zcr->zcr_next; 2980 zcr->zcr_next = NULL; 2981 zcr->zcr_finish(zcr, abuf); 2982 zfs_ereport_free_checksum(zcr); 2983 2984 if (asize != psize) 2985 zio_buf_free(abuf, asize); 2986 } 2987 } 2988 2989 zio_pop_transforms(zio); /* note: may set zio->io_error */ 2990 2991 vdev_stat_update(zio, psize); 2992 2993 if (zio->io_error) { 2994 /* 2995 * If this I/O is attached to a particular vdev, 2996 * generate an error message describing the I/O failure 2997 * at the block level. We ignore these errors if the 2998 * device is currently unavailable. 2999 */ 3000 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 3001 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 3002 3003 if ((zio->io_error == EIO || !(zio->io_flags & 3004 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 3005 zio == lio) { 3006 /* 3007 * For logical I/O requests, tell the SPA to log the 3008 * error and generate a logical data ereport. 3009 */ 3010 spa_log_error(spa, zio); 3011 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 3012 0, 0); 3013 } 3014 } 3015 3016 if (zio->io_error && zio == lio) { 3017 /* 3018 * Determine whether zio should be reexecuted. This will 3019 * propagate all the way to the root via zio_notify_parent(). 3020 */ 3021 ASSERT(vd == NULL && bp != NULL); 3022 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3023 3024 if (IO_IS_ALLOCATING(zio) && 3025 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 3026 if (zio->io_error != ENOSPC) 3027 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 3028 else 3029 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3030 } 3031 3032 if ((zio->io_type == ZIO_TYPE_READ || 3033 zio->io_type == ZIO_TYPE_FREE) && 3034 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 3035 zio->io_error == ENXIO && 3036 spa_load_state(spa) == SPA_LOAD_NONE && 3037 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 3038 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3039 3040 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 3041 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3042 3043 /* 3044 * Here is a possibly good place to attempt to do 3045 * either combinatorial reconstruction or error correction 3046 * based on checksums. It also might be a good place 3047 * to send out preliminary ereports before we suspend 3048 * processing. 3049 */ 3050 } 3051 3052 /* 3053 * If there were logical child errors, they apply to us now. 3054 * We defer this until now to avoid conflating logical child 3055 * errors with errors that happened to the zio itself when 3056 * updating vdev stats and reporting FMA events above. 3057 */ 3058 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 3059 3060 if ((zio->io_error || zio->io_reexecute) && 3061 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 3062 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 3063 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 3064 3065 zio_gang_tree_free(&zio->io_gang_tree); 3066 3067 /* 3068 * Godfather I/Os should never suspend. 3069 */ 3070 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 3071 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 3072 zio->io_reexecute = 0; 3073 3074 if (zio->io_reexecute) { 3075 /* 3076 * This is a logical I/O that wants to reexecute. 3077 * 3078 * Reexecute is top-down. When an i/o fails, if it's not 3079 * the root, it simply notifies its parent and sticks around. 3080 * The parent, seeing that it still has children in zio_done(), 3081 * does the same. This percolates all the way up to the root. 3082 * The root i/o will reexecute or suspend the entire tree. 3083 * 3084 * This approach ensures that zio_reexecute() honors 3085 * all the original i/o dependency relationships, e.g. 3086 * parents not executing until children are ready. 3087 */ 3088 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3089 3090 zio->io_gang_leader = NULL; 3091 3092 mutex_enter(&zio->io_lock); 3093 zio->io_state[ZIO_WAIT_DONE] = 1; 3094 mutex_exit(&zio->io_lock); 3095 3096 /* 3097 * "The Godfather" I/O monitors its children but is 3098 * not a true parent to them. It will track them through 3099 * the pipeline but severs its ties whenever they get into 3100 * trouble (e.g. suspended). This allows "The Godfather" 3101 * I/O to return status without blocking. 3102 */ 3103 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3104 zio_link_t *zl = zio->io_walk_link; 3105 pio_next = zio_walk_parents(zio); 3106 3107 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 3108 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 3109 zio_remove_child(pio, zio, zl); 3110 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3111 } 3112 } 3113 3114 if ((pio = zio_unique_parent(zio)) != NULL) { 3115 /* 3116 * We're not a root i/o, so there's nothing to do 3117 * but notify our parent. Don't propagate errors 3118 * upward since we haven't permanently failed yet. 3119 */ 3120 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 3121 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 3122 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3123 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 3124 /* 3125 * We'd fail again if we reexecuted now, so suspend 3126 * until conditions improve (e.g. device comes online). 3127 */ 3128 zio_suspend(spa, zio); 3129 } else { 3130 /* 3131 * Reexecution is potentially a huge amount of work. 3132 * Hand it off to the otherwise-unused claim taskq. 3133 */ 3134 ASSERT(zio->io_tqent.tqent_next == NULL); 3135 spa_taskq_dispatch_ent(spa, ZIO_TYPE_CLAIM, 3136 ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio, 3137 0, &zio->io_tqent); 3138 } 3139 return (ZIO_PIPELINE_STOP); 3140 } 3141 3142 ASSERT(zio->io_child_count == 0); 3143 ASSERT(zio->io_reexecute == 0); 3144 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 3145 3146 /* 3147 * Report any checksum errors, since the I/O is complete. 3148 */ 3149 while (zio->io_cksum_report != NULL) { 3150 zio_cksum_report_t *zcr = zio->io_cksum_report; 3151 zio->io_cksum_report = zcr->zcr_next; 3152 zcr->zcr_next = NULL; 3153 zcr->zcr_finish(zcr, NULL); 3154 zfs_ereport_free_checksum(zcr); 3155 } 3156 3157 /* 3158 * It is the responsibility of the done callback to ensure that this 3159 * particular zio is no longer discoverable for adoption, and as 3160 * such, cannot acquire any new parents. 3161 */ 3162 if (zio->io_done) 3163 zio->io_done(zio); 3164 3165 mutex_enter(&zio->io_lock); 3166 zio->io_state[ZIO_WAIT_DONE] = 1; 3167 mutex_exit(&zio->io_lock); 3168 3169 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3170 zio_link_t *zl = zio->io_walk_link; 3171 pio_next = zio_walk_parents(zio); 3172 zio_remove_child(pio, zio, zl); 3173 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3174 } 3175 3176 if (zio->io_waiter != NULL) { 3177 mutex_enter(&zio->io_lock); 3178 zio->io_executor = NULL; 3179 cv_broadcast(&zio->io_cv); 3180 mutex_exit(&zio->io_lock); 3181 } else { 3182 zio_destroy(zio); 3183 } 3184 3185 return (ZIO_PIPELINE_STOP); 3186 } 3187 3188 /* 3189 * ========================================================================== 3190 * I/O pipeline definition 3191 * ========================================================================== 3192 */ 3193 static zio_pipe_stage_t *zio_pipeline[] = { 3194 NULL, 3195 zio_read_bp_init, 3196 zio_free_bp_init, 3197 zio_issue_async, 3198 zio_write_bp_init, 3199 zio_checksum_generate, 3200 zio_nop_write, 3201 zio_ddt_read_start, 3202 zio_ddt_read_done, 3203 zio_ddt_write, 3204 zio_ddt_free, 3205 zio_gang_assemble, 3206 zio_gang_issue, 3207 zio_dva_allocate, 3208 zio_dva_free, 3209 zio_dva_claim, 3210 zio_ready, 3211 zio_vdev_io_start, 3212 zio_vdev_io_done, 3213 zio_vdev_io_assess, 3214 zio_checksum_verify, 3215 zio_done 3216 }; 3217 3218 /* dnp is the dnode for zb1->zb_object */ 3219 boolean_t 3220 zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_phys_t *zb1, 3221 const zbookmark_phys_t *zb2) 3222 { 3223 uint64_t zb1nextL0, zb2thisobj; 3224 3225 ASSERT(zb1->zb_objset == zb2->zb_objset); 3226 ASSERT(zb2->zb_level == 0); 3227 3228 /* The objset_phys_t isn't before anything. */ 3229 if (dnp == NULL) 3230 return (B_FALSE); 3231 3232 zb1nextL0 = (zb1->zb_blkid + 1) << 3233 ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)); 3234 3235 zb2thisobj = zb2->zb_object ? zb2->zb_object : 3236 zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT); 3237 3238 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 3239 uint64_t nextobj = zb1nextL0 * 3240 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT; 3241 return (nextobj <= zb2thisobj); 3242 } 3243 3244 if (zb1->zb_object < zb2thisobj) 3245 return (B_TRUE); 3246 if (zb1->zb_object > zb2thisobj) 3247 return (B_FALSE); 3248 if (zb2->zb_object == DMU_META_DNODE_OBJECT) 3249 return (B_FALSE); 3250 return (zb1nextL0 <= zb2->zb_blkid); 3251 }