FILE
raid10.c
File preview with syntax highlighting.
linux-6.19.14/drivers/md/raid10.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * raid10.c : Multiple Devices driver for Linux 4 * 5 * Copyright (C) 2000-2004 Neil Brown 6 * 7 * RAID-10 support for md. 8 * 9 * Base on code in raid1.c. See raid1.c for further copyright information. 10 */ 11 12#include <linux/slab.h> 13#include <linux/delay.h> 14#include <linux/blkdev.h> 15#include <linux/module.h> 16#include <linux/seq_file.h> 17#include <linux/ratelimit.h> 18#include <linux/kthread.h> 19#include <linux/raid/md_p.h> 20#include <trace/events/block.h> 21#include "md.h" 22 23#define RAID_1_10_NAME "raid10" 24#include "raid10.h" 25#include "raid0.h" 26#include "md-bitmap.h" 27#include "md-cluster.h" 28 29/* 30 * RAID10 provides a combination of RAID0 and RAID1 functionality. 31 * The layout of data is defined by 32 * chunk_size 33 * raid_disks 34 * near_copies (stored in low byte of layout) 35 * far_copies (stored in second byte of layout) 36 * far_offset (stored in bit 16 of layout ) 37 * use_far_sets (stored in bit 17 of layout ) 38 * use_far_sets_bugfixed (stored in bit 18 of layout ) 39 * 40 * The data to be stored is divided into chunks using chunksize. Each device 41 * is divided into far_copies sections. In each section, chunks are laid out 42 * in a style similar to raid0, but near_copies copies of each chunk is stored 43 * (each on a different drive). The starting device for each section is offset 44 * near_copies from the starting device of the previous section. Thus there 45 * are (near_copies * far_copies) of each chunk, and each is on a different 46 * drive. near_copies and far_copies must be at least one, and their product 47 * is at most raid_disks. 48 * 49 * If far_offset is true, then the far_copies are handled a bit differently. 50 * The copies are still in different stripes, but instead of being very far 51 * apart on disk, there are adjacent stripes. 52 * 53 * The far and offset algorithms are handled slightly differently if 54 * 'use_far_sets' is true. In this case, the array's devices are grouped into 55 * sets that are (near_copies * far_copies) in size. The far copied stripes 56 * are still shifted by 'near_copies' devices, but this shifting stays confined 57 * to the set rather than the entire array. This is done to improve the number 58 * of device combinations that can fail without causing the array to fail. 59 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk 60 * on a device): 61 * A B C D A B C D E 62 * ... ... 63 * D A B C E A B C D 64 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s): 65 * [A B] [C D] [A B] [C D E] 66 * |...| |...| |...| | ... | 67 * [B A] [D C] [B A] [E C D] 68 */ 69 70static void allow_barrier(struct r10conf *conf); 71static void lower_barrier(struct r10conf *conf); 72static int _enough(struct r10conf *conf, int previous, int ignore); 73static int enough(struct r10conf *conf, int ignore); 74static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 75 int *skipped); 76static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 77static void end_reshape_write(struct bio *bio); 78static void end_reshape(struct r10conf *conf); 79 80#include "raid1-10.c" 81 82#define NULL_CMD 83#define cmd_before(conf, cmd) \ 84 do { \ 85 write_sequnlock_irq(&(conf)->resync_lock); \ 86 cmd; \ 87 } while (0) 88#define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock) 89 90#define wait_event_barrier_cmd(conf, cond, cmd) \ 91 wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \ 92 cmd_after(conf)) 93 94#define wait_event_barrier(conf, cond) \ 95 wait_event_barrier_cmd(conf, cond, NULL_CMD) 96 97/* 98 * for resync bio, r10bio pointer can be retrieved from the per-bio 99 * 'struct resync_pages'. 100 */ 101static inline struct r10bio *get_resync_r10bio(struct bio *bio) 102{ 103 return get_resync_pages(bio)->raid_bio; 104} 105 106static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 107{ 108 struct r10conf *conf = data; 109 int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]); 110 111 /* allocate a r10bio with room for raid_disks entries in the 112 * bios array */ 113 return kzalloc(size, gfp_flags); 114} 115 116#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9) 117/* amount of memory to reserve for resync requests */ 118#define RESYNC_WINDOW (1024*1024) 119/* maximum number of concurrent requests, memory permitting */ 120#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE) 121#define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW) 122#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9) 123 124/* 125 * When performing a resync, we need to read and compare, so 126 * we need as many pages are there are copies. 127 * When performing a recovery, we need 2 bios, one for read, 128 * one for write (we recover only one drive per r10buf) 129 * 130 */ 131static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) 132{ 133 struct r10conf *conf = data; 134 struct r10bio *r10_bio; 135 struct bio *bio; 136 int j; 137 int nalloc, nalloc_rp; 138 struct resync_pages *rps; 139 140 r10_bio = r10bio_pool_alloc(gfp_flags, conf); 141 if (!r10_bio) 142 return NULL; 143 144 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 145 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 146 nalloc = conf->copies; /* resync */ 147 else 148 nalloc = 2; /* recovery */ 149 150 /* allocate once for all bios */ 151 if (!conf->have_replacement) 152 nalloc_rp = nalloc; 153 else 154 nalloc_rp = nalloc * 2; 155 rps = kmalloc_array(nalloc_rp, sizeof(struct resync_pages), gfp_flags); 156 if (!rps) 157 goto out_free_r10bio; 158 159 /* 160 * Allocate bios. 161 */ 162 for (j = nalloc ; j-- ; ) { 163 bio = bio_kmalloc(RESYNC_PAGES, gfp_flags); 164 if (!bio) 165 goto out_free_bio; 166 bio_init_inline(bio, NULL, RESYNC_PAGES, 0); 167 r10_bio->devs[j].bio = bio; 168 if (!conf->have_replacement) 169 continue; 170 bio = bio_kmalloc(RESYNC_PAGES, gfp_flags); 171 if (!bio) 172 goto out_free_bio; 173 bio_init_inline(bio, NULL, RESYNC_PAGES, 0); 174 r10_bio->devs[j].repl_bio = bio; 175 } 176 /* 177 * Allocate RESYNC_PAGES data pages and attach them 178 * where needed. 179 */ 180 for (j = 0; j < nalloc; j++) { 181 struct bio *rbio = r10_bio->devs[j].repl_bio; 182 struct resync_pages *rp, *rp_repl; 183 184 rp = &rps[j]; 185 if (rbio) 186 rp_repl = &rps[nalloc + j]; 187 188 bio = r10_bio->devs[j].bio; 189 190 if (!j || test_bit(MD_RECOVERY_SYNC, 191 &conf->mddev->recovery)) { 192 if (resync_alloc_pages(rp, gfp_flags)) 193 goto out_free_pages; 194 } else { 195 memcpy(rp, &rps[0], sizeof(*rp)); 196 resync_get_all_pages(rp); 197 } 198 199 rp->raid_bio = r10_bio; 200 bio->bi_private = rp; 201 if (rbio) { 202 memcpy(rp_repl, rp, sizeof(*rp)); 203 rbio->bi_private = rp_repl; 204 } 205 } 206 207 return r10_bio; 208 209out_free_pages: 210 while (--j >= 0) 211 resync_free_pages(&rps[j]); 212 213 j = 0; 214out_free_bio: 215 for ( ; j < nalloc; j++) { 216 if (r10_bio->devs[j].bio) 217 bio_uninit(r10_bio->devs[j].bio); 218 kfree(r10_bio->devs[j].bio); 219 if (r10_bio->devs[j].repl_bio) 220 bio_uninit(r10_bio->devs[j].repl_bio); 221 kfree(r10_bio->devs[j].repl_bio); 222 } 223 kfree(rps); 224out_free_r10bio: 225 rbio_pool_free(r10_bio, conf); 226 return NULL; 227} 228 229static void r10buf_pool_free(void *__r10_bio, void *data) 230{ 231 struct r10conf *conf = data; 232 struct r10bio *r10bio = __r10_bio; 233 int j; 234 struct resync_pages *rp = NULL; 235 236 for (j = conf->copies; j--; ) { 237 struct bio *bio = r10bio->devs[j].bio; 238 239 if (bio) { 240 rp = get_resync_pages(bio); 241 resync_free_pages(rp); 242 bio_uninit(bio); 243 kfree(bio); 244 } 245 246 bio = r10bio->devs[j].repl_bio; 247 if (bio) { 248 bio_uninit(bio); 249 kfree(bio); 250 } 251 } 252 253 /* resync pages array stored in the 1st bio's .bi_private */ 254 kfree(rp); 255 256 rbio_pool_free(r10bio, conf); 257} 258 259static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) 260{ 261 int i; 262 263 for (i = 0; i < conf->geo.raid_disks; i++) { 264 struct bio **bio = & r10_bio->devs[i].bio; 265 if (!BIO_SPECIAL(*bio)) 266 bio_put(*bio); 267 *bio = NULL; 268 bio = &r10_bio->devs[i].repl_bio; 269 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) 270 bio_put(*bio); 271 *bio = NULL; 272 } 273} 274 275static void free_r10bio(struct r10bio *r10_bio) 276{ 277 struct r10conf *conf = r10_bio->mddev->private; 278 279 put_all_bios(conf, r10_bio); 280 mempool_free(r10_bio, &conf->r10bio_pool); 281} 282 283static void put_buf(struct r10bio *r10_bio) 284{ 285 struct r10conf *conf = r10_bio->mddev->private; 286 287 mempool_free(r10_bio, &conf->r10buf_pool); 288 289 lower_barrier(conf); 290} 291 292static void wake_up_barrier(struct r10conf *conf) 293{ 294 if (wq_has_sleeper(&conf->wait_barrier)) 295 wake_up(&conf->wait_barrier); 296} 297 298static void reschedule_retry(struct r10bio *r10_bio) 299{ 300 unsigned long flags; 301 struct mddev *mddev = r10_bio->mddev; 302 struct r10conf *conf = mddev->private; 303 304 spin_lock_irqsave(&conf->device_lock, flags); 305 list_add(&r10_bio->retry_list, &conf->retry_list); 306 conf->nr_queued ++; 307 spin_unlock_irqrestore(&conf->device_lock, flags); 308 309 /* wake up frozen array... */ 310 wake_up(&conf->wait_barrier); 311 312 md_wakeup_thread(mddev->thread); 313} 314 315/* 316 * raid_end_bio_io() is called when we have finished servicing a mirrored 317 * operation and are ready to return a success/failure code to the buffer 318 * cache layer. 319 */ 320static void raid_end_bio_io(struct r10bio *r10_bio) 321{ 322 struct bio *bio = r10_bio->master_bio; 323 struct r10conf *conf = r10_bio->mddev->private; 324 325 if (!test_and_set_bit(R10BIO_Returned, &r10_bio->state)) { 326 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 327 bio->bi_status = BLK_STS_IOERR; 328 bio_endio(bio); 329 } 330 331 /* 332 * Wake up any possible resync thread that waits for the device 333 * to go idle. 334 */ 335 allow_barrier(conf); 336 337 free_r10bio(r10_bio); 338} 339 340/* 341 * Update disk head position estimator based on IRQ completion info. 342 */ 343static inline void update_head_pos(int slot, struct r10bio *r10_bio) 344{ 345 struct r10conf *conf = r10_bio->mddev->private; 346 347 conf->mirrors[r10_bio->devs[slot].devnum].head_position = 348 r10_bio->devs[slot].addr + (r10_bio->sectors); 349} 350 351/* 352 * Find the disk number which triggered given bio 353 */ 354static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, 355 struct bio *bio, int *slotp, int *replp) 356{ 357 int slot; 358 int repl = 0; 359 360 for (slot = 0; slot < conf->geo.raid_disks; slot++) { 361 if (r10_bio->devs[slot].bio == bio) 362 break; 363 if (r10_bio->devs[slot].repl_bio == bio) { 364 repl = 1; 365 break; 366 } 367 } 368 369 update_head_pos(slot, r10_bio); 370 371 if (slotp) 372 *slotp = slot; 373 if (replp) 374 *replp = repl; 375 return r10_bio->devs[slot].devnum; 376} 377 378static void raid10_end_read_request(struct bio *bio) 379{ 380 int uptodate = !bio->bi_status; 381 struct r10bio *r10_bio = bio->bi_private; 382 int slot; 383 struct md_rdev *rdev; 384 struct r10conf *conf = r10_bio->mddev->private; 385 386 slot = r10_bio->read_slot; 387 rdev = r10_bio->devs[slot].rdev; 388 /* 389 * this branch is our 'one mirror IO has finished' event handler: 390 */ 391 update_head_pos(slot, r10_bio); 392 393 if (uptodate) { 394 /* 395 * Set R10BIO_Uptodate in our master bio, so that 396 * we will return a good error code to the higher 397 * levels even if IO on some other mirrored buffer fails. 398 * 399 * The 'master' represents the composite IO operation to 400 * user-side. So if something waits for IO, then it will 401 * wait for the 'master' bio. 402 */ 403 set_bit(R10BIO_Uptodate, &r10_bio->state); 404 } else if (!raid1_should_handle_error(bio)) { 405 uptodate = 1; 406 } else { 407 /* If all other devices that store this block have 408 * failed, we want to return the error upwards rather 409 * than fail the last device. Here we redefine 410 * "uptodate" to mean "Don't want to retry" 411 */ 412 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state), 413 rdev->raid_disk)) 414 uptodate = 1; 415 } 416 if (uptodate) { 417 raid_end_bio_io(r10_bio); 418 rdev_dec_pending(rdev, conf->mddev); 419 } else { 420 /* 421 * oops, read error - keep the refcount on the rdev 422 */ 423 pr_err_ratelimited("md/raid10:%s: %pg: rescheduling sector %llu\n", 424 mdname(conf->mddev), 425 rdev->bdev, 426 (unsigned long long)r10_bio->sector); 427 set_bit(R10BIO_ReadError, &r10_bio->state); 428 reschedule_retry(r10_bio); 429 } 430} 431 432static void close_write(struct r10bio *r10_bio) 433{ 434 struct mddev *mddev = r10_bio->mddev; 435 436 md_write_end(mddev); 437} 438 439static void one_write_done(struct r10bio *r10_bio) 440{ 441 if (atomic_dec_and_test(&r10_bio->remaining)) { 442 if (test_bit(R10BIO_WriteError, &r10_bio->state)) 443 reschedule_retry(r10_bio); 444 else { 445 close_write(r10_bio); 446 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) 447 reschedule_retry(r10_bio); 448 else 449 raid_end_bio_io(r10_bio); 450 } 451 } 452} 453 454static void raid10_end_write_request(struct bio *bio) 455{ 456 struct r10bio *r10_bio = bio->bi_private; 457 int dev; 458 int dec_rdev = 1; 459 struct r10conf *conf = r10_bio->mddev->private; 460 int slot, repl; 461 struct md_rdev *rdev = NULL; 462 struct bio *to_put = NULL; 463 bool ignore_error = !raid1_should_handle_error(bio) || 464 (bio->bi_status && bio_op(bio) == REQ_OP_DISCARD); 465 466 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 467 468 if (repl) 469 rdev = conf->mirrors[dev].replacement; 470 if (!rdev) { 471 smp_rmb(); 472 repl = 0; 473 rdev = conf->mirrors[dev].rdev; 474 } 475 /* 476 * this branch is our 'one mirror IO has finished' event handler: 477 */ 478 if (bio->bi_status && !ignore_error) { 479 if (repl) 480 /* Never record new bad blocks to replacement, 481 * just fail it. 482 */ 483 md_error(rdev->mddev, rdev); 484 else { 485 set_bit(WriteErrorSeen, &rdev->flags); 486 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 487 set_bit(MD_RECOVERY_NEEDED, 488 &rdev->mddev->recovery); 489 490 dec_rdev = 0; 491 if (test_bit(FailFast, &rdev->flags) && 492 (bio->bi_opf & MD_FAILFAST)) { 493 md_error(rdev->mddev, rdev); 494 } 495 496 /* 497 * When the device is faulty, it is not necessary to 498 * handle write error. 499 */ 500 if (!test_bit(Faulty, &rdev->flags)) 501 set_bit(R10BIO_WriteError, &r10_bio->state); 502 else { 503 /* Fail the request */ 504 r10_bio->devs[slot].bio = NULL; 505 to_put = bio; 506 dec_rdev = 1; 507 } 508 } 509 } else { 510 /* 511 * Set R10BIO_Uptodate in our master bio, so that 512 * we will return a good error code for to the higher 513 * levels even if IO on some other mirrored buffer fails. 514 * 515 * The 'master' represents the composite IO operation to 516 * user-side. So if something waits for IO, then it will 517 * wait for the 'master' bio. 518 * 519 * Do not set R10BIO_Uptodate if the current device is 520 * rebuilding or Faulty. This is because we cannot use 521 * such device for properly reading the data back (we could 522 * potentially use it, if the current write would have felt 523 * before rdev->recovery_offset, but for simplicity we don't 524 * check this here. 525 */ 526 if (test_bit(In_sync, &rdev->flags) && 527 !test_bit(Faulty, &rdev->flags)) 528 set_bit(R10BIO_Uptodate, &r10_bio->state); 529 530 /* Maybe we can clear some bad blocks. */ 531 if (rdev_has_badblock(rdev, r10_bio->devs[slot].addr, 532 r10_bio->sectors) && 533 !ignore_error) { 534 bio_put(bio); 535 if (repl) 536 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; 537 else 538 r10_bio->devs[slot].bio = IO_MADE_GOOD; 539 dec_rdev = 0; 540 set_bit(R10BIO_MadeGood, &r10_bio->state); 541 } 542 } 543 544 /* 545 * 546 * Let's see if all mirrored write operations have finished 547 * already. 548 */ 549 one_write_done(r10_bio); 550 if (dec_rdev) 551 rdev_dec_pending(rdev, conf->mddev); 552 if (to_put) 553 bio_put(to_put); 554} 555 556/* 557 * RAID10 layout manager 558 * As well as the chunksize and raid_disks count, there are two 559 * parameters: near_copies and far_copies. 560 * near_copies * far_copies must be <= raid_disks. 561 * Normally one of these will be 1. 562 * If both are 1, we get raid0. 563 * If near_copies == raid_disks, we get raid1. 564 * 565 * Chunks are laid out in raid0 style with near_copies copies of the 566 * first chunk, followed by near_copies copies of the next chunk and 567 * so on. 568 * If far_copies > 1, then after 1/far_copies of the array has been assigned 569 * as described above, we start again with a device offset of near_copies. 570 * So we effectively have another copy of the whole array further down all 571 * the drives, but with blocks on different drives. 572 * With this layout, and block is never stored twice on the one device. 573 * 574 * raid10_find_phys finds the sector offset of a given virtual sector 575 * on each device that it is on. 576 * 577 * raid10_find_virt does the reverse mapping, from a device and a 578 * sector offset to a virtual address 579 */ 580 581static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio) 582{ 583 int n,f; 584 sector_t sector; 585 sector_t chunk; 586 sector_t stripe; 587 int dev; 588 int slot = 0; 589 int last_far_set_start, last_far_set_size; 590 591 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 592 last_far_set_start *= geo->far_set_size; 593 594 last_far_set_size = geo->far_set_size; 595 last_far_set_size += (geo->raid_disks % geo->far_set_size); 596 597 /* now calculate first sector/dev */ 598 chunk = r10bio->sector >> geo->chunk_shift; 599 sector = r10bio->sector & geo->chunk_mask; 600 601 chunk *= geo->near_copies; 602 stripe = chunk; 603 dev = sector_div(stripe, geo->raid_disks); 604 if (geo->far_offset) 605 stripe *= geo->far_copies; 606 607 sector += stripe << geo->chunk_shift; 608 609 /* and calculate all the others */ 610 for (n = 0; n < geo->near_copies; n++) { 611 int d = dev; 612 int set; 613 sector_t s = sector; 614 r10bio->devs[slot].devnum = d; 615 r10bio->devs[slot].addr = s; 616 slot++; 617 618 for (f = 1; f < geo->far_copies; f++) { 619 set = d / geo->far_set_size; 620 d += geo->near_copies; 621 622 if ((geo->raid_disks % geo->far_set_size) && 623 (d > last_far_set_start)) { 624 d -= last_far_set_start; 625 d %= last_far_set_size; 626 d += last_far_set_start; 627 } else { 628 d %= geo->far_set_size; 629 d += geo->far_set_size * set; 630 } 631 s += geo->stride; 632 r10bio->devs[slot].devnum = d; 633 r10bio->devs[slot].addr = s; 634 slot++; 635 } 636 dev++; 637 if (dev >= geo->raid_disks) { 638 dev = 0; 639 sector += (geo->chunk_mask + 1); 640 } 641 } 642} 643 644static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio) 645{ 646 struct geom *geo = &conf->geo; 647 648 if (conf->reshape_progress != MaxSector && 649 ((r10bio->sector >= conf->reshape_progress) != 650 conf->mddev->reshape_backwards)) { 651 set_bit(R10BIO_Previous, &r10bio->state); 652 geo = &conf->prev; 653 } else 654 clear_bit(R10BIO_Previous, &r10bio->state); 655 656 __raid10_find_phys(geo, r10bio); 657} 658 659static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) 660{ 661 sector_t offset, chunk, vchunk; 662 /* Never use conf->prev as this is only called during resync 663 * or recovery, so reshape isn't happening 664 */ 665 struct geom *geo = &conf->geo; 666 int far_set_start = (dev / geo->far_set_size) * geo->far_set_size; 667 int far_set_size = geo->far_set_size; 668 int last_far_set_start; 669 670 if (geo->raid_disks % geo->far_set_size) { 671 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 672 last_far_set_start *= geo->far_set_size; 673 674 if (dev >= last_far_set_start) { 675 far_set_size = geo->far_set_size; 676 far_set_size += (geo->raid_disks % geo->far_set_size); 677 far_set_start = last_far_set_start; 678 } 679 } 680 681 offset = sector & geo->chunk_mask; 682 if (geo->far_offset) { 683 int fc; 684 chunk = sector >> geo->chunk_shift; 685 fc = sector_div(chunk, geo->far_copies); 686 dev -= fc * geo->near_copies; 687 if (dev < far_set_start) 688 dev += far_set_size; 689 } else { 690 while (sector >= geo->stride) { 691 sector -= geo->stride; 692 if (dev < (geo->near_copies + far_set_start)) 693 dev += far_set_size - geo->near_copies; 694 else 695 dev -= geo->near_copies; 696 } 697 chunk = sector >> geo->chunk_shift; 698 } 699 vchunk = chunk * geo->raid_disks + dev; 700 sector_div(vchunk, geo->near_copies); 701 return (vchunk << geo->chunk_shift) + offset; 702} 703 704/* 705 * This routine returns the disk from which the requested read should 706 * be done. There is a per-array 'next expected sequential IO' sector 707 * number - if this matches on the next IO then we use the last disk. 708 * There is also a per-disk 'last know head position' sector that is 709 * maintained from IRQ contexts, both the normal and the resync IO 710 * completion handlers update this position correctly. If there is no 711 * perfect sequential match then we pick the disk whose head is closest. 712 * 713 * If there are 2 mirrors in the same 2 devices, performance degrades 714 * because position is mirror, not device based. 715 * 716 * The rdev for the device selected will have nr_pending incremented. 717 */ 718 719/* 720 * FIXME: possibly should rethink readbalancing and do it differently 721 * depending on near_copies / far_copies geometry. 722 */ 723static struct md_rdev *read_balance(struct r10conf *conf, 724 struct r10bio *r10_bio, 725 int *max_sectors) 726{ 727 const sector_t this_sector = r10_bio->sector; 728 int disk, slot; 729 int sectors = r10_bio->sectors; 730 int best_good_sectors; 731 sector_t new_distance, best_dist; 732 struct md_rdev *best_dist_rdev, *best_pending_rdev, *rdev = NULL; 733 int do_balance; 734 int best_dist_slot, best_pending_slot; 735 bool has_nonrot_disk = false; 736 unsigned int min_pending; 737 struct geom *geo = &conf->geo; 738 739 raid10_find_phys(conf, r10_bio); 740 best_dist_slot = -1; 741 min_pending = UINT_MAX; 742 best_dist_rdev = NULL; 743 best_pending_rdev = NULL; 744 best_dist = MaxSector; 745 best_good_sectors = 0; 746 do_balance = 1; 747 clear_bit(R10BIO_FailFast, &r10_bio->state); 748 749 if (raid1_should_read_first(conf->mddev, this_sector, sectors)) 750 do_balance = 0; 751 752 for (slot = 0; slot < conf->copies ; slot++) { 753 sector_t first_bad; 754 sector_t bad_sectors; 755 sector_t dev_sector; 756 unsigned int pending; 757 bool nonrot; 758 759 if (r10_bio->devs[slot].bio == IO_BLOCKED) 760 continue; 761 disk = r10_bio->devs[slot].devnum; 762 rdev = conf->mirrors[disk].replacement; 763 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 764 r10_bio->devs[slot].addr + sectors > 765 rdev->recovery_offset) 766 rdev = conf->mirrors[disk].rdev; 767 if (rdev == NULL || 768 test_bit(Faulty, &rdev->flags)) 769 continue; 770 if (!test_bit(In_sync, &rdev->flags) && 771 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 772 continue; 773 774 dev_sector = r10_bio->devs[slot].addr; 775 if (is_badblock(rdev, dev_sector, sectors, 776 &first_bad, &bad_sectors)) { 777 if (best_dist < MaxSector) 778 /* Already have a better slot */ 779 continue; 780 if (first_bad <= dev_sector) { 781 /* Cannot read here. If this is the 782 * 'primary' device, then we must not read 783 * beyond 'bad_sectors' from another device. 784 */ 785 bad_sectors -= (dev_sector - first_bad); 786 if (!do_balance && sectors > bad_sectors) 787 sectors = bad_sectors; 788 if (best_good_sectors > sectors) 789 best_good_sectors = sectors; 790 } else { 791 sector_t good_sectors = 792 first_bad - dev_sector; 793 if (good_sectors > best_good_sectors) { 794 best_good_sectors = good_sectors; 795 best_dist_slot = slot; 796 best_dist_rdev = rdev; 797 } 798 if (!do_balance) 799 /* Must read from here */ 800 break; 801 } 802 continue; 803 } else 804 best_good_sectors = sectors; 805 806 if (!do_balance) 807 break; 808 809 nonrot = bdev_nonrot(rdev->bdev); 810 has_nonrot_disk |= nonrot; 811 pending = atomic_read(&rdev->nr_pending); 812 if (min_pending > pending && nonrot) { 813 min_pending = pending; 814 best_pending_slot = slot; 815 best_pending_rdev = rdev; 816 } 817 818 if (best_dist_slot >= 0) 819 /* At least 2 disks to choose from so failfast is OK */ 820 set_bit(R10BIO_FailFast, &r10_bio->state); 821 /* This optimisation is debatable, and completely destroys 822 * sequential read speed for 'far copies' arrays. So only 823 * keep it for 'near' arrays, and review those later. 824 */ 825 if (geo->near_copies > 1 && !pending) 826 new_distance = 0; 827 828 /* for far > 1 always use the lowest address */ 829 else if (geo->far_copies > 1) 830 new_distance = r10_bio->devs[slot].addr; 831 else 832 new_distance = abs(r10_bio->devs[slot].addr - 833 conf->mirrors[disk].head_position); 834 835 if (new_distance < best_dist) { 836 best_dist = new_distance; 837 best_dist_slot = slot; 838 best_dist_rdev = rdev; 839 } 840 } 841 if (slot >= conf->copies) { 842 if (has_nonrot_disk) { 843 slot = best_pending_slot; 844 rdev = best_pending_rdev; 845 } else { 846 slot = best_dist_slot; 847 rdev = best_dist_rdev; 848 } 849 } 850 851 if (slot >= 0) { 852 atomic_inc(&rdev->nr_pending); 853 r10_bio->read_slot = slot; 854 } else 855 rdev = NULL; 856 *max_sectors = best_good_sectors; 857 858 return rdev; 859} 860 861static void flush_pending_writes(struct r10conf *conf) 862{ 863 /* Any writes that have been queued but are awaiting 864 * bitmap updates get flushed here. 865 */ 866 spin_lock_irq(&conf->device_lock); 867 868 if (conf->pending_bio_list.head) { 869 struct blk_plug plug; 870 struct bio *bio; 871 872 bio = bio_list_get(&conf->pending_bio_list); 873 spin_unlock_irq(&conf->device_lock); 874 875 /* 876 * As this is called in a wait_event() loop (see freeze_array), 877 * current->state might be TASK_UNINTERRUPTIBLE which will 878 * cause a warning when we prepare to wait again. As it is 879 * rare that this path is taken, it is perfectly safe to force 880 * us to go around the wait_event() loop again, so the warning 881 * is a false-positive. Silence the warning by resetting 882 * thread state 883 */ 884 __set_current_state(TASK_RUNNING); 885 886 blk_start_plug(&plug); 887 raid1_prepare_flush_writes(conf->mddev); 888 wake_up(&conf->wait_barrier); 889 890 while (bio) { /* submit pending writes */ 891 struct bio *next = bio->bi_next; 892 893 raid1_submit_write(bio); 894 bio = next; 895 cond_resched(); 896 } 897 blk_finish_plug(&plug); 898 } else 899 spin_unlock_irq(&conf->device_lock); 900} 901 902/* Barriers.... 903 * Sometimes we need to suspend IO while we do something else, 904 * either some resync/recovery, or reconfigure the array. 905 * To do this we raise a 'barrier'. 906 * The 'barrier' is a counter that can be raised multiple times 907 * to count how many activities are happening which preclude 908 * normal IO. 909 * We can only raise the barrier if there is no pending IO. 910 * i.e. if nr_pending == 0. 911 * We choose only to raise the barrier if no-one is waiting for the 912 * barrier to go down. This means that as soon as an IO request 913 * is ready, no other operations which require a barrier will start 914 * until the IO request has had a chance. 915 * 916 * So: regular IO calls 'wait_barrier'. When that returns there 917 * is no backgroup IO happening, It must arrange to call 918 * allow_barrier when it has finished its IO. 919 * backgroup IO calls must call raise_barrier. Once that returns 920 * there is no normal IO happeing. It must arrange to call 921 * lower_barrier when the particular background IO completes. 922 */ 923 924static void raise_barrier(struct r10conf *conf, int force) 925{ 926 write_seqlock_irq(&conf->resync_lock); 927 928 if (WARN_ON_ONCE(force && !conf->barrier)) 929 force = false; 930 931 /* Wait until no block IO is waiting (unless 'force') */ 932 wait_event_barrier(conf, force || !conf->nr_waiting); 933 934 /* block any new IO from starting */ 935 WRITE_ONCE(conf->barrier, conf->barrier + 1); 936 937 /* Now wait for all pending IO to complete */ 938 wait_event_barrier(conf, !atomic_read(&conf->nr_pending) && 939 conf->barrier < RESYNC_DEPTH); 940 941 write_sequnlock_irq(&conf->resync_lock); 942} 943 944static void lower_barrier(struct r10conf *conf) 945{ 946 unsigned long flags; 947 948 write_seqlock_irqsave(&conf->resync_lock, flags); 949 WRITE_ONCE(conf->barrier, conf->barrier - 1); 950 write_sequnlock_irqrestore(&conf->resync_lock, flags); 951 wake_up(&conf->wait_barrier); 952} 953 954static bool stop_waiting_barrier(struct r10conf *conf) 955{ 956 struct bio_list *bio_list = current->bio_list; 957 struct md_thread *thread; 958 959 /* barrier is dropped */ 960 if (!conf->barrier) 961 return true; 962 963 /* 964 * If there are already pending requests (preventing the barrier from 965 * rising completely), and the pre-process bio queue isn't empty, then 966 * don't wait, as we need to empty that queue to get the nr_pending 967 * count down. 968 */ 969 if (atomic_read(&conf->nr_pending) && bio_list && 970 (!bio_list_empty(&bio_list[0]) || !bio_list_empty(&bio_list[1]))) 971 return true; 972 973 /* daemon thread must exist while handling io */ 974 thread = rcu_dereference_protected(conf->mddev->thread, true); 975 /* 976 * move on if io is issued from raid10d(), nr_pending is not released 977 * from original io(see handle_read_error()). All raise barrier is 978 * blocked until this io is done. 979 */ 980 if (thread->tsk == current) { 981 WARN_ON_ONCE(atomic_read(&conf->nr_pending) == 0); 982 return true; 983 } 984 985 return false; 986} 987 988static bool wait_barrier_nolock(struct r10conf *conf) 989{ 990 unsigned int seq = read_seqbegin(&conf->resync_lock); 991 992 if (READ_ONCE(conf->barrier)) 993 return false; 994 995 atomic_inc(&conf->nr_pending); 996 if (!read_seqretry(&conf->resync_lock, seq)) 997 return true; 998 999 if (atomic_dec_and_test(&conf->nr_pending)) 1000 wake_up_barrier(conf); 1001 1002 return false; 1003} 1004 1005static bool wait_barrier(struct r10conf *conf, bool nowait) 1006{ 1007 bool ret = true; 1008 1009 if (wait_barrier_nolock(conf)) 1010 return true; 1011 1012 write_seqlock_irq(&conf->resync_lock); 1013 if (conf->barrier) { 1014 /* Return false when nowait flag is set */ 1015 if (nowait) { 1016 ret = false; 1017 } else { 1018 conf->nr_waiting++; 1019 mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); 1020 wait_event_barrier(conf, stop_waiting_barrier(conf)); 1021 conf->nr_waiting--; 1022 } 1023 if (!conf->nr_waiting) 1024 wake_up(&conf->wait_barrier); 1025 } 1026 /* Only increment nr_pending when we wait */ 1027 if (ret) 1028 atomic_inc(&conf->nr_pending); 1029 write_sequnlock_irq(&conf->resync_lock); 1030 return ret; 1031} 1032 1033static void allow_barrier(struct r10conf *conf) 1034{ 1035 if ((atomic_dec_and_test(&conf->nr_pending)) || 1036 (conf->array_freeze_pending)) 1037 wake_up_barrier(conf); 1038} 1039 1040static void freeze_array(struct r10conf *conf, int extra) 1041{ 1042 /* stop syncio and normal IO and wait for everything to 1043 * go quiet. 1044 * We increment barrier and nr_waiting, and then 1045 * wait until nr_pending match nr_queued+extra 1046 * This is called in the context of one normal IO request 1047 * that has failed. Thus any sync request that might be pending 1048 * will be blocked by nr_pending, and we need to wait for 1049 * pending IO requests to complete or be queued for re-try. 1050 * Thus the number queued (nr_queued) plus this request (extra) 1051 * must match the number of pending IOs (nr_pending) before 1052 * we continue. 1053 */ 1054 write_seqlock_irq(&conf->resync_lock); 1055 conf->array_freeze_pending++; 1056 WRITE_ONCE(conf->barrier, conf->barrier + 1); 1057 conf->nr_waiting++; 1058 wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) == 1059 conf->nr_queued + extra, flush_pending_writes(conf)); 1060 conf->array_freeze_pending--; 1061 write_sequnlock_irq(&conf->resync_lock); 1062} 1063 1064static void unfreeze_array(struct r10conf *conf) 1065{ 1066 /* reverse the effect of the freeze */ 1067 write_seqlock_irq(&conf->resync_lock); 1068 WRITE_ONCE(conf->barrier, conf->barrier - 1); 1069 conf->nr_waiting--; 1070 wake_up(&conf->wait_barrier); 1071 write_sequnlock_irq(&conf->resync_lock); 1072} 1073 1074static sector_t choose_data_offset(struct r10bio *r10_bio, 1075 struct md_rdev *rdev) 1076{ 1077 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) || 1078 test_bit(R10BIO_Previous, &r10_bio->state)) 1079 return rdev->data_offset; 1080 else 1081 return rdev->new_data_offset; 1082} 1083 1084static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) 1085{ 1086 struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, cb); 1087 struct mddev *mddev = plug->cb.data; 1088 struct r10conf *conf = mddev->private; 1089 struct bio *bio; 1090 1091 if (from_schedule) { 1092 spin_lock_irq(&conf->device_lock); 1093 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1094 spin_unlock_irq(&conf->device_lock); 1095 wake_up_barrier(conf); 1096 md_wakeup_thread(mddev->thread); 1097 kfree(plug); 1098 return; 1099 } 1100 1101 /* we aren't scheduling, so we can do the write-out directly. */ 1102 bio = bio_list_get(&plug->pending); 1103 raid1_prepare_flush_writes(mddev); 1104 wake_up_barrier(conf); 1105 1106 while (bio) { /* submit pending writes */ 1107 struct bio *next = bio->bi_next; 1108 1109 raid1_submit_write(bio); 1110 bio = next; 1111 cond_resched(); 1112 } 1113 kfree(plug); 1114} 1115 1116/* 1117 * 1. Register the new request and wait if the reconstruction thread has put 1118 * up a bar for new requests. Continue immediately if no resync is active 1119 * currently. 1120 * 2. If IO spans the reshape position. Need to wait for reshape to pass. 1121 */ 1122static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, 1123 struct bio *bio, sector_t sectors) 1124{ 1125 /* Bail out if REQ_NOWAIT is set for the bio */ 1126 if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { 1127 bio_wouldblock_error(bio); 1128 return false; 1129 } 1130 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1131 bio->bi_iter.bi_sector < conf->reshape_progress && 1132 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { 1133 allow_barrier(conf); 1134 if (bio->bi_opf & REQ_NOWAIT) { 1135 bio_wouldblock_error(bio); 1136 return false; 1137 } 1138 mddev_add_trace_msg(conf->mddev, "raid10 wait reshape"); 1139 wait_event(conf->wait_barrier, 1140 conf->reshape_progress <= bio->bi_iter.bi_sector || 1141 conf->reshape_progress >= bio->bi_iter.bi_sector + 1142 sectors); 1143 wait_barrier(conf, false); 1144 } 1145 return true; 1146} 1147 1148static void raid10_read_request(struct mddev *mddev, struct bio *bio, 1149 struct r10bio *r10_bio, bool io_accounting) 1150{ 1151 struct r10conf *conf = mddev->private; 1152 struct bio *read_bio; 1153 int max_sectors; 1154 struct md_rdev *rdev; 1155 char b[BDEVNAME_SIZE]; 1156 int slot = r10_bio->read_slot; 1157 struct md_rdev *err_rdev = NULL; 1158 gfp_t gfp = GFP_NOIO; 1159 1160 if (slot >= 0 && r10_bio->devs[slot].rdev) { 1161 /* 1162 * This is an error retry, but we cannot 1163 * safely dereference the rdev in the r10_bio, 1164 * we must use the one in conf. 1165 * If it has already been disconnected (unlikely) 1166 * we lose the device name in error messages. 1167 */ 1168 int disk; 1169 /* 1170 * As we are blocking raid10, it is a little safer to 1171 * use __GFP_HIGH. 1172 */ 1173 gfp = GFP_NOIO | __GFP_HIGH; 1174 1175 disk = r10_bio->devs[slot].devnum; 1176 err_rdev = conf->mirrors[disk].rdev; 1177 if (err_rdev) 1178 snprintf(b, sizeof(b), "%pg", err_rdev->bdev); 1179 else { 1180 strcpy(b, "???"); 1181 /* This never gets dereferenced */ 1182 err_rdev = r10_bio->devs[slot].rdev; 1183 } 1184 } 1185 1186 if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { 1187 raid_end_bio_io(r10_bio); 1188 return; 1189 } 1190 1191 rdev = read_balance(conf, r10_bio, &max_sectors); 1192 if (!rdev) { 1193 if (err_rdev) { 1194 pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n", 1195 mdname(mddev), b, 1196 (unsigned long long)r10_bio->sector); 1197 } 1198 raid_end_bio_io(r10_bio); 1199 return; 1200 } 1201 if (err_rdev) 1202 pr_err_ratelimited("md/raid10:%s: %pg: redirecting sector %llu to another mirror\n", 1203 mdname(mddev), 1204 rdev->bdev, 1205 (unsigned long long)r10_bio->sector); 1206 if (max_sectors < bio_sectors(bio)) { 1207 allow_barrier(conf); 1208 bio = bio_submit_split_bioset(bio, max_sectors, 1209 &conf->bio_split); 1210 wait_barrier(conf, false); 1211 if (!bio) { 1212 set_bit(R10BIO_Returned, &r10_bio->state); 1213 goto err_handle; 1214 } 1215 1216 r10_bio->master_bio = bio; 1217 r10_bio->sectors = max_sectors; 1218 } 1219 slot = r10_bio->read_slot; 1220 1221 if (io_accounting) { 1222 md_account_bio(mddev, &bio); 1223 r10_bio->master_bio = bio; 1224 } 1225 read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set); 1226 read_bio->bi_opf &= ~REQ_NOWAIT; 1227 1228 r10_bio->devs[slot].bio = read_bio; 1229 r10_bio->devs[slot].rdev = rdev; 1230 1231 read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + 1232 choose_data_offset(r10_bio, rdev); 1233 read_bio->bi_end_io = raid10_end_read_request; 1234 if (test_bit(FailFast, &rdev->flags) && 1235 test_bit(R10BIO_FailFast, &r10_bio->state)) 1236 read_bio->bi_opf |= MD_FAILFAST; 1237 read_bio->bi_private = r10_bio; 1238 mddev_trace_remap(mddev, read_bio, r10_bio->sector); 1239 submit_bio_noacct(read_bio); 1240 return; 1241err_handle: 1242 atomic_dec(&rdev->nr_pending); 1243 raid_end_bio_io(r10_bio); 1244} 1245 1246static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, 1247 struct bio *bio, bool replacement, 1248 int n_copy) 1249{ 1250 unsigned long flags; 1251 struct r10conf *conf = mddev->private; 1252 struct md_rdev *rdev; 1253 int devnum = r10_bio->devs[n_copy].devnum; 1254 struct bio *mbio; 1255 1256 rdev = replacement ? conf->mirrors[devnum].replacement : 1257 conf->mirrors[devnum].rdev; 1258 1259 mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set); 1260 mbio->bi_opf &= ~REQ_NOWAIT; 1261 if (replacement) 1262 r10_bio->devs[n_copy].repl_bio = mbio; 1263 else 1264 r10_bio->devs[n_copy].bio = mbio; 1265 1266 mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr + 1267 choose_data_offset(r10_bio, rdev)); 1268 mbio->bi_end_io = raid10_end_write_request; 1269 if (!replacement && test_bit(FailFast, 1270 &conf->mirrors[devnum].rdev->flags) 1271 && enough(conf, devnum)) 1272 mbio->bi_opf |= MD_FAILFAST; 1273 mbio->bi_private = r10_bio; 1274 mddev_trace_remap(mddev, mbio, r10_bio->sector); 1275 /* flush_pending_writes() needs access to the rdev so...*/ 1276 mbio->bi_bdev = (void *)rdev; 1277 1278 atomic_inc(&r10_bio->remaining); 1279 1280 if (!raid1_add_bio_to_plug(mddev, mbio, raid10_unplug, conf->copies)) { 1281 spin_lock_irqsave(&conf->device_lock, flags); 1282 bio_list_add(&conf->pending_bio_list, mbio); 1283 spin_unlock_irqrestore(&conf->device_lock, flags); 1284 md_wakeup_thread(mddev->thread); 1285 } 1286} 1287 1288static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) 1289{ 1290 struct r10conf *conf = mddev->private; 1291 struct md_rdev *blocked_rdev; 1292 int i; 1293 1294retry_wait: 1295 blocked_rdev = NULL; 1296 for (i = 0; i < conf->copies; i++) { 1297 struct md_rdev *rdev, *rrdev; 1298 1299 rdev = conf->mirrors[i].rdev; 1300 if (rdev) { 1301 sector_t dev_sector = r10_bio->devs[i].addr; 1302 1303 /* 1304 * Discard request doesn't care the write result 1305 * so it doesn't need to wait blocked disk here. 1306 */ 1307 if (test_bit(WriteErrorSeen, &rdev->flags) && 1308 r10_bio->sectors && 1309 rdev_has_badblock(rdev, dev_sector, 1310 r10_bio->sectors) < 0) 1311 /* 1312 * Mustn't write here until the bad 1313 * block is acknowledged 1314 */ 1315 set_bit(BlockedBadBlocks, &rdev->flags); 1316 1317 if (rdev_blocked(rdev)) { 1318 blocked_rdev = rdev; 1319 atomic_inc(&rdev->nr_pending); 1320 break; 1321 } 1322 } 1323 1324 rrdev = conf->mirrors[i].replacement; 1325 if (rrdev && rdev_blocked(rrdev)) { 1326 atomic_inc(&rrdev->nr_pending); 1327 blocked_rdev = rrdev; 1328 break; 1329 } 1330 } 1331 1332 if (unlikely(blocked_rdev)) { 1333 /* Have to wait for this device to get unblocked, then retry */ 1334 allow_barrier(conf); 1335 mddev_add_trace_msg(conf->mddev, 1336 "raid10 %s wait rdev %d blocked", 1337 __func__, blocked_rdev->raid_disk); 1338 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1339 wait_barrier(conf, false); 1340 goto retry_wait; 1341 } 1342} 1343 1344static void raid10_write_request(struct mddev *mddev, struct bio *bio, 1345 struct r10bio *r10_bio) 1346{ 1347 struct r10conf *conf = mddev->private; 1348 int i, k; 1349 sector_t sectors; 1350 int max_sectors; 1351 1352 if ((mddev_is_clustered(mddev) && 1353 mddev->cluster_ops->area_resyncing(mddev, WRITE, 1354 bio->bi_iter.bi_sector, 1355 bio_end_sector(bio)))) { 1356 DEFINE_WAIT(w); 1357 /* Bail out if REQ_NOWAIT is set for the bio */ 1358 if (bio->bi_opf & REQ_NOWAIT) { 1359 bio_wouldblock_error(bio); 1360 return; 1361 } 1362 for (;;) { 1363 prepare_to_wait(&conf->wait_barrier, 1364 &w, TASK_IDLE); 1365 if (!mddev->cluster_ops->area_resyncing(mddev, WRITE, 1366 bio->bi_iter.bi_sector, bio_end_sector(bio))) 1367 break; 1368 schedule(); 1369 } 1370 finish_wait(&conf->wait_barrier, &w); 1371 } 1372 1373 sectors = r10_bio->sectors; 1374 if (!regular_request_wait(mddev, conf, bio, sectors)) { 1375 raid_end_bio_io(r10_bio); 1376 return; 1377 } 1378 1379 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1380 (mddev->reshape_backwards 1381 ? (bio->bi_iter.bi_sector < conf->reshape_safe && 1382 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) 1383 : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe && 1384 bio->bi_iter.bi_sector < conf->reshape_progress))) { 1385 /* Need to update reshape_position in metadata */ 1386 mddev->reshape_position = conf->reshape_progress; 1387 set_mask_bits(&mddev->sb_flags, 0, 1388 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1389 md_wakeup_thread(mddev->thread); 1390 if (bio->bi_opf & REQ_NOWAIT) { 1391 allow_barrier(conf); 1392 bio_wouldblock_error(bio); 1393 return; 1394 } 1395 mddev_add_trace_msg(conf->mddev, 1396 "raid10 wait reshape metadata"); 1397 wait_event(mddev->sb_wait, 1398 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)); 1399 1400 conf->reshape_safe = mddev->reshape_position; 1401 } 1402 1403 /* first select target devices under rcu_lock and 1404 * inc refcount on their rdev. Record them by setting 1405 * bios[x] to bio 1406 * If there are known/acknowledged bad blocks on any device 1407 * on which we have seen a write error, we want to avoid 1408 * writing to those blocks. This potentially requires several 1409 * writes to write around the bad blocks. Each set of writes 1410 * gets its own r10_bio with a set of bios attached. 1411 */ 1412 1413 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ 1414 raid10_find_phys(conf, r10_bio); 1415 1416 wait_blocked_dev(mddev, r10_bio); 1417 1418 max_sectors = r10_bio->sectors; 1419 1420 for (i = 0; i < conf->copies; i++) { 1421 int d = r10_bio->devs[i].devnum; 1422 struct md_rdev *rdev, *rrdev; 1423 1424 rdev = conf->mirrors[d].rdev; 1425 rrdev = conf->mirrors[d].replacement; 1426 if (rdev && (test_bit(Faulty, &rdev->flags))) 1427 rdev = NULL; 1428 if (rrdev && (test_bit(Faulty, &rrdev->flags))) 1429 rrdev = NULL; 1430 1431 r10_bio->devs[i].bio = NULL; 1432 r10_bio->devs[i].repl_bio = NULL; 1433 1434 if (!rdev && !rrdev) 1435 continue; 1436 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) { 1437 sector_t first_bad; 1438 sector_t dev_sector = r10_bio->devs[i].addr; 1439 sector_t bad_sectors; 1440 int is_bad; 1441 1442 is_bad = is_badblock(rdev, dev_sector, max_sectors, 1443 &first_bad, &bad_sectors); 1444 if (is_bad && first_bad <= dev_sector) { 1445 /* Cannot write here at all */ 1446 bad_sectors -= (dev_sector - first_bad); 1447 if (bad_sectors < max_sectors) 1448 /* Mustn't write more than bad_sectors 1449 * to other devices yet 1450 */ 1451 max_sectors = bad_sectors; 1452 continue; 1453 } 1454 if (is_bad) { 1455 int good_sectors; 1456 1457 /* 1458 * We cannot atomically write this, so just 1459 * error in that case. It could be possible to 1460 * atomically write other mirrors, but the 1461 * complexity of supporting that is not worth 1462 * the benefit. 1463 */ 1464 if (bio->bi_opf & REQ_ATOMIC) 1465 goto err_handle; 1466 1467 good_sectors = first_bad - dev_sector; 1468 if (good_sectors < max_sectors) 1469 max_sectors = good_sectors; 1470 } 1471 } 1472 if (rdev) { 1473 r10_bio->devs[i].bio = bio; 1474 atomic_inc(&rdev->nr_pending); 1475 } 1476 if (rrdev) { 1477 r10_bio->devs[i].repl_bio = bio; 1478 atomic_inc(&rrdev->nr_pending); 1479 } 1480 } 1481 1482 if (max_sectors < r10_bio->sectors) 1483 r10_bio->sectors = max_sectors; 1484 1485 if (r10_bio->sectors < bio_sectors(bio)) { 1486 allow_barrier(conf); 1487 bio = bio_submit_split_bioset(bio, r10_bio->sectors, 1488 &conf->bio_split); 1489 wait_barrier(conf, false); 1490 if (!bio) { 1491 set_bit(R10BIO_Returned, &r10_bio->state); 1492 goto err_handle; 1493 } 1494 1495 r10_bio->master_bio = bio; 1496 } 1497 1498 md_account_bio(mddev, &bio); 1499 r10_bio->master_bio = bio; 1500 atomic_set(&r10_bio->remaining, 1); 1501 1502 for (i = 0; i < conf->copies; i++) { 1503 if (r10_bio->devs[i].bio) 1504 raid10_write_one_disk(mddev, r10_bio, bio, false, i); 1505 if (r10_bio->devs[i].repl_bio) 1506 raid10_write_one_disk(mddev, r10_bio, bio, true, i); 1507 } 1508 one_write_done(r10_bio); 1509 return; 1510err_handle: 1511 for (k = 0; k < i; k++) { 1512 int d = r10_bio->devs[k].devnum; 1513 struct md_rdev *rdev = conf->mirrors[d].rdev; 1514 struct md_rdev *rrdev = conf->mirrors[d].replacement; 1515 1516 if (r10_bio->devs[k].bio) { 1517 rdev_dec_pending(rdev, mddev); 1518 r10_bio->devs[k].bio = NULL; 1519 } 1520 if (r10_bio->devs[k].repl_bio) { 1521 rdev_dec_pending(rrdev, mddev); 1522 r10_bio->devs[k].repl_bio = NULL; 1523 } 1524 } 1525 1526 raid_end_bio_io(r10_bio); 1527} 1528 1529static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) 1530{ 1531 struct r10conf *conf = mddev->private; 1532 struct r10bio *r10_bio; 1533 1534 r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); 1535 1536 r10_bio->master_bio = bio; 1537 r10_bio->sectors = sectors; 1538 1539 r10_bio->mddev = mddev; 1540 r10_bio->sector = bio->bi_iter.bi_sector; 1541 r10_bio->state = 0; 1542 r10_bio->read_slot = -1; 1543 memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * 1544 conf->geo.raid_disks); 1545 1546 if (bio_data_dir(bio) == READ) 1547 raid10_read_request(mddev, bio, r10_bio, true); 1548 else 1549 raid10_write_request(mddev, bio, r10_bio); 1550} 1551 1552static void raid_end_discard_bio(struct r10bio *r10bio) 1553{ 1554 struct r10conf *conf = r10bio->mddev->private; 1555 struct r10bio *first_r10bio; 1556 1557 while (atomic_dec_and_test(&r10bio->remaining)) { 1558 1559 allow_barrier(conf); 1560 1561 if (!test_bit(R10BIO_Discard, &r10bio->state)) { 1562 first_r10bio = (struct r10bio *)r10bio->master_bio; 1563 free_r10bio(r10bio); 1564 r10bio = first_r10bio; 1565 } else { 1566 md_write_end(r10bio->mddev); 1567 bio_endio(r10bio->master_bio); 1568 free_r10bio(r10bio); 1569 break; 1570 } 1571 } 1572} 1573 1574static void raid10_end_discard_request(struct bio *bio) 1575{ 1576 struct r10bio *r10_bio = bio->bi_private; 1577 struct r10conf *conf = r10_bio->mddev->private; 1578 struct md_rdev *rdev = NULL; 1579 int dev; 1580 int slot, repl; 1581 1582 /* 1583 * We don't care the return value of discard bio 1584 */ 1585 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 1586 set_bit(R10BIO_Uptodate, &r10_bio->state); 1587 1588 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1589 rdev = repl ? conf->mirrors[dev].replacement : 1590 conf->mirrors[dev].rdev; 1591 1592 raid_end_discard_bio(r10_bio); 1593 rdev_dec_pending(rdev, conf->mddev); 1594} 1595 1596/* 1597 * There are some limitations to handle discard bio 1598 * 1st, the discard size is bigger than stripe_size*2. 1599 * 2st, if the discard bio spans reshape progress, we use the old way to 1600 * handle discard bio 1601 */ 1602static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) 1603{ 1604 struct r10conf *conf = mddev->private; 1605 struct geom *geo = &conf->geo; 1606 int far_copies = geo->far_copies; 1607 bool first_copy = true; 1608 struct r10bio *r10_bio, *first_r10bio; 1609 struct bio *split; 1610 int disk; 1611 sector_t chunk; 1612 unsigned int stripe_size; 1613 unsigned int stripe_data_disks; 1614 sector_t split_size; 1615 sector_t bio_start, bio_end; 1616 sector_t first_stripe_index, last_stripe_index; 1617 sector_t start_disk_offset; 1618 unsigned int start_disk_index; 1619 sector_t end_disk_offset; 1620 unsigned int end_disk_index; 1621 unsigned int remainder; 1622 1623 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 1624 return -EAGAIN; 1625 1626 if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { 1627 bio_wouldblock_error(bio); 1628 return 0; 1629 } 1630 1631 /* 1632 * Check reshape again to avoid reshape happens after checking 1633 * MD_RECOVERY_RESHAPE and before wait_barrier 1634 */ 1635 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 1636 goto out; 1637 1638 if (geo->near_copies) 1639 stripe_data_disks = geo->raid_disks / geo->near_copies + 1640 geo->raid_disks % geo->near_copies; 1641 else 1642 stripe_data_disks = geo->raid_disks; 1643 1644 stripe_size = stripe_data_disks << geo->chunk_shift; 1645 1646 bio_start = bio->bi_iter.bi_sector; 1647 bio_end = bio_end_sector(bio); 1648 1649 /* 1650 * Maybe one discard bio is smaller than strip size or across one 1651 * stripe and discard region is larger than one stripe size. For far 1652 * offset layout, if the discard region is not aligned with stripe 1653 * size, there is hole when we submit discard bio to member disk. 1654 * For simplicity, we only handle discard bio which discard region 1655 * is bigger than stripe_size * 2 1656 */ 1657 if (bio_sectors(bio) < stripe_size*2) 1658 goto out; 1659 1660 /* 1661 * Keep bio aligned with strip size. 1662 */ 1663 div_u64_rem(bio_start, stripe_size, &remainder); 1664 if (remainder) { 1665 split_size = stripe_size - remainder; 1666 split = bio_split(bio, split_size, GFP_NOIO, &conf->bio_split); 1667 if (IS_ERR(split)) { 1668 bio->bi_status = errno_to_blk_status(PTR_ERR(split)); 1669 bio_endio(bio); 1670 return 0; 1671 } 1672 1673 bio_chain(split, bio); 1674 trace_block_split(split, bio->bi_iter.bi_sector); 1675 allow_barrier(conf); 1676 /* Resend the fist split part */ 1677 submit_bio_noacct(split); 1678 wait_barrier(conf, false); 1679 } 1680 div_u64_rem(bio_end, stripe_size, &remainder); 1681 if (remainder) { 1682 split_size = bio_sectors(bio) - remainder; 1683 split = bio_split(bio, split_size, GFP_NOIO, &conf->bio_split); 1684 if (IS_ERR(split)) { 1685 bio->bi_status = errno_to_blk_status(PTR_ERR(split)); 1686 bio_endio(bio); 1687 return 0; 1688 } 1689 1690 bio_chain(split, bio); 1691 trace_block_split(split, bio->bi_iter.bi_sector); 1692 allow_barrier(conf); 1693 /* Resend the second split part */ 1694 submit_bio_noacct(bio); 1695 bio = split; 1696 wait_barrier(conf, false); 1697 } 1698 1699 bio_start = bio->bi_iter.bi_sector; 1700 bio_end = bio_end_sector(bio); 1701 1702 /* 1703 * Raid10 uses chunk as the unit to store data. It's similar like raid0. 1704 * One stripe contains the chunks from all member disk (one chunk from 1705 * one disk at the same HBA address). For layout detail, see 'man md 4' 1706 */ 1707 chunk = bio_start >> geo->chunk_shift; 1708 chunk *= geo->near_copies; 1709 first_stripe_index = chunk; 1710 start_disk_index = sector_div(first_stripe_index, geo->raid_disks); 1711 if (geo->far_offset) 1712 first_stripe_index *= geo->far_copies; 1713 start_disk_offset = (bio_start & geo->chunk_mask) + 1714 (first_stripe_index << geo->chunk_shift); 1715 1716 chunk = bio_end >> geo->chunk_shift; 1717 chunk *= geo->near_copies; 1718 last_stripe_index = chunk; 1719 end_disk_index = sector_div(last_stripe_index, geo->raid_disks); 1720 if (geo->far_offset) 1721 last_stripe_index *= geo->far_copies; 1722 end_disk_offset = (bio_end & geo->chunk_mask) + 1723 (last_stripe_index << geo->chunk_shift); 1724 1725retry_discard: 1726 r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); 1727 r10_bio->mddev = mddev; 1728 r10_bio->state = 0; 1729 r10_bio->sectors = 0; 1730 memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks); 1731 wait_blocked_dev(mddev, r10_bio); 1732 1733 /* 1734 * For far layout it needs more than one r10bio to cover all regions. 1735 * Inspired by raid10_sync_request, we can use the first r10bio->master_bio 1736 * to record the discard bio. Other r10bio->master_bio record the first 1737 * r10bio. The first r10bio only release after all other r10bios finish. 1738 * The discard bio returns only first r10bio finishes 1739 */ 1740 if (first_copy) { 1741 md_account_bio(mddev, &bio); 1742 r10_bio->master_bio = bio; 1743 set_bit(R10BIO_Discard, &r10_bio->state); 1744 first_copy = false; 1745 first_r10bio = r10_bio; 1746 } else 1747 r10_bio->master_bio = (struct bio *)first_r10bio; 1748 1749 /* 1750 * first select target devices under rcu_lock and 1751 * inc refcount on their rdev. Record them by setting 1752 * bios[x] to bio 1753 */ 1754 for (disk = 0; disk < geo->raid_disks; disk++) { 1755 struct md_rdev *rdev, *rrdev; 1756 1757 rdev = conf->mirrors[disk].rdev; 1758 rrdev = conf->mirrors[disk].replacement; 1759 r10_bio->devs[disk].bio = NULL; 1760 r10_bio->devs[disk].repl_bio = NULL; 1761 1762 if (rdev && (test_bit(Faulty, &rdev->flags))) 1763 rdev = NULL; 1764 if (rrdev && (test_bit(Faulty, &rrdev->flags))) 1765 rrdev = NULL; 1766 if (!rdev && !rrdev) 1767 continue; 1768 1769 if (rdev) { 1770 r10_bio->devs[disk].bio = bio; 1771 atomic_inc(&rdev->nr_pending); 1772 } 1773 if (rrdev) { 1774 r10_bio->devs[disk].repl_bio = bio; 1775 atomic_inc(&rrdev->nr_pending); 1776 } 1777 } 1778 1779 atomic_set(&r10_bio->remaining, 1); 1780 for (disk = 0; disk < geo->raid_disks; disk++) { 1781 sector_t dev_start, dev_end; 1782 struct bio *mbio, *rbio = NULL; 1783 1784 /* 1785 * Now start to calculate the start and end address for each disk. 1786 * The space between dev_start and dev_end is the discard region. 1787 * 1788 * For dev_start, it needs to consider three conditions: 1789 * 1st, the disk is before start_disk, you can imagine the disk in 1790 * the next stripe. So the dev_start is the start address of next 1791 * stripe. 1792 * 2st, the disk is after start_disk, it means the disk is at the 1793 * same stripe of first disk 1794 * 3st, the first disk itself, we can use start_disk_offset directly 1795 */ 1796 if (disk < start_disk_index) 1797 dev_start = (first_stripe_index + 1) * mddev->chunk_sectors; 1798 else if (disk > start_disk_index) 1799 dev_start = first_stripe_index * mddev->chunk_sectors; 1800 else 1801 dev_start = start_disk_offset; 1802 1803 if (disk < end_disk_index) 1804 dev_end = (last_stripe_index + 1) * mddev->chunk_sectors; 1805 else if (disk > end_disk_index) 1806 dev_end = last_stripe_index * mddev->chunk_sectors; 1807 else 1808 dev_end = end_disk_offset; 1809 1810 /* 1811 * It only handles discard bio which size is >= stripe size, so 1812 * dev_end > dev_start all the time. 1813 * It doesn't need to use rcu lock to get rdev here. We already 1814 * add rdev->nr_pending in the first loop. 1815 */ 1816 if (r10_bio->devs[disk].bio) { 1817 struct md_rdev *rdev = conf->mirrors[disk].rdev; 1818 mbio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, 1819 &mddev->bio_set); 1820 mbio->bi_end_io = raid10_end_discard_request; 1821 mbio->bi_private = r10_bio; 1822 r10_bio->devs[disk].bio = mbio; 1823 r10_bio->devs[disk].devnum = disk; 1824 atomic_inc(&r10_bio->remaining); 1825 md_submit_discard_bio(mddev, rdev, mbio, 1826 dev_start + choose_data_offset(r10_bio, rdev), 1827 dev_end - dev_start); 1828 bio_endio(mbio); 1829 } 1830 if (r10_bio->devs[disk].repl_bio) { 1831 struct md_rdev *rrdev = conf->mirrors[disk].replacement; 1832 rbio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, 1833 &mddev->bio_set); 1834 rbio->bi_end_io = raid10_end_discard_request; 1835 rbio->bi_private = r10_bio; 1836 r10_bio->devs[disk].repl_bio = rbio; 1837 r10_bio->devs[disk].devnum = disk; 1838 atomic_inc(&r10_bio->remaining); 1839 md_submit_discard_bio(mddev, rrdev, rbio, 1840 dev_start + choose_data_offset(r10_bio, rrdev), 1841 dev_end - dev_start); 1842 bio_endio(rbio); 1843 } 1844 } 1845 1846 if (!geo->far_offset && --far_copies) { 1847 first_stripe_index += geo->stride >> geo->chunk_shift; 1848 start_disk_offset += geo->stride; 1849 last_stripe_index += geo->stride >> geo->chunk_shift; 1850 end_disk_offset += geo->stride; 1851 atomic_inc(&first_r10bio->remaining); 1852 raid_end_discard_bio(r10_bio); 1853 wait_barrier(conf, false); 1854 goto retry_discard; 1855 } 1856 1857 raid_end_discard_bio(r10_bio); 1858 1859 return 0; 1860out: 1861 allow_barrier(conf); 1862 return -EAGAIN; 1863} 1864 1865static bool raid10_make_request(struct mddev *mddev, struct bio *bio) 1866{ 1867 struct r10conf *conf = mddev->private; 1868 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); 1869 int chunk_sects = chunk_mask + 1; 1870 int sectors = bio_sectors(bio); 1871 1872 if (unlikely(bio->bi_opf & REQ_PREFLUSH) 1873 && md_flush_request(mddev, bio)) 1874 return true; 1875 1876 md_write_start(mddev, bio); 1877 1878 if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) 1879 if (!raid10_handle_discard(mddev, bio)) 1880 return true; 1881 1882 /* 1883 * If this request crosses a chunk boundary, we need to split 1884 * it. 1885 */ 1886 if (unlikely((bio->bi_iter.bi_sector & chunk_mask) + 1887 sectors > chunk_sects 1888 && (conf->geo.near_copies < conf->geo.raid_disks 1889 || conf->prev.near_copies < 1890 conf->prev.raid_disks))) 1891 sectors = chunk_sects - 1892 (bio->bi_iter.bi_sector & 1893 (chunk_sects - 1)); 1894 __make_request(mddev, bio, sectors); 1895 1896 /* In case raid10d snuck in to freeze_array */ 1897 wake_up_barrier(conf); 1898 return true; 1899} 1900 1901static void raid10_status(struct seq_file *seq, struct mddev *mddev) 1902{ 1903 struct r10conf *conf = mddev->private; 1904 int i; 1905 1906 lockdep_assert_held(&mddev->lock); 1907 1908 if (conf->geo.near_copies < conf->geo.raid_disks) 1909 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1910 if (conf->geo.near_copies > 1) 1911 seq_printf(seq, " %d near-copies", conf->geo.near_copies); 1912 if (conf->geo.far_copies > 1) { 1913 if (conf->geo.far_offset) 1914 seq_printf(seq, " %d offset-copies", conf->geo.far_copies); 1915 else 1916 seq_printf(seq, " %d far-copies", conf->geo.far_copies); 1917 if (conf->geo.far_set_size != conf->geo.raid_disks) 1918 seq_printf(seq, " %d devices per set", conf->geo.far_set_size); 1919 } 1920 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1921 conf->geo.raid_disks - mddev->degraded); 1922 for (i = 0; i < conf->geo.raid_disks; i++) { 1923 struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev); 1924 1925 seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1926 } 1927 seq_printf(seq, "]"); 1928} 1929 1930/* check if there are enough drives for 1931 * every block to appear on atleast one. 1932 * Don't consider the device numbered 'ignore' 1933 * as we might be about to remove it. 1934 */ 1935static int _enough(struct r10conf *conf, int previous, int ignore) 1936{ 1937 int first = 0; 1938 int has_enough = 0; 1939 int disks, ncopies; 1940 if (previous) { 1941 disks = conf->prev.raid_disks; 1942 ncopies = conf->prev.near_copies; 1943 } else { 1944 disks = conf->geo.raid_disks; 1945 ncopies = conf->geo.near_copies; 1946 } 1947 1948 do { 1949 int n = conf->copies; 1950 int cnt = 0; 1951 int this = first; 1952 while (n--) { 1953 struct md_rdev *rdev; 1954 if (this != ignore && 1955 (rdev = conf->mirrors[this].rdev) && 1956 test_bit(In_sync, &rdev->flags)) 1957 cnt++; 1958 this = (this+1) % disks; 1959 } 1960 if (cnt == 0) 1961 goto out; 1962 first = (first + ncopies) % disks; 1963 } while (first != 0); 1964 has_enough = 1; 1965out: 1966 return has_enough; 1967} 1968 1969static int enough(struct r10conf *conf, int ignore) 1970{ 1971 /* when calling 'enough', both 'prev' and 'geo' must 1972 * be stable. 1973 * This is ensured if ->reconfig_mutex or ->device_lock 1974 * is held. 1975 */ 1976 return _enough(conf, 0, ignore) && 1977 _enough(conf, 1, ignore); 1978} 1979 1980/** 1981 * raid10_error() - RAID10 error handler. 1982 * @mddev: affected md device. 1983 * @rdev: member device to fail. 1984 * 1985 * The routine acknowledges &rdev failure and determines new @mddev state. 1986 * If it failed, then: 1987 * - &MD_BROKEN flag is set in &mddev->flags. 1988 * Otherwise, it must be degraded: 1989 * - recovery is interrupted. 1990 * - &mddev->degraded is bumped. 1991 * 1992 * @rdev is marked as &Faulty excluding case when array is failed and 1993 * &mddev->fail_last_dev is off. 1994 */ 1995static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) 1996{ 1997 struct r10conf *conf = mddev->private; 1998 unsigned long flags; 1999 2000 spin_lock_irqsave(&conf->device_lock, flags); 2001 2002 if (test_bit(In_sync, &rdev->flags) && !enough(conf, rdev->raid_disk)) { 2003 set_bit(MD_BROKEN, &mddev->flags); 2004 2005 if (!mddev->fail_last_dev) { 2006 spin_unlock_irqrestore(&conf->device_lock, flags); 2007 return; 2008 } 2009 } 2010 if (test_and_clear_bit(In_sync, &rdev->flags)) 2011 mddev->degraded++; 2012 2013 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2014 set_bit(Blocked, &rdev->flags); 2015 set_bit(Faulty, &rdev->flags); 2016 set_mask_bits(&mddev->sb_flags, 0, 2017 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 2018 spin_unlock_irqrestore(&conf->device_lock, flags); 2019 pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n" 2020 "md/raid10:%s: Operation continuing on %d devices.\n", 2021 mdname(mddev), rdev->bdev, 2022 mdname(mddev), conf->geo.raid_disks - mddev->degraded); 2023} 2024 2025static void print_conf(struct r10conf *conf) 2026{ 2027 int i; 2028 struct md_rdev *rdev; 2029 2030 pr_debug("RAID10 conf printout:\n"); 2031 if (!conf) { 2032 pr_debug("(!conf)\n"); 2033 return; 2034 } 2035 pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 2036 conf->geo.raid_disks); 2037 2038 lockdep_assert_held(&conf->mddev->reconfig_mutex); 2039 for (i = 0; i < conf->geo.raid_disks; i++) { 2040 rdev = conf->mirrors[i].rdev; 2041 if (rdev) 2042 pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n", 2043 i, !test_bit(In_sync, &rdev->flags), 2044 !test_bit(Faulty, &rdev->flags), 2045 rdev->bdev); 2046 } 2047} 2048 2049static void close_sync(struct r10conf *conf) 2050{ 2051 wait_barrier(conf, false); 2052 allow_barrier(conf); 2053 2054 mempool_exit(&conf->r10buf_pool); 2055} 2056 2057static int raid10_spare_active(struct mddev *mddev) 2058{ 2059 int i; 2060 struct r10conf *conf = mddev->private; 2061 struct raid10_info *tmp; 2062 int count = 0; 2063 unsigned long flags; 2064 2065 /* 2066 * Find all non-in_sync disks within the RAID10 configuration 2067 * and mark them in_sync 2068 */ 2069 for (i = 0; i < conf->geo.raid_disks; i++) { 2070 tmp = conf->mirrors + i; 2071 if (tmp->replacement 2072 && tmp->replacement->recovery_offset == MaxSector 2073 && !test_bit(Faulty, &tmp->replacement->flags) 2074 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 2075 /* Replacement has just become active */ 2076 if (!tmp->rdev 2077 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 2078 count++; 2079 if (tmp->rdev) { 2080 /* Replaced device not technically faulty, 2081 * but we need to be sure it gets removed 2082 * and never re-added. 2083 */ 2084 set_bit(Faulty, &tmp->rdev->flags); 2085 sysfs_notify_dirent_safe( 2086 tmp->rdev->sysfs_state); 2087 } 2088 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 2089 } else if (tmp->rdev 2090 && tmp->rdev->recovery_offset == MaxSector 2091 && !test_bit(Faulty, &tmp->rdev->flags) 2092 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 2093 count++; 2094 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 2095 } 2096 } 2097 spin_lock_irqsave(&conf->device_lock, flags); 2098 mddev->degraded -= count; 2099 spin_unlock_irqrestore(&conf->device_lock, flags); 2100 2101 print_conf(conf); 2102 return count; 2103} 2104 2105static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) 2106{ 2107 struct r10conf *conf = mddev->private; 2108 int err = -EEXIST; 2109 int mirror, repl_slot = -1; 2110 int first = 0; 2111 int last = conf->geo.raid_disks - 1; 2112 struct raid10_info *p; 2113 2114 if (mddev->resync_offset < MaxSector) 2115 /* only hot-add to in-sync arrays, as recovery is 2116 * very different from resync 2117 */ 2118 return -EBUSY; 2119 if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1)) 2120 return -EINVAL; 2121 2122 if (rdev->raid_disk >= 0) 2123 first = last = rdev->raid_disk; 2124 2125 if (rdev->saved_raid_disk >= first && 2126 rdev->saved_raid_disk < conf->geo.raid_disks && 2127 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 2128 mirror = rdev->saved_raid_disk; 2129 else 2130 mirror = first; 2131 for ( ; mirror <= last ; mirror++) { 2132 p = &conf->mirrors[mirror]; 2133 if (p->recovery_disabled == mddev->recovery_disabled) 2134 continue; 2135 if (p->rdev) { 2136 if (test_bit(WantReplacement, &p->rdev->flags) && 2137 p->replacement == NULL && repl_slot < 0) 2138 repl_slot = mirror; 2139 continue; 2140 } 2141 2142 err = mddev_stack_new_rdev(mddev, rdev); 2143 if (err) 2144 return err; 2145 p->head_position = 0; 2146 p->recovery_disabled = mddev->recovery_disabled - 1; 2147 rdev->raid_disk = mirror; 2148 err = 0; 2149 if (rdev->saved_raid_disk != mirror) 2150 conf->fullsync = 1; 2151 WRITE_ONCE(p->rdev, rdev); 2152 break; 2153 } 2154 2155 if (err && repl_slot >= 0) { 2156 p = &conf->mirrors[repl_slot]; 2157 clear_bit(In_sync, &rdev->flags); 2158 set_bit(Replacement, &rdev->flags); 2159 rdev->raid_disk = repl_slot; 2160 err = mddev_stack_new_rdev(mddev, rdev); 2161 if (err) 2162 return err; 2163 conf->fullsync = 1; 2164 WRITE_ONCE(p->replacement, rdev); 2165 } 2166 2167 print_conf(conf); 2168 return err; 2169} 2170 2171static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 2172{ 2173 struct r10conf *conf = mddev->private; 2174 int err = 0; 2175 int number = rdev->raid_disk; 2176 struct md_rdev **rdevp; 2177 struct raid10_info *p; 2178 2179 print_conf(conf); 2180 if (unlikely(number >= mddev->raid_disks)) 2181 return 0; 2182 p = conf->mirrors + number; 2183 if (rdev == p->rdev) 2184 rdevp = &p->rdev; 2185 else if (rdev == p->replacement) 2186 rdevp = &p->replacement; 2187 else 2188 return 0; 2189 2190 if (test_bit(In_sync, &rdev->flags) || 2191 atomic_read(&rdev->nr_pending)) { 2192 err = -EBUSY; 2193 goto abort; 2194 } 2195 /* Only remove non-faulty devices if recovery 2196 * is not possible. 2197 */ 2198 if (!test_bit(Faulty, &rdev->flags) && 2199 mddev->recovery_disabled != p->recovery_disabled && 2200 (!p->replacement || p->replacement == rdev) && 2201 number < conf->geo.raid_disks && 2202 enough(conf, -1)) { 2203 err = -EBUSY; 2204 goto abort; 2205 } 2206 WRITE_ONCE(*rdevp, NULL); 2207 if (p->replacement) { 2208 /* We must have just cleared 'rdev' */ 2209 WRITE_ONCE(p->rdev, p->replacement); 2210 clear_bit(Replacement, &p->replacement->flags); 2211 WRITE_ONCE(p->replacement, NULL); 2212 } 2213 2214 clear_bit(WantReplacement, &rdev->flags); 2215 err = md_integrity_register(mddev); 2216 2217abort: 2218 2219 print_conf(conf); 2220 return err; 2221} 2222 2223static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d) 2224{ 2225 struct r10conf *conf = r10_bio->mddev->private; 2226 2227 if (!bio->bi_status) 2228 set_bit(R10BIO_Uptodate, &r10_bio->state); 2229 else 2230 /* The write handler will notice the lack of 2231 * R10BIO_Uptodate and record any errors etc 2232 */ 2233 atomic_add(r10_bio->sectors, 2234 &conf->mirrors[d].rdev->corrected_errors); 2235 2236 /* for reconstruct, we always reschedule after a read. 2237 * for resync, only after all reads 2238 */ 2239 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev); 2240 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || 2241 atomic_dec_and_test(&r10_bio->remaining)) { 2242 /* we have read all the blocks, 2243 * do the comparison in process context in raid10d 2244 */ 2245 reschedule_retry(r10_bio); 2246 } 2247} 2248 2249static void end_sync_read(struct bio *bio) 2250{ 2251 struct r10bio *r10_bio = get_resync_r10bio(bio); 2252 struct r10conf *conf = r10_bio->mddev->private; 2253 int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 2254 2255 __end_sync_read(r10_bio, bio, d); 2256} 2257 2258static void end_reshape_read(struct bio *bio) 2259{ 2260 /* reshape read bio isn't allocated from r10buf_pool */ 2261 struct r10bio *r10_bio = bio->bi_private; 2262 2263 __end_sync_read(r10_bio, bio, r10_bio->read_slot); 2264} 2265 2266static void end_sync_request(struct r10bio *r10_bio) 2267{ 2268 struct mddev *mddev = r10_bio->mddev; 2269 2270 while (atomic_dec_and_test(&r10_bio->remaining)) { 2271 if (r10_bio->master_bio == NULL) { 2272 /* the primary of several recovery bios */ 2273 sector_t s = r10_bio->sectors; 2274 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2275 test_bit(R10BIO_WriteError, &r10_bio->state)) 2276 reschedule_retry(r10_bio); 2277 else 2278 put_buf(r10_bio); 2279 md_done_sync(mddev, s, 1); 2280 break; 2281 } else { 2282 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; 2283 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2284 test_bit(R10BIO_WriteError, &r10_bio->state)) 2285 reschedule_retry(r10_bio); 2286 else 2287 put_buf(r10_bio); 2288 r10_bio = r10_bio2; 2289 } 2290 } 2291} 2292 2293static void end_sync_write(struct bio *bio) 2294{ 2295 struct r10bio *r10_bio = get_resync_r10bio(bio); 2296 struct mddev *mddev = r10_bio->mddev; 2297 struct r10conf *conf = mddev->private; 2298 int d; 2299 int slot; 2300 int repl; 2301 struct md_rdev *rdev = NULL; 2302 2303 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 2304 if (repl) 2305 rdev = conf->mirrors[d].replacement; 2306 else 2307 rdev = conf->mirrors[d].rdev; 2308 2309 if (bio->bi_status) { 2310 if (repl) 2311 md_error(mddev, rdev); 2312 else { 2313 set_bit(WriteErrorSeen, &rdev->flags); 2314 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2315 set_bit(MD_RECOVERY_NEEDED, 2316 &rdev->mddev->recovery); 2317 set_bit(R10BIO_WriteError, &r10_bio->state); 2318 } 2319 } else if (rdev_has_badblock(rdev, r10_bio->devs[slot].addr, 2320 r10_bio->sectors)) { 2321 set_bit(R10BIO_MadeGood, &r10_bio->state); 2322 } 2323 2324 rdev_dec_pending(rdev, mddev); 2325 2326 end_sync_request(r10_bio); 2327} 2328 2329/* 2330 * Note: sync and recover and handled very differently for raid10 2331 * This code is for resync. 2332 * For resync, we read through virtual addresses and read all blocks. 2333 * If there is any error, we schedule a write. The lowest numbered 2334 * drive is authoritative. 2335 * However requests come for physical address, so we need to map. 2336 * For every physical address there are raid_disks/copies virtual addresses, 2337 * which is always are least one, but is not necessarly an integer. 2338 * This means that a physical address can span multiple chunks, so we may 2339 * have to submit multiple io requests for a single sync request. 2340 */ 2341/* 2342 * We check if all blocks are in-sync and only write to blocks that 2343 * aren't in sync 2344 */ 2345static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2346{ 2347 struct r10conf *conf = mddev->private; 2348 int i, first; 2349 struct bio *tbio, *fbio; 2350 int vcnt; 2351 struct page **tpages, **fpages; 2352 2353 atomic_set(&r10_bio->remaining, 1); 2354 2355 /* find the first device with a block */ 2356 for (i=0; i<conf->copies; i++) 2357 if (!r10_bio->devs[i].bio->bi_status) 2358 break; 2359 2360 if (i == conf->copies) 2361 goto done; 2362 2363 first = i; 2364 fbio = r10_bio->devs[i].bio; 2365 fbio->bi_iter.bi_size = r10_bio->sectors << 9; 2366 fbio->bi_iter.bi_idx = 0; 2367 fpages = get_resync_pages(fbio)->pages; 2368 2369 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); 2370 /* now find blocks with errors */ 2371 for (i=0 ; i < conf->copies ; i++) { 2372 int j, d; 2373 struct md_rdev *rdev; 2374 struct resync_pages *rp; 2375 2376 tbio = r10_bio->devs[i].bio; 2377 2378 if (tbio->bi_end_io != end_sync_read) 2379 continue; 2380 if (i == first) 2381 continue; 2382 2383 tpages = get_resync_pages(tbio)->pages; 2384 d = r10_bio->devs[i].devnum; 2385 rdev = conf->mirrors[d].rdev; 2386 if (!r10_bio->devs[i].bio->bi_status) { 2387 /* We know that the bi_io_vec layout is the same for 2388 * both 'first' and 'i', so we just compare them. 2389 * All vec entries are PAGE_SIZE; 2390 */ 2391 int sectors = r10_bio->sectors; 2392 for (j = 0; j < vcnt; j++) { 2393 int len = PAGE_SIZE; 2394 if (sectors < (len / 512)) 2395 len = sectors * 512; 2396 if (memcmp(page_address(fpages[j]), 2397 page_address(tpages[j]), 2398 len)) 2399 break; 2400 sectors -= len/512; 2401 } 2402 if (j == vcnt) 2403 continue; 2404 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); 2405 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) 2406 /* Don't fix anything. */ 2407 continue; 2408 } else if (test_bit(FailFast, &rdev->flags)) { 2409 /* Just give up on this device */ 2410 md_error(rdev->mddev, rdev); 2411 continue; 2412 } 2413 /* Ok, we need to write this bio, either to correct an 2414 * inconsistency or to correct an unreadable block. 2415 * First we need to fixup bv_offset, bv_len and 2416 * bi_vecs, as the read request might have corrupted these 2417 */ 2418 rp = get_resync_pages(tbio); 2419 bio_reset(tbio, conf->mirrors[d].rdev->bdev, REQ_OP_WRITE); 2420 2421 md_bio_reset_resync_pages(tbio, rp, fbio->bi_iter.bi_size); 2422 2423 rp->raid_bio = r10_bio; 2424 tbio->bi_private = rp; 2425 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; 2426 tbio->bi_end_io = end_sync_write; 2427 2428 bio_copy_data(tbio, fbio); 2429 2430 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2431 atomic_inc(&r10_bio->remaining); 2432 2433 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags)) 2434 tbio->bi_opf |= MD_FAILFAST; 2435 tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset; 2436 submit_bio_noacct(tbio); 2437 } 2438 2439 /* Now write out to any replacement devices 2440 * that are active 2441 */ 2442 for (i = 0; i < conf->copies; i++) { 2443 tbio = r10_bio->devs[i].repl_bio; 2444 if (!tbio || !tbio->bi_end_io) 2445 continue; 2446 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2447 && r10_bio->devs[i].bio != fbio) 2448 bio_copy_data(tbio, fbio); 2449 atomic_inc(&r10_bio->remaining); 2450 submit_bio_noacct(tbio); 2451 } 2452 2453done: 2454 if (atomic_dec_and_test(&r10_bio->remaining)) { 2455 md_done_sync(mddev, r10_bio->sectors, 1); 2456 put_buf(r10_bio); 2457 } 2458} 2459 2460/* 2461 * Now for the recovery code. 2462 * Recovery happens across physical sectors. 2463 * We recover all non-is_sync drives by finding the virtual address of 2464 * each, and then choose a working drive that also has that virt address. 2465 * There is a separate r10_bio for each non-in_sync drive. 2466 * Only the first two slots are in use. The first for reading, 2467 * The second for writing. 2468 * 2469 */ 2470static void fix_recovery_read_error(struct r10bio *r10_bio) 2471{ 2472 /* We got a read error during recovery. 2473 * We repeat the read in smaller page-sized sections. 2474 * If a read succeeds, write it to the new device or record 2475 * a bad block if we cannot. 2476 * If a read fails, record a bad block on both old and 2477 * new devices. 2478 */ 2479 struct mddev *mddev = r10_bio->mddev; 2480 struct r10conf *conf = mddev->private; 2481 struct bio *bio = r10_bio->devs[0].bio; 2482 sector_t sect = 0; 2483 int sectors = r10_bio->sectors; 2484 int idx = 0; 2485 int dr = r10_bio->devs[0].devnum; 2486 int dw = r10_bio->devs[1].devnum; 2487 struct page **pages = get_resync_pages(bio)->pages; 2488 2489 while (sectors) { 2490 int s = sectors; 2491 struct md_rdev *rdev; 2492 sector_t addr; 2493 int ok; 2494 2495 if (s > (PAGE_SIZE>>9)) 2496 s = PAGE_SIZE >> 9; 2497 2498 rdev = conf->mirrors[dr].rdev; 2499 addr = r10_bio->devs[0].addr + sect; 2500 ok = sync_page_io(rdev, 2501 addr, 2502 s << 9, 2503 pages[idx], 2504 REQ_OP_READ, false); 2505 if (ok) { 2506 rdev = conf->mirrors[dw].rdev; 2507 addr = r10_bio->devs[1].addr + sect; 2508 ok = sync_page_io(rdev, 2509 addr, 2510 s << 9, 2511 pages[idx], 2512 REQ_OP_WRITE, false); 2513 if (!ok) { 2514 set_bit(WriteErrorSeen, &rdev->flags); 2515 if (!test_and_set_bit(WantReplacement, 2516 &rdev->flags)) 2517 set_bit(MD_RECOVERY_NEEDED, 2518 &rdev->mddev->recovery); 2519 } 2520 } 2521 if (!ok) { 2522 /* We don't worry if we cannot set a bad block - 2523 * it really is bad so there is no loss in not 2524 * recording it yet 2525 */ 2526 rdev_set_badblocks(rdev, addr, s, 0); 2527 2528 if (rdev != conf->mirrors[dw].rdev) { 2529 /* need bad block on destination too */ 2530 struct md_rdev *rdev2 = conf->mirrors[dw].rdev; 2531 addr = r10_bio->devs[1].addr + sect; 2532 ok = rdev_set_badblocks(rdev2, addr, s, 0); 2533 if (!ok) { 2534 /* just abort the recovery */ 2535 pr_notice("md/raid10:%s: recovery aborted due to read error\n", 2536 mdname(mddev)); 2537 2538 conf->mirrors[dw].recovery_disabled 2539 = mddev->recovery_disabled; 2540 set_bit(MD_RECOVERY_INTR, 2541 &mddev->recovery); 2542 break; 2543 } 2544 } 2545 } 2546 2547 sectors -= s; 2548 sect += s; 2549 idx++; 2550 } 2551} 2552 2553static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2554{ 2555 struct r10conf *conf = mddev->private; 2556 int d; 2557 struct bio *wbio = r10_bio->devs[1].bio; 2558 struct bio *wbio2 = r10_bio->devs[1].repl_bio; 2559 2560 /* Need to test wbio2->bi_end_io before we call 2561 * submit_bio_noacct as if the former is NULL, 2562 * the latter is free to free wbio2. 2563 */ 2564 if (wbio2 && !wbio2->bi_end_io) 2565 wbio2 = NULL; 2566 2567 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { 2568 fix_recovery_read_error(r10_bio); 2569 if (wbio->bi_end_io) 2570 end_sync_request(r10_bio); 2571 if (wbio2) 2572 end_sync_request(r10_bio); 2573 return; 2574 } 2575 2576 /* 2577 * share the pages with the first bio 2578 * and submit the write request 2579 */ 2580 d = r10_bio->devs[1].devnum; 2581 if (wbio->bi_end_io) { 2582 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2583 submit_bio_noacct(wbio); 2584 } 2585 if (wbio2) { 2586 atomic_inc(&conf->mirrors[d].replacement->nr_pending); 2587 submit_bio_noacct(wbio2); 2588 } 2589} 2590 2591static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2592 int sectors, struct page *page, enum req_op op) 2593{ 2594 if (rdev_has_badblock(rdev, sector, sectors) && 2595 (op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags))) 2596 return -1; 2597 if (sync_page_io(rdev, sector, sectors << 9, page, op, false)) 2598 /* success */ 2599 return 1; 2600 if (op == REQ_OP_WRITE) { 2601 set_bit(WriteErrorSeen, &rdev->flags); 2602 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2603 set_bit(MD_RECOVERY_NEEDED, 2604 &rdev->mddev->recovery); 2605 } 2606 /* need to record an error - either for the block or the device */ 2607 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2608 md_error(rdev->mddev, rdev); 2609 return 0; 2610} 2611 2612/* 2613 * This is a kernel thread which: 2614 * 2615 * 1. Retries failed read operations on working mirrors. 2616 * 2. Updates the raid superblock when problems encounter. 2617 * 3. Performs writes following reads for array synchronising. 2618 */ 2619 2620static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) 2621{ 2622 int sect = 0; /* Offset from r10_bio->sector */ 2623 int sectors = r10_bio->sectors, slot = r10_bio->read_slot; 2624 struct md_rdev *rdev; 2625 int d = r10_bio->devs[slot].devnum; 2626 2627 /* still own a reference to this rdev, so it cannot 2628 * have been cleared recently. 2629 */ 2630 rdev = conf->mirrors[d].rdev; 2631 2632 if (test_bit(Faulty, &rdev->flags)) 2633 /* drive has already been failed, just ignore any 2634 more fix_read_error() attempts */ 2635 return; 2636 2637 if (exceed_read_errors(mddev, rdev)) { 2638 r10_bio->devs[slot].bio = IO_BLOCKED; 2639 return; 2640 } 2641 2642 while(sectors) { 2643 int s = sectors; 2644 int sl = slot; 2645 int success = 0; 2646 int start; 2647 2648 if (s > (PAGE_SIZE>>9)) 2649 s = PAGE_SIZE >> 9; 2650 2651 do { 2652 d = r10_bio->devs[sl].devnum; 2653 rdev = conf->mirrors[d].rdev; 2654 if (rdev && 2655 test_bit(In_sync, &rdev->flags) && 2656 !test_bit(Faulty, &rdev->flags) && 2657 rdev_has_badblock(rdev, 2658 r10_bio->devs[sl].addr + sect, 2659 s) == 0) { 2660 atomic_inc(&rdev->nr_pending); 2661 success = sync_page_io(rdev, 2662 r10_bio->devs[sl].addr + 2663 sect, 2664 s<<9, 2665 conf->tmppage, 2666 REQ_OP_READ, false); 2667 rdev_dec_pending(rdev, mddev); 2668 if (success) 2669 break; 2670 } 2671 sl++; 2672 if (sl == conf->copies) 2673 sl = 0; 2674 } while (sl != slot); 2675 2676 if (!success) { 2677 /* Cannot read from anywhere, just mark the block 2678 * as bad on the first device to discourage future 2679 * reads. 2680 */ 2681 int dn = r10_bio->devs[slot].devnum; 2682 rdev = conf->mirrors[dn].rdev; 2683 2684 if (!rdev_set_badblocks( 2685 rdev, 2686 r10_bio->devs[slot].addr 2687 + sect, 2688 s, 0)) { 2689 md_error(mddev, rdev); 2690 r10_bio->devs[slot].bio 2691 = IO_BLOCKED; 2692 } 2693 break; 2694 } 2695 2696 start = sl; 2697 /* write it back and re-read */ 2698 while (sl != slot) { 2699 if (sl==0) 2700 sl = conf->copies; 2701 sl--; 2702 d = r10_bio->devs[sl].devnum; 2703 rdev = conf->mirrors[d].rdev; 2704 if (!rdev || 2705 test_bit(Faulty, &rdev->flags) || 2706 !test_bit(In_sync, &rdev->flags)) 2707 continue; 2708 2709 atomic_inc(&rdev->nr_pending); 2710 if (r10_sync_page_io(rdev, 2711 r10_bio->devs[sl].addr + 2712 sect, 2713 s, conf->tmppage, REQ_OP_WRITE) 2714 == 0) { 2715 /* Well, this device is dead */ 2716 pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %pg)\n", 2717 mdname(mddev), s, 2718 (unsigned long long)( 2719 sect + 2720 choose_data_offset(r10_bio, 2721 rdev)), 2722 rdev->bdev); 2723 pr_notice("md/raid10:%s: %pg: failing drive\n", 2724 mdname(mddev), 2725 rdev->bdev); 2726 } 2727 rdev_dec_pending(rdev, mddev); 2728 } 2729 sl = start; 2730 while (sl != slot) { 2731 if (sl==0) 2732 sl = conf->copies; 2733 sl--; 2734 d = r10_bio->devs[sl].devnum; 2735 rdev = conf->mirrors[d].rdev; 2736 if (!rdev || 2737 test_bit(Faulty, &rdev->flags) || 2738 !test_bit(In_sync, &rdev->flags)) 2739 continue; 2740 2741 atomic_inc(&rdev->nr_pending); 2742 switch (r10_sync_page_io(rdev, 2743 r10_bio->devs[sl].addr + 2744 sect, 2745 s, conf->tmppage, REQ_OP_READ)) { 2746 case 0: 2747 /* Well, this device is dead */ 2748 pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %pg)\n", 2749 mdname(mddev), s, 2750 (unsigned long long)( 2751 sect + 2752 choose_data_offset(r10_bio, rdev)), 2753 rdev->bdev); 2754 pr_notice("md/raid10:%s: %pg: failing drive\n", 2755 mdname(mddev), 2756 rdev->bdev); 2757 break; 2758 case 1: 2759 pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %pg)\n", 2760 mdname(mddev), s, 2761 (unsigned long long)( 2762 sect + 2763 choose_data_offset(r10_bio, rdev)), 2764 rdev->bdev); 2765 atomic_add(s, &rdev->corrected_errors); 2766 } 2767 2768 rdev_dec_pending(rdev, mddev); 2769 } 2770 2771 sectors -= s; 2772 sect += s; 2773 } 2774} 2775 2776static bool narrow_write_error(struct r10bio *r10_bio, int i) 2777{ 2778 struct bio *bio = r10_bio->master_bio; 2779 struct mddev *mddev = r10_bio->mddev; 2780 struct r10conf *conf = mddev->private; 2781 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; 2782 /* bio has the data to be written to slot 'i' where 2783 * we just recently had a write error. 2784 * We repeatedly clone the bio and trim down to one block, 2785 * then try the write. Where the write fails we record 2786 * a bad block. 2787 * It is conceivable that the bio doesn't exactly align with 2788 * blocks. We must handle this. 2789 * 2790 * We currently own a reference to the rdev. 2791 */ 2792 2793 int block_sectors; 2794 sector_t sector; 2795 int sectors; 2796 int sect_to_write = r10_bio->sectors; 2797 bool ok = true; 2798 2799 if (rdev->badblocks.shift < 0) 2800 return false; 2801 2802 block_sectors = roundup(1 << rdev->badblocks.shift, 2803 bdev_logical_block_size(rdev->bdev) >> 9); 2804 sector = r10_bio->sector; 2805 sectors = ((r10_bio->sector + block_sectors) 2806 & ~(sector_t)(block_sectors - 1)) 2807 - sector; 2808 2809 while (sect_to_write) { 2810 struct bio *wbio; 2811 sector_t wsector; 2812 if (sectors > sect_to_write) 2813 sectors = sect_to_write; 2814 /* Write at 'sector' for 'sectors' */ 2815 wbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, 2816 &mddev->bio_set); 2817 bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors); 2818 wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector); 2819 wbio->bi_iter.bi_sector = wsector + 2820 choose_data_offset(r10_bio, rdev); 2821 wbio->bi_opf = REQ_OP_WRITE; 2822 2823 if (submit_bio_wait(wbio) < 0) 2824 /* Failure! */ 2825 ok = rdev_set_badblocks(rdev, wsector, 2826 sectors, 0) 2827 && ok; 2828 2829 bio_put(wbio); 2830 sect_to_write -= sectors; 2831 sector += sectors; 2832 sectors = block_sectors; 2833 } 2834 return ok; 2835} 2836 2837static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) 2838{ 2839 int slot = r10_bio->read_slot; 2840 struct bio *bio; 2841 struct r10conf *conf = mddev->private; 2842 struct md_rdev *rdev = r10_bio->devs[slot].rdev; 2843 2844 /* we got a read error. Maybe the drive is bad. Maybe just 2845 * the block and we can fix it. 2846 * We freeze all other IO, and try reading the block from 2847 * other devices. When we find one, we re-write 2848 * and check it that fixes the read error. 2849 * This is all done synchronously while the array is 2850 * frozen. 2851 */ 2852 bio = r10_bio->devs[slot].bio; 2853 bio_put(bio); 2854 r10_bio->devs[slot].bio = NULL; 2855 2856 if (mddev->ro) 2857 r10_bio->devs[slot].bio = IO_BLOCKED; 2858 else if (!test_bit(FailFast, &rdev->flags)) { 2859 freeze_array(conf, 1); 2860 fix_read_error(conf, mddev, r10_bio); 2861 unfreeze_array(conf); 2862 } else 2863 md_error(mddev, rdev); 2864 2865 rdev_dec_pending(rdev, mddev); 2866 r10_bio->state = 0; 2867 raid10_read_request(mddev, r10_bio->master_bio, r10_bio, false); 2868 /* 2869 * allow_barrier after re-submit to ensure no sync io 2870 * can be issued while regular io pending. 2871 */ 2872 allow_barrier(conf); 2873} 2874 2875static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) 2876{ 2877 /* Some sort of write request has finished and it 2878 * succeeded in writing where we thought there was a 2879 * bad block. So forget the bad block. 2880 * Or possibly if failed and we need to record 2881 * a bad block. 2882 */ 2883 int m; 2884 struct md_rdev *rdev; 2885 2886 if (test_bit(R10BIO_IsSync, &r10_bio->state) || 2887 test_bit(R10BIO_IsRecover, &r10_bio->state)) { 2888 for (m = 0; m < conf->copies; m++) { 2889 int dev = r10_bio->devs[m].devnum; 2890 rdev = conf->mirrors[dev].rdev; 2891 if (r10_bio->devs[m].bio == NULL || 2892 r10_bio->devs[m].bio->bi_end_io == NULL) 2893 continue; 2894 if (!r10_bio->devs[m].bio->bi_status) { 2895 rdev_clear_badblocks( 2896 rdev, 2897 r10_bio->devs[m].addr, 2898 r10_bio->sectors, 0); 2899 } else { 2900 if (!rdev_set_badblocks( 2901 rdev, 2902 r10_bio->devs[m].addr, 2903 r10_bio->sectors, 0)) 2904 md_error(conf->mddev, rdev); 2905 } 2906 rdev = conf->mirrors[dev].replacement; 2907 if (r10_bio->devs[m].repl_bio == NULL || 2908 r10_bio->devs[m].repl_bio->bi_end_io == NULL) 2909 continue; 2910 2911 if (!r10_bio->devs[m].repl_bio->bi_status) { 2912 rdev_clear_badblocks( 2913 rdev, 2914 r10_bio->devs[m].addr, 2915 r10_bio->sectors, 0); 2916 } else { 2917 if (!rdev_set_badblocks( 2918 rdev, 2919 r10_bio->devs[m].addr, 2920 r10_bio->sectors, 0)) 2921 md_error(conf->mddev, rdev); 2922 } 2923 } 2924 put_buf(r10_bio); 2925 } else { 2926 bool fail = false; 2927 for (m = 0; m < conf->copies; m++) { 2928 int dev = r10_bio->devs[m].devnum; 2929 struct bio *bio = r10_bio->devs[m].bio; 2930 rdev = conf->mirrors[dev].rdev; 2931 if (bio == IO_MADE_GOOD) { 2932 rdev_clear_badblocks( 2933 rdev, 2934 r10_bio->devs[m].addr, 2935 r10_bio->sectors, 0); 2936 rdev_dec_pending(rdev, conf->mddev); 2937 } else if (bio != NULL && bio->bi_status) { 2938 fail = true; 2939 if (!narrow_write_error(r10_bio, m)) 2940 md_error(conf->mddev, rdev); 2941 rdev_dec_pending(rdev, conf->mddev); 2942 } 2943 bio = r10_bio->devs[m].repl_bio; 2944 rdev = conf->mirrors[dev].replacement; 2945 if (rdev && bio == IO_MADE_GOOD) { 2946 rdev_clear_badblocks( 2947 rdev, 2948 r10_bio->devs[m].addr, 2949 r10_bio->sectors, 0); 2950 rdev_dec_pending(rdev, conf->mddev); 2951 } 2952 } 2953 if (fail) { 2954 spin_lock_irq(&conf->device_lock); 2955 list_add(&r10_bio->retry_list, &conf->bio_end_io_list); 2956 conf->nr_queued++; 2957 spin_unlock_irq(&conf->device_lock); 2958 /* 2959 * In case freeze_array() is waiting for condition 2960 * nr_pending == nr_queued + extra to be true. 2961 */ 2962 wake_up(&conf->wait_barrier); 2963 md_wakeup_thread(conf->mddev->thread); 2964 } else { 2965 if (test_bit(R10BIO_WriteError, 2966 &r10_bio->state)) 2967 close_write(r10_bio); 2968 raid_end_bio_io(r10_bio); 2969 } 2970 } 2971} 2972 2973static void raid10d(struct md_thread *thread) 2974{ 2975 struct mddev *mddev = thread->mddev; 2976 struct r10bio *r10_bio; 2977 unsigned long flags; 2978 struct r10conf *conf = mddev->private; 2979 struct list_head *head = &conf->retry_list; 2980 struct blk_plug plug; 2981 2982 md_check_recovery(mddev); 2983 2984 if (!list_empty_careful(&conf->bio_end_io_list) && 2985 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2986 LIST_HEAD(tmp); 2987 spin_lock_irqsave(&conf->device_lock, flags); 2988 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2989 while (!list_empty(&conf->bio_end_io_list)) { 2990 list_move(conf->bio_end_io_list.prev, &tmp); 2991 conf->nr_queued--; 2992 } 2993 } 2994 spin_unlock_irqrestore(&conf->device_lock, flags); 2995 while (!list_empty(&tmp)) { 2996 r10_bio = list_first_entry(&tmp, struct r10bio, 2997 retry_list); 2998 list_del(&r10_bio->retry_list); 2999 3000 if (test_bit(R10BIO_WriteError, 3001 &r10_bio->state)) 3002 close_write(r10_bio); 3003 raid_end_bio_io(r10_bio); 3004 } 3005 } 3006 3007 blk_start_plug(&plug); 3008 for (;;) { 3009 3010 flush_pending_writes(conf); 3011 3012 spin_lock_irqsave(&conf->device_lock, flags); 3013 if (list_empty(head)) { 3014 spin_unlock_irqrestore(&conf->device_lock, flags); 3015 break; 3016 } 3017 r10_bio = list_entry(head->prev, struct r10bio, retry_list); 3018 list_del(head->prev); 3019 conf->nr_queued--; 3020 spin_unlock_irqrestore(&conf->device_lock, flags); 3021 3022 mddev = r10_bio->mddev; 3023 conf = mddev->private; 3024 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 3025 test_bit(R10BIO_WriteError, &r10_bio->state)) 3026 handle_write_completed(conf, r10_bio); 3027 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) 3028 reshape_request_write(mddev, r10_bio); 3029 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) 3030 sync_request_write(mddev, r10_bio); 3031 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) 3032 recovery_request_write(mddev, r10_bio); 3033 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) 3034 handle_read_error(mddev, r10_bio); 3035 else 3036 WARN_ON_ONCE(1); 3037 3038 cond_resched(); 3039 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING)) 3040 md_check_recovery(mddev); 3041 } 3042 blk_finish_plug(&plug); 3043} 3044 3045static int init_resync(struct r10conf *conf) 3046{ 3047 int ret, buffs, i; 3048 3049 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 3050 BUG_ON(mempool_initialized(&conf->r10buf_pool)); 3051 conf->have_replacement = 0; 3052 for (i = 0; i < conf->geo.raid_disks; i++) 3053 if (conf->mirrors[i].replacement) 3054 conf->have_replacement = 1; 3055 ret = mempool_init(&conf->r10buf_pool, buffs, 3056 r10buf_pool_alloc, r10buf_pool_free, conf); 3057 if (ret) 3058 return ret; 3059 conf->next_resync = 0; 3060 return 0; 3061} 3062 3063static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf) 3064{ 3065 struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO); 3066 struct rsync_pages *rp; 3067 struct bio *bio; 3068 int nalloc; 3069 int i; 3070 3071 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 3072 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 3073 nalloc = conf->copies; /* resync */ 3074 else 3075 nalloc = 2; /* recovery */ 3076 3077 for (i = 0; i < nalloc; i++) { 3078 bio = r10bio->devs[i].bio; 3079 rp = bio->bi_private; 3080 bio_reset(bio, NULL, 0); 3081 bio->bi_private = rp; 3082 bio = r10bio->devs[i].repl_bio; 3083 if (bio) { 3084 rp = bio->bi_private; 3085 bio_reset(bio, NULL, 0); 3086 bio->bi_private = rp; 3087 } 3088 } 3089 return r10bio; 3090} 3091 3092/* 3093 * Set cluster_sync_high since we need other nodes to add the 3094 * range [cluster_sync_low, cluster_sync_high] to suspend list. 3095 */ 3096static void raid10_set_cluster_sync_high(struct r10conf *conf) 3097{ 3098 sector_t window_size; 3099 int extra_chunk, chunks; 3100 3101 /* 3102 * First, here we define "stripe" as a unit which across 3103 * all member devices one time, so we get chunks by use 3104 * raid_disks / near_copies. Otherwise, if near_copies is 3105 * close to raid_disks, then resync window could increases 3106 * linearly with the increase of raid_disks, which means 3107 * we will suspend a really large IO window while it is not 3108 * necessary. If raid_disks is not divisible by near_copies, 3109 * an extra chunk is needed to ensure the whole "stripe" is 3110 * covered. 3111 */ 3112 3113 chunks = conf->geo.raid_disks / conf->geo.near_copies; 3114 if (conf->geo.raid_disks % conf->geo.near_copies == 0) 3115 extra_chunk = 0; 3116 else 3117 extra_chunk = 1; 3118 window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors; 3119 3120 /* 3121 * At least use a 32M window to align with raid1's resync window 3122 */ 3123 window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ? 3124 CLUSTER_RESYNC_WINDOW_SECTORS : window_size; 3125 3126 conf->cluster_sync_high = conf->cluster_sync_low + window_size; 3127} 3128 3129/* 3130 * perform a "sync" on one "block" 3131 * 3132 * We need to make sure that no normal I/O request - particularly write 3133 * requests - conflict with active sync requests. 3134 * 3135 * This is achieved by tracking pending requests and a 'barrier' concept 3136 * that can be installed to exclude normal IO requests. 3137 * 3138 * Resync and recovery are handled very differently. 3139 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery. 3140 * 3141 * For resync, we iterate over virtual addresses, read all copies, 3142 * and update if there are differences. If only one copy is live, 3143 * skip it. 3144 * For recovery, we iterate over physical addresses, read a good 3145 * value for each non-in_sync drive, and over-write. 3146 * 3147 * So, for recovery we may have several outstanding complex requests for a 3148 * given address, one for each out-of-sync device. We model this by allocating 3149 * a number of r10_bio structures, one for each out-of-sync device. 3150 * As we setup these structures, we collect all bio's together into a list 3151 * which we then process collectively to add pages, and then process again 3152 * to pass to submit_bio_noacct. 3153 * 3154 * The r10_bio structures are linked using a borrowed master_bio pointer. 3155 * This link is counted in ->remaining. When the r10_bio that points to NULL 3156 * has its remaining count decremented to 0, the whole complex operation 3157 * is complete. 3158 * 3159 */ 3160 3161static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, 3162 sector_t max_sector, int *skipped) 3163{ 3164 struct r10conf *conf = mddev->private; 3165 struct r10bio *r10_bio; 3166 struct bio *biolist = NULL, *bio; 3167 sector_t nr_sectors; 3168 int i; 3169 int max_sync; 3170 sector_t sync_blocks; 3171 sector_t sectors_skipped = 0; 3172 int chunks_skipped = 0; 3173 sector_t chunk_mask = conf->geo.chunk_mask; 3174 int page_idx = 0; 3175 int error_disk = -1; 3176 3177 /* 3178 * Allow skipping a full rebuild for incremental assembly 3179 * of a clean array, like RAID1 does. 3180 */ 3181 if (mddev->bitmap == NULL && 3182 mddev->resync_offset == MaxSector && 3183 mddev->reshape_position == MaxSector && 3184 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 3185 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 3186 !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 3187 conf->fullsync == 0) { 3188 *skipped = 1; 3189 return mddev->dev_sectors - sector_nr; 3190 } 3191 3192 if (!mempool_initialized(&conf->r10buf_pool)) 3193 if (init_resync(conf)) 3194 return 0; 3195 3196 skipped: 3197 if (sector_nr >= max_sector) { 3198 conf->cluster_sync_low = 0; 3199 conf->cluster_sync_high = 0; 3200 3201 /* If we aborted, we need to abort the 3202 * sync on the 'current' bitmap chucks (there can 3203 * be several when recovering multiple devices). 3204 * as we may have started syncing it but not finished. 3205 * We can find the current address in 3206 * mddev->curr_resync, but for recovery, 3207 * we need to convert that to several 3208 * virtual addresses. 3209 */ 3210 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 3211 end_reshape(conf); 3212 close_sync(conf); 3213 return 0; 3214 } 3215 3216 if (mddev->curr_resync < max_sector) { /* aborted */ 3217 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) 3218 md_bitmap_end_sync(mddev, mddev->curr_resync, 3219 &sync_blocks); 3220 else for (i = 0; i < conf->geo.raid_disks; i++) { 3221 sector_t sect = 3222 raid10_find_virt(conf, mddev->curr_resync, i); 3223 3224 md_bitmap_end_sync(mddev, sect, &sync_blocks); 3225 } 3226 } else { 3227 /* completed sync */ 3228 if ((!mddev->bitmap || conf->fullsync) 3229 && conf->have_replacement 3230 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3231 /* Completed a full sync so the replacements 3232 * are now fully recovered. 3233 */ 3234 for (i = 0; i < conf->geo.raid_disks; i++) { 3235 struct md_rdev *rdev = 3236 conf->mirrors[i].replacement; 3237 3238 if (rdev) 3239 rdev->recovery_offset = MaxSector; 3240 } 3241 } 3242 conf->fullsync = 0; 3243 } 3244 if (md_bitmap_enabled(mddev, false)) 3245 mddev->bitmap_ops->close_sync(mddev); 3246 close_sync(conf); 3247 *skipped = 1; 3248 return sectors_skipped; 3249 } 3250 3251 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 3252 return reshape_request(mddev, sector_nr, skipped); 3253 3254 if (chunks_skipped >= conf->geo.raid_disks) { 3255 pr_err("md/raid10:%s: %s fails\n", mdname(mddev), 3256 test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ? "resync" : "recovery"); 3257 if (error_disk >= 0 && 3258 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3259 /* 3260 * recovery fails, set mirrors.recovery_disabled, 3261 * device shouldn't be added to there. 3262 */ 3263 conf->mirrors[error_disk].recovery_disabled = 3264 mddev->recovery_disabled; 3265 return 0; 3266 } 3267 /* 3268 * if there has been nothing to do on any drive, 3269 * then there is nothing to do at all. 3270 */ 3271 *skipped = 1; 3272 return (max_sector - sector_nr) + sectors_skipped; 3273 } 3274 3275 if (max_sector > mddev->resync_max) 3276 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 3277 3278 /* make sure whole request will fit in a chunk - if chunks 3279 * are meaningful 3280 */ 3281 if (conf->geo.near_copies < conf->geo.raid_disks && 3282 max_sector > (sector_nr | chunk_mask)) 3283 max_sector = (sector_nr | chunk_mask) + 1; 3284 3285 /* 3286 * If there is non-resync activity waiting for a turn, then let it 3287 * though before starting on this new sync request. 3288 */ 3289 if (conf->nr_waiting) 3290 schedule_timeout_uninterruptible(1); 3291 3292 /* Again, very different code for resync and recovery. 3293 * Both must result in an r10bio with a list of bios that 3294 * have bi_end_io, bi_sector, bi_bdev set, 3295 * and bi_private set to the r10bio. 3296 * For recovery, we may actually create several r10bios 3297 * with 2 bios in each, that correspond to the bios in the main one. 3298 * In this case, the subordinate r10bios link back through a 3299 * borrowed master_bio pointer, and the counter in the master 3300 * includes a ref from each subordinate. 3301 */ 3302 /* First, we decide what to do and set ->bi_end_io 3303 * To end_sync_read if we want to read, and 3304 * end_sync_write if we will want to write. 3305 */ 3306 3307 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9); 3308 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3309 /* recovery... the complicated one */ 3310 int j; 3311 r10_bio = NULL; 3312 3313 for (i = 0 ; i < conf->geo.raid_disks; i++) { 3314 bool still_degraded; 3315 struct r10bio *rb2; 3316 sector_t sect; 3317 bool must_sync; 3318 int any_working; 3319 struct raid10_info *mirror = &conf->mirrors[i]; 3320 struct md_rdev *mrdev, *mreplace; 3321 3322 mrdev = mirror->rdev; 3323 mreplace = mirror->replacement; 3324 3325 if (mrdev && (test_bit(Faulty, &mrdev->flags) || 3326 test_bit(In_sync, &mrdev->flags))) 3327 mrdev = NULL; 3328 if (mreplace && test_bit(Faulty, &mreplace->flags)) 3329 mreplace = NULL; 3330 3331 if (!mrdev && !mreplace) 3332 continue; 3333 3334 still_degraded = false; 3335 /* want to reconstruct this device */ 3336 rb2 = r10_bio; 3337 sect = raid10_find_virt(conf, sector_nr, i); 3338 if (sect >= mddev->resync_max_sectors) 3339 /* last stripe is not complete - don't 3340 * try to recover this sector. 3341 */ 3342 continue; 3343 /* Unless we are doing a full sync, or a replacement 3344 * we only need to recover the block if it is set in 3345 * the bitmap 3346 */ 3347 must_sync = md_bitmap_start_sync(mddev, sect, 3348 &sync_blocks, true); 3349 if (sync_blocks < max_sync) 3350 max_sync = sync_blocks; 3351 if (!must_sync && 3352 mreplace == NULL && 3353 !conf->fullsync) { 3354 /* yep, skip the sync_blocks here, but don't assume 3355 * that there will never be anything to do here 3356 */ 3357 chunks_skipped = -1; 3358 continue; 3359 } 3360 if (mrdev) 3361 atomic_inc(&mrdev->nr_pending); 3362 if (mreplace) 3363 atomic_inc(&mreplace->nr_pending); 3364 3365 r10_bio = raid10_alloc_init_r10buf(conf); 3366 r10_bio->state = 0; 3367 raise_barrier(conf, rb2 != NULL); 3368 atomic_set(&r10_bio->remaining, 0); 3369 3370 r10_bio->master_bio = (struct bio*)rb2; 3371 if (rb2) 3372 atomic_inc(&rb2->remaining); 3373 r10_bio->mddev = mddev; 3374 set_bit(R10BIO_IsRecover, &r10_bio->state); 3375 r10_bio->sector = sect; 3376 3377 raid10_find_phys(conf, r10_bio); 3378 3379 /* Need to check if the array will still be 3380 * degraded 3381 */ 3382 for (j = 0; j < conf->geo.raid_disks; j++) { 3383 struct md_rdev *rdev = conf->mirrors[j].rdev; 3384 3385 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3386 still_degraded = false; 3387 break; 3388 } 3389 } 3390 3391 md_bitmap_start_sync(mddev, sect, &sync_blocks, 3392 still_degraded); 3393 any_working = 0; 3394 for (j=0; j<conf->copies;j++) { 3395 int k; 3396 int d = r10_bio->devs[j].devnum; 3397 sector_t from_addr, to_addr; 3398 struct md_rdev *rdev = conf->mirrors[d].rdev; 3399 sector_t sector, first_bad; 3400 sector_t bad_sectors; 3401 if (!rdev || 3402 !test_bit(In_sync, &rdev->flags)) 3403 continue; 3404 /* This is where we read from */ 3405 sector = r10_bio->devs[j].addr; 3406 3407 if (is_badblock(rdev, sector, max_sync, 3408 &first_bad, &bad_sectors)) { 3409 if (first_bad > sector) 3410 max_sync = first_bad - sector; 3411 else { 3412 bad_sectors -= (sector 3413 - first_bad); 3414 if (max_sync > bad_sectors) 3415 max_sync = bad_sectors; 3416 continue; 3417 } 3418 } 3419 any_working = 1; 3420 bio = r10_bio->devs[0].bio; 3421 bio->bi_next = biolist; 3422 biolist = bio; 3423 bio->bi_end_io = end_sync_read; 3424 bio->bi_opf = REQ_OP_READ; 3425 if (test_bit(FailFast, &rdev->flags)) 3426 bio->bi_opf |= MD_FAILFAST; 3427 from_addr = r10_bio->devs[j].addr; 3428 bio->bi_iter.bi_sector = from_addr + 3429 rdev->data_offset; 3430 bio_set_dev(bio, rdev->bdev); 3431 atomic_inc(&rdev->nr_pending); 3432 /* and we write to 'i' (if not in_sync) */ 3433 3434 for (k=0; k<conf->copies; k++) 3435 if (r10_bio->devs[k].devnum == i) 3436 break; 3437 BUG_ON(k == conf->copies); 3438 to_addr = r10_bio->devs[k].addr; 3439 r10_bio->devs[0].devnum = d; 3440 r10_bio->devs[0].addr = from_addr; 3441 r10_bio->devs[1].devnum = i; 3442 r10_bio->devs[1].addr = to_addr; 3443 3444 if (mrdev) { 3445 bio = r10_bio->devs[1].bio; 3446 bio->bi_next = biolist; 3447 biolist = bio; 3448 bio->bi_end_io = end_sync_write; 3449 bio->bi_opf = REQ_OP_WRITE; 3450 bio->bi_iter.bi_sector = to_addr 3451 + mrdev->data_offset; 3452 bio_set_dev(bio, mrdev->bdev); 3453 atomic_inc(&r10_bio->remaining); 3454 } else 3455 r10_bio->devs[1].bio->bi_end_io = NULL; 3456 3457 /* and maybe write to replacement */ 3458 bio = r10_bio->devs[1].repl_bio; 3459 if (bio) 3460 bio->bi_end_io = NULL; 3461 /* Note: if replace is not NULL, then bio 3462 * cannot be NULL as r10buf_pool_alloc will 3463 * have allocated it. 3464 */ 3465 if (!mreplace) 3466 break; 3467 bio->bi_next = biolist; 3468 biolist = bio; 3469 bio->bi_end_io = end_sync_write; 3470 bio->bi_opf = REQ_OP_WRITE; 3471 bio->bi_iter.bi_sector = to_addr + 3472 mreplace->data_offset; 3473 bio_set_dev(bio, mreplace->bdev); 3474 atomic_inc(&r10_bio->remaining); 3475 break; 3476 } 3477 if (j == conf->copies) { 3478 /* Cannot recover, so abort the recovery or 3479 * record a bad block */ 3480 if (any_working) { 3481 /* problem is that there are bad blocks 3482 * on other device(s) 3483 */ 3484 int k; 3485 for (k = 0; k < conf->copies; k++) 3486 if (r10_bio->devs[k].devnum == i) 3487 break; 3488 if (mrdev && !test_bit(In_sync, 3489 &mrdev->flags) 3490 && !rdev_set_badblocks( 3491 mrdev, 3492 r10_bio->devs[k].addr, 3493 max_sync, 0)) 3494 any_working = 0; 3495 if (mreplace && 3496 !rdev_set_badblocks( 3497 mreplace, 3498 r10_bio->devs[k].addr, 3499 max_sync, 0)) 3500 any_working = 0; 3501 } 3502 if (!any_working) { 3503 if (!test_and_set_bit(MD_RECOVERY_INTR, 3504 &mddev->recovery)) 3505 pr_warn("md/raid10:%s: insufficient working devices for recovery.\n", 3506 mdname(mddev)); 3507 mirror->recovery_disabled 3508 = mddev->recovery_disabled; 3509 } else { 3510 error_disk = i; 3511 } 3512 put_buf(r10_bio); 3513 if (rb2) 3514 atomic_dec(&rb2->remaining); 3515 r10_bio = rb2; 3516 if (mrdev) 3517 rdev_dec_pending(mrdev, mddev); 3518 if (mreplace) 3519 rdev_dec_pending(mreplace, mddev); 3520 break; 3521 } 3522 if (mrdev) 3523 rdev_dec_pending(mrdev, mddev); 3524 if (mreplace) 3525 rdev_dec_pending(mreplace, mddev); 3526 if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) { 3527 /* Only want this if there is elsewhere to 3528 * read from. 'j' is currently the first 3529 * readable copy. 3530 */ 3531 int targets = 1; 3532 for (; j < conf->copies; j++) { 3533 int d = r10_bio->devs[j].devnum; 3534 if (conf->mirrors[d].rdev && 3535 test_bit(In_sync, 3536 &conf->mirrors[d].rdev->flags)) 3537 targets++; 3538 } 3539 if (targets == 1) 3540 r10_bio->devs[0].bio->bi_opf 3541 &= ~MD_FAILFAST; 3542 } 3543 } 3544 if (biolist == NULL) { 3545 while (r10_bio) { 3546 struct r10bio *rb2 = r10_bio; 3547 r10_bio = (struct r10bio*) rb2->master_bio; 3548 rb2->master_bio = NULL; 3549 put_buf(rb2); 3550 } 3551 goto giveup; 3552 } 3553 } else { 3554 /* resync. Schedule a read for every block at this virt offset */ 3555 int count = 0; 3556 3557 /* 3558 * Since curr_resync_completed could probably not update in 3559 * time, and we will set cluster_sync_low based on it. 3560 * Let's check against "sector_nr + 2 * RESYNC_SECTORS" for 3561 * safety reason, which ensures curr_resync_completed is 3562 * updated in bitmap_cond_end_sync. 3563 */ 3564 if (md_bitmap_enabled(mddev, false)) 3565 mddev->bitmap_ops->cond_end_sync(mddev, sector_nr, 3566 mddev_is_clustered(mddev) && 3567 (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high)); 3568 3569 if (!md_bitmap_start_sync(mddev, sector_nr, &sync_blocks, 3570 mddev->degraded) && 3571 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, 3572 &mddev->recovery)) { 3573 /* We can skip this block */ 3574 *skipped = 1; 3575 return sync_blocks + sectors_skipped; 3576 } 3577 if (sync_blocks < max_sync) 3578 max_sync = sync_blocks; 3579 r10_bio = raid10_alloc_init_r10buf(conf); 3580 r10_bio->state = 0; 3581 3582 r10_bio->mddev = mddev; 3583 atomic_set(&r10_bio->remaining, 0); 3584 raise_barrier(conf, 0); 3585 conf->next_resync = sector_nr; 3586 3587 r10_bio->master_bio = NULL; 3588 r10_bio->sector = sector_nr; 3589 set_bit(R10BIO_IsSync, &r10_bio->state); 3590 raid10_find_phys(conf, r10_bio); 3591 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; 3592 3593 for (i = 0; i < conf->copies; i++) { 3594 int d = r10_bio->devs[i].devnum; 3595 sector_t first_bad, sector; 3596 sector_t bad_sectors; 3597 struct md_rdev *rdev; 3598 3599 if (r10_bio->devs[i].repl_bio) 3600 r10_bio->devs[i].repl_bio->bi_end_io = NULL; 3601 3602 bio = r10_bio->devs[i].bio; 3603 bio->bi_status = BLK_STS_IOERR; 3604 rdev = conf->mirrors[d].rdev; 3605 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 3606 continue; 3607 3608 sector = r10_bio->devs[i].addr; 3609 if (is_badblock(rdev, sector, max_sync, 3610 &first_bad, &bad_sectors)) { 3611 if (first_bad > sector) 3612 max_sync = first_bad - sector; 3613 else { 3614 bad_sectors -= (sector - first_bad); 3615 if (max_sync > bad_sectors) 3616 max_sync = bad_sectors; 3617 continue; 3618 } 3619 } 3620 atomic_inc(&rdev->nr_pending); 3621 atomic_inc(&r10_bio->remaining); 3622 bio->bi_next = biolist; 3623 biolist = bio; 3624 bio->bi_end_io = end_sync_read; 3625 bio->bi_opf = REQ_OP_READ; 3626 if (test_bit(FailFast, &rdev->flags)) 3627 bio->bi_opf |= MD_FAILFAST; 3628 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3629 bio_set_dev(bio, rdev->bdev); 3630 count++; 3631 3632 rdev = conf->mirrors[d].replacement; 3633 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 3634 continue; 3635 3636 atomic_inc(&rdev->nr_pending); 3637 3638 /* Need to set up for writing to the replacement */ 3639 bio = r10_bio->devs[i].repl_bio; 3640 bio->bi_status = BLK_STS_IOERR; 3641 3642 sector = r10_bio->devs[i].addr; 3643 bio->bi_next = biolist; 3644 biolist = bio; 3645 bio->bi_end_io = end_sync_write; 3646 bio->bi_opf = REQ_OP_WRITE; 3647 if (test_bit(FailFast, &rdev->flags)) 3648 bio->bi_opf |= MD_FAILFAST; 3649 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3650 bio_set_dev(bio, rdev->bdev); 3651 count++; 3652 } 3653 3654 if (count < 2) { 3655 for (i=0; i<conf->copies; i++) { 3656 int d = r10_bio->devs[i].devnum; 3657 if (r10_bio->devs[i].bio->bi_end_io) 3658 rdev_dec_pending(conf->mirrors[d].rdev, 3659 mddev); 3660 if (r10_bio->devs[i].repl_bio && 3661 r10_bio->devs[i].repl_bio->bi_end_io) 3662 rdev_dec_pending( 3663 conf->mirrors[d].replacement, 3664 mddev); 3665 } 3666 put_buf(r10_bio); 3667 biolist = NULL; 3668 goto giveup; 3669 } 3670 } 3671 3672 nr_sectors = 0; 3673 if (sector_nr + max_sync < max_sector) 3674 max_sector = sector_nr + max_sync; 3675 do { 3676 struct page *page; 3677 int len = PAGE_SIZE; 3678 if (sector_nr + (len>>9) > max_sector) 3679 len = (max_sector - sector_nr) << 9; 3680 if (len == 0) 3681 break; 3682 for (bio= biolist ; bio ; bio=bio->bi_next) { 3683 struct resync_pages *rp = get_resync_pages(bio); 3684 page = resync_fetch_page(rp, page_idx); 3685 if (WARN_ON(!bio_add_page(bio, page, len, 0))) { 3686 bio->bi_status = BLK_STS_RESOURCE; 3687 bio_endio(bio); 3688 goto giveup; 3689 } 3690 } 3691 nr_sectors += len>>9; 3692 sector_nr += len>>9; 3693 } while (++page_idx < RESYNC_PAGES); 3694 r10_bio->sectors = nr_sectors; 3695 3696 if (mddev_is_clustered(mddev) && 3697 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3698 /* It is resync not recovery */ 3699 if (conf->cluster_sync_high < sector_nr + nr_sectors) { 3700 conf->cluster_sync_low = mddev->curr_resync_completed; 3701 raid10_set_cluster_sync_high(conf); 3702 /* Send resync message */ 3703 mddev->cluster_ops->resync_info_update(mddev, 3704 conf->cluster_sync_low, 3705 conf->cluster_sync_high); 3706 } 3707 } else if (mddev_is_clustered(mddev)) { 3708 /* This is recovery not resync */ 3709 sector_t sect_va1, sect_va2; 3710 bool broadcast_msg = false; 3711 3712 for (i = 0; i < conf->geo.raid_disks; i++) { 3713 /* 3714 * sector_nr is a device address for recovery, so we 3715 * need translate it to array address before compare 3716 * with cluster_sync_high. 3717 */ 3718 sect_va1 = raid10_find_virt(conf, sector_nr, i); 3719 3720 if (conf->cluster_sync_high < sect_va1 + nr_sectors) { 3721 broadcast_msg = true; 3722 /* 3723 * curr_resync_completed is similar as 3724 * sector_nr, so make the translation too. 3725 */ 3726 sect_va2 = raid10_find_virt(conf, 3727 mddev->curr_resync_completed, i); 3728 3729 if (conf->cluster_sync_low == 0 || 3730 conf->cluster_sync_low > sect_va2) 3731 conf->cluster_sync_low = sect_va2; 3732 } 3733 } 3734 if (broadcast_msg) { 3735 raid10_set_cluster_sync_high(conf); 3736 mddev->cluster_ops->resync_info_update(mddev, 3737 conf->cluster_sync_low, 3738 conf->cluster_sync_high); 3739 } 3740 } 3741 3742 while (biolist) { 3743 bio = biolist; 3744 biolist = biolist->bi_next; 3745 3746 bio->bi_next = NULL; 3747 r10_bio = get_resync_r10bio(bio); 3748 r10_bio->sectors = nr_sectors; 3749 3750 if (bio->bi_end_io == end_sync_read) { 3751 bio->bi_status = 0; 3752 submit_bio_noacct(bio); 3753 } 3754 } 3755 3756 if (sectors_skipped) 3757 /* pretend they weren't skipped, it makes 3758 * no important difference in this case 3759 */ 3760 md_done_sync(mddev, sectors_skipped, 1); 3761 3762 return sectors_skipped + nr_sectors; 3763 giveup: 3764 /* There is nowhere to write, so all non-sync 3765 * drives must be failed or in resync, all drives 3766 * have a bad block, so try the next chunk... 3767 */ 3768 if (sector_nr + max_sync < max_sector) 3769 max_sector = sector_nr + max_sync; 3770 3771 sectors_skipped += (max_sector - sector_nr); 3772 chunks_skipped ++; 3773 sector_nr = max_sector; 3774 goto skipped; 3775} 3776 3777static sector_t 3778raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3779{ 3780 sector_t size; 3781 struct r10conf *conf = mddev->private; 3782 3783 if (!raid_disks) 3784 raid_disks = min(conf->geo.raid_disks, 3785 conf->prev.raid_disks); 3786 if (!sectors) 3787 sectors = conf->dev_sectors; 3788 3789 size = sectors >> conf->geo.chunk_shift; 3790 sector_div(size, conf->geo.far_copies); 3791 size = size * raid_disks; 3792 sector_div(size, conf->geo.near_copies); 3793 3794 return size << conf->geo.chunk_shift; 3795} 3796 3797static void calc_sectors(struct r10conf *conf, sector_t size) 3798{ 3799 /* Calculate the number of sectors-per-device that will 3800 * actually be used, and set conf->dev_sectors and 3801 * conf->stride 3802 */ 3803 3804 size = size >> conf->geo.chunk_shift; 3805 sector_div(size, conf->geo.far_copies); 3806 size = size * conf->geo.raid_disks; 3807 sector_div(size, conf->geo.near_copies); 3808 /* 'size' is now the number of chunks in the array */ 3809 /* calculate "used chunks per device" */ 3810 size = size * conf->copies; 3811 3812 /* We need to round up when dividing by raid_disks to 3813 * get the stride size. 3814 */ 3815 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks); 3816 3817 conf->dev_sectors = size << conf->geo.chunk_shift; 3818 3819 if (conf->geo.far_offset) 3820 conf->geo.stride = 1 << conf->geo.chunk_shift; 3821 else { 3822 sector_div(size, conf->geo.far_copies); 3823 conf->geo.stride = size << conf->geo.chunk_shift; 3824 } 3825} 3826 3827enum geo_type {geo_new, geo_old, geo_start}; 3828static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) 3829{ 3830 int nc, fc, fo; 3831 int layout, chunk, disks; 3832 switch (new) { 3833 case geo_old: 3834 layout = mddev->layout; 3835 chunk = mddev->chunk_sectors; 3836 disks = mddev->raid_disks - mddev->delta_disks; 3837 break; 3838 case geo_new: 3839 layout = mddev->new_layout; 3840 chunk = mddev->new_chunk_sectors; 3841 disks = mddev->raid_disks; 3842 break; 3843 default: /* avoid 'may be unused' warnings */ 3844 case geo_start: /* new when starting reshape - raid_disks not 3845 * updated yet. */ 3846 layout = mddev->new_layout; 3847 chunk = mddev->new_chunk_sectors; 3848 disks = mddev->raid_disks + mddev->delta_disks; 3849 break; 3850 } 3851 if (layout >> 19) 3852 return -1; 3853 if (chunk < (PAGE_SIZE >> 9) || 3854 !is_power_of_2(chunk)) 3855 return -2; 3856 nc = layout & 255; 3857 fc = (layout >> 8) & 255; 3858 fo = layout & (1<<16); 3859 geo->raid_disks = disks; 3860 geo->near_copies = nc; 3861 geo->far_copies = fc; 3862 geo->far_offset = fo; 3863 switch (layout >> 17) { 3864 case 0: /* original layout. simple but not always optimal */ 3865 geo->far_set_size = disks; 3866 break; 3867 case 1: /* "improved" layout which was buggy. Hopefully no-one is 3868 * actually using this, but leave code here just in case.*/ 3869 geo->far_set_size = disks/fc; 3870 WARN(geo->far_set_size < fc, 3871 "This RAID10 layout does not provide data safety - please backup and create new array\n"); 3872 break; 3873 case 2: /* "improved" layout fixed to match documentation */ 3874 geo->far_set_size = fc * nc; 3875 break; 3876 default: /* Not a valid layout */ 3877 return -1; 3878 } 3879 geo->chunk_mask = chunk - 1; 3880 geo->chunk_shift = ffz(~chunk); 3881 return nc*fc; 3882} 3883 3884static void raid10_free_conf(struct r10conf *conf) 3885{ 3886 if (!conf) 3887 return; 3888 3889 mempool_exit(&conf->r10bio_pool); 3890 kfree(conf->mirrors); 3891 kfree(conf->mirrors_old); 3892 kfree(conf->mirrors_new); 3893 safe_put_page(conf->tmppage); 3894 bioset_exit(&conf->bio_split); 3895 kfree(conf); 3896} 3897 3898static struct r10conf *setup_conf(struct mddev *mddev) 3899{ 3900 struct r10conf *conf = NULL; 3901 int err = -EINVAL; 3902 struct geom geo; 3903 int copies; 3904 3905 copies = setup_geo(&geo, mddev, geo_new); 3906 3907 if (copies == -2) { 3908 pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n", 3909 mdname(mddev), PAGE_SIZE); 3910 goto out; 3911 } 3912 3913 if (copies < 2 || copies > mddev->raid_disks) { 3914 pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n", 3915 mdname(mddev), mddev->new_layout); 3916 goto out; 3917 } 3918 3919 err = -ENOMEM; 3920 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL); 3921 if (!conf) 3922 goto out; 3923 3924 /* FIXME calc properly */ 3925 conf->mirrors = kcalloc(mddev->raid_disks + max(0, -mddev->delta_disks), 3926 sizeof(struct raid10_info), 3927 GFP_KERNEL); 3928 if (!conf->mirrors) 3929 goto out; 3930 3931 conf->tmppage = alloc_page(GFP_KERNEL); 3932 if (!conf->tmppage) 3933 goto out; 3934 3935 conf->geo = geo; 3936 conf->copies = copies; 3937 err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc, 3938 rbio_pool_free, conf); 3939 if (err) 3940 goto out; 3941 3942 err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); 3943 if (err) 3944 goto out; 3945 3946 calc_sectors(conf, mddev->dev_sectors); 3947 if (mddev->reshape_position == MaxSector) { 3948 conf->prev = conf->geo; 3949 conf->reshape_progress = MaxSector; 3950 } else { 3951 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) { 3952 err = -EINVAL; 3953 goto out; 3954 } 3955 conf->reshape_progress = mddev->reshape_position; 3956 if (conf->prev.far_offset) 3957 conf->prev.stride = 1 << conf->prev.chunk_shift; 3958 else 3959 /* far_copies must be 1 */ 3960 conf->prev.stride = conf->dev_sectors; 3961 } 3962 conf->reshape_safe = conf->reshape_progress; 3963 spin_lock_init(&conf->device_lock); 3964 INIT_LIST_HEAD(&conf->retry_list); 3965 INIT_LIST_HEAD(&conf->bio_end_io_list); 3966 3967 seqlock_init(&conf->resync_lock); 3968 init_waitqueue_head(&conf->wait_barrier); 3969 atomic_set(&conf->nr_pending, 0); 3970 3971 err = -ENOMEM; 3972 rcu_assign_pointer(conf->thread, 3973 md_register_thread(raid10d, mddev, "raid10")); 3974 if (!conf->thread) 3975 goto out; 3976 3977 conf->mddev = mddev; 3978 return conf; 3979 3980 out: 3981 raid10_free_conf(conf); 3982 return ERR_PTR(err); 3983} 3984 3985static unsigned int raid10_nr_stripes(struct r10conf *conf) 3986{ 3987 unsigned int raid_disks = conf->geo.raid_disks; 3988 3989 if (conf->geo.raid_disks % conf->geo.near_copies) 3990 return raid_disks; 3991 return raid_disks / conf->geo.near_copies; 3992} 3993 3994static int raid10_set_queue_limits(struct mddev *mddev) 3995{ 3996 struct r10conf *conf = mddev->private; 3997 struct queue_limits lim; 3998 int err; 3999 4000 md_init_stacking_limits(&lim); 4001 lim.max_write_zeroes_sectors = 0; 4002 lim.max_hw_wzeroes_unmap_sectors = 0; 4003 lim.logical_block_size = mddev->logical_block_size; 4004 lim.io_min = mddev->chunk_sectors << 9; 4005 lim.chunk_sectors = mddev->chunk_sectors; 4006 lim.io_opt = lim.io_min * raid10_nr_stripes(conf); 4007 lim.features |= BLK_FEAT_ATOMIC_WRITES; 4008 err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); 4009 if (err) 4010 return err; 4011 return queue_limits_set(mddev->gendisk->queue, &lim); 4012} 4013 4014static int raid10_run(struct mddev *mddev) 4015{ 4016 struct r10conf *conf; 4017 int i, disk_idx; 4018 struct raid10_info *disk; 4019 struct md_rdev *rdev; 4020 sector_t size; 4021 sector_t min_offset_diff = 0; 4022 int first = 1; 4023 int ret = -EIO; 4024 4025 if (mddev->private == NULL) { 4026 conf = setup_conf(mddev); 4027 if (IS_ERR(conf)) 4028 return PTR_ERR(conf); 4029 mddev->private = conf; 4030 } 4031 conf = mddev->private; 4032 if (!conf) 4033 goto out; 4034 4035 rcu_assign_pointer(mddev->thread, conf->thread); 4036 rcu_assign_pointer(conf->thread, NULL); 4037 4038 if (mddev_is_clustered(conf->mddev)) { 4039 int fc, fo; 4040 4041 fc = (mddev->layout >> 8) & 255; 4042 fo = mddev->layout & (1<<16); 4043 if (fc > 1 || fo > 0) { 4044 pr_err("only near layout is supported by clustered" 4045 " raid10\n"); 4046 goto out_free_conf; 4047 } 4048 } 4049 4050 rdev_for_each(rdev, mddev) { 4051 long long diff; 4052 4053 disk_idx = rdev->raid_disk; 4054 if (disk_idx < 0) 4055 continue; 4056 if (disk_idx >= conf->geo.raid_disks && 4057 disk_idx >= conf->prev.raid_disks) 4058 continue; 4059 disk = conf->mirrors + disk_idx; 4060 4061 if (test_bit(Replacement, &rdev->flags)) { 4062 if (disk->replacement) 4063 goto out_free_conf; 4064 disk->replacement = rdev; 4065 } else { 4066 if (disk->rdev) 4067 goto out_free_conf; 4068 disk->rdev = rdev; 4069 } 4070 diff = (rdev->new_data_offset - rdev->data_offset); 4071 if (!mddev->reshape_backwards) 4072 diff = -diff; 4073 if (diff < 0) 4074 diff = 0; 4075 if (first || diff < min_offset_diff) 4076 min_offset_diff = diff; 4077 4078 disk->head_position = 0; 4079 first = 0; 4080 } 4081 4082 if (!mddev_is_dm(conf->mddev)) { 4083 int err = raid10_set_queue_limits(mddev); 4084 4085 if (err) { 4086 ret = err; 4087 goto out_free_conf; 4088 } 4089 } 4090 4091 /* need to check that every block has at least one working mirror */ 4092 if (!enough(conf, -1)) { 4093 pr_err("md/raid10:%s: not enough operational mirrors.\n", 4094 mdname(mddev)); 4095 goto out_free_conf; 4096 } 4097 4098 if (conf->reshape_progress != MaxSector) { 4099 /* must ensure that shape change is supported */ 4100 if (conf->geo.far_copies != 1 && 4101 conf->geo.far_offset == 0) 4102 goto out_free_conf; 4103 if (conf->prev.far_copies != 1 && 4104 conf->prev.far_offset == 0) 4105 goto out_free_conf; 4106 } 4107 4108 mddev->degraded = 0; 4109 for (i = 0; 4110 i < conf->geo.raid_disks 4111 || i < conf->prev.raid_disks; 4112 i++) { 4113 4114 disk = conf->mirrors + i; 4115 4116 if (!disk->rdev && disk->replacement) { 4117 /* The replacement is all we have - use it */ 4118 disk->rdev = disk->replacement; 4119 disk->replacement = NULL; 4120 clear_bit(Replacement, &disk->rdev->flags); 4121 } 4122 4123 if (!disk->rdev || 4124 !test_bit(In_sync, &disk->rdev->flags)) { 4125 disk->head_position = 0; 4126 mddev->degraded++; 4127 if (disk->rdev && 4128 disk->rdev->saved_raid_disk < 0) 4129 conf->fullsync = 1; 4130 } 4131 4132 if (disk->replacement && 4133 !test_bit(In_sync, &disk->replacement->flags) && 4134 disk->replacement->saved_raid_disk < 0) { 4135 conf->fullsync = 1; 4136 } 4137 4138 disk->recovery_disabled = mddev->recovery_disabled - 1; 4139 } 4140 4141 if (mddev->resync_offset != MaxSector) 4142 pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n", 4143 mdname(mddev)); 4144 pr_info("md/raid10:%s: active with %d out of %d devices\n", 4145 mdname(mddev), conf->geo.raid_disks - mddev->degraded, 4146 conf->geo.raid_disks); 4147 /* 4148 * Ok, everything is just fine now 4149 */ 4150 mddev->dev_sectors = conf->dev_sectors; 4151 size = raid10_size(mddev, 0, 0); 4152 md_set_array_sectors(mddev, size); 4153 mddev->resync_max_sectors = size; 4154 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags); 4155 4156 if (md_integrity_register(mddev)) 4157 goto out_free_conf; 4158 4159 if (conf->reshape_progress != MaxSector) { 4160 unsigned long before_length, after_length; 4161 4162 before_length = ((1 << conf->prev.chunk_shift) * 4163 conf->prev.far_copies); 4164 after_length = ((1 << conf->geo.chunk_shift) * 4165 conf->geo.far_copies); 4166 4167 if (max(before_length, after_length) > min_offset_diff) { 4168 /* This cannot work */ 4169 pr_warn("md/raid10: offset difference not enough to continue reshape\n"); 4170 goto out_free_conf; 4171 } 4172 conf->offset_diff = min_offset_diff; 4173 4174 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4175 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4176 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4177 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4178 } 4179 4180 return 0; 4181 4182out_free_conf: 4183 md_unregister_thread(mddev, &mddev->thread); 4184 raid10_free_conf(conf); 4185 mddev->private = NULL; 4186out: 4187 return ret; 4188} 4189 4190static void raid10_free(struct mddev *mddev, void *priv) 4191{ 4192 raid10_free_conf(priv); 4193} 4194 4195static void raid10_quiesce(struct mddev *mddev, int quiesce) 4196{ 4197 struct r10conf *conf = mddev->private; 4198 4199 if (quiesce) 4200 raise_barrier(conf, 0); 4201 else 4202 lower_barrier(conf); 4203} 4204 4205static int raid10_resize(struct mddev *mddev, sector_t sectors) 4206{ 4207 /* Resize of 'far' arrays is not supported. 4208 * For 'near' and 'offset' arrays we can set the 4209 * number of sectors used to be an appropriate multiple 4210 * of the chunk size. 4211 * For 'offset', this is far_copies*chunksize. 4212 * For 'near' the multiplier is the LCM of 4213 * near_copies and raid_disks. 4214 * So if far_copies > 1 && !far_offset, fail. 4215 * Else find LCM(raid_disks, near_copy)*far_copies and 4216 * multiply by chunk_size. Then round to this number. 4217 * This is mostly done by raid10_size() 4218 */ 4219 struct r10conf *conf = mddev->private; 4220 sector_t oldsize, size; 4221 4222 if (mddev->reshape_position != MaxSector) 4223 return -EBUSY; 4224 4225 if (conf->geo.far_copies > 1 && !conf->geo.far_offset) 4226 return -EINVAL; 4227 4228 oldsize = raid10_size(mddev, 0, 0); 4229 size = raid10_size(mddev, sectors, 0); 4230 if (mddev->external_size && 4231 mddev->array_sectors > size) 4232 return -EINVAL; 4233 4234 if (md_bitmap_enabled(mddev, false)) { 4235 int ret = mddev->bitmap_ops->resize(mddev, size, 0); 4236 4237 if (ret) 4238 return ret; 4239 } 4240 4241 md_set_array_sectors(mddev, size); 4242 if (sectors > mddev->dev_sectors && 4243 mddev->resync_offset > oldsize) { 4244 mddev->resync_offset = oldsize; 4245 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4246 } 4247 calc_sectors(conf, sectors); 4248 mddev->dev_sectors = conf->dev_sectors; 4249 mddev->resync_max_sectors = size; 4250 return 0; 4251} 4252 4253static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) 4254{ 4255 struct md_rdev *rdev; 4256 struct r10conf *conf; 4257 4258 if (mddev->degraded > 0) { 4259 pr_warn("md/raid10:%s: Error: degraded raid0!\n", 4260 mdname(mddev)); 4261 return ERR_PTR(-EINVAL); 4262 } 4263 sector_div(size, devs); 4264 4265 /* Set new parameters */ 4266 mddev->new_level = 10; 4267 /* new layout: far_copies = 1, near_copies = 2 */ 4268 mddev->new_layout = (1<<8) + 2; 4269 mddev->new_chunk_sectors = mddev->chunk_sectors; 4270 mddev->delta_disks = mddev->raid_disks; 4271 mddev->raid_disks *= 2; 4272 /* make sure it will be not marked as dirty */ 4273 mddev->resync_offset = MaxSector; 4274 mddev->dev_sectors = size; 4275 4276 conf = setup_conf(mddev); 4277 if (!IS_ERR(conf)) { 4278 rdev_for_each(rdev, mddev) 4279 if (rdev->raid_disk >= 0) { 4280 rdev->new_raid_disk = rdev->raid_disk * 2; 4281 rdev->sectors = size; 4282 } 4283 } 4284 4285 return conf; 4286} 4287 4288static void *raid10_takeover(struct mddev *mddev) 4289{ 4290 struct r0conf *raid0_conf; 4291 4292 /* raid10 can take over: 4293 * raid0 - providing it has only two drives 4294 */ 4295 if (mddev->level == 0) { 4296 /* for raid0 takeover only one zone is supported */ 4297 raid0_conf = mddev->private; 4298 if (raid0_conf->nr_strip_zones > 1) { 4299 pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n", 4300 mdname(mddev)); 4301 return ERR_PTR(-EINVAL); 4302 } 4303 return raid10_takeover_raid0(mddev, 4304 raid0_conf->strip_zone->zone_end, 4305 raid0_conf->strip_zone->nb_dev); 4306 } 4307 return ERR_PTR(-EINVAL); 4308} 4309 4310static int raid10_check_reshape(struct mddev *mddev) 4311{ 4312 /* Called when there is a request to change 4313 * - layout (to ->new_layout) 4314 * - chunk size (to ->new_chunk_sectors) 4315 * - raid_disks (by delta_disks) 4316 * or when trying to restart a reshape that was ongoing. 4317 * 4318 * We need to validate the request and possibly allocate 4319 * space if that might be an issue later. 4320 * 4321 * Currently we reject any reshape of a 'far' mode array, 4322 * allow chunk size to change if new is generally acceptable, 4323 * allow raid_disks to increase, and allow 4324 * a switch between 'near' mode and 'offset' mode. 4325 */ 4326 struct r10conf *conf = mddev->private; 4327 struct geom geo; 4328 4329 if (conf->geo.far_copies != 1 && !conf->geo.far_offset) 4330 return -EINVAL; 4331 4332 if (setup_geo(&geo, mddev, geo_start) != conf->copies) 4333 /* mustn't change number of copies */ 4334 return -EINVAL; 4335 if (geo.far_copies > 1 && !geo.far_offset) 4336 /* Cannot switch to 'far' mode */ 4337 return -EINVAL; 4338 4339 if (mddev->array_sectors & geo.chunk_mask) 4340 /* not factor of array size */ 4341 return -EINVAL; 4342 4343 if (!enough(conf, -1)) 4344 return -EINVAL; 4345 4346 kfree(conf->mirrors_new); 4347 conf->mirrors_new = NULL; 4348 if (mddev->delta_disks > 0) { 4349 /* allocate new 'mirrors' list */ 4350 conf->mirrors_new = 4351 kcalloc(mddev->raid_disks + mddev->delta_disks, 4352 sizeof(struct raid10_info), 4353 GFP_KERNEL); 4354 if (!conf->mirrors_new) 4355 return -ENOMEM; 4356 } 4357 return 0; 4358} 4359 4360/* 4361 * Need to check if array has failed when deciding whether to: 4362 * - start an array 4363 * - remove non-faulty devices 4364 * - add a spare 4365 * - allow a reshape 4366 * This determination is simple when no reshape is happening. 4367 * However if there is a reshape, we need to carefully check 4368 * both the before and after sections. 4369 * This is because some failed devices may only affect one 4370 * of the two sections, and some non-in_sync devices may 4371 * be insync in the section most affected by failed devices. 4372 */ 4373static int calc_degraded(struct r10conf *conf) 4374{ 4375 int degraded, degraded2; 4376 int i; 4377 4378 degraded = 0; 4379 /* 'prev' section first */ 4380 for (i = 0; i < conf->prev.raid_disks; i++) { 4381 struct md_rdev *rdev = conf->mirrors[i].rdev; 4382 4383 if (!rdev || test_bit(Faulty, &rdev->flags)) 4384 degraded++; 4385 else if (!test_bit(In_sync, &rdev->flags)) 4386 /* When we can reduce the number of devices in 4387 * an array, this might not contribute to 4388 * 'degraded'. It does now. 4389 */ 4390 degraded++; 4391 } 4392 if (conf->geo.raid_disks == conf->prev.raid_disks) 4393 return degraded; 4394 degraded2 = 0; 4395 for (i = 0; i < conf->geo.raid_disks; i++) { 4396 struct md_rdev *rdev = conf->mirrors[i].rdev; 4397 4398 if (!rdev || test_bit(Faulty, &rdev->flags)) 4399 degraded2++; 4400 else if (!test_bit(In_sync, &rdev->flags)) { 4401 /* If reshape is increasing the number of devices, 4402 * this section has already been recovered, so 4403 * it doesn't contribute to degraded. 4404 * else it does. 4405 */ 4406 if (conf->geo.raid_disks <= conf->prev.raid_disks) 4407 degraded2++; 4408 } 4409 } 4410 if (degraded2 > degraded) 4411 return degraded2; 4412 return degraded; 4413} 4414 4415static int raid10_start_reshape(struct mddev *mddev) 4416{ 4417 /* A 'reshape' has been requested. This commits 4418 * the various 'new' fields and sets MD_RECOVER_RESHAPE 4419 * This also checks if there are enough spares and adds them 4420 * to the array. 4421 * We currently require enough spares to make the final 4422 * array non-degraded. We also require that the difference 4423 * between old and new data_offset - on each device - is 4424 * enough that we never risk over-writing. 4425 */ 4426 4427 unsigned long before_length, after_length; 4428 sector_t min_offset_diff = 0; 4429 int first = 1; 4430 struct geom new; 4431 struct r10conf *conf = mddev->private; 4432 struct md_rdev *rdev; 4433 int spares = 0; 4434 int ret; 4435 4436 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 4437 return -EBUSY; 4438 4439 if (setup_geo(&new, mddev, geo_start) != conf->copies) 4440 return -EINVAL; 4441 4442 before_length = ((1 << conf->prev.chunk_shift) * 4443 conf->prev.far_copies); 4444 after_length = ((1 << conf->geo.chunk_shift) * 4445 conf->geo.far_copies); 4446 4447 rdev_for_each(rdev, mddev) { 4448 if (!test_bit(In_sync, &rdev->flags) 4449 && !test_bit(Faulty, &rdev->flags)) 4450 spares++; 4451 if (rdev->raid_disk >= 0) { 4452 long long diff = (rdev->new_data_offset 4453 - rdev->data_offset); 4454 if (!mddev->reshape_backwards) 4455 diff = -diff; 4456 if (diff < 0) 4457 diff = 0; 4458 if (first || diff < min_offset_diff) 4459 min_offset_diff = diff; 4460 first = 0; 4461 } 4462 } 4463 4464 if (max(before_length, after_length) > min_offset_diff) 4465 return -EINVAL; 4466 4467 if (spares < mddev->delta_disks) 4468 return -EINVAL; 4469 4470 conf->offset_diff = min_offset_diff; 4471 spin_lock_irq(&conf->device_lock); 4472 if (conf->mirrors_new) { 4473 memcpy(conf->mirrors_new, conf->mirrors, 4474 sizeof(struct raid10_info)*conf->prev.raid_disks); 4475 smp_mb(); 4476 kfree(conf->mirrors_old); 4477 conf->mirrors_old = conf->mirrors; 4478 conf->mirrors = conf->mirrors_new; 4479 conf->mirrors_new = NULL; 4480 } 4481 setup_geo(&conf->geo, mddev, geo_start); 4482 smp_mb(); 4483 if (mddev->reshape_backwards) { 4484 sector_t size = raid10_size(mddev, 0, 0); 4485 if (size < mddev->array_sectors) { 4486 spin_unlock_irq(&conf->device_lock); 4487 pr_warn("md/raid10:%s: array size must be reduce before number of disks\n", 4488 mdname(mddev)); 4489 return -EINVAL; 4490 } 4491 mddev->resync_max_sectors = size; 4492 conf->reshape_progress = size; 4493 } else 4494 conf->reshape_progress = 0; 4495 conf->reshape_safe = conf->reshape_progress; 4496 spin_unlock_irq(&conf->device_lock); 4497 4498 if (mddev->delta_disks && mddev->bitmap) { 4499 struct mdp_superblock_1 *sb = NULL; 4500 sector_t oldsize, newsize; 4501 4502 oldsize = raid10_size(mddev, 0, 0); 4503 newsize = raid10_size(mddev, 0, conf->geo.raid_disks); 4504 4505 if (!mddev_is_clustered(mddev) && 4506 md_bitmap_enabled(mddev, false)) { 4507 ret = mddev->bitmap_ops->resize(mddev, newsize, 0); 4508 if (ret) 4509 goto abort; 4510 else 4511 goto out; 4512 } 4513 4514 rdev_for_each(rdev, mddev) { 4515 if (rdev->raid_disk > -1 && 4516 !test_bit(Faulty, &rdev->flags)) 4517 sb = page_address(rdev->sb_page); 4518 } 4519 4520 /* 4521 * some node is already performing reshape, and no need to 4522 * call bitmap_ops->resize again since it should be called when 4523 * receiving BITMAP_RESIZE msg 4524 */ 4525 if ((sb && (le32_to_cpu(sb->feature_map) & 4526 MD_FEATURE_RESHAPE_ACTIVE)) || (oldsize == newsize)) 4527 goto out; 4528 4529 /* cluster can't be setup without bitmap */ 4530 ret = mddev->bitmap_ops->resize(mddev, newsize, 0); 4531 if (ret) 4532 goto abort; 4533 4534 ret = mddev->cluster_ops->resize_bitmaps(mddev, newsize, oldsize); 4535 if (ret) { 4536 mddev->bitmap_ops->resize(mddev, oldsize, 0); 4537 goto abort; 4538 } 4539 } 4540out: 4541 if (mddev->delta_disks > 0) { 4542 rdev_for_each(rdev, mddev) 4543 if (rdev->raid_disk < 0 && 4544 !test_bit(Faulty, &rdev->flags)) { 4545 if (raid10_add_disk(mddev, rdev) == 0) { 4546 if (rdev->raid_disk >= 4547 conf->prev.raid_disks) 4548 set_bit(In_sync, &rdev->flags); 4549 else 4550 rdev->recovery_offset = 0; 4551 4552 /* Failure here is OK */ 4553 sysfs_link_rdev(mddev, rdev); 4554 } 4555 } else if (rdev->raid_disk >= conf->prev.raid_disks 4556 && !test_bit(Faulty, &rdev->flags)) { 4557 /* This is a spare that was manually added */ 4558 set_bit(In_sync, &rdev->flags); 4559 } 4560 } 4561 /* When a reshape changes the number of devices, 4562 * ->degraded is measured against the larger of the 4563 * pre and post numbers. 4564 */ 4565 spin_lock_irq(&conf->device_lock); 4566 mddev->degraded = calc_degraded(conf); 4567 spin_unlock_irq(&conf->device_lock); 4568 mddev->raid_disks = conf->geo.raid_disks; 4569 mddev->reshape_position = conf->reshape_progress; 4570 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 4571 4572 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4573 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4574 clear_bit(MD_RECOVERY_DONE, &mddev->recovery); 4575 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4576 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4577 conf->reshape_checkpoint = jiffies; 4578 md_new_event(); 4579 return 0; 4580 4581abort: 4582 mddev->recovery = 0; 4583 spin_lock_irq(&conf->device_lock); 4584 conf->geo = conf->prev; 4585 mddev->raid_disks = conf->geo.raid_disks; 4586 rdev_for_each(rdev, mddev) 4587 rdev->new_data_offset = rdev->data_offset; 4588 smp_wmb(); 4589 conf->reshape_progress = MaxSector; 4590 conf->reshape_safe = MaxSector; 4591 mddev->reshape_position = MaxSector; 4592 spin_unlock_irq(&conf->device_lock); 4593 return ret; 4594} 4595 4596/* Calculate the last device-address that could contain 4597 * any block from the chunk that includes the array-address 's' 4598 * and report the next address. 4599 * i.e. the address returned will be chunk-aligned and after 4600 * any data that is in the chunk containing 's'. 4601 */ 4602static sector_t last_dev_address(sector_t s, struct geom *geo) 4603{ 4604 s = (s | geo->chunk_mask) + 1; 4605 s >>= geo->chunk_shift; 4606 s *= geo->near_copies; 4607 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks); 4608 s *= geo->far_copies; 4609 s <<= geo->chunk_shift; 4610 return s; 4611} 4612 4613/* Calculate the first device-address that could contain 4614 * any block from the chunk that includes the array-address 's'. 4615 * This too will be the start of a chunk 4616 */ 4617static sector_t first_dev_address(sector_t s, struct geom *geo) 4618{ 4619 s >>= geo->chunk_shift; 4620 s *= geo->near_copies; 4621 sector_div(s, geo->raid_disks); 4622 s *= geo->far_copies; 4623 s <<= geo->chunk_shift; 4624 return s; 4625} 4626 4627static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 4628 int *skipped) 4629{ 4630 /* We simply copy at most one chunk (smallest of old and new) 4631 * at a time, possibly less if that exceeds RESYNC_PAGES, 4632 * or we hit a bad block or something. 4633 * This might mean we pause for normal IO in the middle of 4634 * a chunk, but that is not a problem as mddev->reshape_position 4635 * can record any location. 4636 * 4637 * If we will want to write to a location that isn't 4638 * yet recorded as 'safe' (i.e. in metadata on disk) then 4639 * we need to flush all reshape requests and update the metadata. 4640 * 4641 * When reshaping forwards (e.g. to more devices), we interpret 4642 * 'safe' as the earliest block which might not have been copied 4643 * down yet. We divide this by previous stripe size and multiply 4644 * by previous stripe length to get lowest device offset that we 4645 * cannot write to yet. 4646 * We interpret 'sector_nr' as an address that we want to write to. 4647 * From this we use last_device_address() to find where we might 4648 * write to, and first_device_address on the 'safe' position. 4649 * If this 'next' write position is after the 'safe' position, 4650 * we must update the metadata to increase the 'safe' position. 4651 * 4652 * When reshaping backwards, we round in the opposite direction 4653 * and perform the reverse test: next write position must not be 4654 * less than current safe position. 4655 * 4656 * In all this the minimum difference in data offsets 4657 * (conf->offset_diff - always positive) allows a bit of slack, 4658 * so next can be after 'safe', but not by more than offset_diff 4659 * 4660 * We need to prepare all the bios here before we start any IO 4661 * to ensure the size we choose is acceptable to all devices. 4662 * The means one for each copy for write-out and an extra one for 4663 * read-in. 4664 * We store the read-in bio in ->master_bio and the others in 4665 * ->devs[x].bio and ->devs[x].repl_bio. 4666 */ 4667 struct r10conf *conf = mddev->private; 4668 struct r10bio *r10_bio; 4669 sector_t next, safe, last; 4670 int max_sectors; 4671 int nr_sectors; 4672 int s; 4673 struct md_rdev *rdev; 4674 int need_flush = 0; 4675 struct bio *blist; 4676 struct bio *bio, *read_bio; 4677 int sectors_done = 0; 4678 struct page **pages; 4679 4680 if (sector_nr == 0) { 4681 /* If restarting in the middle, skip the initial sectors */ 4682 if (mddev->reshape_backwards && 4683 conf->reshape_progress < raid10_size(mddev, 0, 0)) { 4684 sector_nr = (raid10_size(mddev, 0, 0) 4685 - conf->reshape_progress); 4686 } else if (!mddev->reshape_backwards && 4687 conf->reshape_progress > 0) 4688 sector_nr = conf->reshape_progress; 4689 if (sector_nr) { 4690 mddev->curr_resync_completed = sector_nr; 4691 sysfs_notify_dirent_safe(mddev->sysfs_completed); 4692 *skipped = 1; 4693 return sector_nr; 4694 } 4695 } 4696 4697 /* We don't use sector_nr to track where we are up to 4698 * as that doesn't work well for ->reshape_backwards. 4699 * So just use ->reshape_progress. 4700 */ 4701 if (mddev->reshape_backwards) { 4702 /* 'next' is the earliest device address that we might 4703 * write to for this chunk in the new layout 4704 */ 4705 next = first_dev_address(conf->reshape_progress - 1, 4706 &conf->geo); 4707 4708 /* 'safe' is the last device address that we might read from 4709 * in the old layout after a restart 4710 */ 4711 safe = last_dev_address(conf->reshape_safe - 1, 4712 &conf->prev); 4713 4714 if (next + conf->offset_diff < safe) 4715 need_flush = 1; 4716 4717 last = conf->reshape_progress - 1; 4718 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask 4719 & conf->prev.chunk_mask); 4720 if (sector_nr + RESYNC_SECTORS < last) 4721 sector_nr = last + 1 - RESYNC_SECTORS; 4722 } else { 4723 /* 'next' is after the last device address that we 4724 * might write to for this chunk in the new layout 4725 */ 4726 next = last_dev_address(conf->reshape_progress, &conf->geo); 4727 4728 /* 'safe' is the earliest device address that we might 4729 * read from in the old layout after a restart 4730 */ 4731 safe = first_dev_address(conf->reshape_safe, &conf->prev); 4732 4733 /* Need to update metadata if 'next' might be beyond 'safe' 4734 * as that would possibly corrupt data 4735 */ 4736 if (next > safe + conf->offset_diff) 4737 need_flush = 1; 4738 4739 sector_nr = conf->reshape_progress; 4740 last = sector_nr | (conf->geo.chunk_mask 4741 & conf->prev.chunk_mask); 4742 4743 if (sector_nr + RESYNC_SECTORS <= last) 4744 last = sector_nr + RESYNC_SECTORS - 1; 4745 } 4746 4747 if (need_flush || 4748 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4749 /* Need to update reshape_position in metadata */ 4750 wait_barrier(conf, false); 4751 mddev->reshape_position = conf->reshape_progress; 4752 if (mddev->reshape_backwards) 4753 mddev->curr_resync_completed = raid10_size(mddev, 0, 0) 4754 - conf->reshape_progress; 4755 else 4756 mddev->curr_resync_completed = conf->reshape_progress; 4757 conf->reshape_checkpoint = jiffies; 4758 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 4759 md_wakeup_thread(mddev->thread); 4760 wait_event(mddev->sb_wait, mddev->sb_flags == 0 || 4761 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4762 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 4763 allow_barrier(conf); 4764 return sectors_done; 4765 } 4766 conf->reshape_safe = mddev->reshape_position; 4767 allow_barrier(conf); 4768 } 4769 4770 raise_barrier(conf, 0); 4771read_more: 4772 /* Now schedule reads for blocks from sector_nr to last */ 4773 r10_bio = raid10_alloc_init_r10buf(conf); 4774 r10_bio->state = 0; 4775 raise_barrier(conf, 1); 4776 atomic_set(&r10_bio->remaining, 0); 4777 r10_bio->mddev = mddev; 4778 r10_bio->sector = sector_nr; 4779 set_bit(R10BIO_IsReshape, &r10_bio->state); 4780 r10_bio->sectors = last - sector_nr + 1; 4781 rdev = read_balance(conf, r10_bio, &max_sectors); 4782 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); 4783 4784 if (!rdev) { 4785 /* Cannot read from here, so need to record bad blocks 4786 * on all the target devices. 4787 */ 4788 // FIXME 4789 mempool_free(r10_bio, &conf->r10buf_pool); 4790 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4791 return sectors_done; 4792 } 4793 4794 read_bio = bio_alloc_bioset(rdev->bdev, RESYNC_PAGES, REQ_OP_READ, 4795 GFP_KERNEL, &mddev->bio_set); 4796 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr 4797 + rdev->data_offset); 4798 read_bio->bi_private = r10_bio; 4799 read_bio->bi_end_io = end_reshape_read; 4800 r10_bio->master_bio = read_bio; 4801 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; 4802 4803 /* 4804 * Broadcast RESYNC message to other nodes, so all nodes would not 4805 * write to the region to avoid conflict. 4806 */ 4807 if (mddev_is_clustered(mddev) && conf->cluster_sync_high <= sector_nr) { 4808 struct mdp_superblock_1 *sb = NULL; 4809 int sb_reshape_pos = 0; 4810 4811 conf->cluster_sync_low = sector_nr; 4812 conf->cluster_sync_high = sector_nr + CLUSTER_RESYNC_WINDOW_SECTORS; 4813 sb = page_address(rdev->sb_page); 4814 if (sb) { 4815 sb_reshape_pos = le64_to_cpu(sb->reshape_position); 4816 /* 4817 * Set cluster_sync_low again if next address for array 4818 * reshape is less than cluster_sync_low. Since we can't 4819 * update cluster_sync_low until it has finished reshape. 4820 */ 4821 if (sb_reshape_pos < conf->cluster_sync_low) 4822 conf->cluster_sync_low = sb_reshape_pos; 4823 } 4824 4825 mddev->cluster_ops->resync_info_update(mddev, conf->cluster_sync_low, 4826 conf->cluster_sync_high); 4827 } 4828 4829 /* Now find the locations in the new layout */ 4830 __raid10_find_phys(&conf->geo, r10_bio); 4831 4832 blist = read_bio; 4833 read_bio->bi_next = NULL; 4834 4835 for (s = 0; s < conf->copies*2; s++) { 4836 struct bio *b; 4837 int d = r10_bio->devs[s/2].devnum; 4838 struct md_rdev *rdev2; 4839 if (s&1) { 4840 rdev2 = conf->mirrors[d].replacement; 4841 b = r10_bio->devs[s/2].repl_bio; 4842 } else { 4843 rdev2 = conf->mirrors[d].rdev; 4844 b = r10_bio->devs[s/2].bio; 4845 } 4846 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) 4847 continue; 4848 4849 bio_set_dev(b, rdev2->bdev); 4850 b->bi_iter.bi_sector = r10_bio->devs[s/2].addr + 4851 rdev2->new_data_offset; 4852 b->bi_end_io = end_reshape_write; 4853 b->bi_opf = REQ_OP_WRITE; 4854 b->bi_next = blist; 4855 blist = b; 4856 } 4857 4858 /* Now add as many pages as possible to all of these bios. */ 4859 4860 nr_sectors = 0; 4861 pages = get_resync_pages(r10_bio->devs[0].bio)->pages; 4862 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) { 4863 struct page *page = pages[s / (PAGE_SIZE >> 9)]; 4864 int len = (max_sectors - s) << 9; 4865 if (len > PAGE_SIZE) 4866 len = PAGE_SIZE; 4867 for (bio = blist; bio ; bio = bio->bi_next) { 4868 if (WARN_ON(!bio_add_page(bio, page, len, 0))) { 4869 bio->bi_status = BLK_STS_RESOURCE; 4870 bio_endio(bio); 4871 return sectors_done; 4872 } 4873 } 4874 sector_nr += len >> 9; 4875 nr_sectors += len >> 9; 4876 } 4877 r10_bio->sectors = nr_sectors; 4878 4879 /* Now submit the read */ 4880 atomic_inc(&r10_bio->remaining); 4881 read_bio->bi_next = NULL; 4882 submit_bio_noacct(read_bio); 4883 sectors_done += nr_sectors; 4884 if (sector_nr <= last) 4885 goto read_more; 4886 4887 lower_barrier(conf); 4888 4889 /* Now that we have done the whole section we can 4890 * update reshape_progress 4891 */ 4892 if (mddev->reshape_backwards) 4893 conf->reshape_progress -= sectors_done; 4894 else 4895 conf->reshape_progress += sectors_done; 4896 4897 return sectors_done; 4898} 4899 4900static void end_reshape_request(struct r10bio *r10_bio); 4901static int handle_reshape_read_error(struct mddev *mddev, 4902 struct r10bio *r10_bio); 4903static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) 4904{ 4905 /* Reshape read completed. Hopefully we have a block 4906 * to write out. 4907 * If we got a read error then we do sync 1-page reads from 4908 * elsewhere until we find the data - or give up. 4909 */ 4910 struct r10conf *conf = mddev->private; 4911 int s; 4912 4913 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 4914 if (handle_reshape_read_error(mddev, r10_bio) < 0) { 4915 /* Reshape has been aborted */ 4916 md_done_sync(mddev, r10_bio->sectors, 0); 4917 return; 4918 } 4919 4920 /* We definitely have the data in the pages, schedule the 4921 * writes. 4922 */ 4923 atomic_set(&r10_bio->remaining, 1); 4924 for (s = 0; s < conf->copies*2; s++) { 4925 struct bio *b; 4926 int d = r10_bio->devs[s/2].devnum; 4927 struct md_rdev *rdev; 4928 if (s&1) { 4929 rdev = conf->mirrors[d].replacement; 4930 b = r10_bio->devs[s/2].repl_bio; 4931 } else { 4932 rdev = conf->mirrors[d].rdev; 4933 b = r10_bio->devs[s/2].bio; 4934 } 4935 if (!rdev || test_bit(Faulty, &rdev->flags)) 4936 continue; 4937 4938 atomic_inc(&rdev->nr_pending); 4939 atomic_inc(&r10_bio->remaining); 4940 b->bi_next = NULL; 4941 submit_bio_noacct(b); 4942 } 4943 end_reshape_request(r10_bio); 4944} 4945 4946static void end_reshape(struct r10conf *conf) 4947{ 4948 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) 4949 return; 4950 4951 spin_lock_irq(&conf->device_lock); 4952 conf->prev = conf->geo; 4953 md_finish_reshape(conf->mddev); 4954 smp_wmb(); 4955 conf->reshape_progress = MaxSector; 4956 conf->reshape_safe = MaxSector; 4957 spin_unlock_irq(&conf->device_lock); 4958 4959 mddev_update_io_opt(conf->mddev, raid10_nr_stripes(conf)); 4960 conf->fullsync = 0; 4961} 4962 4963static void raid10_update_reshape_pos(struct mddev *mddev) 4964{ 4965 struct r10conf *conf = mddev->private; 4966 sector_t lo, hi; 4967 4968 mddev->cluster_ops->resync_info_get(mddev, &lo, &hi); 4969 if (((mddev->reshape_position <= hi) && (mddev->reshape_position >= lo)) 4970 || mddev->reshape_position == MaxSector) 4971 conf->reshape_progress = mddev->reshape_position; 4972 else 4973 WARN_ON_ONCE(1); 4974} 4975 4976static int handle_reshape_read_error(struct mddev *mddev, 4977 struct r10bio *r10_bio) 4978{ 4979 /* Use sync reads to get the blocks from somewhere else */ 4980 int sectors = r10_bio->sectors; 4981 struct r10conf *conf = mddev->private; 4982 struct r10bio *r10b; 4983 int slot = 0; 4984 int idx = 0; 4985 struct page **pages; 4986 4987 r10b = kmalloc(struct_size(r10b, devs, conf->copies), GFP_NOIO); 4988 if (!r10b) { 4989 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4990 return -ENOMEM; 4991 } 4992 4993 /* reshape IOs share pages from .devs[0].bio */ 4994 pages = get_resync_pages(r10_bio->devs[0].bio)->pages; 4995 4996 r10b->sector = r10_bio->sector; 4997 __raid10_find_phys(&conf->prev, r10b); 4998 4999 while (sectors) { 5000 int s = sectors; 5001 int success = 0; 5002 int first_slot = slot; 5003 5004 if (s > (PAGE_SIZE >> 9)) 5005 s = PAGE_SIZE >> 9; 5006 5007 while (!success) { 5008 int d = r10b->devs[slot].devnum; 5009 struct md_rdev *rdev = conf->mirrors[d].rdev; 5010 sector_t addr; 5011 if (rdev == NULL || 5012 test_bit(Faulty, &rdev->flags) || 5013 !test_bit(In_sync, &rdev->flags)) 5014 goto failed; 5015 5016 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 5017 atomic_inc(&rdev->nr_pending); 5018 success = sync_page_io(rdev, 5019 addr, 5020 s << 9, 5021 pages[idx], 5022 REQ_OP_READ, false); 5023 rdev_dec_pending(rdev, mddev); 5024 if (success) 5025 break; 5026 failed: 5027 slot++; 5028 if (slot >= conf->copies) 5029 slot = 0; 5030 if (slot == first_slot) 5031 break; 5032 } 5033 if (!success) { 5034 /* couldn't read this block, must give up */ 5035 set_bit(MD_RECOVERY_INTR, 5036 &mddev->recovery); 5037 kfree(r10b); 5038 return -EIO; 5039 } 5040 sectors -= s; 5041 idx++; 5042 } 5043 kfree(r10b); 5044 return 0; 5045} 5046 5047static void end_reshape_write(struct bio *bio) 5048{ 5049 struct r10bio *r10_bio = get_resync_r10bio(bio); 5050 struct mddev *mddev = r10_bio->mddev; 5051 struct r10conf *conf = mddev->private; 5052 int d; 5053 int slot; 5054 int repl; 5055 struct md_rdev *rdev = NULL; 5056 5057 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 5058 rdev = repl ? conf->mirrors[d].replacement : 5059 conf->mirrors[d].rdev; 5060 5061 if (bio->bi_status) { 5062 /* FIXME should record badblock */ 5063 md_error(mddev, rdev); 5064 } 5065 5066 rdev_dec_pending(rdev, mddev); 5067 end_reshape_request(r10_bio); 5068} 5069 5070static void end_reshape_request(struct r10bio *r10_bio) 5071{ 5072 if (!atomic_dec_and_test(&r10_bio->remaining)) 5073 return; 5074 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); 5075 bio_put(r10_bio->master_bio); 5076 put_buf(r10_bio); 5077} 5078 5079static void raid10_finish_reshape(struct mddev *mddev) 5080{ 5081 struct r10conf *conf = mddev->private; 5082 5083 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 5084 return; 5085 5086 if (mddev->delta_disks > 0) { 5087 if (mddev->resync_offset > mddev->resync_max_sectors) { 5088 mddev->resync_offset = mddev->resync_max_sectors; 5089 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 5090 } 5091 mddev->resync_max_sectors = mddev->array_sectors; 5092 } else { 5093 int d; 5094 for (d = conf->geo.raid_disks ; 5095 d < conf->geo.raid_disks - mddev->delta_disks; 5096 d++) { 5097 struct md_rdev *rdev = conf->mirrors[d].rdev; 5098 if (rdev) 5099 clear_bit(In_sync, &rdev->flags); 5100 rdev = conf->mirrors[d].replacement; 5101 if (rdev) 5102 clear_bit(In_sync, &rdev->flags); 5103 } 5104 } 5105 mddev->layout = mddev->new_layout; 5106 mddev->chunk_sectors = 1 << conf->geo.chunk_shift; 5107 mddev->reshape_position = MaxSector; 5108 mddev->delta_disks = 0; 5109 mddev->reshape_backwards = 0; 5110} 5111 5112static struct md_personality raid10_personality = 5113{ 5114 .head = { 5115 .type = MD_PERSONALITY, 5116 .id = ID_RAID10, 5117 .name = "raid10", 5118 .owner = THIS_MODULE, 5119 }, 5120 5121 .make_request = raid10_make_request, 5122 .run = raid10_run, 5123 .free = raid10_free, 5124 .status = raid10_status, 5125 .error_handler = raid10_error, 5126 .hot_add_disk = raid10_add_disk, 5127 .hot_remove_disk= raid10_remove_disk, 5128 .spare_active = raid10_spare_active, 5129 .sync_request = raid10_sync_request, 5130 .quiesce = raid10_quiesce, 5131 .size = raid10_size, 5132 .resize = raid10_resize, 5133 .takeover = raid10_takeover, 5134 .check_reshape = raid10_check_reshape, 5135 .start_reshape = raid10_start_reshape, 5136 .finish_reshape = raid10_finish_reshape, 5137 .update_reshape_pos = raid10_update_reshape_pos, 5138}; 5139 5140static int __init raid10_init(void) 5141{ 5142 return register_md_submodule(&raid10_personality.head); 5143} 5144 5145static void __exit raid10_exit(void) 5146{ 5147 unregister_md_submodule(&raid10_personality.head); 5148} 5149 5150module_init(raid10_init); 5151module_exit(raid10_exit); 5152MODULE_LICENSE("GPL"); 5153MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); 5154MODULE_ALIAS("md-personality-9"); /* RAID10 */ 5155MODULE_ALIAS("md-raid10"); 5156MODULE_ALIAS("md-level-10");
~/
Read-only file preview.