raid5-cache.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2015 Shaohua Li <shli@fb.com>
   4 * Copyright (C) 2016 Song Liu <songliubraving@fb.com>
   5 */
   6#include <linux/kernel.h>
   7#include <linux/wait.h>
   8#include <linux/blkdev.h>
   9#include <linux/slab.h>
  10#include <linux/raid/md_p.h>
  11#include <linux/crc32c.h>
  12#include <linux/random.h>
  13#include <linux/kthread.h>
  14#include <linux/types.h>
  15#include "md.h"
  16#include "raid5.h"
  17#include "md-bitmap.h"
  18#include "raid5-log.h"
  19
  20/*
  21 * metadata/data stored in disk with 4k size unit (a block) regardless
  22 * underneath hardware sector size. only works with PAGE_SIZE == 4096
  23 */
  24#define BLOCK_SECTORS (8)
  25#define BLOCK_SECTOR_SHIFT (3)
  26
  27/*
  28 * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
  29 *
  30 * In write through mode, the reclaim runs every log->max_free_space.
  31 * This can prevent the recovery scans for too long
  32 */
  33#define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
  34#define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
  35
  36/* wake up reclaim thread periodically */
  37#define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
  38/* start flush with these full stripes */
  39#define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4)
  40/* reclaim stripes in groups */
  41#define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
  42
  43/*
  44 * We only need 2 bios per I/O unit to make progress, but ensure we
  45 * have a few more available to not get too tight.
  46 */
  47#define R5L_POOL_SIZE	4
  48
  49static char *r5c_journal_mode_str[] = {"write-through",
  50				       "write-back"};
  51/*
  52 * raid5 cache state machine
  53 *
  54 * With the RAID cache, each stripe works in two phases:
  55 *	- caching phase
  56 *	- writing-out phase
  57 *
  58 * These two phases are controlled by bit STRIPE_R5C_CACHING:
  59 *   if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase
  60 *   if STRIPE_R5C_CACHING == 1, the stripe is in caching phase
  61 *
  62 * When there is no journal, or the journal is in write-through mode,
  63 * the stripe is always in writing-out phase.
  64 *
  65 * For write-back journal, the stripe is sent to caching phase on write
  66 * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off
  67 * the write-out phase by clearing STRIPE_R5C_CACHING.
  68 *
  69 * Stripes in caching phase do not write the raid disks. Instead, all
  70 * writes are committed from the log device. Therefore, a stripe in
  71 * caching phase handles writes as:
  72 *	- write to log device
  73 *	- return IO
  74 *
  75 * Stripes in writing-out phase handle writes as:
  76 *	- calculate parity
  77 *	- write pending data and parity to journal
  78 *	- write data and parity to raid disks
  79 *	- return IO for pending writes
  80 */
  81
  82struct r5l_log {
  83	struct md_rdev *rdev;
  84
  85	u32 uuid_checksum;
  86
  87	sector_t device_size;		/* log device size, round to
  88					 * BLOCK_SECTORS */
  89	sector_t max_free_space;	/* reclaim run if free space is at
  90					 * this size */
  91
  92	sector_t last_checkpoint;	/* log tail. where recovery scan
  93					 * starts from */
  94	u64 last_cp_seq;		/* log tail sequence */
  95
  96	sector_t log_start;		/* log head. where new data appends */
  97	u64 seq;			/* log head sequence */
  98
  99	sector_t next_checkpoint;
 100
 101	struct mutex io_mutex;
 102	struct r5l_io_unit *current_io;	/* current io_unit accepting new data */
 103
 104	spinlock_t io_list_lock;
 105	struct list_head running_ios;	/* io_units which are still running,
 106					 * and have not yet been completely
 107					 * written to the log */
 108	struct list_head io_end_ios;	/* io_units which have been completely
 109					 * written to the log but not yet written
 110					 * to the RAID */
 111	struct list_head flushing_ios;	/* io_units which are waiting for log
 112					 * cache flush */
 113	struct list_head finished_ios;	/* io_units which settle down in log disk */
 114	struct bio flush_bio;
 115
 116	struct list_head no_mem_stripes;   /* pending stripes, -ENOMEM */
 117
 118	struct kmem_cache *io_kc;
 119	mempool_t io_pool;
 120	struct bio_set bs;
 121	mempool_t meta_pool;
 122
 123	struct md_thread __rcu *reclaim_thread;
 124	unsigned long reclaim_target;	/* number of space that need to be
 125					 * reclaimed.  if it's 0, reclaim spaces
 126					 * used by io_units which are in
 127					 * IO_UNIT_STRIPE_END state (eg, reclaim
 128					 * doesn't wait for specific io_unit
 129					 * switching to IO_UNIT_STRIPE_END
 130					 * state) */
 131	wait_queue_head_t iounit_wait;
 132
 133	struct list_head no_space_stripes; /* pending stripes, log has no space */
 134	spinlock_t no_space_stripes_lock;
 135
 136	bool need_cache_flush;
 137
 138	/* for r5c_cache */
 139	enum r5c_journal_mode r5c_journal_mode;
 140
 141	/* all stripes in r5cache, in the order of seq at sh->log_start */
 142	struct list_head stripe_in_journal_list;
 143
 144	spinlock_t stripe_in_journal_lock;
 145	atomic_t stripe_in_journal_count;
 146
 147	/* to submit async io_units, to fulfill ordering of flush */
 148	struct work_struct deferred_io_work;
 149	/* to disable write back during in degraded mode */
 150	struct work_struct disable_writeback_work;
 151
 152	/* to for chunk_aligned_read in writeback mode, details below */
 153	spinlock_t tree_lock;
 154	struct radix_tree_root big_stripe_tree;
 155};
 156
 157/*
 158 * Enable chunk_aligned_read() with write back cache.
 159 *
 160 * Each chunk may contain more than one stripe (for example, a 256kB
 161 * chunk contains 64 4kB-page, so this chunk contain 64 stripes). For
 162 * chunk_aligned_read, these stripes are grouped into one "big_stripe".
 163 * For each big_stripe, we count how many stripes of this big_stripe
 164 * are in the write back cache. These data are tracked in a radix tree
 165 * (big_stripe_tree). We use radix_tree item pointer as the counter.
 166 * r5c_tree_index() is used to calculate keys for the radix tree.
 167 *
 168 * chunk_aligned_read() calls r5c_big_stripe_cached() to look up
 169 * big_stripe of each chunk in the tree. If this big_stripe is in the
 170 * tree, chunk_aligned_read() aborts. This look up is protected by
 171 * rcu_read_lock().
 172 *
 173 * It is necessary to remember whether a stripe is counted in
 174 * big_stripe_tree. Instead of adding new flag, we reuses existing flags:
 175 * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE. If either of these
 176 * two flags are set, the stripe is counted in big_stripe_tree. This
 177 * requires moving set_bit(STRIPE_R5C_PARTIAL_STRIPE) to
 178 * r5c_try_caching_write(); and moving clear_bit of
 179 * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE to
 180 * r5c_finish_stripe_write_out().
 181 */
 182
 183/*
 184 * radix tree requests lowest 2 bits of data pointer to be 2b'00.
 185 * So it is necessary to left shift the counter by 2 bits before using it
 186 * as data pointer of the tree.
 187 */
 188#define R5C_RADIX_COUNT_SHIFT 2
 189
 190/*
 191 * calculate key for big_stripe_tree
 192 *
 193 * sect: align_bi->bi_iter.bi_sector or sh->sector
 194 */
 195static inline sector_t r5c_tree_index(struct r5conf *conf,
 196				      sector_t sect)
 197{
 198	sector_div(sect, conf->chunk_sectors);
 199	return sect;
 200}
 201
 202/*
 203 * an IO range starts from a meta data block and end at the next meta data
 204 * block. The io unit's the meta data block tracks data/parity followed it. io
 205 * unit is written to log disk with normal write, as we always flush log disk
 206 * first and then start move data to raid disks, there is no requirement to
 207 * write io unit with FLUSH/FUA
 208 */
 209struct r5l_io_unit {
 210	struct r5l_log *log;
 211
 212	struct page *meta_page;	/* store meta block */
 213	int meta_offset;	/* current offset in meta_page */
 214
 215	struct bio *current_bio;/* current_bio accepting new data */
 216
 217	atomic_t pending_stripe;/* how many stripes not flushed to raid */
 218	u64 seq;		/* seq number of the metablock */
 219	sector_t log_start;	/* where the io_unit starts */
 220	sector_t log_end;	/* where the io_unit ends */
 221	struct list_head log_sibling; /* log->running_ios */
 222	struct list_head stripe_list; /* stripes added to the io_unit */
 223
 224	int state;
 225	bool need_split_bio;
 226	struct bio *split_bio;
 227
 228	unsigned int has_flush:1;		/* include flush request */
 229	unsigned int has_fua:1;			/* include fua request */
 230	unsigned int has_null_flush:1;		/* include null flush request */
 231	unsigned int has_flush_payload:1;	/* include flush payload  */
 232	/*
 233	 * io isn't sent yet, flush/fua request can only be submitted till it's
 234	 * the first IO in running_ios list
 235	 */
 236	unsigned int io_deferred:1;
 237
 238	struct bio_list flush_barriers;   /* size == 0 flush bios */
 239};
 240
 241/* r5l_io_unit state */
 242enum r5l_io_unit_state {
 243	IO_UNIT_RUNNING = 0,	/* accepting new IO */
 244	IO_UNIT_IO_START = 1,	/* io_unit bio start writing to log,
 245				 * don't accepting new bio */
 246	IO_UNIT_IO_END = 2,	/* io_unit bio finish writing to log */
 247	IO_UNIT_STRIPE_END = 3,	/* stripes data finished writing to raid */
 248};
 249
 250bool r5c_is_writeback(struct r5l_log *log)
 251{
 252	return (log != NULL &&
 253		log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);
 254}
 255
 256static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
 257{
 258	start += inc;
 259	if (start >= log->device_size)
 260		start = start - log->device_size;
 261	return start;
 262}
 263
 264static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start,
 265				  sector_t end)
 266{
 267	if (end >= start)
 268		return end - start;
 269	else
 270		return end + log->device_size - start;
 271}
 272
 273static bool r5l_has_free_space(struct r5l_log *log, sector_t size)
 274{
 275	sector_t used_size;
 276
 277	used_size = r5l_ring_distance(log, log->last_checkpoint,
 278					log->log_start);
 279
 280	return log->device_size > used_size + size;
 281}
 282
 283static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
 284				    enum r5l_io_unit_state state)
 285{
 286	if (WARN_ON(io->state >= state))
 287		return;
 288	io->state = state;
 289}
 290
 291static void
 292r5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev)
 293{
 294	struct bio *wbi, *wbi2;
 295
 296	wbi = dev->written;
 297	dev->written = NULL;
 298	while (wbi && wbi->bi_iter.bi_sector <
 299	       dev->sector + RAID5_STRIPE_SECTORS(conf)) {
 300		wbi2 = r5_next_bio(conf, wbi, dev->sector);
 301		md_write_end(conf->mddev);
 302		bio_endio(wbi);
 303		wbi = wbi2;
 304	}
 305}
 306
 307void r5c_handle_cached_data_endio(struct r5conf *conf,
 308				  struct stripe_head *sh, int disks)
 309{
 310	int i;
 311
 312	for (i = sh->disks; i--; ) {
 313		if (sh->dev[i].written) {
 314			set_bit(R5_UPTODATE, &sh->dev[i].flags);
 315			r5c_return_dev_pending_writes(conf, &sh->dev[i]);
 316		}
 317	}
 318}
 319
 320void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
 321
 322/* Check whether we should flush some stripes to free up stripe cache */
 323void r5c_check_stripe_cache_usage(struct r5conf *conf)
 324{
 325	int total_cached;
 326	struct r5l_log *log = READ_ONCE(conf->log);
 327
 328	if (!r5c_is_writeback(log))
 329		return;
 330
 331	total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
 332		atomic_read(&conf->r5c_cached_full_stripes);
 333
 334	/*
 335	 * The following condition is true for either of the following:
 336	 *   - stripe cache pressure high:
 337	 *          total_cached > 3/4 min_nr_stripes ||
 338	 *          empty_inactive_list_nr > 0
 339	 *   - stripe cache pressure moderate:
 340	 *          total_cached > 1/2 min_nr_stripes
 341	 */
 342	if (total_cached > conf->min_nr_stripes * 1 / 2 ||
 343	    atomic_read(&conf->empty_inactive_list_nr) > 0)
 344		r5l_wake_reclaim(log, 0);
 345}
 346
 347/*
 348 * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full
 349 * stripes in the cache
 350 */
 351void r5c_check_cached_full_stripe(struct r5conf *conf)
 352{
 353	struct r5l_log *log = READ_ONCE(conf->log);
 354
 355	if (!r5c_is_writeback(log))
 356		return;
 357
 358	/*
 359	 * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes
 360	 * or a full stripe (chunk size / 4k stripes).
 361	 */
 362	if (atomic_read(&conf->r5c_cached_full_stripes) >=
 363	    min(R5C_FULL_STRIPE_FLUSH_BATCH(conf),
 364		conf->chunk_sectors >> RAID5_STRIPE_SHIFT(conf)))
 365		r5l_wake_reclaim(log, 0);
 366}
 367
 368/*
 369 * Total log space (in sectors) needed to flush all data in cache
 370 *
 371 * To avoid deadlock due to log space, it is necessary to reserve log
 372 * space to flush critical stripes (stripes that occupying log space near
 373 * last_checkpoint). This function helps check how much log space is
 374 * required to flush all cached stripes.
 375 *
 376 * To reduce log space requirements, two mechanisms are used to give cache
 377 * flush higher priorities:
 378 *    1. In handle_stripe_dirtying() and schedule_reconstruction(),
 379 *       stripes ALREADY in journal can be flushed w/o pending writes;
 380 *    2. In r5l_write_stripe() and r5c_cache_data(), stripes NOT in journal
 381 *       can be delayed (r5l_add_no_space_stripe).
 382 *
 383 * In cache flush, the stripe goes through 1 and then 2. For a stripe that
 384 * already passed 1, flushing it requires at most (conf->max_degraded + 1)
 385 * pages of journal space. For stripes that has not passed 1, flushing it
 386 * requires (conf->raid_disks + 1) pages of journal space. There are at
 387 * most (conf->group_cnt + 1) stripe that passed 1. So total journal space
 388 * required to flush all cached stripes (in pages) is:
 389 *
 390 *     (stripe_in_journal_count - group_cnt - 1) * (max_degraded + 1) +
 391 *     (group_cnt + 1) * (raid_disks + 1)
 392 * or
 393 *     (stripe_in_journal_count) * (max_degraded + 1) +
 394 *     (group_cnt + 1) * (raid_disks - max_degraded)
 395 */
 396static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf)
 397{
 398	struct r5l_log *log = READ_ONCE(conf->log);
 399
 400	if (!r5c_is_writeback(log))
 401		return 0;
 402
 403	return BLOCK_SECTORS *
 404		((conf->max_degraded + 1) * atomic_read(&log->stripe_in_journal_count) +
 405		 (conf->raid_disks - conf->max_degraded) * (conf->group_cnt + 1));
 406}
 407
 408/*
 409 * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL
 410 *
 411 * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of
 412 * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log
 413 * device is less than 2x of reclaim_required_space.
 414 */
 415static inline void r5c_update_log_state(struct r5l_log *log)
 416{
 417	struct r5conf *conf = log->rdev->mddev->private;
 418	sector_t free_space;
 419	sector_t reclaim_space;
 420	bool wake_reclaim = false;
 421
 422	if (!r5c_is_writeback(log))
 423		return;
 424
 425	free_space = r5l_ring_distance(log, log->log_start,
 426				       log->last_checkpoint);
 427	reclaim_space = r5c_log_required_to_flush_cache(conf);
 428	if (free_space < 2 * reclaim_space)
 429		set_bit(R5C_LOG_CRITICAL, &conf->cache_state);
 430	else {
 431		if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
 432			wake_reclaim = true;
 433		clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
 434	}
 435	if (free_space < 3 * reclaim_space)
 436		set_bit(R5C_LOG_TIGHT, &conf->cache_state);
 437	else
 438		clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
 439
 440	if (wake_reclaim)
 441		r5l_wake_reclaim(log, 0);
 442}
 443
 444/*
 445 * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING.
 446 * This function should only be called in write-back mode.
 447 */
 448void r5c_make_stripe_write_out(struct stripe_head *sh)
 449{
 450	struct r5conf *conf = sh->raid_conf;
 451	struct r5l_log *log = READ_ONCE(conf->log);
 452
 453	BUG_ON(!r5c_is_writeback(log));
 454
 455	WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
 456	clear_bit(STRIPE_R5C_CACHING, &sh->state);
 457
 458	if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
 459		atomic_inc(&conf->preread_active_stripes);
 460}
 461
 462static void r5c_handle_data_cached(struct stripe_head *sh)
 463{
 464	int i;
 465
 466	for (i = sh->disks; i--; )
 467		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
 468			set_bit(R5_InJournal, &sh->dev[i].flags);
 469			clear_bit(R5_LOCKED, &sh->dev[i].flags);
 470		}
 471	clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
 472}
 473
 474/*
 475 * this journal write must contain full parity,
 476 * it may also contain some data pages
 477 */
 478static void r5c_handle_parity_cached(struct stripe_head *sh)
 479{
 480	int i;
 481
 482	for (i = sh->disks; i--; )
 483		if (test_bit(R5_InJournal, &sh->dev[i].flags))
 484			set_bit(R5_Wantwrite, &sh->dev[i].flags);
 485}
 486
 487/*
 488 * Setting proper flags after writing (or flushing) data and/or parity to the
 489 * log device. This is called from r5l_log_endio() or r5l_log_flush_endio().
 490 */
 491static void r5c_finish_cache_stripe(struct stripe_head *sh)
 492{
 493	struct r5l_log *log = READ_ONCE(sh->raid_conf->log);
 494
 495	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
 496		BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
 497		/*
 498		 * Set R5_InJournal for parity dev[pd_idx]. This means
 499		 * all data AND parity in the journal. For RAID 6, it is
 500		 * NOT necessary to set the flag for dev[qd_idx], as the
 501		 * two parities are written out together.
 502		 */
 503		set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
 504	} else if (test_bit(STRIPE_R5C_CACHING, &sh->state)) {
 505		r5c_handle_data_cached(sh);
 506	} else {
 507		r5c_handle_parity_cached(sh);
 508		set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
 509	}
 510}
 511
 512static void r5l_io_run_stripes(struct r5l_io_unit *io)
 513{
 514	struct stripe_head *sh, *next;
 515
 516	list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
 517		list_del_init(&sh->log_list);
 518
 519		r5c_finish_cache_stripe(sh);
 520
 521		set_bit(STRIPE_HANDLE, &sh->state);
 522		raid5_release_stripe(sh);
 523	}
 524}
 525
 526static void r5l_log_run_stripes(struct r5l_log *log)
 527{
 528	struct r5l_io_unit *io, *next;
 529
 530	lockdep_assert_held(&log->io_list_lock);
 531
 532	list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
 533		/* don't change list order */
 534		if (io->state < IO_UNIT_IO_END)
 535			break;
 536
 537		list_move_tail(&io->log_sibling, &log->finished_ios);
 538		r5l_io_run_stripes(io);
 539	}
 540}
 541
 542static void r5l_move_to_end_ios(struct r5l_log *log)
 543{
 544	struct r5l_io_unit *io, *next;
 545
 546	lockdep_assert_held(&log->io_list_lock);
 547
 548	list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
 549		/* don't change list order */
 550		if (io->state < IO_UNIT_IO_END)
 551			break;
 552		list_move_tail(&io->log_sibling, &log->io_end_ios);
 553	}
 554}
 555
 556static void __r5l_stripe_write_finished(struct r5l_io_unit *io);
 557static void r5l_log_endio(struct bio *bio)
 558{
 559	struct r5l_io_unit *io = bio->bi_private;
 560	struct r5l_io_unit *io_deferred;
 561	struct r5l_log *log = io->log;
 562	unsigned long flags;
 563	bool has_null_flush;
 564	bool has_flush_payload;
 565
 566	if (bio->bi_status)
 567		md_error(log->rdev->mddev, log->rdev);
 568
 569	bio_put(bio);
 570	mempool_free(io->meta_page, &log->meta_pool);
 571
 572	spin_lock_irqsave(&log->io_list_lock, flags);
 573	__r5l_set_io_unit_state(io, IO_UNIT_IO_END);
 574
 575	/*
 576	 * if the io doesn't not have null_flush or flush payload,
 577	 * it is not safe to access it after releasing io_list_lock.
 578	 * Therefore, it is necessary to check the condition with
 579	 * the lock held.
 580	 */
 581	has_null_flush = io->has_null_flush;
 582	has_flush_payload = io->has_flush_payload;
 583
 584	if (log->need_cache_flush && !list_empty(&io->stripe_list))
 585		r5l_move_to_end_ios(log);
 586	else
 587		r5l_log_run_stripes(log);
 588	if (!list_empty(&log->running_ios)) {
 589		/*
 590		 * FLUSH/FUA io_unit is deferred because of ordering, now we
 591		 * can dispatch it
 592		 */
 593		io_deferred = list_first_entry(&log->running_ios,
 594					       struct r5l_io_unit, log_sibling);
 595		if (io_deferred->io_deferred)
 596			schedule_work(&log->deferred_io_work);
 597	}
 598
 599	spin_unlock_irqrestore(&log->io_list_lock, flags);
 600
 601	if (log->need_cache_flush)
 602		md_wakeup_thread(log->rdev->mddev->thread);
 603
 604	/* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
 605	if (has_null_flush) {
 606		struct bio *bi;
 607
 608		WARN_ON(bio_list_empty(&io->flush_barriers));
 609		while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
 610			bio_endio(bi);
 611			if (atomic_dec_and_test(&io->pending_stripe)) {
 612				__r5l_stripe_write_finished(io);
 613				return;
 614			}
 615		}
 616	}
 617	/* decrease pending_stripe for flush payload */
 618	if (has_flush_payload)
 619		if (atomic_dec_and_test(&io->pending_stripe))
 620			__r5l_stripe_write_finished(io);
 621}
 622
 623static void r5l_do_submit_io(struct r5l_log *log, struct r5l_io_unit *io)
 624{
 625	unsigned long flags;
 626
 627	spin_lock_irqsave(&log->io_list_lock, flags);
 628	__r5l_set_io_unit_state(io, IO_UNIT_IO_START);
 629	spin_unlock_irqrestore(&log->io_list_lock, flags);
 630
 631	/*
 632	 * In case of journal device failures, submit_bio will get error
 633	 * and calls endio, then active stripes will continue write
 634	 * process. Therefore, it is not necessary to check Faulty bit
 635	 * of journal device here.
 636	 *
 637	 * We can't check split_bio after current_bio is submitted. If
 638	 * io->split_bio is null, after current_bio is submitted, current_bio
 639	 * might already be completed and the io_unit is freed. We submit
 640	 * split_bio first to avoid the issue.
 641	 */
 642	if (io->split_bio) {
 643		if (io->has_flush)
 644			io->split_bio->bi_opf |= REQ_PREFLUSH;
 645		if (io->has_fua)
 646			io->split_bio->bi_opf |= REQ_FUA;
 647		submit_bio(io->split_bio);
 648	}
 649
 650	if (io->has_flush)
 651		io->current_bio->bi_opf |= REQ_PREFLUSH;
 652	if (io->has_fua)
 653		io->current_bio->bi_opf |= REQ_FUA;
 654	submit_bio(io->current_bio);
 655}
 656
 657/* deferred io_unit will be dispatched here */
 658static void r5l_submit_io_async(struct work_struct *work)
 659{
 660	struct r5l_log *log = container_of(work, struct r5l_log,
 661					   deferred_io_work);
 662	struct r5l_io_unit *io = NULL;
 663	unsigned long flags;
 664
 665	spin_lock_irqsave(&log->io_list_lock, flags);
 666	if (!list_empty(&log->running_ios)) {
 667		io = list_first_entry(&log->running_ios, struct r5l_io_unit,
 668				      log_sibling);
 669		if (!io->io_deferred)
 670			io = NULL;
 671		else
 672			io->io_deferred = 0;
 673	}
 674	spin_unlock_irqrestore(&log->io_list_lock, flags);
 675	if (io)
 676		r5l_do_submit_io(log, io);
 677}
 678
 679static void r5c_disable_writeback_async(struct work_struct *work)
 680{
 681	struct r5l_log *log = container_of(work, struct r5l_log,
 682					   disable_writeback_work);
 683	struct mddev *mddev = log->rdev->mddev;
 684	struct r5conf *conf = mddev->private;
 685
 686	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
 687		return;
 688	pr_info("md/raid:%s: Disabling writeback cache for degraded array.\n",
 689		mdname(mddev));
 690
 691	/* wait superblock change before suspend */
 692	wait_event(mddev->sb_wait,
 693		   !READ_ONCE(conf->log) ||
 694		   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
 695
 696	log = READ_ONCE(conf->log);
 697	if (log) {
 698		mddev_suspend(mddev, false);
 699		log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
 700		mddev_resume(mddev);
 701	}
 702}
 703
 704static void r5l_submit_current_io(struct r5l_log *log)
 705{
 706	struct r5l_io_unit *io = log->current_io;
 707	struct r5l_meta_block *block;
 708	unsigned long flags;
 709	u32 crc;
 710	bool do_submit = true;
 711
 712	if (!io)
 713		return;
 714
 715	block = page_address(io->meta_page);
 716	block->meta_size = cpu_to_le32(io->meta_offset);
 717	crc = crc32c(log->uuid_checksum, block, PAGE_SIZE);
 718	block->checksum = cpu_to_le32(crc);
 719
 720	log->current_io = NULL;
 721	spin_lock_irqsave(&log->io_list_lock, flags);
 722	if (io->has_flush || io->has_fua) {
 723		if (io != list_first_entry(&log->running_ios,
 724					   struct r5l_io_unit, log_sibling)) {
 725			io->io_deferred = 1;
 726			do_submit = false;
 727		}
 728	}
 729	spin_unlock_irqrestore(&log->io_list_lock, flags);
 730	if (do_submit)
 731		r5l_do_submit_io(log, io);
 732}
 733
 734static struct bio *r5l_bio_alloc(struct r5l_log *log)
 735{
 736	struct bio *bio = bio_alloc_bioset(log->rdev->bdev, BIO_MAX_VECS,
 737					   REQ_OP_WRITE, GFP_NOIO, &log->bs);
 738
 739	bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start;
 740
 741	return bio;
 742}
 743
 744static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)
 745{
 746	log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
 747
 748	r5c_update_log_state(log);
 749	/*
 750	 * If we filled up the log device start from the beginning again,
 751	 * which will require a new bio.
 752	 *
 753	 * Note: for this to work properly the log size needs to me a multiple
 754	 * of BLOCK_SECTORS.
 755	 */
 756	if (log->log_start == 0)
 757		io->need_split_bio = true;
 758
 759	io->log_end = log->log_start;
 760}
 761
 762static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
 763{
 764	struct r5l_io_unit *io;
 765	struct r5l_meta_block *block;
 766
 767	io = mempool_alloc(&log->io_pool, GFP_ATOMIC);
 768	if (!io)
 769		return NULL;
 770	memset(io, 0, sizeof(*io));
 771
 772	io->log = log;
 773	INIT_LIST_HEAD(&io->log_sibling);
 774	INIT_LIST_HEAD(&io->stripe_list);
 775	bio_list_init(&io->flush_barriers);
 776	io->state = IO_UNIT_RUNNING;
 777
 778	io->meta_page = mempool_alloc(&log->meta_pool, GFP_NOIO);
 779	block = page_address(io->meta_page);
 780	clear_page(block);
 781	block->magic = cpu_to_le32(R5LOG_MAGIC);
 782	block->version = R5LOG_VERSION;
 783	block->seq = cpu_to_le64(log->seq);
 784	block->position = cpu_to_le64(log->log_start);
 785
 786	io->log_start = log->log_start;
 787	io->meta_offset = sizeof(struct r5l_meta_block);
 788	io->seq = log->seq++;
 789
 790	io->current_bio = r5l_bio_alloc(log);
 791	io->current_bio->bi_end_io = r5l_log_endio;
 792	io->current_bio->bi_private = io;
 793	__bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0);
 794
 795	r5_reserve_log_entry(log, io);
 796
 797	spin_lock_irq(&log->io_list_lock);
 798	list_add_tail(&io->log_sibling, &log->running_ios);
 799	spin_unlock_irq(&log->io_list_lock);
 800
 801	return io;
 802}
 803
 804static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
 805{
 806	if (log->current_io &&
 807	    log->current_io->meta_offset + payload_size > PAGE_SIZE)
 808		r5l_submit_current_io(log);
 809
 810	if (!log->current_io) {
 811		log->current_io = r5l_new_meta(log);
 812		if (!log->current_io)
 813			return -ENOMEM;
 814	}
 815
 816	return 0;
 817}
 818
 819static void r5l_append_payload_meta(struct r5l_log *log, u16 type,
 820				    sector_t location,
 821				    u32 checksum1, u32 checksum2,
 822				    bool checksum2_valid)
 823{
 824	struct r5l_io_unit *io = log->current_io;
 825	struct r5l_payload_data_parity *payload;
 826
 827	payload = page_address(io->meta_page) + io->meta_offset;
 828	payload->header.type = cpu_to_le16(type);
 829	payload->header.flags = cpu_to_le16(0);
 830	payload->size = cpu_to_le32((1 + !!checksum2_valid) <<
 831				    (PAGE_SHIFT - 9));
 832	payload->location = cpu_to_le64(location);
 833	payload->checksum[0] = cpu_to_le32(checksum1);
 834	if (checksum2_valid)
 835		payload->checksum[1] = cpu_to_le32(checksum2);
 836
 837	io->meta_offset += sizeof(struct r5l_payload_data_parity) +
 838		sizeof(__le32) * (1 + !!checksum2_valid);
 839}
 840
 841static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
 842{
 843	struct r5l_io_unit *io = log->current_io;
 844
 845	if (io->need_split_bio) {
 846		BUG_ON(io->split_bio);
 847		io->split_bio = io->current_bio;
 848		io->current_bio = r5l_bio_alloc(log);
 849		bio_chain(io->current_bio, io->split_bio);
 850		io->need_split_bio = false;
 851	}
 852
 853	if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
 854		BUG();
 855
 856	r5_reserve_log_entry(log, io);
 857}
 858
 859static void r5l_append_flush_payload(struct r5l_log *log, sector_t sect)
 860{
 861	struct mddev *mddev = log->rdev->mddev;
 862	struct r5conf *conf = mddev->private;
 863	struct r5l_io_unit *io;
 864	struct r5l_payload_flush *payload;
 865	int meta_size;
 866
 867	/*
 868	 * payload_flush requires extra writes to the journal.
 869	 * To avoid handling the extra IO in quiesce, just skip
 870	 * flush_payload
 871	 */
 872	if (conf->quiesce)
 873		return;
 874
 875	mutex_lock(&log->io_mutex);
 876	meta_size = sizeof(struct r5l_payload_flush) + sizeof(__le64);
 877
 878	if (r5l_get_meta(log, meta_size)) {
 879		mutex_unlock(&log->io_mutex);
 880		return;
 881	}
 882
 883	/* current implementation is one stripe per flush payload */
 884	io = log->current_io;
 885	payload = page_address(io->meta_page) + io->meta_offset;
 886	payload->header.type = cpu_to_le16(R5LOG_PAYLOAD_FLUSH);
 887	payload->header.flags = cpu_to_le16(0);
 888	payload->size = cpu_to_le32(sizeof(__le64));
 889	payload->flush_stripes[0] = cpu_to_le64(sect);
 890	io->meta_offset += meta_size;
 891	/* multiple flush payloads count as one pending_stripe */
 892	if (!io->has_flush_payload) {
 893		io->has_flush_payload = 1;
 894		atomic_inc(&io->pending_stripe);
 895	}
 896	mutex_unlock(&log->io_mutex);
 897}
 898
 899static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
 900			   int data_pages, int parity_pages)
 901{
 902	int i;
 903	int meta_size;
 904	int ret;
 905	struct r5l_io_unit *io;
 906
 907	meta_size =
 908		((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
 909		 * data_pages) +
 910		sizeof(struct r5l_payload_data_parity) +
 911		sizeof(__le32) * parity_pages;
 912
 913	ret = r5l_get_meta(log, meta_size);
 914	if (ret)
 915		return ret;
 916
 917	io = log->current_io;
 918
 919	if (test_and_clear_bit(STRIPE_R5C_PREFLUSH, &sh->state))
 920		io->has_flush = 1;
 921
 922	for (i = 0; i < sh->disks; i++) {
 923		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
 924		    test_bit(R5_InJournal, &sh->dev[i].flags))
 925			continue;
 926		if (i == sh->pd_idx || i == sh->qd_idx)
 927			continue;
 928		if (test_bit(R5_WantFUA, &sh->dev[i].flags) &&
 929		    log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK) {
 930			io->has_fua = 1;
 931			/*
 932			 * we need to flush journal to make sure recovery can
 933			 * reach the data with fua flag
 934			 */
 935			io->has_flush = 1;
 936		}
 937		r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
 938					raid5_compute_blocknr(sh, i, 0),
 939					sh->dev[i].log_checksum, 0, false);
 940		r5l_append_payload_page(log, sh->dev[i].page);
 941	}
 942
 943	if (parity_pages == 2) {
 944		r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
 945					sh->sector, sh->dev[sh->pd_idx].log_checksum,
 946					sh->dev[sh->qd_idx].log_checksum, true);
 947		r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
 948		r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
 949	} else if (parity_pages == 1) {
 950		r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
 951					sh->sector, sh->dev[sh->pd_idx].log_checksum,
 952					0, false);
 953		r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
 954	} else  /* Just writing data, not parity, in caching phase */
 955		BUG_ON(parity_pages != 0);
 956
 957	list_add_tail(&sh->log_list, &io->stripe_list);
 958	atomic_inc(&io->pending_stripe);
 959	sh->log_io = io;
 960
 961	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
 962		return 0;
 963
 964	if (sh->log_start == MaxSector) {
 965		BUG_ON(!list_empty(&sh->r5c));
 966		sh->log_start = io->log_start;
 967		spin_lock_irq(&log->stripe_in_journal_lock);
 968		list_add_tail(&sh->r5c,
 969			      &log->stripe_in_journal_list);
 970		spin_unlock_irq(&log->stripe_in_journal_lock);
 971		atomic_inc(&log->stripe_in_journal_count);
 972	}
 973	return 0;
 974}
 975
 976/* add stripe to no_space_stripes, and then wake up reclaim */
 977static inline void r5l_add_no_space_stripe(struct r5l_log *log,
 978					   struct stripe_head *sh)
 979{
 980	spin_lock(&log->no_space_stripes_lock);
 981	list_add_tail(&sh->log_list, &log->no_space_stripes);
 982	spin_unlock(&log->no_space_stripes_lock);
 983}
 984
 985/*
 986 * running in raid5d, where reclaim could wait for raid5d too (when it flushes
 987 * data from log to raid disks), so we shouldn't wait for reclaim here
 988 */
 989int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 990{
 991	struct r5conf *conf = sh->raid_conf;
 992	int write_disks = 0;
 993	int data_pages, parity_pages;
 994	int reserve;
 995	int i;
 996	int ret = 0;
 997	bool wake_reclaim = false;
 998
 999	if (!log)
1000		return -EAGAIN;
1001	/* Don't support stripe batch */
1002	if (sh->log_io || !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) ||
1003	    test_bit(STRIPE_SYNCING, &sh->state)) {
1004		/* the stripe is written to log, we start writing it to raid */
1005		clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
1006		return -EAGAIN;
1007	}
1008
1009	WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
1010
1011	for (i = 0; i < sh->disks; i++) {
1012		void *addr;
1013
1014		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
1015		    test_bit(R5_InJournal, &sh->dev[i].flags))
1016			continue;
1017
1018		write_disks++;
1019		/* checksum is already calculated in last run */
1020		if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
1021			continue;
1022		addr = kmap_local_page(sh->dev[i].page);
1023		sh->dev[i].log_checksum = crc32c(log->uuid_checksum,
1024						 addr, PAGE_SIZE);
1025		kunmap_local(addr);
1026	}
1027	parity_pages = 1 + !!(sh->qd_idx >= 0);
1028	data_pages = write_disks - parity_pages;
1029
1030	set_bit(STRIPE_LOG_TRAPPED, &sh->state);
1031	/*
1032	 * The stripe must enter state machine again to finish the write, so
1033	 * don't delay.
1034	 */
1035	clear_bit(STRIPE_DELAYED, &sh->state);
1036	atomic_inc(&sh->count);
1037
1038	mutex_lock(&log->io_mutex);
1039	/* meta + data */
1040	reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
1041
1042	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
1043		if (!r5l_has_free_space(log, reserve)) {
1044			r5l_add_no_space_stripe(log, sh);
1045			wake_reclaim = true;
1046		} else {
1047			ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
1048			if (ret) {
1049				spin_lock_irq(&log->io_list_lock);
1050				list_add_tail(&sh->log_list,
1051					      &log->no_mem_stripes);
1052				spin_unlock_irq(&log->io_list_lock);
1053			}
1054		}
1055	} else {  /* R5C_JOURNAL_MODE_WRITE_BACK */
1056		/*
1057		 * log space critical, do not process stripes that are
1058		 * not in cache yet (sh->log_start == MaxSector).
1059		 */
1060		if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
1061		    sh->log_start == MaxSector) {
1062			r5l_add_no_space_stripe(log, sh);
1063			wake_reclaim = true;
1064			reserve = 0;
1065		} else if (!r5l_has_free_space(log, reserve)) {
1066			if (sh->log_start == log->last_checkpoint)
1067				BUG();
1068			else
1069				r5l_add_no_space_stripe(log, sh);
1070		} else {
1071			ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
1072			if (ret) {
1073				spin_lock_irq(&log->io_list_lock);
1074				list_add_tail(&sh->log_list,
1075					      &log->no_mem_stripes);
1076				spin_unlock_irq(&log->io_list_lock);
1077			}
1078		}
1079	}
1080
1081	mutex_unlock(&log->io_mutex);
1082	if (wake_reclaim)
1083		r5l_wake_reclaim(log, reserve);
1084	return 0;
1085}
1086
1087void r5l_write_stripe_run(struct r5l_log *log)
1088{
1089	if (!log)
1090		return;
1091	mutex_lock(&log->io_mutex);
1092	r5l_submit_current_io(log);
1093	mutex_unlock(&log->io_mutex);
1094}
1095
1096int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
1097{
1098	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
1099		/*
1100		 * in write through (journal only)
1101		 * we flush log disk cache first, then write stripe data to
1102		 * raid disks. So if bio is finished, the log disk cache is
1103		 * flushed already. The recovery guarantees we can recovery
1104		 * the bio from log disk, so we don't need to flush again
1105		 */
1106		if (bio->bi_iter.bi_size == 0) {
1107			bio_endio(bio);
1108			return 0;
1109		}
1110		bio->bi_opf &= ~REQ_PREFLUSH;
1111	} else {
1112		/* write back (with cache) */
1113		if (bio->bi_iter.bi_size == 0) {
1114			mutex_lock(&log->io_mutex);
1115			r5l_get_meta(log, 0);
1116			bio_list_add(&log->current_io->flush_barriers, bio);
1117			log->current_io->has_flush = 1;
1118			log->current_io->has_null_flush = 1;
1119			atomic_inc(&log->current_io->pending_stripe);
1120			r5l_submit_current_io(log);
1121			mutex_unlock(&log->io_mutex);
1122			return 0;
1123		}
1124	}
1125	return -EAGAIN;
1126}
1127
1128/* This will run after log space is reclaimed */
1129static void r5l_run_no_space_stripes(struct r5l_log *log)
1130{
1131	struct stripe_head *sh;
1132
1133	spin_lock(&log->no_space_stripes_lock);
1134	while (!list_empty(&log->no_space_stripes)) {
1135		sh = list_first_entry(&log->no_space_stripes,
1136				      struct stripe_head, log_list);
1137		list_del_init(&sh->log_list);
1138		set_bit(STRIPE_HANDLE, &sh->state);
1139		raid5_release_stripe(sh);
1140	}
1141	spin_unlock(&log->no_space_stripes_lock);
1142}
1143
1144/*
1145 * calculate new last_checkpoint
1146 * for write through mode, returns log->next_checkpoint
1147 * for write back, returns log_start of first sh in stripe_in_journal_list
1148 */
1149static sector_t r5c_calculate_new_cp(struct r5conf *conf)
1150{
1151	struct stripe_head *sh;
1152	struct r5l_log *log = READ_ONCE(conf->log);
1153	sector_t new_cp;
1154	unsigned long flags;
1155
1156	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
1157		return log->next_checkpoint;
1158
1159	spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
1160	if (list_empty(&log->stripe_in_journal_list)) {
1161		/* all stripes flushed */
1162		spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1163		return log->next_checkpoint;
1164	}
1165	sh = list_first_entry(&log->stripe_in_journal_list,
1166			      struct stripe_head, r5c);
1167	new_cp = sh->log_start;
1168	spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1169	return new_cp;
1170}
1171
1172static sector_t r5l_reclaimable_space(struct r5l_log *log)
1173{
1174	struct r5conf *conf = log->rdev->mddev->private;
1175
1176	return r5l_ring_distance(log, log->last_checkpoint,
1177				 r5c_calculate_new_cp(conf));
1178}
1179
1180static void r5l_run_no_mem_stripe(struct r5l_log *log)
1181{
1182	struct stripe_head *sh;
1183
1184	lockdep_assert_held(&log->io_list_lock);
1185
1186	if (!list_empty(&log->no_mem_stripes)) {
1187		sh = list_first_entry(&log->no_mem_stripes,
1188				      struct stripe_head, log_list);
1189		list_del_init(&sh->log_list);
1190		set_bit(STRIPE_HANDLE, &sh->state);
1191		raid5_release_stripe(sh);
1192	}
1193}
1194
1195static bool r5l_complete_finished_ios(struct r5l_log *log)
1196{
1197	struct r5l_io_unit *io, *next;
1198	bool found = false;
1199
1200	lockdep_assert_held(&log->io_list_lock);
1201
1202	list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) {
1203		/* don't change list order */
1204		if (io->state < IO_UNIT_STRIPE_END)
1205			break;
1206
1207		log->next_checkpoint = io->log_start;
1208
1209		list_del(&io->log_sibling);
1210		mempool_free(io, &log->io_pool);
1211		r5l_run_no_mem_stripe(log);
1212
1213		found = true;
1214	}
1215
1216	return found;
1217}
1218
1219static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
1220{
1221	struct r5l_log *log = io->log;
1222	struct r5conf *conf = log->rdev->mddev->private;
1223	unsigned long flags;
1224
1225	spin_lock_irqsave(&log->io_list_lock, flags);
1226	__r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END);
1227
1228	if (!r5l_complete_finished_ios(log)) {
1229		spin_unlock_irqrestore(&log->io_list_lock, flags);
1230		return;
1231	}
1232
1233	if (r5l_reclaimable_space(log) > log->max_free_space ||
1234	    test_bit(R5C_LOG_TIGHT, &conf->cache_state))
1235		r5l_wake_reclaim(log, 0);
1236
1237	spin_unlock_irqrestore(&log->io_list_lock, flags);
1238	wake_up(&log->iounit_wait);
1239}
1240
1241void r5l_stripe_write_finished(struct stripe_head *sh)
1242{
1243	struct r5l_io_unit *io;
1244
1245	io = sh->log_io;
1246	sh->log_io = NULL;
1247
1248	if (io && atomic_dec_and_test(&io->pending_stripe))
1249		__r5l_stripe_write_finished(io);
1250}
1251
1252static void r5l_log_flush_endio(struct bio *bio)
1253{
1254	struct r5l_log *log = container_of(bio, struct r5l_log,
1255		flush_bio);
1256	unsigned long flags;
1257	struct r5l_io_unit *io;
1258
1259	if (bio->bi_status)
1260		md_error(log->rdev->mddev, log->rdev);
1261	bio_uninit(bio);
1262
1263	spin_lock_irqsave(&log->io_list_lock, flags);
1264	list_for_each_entry(io, &log->flushing_ios, log_sibling)
1265		r5l_io_run_stripes(io);
1266	list_splice_tail_init(&log->flushing_ios, &log->finished_ios);
1267	spin_unlock_irqrestore(&log->io_list_lock, flags);
1268}
1269
1270/*
1271 * Starting dispatch IO to raid.
1272 * io_unit(meta) consists of a log. There is one situation we want to avoid. A
1273 * broken meta in the middle of a log causes recovery can't find meta at the
1274 * head of log. If operations require meta at the head persistent in log, we
1275 * must make sure meta before it persistent in log too. A case is:
1276 *
1277 * stripe data/parity is in log, we start write stripe to raid disks. stripe
1278 * data/parity must be persistent in log before we do the write to raid disks.
1279 *
1280 * The solution is we restrictly maintain io_unit list order. In this case, we
1281 * only write stripes of an io_unit to raid disks till the io_unit is the first
1282 * one whose data/parity is in log.
1283 */
1284void r5l_flush_stripe_to_raid(struct r5l_log *log)
1285{
1286	bool do_flush;
1287
1288	if (!log || !log->need_cache_flush)
1289		return;
1290
1291	spin_lock_irq(&log->io_list_lock);
1292	/* flush bio is running */
1293	if (!list_empty(&log->flushing_ios)) {
1294		spin_unlock_irq(&log->io_list_lock);
1295		return;
1296	}
1297	list_splice_tail_init(&log->io_end_ios, &log->flushing_ios);
1298	do_flush = !list_empty(&log->flushing_ios);
1299	spin_unlock_irq(&log->io_list_lock);
1300
1301	if (!do_flush)
1302		return;
1303	bio_init(&log->flush_bio, log->rdev->bdev, NULL, 0,
1304		  REQ_OP_WRITE | REQ_PREFLUSH);
1305	log->flush_bio.bi_end_io = r5l_log_flush_endio;
1306	submit_bio(&log->flush_bio);
1307}
1308
1309static void r5l_write_super(struct r5l_log *log, sector_t cp);
1310static void r5l_write_super_and_discard_space(struct r5l_log *log,
1311	sector_t end)
1312{
1313	struct block_device *bdev = log->rdev->bdev;
1314	struct mddev *mddev;
1315
1316	r5l_write_super(log, end);
1317
1318	if (!bdev_max_discard_sectors(bdev))
1319		return;
1320
1321	mddev = log->rdev->mddev;
1322	/*
1323	 * Discard could zero data, so before discard we must make sure
1324	 * superblock is updated to new log tail. Updating superblock (either
1325	 * directly call md_update_sb() or depend on md thread) must hold
1326	 * reconfig mutex. On the other hand, raid5_quiesce is called with
1327	 * reconfig_mutex hold. The first step of raid5_quiesce() is waiting
1328	 * for all IO finish, hence waiting for reclaim thread, while reclaim
1329	 * thread is calling this function and waiting for reconfig mutex. So
1330	 * there is a deadlock. We workaround this issue with a trylock.
1331	 * FIXME: we could miss discard if we can't take reconfig mutex
1332	 */
1333	set_mask_bits(&mddev->sb_flags, 0,
1334		BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1335	if (!mddev_trylock(mddev))
1336		return;
1337	md_update_sb(mddev, 1);
1338	mddev_unlock(mddev);
1339
1340	/* discard IO error really doesn't matter, ignore it */
1341	if (log->last_checkpoint < end) {
1342		blkdev_issue_discard(bdev,
1343				log->last_checkpoint + log->rdev->data_offset,
1344				end - log->last_checkpoint, GFP_NOIO);
1345	} else {
1346		blkdev_issue_discard(bdev,
1347				log->last_checkpoint + log->rdev->data_offset,
1348				log->device_size - log->last_checkpoint,
1349				GFP_NOIO);
1350		blkdev_issue_discard(bdev, log->rdev->data_offset, end,
1351				GFP_NOIO);
1352	}
1353}
1354
1355/*
1356 * r5c_flush_stripe moves stripe from cached list to handle_list. When called,
1357 * the stripe must be on r5c_cached_full_stripes or r5c_cached_partial_stripes.
1358 *
1359 * must hold conf->device_lock
1360 */
1361static void r5c_flush_stripe(struct r5conf *conf, struct stripe_head *sh)
1362{
1363	BUG_ON(list_empty(&sh->lru));
1364	BUG_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
1365	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
1366
1367	/*
1368	 * The stripe is not ON_RELEASE_LIST, so it is safe to call
1369	 * raid5_release_stripe() while holding conf->device_lock
1370	 */
1371	BUG_ON(test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
1372	lockdep_assert_held(&conf->device_lock);
1373
1374	list_del_init(&sh->lru);
1375	atomic_inc(&sh->count);
1376
1377	set_bit(STRIPE_HANDLE, &sh->state);
1378	atomic_inc(&conf->active_stripes);
1379	r5c_make_stripe_write_out(sh);
1380
1381	if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
1382		atomic_inc(&conf->r5c_flushing_partial_stripes);
1383	else
1384		atomic_inc(&conf->r5c_flushing_full_stripes);
1385	raid5_release_stripe(sh);
1386}
1387
1388/*
1389 * if num == 0, flush all full stripes
1390 * if num > 0, flush all full stripes. If less than num full stripes are
1391 *             flushed, flush some partial stripes until totally num stripes are
1392 *             flushed or there is no more cached stripes.
1393 */
1394void r5c_flush_cache(struct r5conf *conf, int num)
1395{
1396	int count;
1397	struct stripe_head *sh, *next;
1398
1399	lockdep_assert_held(&conf->device_lock);
1400	if (!READ_ONCE(conf->log))
1401		return;
1402
1403	count = 0;
1404	list_for_each_entry_safe(sh, next, &conf->r5c_full_stripe_list, lru) {
1405		r5c_flush_stripe(conf, sh);
1406		count++;
1407	}
1408
1409	if (count >= num)
1410		return;
1411	list_for_each_entry_safe(sh, next,
1412				 &conf->r5c_partial_stripe_list, lru) {
1413		r5c_flush_stripe(conf, sh);
1414		if (++count >= num)
1415			break;
1416	}
1417}
1418
1419static void r5c_do_reclaim(struct r5conf *conf)
1420{
1421	struct r5l_log *log = READ_ONCE(conf->log);
1422	struct stripe_head *sh;
1423	int count = 0;
1424	unsigned long flags;
1425	int total_cached;
1426	int stripes_to_flush;
1427	int flushing_partial, flushing_full;
1428
1429	if (!r5c_is_writeback(log))
1430		return;
1431
1432	flushing_partial = atomic_read(&conf->r5c_flushing_partial_stripes);
1433	flushing_full = atomic_read(&conf->r5c_flushing_full_stripes);
1434	total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
1435		atomic_read(&conf->r5c_cached_full_stripes) -
1436		flushing_full - flushing_partial;
1437
1438	if (total_cached > conf->min_nr_stripes * 3 / 4 ||
1439	    atomic_read(&conf->empty_inactive_list_nr) > 0)
1440		/*
1441		 * if stripe cache pressure high, flush all full stripes and
1442		 * some partial stripes
1443		 */
1444		stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
1445	else if (total_cached > conf->min_nr_stripes * 1 / 2 ||
1446		 atomic_read(&conf->r5c_cached_full_stripes) - flushing_full >
1447		 R5C_FULL_STRIPE_FLUSH_BATCH(conf))
1448		/*
1449		 * if stripe cache pressure moderate, or if there is many full
1450		 * stripes,flush all full stripes
1451		 */
1452		stripes_to_flush = 0;
1453	else
1454		/* no need to flush */
1455		stripes_to_flush = -1;
1456
1457	if (stripes_to_flush >= 0) {
1458		spin_lock_irqsave(&conf->device_lock, flags);
1459		r5c_flush_cache(conf, stripes_to_flush);
1460		spin_unlock_irqrestore(&conf->device_lock, flags);
1461	}
1462
1463	/* if log space is tight, flush stripes on stripe_in_journal_list */
1464	if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) {
1465		spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
1466		spin_lock(&conf->device_lock);
1467		list_for_each_entry(sh, &log->stripe_in_journal_list, r5c) {
1468			/*
1469			 * stripes on stripe_in_journal_list could be in any
1470			 * state of the stripe_cache state machine. In this
1471			 * case, we only want to flush stripe on
1472			 * r5c_cached_full/partial_stripes. The following
1473			 * condition makes sure the stripe is on one of the
1474			 * two lists.
1475			 */
1476			if (!list_empty(&sh->lru) &&
1477			    !test_bit(STRIPE_HANDLE, &sh->state) &&
1478			    atomic_read(&sh->count) == 0) {
1479				r5c_flush_stripe(conf, sh);
1480				if (count++ >= R5C_RECLAIM_STRIPE_GROUP)
1481					break;
1482			}
1483		}
1484		spin_unlock(&conf->device_lock);
1485		spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1486	}
1487
1488	if (!test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
1489		r5l_run_no_space_stripes(log);
1490
1491	md_wakeup_thread(conf->mddev->thread);
1492}
1493
1494static void r5l_do_reclaim(struct r5l_log *log)
1495{
1496	struct r5conf *conf = log->rdev->mddev->private;
1497	sector_t reclaim_target = xchg(&log->reclaim_target, 0);
1498	sector_t reclaimable;
1499	sector_t next_checkpoint;
1500	bool write_super;
1501
1502	spin_lock_irq(&log->io_list_lock);
1503	write_super = r5l_reclaimable_space(log) > log->max_free_space ||
1504		reclaim_target != 0 || !list_empty(&log->no_space_stripes);
1505	/*
1506	 * move proper io_unit to reclaim list. We should not change the order.
1507	 * reclaimable/unreclaimable io_unit can be mixed in the list, we
1508	 * shouldn't reuse space of an unreclaimable io_unit
1509	 */
1510	while (1) {
1511		reclaimable = r5l_reclaimable_space(log);
1512		if (reclaimable >= reclaim_target ||
1513		    (list_empty(&log->running_ios) &&
1514		     list_empty(&log->io_end_ios) &&
1515		     list_empty(&log->flushing_ios) &&
1516		     list_empty(&log->finished_ios)))
1517			break;
1518
1519		md_wakeup_thread(log->rdev->mddev->thread);
1520		wait_event_lock_irq(log->iounit_wait,
1521				    r5l_reclaimable_space(log) > reclaimable,
1522				    log->io_list_lock);
1523	}
1524
1525	next_checkpoint = r5c_calculate_new_cp(conf);
1526	spin_unlock_irq(&log->io_list_lock);
1527
1528	if (reclaimable == 0 || !write_super)
1529		return;
1530
1531	/*
1532	 * write_super will flush cache of each raid disk. We must write super
1533	 * here, because the log area might be reused soon and we don't want to
1534	 * confuse recovery
1535	 */
1536	r5l_write_super_and_discard_space(log, next_checkpoint);
1537
1538	mutex_lock(&log->io_mutex);
1539	log->last_checkpoint = next_checkpoint;
1540	r5c_update_log_state(log);
1541	mutex_unlock(&log->io_mutex);
1542
1543	r5l_run_no_space_stripes(log);
1544}
1545
1546static void r5l_reclaim_thread(struct md_thread *thread)
1547{
1548	struct mddev *mddev = thread->mddev;
1549	struct r5conf *conf = mddev->private;
1550	struct r5l_log *log = READ_ONCE(conf->log);
1551
1552	if (!log)
1553		return;
1554	r5c_do_reclaim(conf);
1555	r5l_do_reclaim(log);
1556}
1557
1558void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
1559{
1560	unsigned long target;
1561	unsigned long new = (unsigned long)space; /* overflow in theory */
1562
1563	if (!log)
1564		return;
1565
1566	target = READ_ONCE(log->reclaim_target);
1567	do {
1568		if (new < target)
1569			return;
1570	} while (!try_cmpxchg(&log->reclaim_target, &target, new));
1571	md_wakeup_thread(log->reclaim_thread);
1572}
1573
1574void r5l_quiesce(struct r5l_log *log, int quiesce)
1575{
1576	struct mddev *mddev = log->rdev->mddev;
1577	struct md_thread *thread = rcu_dereference_protected(
1578		log->reclaim_thread, lockdep_is_held(&mddev->reconfig_mutex));
1579
1580	if (quiesce) {
1581		/* make sure r5l_write_super_and_discard_space exits */
1582		wake_up(&mddev->sb_wait);
1583		kthread_park(thread->tsk);
1584		r5l_wake_reclaim(log, MaxSector);
1585		r5l_do_reclaim(log);
1586	} else
1587		kthread_unpark(thread->tsk);
1588}
1589
1590bool r5l_log_disk_error(struct r5conf *conf)
1591{
1592	struct r5l_log *log = READ_ONCE(conf->log);
1593
1594	/* don't allow write if journal disk is missing */
1595	if (!log)
1596		return test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
1597	else
1598		return test_bit(Faulty, &log->rdev->flags);
1599}
1600
1601#define R5L_RECOVERY_PAGE_POOL_SIZE 256
1602
1603struct r5l_recovery_ctx {
1604	struct page *meta_page;		/* current meta */
1605	sector_t meta_total_blocks;	/* total size of current meta and data */
1606	sector_t pos;			/* recovery position */
1607	u64 seq;			/* recovery position seq */
1608	int data_parity_stripes;	/* number of data_parity stripes */
1609	int data_only_stripes;		/* number of data_only stripes */
1610	struct list_head cached_list;
1611
1612	/*
1613	 * read ahead page pool (ra_pool)
1614	 * in recovery, log is read sequentially. It is not efficient to
1615	 * read every page with sync_page_io(). The read ahead page pool
1616	 * reads multiple pages with one IO, so further log read can
1617	 * just copy data from the pool.
1618	 */
1619	struct page *ra_pool[R5L_RECOVERY_PAGE_POOL_SIZE];
1620	struct bio_vec ra_bvec[R5L_RECOVERY_PAGE_POOL_SIZE];
1621	sector_t pool_offset;	/* offset of first page in the pool */
1622	int total_pages;	/* total allocated pages */
1623	int valid_pages;	/* pages with valid data */
1624};
1625
1626static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
1627					    struct r5l_recovery_ctx *ctx)
1628{
1629	struct page *page;
1630
1631	ctx->valid_pages = 0;
1632	ctx->total_pages = 0;
1633	while (ctx->total_pages < R5L_RECOVERY_PAGE_POOL_SIZE) {
1634		page = alloc_page(GFP_KERNEL);
1635
1636		if (!page)
1637			break;
1638		ctx->ra_pool[ctx->total_pages] = page;
1639		ctx->total_pages += 1;
1640	}
1641
1642	if (ctx->total_pages == 0)
1643		return -ENOMEM;
1644
1645	ctx->pool_offset = 0;
1646	return 0;
1647}
1648
1649static void r5l_recovery_free_ra_pool(struct r5l_log *log,
1650					struct r5l_recovery_ctx *ctx)
1651{
1652	int i;
1653
1654	for (i = 0; i < ctx->total_pages; ++i)
1655		put_page(ctx->ra_pool[i]);
1656}
1657
1658/*
1659 * fetch ctx->valid_pages pages from offset
1660 * In normal cases, ctx->valid_pages == ctx->total_pages after the call.
1661 * However, if the offset is close to the end of the journal device,
1662 * ctx->valid_pages could be smaller than ctx->total_pages
1663 */
1664static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
1665				      struct r5l_recovery_ctx *ctx,
1666				      sector_t offset)
1667{
1668	struct bio bio;
1669	int ret;
1670
1671	bio_init(&bio, log->rdev->bdev, ctx->ra_bvec,
1672		 R5L_RECOVERY_PAGE_POOL_SIZE, REQ_OP_READ);
1673	bio.bi_iter.bi_sector = log->rdev->data_offset + offset;
1674
1675	ctx->valid_pages = 0;
1676	ctx->pool_offset = offset;
1677
1678	while (ctx->valid_pages < ctx->total_pages) {
1679		__bio_add_page(&bio, ctx->ra_pool[ctx->valid_pages], PAGE_SIZE,
1680			       0);
1681		ctx->valid_pages += 1;
1682
1683		offset = r5l_ring_add(log, offset, BLOCK_SECTORS);
1684
1685		if (offset == 0)  /* reached end of the device */
1686			break;
1687	}
1688
1689	ret = submit_bio_wait(&bio);
1690	bio_uninit(&bio);
1691	return ret;
1692}
1693
1694/*
1695 * try read a page from the read ahead page pool, if the page is not in the
1696 * pool, call r5l_recovery_fetch_ra_pool
1697 */
1698static int r5l_recovery_read_page(struct r5l_log *log,
1699				  struct r5l_recovery_ctx *ctx,
1700				  struct page *page,
1701				  sector_t offset)
1702{
1703	int ret;
1704
1705	if (offset < ctx->pool_offset ||
1706	    offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS) {
1707		ret = r5l_recovery_fetch_ra_pool(log, ctx, offset);
1708		if (ret)
1709			return ret;
1710	}
1711
1712	BUG_ON(offset < ctx->pool_offset ||
1713	       offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS);
1714
1715	memcpy(page_address(page),
1716	       page_address(ctx->ra_pool[(offset - ctx->pool_offset) >>
1717					 BLOCK_SECTOR_SHIFT]),
1718	       PAGE_SIZE);
1719	return 0;
1720}
1721
1722static int r5l_recovery_read_meta_block(struct r5l_log *log,
1723					struct r5l_recovery_ctx *ctx)
1724{
1725	struct page *page = ctx->meta_page;
1726	struct r5l_meta_block *mb;
1727	u32 crc, stored_crc;
1728	int ret;
1729
1730	ret = r5l_recovery_read_page(log, ctx, page, ctx->pos);
1731	if (ret != 0)
1732		return ret;
1733
1734	mb = page_address(page);
1735	stored_crc = le32_to_cpu(mb->checksum);
1736	mb->checksum = 0;
1737
1738	if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
1739	    le64_to_cpu(mb->seq) != ctx->seq ||
1740	    mb->version != R5LOG_VERSION ||
1741	    le64_to_cpu(mb->position) != ctx->pos)
1742		return -EINVAL;
1743
1744	crc = crc32c(log->uuid_checksum, mb, PAGE_SIZE);
1745	if (stored_crc != crc)
1746		return -EINVAL;
1747
1748	if (le32_to_cpu(mb->meta_size) > PAGE_SIZE)
1749		return -EINVAL;
1750
1751	ctx->meta_total_blocks = BLOCK_SECTORS;
1752
1753	return 0;
1754}
1755
1756static void
1757r5l_recovery_create_empty_meta_block(struct r5l_log *log,
1758				     struct page *page,
1759				     sector_t pos, u64 seq)
1760{
1761	struct r5l_meta_block *mb;
1762
1763	mb = page_address(page);
1764	clear_page(mb);
1765	mb->magic = cpu_to_le32(R5LOG_MAGIC);
1766	mb->version = R5LOG_VERSION;
1767	mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
1768	mb->seq = cpu_to_le64(seq);
1769	mb->position = cpu_to_le64(pos);
1770}
1771
1772static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
1773					  u64 seq)
1774{
1775	struct page *page;
1776	struct r5l_meta_block *mb;
1777
1778	page = alloc_page(GFP_KERNEL);
1779	if (!page)
1780		return -ENOMEM;
1781	r5l_recovery_create_empty_meta_block(log, page, pos, seq);
1782	mb = page_address(page);
1783	mb->checksum = cpu_to_le32(crc32c(log->uuid_checksum, mb, PAGE_SIZE));
1784	if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE |
1785			  REQ_SYNC | REQ_FUA, false)) {
1786		__free_page(page);
1787		return -EIO;
1788	}
1789	__free_page(page);
1790	return 0;
1791}
1792
1793/*
1794 * r5l_recovery_load_data and r5l_recovery_load_parity uses flag R5_Wantwrite
1795 * to mark valid (potentially not flushed) data in the journal.
1796 *
1797 * We already verified checksum in r5l_recovery_verify_data_checksum_for_mb,
1798 * so there should not be any mismatch here.
1799 */
1800static void r5l_recovery_load_data(struct r5l_log *log,
1801				   struct stripe_head *sh,
1802				   struct r5l_recovery_ctx *ctx,
1803				   struct r5l_payload_data_parity *payload,
1804				   sector_t log_offset)
1805{
1806	struct mddev *mddev = log->rdev->mddev;
1807	struct r5conf *conf = mddev->private;
1808	int dd_idx;
1809
1810	raid5_compute_sector(conf,
1811			     le64_to_cpu(payload->location), 0,
1812			     &dd_idx, sh);
1813	r5l_recovery_read_page(log, ctx, sh->dev[dd_idx].page, log_offset);
1814	sh->dev[dd_idx].log_checksum =
1815		le32_to_cpu(payload->checksum[0]);
1816	ctx->meta_total_blocks += BLOCK_SECTORS;
1817
1818	set_bit(R5_Wantwrite, &sh->dev[dd_idx].flags);
1819	set_bit(STRIPE_R5C_CACHING, &sh->state);
1820}
1821
1822static void r5l_recovery_load_parity(struct r5l_log *log,
1823				     struct stripe_head *sh,
1824				     struct r5l_recovery_ctx *ctx,
1825				     struct r5l_payload_data_parity *payload,
1826				     sector_t log_offset)
1827{
1828	struct mddev *mddev = log->rdev->mddev;
1829	struct r5conf *conf = mddev->private;
1830
1831	ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
1832	r5l_recovery_read_page(log, ctx, sh->dev[sh->pd_idx].page, log_offset);
1833	sh->dev[sh->pd_idx].log_checksum =
1834		le32_to_cpu(payload->checksum[0]);
1835	set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
1836
1837	if (sh->qd_idx >= 0) {
1838		r5l_recovery_read_page(
1839			log, ctx, sh->dev[sh->qd_idx].page,
1840			r5l_ring_add(log, log_offset, BLOCK_SECTORS));
1841		sh->dev[sh->qd_idx].log_checksum =
1842			le32_to_cpu(payload->checksum[1]);
1843		set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
1844	}
1845	clear_bit(STRIPE_R5C_CACHING, &sh->state);
1846}
1847
1848static void r5l_recovery_reset_stripe(struct stripe_head *sh)
1849{
1850	int i;
1851
1852	sh->state = 0;
1853	sh->log_start = MaxSector;
1854	for (i = sh->disks; i--; )
1855		sh->dev[i].flags = 0;
1856}
1857
1858static void
1859r5l_recovery_replay_one_stripe(struct r5conf *conf,
1860			       struct stripe_head *sh,
1861			       struct r5l_recovery_ctx *ctx)
1862{
1863	struct md_rdev *rdev, *rrdev;
1864	int disk_index;
1865	int data_count = 0;
1866
1867	for (disk_index = 0; disk_index < sh->disks; disk_index++) {
1868		if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
1869			continue;
1870		if (disk_index == sh->qd_idx || disk_index == sh->pd_idx)
1871			continue;
1872		data_count++;
1873	}
1874
1875	/*
1876	 * stripes that only have parity must have been flushed
1877	 * before the crash that we are now recovering from, so
1878	 * there is nothing more to recovery.
1879	 */
1880	if (data_count == 0)
1881		goto out;
1882
1883	for (disk_index = 0; disk_index < sh->disks; disk_index++) {
1884		if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
1885			continue;
1886
1887		/* in case device is broken */
1888		rdev = conf->disks[disk_index].rdev;
1889		if (rdev) {
1890			atomic_inc(&rdev->nr_pending);
1891			sync_page_io(rdev, sh->sector, PAGE_SIZE,
1892				     sh->dev[disk_index].page, REQ_OP_WRITE,
1893				     false);
1894			rdev_dec_pending(rdev, rdev->mddev);
1895		}
1896		rrdev = conf->disks[disk_index].replacement;
1897		if (rrdev) {
1898			atomic_inc(&rrdev->nr_pending);
1899			sync_page_io(rrdev, sh->sector, PAGE_SIZE,
1900				     sh->dev[disk_index].page, REQ_OP_WRITE,
1901				     false);
1902			rdev_dec_pending(rrdev, rrdev->mddev);
1903		}
1904	}
1905	ctx->data_parity_stripes++;
1906out:
1907	r5l_recovery_reset_stripe(sh);
1908}
1909
1910static struct stripe_head *
1911r5c_recovery_alloc_stripe(
1912		struct r5conf *conf,
1913		sector_t stripe_sect,
1914		int noblock)
1915{
1916	struct stripe_head *sh;
1917
1918	sh = raid5_get_active_stripe(conf, NULL, stripe_sect,
1919				     noblock ? R5_GAS_NOBLOCK : 0);
1920	if (!sh)
1921		return NULL;  /* no more stripe available */
1922
1923	r5l_recovery_reset_stripe(sh);
1924
1925	return sh;
1926}
1927
1928static struct stripe_head *
1929r5c_recovery_lookup_stripe(struct list_head *list, sector_t sect)
1930{
1931	struct stripe_head *sh;
1932
1933	list_for_each_entry(sh, list, lru)
1934		if (sh->sector == sect)
1935			return sh;
1936	return NULL;
1937}
1938
1939static void
1940r5c_recovery_drop_stripes(struct list_head *cached_stripe_list,
1941			  struct r5l_recovery_ctx *ctx)
1942{
1943	struct stripe_head *sh, *next;
1944
1945	list_for_each_entry_safe(sh, next, cached_stripe_list, lru) {
1946		r5l_recovery_reset_stripe(sh);
1947		list_del_init(&sh->lru);
1948		raid5_release_stripe(sh);
1949	}
1950}
1951
1952static void
1953r5c_recovery_replay_stripes(struct list_head *cached_stripe_list,
1954			    struct r5l_recovery_ctx *ctx)
1955{
1956	struct stripe_head *sh, *next;
1957
1958	list_for_each_entry_safe(sh, next, cached_stripe_list, lru)
1959		if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
1960			r5l_recovery_replay_one_stripe(sh->raid_conf, sh, ctx);
1961			list_del_init(&sh->lru);
1962			raid5_release_stripe(sh);
1963		}
1964}
1965
1966/* if matches return 0; otherwise return -EINVAL */
1967static int
1968r5l_recovery_verify_data_checksum(struct r5l_log *log,
1969				  struct r5l_recovery_ctx *ctx,
1970				  struct page *page,
1971				  sector_t log_offset, __le32 log_checksum)
1972{
1973	void *addr;
1974	u32 checksum;
1975
1976	r5l_recovery_read_page(log, ctx, page, log_offset);
1977	addr = kmap_local_page(page);
1978	checksum = crc32c(log->uuid_checksum, addr, PAGE_SIZE);
1979	kunmap_local(addr);
1980	return (le32_to_cpu(log_checksum) == checksum) ? 0 : -EINVAL;
1981}
1982
1983/*
1984 * before loading data to stripe cache, we need verify checksum for all data,
1985 * if there is mismatch for any data page, we drop all data in the mata block
1986 */
1987static int
1988r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
1989					 struct r5l_recovery_ctx *ctx)
1990{
1991	struct mddev *mddev = log->rdev->mddev;
1992	struct r5conf *conf = mddev->private;
1993	struct r5l_meta_block *mb = page_address(ctx->meta_page);
1994	sector_t mb_offset = sizeof(struct r5l_meta_block);
1995	sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
1996	struct page *page;
1997	struct r5l_payload_data_parity *payload;
1998	struct r5l_payload_flush *payload_flush;
1999
2000	page = alloc_page(GFP_KERNEL);
2001	if (!page)
2002		return -ENOMEM;
2003
2004	while (mb_offset < le32_to_cpu(mb->meta_size)) {
2005		payload = (void *)mb + mb_offset;
2006		payload_flush = (void *)mb + mb_offset;
2007
2008		if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
2009			if (r5l_recovery_verify_data_checksum(
2010				    log, ctx, page, log_offset,
2011				    payload->checksum[0]) < 0)
2012				goto mismatch;
2013		} else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) {
2014			if (r5l_recovery_verify_data_checksum(
2015				    log, ctx, page, log_offset,
2016				    payload->checksum[0]) < 0)
2017				goto mismatch;
2018			if (conf->max_degraded == 2 && /* q for RAID 6 */
2019			    r5l_recovery_verify_data_checksum(
2020				    log, ctx, page,
2021				    r5l_ring_add(log, log_offset,
2022						 BLOCK_SECTORS),
2023				    payload->checksum[1]) < 0)
2024				goto mismatch;
2025		} else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
2026			/* nothing to do for R5LOG_PAYLOAD_FLUSH here */
2027		} else /* not R5LOG_PAYLOAD_DATA/PARITY/FLUSH */
2028			goto mismatch;
2029
2030		if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
2031			mb_offset += sizeof(struct r5l_payload_flush) +
2032				le32_to_cpu(payload_flush->size);
2033		} else {
2034			/* DATA or PARITY payload */
2035			log_offset = r5l_ring_add(log, log_offset,
2036						  le32_to_cpu(payload->size));
2037			mb_offset += sizeof(struct r5l_payload_data_parity) +
2038				sizeof(__le32) *
2039				(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
2040		}
2041
2042	}
2043
2044	put_page(page);
2045	return 0;
2046
2047mismatch:
2048	put_page(page);
2049	return -EINVAL;
2050}
2051
2052/*
2053 * Analyze all data/parity pages in one meta block
2054 * Returns:
2055 * 0 for success
2056 * -EINVAL for unknown playload type
2057 * -EAGAIN for checksum mismatch of data page
2058 * -ENOMEM for run out of memory (alloc_page failed or run out of stripes)
2059 */
2060static int
2061r5c_recovery_analyze_meta_block(struct r5l_log *log,
2062				struct r5l_recovery_ctx *ctx,
2063				struct list_head *cached_stripe_list)
2064{
2065	struct mddev *mddev = log->rdev->mddev;
2066	struct r5conf *conf = mddev->private;
2067	struct r5l_meta_block *mb;
2068	struct r5l_payload_data_parity *payload;
2069	struct r5l_payload_flush *payload_flush;
2070	int mb_offset;
2071	sector_t log_offset;
2072	sector_t stripe_sect;
2073	struct stripe_head *sh;
2074	int ret;
2075
2076	/*
2077	 * for mismatch in data blocks, we will drop all data in this mb, but
2078	 * we will still read next mb for other data with FLUSH flag, as
2079	 * io_unit could finish out of order.
2080	 */
2081	ret = r5l_recovery_verify_data_checksum_for_mb(log, ctx);
2082	if (ret == -EINVAL)
2083		return -EAGAIN;
2084	else if (ret)
2085		return ret;   /* -ENOMEM duo to alloc_page() failed */
2086
2087	mb = page_address(ctx->meta_page);
2088	mb_offset = sizeof(struct r5l_meta_block);
2089	log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2090
2091	while (mb_offset < le32_to_cpu(mb->meta_size)) {
2092		int dd;
2093
2094		payload = (void *)mb + mb_offset;
2095		payload_flush = (void *)mb + mb_offset;
2096
2097		if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
2098			int i, count;
2099
2100			count = le32_to_cpu(payload_flush->size) / sizeof(__le64);
2101			for (i = 0; i < count; ++i) {
2102				stripe_sect = le64_to_cpu(payload_flush->flush_stripes[i]);
2103				sh = r5c_recovery_lookup_stripe(cached_stripe_list,
2104								stripe_sect);
2105				if (sh) {
2106					WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
2107					r5l_recovery_reset_stripe(sh);
2108					list_del_init(&sh->lru);
2109					raid5_release_stripe(sh);
2110				}
2111			}
2112
2113			mb_offset += sizeof(struct r5l_payload_flush) +
2114				le32_to_cpu(payload_flush->size);
2115			continue;
2116		}
2117
2118		/* DATA or PARITY payload */
2119		stripe_sect = (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) ?
2120			raid5_compute_sector(
2121				conf, le64_to_cpu(payload->location), 0, &dd,
2122				NULL)
2123			: le64_to_cpu(payload->location);
2124
2125		sh = r5c_recovery_lookup_stripe(cached_stripe_list,
2126						stripe_sect);
2127
2128		if (!sh) {
2129			sh = r5c_recovery_alloc_stripe(conf, stripe_sect, 1);
2130			/*
2131			 * cannot get stripe from raid5_get_active_stripe
2132			 * try replay some stripes
2133			 */
2134			if (!sh) {
2135				r5c_recovery_replay_stripes(
2136					cached_stripe_list, ctx);
2137				sh = r5c_recovery_alloc_stripe(
2138					conf, stripe_sect, 1);
2139			}
2140			if (!sh) {
2141				int new_size = conf->min_nr_stripes * 2;
2142				pr_debug("md/raid:%s: Increasing stripe cache size to %d to recovery data on journal.\n",
2143					mdname(mddev),
2144					new_size);
2145				ret = raid5_set_cache_size(mddev, new_size);
2146				if (conf->min_nr_stripes <= new_size / 2) {
2147					pr_err("md/raid:%s: Cannot increase cache size, ret=%d, new_size=%d, min_nr_stripes=%d, max_nr_stripes=%d\n",
2148						mdname(mddev),
2149						ret,
2150						new_size,
2151						conf->min_nr_stripes,
2152						conf->max_nr_stripes);
2153					return -ENOMEM;
2154				}
2155				sh = r5c_recovery_alloc_stripe(
2156					conf, stripe_sect, 0);
2157			}
2158			if (!sh) {
2159				pr_err("md/raid:%s: Cannot get enough stripes due to memory pressure. Recovery failed.\n",
2160					mdname(mddev));
2161				return -ENOMEM;
2162			}
2163			list_add_tail(&sh->lru, cached_stripe_list);
2164		}
2165
2166		if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
2167			if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
2168			    test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
2169				r5l_recovery_replay_one_stripe(conf, sh, ctx);
2170				list_move_tail(&sh->lru, cached_stripe_list);
2171			}
2172			r5l_recovery_load_data(log, sh, ctx, payload,
2173					       log_offset);
2174		} else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
2175			r5l_recovery_load_parity(log, sh, ctx, payload,
2176						 log_offset);
2177		else
2178			return -EINVAL;
2179
2180		log_offset = r5l_ring_add(log, log_offset,
2181					  le32_to_cpu(payload->size));
2182
2183		mb_offset += sizeof(struct r5l_payload_data_parity) +
2184			sizeof(__le32) *
2185			(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
2186	}
2187
2188	return 0;
2189}
2190
2191/*
2192 * Load the stripe into cache. The stripe will be written out later by
2193 * the stripe cache state machine.
2194 */
2195static void r5c_recovery_load_one_stripe(struct r5l_log *log,
2196					 struct stripe_head *sh)
2197{
2198	struct r5dev *dev;
2199	int i;
2200
2201	for (i = sh->disks; i--; ) {
2202		dev = sh->dev + i;
2203		if (test_and_clear_bit(R5_Wantwrite, &dev->flags)) {
2204			set_bit(R5_InJournal, &dev->flags);
2205			set_bit(R5_UPTODATE, &dev->flags);
2206		}
2207	}
2208}
2209
2210/*
2211 * Scan through the log for all to-be-flushed data
2212 *
2213 * For stripes with data and parity, namely Data-Parity stripe
2214 * (STRIPE_R5C_CACHING == 0), we simply replay all the writes.
2215 *
2216 * For stripes with only data, namely Data-Only stripe
2217 * (STRIPE_R5C_CACHING == 1), we load them to stripe cache state machine.
2218 *
2219 * For a stripe, if we see data after parity, we should discard all previous
2220 * data and parity for this stripe, as these data are already flushed to
2221 * the array.
2222 *
2223 * At the end of the scan, we return the new journal_tail, which points to
2224 * first data-only stripe on the journal device, or next invalid meta block.
2225 */
2226static int r5c_recovery_flush_log(struct r5l_log *log,
2227				  struct r5l_recovery_ctx *ctx)
2228{
2229	struct stripe_head *sh;
2230	int ret = 0;
2231
2232	/* scan through the log */
2233	while (1) {
2234		if (r5l_recovery_read_meta_block(log, ctx))
2235			break;
2236
2237		ret = r5c_recovery_analyze_meta_block(log, ctx,
2238						      &ctx->cached_list);
2239		/*
2240		 * -EAGAIN means mismatch in data block, in this case, we still
2241		 * try scan the next metablock
2242		 */
2243		if (ret && ret != -EAGAIN)
2244			break;   /* ret == -EINVAL or -ENOMEM */
2245		ctx->seq++;
2246		ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
2247	}
2248
2249	if (ret == -ENOMEM) {
2250		r5c_recovery_drop_stripes(&ctx->cached_list, ctx);
2251		return ret;
2252	}
2253
2254	/* replay data-parity stripes */
2255	r5c_recovery_replay_stripes(&ctx->cached_list, ctx);
2256
2257	/* load data-only stripes to stripe cache */
2258	list_for_each_entry(sh, &ctx->cached_list, lru) {
2259		WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
2260		r5c_recovery_load_one_stripe(log, sh);
2261		ctx->data_only_stripes++;
2262	}
2263
2264	return 0;
2265}
2266
2267/*
2268 * we did a recovery. Now ctx.pos points to an invalid meta block. New
2269 * log will start here. but we can't let superblock point to last valid
2270 * meta block. The log might looks like:
2271 * | meta 1| meta 2| meta 3|
2272 * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
2273 * superblock points to meta 1, we write a new valid meta 2n.  if crash
2274 * happens again, new recovery will start from meta 1. Since meta 2n is
2275 * valid now, recovery will think meta 3 is valid, which is wrong.
2276 * The solution is we create a new meta in meta2 with its seq == meta
2277 * 1's seq + 10000 and let superblock points to meta2. The same recovery
2278 * will not think meta 3 is a valid meta, because its seq doesn't match
2279 */
2280
2281/*
2282 * Before recovery, the log looks like the following
2283 *
2284 *   ---------------------------------------------
2285 *   |           valid log        | invalid log  |
2286 *   ---------------------------------------------
2287 *   ^
2288 *   |- log->last_checkpoint
2289 *   |- log->last_cp_seq
2290 *
2291 * Now we scan through the log until we see invalid entry
2292 *
2293 *   ---------------------------------------------
2294 *   |           valid log        | invalid log  |
2295 *   ---------------------------------------------
2296 *   ^                            ^
2297 *   |- log->last_checkpoint      |- ctx->pos
2298 *   |- log->last_cp_seq          |- ctx->seq
2299 *
2300 * From this point, we need to increase seq number by 10 to avoid
2301 * confusing next recovery.
2302 *
2303 *   ---------------------------------------------
2304 *   |           valid log        | invalid log  |
2305 *   ---------------------------------------------
2306 *   ^                              ^
2307 *   |- log->last_checkpoint        |- ctx->pos+1
2308 *   |- log->last_cp_seq            |- ctx->seq+10001
2309 *
2310 * However, it is not safe to start the state machine yet, because data only
2311 * parities are not yet secured in RAID. To save these data only parities, we
2312 * rewrite them from seq+11.
2313 *
2314 *   -----------------------------------------------------------------
2315 *   |           valid log        | data only stripes | invalid log  |
2316 *   -----------------------------------------------------------------
2317 *   ^                                                ^
2318 *   |- log->last_checkpoint                          |- ctx->pos+n
2319 *   |- log->last_cp_seq                              |- ctx->seq+10000+n
2320 *
2321 * If failure happens again during this process, the recovery can safe start
2322 * again from log->last_checkpoint.
2323 *
2324 * Once data only stripes are rewritten to journal, we move log_tail
2325 *
2326 *   -----------------------------------------------------------------
2327 *   |     old log        |    data only stripes    | invalid log  |
2328 *   -----------------------------------------------------------------
2329 *                        ^                         ^
2330 *                        |- log->last_checkpoint   |- ctx->pos+n
2331 *                        |- log->last_cp_seq       |- ctx->seq+10000+n
2332 *
2333 * Then we can safely start the state machine. If failure happens from this
2334 * point on, the recovery will start from new log->last_checkpoint.
2335 */
2336static int
2337r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log,
2338				       struct r5l_recovery_ctx *ctx)
2339{
2340	struct stripe_head *sh;
2341	struct mddev *mddev = log->rdev->mddev;
2342	struct page *page;
2343	sector_t next_checkpoint = MaxSector;
2344
2345	page = alloc_page(GFP_KERNEL);
2346	if (!page) {
2347		pr_err("md/raid:%s: cannot allocate memory to rewrite data only stripes\n",
2348		       mdname(mddev));
2349		return -ENOMEM;
2350	}
2351
2352	WARN_ON(list_empty(&ctx->cached_list));
2353
2354	list_for_each_entry(sh, &ctx->cached_list, lru) {
2355		struct r5l_meta_block *mb;
2356		int i;
2357		int offset;
2358		sector_t write_pos;
2359
2360		WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
2361		r5l_recovery_create_empty_meta_block(log, page,
2362						     ctx->pos, ctx->seq);
2363		mb = page_address(page);
2364		offset = le32_to_cpu(mb->meta_size);
2365		write_pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2366
2367		for (i = sh->disks; i--; ) {
2368			struct r5dev *dev = &sh->dev[i];
2369			struct r5l_payload_data_parity *payload;
2370			void *addr;
2371
2372			if (test_bit(R5_InJournal, &dev->flags)) {
2373				payload = (void *)mb + offset;
2374				payload->header.type = cpu_to_le16(
2375					R5LOG_PAYLOAD_DATA);
2376				payload->size = cpu_to_le32(BLOCK_SECTORS);
2377				payload->location = cpu_to_le64(
2378					raid5_compute_blocknr(sh, i, 0));
2379				addr = kmap_local_page(dev->page);
2380				payload->checksum[0] = cpu_to_le32(
2381					crc32c(log->uuid_checksum, addr,
2382					       PAGE_SIZE));
2383				kunmap_local(addr);
2384				sync_page_io(log->rdev, write_pos, PAGE_SIZE,
2385					     dev->page, REQ_OP_WRITE, false);
2386				write_pos = r5l_ring_add(log, write_pos,
2387							 BLOCK_SECTORS);
2388				offset += sizeof(__le32) +
2389					sizeof(struct r5l_payload_data_parity);
2390
2391			}
2392		}
2393		mb->meta_size = cpu_to_le32(offset);
2394		mb->checksum = cpu_to_le32(crc32c(log->uuid_checksum,
2395						  mb, PAGE_SIZE));
2396		sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
2397			     REQ_OP_WRITE | REQ_SYNC | REQ_FUA, false);
2398		sh->log_start = ctx->pos;
2399		list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
2400		atomic_inc(&log->stripe_in_journal_count);
2401		ctx->pos = write_pos;
2402		ctx->seq += 1;
2403		next_checkpoint = sh->log_start;
2404	}
2405	log->next_checkpoint = next_checkpoint;
2406	__free_page(page);
2407	return 0;
2408}
2409
2410static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log,
2411						 struct r5l_recovery_ctx *ctx)
2412{
2413	struct mddev *mddev = log->rdev->mddev;
2414	struct r5conf *conf = mddev->private;
2415	struct stripe_head *sh, *next;
2416	bool cleared_pending = false;
2417
2418	if (ctx->data_only_stripes == 0)
2419		return;
2420
2421	if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2422		cleared_pending = true;
2423		clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
2424	}
2425	log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_BACK;
2426
2427	list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) {
2428		r5c_make_stripe_write_out(sh);
2429		set_bit(STRIPE_HANDLE, &sh->state);
2430		list_del_init(&sh->lru);
2431		raid5_release_stripe(sh);
2432	}
2433
2434	/* reuse conf->wait_for_quiescent in recovery */
2435	wait_event(conf->wait_for_quiescent,
2436		   atomic_read(&conf->active_stripes) == 0);
2437
2438	log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
2439	if (cleared_pending)
2440		set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
2441}
2442
2443static int r5l_recovery_log(struct r5l_log *log)
2444{
2445	struct mddev *mddev = log->rdev->mddev;
2446	struct r5l_recovery_ctx *ctx;
2447	int ret;
2448	sector_t pos;
2449
2450	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2451	if (!ctx)
2452		return -ENOMEM;
2453
2454	ctx->pos = log->last_checkpoint;
2455	ctx->seq = log->last_cp_seq;
2456	INIT_LIST_HEAD(&ctx->cached_list);
2457	ctx->meta_page = alloc_page(GFP_KERNEL);
2458
2459	if (!ctx->meta_page) {
2460		ret =  -ENOMEM;
2461		goto meta_page;
2462	}
2463
2464	if (r5l_recovery_allocate_ra_pool(log, ctx) != 0) {
2465		ret = -ENOMEM;
2466		goto ra_pool;
2467	}
2468
2469	ret = r5c_recovery_flush_log(log, ctx);
2470
2471	if (ret)
2472		goto error;
2473
2474	pos = ctx->pos;
2475	ctx->seq += 10000;
2476
2477	if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
2478		pr_info("md/raid:%s: starting from clean shutdown\n",
2479			 mdname(mddev));
2480	else
2481		pr_info("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
2482			 mdname(mddev), ctx->data_only_stripes,
2483			 ctx->data_parity_stripes);
2484
2485	if (ctx->data_only_stripes == 0) {
2486		log->next_checkpoint = ctx->pos;
2487		r5l_log_write_empty_meta_block(log, ctx->pos, ctx->seq++);
2488		ctx->pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2489	} else if (r5c_recovery_rewrite_data_only_stripes(log, ctx)) {
2490		pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
2491		       mdname(mddev));
2492		ret =  -EIO;
2493		goto error;
2494	}
2495
2496	log->log_start = ctx->pos;
2497	log->seq = ctx->seq;
2498	log->last_checkpoint = pos;
2499	r5l_write_super(log, pos);
2500
2501	r5c_recovery_flush_data_only_stripes(log, ctx);
2502	ret = 0;
2503error:
2504	r5l_recovery_free_ra_pool(log, ctx);
2505ra_pool:
2506	__free_page(ctx->meta_page);
2507meta_page:
2508	kfree(ctx);
2509	return ret;
2510}
2511
2512static void r5l_write_super(struct r5l_log *log, sector_t cp)
2513{
2514	struct mddev *mddev = log->rdev->mddev;
2515
2516	log->rdev->journal_tail = cp;
2517	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2518}
2519
2520static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
2521{
2522	struct r5conf *conf;
2523	int ret;
2524
2525	ret = mddev_lock(mddev);
2526	if (ret)
2527		return ret;
2528
2529	conf = mddev->private;
2530	if (!conf || !conf->log)
2531		goto out_unlock;
2532
2533	switch (conf->log->r5c_journal_mode) {
2534	case R5C_JOURNAL_MODE_WRITE_THROUGH:
2535		ret = snprintf(
2536			page, PAGE_SIZE, "[%s] %s\n",
2537			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
2538			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
2539		break;
2540	case R5C_JOURNAL_MODE_WRITE_BACK:
2541		ret = snprintf(
2542			page, PAGE_SIZE, "%s [%s]\n",
2543			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
2544			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
2545		break;
2546	default:
2547		ret = 0;
2548	}
2549
2550out_unlock:
2551	mddev_unlock(mddev);
2552	return ret;
2553}
2554
2555/*
2556 * Set journal cache mode on @mddev (external API initially needed by dm-raid).
2557 *
2558 * @mode as defined in 'enum r5c_journal_mode'.
2559 *
2560 */
2561int r5c_journal_mode_set(struct mddev *mddev, int mode)
2562{
2563	struct r5conf *conf;
2564
2565	if (mode < R5C_JOURNAL_MODE_WRITE_THROUGH ||
2566	    mode > R5C_JOURNAL_MODE_WRITE_BACK)
2567		return -EINVAL;
2568
2569	conf = mddev->private;
2570	if (!conf || !conf->log)
2571		return -ENODEV;
2572
2573	if (raid5_calc_degraded(conf) > 0 &&
2574	    mode == R5C_JOURNAL_MODE_WRITE_BACK)
2575		return -EINVAL;
2576
2577	conf->log->r5c_journal_mode = mode;
2578
2579	pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
2580		 mdname(mddev), mode, r5c_journal_mode_str[mode]);
2581	return 0;
2582}
2583EXPORT_SYMBOL(r5c_journal_mode_set);
2584
2585static ssize_t r5c_journal_mode_store(struct mddev *mddev,
2586				      const char *page, size_t length)
2587{
2588	int mode = ARRAY_SIZE(r5c_journal_mode_str);
2589	size_t len = length;
2590	int ret;
2591
2592	if (len < 2)
2593		return -EINVAL;
2594
2595	if (page[len - 1] == '\n')
2596		len--;
2597
2598	while (mode--)
2599		if (strlen(r5c_journal_mode_str[mode]) == len &&
2600		    !strncmp(page, r5c_journal_mode_str[mode], len))
2601			break;
2602	ret = mddev_suspend_and_lock(mddev);
2603	if (ret)
2604		return ret;
2605	ret = r5c_journal_mode_set(mddev, mode);
2606	mddev_unlock_and_resume(mddev);
2607	return ret ?: length;
2608}
2609
2610struct md_sysfs_entry
2611r5c_journal_mode = __ATTR(journal_mode, 0644,
2612			  r5c_journal_mode_show, r5c_journal_mode_store);
2613
2614/*
2615 * Try handle write operation in caching phase. This function should only
2616 * be called in write-back mode.
2617 *
2618 * If all outstanding writes can be handled in caching phase, returns 0
2619 * If writes requires write-out phase, call r5c_make_stripe_write_out()
2620 * and returns -EAGAIN
2621 */
2622int r5c_try_caching_write(struct r5conf *conf,
2623			  struct stripe_head *sh,
2624			  struct stripe_head_state *s,
2625			  int disks)
2626{
2627	struct r5l_log *log = READ_ONCE(conf->log);
2628	int i;
2629	struct r5dev *dev;
2630	int to_cache = 0;
2631	void __rcu **pslot;
2632	sector_t tree_index;
2633	int ret;
2634	uintptr_t refcount;
2635
2636	BUG_ON(!r5c_is_writeback(log));
2637
2638	if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
2639		/*
2640		 * There are two different scenarios here:
2641		 *  1. The stripe has some data cached, and it is sent to
2642		 *     write-out phase for reclaim
2643		 *  2. The stripe is clean, and this is the first write
2644		 *
2645		 * For 1, return -EAGAIN, so we continue with
2646		 * handle_stripe_dirtying().
2647		 *
2648		 * For 2, set STRIPE_R5C_CACHING and continue with caching
2649		 * write.
2650		 */
2651
2652		/* case 1: anything injournal or anything in written */
2653		if (s->injournal > 0 || s->written > 0)
2654			return -EAGAIN;
2655		/* case 2 */
2656		set_bit(STRIPE_R5C_CACHING, &sh->state);
2657	}
2658
2659	/*
2660	 * When run in degraded mode, array is set to write-through mode.
2661	 * This check helps drain pending write safely in the transition to
2662	 * write-through mode.
2663	 *
2664	 * When a stripe is syncing, the write is also handled in write
2665	 * through mode.
2666	 */
2667	if (s->failed || test_bit(STRIPE_SYNCING, &sh->state)) {
2668		r5c_make_stripe_write_out(sh);
2669		return -EAGAIN;
2670	}
2671
2672	for (i = disks; i--; ) {
2673		dev = &sh->dev[i];
2674		/* if non-overwrite, use writing-out phase */
2675		if (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags) &&
2676		    !test_bit(R5_InJournal, &dev->flags)) {
2677			r5c_make_stripe_write_out(sh);
2678			return -EAGAIN;
2679		}
2680	}
2681
2682	/* if the stripe is not counted in big_stripe_tree, add it now */
2683	if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
2684	    !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2685		tree_index = r5c_tree_index(conf, sh->sector);
2686		spin_lock(&log->tree_lock);
2687		pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
2688					       tree_index);
2689		if (pslot) {
2690			refcount = (uintptr_t)radix_tree_deref_slot_protected(
2691				pslot, &log->tree_lock) >>
2692				R5C_RADIX_COUNT_SHIFT;
2693			radix_tree_replace_slot(
2694				&log->big_stripe_tree, pslot,
2695				(void *)((refcount + 1) << R5C_RADIX_COUNT_SHIFT));
2696		} else {
2697			/*
2698			 * this radix_tree_insert can fail safely, so no
2699			 * need to call radix_tree_preload()
2700			 */
2701			ret = radix_tree_insert(
2702				&log->big_stripe_tree, tree_index,
2703				(void *)(1 << R5C_RADIX_COUNT_SHIFT));
2704			if (ret) {
2705				spin_unlock(&log->tree_lock);
2706				r5c_make_stripe_write_out(sh);
2707				return -EAGAIN;
2708			}
2709		}
2710		spin_unlock(&log->tree_lock);
2711
2712		/*
2713		 * set STRIPE_R5C_PARTIAL_STRIPE, this shows the stripe is
2714		 * counted in the radix tree
2715		 */
2716		set_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state);
2717		atomic_inc(&conf->r5c_cached_partial_stripes);
2718	}
2719
2720	for (i = disks; i--; ) {
2721		dev = &sh->dev[i];
2722		if (dev->towrite) {
2723			set_bit(R5_Wantwrite, &dev->flags);
2724			set_bit(R5_Wantdrain, &dev->flags);
2725			set_bit(R5_LOCKED, &dev->flags);
2726			to_cache++;
2727		}
2728	}
2729
2730	if (to_cache) {
2731		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2732		/*
2733		 * set STRIPE_LOG_TRAPPED, which triggers r5c_cache_data()
2734		 * in ops_run_io(). STRIPE_LOG_TRAPPED will be cleared in
2735		 * r5c_handle_data_cached()
2736		 */
2737		set_bit(STRIPE_LOG_TRAPPED, &sh->state);
2738	}
2739
2740	return 0;
2741}
2742
2743/*
2744 * free extra pages (orig_page) we allocated for prexor
2745 */
2746void r5c_release_extra_page(struct stripe_head *sh)
2747{
2748	struct r5conf *conf = sh->raid_conf;
2749	int i;
2750	bool using_disk_info_extra_page;
2751
2752	using_disk_info_extra_page =
2753		sh->dev[0].orig_page == conf->disks[0].extra_page;
2754
2755	for (i = sh->disks; i--; )
2756		if (sh->dev[i].page != sh->dev[i].orig_page) {
2757			struct page *p = sh->dev[i].orig_page;
2758
2759			sh->dev[i].orig_page = sh->dev[i].page;
2760			clear_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
2761
2762			if (!using_disk_info_extra_page)
2763				put_page(p);
2764		}
2765
2766	if (using_disk_info_extra_page) {
2767		clear_bit(R5C_EXTRA_PAGE_IN_USE, &conf->cache_state);
2768		md_wakeup_thread(conf->mddev->thread);
2769	}
2770}
2771
2772void r5c_use_extra_page(struct stripe_head *sh)
2773{
2774	struct r5conf *conf = sh->raid_conf;
2775	int i;
2776	struct r5dev *dev;
2777
2778	for (i = sh->disks; i--; ) {
2779		dev = &sh->dev[i];
2780		if (dev->orig_page != dev->page)
2781			put_page(dev->orig_page);
2782		dev->orig_page = conf->disks[i].extra_page;
2783	}
2784}
2785
2786/*
2787 * clean up the stripe (clear R5_InJournal for dev[pd_idx] etc.) after the
2788 * stripe is committed to RAID disks.
2789 */
2790void r5c_finish_stripe_write_out(struct r5conf *conf,
2791				 struct stripe_head *sh,
2792				 struct stripe_head_state *s)
2793{
2794	struct r5l_log *log = READ_ONCE(conf->log);
2795	int i;
2796	sector_t tree_index;
2797	void __rcu **pslot;
2798	uintptr_t refcount;
2799
2800	if (!log || !test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags))
2801		return;
2802
2803	WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
2804	clear_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
2805
2806	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
2807		return;
2808
2809	for (i = sh->disks; i--; ) {
2810		clear_bit(R5_InJournal, &sh->dev[i].flags);
2811		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2812			wake_up_bit(&sh->dev[i].flags, R5_Overlap);
2813	}
2814
2815	/*
2816	 * analyse_stripe() runs before r5c_finish_stripe_write_out(),
2817	 * We updated R5_InJournal, so we also update s->injournal.
2818	 */
2819	s->injournal = 0;
2820
2821	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2822		if (atomic_dec_and_test(&conf->pending_full_writes))
2823			md_wakeup_thread(conf->mddev->thread);
2824
2825	spin_lock_irq(&log->stripe_in_journal_lock);
2826	list_del_init(&sh->r5c);
2827	spin_unlock_irq(&log->stripe_in_journal_lock);
2828	sh->log_start = MaxSector;
2829
2830	atomic_dec(&log->stripe_in_journal_count);
2831	r5c_update_log_state(log);
2832
2833	/* stop counting this stripe in big_stripe_tree */
2834	if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) ||
2835	    test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2836		tree_index = r5c_tree_index(conf, sh->sector);
2837		spin_lock(&log->tree_lock);
2838		pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
2839					       tree_index);
2840		BUG_ON(pslot == NULL);
2841		refcount = (uintptr_t)radix_tree_deref_slot_protected(
2842			pslot, &log->tree_lock) >>
2843			R5C_RADIX_COUNT_SHIFT;
2844		if (refcount == 1)
2845			radix_tree_delete(&log->big_stripe_tree, tree_index);
2846		else
2847			radix_tree_replace_slot(
2848				&log->big_stripe_tree, pslot,
2849				(void *)((refcount - 1) << R5C_RADIX_COUNT_SHIFT));
2850		spin_unlock(&log->tree_lock);
2851	}
2852
2853	if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) {
2854		BUG_ON(atomic_read(&conf->r5c_cached_partial_stripes) == 0);
2855		atomic_dec(&conf->r5c_flushing_partial_stripes);
2856		atomic_dec(&conf->r5c_cached_partial_stripes);
2857	}
2858
2859	if (test_and_clear_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2860		BUG_ON(atomic_read(&conf->r5c_cached_full_stripes) == 0);
2861		atomic_dec(&conf->r5c_flushing_full_stripes);
2862		atomic_dec(&conf->r5c_cached_full_stripes);
2863	}
2864
2865	r5l_append_flush_payload(log, sh->sector);
2866	/* stripe is flused to raid disks, we can do resync now */
2867	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
2868		set_bit(STRIPE_HANDLE, &sh->state);
2869}
2870
2871int r5c_cache_data(struct r5l_log *log, struct stripe_head *sh)
2872{
2873	struct r5conf *conf = sh->raid_conf;
2874	int pages = 0;
2875	int reserve;
2876	int i;
2877	int ret = 0;
2878
2879	BUG_ON(!log);
2880
2881	for (i = 0; i < sh->disks; i++) {
2882		void *addr;
2883
2884		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
2885			continue;
2886		addr = kmap_local_page(sh->dev[i].page);
2887		sh->dev[i].log_checksum = crc32c(log->uuid_checksum,
2888						 addr, PAGE_SIZE);
2889		kunmap_local(addr);
2890		pages++;
2891	}
2892	WARN_ON(pages == 0);
2893
2894	/*
2895	 * The stripe must enter state machine again to call endio, so
2896	 * don't delay.
2897	 */
2898	clear_bit(STRIPE_DELAYED, &sh->state);
2899	atomic_inc(&sh->count);
2900
2901	mutex_lock(&log->io_mutex);
2902	/* meta + data */
2903	reserve = (1 + pages) << (PAGE_SHIFT - 9);
2904
2905	if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
2906	    sh->log_start == MaxSector)
2907		r5l_add_no_space_stripe(log, sh);
2908	else if (!r5l_has_free_space(log, reserve)) {
2909		if (sh->log_start == log->last_checkpoint)
2910			BUG();
2911		else
2912			r5l_add_no_space_stripe(log, sh);
2913	} else {
2914		ret = r5l_log_stripe(log, sh, pages, 0);
2915		if (ret) {
2916			spin_lock_irq(&log->io_list_lock);
2917			list_add_tail(&sh->log_list, &log->no_mem_stripes);
2918			spin_unlock_irq(&log->io_list_lock);
2919		}
2920	}
2921
2922	mutex_unlock(&log->io_mutex);
2923	return 0;
2924}
2925
2926/* check whether this big stripe is in write back cache. */
2927bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect)
2928{
2929	struct r5l_log *log = READ_ONCE(conf->log);
2930	sector_t tree_index;
2931	void *slot;
2932
2933	if (!log)
2934		return false;
2935
2936	tree_index = r5c_tree_index(conf, sect);
2937	slot = radix_tree_lookup(&log->big_stripe_tree, tree_index);
2938	return slot != NULL;
2939}
2940
2941static int r5l_load_log(struct r5l_log *log)
2942{
2943	struct md_rdev *rdev = log->rdev;
2944	struct page *page;
2945	struct r5l_meta_block *mb;
2946	sector_t cp = log->rdev->journal_tail;
2947	u32 stored_crc, expected_crc;
2948	bool create_super = false;
2949	int ret = 0;
2950
2951	/* Make sure it's valid */
2952	if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp)
2953		cp = 0;
2954	page = alloc_page(GFP_KERNEL);
2955	if (!page)
2956		return -ENOMEM;
2957
2958	if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, false)) {
2959		ret = -EIO;
2960		goto ioerr;
2961	}
2962	mb = page_address(page);
2963
2964	if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
2965	    mb->version != R5LOG_VERSION) {
2966		create_super = true;
2967		goto create;
2968	}
2969	stored_crc = le32_to_cpu(mb->checksum);
2970	mb->checksum = 0;
2971	expected_crc = crc32c(log->uuid_checksum, mb, PAGE_SIZE);
2972	if (stored_crc != expected_crc) {
2973		create_super = true;
2974		goto create;
2975	}
2976	if (le64_to_cpu(mb->position) != cp) {
2977		create_super = true;
2978		goto create;
2979	}
2980create:
2981	if (create_super) {
2982		log->last_cp_seq = get_random_u32();
2983		cp = 0;
2984		r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq);
2985		/*
2986		 * Make sure super points to correct address. Log might have
2987		 * data very soon. If super hasn't correct log tail address,
2988		 * recovery can't find the log
2989		 */
2990		r5l_write_super(log, cp);
2991	} else
2992		log->last_cp_seq = le64_to_cpu(mb->seq);
2993
2994	log->device_size = round_down(rdev->sectors, BLOCK_SECTORS);
2995	log->max_free_space = log->device_size >> RECLAIM_MAX_FREE_SPACE_SHIFT;
2996	if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
2997		log->max_free_space = RECLAIM_MAX_FREE_SPACE;
2998	log->last_checkpoint = cp;
2999
3000	__free_page(page);
3001
3002	if (create_super) {
3003		log->log_start = r5l_ring_add(log, cp, BLOCK_SECTORS);
3004		log->seq = log->last_cp_seq + 1;
3005		log->next_checkpoint = cp;
3006	} else
3007		ret = r5l_recovery_log(log);
3008
3009	r5c_update_log_state(log);
3010	return ret;
3011ioerr:
3012	__free_page(page);
3013	return ret;
3014}
3015
3016int r5l_start(struct r5l_log *log)
3017{
3018	int ret;
3019
3020	if (!log)
3021		return 0;
3022
3023	ret = r5l_load_log(log);
3024	if (ret) {
3025		struct mddev *mddev = log->rdev->mddev;
3026		struct r5conf *conf = mddev->private;
3027
3028		r5l_exit_log(conf);
3029	}
3030	return ret;
3031}
3032
3033void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev)
3034{
3035	struct r5conf *conf = mddev->private;
3036	struct r5l_log *log = READ_ONCE(conf->log);
3037
3038	if (!log)
3039		return;
3040
3041	if ((raid5_calc_degraded(conf) > 0 ||
3042	     test_bit(Journal, &rdev->flags)) &&
3043	    log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK)
3044		schedule_work(&log->disable_writeback_work);
3045}
3046
3047int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
3048{
3049	struct r5l_log *log;
3050	struct md_thread *thread;
3051	int ret;
3052
3053	pr_debug("md/raid:%s: using device %pg as journal\n",
3054		 mdname(conf->mddev), rdev->bdev);
3055
3056	if (PAGE_SIZE != 4096)
3057		return -EINVAL;
3058
3059	/*
3060	 * The PAGE_SIZE must be big enough to hold 1 r5l_meta_block and
3061	 * raid_disks r5l_payload_data_parity.
3062	 *
3063	 * Write journal and cache does not work for very big array
3064	 * (raid_disks > 203)
3065	 */
3066	if (sizeof(struct r5l_meta_block) +
3067	    ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) *
3068	     conf->raid_disks) > PAGE_SIZE) {
3069		pr_err("md/raid:%s: write journal/cache doesn't work for array with %d disks\n",
3070		       mdname(conf->mddev), conf->raid_disks);
3071		return -EINVAL;
3072	}
3073
3074	log = kzalloc(sizeof(*log), GFP_KERNEL);
3075	if (!log)
3076		return -ENOMEM;
3077	log->rdev = rdev;
3078	log->need_cache_flush = bdev_write_cache(rdev->bdev);
3079	log->uuid_checksum = crc32c(~0, rdev->mddev->uuid,
3080				    sizeof(rdev->mddev->uuid));
3081
3082	mutex_init(&log->io_mutex);
3083
3084	spin_lock_init(&log->io_list_lock);
3085	INIT_LIST_HEAD(&log->running_ios);
3086	INIT_LIST_HEAD(&log->io_end_ios);
3087	INIT_LIST_HEAD(&log->flushing_ios);
3088	INIT_LIST_HEAD(&log->finished_ios);
3089
3090	log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
3091	if (!log->io_kc)
3092		goto io_kc;
3093
3094	ret = mempool_init_slab_pool(&log->io_pool, R5L_POOL_SIZE, log->io_kc);
3095	if (ret)
3096		goto io_pool;
3097
3098	ret = bioset_init(&log->bs, R5L_POOL_SIZE, 0, BIOSET_NEED_BVECS);
3099	if (ret)
3100		goto io_bs;
3101
3102	ret = mempool_init_page_pool(&log->meta_pool, R5L_POOL_SIZE, 0);
3103	if (ret)
3104		goto out_mempool;
3105
3106	spin_lock_init(&log->tree_lock);
3107	INIT_RADIX_TREE(&log->big_stripe_tree, GFP_NOWAIT);
3108
3109	thread = md_register_thread(r5l_reclaim_thread, log->rdev->mddev,
3110				    "reclaim");
3111	if (!thread)
3112		goto reclaim_thread;
3113
3114	thread->timeout = R5C_RECLAIM_WAKEUP_INTERVAL;
3115	rcu_assign_pointer(log->reclaim_thread, thread);
3116
3117	init_waitqueue_head(&log->iounit_wait);
3118
3119	INIT_LIST_HEAD(&log->no_mem_stripes);
3120
3121	INIT_LIST_HEAD(&log->no_space_stripes);
3122	spin_lock_init(&log->no_space_stripes_lock);
3123
3124	INIT_WORK(&log->deferred_io_work, r5l_submit_io_async);
3125	INIT_WORK(&log->disable_writeback_work, r5c_disable_writeback_async);
3126
3127	log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
3128	INIT_LIST_HEAD(&log->stripe_in_journal_list);
3129	spin_lock_init(&log->stripe_in_journal_lock);
3130	atomic_set(&log->stripe_in_journal_count, 0);
3131
3132	WRITE_ONCE(conf->log, log);
3133
3134	set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
3135	return 0;
3136
3137reclaim_thread:
3138	mempool_exit(&log->meta_pool);
3139out_mempool:
3140	bioset_exit(&log->bs);
3141io_bs:
3142	mempool_exit(&log->io_pool);
3143io_pool:
3144	kmem_cache_destroy(log->io_kc);
3145io_kc:
3146	kfree(log);
3147	return -EINVAL;
3148}
3149
3150void r5l_exit_log(struct r5conf *conf)
3151{
3152	struct r5l_log *log = conf->log;
3153
3154	md_unregister_thread(conf->mddev, &log->reclaim_thread);
3155
3156	/*
3157	 * 'reconfig_mutex' is held by caller, set 'confg->log' to NULL to
3158	 * ensure disable_writeback_work wakes up and exits.
3159	 */
3160	WRITE_ONCE(conf->log, NULL);
3161	wake_up(&conf->mddev->sb_wait);
3162	flush_work(&log->disable_writeback_work);
3163
3164	mempool_exit(&log->meta_pool);
3165	bioset_exit(&log->bs);
3166	mempool_exit(&log->io_pool);
3167	kmem_cache_destroy(log->io_kc);
3168	kfree(log);
3169}