buffered-io.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2010 Red Hat, Inc.
   4 * Copyright (C) 2016-2023 Christoph Hellwig.
   5 */
   6#include <linux/iomap.h>
   7#include <linux/buffer_head.h>
   8#include <linux/writeback.h>
   9#include <linux/swap.h>
  10#include <linux/migrate.h>
  11#include "internal.h"
  12#include "trace.h"
  13
  14#include "../internal.h"
  15
  16/*
  17 * Structure allocated for each folio to track per-block uptodate, dirty state
  18 * and I/O completions.
  19 */
  20struct iomap_folio_state {
  21	spinlock_t		state_lock;
  22	unsigned int		read_bytes_pending;
  23	atomic_t		write_bytes_pending;
  24
  25	/*
  26	 * Each block has two bits in this bitmap:
  27	 * Bits [0..blocks_per_folio) has the uptodate status.
  28	 * Bits [b_p_f...(2*b_p_f))   has the dirty status.
  29	 */
  30	unsigned long		state[];
  31};
  32
  33static inline bool ifs_is_fully_uptodate(struct folio *folio,
  34		struct iomap_folio_state *ifs)
  35{
  36	struct inode *inode = folio->mapping->host;
  37
  38	return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio));
  39}
  40
  41/*
  42 * Find the next uptodate block in the folio. end_blk is inclusive.
  43 * If no uptodate block is found, this will return end_blk + 1.
  44 */
  45static unsigned ifs_next_uptodate_block(struct folio *folio,
  46		unsigned start_blk, unsigned end_blk)
  47{
  48	struct iomap_folio_state *ifs = folio->private;
  49
  50	return find_next_bit(ifs->state, end_blk + 1, start_blk);
  51}
  52
  53/*
  54 * Find the next non-uptodate block in the folio. end_blk is inclusive.
  55 * If no non-uptodate block is found, this will return end_blk + 1.
  56 */
  57static unsigned ifs_next_nonuptodate_block(struct folio *folio,
  58		unsigned start_blk, unsigned end_blk)
  59{
  60	struct iomap_folio_state *ifs = folio->private;
  61
  62	return find_next_zero_bit(ifs->state, end_blk + 1, start_blk);
  63}
  64
  65static bool ifs_set_range_uptodate(struct folio *folio,
  66		struct iomap_folio_state *ifs, size_t off, size_t len)
  67{
  68	struct inode *inode = folio->mapping->host;
  69	unsigned int first_blk = off >> inode->i_blkbits;
  70	unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
  71	unsigned int nr_blks = last_blk - first_blk + 1;
  72
  73	bitmap_set(ifs->state, first_blk, nr_blks);
  74	return ifs_is_fully_uptodate(folio, ifs);
  75}
  76
  77static void iomap_set_range_uptodate(struct folio *folio, size_t off,
  78		size_t len)
  79{
  80	struct iomap_folio_state *ifs = folio->private;
  81	unsigned long flags;
  82	bool mark_uptodate = true;
  83
  84	if (folio_test_uptodate(folio))
  85		return;
  86
  87	if (ifs) {
  88		spin_lock_irqsave(&ifs->state_lock, flags);
  89		/*
  90		 * If a read with bytes pending is in progress, we must not call
  91		 * folio_mark_uptodate(). The read completion path
  92		 * (iomap_read_end()) will call folio_end_read(), which uses XOR
  93		 * semantics to set the uptodate bit. If we set it here, the XOR
  94		 * in folio_end_read() will clear it, leaving the folio not
  95		 * uptodate.
  96		 */
  97		mark_uptodate = ifs_set_range_uptodate(folio, ifs, off, len) &&
  98				!ifs->read_bytes_pending;
  99		spin_unlock_irqrestore(&ifs->state_lock, flags);
 100	}
 101
 102	if (mark_uptodate)
 103		folio_mark_uptodate(folio);
 104}
 105
 106/*
 107 * Find the next dirty block in the folio. end_blk is inclusive.
 108 * If no dirty block is found, this will return end_blk + 1.
 109 */
 110static unsigned ifs_next_dirty_block(struct folio *folio,
 111		unsigned start_blk, unsigned end_blk)
 112{
 113	struct iomap_folio_state *ifs = folio->private;
 114	struct inode *inode = folio->mapping->host;
 115	unsigned int blks = i_blocks_per_folio(inode, folio);
 116
 117	return find_next_bit(ifs->state, blks + end_blk + 1,
 118			blks + start_blk) - blks;
 119}
 120
 121/*
 122 * Find the next clean block in the folio. end_blk is inclusive.
 123 * If no clean block is found, this will return end_blk + 1.
 124 */
 125static unsigned ifs_next_clean_block(struct folio *folio,
 126		unsigned start_blk, unsigned end_blk)
 127{
 128	struct iomap_folio_state *ifs = folio->private;
 129	struct inode *inode = folio->mapping->host;
 130	unsigned int blks = i_blocks_per_folio(inode, folio);
 131
 132	return find_next_zero_bit(ifs->state, blks + end_blk + 1,
 133			blks + start_blk) - blks;
 134}
 135
 136static unsigned ifs_find_dirty_range(struct folio *folio,
 137		struct iomap_folio_state *ifs, u64 *range_start, u64 range_end)
 138{
 139	struct inode *inode = folio->mapping->host;
 140	unsigned start_blk =
 141		offset_in_folio(folio, *range_start) >> inode->i_blkbits;
 142	unsigned end_blk = min_not_zero(
 143		offset_in_folio(folio, range_end) >> inode->i_blkbits,
 144		i_blocks_per_folio(inode, folio)) - 1;
 145	unsigned nblks;
 146
 147	start_blk = ifs_next_dirty_block(folio, start_blk, end_blk);
 148	if (start_blk > end_blk)
 149		return 0;
 150	if (start_blk == end_blk)
 151		nblks = 1;
 152	else
 153		nblks = ifs_next_clean_block(folio, start_blk + 1, end_blk) -
 154				start_blk;
 155
 156	*range_start = folio_pos(folio) + (start_blk << inode->i_blkbits);
 157	return nblks << inode->i_blkbits;
 158}
 159
 160static unsigned iomap_find_dirty_range(struct folio *folio, u64 *range_start,
 161		u64 range_end)
 162{
 163	struct iomap_folio_state *ifs = folio->private;
 164
 165	if (*range_start >= range_end)
 166		return 0;
 167
 168	if (ifs)
 169		return ifs_find_dirty_range(folio, ifs, range_start, range_end);
 170	return range_end - *range_start;
 171}
 172
 173static void ifs_clear_range_dirty(struct folio *folio,
 174		struct iomap_folio_state *ifs, size_t off, size_t len)
 175{
 176	struct inode *inode = folio->mapping->host;
 177	unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
 178	unsigned int first_blk = (off >> inode->i_blkbits);
 179	unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
 180	unsigned int nr_blks = last_blk - first_blk + 1;
 181	unsigned long flags;
 182
 183	spin_lock_irqsave(&ifs->state_lock, flags);
 184	bitmap_clear(ifs->state, first_blk + blks_per_folio, nr_blks);
 185	spin_unlock_irqrestore(&ifs->state_lock, flags);
 186}
 187
 188static void iomap_clear_range_dirty(struct folio *folio, size_t off, size_t len)
 189{
 190	struct iomap_folio_state *ifs = folio->private;
 191
 192	if (ifs)
 193		ifs_clear_range_dirty(folio, ifs, off, len);
 194}
 195
 196static void ifs_set_range_dirty(struct folio *folio,
 197		struct iomap_folio_state *ifs, size_t off, size_t len)
 198{
 199	struct inode *inode = folio->mapping->host;
 200	unsigned int blks_per_folio = i_blocks_per_folio(inode, folio);
 201	unsigned int first_blk = (off >> inode->i_blkbits);
 202	unsigned int last_blk = (off + len - 1) >> inode->i_blkbits;
 203	unsigned int nr_blks = last_blk - first_blk + 1;
 204	unsigned long flags;
 205
 206	spin_lock_irqsave(&ifs->state_lock, flags);
 207	bitmap_set(ifs->state, first_blk + blks_per_folio, nr_blks);
 208	spin_unlock_irqrestore(&ifs->state_lock, flags);
 209}
 210
 211static void iomap_set_range_dirty(struct folio *folio, size_t off, size_t len)
 212{
 213	struct iomap_folio_state *ifs = folio->private;
 214
 215	if (ifs)
 216		ifs_set_range_dirty(folio, ifs, off, len);
 217}
 218
 219static struct iomap_folio_state *ifs_alloc(struct inode *inode,
 220		struct folio *folio, unsigned int flags)
 221{
 222	struct iomap_folio_state *ifs = folio->private;
 223	unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
 224	gfp_t gfp;
 225
 226	if (ifs || nr_blocks <= 1)
 227		return ifs;
 228
 229	if (flags & IOMAP_NOWAIT)
 230		gfp = GFP_NOWAIT;
 231	else
 232		gfp = GFP_NOFS | __GFP_NOFAIL;
 233
 234	/*
 235	 * ifs->state tracks two sets of state flags when the
 236	 * filesystem block size is smaller than the folio size.
 237	 * The first state tracks per-block uptodate and the
 238	 * second tracks per-block dirty state.
 239	 */
 240	ifs = kzalloc(struct_size(ifs, state,
 241		      BITS_TO_LONGS(2 * nr_blocks)), gfp);
 242	if (!ifs)
 243		return ifs;
 244
 245	spin_lock_init(&ifs->state_lock);
 246	if (folio_test_uptodate(folio))
 247		bitmap_set(ifs->state, 0, nr_blocks);
 248	if (folio_test_dirty(folio))
 249		bitmap_set(ifs->state, nr_blocks, nr_blocks);
 250	folio_attach_private(folio, ifs);
 251
 252	return ifs;
 253}
 254
 255static void ifs_free(struct folio *folio)
 256{
 257	struct iomap_folio_state *ifs = folio_detach_private(folio);
 258
 259	if (!ifs)
 260		return;
 261	WARN_ON_ONCE(ifs->read_bytes_pending != 0);
 262	WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending));
 263	WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) !=
 264			folio_test_uptodate(folio));
 265	kfree(ifs);
 266}
 267
 268/*
 269 * Calculate how many bytes to truncate based off the number of blocks to
 270 * truncate and the end position to start truncating from.
 271 */
 272static size_t iomap_bytes_to_truncate(loff_t end_pos, unsigned block_bits,
 273		unsigned blocks_truncated)
 274{
 275	unsigned block_size = 1 << block_bits;
 276	unsigned block_offset = end_pos & (block_size - 1);
 277
 278	if (!block_offset)
 279		return blocks_truncated << block_bits;
 280
 281	return ((blocks_truncated - 1) << block_bits) + block_offset;
 282}
 283
 284/*
 285 * Calculate the range inside the folio that we actually need to read.
 286 */
 287static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,
 288		loff_t *pos, loff_t length, size_t *offp, size_t *lenp)
 289{
 290	struct iomap_folio_state *ifs = folio->private;
 291	loff_t orig_pos = *pos;
 292	loff_t isize = i_size_read(inode);
 293	unsigned block_bits = inode->i_blkbits;
 294	unsigned block_size = (1 << block_bits);
 295	size_t poff = offset_in_folio(folio, *pos);
 296	size_t plen = min_t(loff_t, folio_size(folio) - poff, length);
 297	size_t orig_plen = plen;
 298	unsigned first = poff >> block_bits;
 299	unsigned last = (poff + plen - 1) >> block_bits;
 300
 301	/*
 302	 * If the block size is smaller than the page size, we need to check the
 303	 * per-block uptodate status and adjust the offset and length if needed
 304	 * to avoid reading in already uptodate ranges.
 305	 */
 306	if (ifs) {
 307		unsigned int next, blocks_skipped;
 308
 309		next = ifs_next_nonuptodate_block(folio, first, last);
 310		blocks_skipped = next - first;
 311
 312		if (blocks_skipped) {
 313			unsigned long block_offset = *pos & (block_size - 1);
 314			unsigned bytes_skipped =
 315				(blocks_skipped << block_bits) - block_offset;
 316
 317			*pos += bytes_skipped;
 318			poff += bytes_skipped;
 319			plen -= bytes_skipped;
 320		}
 321		first = next;
 322
 323		/* truncate len if we find any trailing uptodate block(s) */
 324		if (++next <= last) {
 325			next = ifs_next_uptodate_block(folio, next, last);
 326			if (next <= last) {
 327				plen -= iomap_bytes_to_truncate(*pos + plen,
 328						block_bits, last - next + 1);
 329				last = next - 1;
 330			}
 331		}
 332	}
 333
 334	/*
 335	 * If the extent spans the block that contains the i_size, we need to
 336	 * handle both halves separately so that we properly zero data in the
 337	 * page cache for blocks that are entirely outside of i_size.
 338	 */
 339	if (orig_pos <= isize && orig_pos + orig_plen > isize) {
 340		unsigned end = offset_in_folio(folio, isize - 1) >> block_bits;
 341
 342		if (first <= end && last > end)
 343			plen -= iomap_bytes_to_truncate(*pos + plen, block_bits,
 344					last - end);
 345	}
 346
 347	*offp = poff;
 348	*lenp = plen;
 349}
 350
 351static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter,
 352		loff_t pos)
 353{
 354	const struct iomap *srcmap = iomap_iter_srcmap(iter);
 355
 356	return srcmap->type != IOMAP_MAPPED ||
 357		(srcmap->flags & IOMAP_F_NEW) ||
 358		pos >= i_size_read(iter->inode);
 359}
 360
 361/**
 362 * iomap_read_inline_data - copy inline data into the page cache
 363 * @iter: iteration structure
 364 * @folio: folio to copy to
 365 *
 366 * Copy the inline data in @iter into @folio and zero out the rest of the folio.
 367 * Only a single IOMAP_INLINE extent is allowed at the end of each file.
 368 * Returns zero for success to complete the read, or the usual negative errno.
 369 */
 370static int iomap_read_inline_data(const struct iomap_iter *iter,
 371		struct folio *folio)
 372{
 373	const struct iomap *iomap = iomap_iter_srcmap(iter);
 374	size_t size = i_size_read(iter->inode) - iomap->offset;
 375	size_t offset = offset_in_folio(folio, iomap->offset);
 376
 377	if (WARN_ON_ONCE(!iomap->inline_data))
 378		return -EIO;
 379
 380	if (folio_test_uptodate(folio))
 381		return 0;
 382
 383	if (WARN_ON_ONCE(size > iomap->length))
 384		return -EIO;
 385	if (offset > 0)
 386		ifs_alloc(iter->inode, folio, iter->flags);
 387
 388	folio_fill_tail(folio, offset, iomap->inline_data, size);
 389	iomap_set_range_uptodate(folio, offset, folio_size(folio) - offset);
 390	return 0;
 391}
 392
 393void iomap_finish_folio_read(struct folio *folio, size_t off, size_t len,
 394		int error)
 395{
 396	struct iomap_folio_state *ifs = folio->private;
 397	bool uptodate = !error;
 398	bool finished = true;
 399
 400	if (ifs) {
 401		unsigned long flags;
 402
 403		spin_lock_irqsave(&ifs->state_lock, flags);
 404		if (!error)
 405			uptodate = ifs_set_range_uptodate(folio, ifs, off, len);
 406		ifs->read_bytes_pending -= len;
 407		finished = !ifs->read_bytes_pending;
 408		spin_unlock_irqrestore(&ifs->state_lock, flags);
 409	}
 410
 411	if (finished)
 412		folio_end_read(folio, uptodate);
 413}
 414EXPORT_SYMBOL_GPL(iomap_finish_folio_read);
 415
 416static void iomap_read_init(struct folio *folio)
 417{
 418	struct iomap_folio_state *ifs = folio->private;
 419
 420	if (ifs) {
 421		/*
 422		 * ifs->read_bytes_pending is used to track how many bytes are
 423		 * read in asynchronously by the IO helper. We need to track
 424		 * this so that we can know when the IO helper has finished
 425		 * reading in all the necessary ranges of the folio and can end
 426		 * the read.
 427		 *
 428		 * Increase ->read_bytes_pending by the folio size to start.
 429		 * We'll subtract any uptodate / zeroed ranges that did not
 430		 * require IO in iomap_read_end() after we're done processing
 431		 * the folio.
 432		 *
 433		 * We do this because otherwise, we would have to increment
 434		 * ifs->read_bytes_pending every time a range in the folio needs
 435		 * to be read in, which can get expensive since the spinlock
 436		 * needs to be held whenever modifying ifs->read_bytes_pending.
 437		 */
 438		spin_lock_irq(&ifs->state_lock);
 439		WARN_ON_ONCE(ifs->read_bytes_pending != 0);
 440		ifs->read_bytes_pending = folio_size(folio);
 441		spin_unlock_irq(&ifs->state_lock);
 442	}
 443}
 444
 445/*
 446 * This ends IO if no bytes were submitted to an IO helper.
 447 *
 448 * Otherwise, this calibrates ifs->read_bytes_pending to represent only the
 449 * submitted bytes (see comment in iomap_read_init()). If all bytes submitted
 450 * have already been completed by the IO helper, then this will end the read.
 451 * Else the IO helper will end the read after all submitted ranges have been
 452 * read.
 453 */
 454static void iomap_read_end(struct folio *folio, size_t bytes_submitted)
 455{
 456	struct iomap_folio_state *ifs = folio->private;
 457
 458	if (ifs) {
 459		bool end_read, uptodate;
 460
 461		spin_lock_irq(&ifs->state_lock);
 462		if (!ifs->read_bytes_pending) {
 463			WARN_ON_ONCE(bytes_submitted);
 464			spin_unlock_irq(&ifs->state_lock);
 465			folio_unlock(folio);
 466			return;
 467		}
 468
 469		/*
 470		 * Subtract any bytes that were initially accounted to
 471		 * read_bytes_pending but skipped for IO.
 472		 */
 473		ifs->read_bytes_pending -= folio_size(folio) - bytes_submitted;
 474
 475		/*
 476		 * If !ifs->read_bytes_pending, this means all pending reads by
 477		 * the IO helper have already completed, which means we need to
 478		 * end the folio read here. If ifs->read_bytes_pending != 0,
 479		 * the IO helper will end the folio read.
 480		 */
 481		end_read = !ifs->read_bytes_pending;
 482		if (end_read)
 483			uptodate = ifs_is_fully_uptodate(folio, ifs);
 484		spin_unlock_irq(&ifs->state_lock);
 485		if (end_read)
 486			folio_end_read(folio, uptodate);
 487	} else {
 488		/*
 489		 * If a folio without an ifs is submitted to the IO helper, the
 490		 * read must be on the entire folio and the IO helper takes
 491		 * ownership of the folio. This means we should only enter
 492		 * iomap_read_end() for the !ifs case if no bytes were submitted
 493		 * to the IO helper, in which case we are responsible for
 494		 * unlocking the folio here.
 495		 */
 496		WARN_ON_ONCE(bytes_submitted);
 497		folio_unlock(folio);
 498	}
 499}
 500
 501static int iomap_read_folio_iter(struct iomap_iter *iter,
 502		struct iomap_read_folio_ctx *ctx, size_t *bytes_submitted)
 503{
 504	const struct iomap *iomap = &iter->iomap;
 505	loff_t pos = iter->pos;
 506	loff_t length = iomap_length(iter);
 507	struct folio *folio = ctx->cur_folio;
 508	size_t folio_len = folio_size(folio);
 509	struct iomap_folio_state *ifs;
 510	size_t poff, plen;
 511	loff_t pos_diff;
 512	int ret;
 513
 514	if (iomap->type == IOMAP_INLINE) {
 515		ret = iomap_read_inline_data(iter, folio);
 516		if (ret)
 517			return ret;
 518		return iomap_iter_advance(iter, length);
 519	}
 520
 521	ifs = ifs_alloc(iter->inode, folio, iter->flags);
 522
 523	length = min_t(loff_t, length, folio_len - offset_in_folio(folio, pos));
 524	while (length) {
 525		iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff,
 526				&plen);
 527
 528		pos_diff = pos - iter->pos;
 529		if (WARN_ON_ONCE(pos_diff + plen > length))
 530			return -EIO;
 531
 532		ret = iomap_iter_advance(iter, pos_diff);
 533		if (ret)
 534			return ret;
 535
 536		if (plen == 0)
 537			return 0;
 538
 539		/* zero post-eof blocks as the page may be mapped */
 540		if (iomap_block_needs_zeroing(iter, pos)) {
 541			folio_zero_range(folio, poff, plen);
 542			iomap_set_range_uptodate(folio, poff, plen);
 543		} else {
 544			if (!*bytes_submitted)
 545				iomap_read_init(folio);
 546			ret = ctx->ops->read_folio_range(iter, ctx, plen);
 547			if (ret)
 548				return ret;
 549
 550			*bytes_submitted += plen;
 551			/*
 552			 * Hand off folio ownership to the IO helper when:
 553			 * 1) The entire folio has been submitted for IO, or
 554			 * 2) There is no ifs attached to the folio
 555			 *
 556			 * Case (2) occurs when 1 << i_blkbits matches the folio
 557			 * size but the underlying filesystem or block device
 558			 * uses a smaller granularity for IO.
 559			 */
 560			if (*bytes_submitted == folio_len || !ifs)
 561				ctx->cur_folio = NULL;
 562		}
 563
 564		ret = iomap_iter_advance(iter, plen);
 565		if (ret)
 566			return ret;
 567		length -= pos_diff + plen;
 568		pos = iter->pos;
 569	}
 570	return 0;
 571}
 572
 573void iomap_read_folio(const struct iomap_ops *ops,
 574		struct iomap_read_folio_ctx *ctx)
 575{
 576	struct folio *folio = ctx->cur_folio;
 577	struct iomap_iter iter = {
 578		.inode		= folio->mapping->host,
 579		.pos		= folio_pos(folio),
 580		.len		= folio_size(folio),
 581	};
 582	size_t bytes_submitted = 0;
 583	int ret;
 584
 585	trace_iomap_readpage(iter.inode, 1);
 586
 587	while ((ret = iomap_iter(&iter, ops)) > 0)
 588		iter.status = iomap_read_folio_iter(&iter, ctx,
 589				&bytes_submitted);
 590
 591	if (ctx->ops->submit_read)
 592		ctx->ops->submit_read(ctx);
 593
 594	if (ctx->cur_folio)
 595		iomap_read_end(ctx->cur_folio, bytes_submitted);
 596}
 597EXPORT_SYMBOL_GPL(iomap_read_folio);
 598
 599static int iomap_readahead_iter(struct iomap_iter *iter,
 600		struct iomap_read_folio_ctx *ctx, size_t *cur_bytes_submitted)
 601{
 602	int ret;
 603
 604	while (iomap_length(iter)) {
 605		if (ctx->cur_folio &&
 606		    offset_in_folio(ctx->cur_folio, iter->pos) == 0) {
 607			iomap_read_end(ctx->cur_folio, *cur_bytes_submitted);
 608			ctx->cur_folio = NULL;
 609		}
 610		if (!ctx->cur_folio) {
 611			ctx->cur_folio = readahead_folio(ctx->rac);
 612			if (WARN_ON_ONCE(!ctx->cur_folio))
 613				return -EINVAL;
 614			*cur_bytes_submitted = 0;
 615		}
 616		ret = iomap_read_folio_iter(iter, ctx, cur_bytes_submitted);
 617		if (ret)
 618			return ret;
 619	}
 620
 621	return 0;
 622}
 623
 624/**
 625 * iomap_readahead - Attempt to read pages from a file.
 626 * @ops: The operations vector for the filesystem.
 627 * @ctx: The ctx used for issuing readahead.
 628 *
 629 * This function is for filesystems to call to implement their readahead
 630 * address_space operation.
 631 *
 632 * Context: The @ops callbacks may submit I/O (eg to read the addresses of
 633 * blocks from disc), and may wait for it.  The caller may be trying to
 634 * access a different page, and so sleeping excessively should be avoided.
 635 * It may allocate memory, but should avoid costly allocations.  This
 636 * function is called with memalloc_nofs set, so allocations will not cause
 637 * the filesystem to be reentered.
 638 */
 639void iomap_readahead(const struct iomap_ops *ops,
 640		struct iomap_read_folio_ctx *ctx)
 641{
 642	struct readahead_control *rac = ctx->rac;
 643	struct iomap_iter iter = {
 644		.inode	= rac->mapping->host,
 645		.pos	= readahead_pos(rac),
 646		.len	= readahead_length(rac),
 647	};
 648	size_t cur_bytes_submitted;
 649
 650	trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
 651
 652	while (iomap_iter(&iter, ops) > 0)
 653		iter.status = iomap_readahead_iter(&iter, ctx,
 654					&cur_bytes_submitted);
 655
 656	if (ctx->ops->submit_read)
 657		ctx->ops->submit_read(ctx);
 658
 659	if (ctx->cur_folio)
 660		iomap_read_end(ctx->cur_folio, cur_bytes_submitted);
 661}
 662EXPORT_SYMBOL_GPL(iomap_readahead);
 663
 664/*
 665 * iomap_is_partially_uptodate checks whether blocks within a folio are
 666 * uptodate or not.
 667 *
 668 * Returns true if all blocks which correspond to the specified part
 669 * of the folio are uptodate.
 670 */
 671bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
 672{
 673	struct iomap_folio_state *ifs = folio->private;
 674	struct inode *inode = folio->mapping->host;
 675	unsigned first, last;
 676
 677	if (!ifs)
 678		return false;
 679
 680	/* Caller's range may extend past the end of this folio */
 681	count = min(folio_size(folio) - from, count);
 682
 683	/* First and last blocks in range within folio */
 684	first = from >> inode->i_blkbits;
 685	last = (from + count - 1) >> inode->i_blkbits;
 686
 687	return ifs_next_nonuptodate_block(folio, first, last) > last;
 688}
 689EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
 690
 691/**
 692 * iomap_get_folio - get a folio reference for writing
 693 * @iter: iteration structure
 694 * @pos: start offset of write
 695 * @len: Suggested size of folio to create.
 696 *
 697 * Returns a locked reference to the folio at @pos, or an error pointer if the
 698 * folio could not be obtained.
 699 */
 700struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)
 701{
 702	fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS;
 703
 704	if (iter->flags & IOMAP_NOWAIT)
 705		fgp |= FGP_NOWAIT;
 706	if (iter->flags & IOMAP_DONTCACHE)
 707		fgp |= FGP_DONTCACHE;
 708	fgp |= fgf_set_order(len);
 709
 710	return __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT,
 711			fgp, mapping_gfp_mask(iter->inode->i_mapping));
 712}
 713EXPORT_SYMBOL_GPL(iomap_get_folio);
 714
 715bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags)
 716{
 717	trace_iomap_release_folio(folio->mapping->host, folio_pos(folio),
 718			folio_size(folio));
 719
 720	/*
 721	 * If the folio is dirty, we refuse to release our metadata because
 722	 * it may be partially dirty.  Once we track per-block dirty state,
 723	 * we can release the metadata if every block is dirty.
 724	 */
 725	if (folio_test_dirty(folio))
 726		return false;
 727	ifs_free(folio);
 728	return true;
 729}
 730EXPORT_SYMBOL_GPL(iomap_release_folio);
 731
 732void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len)
 733{
 734	trace_iomap_invalidate_folio(folio->mapping->host,
 735					folio_pos(folio) + offset, len);
 736
 737	/*
 738	 * If we're invalidating the entire folio, clear the dirty state
 739	 * from it and release it to avoid unnecessary buildup of the LRU.
 740	 */
 741	if (offset == 0 && len == folio_size(folio)) {
 742		WARN_ON_ONCE(folio_test_writeback(folio));
 743		folio_cancel_dirty(folio);
 744		ifs_free(folio);
 745	}
 746}
 747EXPORT_SYMBOL_GPL(iomap_invalidate_folio);
 748
 749bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio)
 750{
 751	struct inode *inode = mapping->host;
 752	size_t len = folio_size(folio);
 753
 754	ifs_alloc(inode, folio, 0);
 755	iomap_set_range_dirty(folio, 0, len);
 756	return filemap_dirty_folio(mapping, folio);
 757}
 758EXPORT_SYMBOL_GPL(iomap_dirty_folio);
 759
 760static void
 761iomap_write_failed(struct inode *inode, loff_t pos, unsigned len)
 762{
 763	loff_t i_size = i_size_read(inode);
 764
 765	/*
 766	 * Only truncate newly allocated pages beyoned EOF, even if the
 767	 * write started inside the existing inode size.
 768	 */
 769	if (pos + len > i_size)
 770		truncate_pagecache_range(inode, max(pos, i_size),
 771					 pos + len - 1);
 772}
 773
 774static int __iomap_write_begin(const struct iomap_iter *iter,
 775		const struct iomap_write_ops *write_ops, size_t len,
 776		struct folio *folio)
 777{
 778	struct iomap_folio_state *ifs;
 779	loff_t pos = iter->pos;
 780	loff_t block_size = i_blocksize(iter->inode);
 781	loff_t block_start = round_down(pos, block_size);
 782	loff_t block_end = round_up(pos + len, block_size);
 783	unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio);
 784	size_t from = offset_in_folio(folio, pos), to = from + len;
 785	size_t poff, plen;
 786
 787	/*
 788	 * If the write or zeroing completely overlaps the current folio, then
 789	 * entire folio will be dirtied so there is no need for
 790	 * per-block state tracking structures to be attached to this folio.
 791	 * For the unshare case, we must read in the ondisk contents because we
 792	 * are not changing pagecache contents.
 793	 */
 794	if (!(iter->flags & IOMAP_UNSHARE) && pos <= folio_pos(folio) &&
 795	    pos + len >= folio_next_pos(folio))
 796		return 0;
 797
 798	ifs = ifs_alloc(iter->inode, folio, iter->flags);
 799	if ((iter->flags & IOMAP_NOWAIT) && !ifs && nr_blocks > 1)
 800		return -EAGAIN;
 801
 802	if (folio_test_uptodate(folio))
 803		return 0;
 804
 805	do {
 806		iomap_adjust_read_range(iter->inode, folio, &block_start,
 807				block_end - block_start, &poff, &plen);
 808		if (plen == 0)
 809			break;
 810
 811		/*
 812		 * If the read range will be entirely overwritten by the write,
 813		 * we can skip having to zero/read it in.
 814		 */
 815		if (!(iter->flags & IOMAP_UNSHARE) && from <= poff &&
 816		    to >= poff + plen)
 817			continue;
 818
 819		if (iomap_block_needs_zeroing(iter, block_start)) {
 820			if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE))
 821				return -EIO;
 822			folio_zero_segments(folio, poff, from, to, poff + plen);
 823		} else {
 824			int status;
 825
 826			if (iter->flags & IOMAP_NOWAIT)
 827				return -EAGAIN;
 828
 829			if (write_ops && write_ops->read_folio_range)
 830				status = write_ops->read_folio_range(iter,
 831						folio, block_start, plen);
 832			else
 833				status = iomap_bio_read_folio_range_sync(iter,
 834						folio, block_start, plen);
 835			if (status)
 836				return status;
 837		}
 838		iomap_set_range_uptodate(folio, poff, plen);
 839	} while ((block_start += plen) < block_end);
 840
 841	return 0;
 842}
 843
 844static struct folio *__iomap_get_folio(struct iomap_iter *iter,
 845		const struct iomap_write_ops *write_ops, size_t len)
 846{
 847	loff_t pos = iter->pos;
 848
 849	if (!mapping_large_folio_support(iter->inode->i_mapping))
 850		len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos));
 851
 852	if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
 853		struct folio *folio = folio_batch_next(iter->fbatch);
 854
 855		if (!folio)
 856			return NULL;
 857
 858		/*
 859		 * The folio mapping generally shouldn't have changed based on
 860		 * fs locks, but be consistent with filemap lookup and retry
 861		 * the iter if it does.
 862		 */
 863		folio_lock(folio);
 864		if (unlikely(folio->mapping != iter->inode->i_mapping)) {
 865			iter->iomap.flags |= IOMAP_F_STALE;
 866			folio_unlock(folio);
 867			return NULL;
 868		}
 869
 870		folio_get(folio);
 871		folio_wait_stable(folio);
 872		return folio;
 873	}
 874
 875	if (write_ops && write_ops->get_folio)
 876		return write_ops->get_folio(iter, pos, len);
 877	return iomap_get_folio(iter, pos, len);
 878}
 879
 880static void __iomap_put_folio(struct iomap_iter *iter,
 881		const struct iomap_write_ops *write_ops, size_t ret,
 882		struct folio *folio)
 883{
 884	loff_t pos = iter->pos;
 885
 886	if (write_ops && write_ops->put_folio) {
 887		write_ops->put_folio(iter->inode, pos, ret, folio);
 888	} else {
 889		folio_unlock(folio);
 890		folio_put(folio);
 891	}
 892}
 893
 894/* trim pos and bytes to within a given folio */
 895static loff_t iomap_trim_folio_range(struct iomap_iter *iter,
 896		struct folio *folio, size_t *offset, u64 *bytes)
 897{
 898	loff_t pos = iter->pos;
 899	size_t fsize = folio_size(folio);
 900
 901	WARN_ON_ONCE(pos < folio_pos(folio));
 902	WARN_ON_ONCE(pos >= folio_pos(folio) + fsize);
 903
 904	*offset = offset_in_folio(folio, pos);
 905	*bytes = min(*bytes, fsize - *offset);
 906
 907	return pos;
 908}
 909
 910static int iomap_write_begin_inline(const struct iomap_iter *iter,
 911		struct folio *folio)
 912{
 913	/* needs more work for the tailpacking case; disable for now */
 914	if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0))
 915		return -EIO;
 916	return iomap_read_inline_data(iter, folio);
 917}
 918
 919/*
 920 * Grab and prepare a folio for write based on iter state. Returns the folio,
 921 * offset, and length. Callers can optionally pass a max length *plen,
 922 * otherwise init to zero.
 923 */
 924static int iomap_write_begin(struct iomap_iter *iter,
 925		const struct iomap_write_ops *write_ops, struct folio **foliop,
 926		size_t *poffset, u64 *plen)
 927{
 928	const struct iomap *srcmap = iomap_iter_srcmap(iter);
 929	loff_t pos;
 930	u64 len = min_t(u64, SIZE_MAX, iomap_length(iter));
 931	struct folio *folio;
 932	int status = 0;
 933
 934	len = min_not_zero(len, *plen);
 935	*foliop = NULL;
 936	*plen = 0;
 937
 938	if (fatal_signal_pending(current))
 939		return -EINTR;
 940
 941	folio = __iomap_get_folio(iter, write_ops, len);
 942	if (IS_ERR(folio))
 943		return PTR_ERR(folio);
 944
 945	/*
 946	 * No folio means we're done with a batch. We still have range to
 947	 * process so return and let the caller iterate and refill the batch.
 948	 */
 949	if (!folio) {
 950		WARN_ON_ONCE(!(iter->iomap.flags & IOMAP_F_FOLIO_BATCH));
 951		return 0;
 952	}
 953
 954	/*
 955	 * Now we have a locked folio, before we do anything with it we need to
 956	 * check that the iomap we have cached is not stale. The inode extent
 957	 * mapping can change due to concurrent IO in flight (e.g.
 958	 * IOMAP_UNWRITTEN state can change and memory reclaim could have
 959	 * reclaimed a previously partially written page at this index after IO
 960	 * completion before this write reaches this file offset) and hence we
 961	 * could do the wrong thing here (zero a page range incorrectly or fail
 962	 * to zero) and corrupt data.
 963	 */
 964	if (write_ops && write_ops->iomap_valid) {
 965		bool iomap_valid = write_ops->iomap_valid(iter->inode,
 966							 &iter->iomap);
 967		if (!iomap_valid) {
 968			iter->iomap.flags |= IOMAP_F_STALE;
 969			status = 0;
 970			goto out_unlock;
 971		}
 972	}
 973
 974	/*
 975	 * The folios in a batch may not be contiguous. If we've skipped
 976	 * forward, advance the iter to the pos of the current folio. If the
 977	 * folio starts beyond the end of the mapping, it may have been trimmed
 978	 * since the lookup for whatever reason. Return a NULL folio to
 979	 * terminate the op.
 980	 */
 981	if (folio_pos(folio) > iter->pos) {
 982		len = min_t(u64, folio_pos(folio) - iter->pos,
 983				 iomap_length(iter));
 984		status = iomap_iter_advance(iter, len);
 985		len = iomap_length(iter);
 986		if (status || !len)
 987			goto out_unlock;
 988	}
 989
 990	pos = iomap_trim_folio_range(iter, folio, poffset, &len);
 991
 992	if (srcmap->type == IOMAP_INLINE)
 993		status = iomap_write_begin_inline(iter, folio);
 994	else if (srcmap->flags & IOMAP_F_BUFFER_HEAD)
 995		status = __block_write_begin_int(folio, pos, len, NULL, srcmap);
 996	else
 997		status = __iomap_write_begin(iter, write_ops, len, folio);
 998
 999	if (unlikely(status))
1000		goto out_unlock;
1001
1002	*foliop = folio;
1003	*plen = len;
1004	return 0;
1005
1006out_unlock:
1007	__iomap_put_folio(iter, write_ops, 0, folio);
1008	return status;
1009}
1010
1011static bool __iomap_write_end(struct inode *inode, loff_t pos, size_t len,
1012		size_t copied, struct folio *folio)
1013{
1014	flush_dcache_folio(folio);
1015
1016	/*
1017	 * The blocks that were entirely written will now be uptodate, so we
1018	 * don't have to worry about a read_folio reading them and overwriting a
1019	 * partial write.  However, if we've encountered a short write and only
1020	 * partially written into a block, it will not be marked uptodate, so a
1021	 * read_folio might come in and destroy our partial write.
1022	 *
1023	 * Do the simplest thing and just treat any short write to a
1024	 * non-uptodate page as a zero-length write, and force the caller to
1025	 * redo the whole thing.
1026	 */
1027	if (unlikely(copied < len && !folio_test_uptodate(folio)))
1028		return false;
1029	iomap_set_range_uptodate(folio, offset_in_folio(folio, pos), len);
1030	iomap_set_range_dirty(folio, offset_in_folio(folio, pos), copied);
1031	filemap_dirty_folio(inode->i_mapping, folio);
1032	return true;
1033}
1034
1035static bool iomap_write_end_inline(const struct iomap_iter *iter,
1036		struct folio *folio, loff_t pos, size_t copied)
1037{
1038	const struct iomap *iomap = &iter->iomap;
1039	void *addr;
1040
1041	WARN_ON_ONCE(!folio_test_uptodate(folio));
1042	BUG_ON(!iomap_inline_data_valid(iomap));
1043
1044	if (WARN_ON_ONCE(!iomap->inline_data))
1045		return false;
1046
1047	flush_dcache_folio(folio);
1048	addr = kmap_local_folio(folio, pos);
1049	memcpy(iomap_inline_data(iomap, pos), addr, copied);
1050	kunmap_local(addr);
1051
1052	mark_inode_dirty(iter->inode);
1053	return true;
1054}
1055
1056/*
1057 * Returns true if all copied bytes have been written to the pagecache,
1058 * otherwise return false.
1059 */
1060static bool iomap_write_end(struct iomap_iter *iter, size_t len, size_t copied,
1061		struct folio *folio)
1062{
1063	const struct iomap *srcmap = iomap_iter_srcmap(iter);
1064	loff_t pos = iter->pos;
1065
1066	if (srcmap->type == IOMAP_INLINE)
1067		return iomap_write_end_inline(iter, folio, pos, copied);
1068
1069	if (srcmap->flags & IOMAP_F_BUFFER_HEAD) {
1070		size_t bh_written;
1071
1072		bh_written = block_write_end(pos, len, copied, folio);
1073		WARN_ON_ONCE(bh_written != copied && bh_written != 0);
1074		return bh_written == copied;
1075	}
1076
1077	return __iomap_write_end(iter->inode, pos, len, copied, folio);
1078}
1079
1080static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i,
1081		const struct iomap_write_ops *write_ops)
1082{
1083	ssize_t total_written = 0;
1084	int status = 0;
1085	struct address_space *mapping = iter->inode->i_mapping;
1086	size_t chunk = mapping_max_folio_size(mapping);
1087	unsigned int bdp_flags = (iter->flags & IOMAP_NOWAIT) ? BDP_ASYNC : 0;
1088
1089	do {
1090		struct folio *folio;
1091		loff_t old_size;
1092		size_t offset;		/* Offset into folio */
1093		u64 bytes;		/* Bytes to write to folio */
1094		size_t copied;		/* Bytes copied from user */
1095		u64 written;		/* Bytes have been written */
1096		loff_t pos;
1097
1098		bytes = iov_iter_count(i);
1099retry:
1100		offset = iter->pos & (chunk - 1);
1101		bytes = min(chunk - offset, bytes);
1102		status = balance_dirty_pages_ratelimited_flags(mapping,
1103							       bdp_flags);
1104		if (unlikely(status))
1105			break;
1106
1107		if (bytes > iomap_length(iter))
1108			bytes = iomap_length(iter);
1109
1110		/*
1111		 * Bring in the user page that we'll copy from _first_.
1112		 * Otherwise there's a nasty deadlock on copying from the
1113		 * same page as we're writing to, without it being marked
1114		 * up-to-date.
1115		 *
1116		 * For async buffered writes the assumption is that the user
1117		 * page has already been faulted in. This can be optimized by
1118		 * faulting the user page.
1119		 */
1120		if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) {
1121			status = -EFAULT;
1122			break;
1123		}
1124
1125		status = iomap_write_begin(iter, write_ops, &folio, &offset,
1126				&bytes);
1127		if (unlikely(status)) {
1128			iomap_write_failed(iter->inode, iter->pos, bytes);
1129			break;
1130		}
1131		if (iter->iomap.flags & IOMAP_F_STALE)
1132			break;
1133
1134		pos = iter->pos;
1135
1136		if (mapping_writably_mapped(mapping))
1137			flush_dcache_folio(folio);
1138
1139		copied = copy_folio_from_iter_atomic(folio, offset, bytes, i);
1140		written = iomap_write_end(iter, bytes, copied, folio) ?
1141			  copied : 0;
1142
1143		/*
1144		 * Update the in-memory inode size after copying the data into
1145		 * the page cache.  It's up to the file system to write the
1146		 * updated size to disk, preferably after I/O completion so that
1147		 * no stale data is exposed.  Only once that's done can we
1148		 * unlock and release the folio.
1149		 */
1150		old_size = iter->inode->i_size;
1151		if (pos + written > old_size) {
1152			i_size_write(iter->inode, pos + written);
1153			iter->iomap.flags |= IOMAP_F_SIZE_CHANGED;
1154		}
1155		__iomap_put_folio(iter, write_ops, written, folio);
1156
1157		if (old_size < pos)
1158			pagecache_isize_extended(iter->inode, old_size, pos);
1159
1160		cond_resched();
1161		if (unlikely(written == 0)) {
1162			/*
1163			 * A short copy made iomap_write_end() reject the
1164			 * thing entirely.  Might be memory poisoning
1165			 * halfway through, might be a race with munmap,
1166			 * might be severe memory pressure.
1167			 */
1168			iomap_write_failed(iter->inode, pos, bytes);
1169			iov_iter_revert(i, copied);
1170
1171			if (chunk > PAGE_SIZE)
1172				chunk /= 2;
1173			if (copied) {
1174				bytes = copied;
1175				goto retry;
1176			}
1177		} else {
1178			total_written += written;
1179			iomap_iter_advance(iter, written);
1180		}
1181	} while (iov_iter_count(i) && iomap_length(iter));
1182
1183	return total_written ? 0 : status;
1184}
1185
1186ssize_t
1187iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
1188		const struct iomap_ops *ops,
1189		const struct iomap_write_ops *write_ops, void *private)
1190{
1191	struct iomap_iter iter = {
1192		.inode		= iocb->ki_filp->f_mapping->host,
1193		.pos		= iocb->ki_pos,
1194		.len		= iov_iter_count(i),
1195		.flags		= IOMAP_WRITE,
1196		.private	= private,
1197	};
1198	ssize_t ret;
1199
1200	if (iocb->ki_flags & IOCB_NOWAIT)
1201		iter.flags |= IOMAP_NOWAIT;
1202	if (iocb->ki_flags & IOCB_DONTCACHE)
1203		iter.flags |= IOMAP_DONTCACHE;
1204
1205	while ((ret = iomap_iter(&iter, ops)) > 0)
1206		iter.status = iomap_write_iter(&iter, i, write_ops);
1207
1208	if (unlikely(iter.pos == iocb->ki_pos))
1209		return ret;
1210	ret = iter.pos - iocb->ki_pos;
1211	iocb->ki_pos = iter.pos;
1212	return ret;
1213}
1214EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
1215
1216static void iomap_write_delalloc_ifs_punch(struct inode *inode,
1217		struct folio *folio, loff_t start_byte, loff_t end_byte,
1218		struct iomap *iomap, iomap_punch_t punch)
1219{
1220	unsigned int first_blk, last_blk;
1221	loff_t last_byte;
1222	u8 blkbits = inode->i_blkbits;
1223	struct iomap_folio_state *ifs;
1224
1225	/*
1226	 * When we have per-block dirty tracking, there can be
1227	 * blocks within a folio which are marked uptodate
1228	 * but not dirty. In that case it is necessary to punch
1229	 * out such blocks to avoid leaking any delalloc blocks.
1230	 */
1231	ifs = folio->private;
1232	if (!ifs)
1233		return;
1234
1235	last_byte = min_t(loff_t, end_byte - 1, folio_next_pos(folio) - 1);
1236	first_blk = offset_in_folio(folio, start_byte) >> blkbits;
1237	last_blk = offset_in_folio(folio, last_byte) >> blkbits;
1238	while ((first_blk = ifs_next_clean_block(folio, first_blk, last_blk))
1239		       <= last_blk) {
1240		punch(inode, folio_pos(folio) + (first_blk << blkbits),
1241				1 << blkbits, iomap);
1242		first_blk++;
1243	}
1244}
1245
1246static void iomap_write_delalloc_punch(struct inode *inode, struct folio *folio,
1247		loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte,
1248		struct iomap *iomap, iomap_punch_t punch)
1249{
1250	if (!folio_test_dirty(folio))
1251		return;
1252
1253	/* if dirty, punch up to offset */
1254	if (start_byte > *punch_start_byte) {
1255		punch(inode, *punch_start_byte, start_byte - *punch_start_byte,
1256				iomap);
1257	}
1258
1259	/* Punch non-dirty blocks within folio */
1260	iomap_write_delalloc_ifs_punch(inode, folio, start_byte, end_byte,
1261			iomap, punch);
1262
1263	/*
1264	 * Make sure the next punch start is correctly bound to
1265	 * the end of this data range, not the end of the folio.
1266	 */
1267	*punch_start_byte = min_t(loff_t, end_byte, folio_next_pos(folio));
1268}
1269
1270/*
1271 * Scan the data range passed to us for dirty page cache folios. If we find a
1272 * dirty folio, punch out the preceding range and update the offset from which
1273 * the next punch will start from.
1274 *
1275 * We can punch out storage reservations under clean pages because they either
1276 * contain data that has been written back - in which case the delalloc punch
1277 * over that range is a no-op - or they have been read faults in which case they
1278 * contain zeroes and we can remove the delalloc backing range and any new
1279 * writes to those pages will do the normal hole filling operation...
1280 *
1281 * This makes the logic simple: we only need to keep the delalloc extents only
1282 * over the dirty ranges of the page cache.
1283 *
1284 * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1285 * simplify range iterations.
1286 */
1287static void iomap_write_delalloc_scan(struct inode *inode,
1288		loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte,
1289		struct iomap *iomap, iomap_punch_t punch)
1290{
1291	while (start_byte < end_byte) {
1292		struct folio	*folio;
1293
1294		/* grab locked page */
1295		folio = filemap_lock_folio(inode->i_mapping,
1296				start_byte >> PAGE_SHIFT);
1297		if (IS_ERR(folio)) {
1298			start_byte = ALIGN_DOWN(start_byte, PAGE_SIZE) +
1299					PAGE_SIZE;
1300			continue;
1301		}
1302
1303		iomap_write_delalloc_punch(inode, folio, punch_start_byte,
1304				start_byte, end_byte, iomap, punch);
1305
1306		/* move offset to start of next folio in range */
1307		start_byte = folio_next_pos(folio);
1308		folio_unlock(folio);
1309		folio_put(folio);
1310	}
1311}
1312
1313/*
1314 * When a short write occurs, the filesystem might need to use ->iomap_end
1315 * to remove space reservations created in ->iomap_begin.
1316 *
1317 * For filesystems that use delayed allocation, there can be dirty pages over
1318 * the delalloc extent outside the range of a short write but still within the
1319 * delalloc extent allocated for this iomap if the write raced with page
1320 * faults.
1321 *
1322 * Punch out all the delalloc blocks in the range given except for those that
1323 * have dirty data still pending in the page cache - those are going to be
1324 * written and so must still retain the delalloc backing for writeback.
1325 *
1326 * The punch() callback *must* only punch delalloc extents in the range passed
1327 * to it. It must skip over all other types of extents in the range and leave
1328 * them completely unchanged. It must do this punch atomically with respect to
1329 * other extent modifications.
1330 *
1331 * The punch() callback may be called with a folio locked to prevent writeback
1332 * extent allocation racing at the edge of the range we are currently punching.
1333 * The locked folio may or may not cover the range being punched, so it is not
1334 * safe for the punch() callback to lock folios itself.
1335 *
1336 * Lock order is:
1337 *
1338 * inode->i_rwsem (shared or exclusive)
1339 *   inode->i_mapping->invalidate_lock (exclusive)
1340 *     folio_lock()
1341 *       ->punch
1342 *         internal filesystem allocation lock
1343 *
1344 * As we are scanning the page cache for data, we don't need to reimplement the
1345 * wheel - mapping_seek_hole_data() does exactly what we need to identify the
1346 * start and end of data ranges correctly even for sub-folio block sizes. This
1347 * byte range based iteration is especially convenient because it means we
1348 * don't have to care about variable size folios, nor where the start or end of
1349 * the data range lies within a folio, if they lie within the same folio or even
1350 * if there are multiple discontiguous data ranges within the folio.
1351 *
1352 * It should be noted that mapping_seek_hole_data() is not aware of EOF, and so
1353 * can return data ranges that exist in the cache beyond EOF. e.g. a page fault
1354 * spanning EOF will initialise the post-EOF data to zeroes and mark it up to
1355 * date. A write page fault can then mark it dirty. If we then fail a write()
1356 * beyond EOF into that up to date cached range, we allocate a delalloc block
1357 * beyond EOF and then have to punch it out. Because the range is up to date,
1358 * mapping_seek_hole_data() will return it, and we will skip the punch because
1359 * the folio is dirty. THis is incorrect - we always need to punch out delalloc
1360 * beyond EOF in this case as writeback will never write back and covert that
1361 * delalloc block beyond EOF. Hence we limit the cached data scan range to EOF,
1362 * resulting in always punching out the range from the EOF to the end of the
1363 * range the iomap spans.
1364 *
1365 * Intervals are of the form [start_byte, end_byte) (i.e. open ended) because it
1366 * matches the intervals returned by mapping_seek_hole_data(). i.e. SEEK_DATA
1367 * returns the start of a data range (start_byte), and SEEK_HOLE(start_byte)
1368 * returns the end of the data range (data_end). Using closed intervals would
1369 * require sprinkling this code with magic "+ 1" and "- 1" arithmetic and expose
1370 * the code to subtle off-by-one bugs....
1371 */
1372void iomap_write_delalloc_release(struct inode *inode, loff_t start_byte,
1373		loff_t end_byte, unsigned flags, struct iomap *iomap,
1374		iomap_punch_t punch)
1375{
1376	loff_t punch_start_byte = start_byte;
1377	loff_t scan_end_byte = min(i_size_read(inode), end_byte);
1378
1379	/*
1380	 * The caller must hold invalidate_lock to avoid races with page faults
1381	 * re-instantiating folios and dirtying them via ->page_mkwrite whilst
1382	 * we walk the cache and perform delalloc extent removal.  Failing to do
1383	 * this can leave dirty pages with no space reservation in the cache.
1384	 */
1385	lockdep_assert_held_write(&inode->i_mapping->invalidate_lock);
1386
1387	while (start_byte < scan_end_byte) {
1388		loff_t		data_end;
1389
1390		start_byte = mapping_seek_hole_data(inode->i_mapping,
1391				start_byte, scan_end_byte, SEEK_DATA);
1392		/*
1393		 * If there is no more data to scan, all that is left is to
1394		 * punch out the remaining range.
1395		 *
1396		 * Note that mapping_seek_hole_data is only supposed to return
1397		 * either an offset or -ENXIO, so WARN on any other error as
1398		 * that would be an API change without updating the callers.
1399		 */
1400		if (start_byte == -ENXIO || start_byte == scan_end_byte)
1401			break;
1402		if (WARN_ON_ONCE(start_byte < 0))
1403			return;
1404		WARN_ON_ONCE(start_byte < punch_start_byte);
1405		WARN_ON_ONCE(start_byte > scan_end_byte);
1406
1407		/*
1408		 * We find the end of this contiguous cached data range by
1409		 * seeking from start_byte to the beginning of the next hole.
1410		 */
1411		data_end = mapping_seek_hole_data(inode->i_mapping, start_byte,
1412				scan_end_byte, SEEK_HOLE);
1413		if (WARN_ON_ONCE(data_end < 0))
1414			return;
1415
1416		/*
1417		 * If we race with post-direct I/O invalidation of the page cache,
1418		 * there might be no data left at start_byte.
1419		 */
1420		if (data_end == start_byte)
1421			continue;
1422
1423		WARN_ON_ONCE(data_end < start_byte);
1424		WARN_ON_ONCE(data_end > scan_end_byte);
1425
1426		iomap_write_delalloc_scan(inode, &punch_start_byte, start_byte,
1427				data_end, iomap, punch);
1428
1429		/* The next data search starts at the end of this one. */
1430		start_byte = data_end;
1431	}
1432
1433	if (punch_start_byte < end_byte)
1434		punch(inode, punch_start_byte, end_byte - punch_start_byte,
1435				iomap);
1436}
1437EXPORT_SYMBOL_GPL(iomap_write_delalloc_release);
1438
1439static int iomap_unshare_iter(struct iomap_iter *iter,
1440		const struct iomap_write_ops *write_ops)
1441{
1442	struct iomap *iomap = &iter->iomap;
1443	u64 bytes = iomap_length(iter);
1444	int status;
1445
1446	if (!iomap_want_unshare_iter(iter))
1447		return iomap_iter_advance(iter, bytes);
1448
1449	do {
1450		struct folio *folio;
1451		size_t offset;
1452		bool ret;
1453
1454		bytes = min_t(u64, SIZE_MAX, bytes);
1455		status = iomap_write_begin(iter, write_ops, &folio, &offset,
1456				&bytes);
1457		if (unlikely(status))
1458			return status;
1459		if (iomap->flags & IOMAP_F_STALE)
1460			break;
1461
1462		ret = iomap_write_end(iter, bytes, bytes, folio);
1463		__iomap_put_folio(iter, write_ops, bytes, folio);
1464		if (WARN_ON_ONCE(!ret))
1465			return -EIO;
1466
1467		cond_resched();
1468
1469		balance_dirty_pages_ratelimited(iter->inode->i_mapping);
1470
1471		status = iomap_iter_advance(iter, bytes);
1472		if (status)
1473			break;
1474	} while ((bytes = iomap_length(iter)) > 0);
1475
1476	return status;
1477}
1478
1479int
1480iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
1481		const struct iomap_ops *ops,
1482		const struct iomap_write_ops *write_ops)
1483{
1484	struct iomap_iter iter = {
1485		.inode		= inode,
1486		.pos		= pos,
1487		.flags		= IOMAP_WRITE | IOMAP_UNSHARE,
1488	};
1489	loff_t size = i_size_read(inode);
1490	int ret;
1491
1492	if (pos < 0 || pos >= size)
1493		return 0;
1494
1495	iter.len = min(len, size - pos);
1496	while ((ret = iomap_iter(&iter, ops)) > 0)
1497		iter.status = iomap_unshare_iter(&iter, write_ops);
1498	return ret;
1499}
1500EXPORT_SYMBOL_GPL(iomap_file_unshare);
1501
1502/*
1503 * Flush the remaining range of the iter and mark the current mapping stale.
1504 * This is used when zero range sees an unwritten mapping that may have had
1505 * dirty pagecache over it.
1506 */
1507static inline int iomap_zero_iter_flush_and_stale(struct iomap_iter *i)
1508{
1509	struct address_space *mapping = i->inode->i_mapping;
1510	loff_t end = i->pos + i->len - 1;
1511
1512	i->iomap.flags |= IOMAP_F_STALE;
1513	return filemap_write_and_wait_range(mapping, i->pos, end);
1514}
1515
1516static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero,
1517		const struct iomap_write_ops *write_ops)
1518{
1519	u64 bytes = iomap_length(iter);
1520	int status;
1521
1522	do {
1523		struct folio *folio;
1524		size_t offset;
1525		bool ret;
1526
1527		bytes = min_t(u64, SIZE_MAX, bytes);
1528		status = iomap_write_begin(iter, write_ops, &folio, &offset,
1529				&bytes);
1530		if (status)
1531			return status;
1532		if (iter->iomap.flags & IOMAP_F_STALE)
1533			break;
1534
1535		/* a NULL folio means we're done with a folio batch */
1536		if (!folio) {
1537			status = iomap_iter_advance_full(iter);
1538			break;
1539		}
1540
1541		/* warn about zeroing folios beyond eof that won't write back */
1542		WARN_ON_ONCE(folio_pos(folio) > iter->inode->i_size);
1543
1544		trace_iomap_zero_iter(iter->inode, folio_pos(folio) + offset,
1545				bytes);
1546
1547		folio_zero_range(folio, offset, bytes);
1548		folio_mark_accessed(folio);
1549
1550		ret = iomap_write_end(iter, bytes, bytes, folio);
1551		__iomap_put_folio(iter, write_ops, bytes, folio);
1552		if (WARN_ON_ONCE(!ret))
1553			return -EIO;
1554
1555		status = iomap_iter_advance(iter, bytes);
1556		if (status)
1557			break;
1558	} while ((bytes = iomap_length(iter)) > 0);
1559
1560	if (did_zero)
1561		*did_zero = true;
1562	return status;
1563}
1564
1565/**
1566 * iomap_fill_dirty_folios - fill a folio batch with dirty folios
1567 * @iter: Iteration structure
1568 * @start: Start offset of range. Updated based on lookup progress.
1569 * @end: End offset of range
1570 * @iomap_flags: Flags to set on the associated iomap to track the batch.
1571 *
1572 * Returns the folio count directly. Also returns the associated control flag if
1573 * the the batch lookup is performed and the expected offset of a subsequent
1574 * lookup via out params. The caller is responsible to set the flag on the
1575 * associated iomap.
1576 */
1577unsigned int
1578iomap_fill_dirty_folios(
1579	struct iomap_iter	*iter,
1580	loff_t			*start,
1581	loff_t			end,
1582	unsigned int		*iomap_flags)
1583{
1584	struct address_space	*mapping = iter->inode->i_mapping;
1585	pgoff_t			pstart = *start >> PAGE_SHIFT;
1586	pgoff_t			pend = (end - 1) >> PAGE_SHIFT;
1587	unsigned int		count;
1588
1589	if (!iter->fbatch) {
1590		*start = end;
1591		return 0;
1592	}
1593
1594	count = filemap_get_folios_dirty(mapping, &pstart, pend, iter->fbatch);
1595	*start = (pstart << PAGE_SHIFT);
1596	*iomap_flags |= IOMAP_F_FOLIO_BATCH;
1597	return count;
1598}
1599EXPORT_SYMBOL_GPL(iomap_fill_dirty_folios);
1600
1601int
1602iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
1603		const struct iomap_ops *ops,
1604		const struct iomap_write_ops *write_ops, void *private)
1605{
1606	struct folio_batch fbatch;
1607	struct iomap_iter iter = {
1608		.inode		= inode,
1609		.pos		= pos,
1610		.len		= len,
1611		.flags		= IOMAP_ZERO,
1612		.private	= private,
1613		.fbatch		= &fbatch,
1614	};
1615	struct address_space *mapping = inode->i_mapping;
1616	int ret;
1617	bool range_dirty;
1618
1619	folio_batch_init(&fbatch);
1620
1621	/*
1622	 * To avoid an unconditional flush, check pagecache state and only flush
1623	 * if dirty and the fs returns a mapping that might convert on
1624	 * writeback.
1625	 */
1626	range_dirty = filemap_range_needs_writeback(mapping, iter.pos,
1627					iter.pos + iter.len - 1);
1628	while ((ret = iomap_iter(&iter, ops)) > 0) {
1629		const struct iomap *srcmap = iomap_iter_srcmap(&iter);
1630
1631		if (WARN_ON_ONCE((iter.iomap.flags & IOMAP_F_FOLIO_BATCH) &&
1632				 srcmap->type != IOMAP_UNWRITTEN))
1633			return -EIO;
1634
1635		if (!(iter.iomap.flags & IOMAP_F_FOLIO_BATCH) &&
1636		    (srcmap->type == IOMAP_HOLE ||
1637		     srcmap->type == IOMAP_UNWRITTEN)) {
1638			s64 status;
1639
1640			if (range_dirty) {
1641				range_dirty = false;
1642				status = iomap_zero_iter_flush_and_stale(&iter);
1643			} else {
1644				status = iomap_iter_advance_full(&iter);
1645			}
1646			iter.status = status;
1647			continue;
1648		}
1649
1650		iter.status = iomap_zero_iter(&iter, did_zero, write_ops);
1651	}
1652	return ret;
1653}
1654EXPORT_SYMBOL_GPL(iomap_zero_range);
1655
1656int
1657iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
1658		const struct iomap_ops *ops,
1659		const struct iomap_write_ops *write_ops, void *private)
1660{
1661	unsigned int blocksize = i_blocksize(inode);
1662	unsigned int off = pos & (blocksize - 1);
1663
1664	/* Block boundary? Nothing to do */
1665	if (!off)
1666		return 0;
1667	return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops,
1668			write_ops, private);
1669}
1670EXPORT_SYMBOL_GPL(iomap_truncate_page);
1671
1672static int iomap_folio_mkwrite_iter(struct iomap_iter *iter,
1673		struct folio *folio)
1674{
1675	loff_t length = iomap_length(iter);
1676	int ret;
1677
1678	if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) {
1679		ret = __block_write_begin_int(folio, iter->pos, length, NULL,
1680					      &iter->iomap);
1681		if (ret)
1682			return ret;
1683		block_commit_write(folio, 0, length);
1684	} else {
1685		WARN_ON_ONCE(!folio_test_uptodate(folio));
1686		folio_mark_dirty(folio);
1687	}
1688
1689	return iomap_iter_advance(iter, length);
1690}
1691
1692vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops,
1693		void *private)
1694{
1695	struct iomap_iter iter = {
1696		.inode		= file_inode(vmf->vma->vm_file),
1697		.flags		= IOMAP_WRITE | IOMAP_FAULT,
1698		.private	= private,
1699	};
1700	struct folio *folio = page_folio(vmf->page);
1701	ssize_t ret;
1702
1703	folio_lock(folio);
1704	ret = folio_mkwrite_check_truncate(folio, iter.inode);
1705	if (ret < 0)
1706		goto out_unlock;
1707	iter.pos = folio_pos(folio);
1708	iter.len = ret;
1709	while ((ret = iomap_iter(&iter, ops)) > 0)
1710		iter.status = iomap_folio_mkwrite_iter(&iter, folio);
1711
1712	if (ret < 0)
1713		goto out_unlock;
1714	folio_wait_stable(folio);
1715	return VM_FAULT_LOCKED;
1716out_unlock:
1717	folio_unlock(folio);
1718	return vmf_fs_error(ret);
1719}
1720EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1721
1722static void iomap_writeback_init(struct inode *inode, struct folio *folio)
1723{
1724	struct iomap_folio_state *ifs = folio->private;
1725
1726	WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);
1727	if (ifs) {
1728		WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending) != 0);
1729		/*
1730		 * Set this to the folio size. After processing the folio for
1731		 * writeback in iomap_writeback_folio(), we'll subtract any
1732		 * ranges not written back.
1733		 *
1734		 * We do this because otherwise, we would have to atomically
1735		 * increment ifs->write_bytes_pending every time a range in the
1736		 * folio needs to be written back.
1737		 */
1738		atomic_set(&ifs->write_bytes_pending, folio_size(folio));
1739	}
1740}
1741
1742void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
1743		size_t len)
1744{
1745	struct iomap_folio_state *ifs = folio->private;
1746
1747	WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs);
1748	WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) <= 0);
1749
1750	if (!ifs || atomic_sub_and_test(len, &ifs->write_bytes_pending))
1751		folio_end_writeback(folio);
1752}
1753EXPORT_SYMBOL_GPL(iomap_finish_folio_write);
1754
1755static int iomap_writeback_range(struct iomap_writepage_ctx *wpc,
1756		struct folio *folio, u64 pos, u32 rlen, u64 end_pos,
1757		size_t *bytes_submitted)
1758{
1759	do {
1760		ssize_t ret;
1761
1762		ret = wpc->ops->writeback_range(wpc, folio, pos, rlen, end_pos);
1763		if (WARN_ON_ONCE(ret == 0 || ret > rlen))
1764			return -EIO;
1765		if (ret < 0)
1766			return ret;
1767		rlen -= ret;
1768		pos += ret;
1769
1770		/*
1771		 * Holes are not written back by ->writeback_range, so track
1772		 * if we did handle anything that is not a hole here.
1773		 */
1774		if (wpc->iomap.type != IOMAP_HOLE)
1775			*bytes_submitted += ret;
1776	} while (rlen);
1777
1778	return 0;
1779}
1780
1781/*
1782 * Check interaction of the folio with the file end.
1783 *
1784 * If the folio is entirely beyond i_size, return false.  If it straddles
1785 * i_size, adjust end_pos and zero all data beyond i_size.
1786 */
1787static bool iomap_writeback_handle_eof(struct folio *folio, struct inode *inode,
1788		u64 *end_pos)
1789{
1790	u64 isize = i_size_read(inode);
1791
1792	if (*end_pos > isize) {
1793		size_t poff = offset_in_folio(folio, isize);
1794		pgoff_t end_index = isize >> PAGE_SHIFT;
1795
1796		/*
1797		 * If the folio is entirely ouside of i_size, skip it.
1798		 *
1799		 * This can happen due to a truncate operation that is in
1800		 * progress and in that case truncate will finish it off once
1801		 * we've dropped the folio lock.
1802		 *
1803		 * Note that the pgoff_t used for end_index is an unsigned long.
1804		 * If the given offset is greater than 16TB on a 32-bit system,
1805		 * then if we checked if the folio is fully outside i_size with
1806		 * "if (folio->index >= end_index + 1)", "end_index + 1" would
1807		 * overflow and evaluate to 0.  Hence this folio would be
1808		 * redirtied and written out repeatedly, which would result in
1809		 * an infinite loop; the user program performing this operation
1810		 * would hang.  Instead, we can detect this situation by
1811		 * checking if the folio is totally beyond i_size or if its
1812		 * offset is just equal to the EOF.
1813		 */
1814		if (folio->index > end_index ||
1815		    (folio->index == end_index && poff == 0))
1816			return false;
1817
1818		/*
1819		 * The folio straddles i_size.
1820		 *
1821		 * It must be zeroed out on each and every writepage invocation
1822		 * because it may be mmapped:
1823		 *
1824		 *    A file is mapped in multiples of the page size.  For a
1825		 *    file that is not a multiple of the page size, the
1826		 *    remaining memory is zeroed when mapped, and writes to that
1827		 *    region are not written out to the file.
1828		 *
1829		 * Also adjust the end_pos to the end of file and skip writeback
1830		 * for all blocks entirely beyond i_size.
1831		 */
1832		folio_zero_segment(folio, poff, folio_size(folio));
1833		*end_pos = isize;
1834	}
1835
1836	return true;
1837}
1838
1839int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio)
1840{
1841	struct iomap_folio_state *ifs = folio->private;
1842	struct inode *inode = wpc->inode;
1843	u64 pos = folio_pos(folio);
1844	u64 end_pos = pos + folio_size(folio);
1845	u64 end_aligned = 0;
1846	size_t bytes_submitted = 0;
1847	int error = 0;
1848	u32 rlen;
1849
1850	WARN_ON_ONCE(!folio_test_locked(folio));
1851	WARN_ON_ONCE(folio_test_dirty(folio));
1852	WARN_ON_ONCE(folio_test_writeback(folio));
1853
1854	trace_iomap_writeback_folio(inode, pos, folio_size(folio));
1855
1856	if (!iomap_writeback_handle_eof(folio, inode, &end_pos))
1857		return 0;
1858	WARN_ON_ONCE(end_pos <= pos);
1859
1860	if (i_blocks_per_folio(inode, folio) > 1) {
1861		if (!ifs) {
1862			ifs = ifs_alloc(inode, folio, 0);
1863			iomap_set_range_dirty(folio, 0, end_pos - pos);
1864		}
1865
1866		iomap_writeback_init(inode, folio);
1867	}
1868
1869	/*
1870	 * Set the writeback bit ASAP, as the I/O completion for the single
1871	 * block per folio case happen hit as soon as we're submitting the bio.
1872	 */
1873	folio_start_writeback(folio);
1874
1875	/*
1876	 * Walk through the folio to find dirty areas to write back.
1877	 */
1878	end_aligned = round_up(end_pos, i_blocksize(inode));
1879	while ((rlen = iomap_find_dirty_range(folio, &pos, end_aligned))) {
1880		error = iomap_writeback_range(wpc, folio, pos, rlen, end_pos,
1881				&bytes_submitted);
1882		if (error)
1883			break;
1884		pos += rlen;
1885	}
1886
1887	if (bytes_submitted)
1888		wpc->nr_folios++;
1889
1890	/*
1891	 * We can have dirty bits set past end of file in page_mkwrite path
1892	 * while mapping the last partial folio. Hence it's better to clear
1893	 * all the dirty bits in the folio here.
1894	 */
1895	iomap_clear_range_dirty(folio, 0, folio_size(folio));
1896
1897	/*
1898	 * Usually the writeback bit is cleared by the I/O completion handler.
1899	 * But we may end up either not actually writing any blocks, or (when
1900	 * there are multiple blocks in a folio) all I/O might have finished
1901	 * already at this point.  In that case we need to clear the writeback
1902	 * bit ourselves right after unlocking the page.
1903	 */
1904	if (ifs) {
1905		/*
1906		 * Subtract any bytes that were initially accounted to
1907		 * write_bytes_pending but skipped for writeback.
1908		 */
1909		size_t bytes_not_submitted = folio_size(folio) -
1910				bytes_submitted;
1911
1912		if (bytes_not_submitted)
1913			iomap_finish_folio_write(inode, folio,
1914					bytes_not_submitted);
1915	} else if (!bytes_submitted) {
1916		folio_end_writeback(folio);
1917	}
1918
1919	mapping_set_error(inode->i_mapping, error);
1920	return error;
1921}
1922EXPORT_SYMBOL_GPL(iomap_writeback_folio);
1923
1924int
1925iomap_writepages(struct iomap_writepage_ctx *wpc)
1926{
1927	struct address_space *mapping = wpc->inode->i_mapping;
1928	struct folio *folio = NULL;
1929	int error;
1930
1931	/*
1932	 * Writeback from reclaim context should never happen except in the case
1933	 * of a VM regression so warn about it and refuse to write the data.
1934	 */
1935	if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC | PF_KSWAPD)) ==
1936			PF_MEMALLOC))
1937		return -EIO;
1938
1939	while ((folio = writeback_iter(mapping, wpc->wbc, folio, &error))) {
1940		error = iomap_writeback_folio(wpc, folio);
1941		folio_unlock(folio);
1942	}
1943
1944	/*
1945	 * If @error is non-zero, it means that we have a situation where some
1946	 * part of the submission process has failed after we've marked pages
1947	 * for writeback.
1948	 *
1949	 * We cannot cancel the writeback directly in that case, so always call
1950	 * ->writeback_submit to run the I/O completion handler to clear the
1951	 * writeback bit and let the file system proess the errors.
1952	 */
1953	if (wpc->wb_ctx)
1954		return wpc->ops->writeback_submit(wpc, error);
1955	return error;
1956}
1957EXPORT_SYMBOL_GPL(iomap_writepages);