ioend.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2016-2025 Christoph Hellwig.
  4 */
  5#include <linux/iomap.h>
  6#include <linux/list_sort.h>
  7#include <linux/pagemap.h>
  8#include <linux/writeback.h>
  9#include "internal.h"
 10#include "trace.h"
 11
 12struct bio_set iomap_ioend_bioset;
 13EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
 14
 15struct iomap_ioend *iomap_init_ioend(struct inode *inode,
 16		struct bio *bio, loff_t file_offset, u16 ioend_flags)
 17{
 18	struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
 19
 20	atomic_set(&ioend->io_remaining, 1);
 21	ioend->io_error = 0;
 22	ioend->io_parent = NULL;
 23	INIT_LIST_HEAD(&ioend->io_list);
 24	ioend->io_flags = ioend_flags;
 25	ioend->io_inode = inode;
 26	ioend->io_offset = file_offset;
 27	ioend->io_size = bio->bi_iter.bi_size;
 28	ioend->io_sector = bio->bi_iter.bi_sector;
 29	ioend->io_private = NULL;
 30	return ioend;
 31}
 32EXPORT_SYMBOL_GPL(iomap_init_ioend);
 33
 34/*
 35 * We're now finished for good with this ioend structure.  Update the folio
 36 * state, release holds on bios, and finally free up memory.  Do not use the
 37 * ioend after this.
 38 */
 39static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
 40{
 41	struct inode *inode = ioend->io_inode;
 42	struct bio *bio = &ioend->io_bio;
 43	struct folio_iter fi;
 44	u32 folio_count = 0;
 45
 46	if (ioend->io_error) {
 47		mapping_set_error(inode->i_mapping, ioend->io_error);
 48		if (!bio_flagged(bio, BIO_QUIET)) {
 49			pr_err_ratelimited(
 50"%s: writeback error on inode %lu, offset %lld, sector %llu",
 51				inode->i_sb->s_id, inode->i_ino,
 52				ioend->io_offset, ioend->io_sector);
 53		}
 54	}
 55
 56	/* walk all folios in bio, ending page IO on them */
 57	bio_for_each_folio_all(fi, bio) {
 58		iomap_finish_folio_write(inode, fi.folio, fi.length);
 59		folio_count++;
 60	}
 61
 62	bio_put(bio);	/* frees the ioend */
 63	return folio_count;
 64}
 65
 66static void ioend_writeback_end_bio(struct bio *bio)
 67{
 68	struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
 69
 70	ioend->io_error = blk_status_to_errno(bio->bi_status);
 71	iomap_finish_ioend_buffered(ioend);
 72}
 73
 74/*
 75 * We cannot cancel the ioend directly in case of an error, so call the bio end
 76 * I/O handler with the error status here to run the normal I/O completion
 77 * handler.
 78 */
 79int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
 80{
 81	struct iomap_ioend *ioend = wpc->wb_ctx;
 82
 83	if (!ioend->io_bio.bi_end_io)
 84		ioend->io_bio.bi_end_io = ioend_writeback_end_bio;
 85
 86	if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
 87		error = -EIO;
 88
 89	if (error) {
 90		ioend->io_bio.bi_status = errno_to_blk_status(error);
 91		bio_endio(&ioend->io_bio);
 92		return error;
 93	}
 94
 95	submit_bio(&ioend->io_bio);
 96	return 0;
 97}
 98EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit);
 99
100static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
101		loff_t pos, u16 ioend_flags)
102{
103	struct bio *bio;
104
105	bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
106			       REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
107			       GFP_NOFS, &iomap_ioend_bioset);
108	bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
109	bio->bi_write_hint = wpc->inode->i_write_hint;
110	wbc_init_bio(wpc->wbc, bio);
111	wpc->nr_folios = 0;
112	return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
113}
114
115static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
116		u16 ioend_flags)
117{
118	struct iomap_ioend *ioend = wpc->wb_ctx;
119
120	if (ioend_flags & IOMAP_IOEND_BOUNDARY)
121		return false;
122	if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
123	    (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
124		return false;
125	if (pos != ioend->io_offset + ioend->io_size)
126		return false;
127	if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
128	    iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
129		return false;
130	/*
131	 * Limit ioend bio chain lengths to minimise IO completion latency. This
132	 * also prevents long tight loops ending page writeback on all the
133	 * folios in the ioend.
134	 */
135	if (wpc->nr_folios >= IOEND_BATCH_SIZE)
136		return false;
137	return true;
138}
139
140/*
141 * Test to see if we have an existing ioend structure that we could append to
142 * first; otherwise finish off the current ioend and start another.
143 *
144 * If a new ioend is created and cached, the old ioend is submitted to the block
145 * layer instantly.  Batching optimisations are provided by higher level block
146 * plugging.
147 *
148 * At the end of a writeback pass, there will be a cached ioend remaining on the
149 * writepage context that the caller will need to submit.
150 */
151ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
152		loff_t pos, loff_t end_pos, unsigned int dirty_len)
153{
154	struct iomap_ioend *ioend = wpc->wb_ctx;
155	size_t poff = offset_in_folio(folio, pos);
156	unsigned int ioend_flags = 0;
157	unsigned int map_len = min_t(u64, dirty_len,
158		wpc->iomap.offset + wpc->iomap.length - pos);
159	int error;
160
161	trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);
162
163	WARN_ON_ONCE(!folio->private && map_len < dirty_len);
164
165	switch (wpc->iomap.type) {
166	case IOMAP_UNWRITTEN:
167		ioend_flags |= IOMAP_IOEND_UNWRITTEN;
168		break;
169	case IOMAP_MAPPED:
170		break;
171	case IOMAP_HOLE:
172		return map_len;
173	default:
174		WARN_ON_ONCE(1);
175		return -EIO;
176	}
177
178	if (wpc->iomap.flags & IOMAP_F_SHARED)
179		ioend_flags |= IOMAP_IOEND_SHARED;
180	if (folio_test_dropbehind(folio))
181		ioend_flags |= IOMAP_IOEND_DONTCACHE;
182	if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
183		ioend_flags |= IOMAP_IOEND_BOUNDARY;
184
185	if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
186new_ioend:
187		if (ioend) {
188			error = wpc->ops->writeback_submit(wpc, 0);
189			if (error)
190				return error;
191		}
192		wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags);
193	}
194
195	if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff))
196		goto new_ioend;
197
198	/*
199	 * Clamp io_offset and io_size to the incore EOF so that ondisk
200	 * file size updates in the ioend completion are byte-accurate.
201	 * This avoids recovering files with zeroed tail regions when
202	 * writeback races with appending writes:
203	 *
204	 *    Thread 1:                  Thread 2:
205	 *    ------------               -----------
206	 *    write [A, A+B]
207	 *    update inode size to A+B
208	 *    submit I/O [A, A+BS]
209	 *                               write [A+B, A+B+C]
210	 *                               update inode size to A+B+C
211	 *    <I/O completes, updates disk size to min(A+B+C, A+BS)>
212	 *    <power failure>
213	 *
214	 *  After reboot:
215	 *    1) with A+B+C < A+BS, the file has zero padding in range
216	 *       [A+B, A+B+C]
217	 *
218	 *    |<     Block Size (BS)   >|
219	 *    |DDDDDDDDDDDD0000000000000|
220	 *    ^           ^        ^
221	 *    A          A+B     A+B+C
222	 *                       (EOF)
223	 *
224	 *    2) with A+B+C > A+BS, the file has zero padding in range
225	 *       [A+B, A+BS]
226	 *
227	 *    |<     Block Size (BS)   >|<     Block Size (BS)    >|
228	 *    |DDDDDDDDDDDD0000000000000|00000000000000000000000000|
229	 *    ^           ^             ^           ^
230	 *    A          A+B           A+BS       A+B+C
231	 *                             (EOF)
232	 *
233	 *    D = Valid Data
234	 *    0 = Zero Padding
235	 *
236	 * Note that this defeats the ability to chain the ioends of
237	 * appending writes.
238	 */
239	ioend->io_size += map_len;
240	if (ioend->io_offset + ioend->io_size > end_pos)
241		ioend->io_size = end_pos - ioend->io_offset;
242
243	wbc_account_cgroup_owner(wpc->wbc, folio, map_len);
244	return map_len;
245}
246EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
247
248static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
249{
250	if (ioend->io_parent) {
251		struct bio *bio = &ioend->io_bio;
252
253		ioend = ioend->io_parent;
254		bio_put(bio);
255	}
256
257	if (error)
258		cmpxchg(&ioend->io_error, 0, error);
259
260	if (!atomic_dec_and_test(&ioend->io_remaining))
261		return 0;
262	if (ioend->io_flags & IOMAP_IOEND_DIRECT)
263		return iomap_finish_ioend_direct(ioend);
264	return iomap_finish_ioend_buffered(ioend);
265}
266
267/*
268 * Ioend completion routine for merged bios. This can only be called from task
269 * contexts as merged ioends can be of unbound length. Hence we have to break up
270 * the writeback completions into manageable chunks to avoid long scheduler
271 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
272 * good batch processing throughput without creating adverse scheduler latency
273 * conditions.
274 */
275void iomap_finish_ioends(struct iomap_ioend *ioend, int error)
276{
277	struct list_head tmp;
278	u32 completions;
279
280	might_sleep();
281
282	list_replace_init(&ioend->io_list, &tmp);
283	completions = iomap_finish_ioend(ioend, error);
284
285	while (!list_empty(&tmp)) {
286		if (completions > IOEND_BATCH_SIZE * 8) {
287			cond_resched();
288			completions = 0;
289		}
290		ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
291		list_del_init(&ioend->io_list);
292		completions += iomap_finish_ioend(ioend, error);
293	}
294}
295EXPORT_SYMBOL_GPL(iomap_finish_ioends);
296
297/*
298 * We can merge two adjacent ioends if they have the same set of work to do.
299 */
300static bool iomap_ioend_can_merge(struct iomap_ioend *ioend,
301		struct iomap_ioend *next)
302{
303	if (ioend->io_bio.bi_status != next->io_bio.bi_status)
304		return false;
305	if (next->io_flags & IOMAP_IOEND_BOUNDARY)
306		return false;
307	if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
308	    (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
309		return false;
310	if (ioend->io_offset + ioend->io_size != next->io_offset)
311		return false;
312	/*
313	 * Do not merge physically discontiguous ioends. The filesystem
314	 * completion functions will have to iterate the physical
315	 * discontiguities even if we merge the ioends at a logical level, so
316	 * we don't gain anything by merging physical discontiguities here.
317	 *
318	 * We cannot use bio->bi_iter.bi_sector here as it is modified during
319	 * submission so does not point to the start sector of the bio at
320	 * completion.
321	 */
322	if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) !=
323	    next->io_sector)
324		return false;
325	return true;
326}
327
328void iomap_ioend_try_merge(struct iomap_ioend *ioend,
329		struct list_head *more_ioends)
330{
331	struct iomap_ioend *next;
332
333	INIT_LIST_HEAD(&ioend->io_list);
334
335	while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
336			io_list))) {
337		if (!iomap_ioend_can_merge(ioend, next))
338			break;
339		list_move_tail(&next->io_list, &ioend->io_list);
340		ioend->io_size += next->io_size;
341	}
342}
343EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
344
345static int iomap_ioend_compare(void *priv, const struct list_head *a,
346		const struct list_head *b)
347{
348	struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
349	struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
350
351	if (ia->io_offset < ib->io_offset)
352		return -1;
353	if (ia->io_offset > ib->io_offset)
354		return 1;
355	return 0;
356}
357
358void iomap_sort_ioends(struct list_head *ioend_list)
359{
360	list_sort(NULL, ioend_list, iomap_ioend_compare);
361}
362EXPORT_SYMBOL_GPL(iomap_sort_ioends);
363
364/*
365 * Split up to the first @max_len bytes from @ioend if the ioend covers more
366 * than @max_len bytes.
367 *
368 * If @is_append is set, the split will be based on the hardware limits for
369 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware
370 * limits don't allow the entire @max_len length.
371 *
372 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer
373 * does not allow splitting REQ_OP_ZONE_APPEND bios.  The file systems has to
374 * switch the operation after this call, but before submitting the bio.
375 */
376struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
377		unsigned int max_len, bool is_append)
378{
379	struct bio *bio = &ioend->io_bio;
380	struct iomap_ioend *split_ioend;
381	unsigned int nr_segs;
382	int sector_offset;
383	struct bio *split;
384
385	if (is_append) {
386		struct queue_limits *lim = bdev_limits(bio->bi_bdev);
387
388		max_len = min(max_len,
389			      lim->max_zone_append_sectors << SECTOR_SHIFT);
390
391		sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len);
392		if (unlikely(sector_offset < 0))
393			return ERR_PTR(sector_offset);
394		if (!sector_offset)
395			return NULL;
396	} else {
397		if (bio->bi_iter.bi_size <= max_len)
398			return NULL;
399		sector_offset = max_len >> SECTOR_SHIFT;
400	}
401
402	/* ensure the split ioend is still block size aligned */
403	sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT,
404			i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT;
405
406	split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset);
407	if (IS_ERR(split))
408		return ERR_CAST(split);
409	split->bi_private = bio->bi_private;
410	split->bi_end_io = bio->bi_end_io;
411
412	split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset,
413			ioend->io_flags);
414	split_ioend->io_parent = ioend;
415
416	atomic_inc(&ioend->io_remaining);
417	ioend->io_offset += split_ioend->io_size;
418	ioend->io_size -= split_ioend->io_size;
419
420	split_ioend->io_sector = ioend->io_sector;
421	if (!is_append)
422		ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT);
423	return split_ioend;
424}
425EXPORT_SYMBOL_GPL(iomap_split_ioend);
426
427static int __init iomap_ioend_init(void)
428{
429	return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
430			   offsetof(struct iomap_ioend, io_bio),
431			   BIOSET_NEED_BVECS);
432}
433fs_initcall(iomap_ioend_init);