blk-merge.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Functions related to segment and merge handling
   4 */
   5#include <linux/kernel.h>
   6#include <linux/module.h>
   7#include <linux/bio.h>
   8#include <linux/blkdev.h>
   9#include <linux/blk-integrity.h>
  10#include <linux/part_stat.h>
  11#include <linux/blk-cgroup.h>
  12
  13#include <trace/events/block.h>
  14
  15#include "blk.h"
  16#include "blk-mq-sched.h"
  17#include "blk-rq-qos.h"
  18#include "blk-throttle.h"
  19
  20static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
  21{
  22	*bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
  23}
  24
  25static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
  26{
  27	struct bvec_iter iter = bio->bi_iter;
  28	int idx;
  29
  30	bio_get_first_bvec(bio, bv);
  31	if (bv->bv_len == bio->bi_iter.bi_size)
  32		return;		/* this bio only has a single bvec */
  33
  34	bio_advance_iter(bio, &iter, iter.bi_size);
  35
  36	if (!iter.bi_bvec_done)
  37		idx = iter.bi_idx - 1;
  38	else	/* in the middle of bvec */
  39		idx = iter.bi_idx;
  40
  41	*bv = bio->bi_io_vec[idx];
  42
  43	/*
  44	 * iter.bi_bvec_done records actual length of the last bvec
  45	 * if this bio ends in the middle of one io vector
  46	 */
  47	if (iter.bi_bvec_done)
  48		bv->bv_len = iter.bi_bvec_done;
  49}
  50
  51static inline bool bio_will_gap(struct request_queue *q,
  52		struct request *prev_rq, struct bio *prev, struct bio *next)
  53{
  54	struct bio_vec pb, nb;
  55
  56	if (!bio_has_data(prev) || !queue_virt_boundary(q))
  57		return false;
  58
  59	/*
  60	 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
  61	 * is quite difficult to respect the sg gap limit.  We work hard to
  62	 * merge a huge number of small single bios in case of mkfs.
  63	 */
  64	if (prev_rq)
  65		bio_get_first_bvec(prev_rq->bio, &pb);
  66	else
  67		bio_get_first_bvec(prev, &pb);
  68	if (pb.bv_offset & queue_virt_boundary(q))
  69		return true;
  70
  71	/*
  72	 * We don't need to worry about the situation that the merged segment
  73	 * ends in unaligned virt boundary:
  74	 *
  75	 * - if 'pb' ends aligned, the merged segment ends aligned
  76	 * - if 'pb' ends unaligned, the next bio must include
  77	 *   one single bvec of 'nb', otherwise the 'nb' can't
  78	 *   merge with 'pb'
  79	 */
  80	bio_get_last_bvec(prev, &pb);
  81	bio_get_first_bvec(next, &nb);
  82	if (biovec_phys_mergeable(q, &pb, &nb))
  83		return false;
  84	return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
  85}
  86
  87static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
  88{
  89	return bio_will_gap(req->q, req, req->biotail, bio);
  90}
  91
  92static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
  93{
  94	return bio_will_gap(req->q, NULL, bio, req->bio);
  95}
  96
  97/*
  98 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
  99 * is defined as 'unsigned int', meantime it has to be aligned to with the
 100 * logical block size, which is the minimum accepted unit by hardware.
 101 */
 102static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim)
 103{
 104	return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
 105}
 106
 107/*
 108 * bio_submit_split_bioset - Submit a bio, splitting it at a designated sector
 109 * @bio:		the original bio to be submitted and split
 110 * @split_sectors:	the sector count at which to split
 111 * @bs:			the bio set used for allocating the new split bio
 112 *
 113 * The original bio is modified to contain the remaining sectors and submitted.
 114 * The caller is responsible for submitting the returned bio.
 115 *
 116 * If succeed, the newly allocated bio representing the initial part will be
 117 * returned, on failure NULL will be returned and original bio will fail.
 118 */
 119struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors,
 120				    struct bio_set *bs)
 121{
 122	struct bio *split = bio_split(bio, split_sectors, GFP_NOIO, bs);
 123
 124	if (IS_ERR(split)) {
 125		bio->bi_status = errno_to_blk_status(PTR_ERR(split));
 126		bio_endio(bio);
 127		return NULL;
 128	}
 129
 130	bio_chain(split, bio);
 131	trace_block_split(split, bio->bi_iter.bi_sector);
 132	WARN_ON_ONCE(bio_zone_write_plugging(bio));
 133
 134	if (should_fail_bio(bio))
 135		bio_io_error(bio);
 136	else if (!blk_throtl_bio(bio))
 137		submit_bio_noacct_nocheck(bio, true);
 138
 139	return split;
 140}
 141EXPORT_SYMBOL_GPL(bio_submit_split_bioset);
 142
 143static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
 144{
 145	if (unlikely(split_sectors < 0)) {
 146		bio->bi_status = errno_to_blk_status(split_sectors);
 147		bio_endio(bio);
 148		return NULL;
 149	}
 150
 151	if (split_sectors) {
 152		bio = bio_submit_split_bioset(bio, split_sectors,
 153				&bio->bi_bdev->bd_disk->bio_split);
 154		if (bio)
 155			bio->bi_opf |= REQ_NOMERGE;
 156	}
 157
 158	return bio;
 159}
 160
 161static struct bio *__bio_split_discard(struct bio *bio,
 162		const struct queue_limits *lim, unsigned *nsegs,
 163		unsigned int max_sectors)
 164{
 165	unsigned int max_discard_sectors, granularity;
 166	sector_t tmp;
 167	unsigned split_sectors;
 168
 169	*nsegs = 1;
 170
 171	granularity = max(lim->discard_granularity >> 9, 1U);
 172
 173	max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim));
 174	max_discard_sectors -= max_discard_sectors % granularity;
 175	if (unlikely(!max_discard_sectors))
 176		return bio;
 177
 178	if (bio_sectors(bio) <= max_discard_sectors)
 179		return bio;
 180
 181	split_sectors = max_discard_sectors;
 182
 183	/*
 184	 * If the next starting sector would be misaligned, stop the discard at
 185	 * the previous aligned sector.
 186	 */
 187	tmp = bio->bi_iter.bi_sector + split_sectors -
 188		((lim->discard_alignment >> 9) % granularity);
 189	tmp = sector_div(tmp, granularity);
 190
 191	if (split_sectors > tmp)
 192		split_sectors -= tmp;
 193
 194	return bio_submit_split(bio, split_sectors);
 195}
 196
 197struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
 198		unsigned *nsegs)
 199{
 200	unsigned int max_sectors;
 201
 202	if (bio_op(bio) == REQ_OP_SECURE_ERASE)
 203		max_sectors = lim->max_secure_erase_sectors;
 204	else
 205		max_sectors = lim->max_discard_sectors;
 206
 207	return __bio_split_discard(bio, lim, nsegs, max_sectors);
 208}
 209
 210static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim,
 211						bool is_atomic)
 212{
 213	/*
 214	 * chunk_sectors must be a multiple of atomic_write_boundary_sectors if
 215	 * both non-zero.
 216	 */
 217	if (is_atomic && lim->atomic_write_boundary_sectors)
 218		return lim->atomic_write_boundary_sectors;
 219
 220	return lim->chunk_sectors;
 221}
 222
 223/*
 224 * Return the maximum number of sectors from the start of a bio that may be
 225 * submitted as a single request to a block device. If enough sectors remain,
 226 * align the end to the physical block size. Otherwise align the end to the
 227 * logical block size. This approach minimizes the number of non-aligned
 228 * requests that are submitted to a block device if the start of a bio is not
 229 * aligned to a physical block boundary.
 230 */
 231static inline unsigned get_max_io_size(struct bio *bio,
 232				       const struct queue_limits *lim)
 233{
 234	unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
 235	unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
 236	bool is_atomic = bio->bi_opf & REQ_ATOMIC;
 237	unsigned boundary_sectors = blk_boundary_sectors(lim, is_atomic);
 238	unsigned max_sectors, start, end;
 239
 240	/*
 241	 * We ignore lim->max_sectors for atomic writes because it may less
 242	 * than the actual bio size, which we cannot tolerate.
 243	 */
 244	if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
 245		max_sectors = lim->max_write_zeroes_sectors;
 246	else if (is_atomic)
 247		max_sectors = lim->atomic_write_max_sectors;
 248	else
 249		max_sectors = lim->max_sectors;
 250
 251	if (boundary_sectors) {
 252		max_sectors = min(max_sectors,
 253			blk_boundary_sectors_left(bio->bi_iter.bi_sector,
 254					      boundary_sectors));
 255	}
 256
 257	start = bio->bi_iter.bi_sector & (pbs - 1);
 258	end = (start + max_sectors) & ~(pbs - 1);
 259	if (end > start)
 260		return end - start;
 261	return max_sectors & ~(lbs - 1);
 262}
 263
 264/**
 265 * bvec_split_segs - verify whether or not a bvec should be split in the middle
 266 * @lim:      [in] queue limits to split based on
 267 * @bv:       [in] bvec to examine
 268 * @nsegs:    [in,out] Number of segments in the bio being built. Incremented
 269 *            by the number of segments from @bv that may be appended to that
 270 *            bio without exceeding @max_segs
 271 * @bytes:    [in,out] Number of bytes in the bio being built. Incremented
 272 *            by the number of bytes from @bv that may be appended to that
 273 *            bio without exceeding @max_bytes
 274 * @max_segs: [in] upper bound for *@nsegs
 275 * @max_bytes: [in] upper bound for *@bytes
 276 *
 277 * When splitting a bio, it can happen that a bvec is encountered that is too
 278 * big to fit in a single segment and hence that it has to be split in the
 279 * middle. This function verifies whether or not that should happen. The value
 280 * %true is returned if and only if appending the entire @bv to a bio with
 281 * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
 282 * the block driver.
 283 */
 284static bool bvec_split_segs(const struct queue_limits *lim,
 285		const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes,
 286		unsigned max_segs, unsigned max_bytes)
 287{
 288	unsigned max_len = max_bytes - *bytes;
 289	unsigned len = min(bv->bv_len, max_len);
 290	unsigned total_len = 0;
 291	unsigned seg_size = 0;
 292
 293	while (len && *nsegs < max_segs) {
 294		seg_size = get_max_segment_size(lim, bvec_phys(bv) + total_len, len);
 295
 296		(*nsegs)++;
 297		total_len += seg_size;
 298		len -= seg_size;
 299
 300		if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
 301			break;
 302	}
 303
 304	*bytes += total_len;
 305
 306	/* tell the caller to split the bvec if it is too big to fit */
 307	return len > 0 || bv->bv_len > max_len;
 308}
 309
 310static unsigned int bio_split_alignment(struct bio *bio,
 311		const struct queue_limits *lim)
 312{
 313	if (op_is_write(bio_op(bio)) && lim->zone_write_granularity)
 314		return lim->zone_write_granularity;
 315	return lim->logical_block_size;
 316}
 317
 318static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv,
 319					struct bio_vec *bv)
 320{
 321	return bv->bv_offset | (bvprv->bv_offset + bvprv->bv_len);
 322}
 323
 324/**
 325 * bio_split_io_at - check if and where to split a bio
 326 * @bio:  [in] bio to be split
 327 * @lim:  [in] queue limits to split based on
 328 * @segs: [out] number of segments in the bio with the first half of the sectors
 329 * @max_bytes: [in] maximum number of bytes per bio
 330 * @len_align_mask: [in] length alignment mask for each vector
 331 *
 332 * Find out if @bio needs to be split to fit the queue limits in @lim and a
 333 * maximum size of @max_bytes.  Returns a negative error number if @bio can't be
 334 * split, 0 if the bio doesn't have to be split, or a positive sector offset if
 335 * @bio needs to be split.
 336 */
 337int bio_split_io_at(struct bio *bio, const struct queue_limits *lim,
 338		unsigned *segs, unsigned max_bytes, unsigned len_align_mask)
 339{
 340	struct bio_vec bv, bvprv, *bvprvp = NULL;
 341	unsigned nsegs = 0, bytes = 0, gaps = 0;
 342	struct bvec_iter iter;
 343
 344	bio_for_each_bvec(bv, bio, iter) {
 345		if (bv.bv_offset & lim->dma_alignment ||
 346		    bv.bv_len & len_align_mask)
 347			return -EINVAL;
 348
 349		/*
 350		 * If the queue doesn't support SG gaps and adding this
 351		 * offset would create a gap, disallow it.
 352		 */
 353		if (bvprvp) {
 354			if (bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
 355				goto split;
 356			gaps |= bvec_seg_gap(bvprvp, &bv);
 357		}
 358
 359		if (nsegs < lim->max_segments &&
 360		    bytes + bv.bv_len <= max_bytes &&
 361		    bv.bv_offset + bv.bv_len <= lim->max_fast_segment_size) {
 362			nsegs++;
 363			bytes += bv.bv_len;
 364		} else {
 365			if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
 366					lim->max_segments, max_bytes))
 367				goto split;
 368		}
 369
 370		bvprv = bv;
 371		bvprvp = &bvprv;
 372	}
 373
 374	*segs = nsegs;
 375	bio->bi_bvec_gap_bit = ffs(gaps);
 376	return 0;
 377split:
 378	if (bio->bi_opf & REQ_ATOMIC)
 379		return -EINVAL;
 380
 381	/*
 382	 * We can't sanely support splitting for a REQ_NOWAIT bio. End it
 383	 * with EAGAIN if splitting is required and return an error pointer.
 384	 */
 385	if (bio->bi_opf & REQ_NOWAIT)
 386		return -EAGAIN;
 387
 388	*segs = nsegs;
 389
 390	/*
 391	 * Individual bvecs might not be logical block aligned. Round down the
 392	 * split size so that each bio is properly block size aligned, even if
 393	 * we do not use the full hardware limits.
 394	 *
 395	 * It is possible to submit a bio that can't be split into a valid io:
 396	 * there may either be too many discontiguous vectors for the max
 397	 * segments limit, or contain virtual boundary gaps without having a
 398	 * valid block sized split. A zero byte result means one of those
 399	 * conditions occured.
 400	 */
 401	bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim));
 402	if (!bytes)
 403		return -EINVAL;
 404
 405	/*
 406	 * Bio splitting may cause subtle trouble such as hang when doing sync
 407	 * iopoll in direct IO routine. Given performance gain of iopoll for
 408	 * big IO can be trival, disable iopoll when split needed.
 409	 */
 410	bio_clear_polled(bio);
 411	bio->bi_bvec_gap_bit = ffs(gaps);
 412	return bytes >> SECTOR_SHIFT;
 413}
 414EXPORT_SYMBOL_GPL(bio_split_io_at);
 415
 416struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
 417		unsigned *nr_segs)
 418{
 419	return bio_submit_split(bio,
 420		bio_split_rw_at(bio, lim, nr_segs,
 421			get_max_io_size(bio, lim) << SECTOR_SHIFT));
 422}
 423
 424/*
 425 * REQ_OP_ZONE_APPEND bios must never be split by the block layer.
 426 *
 427 * But we want the nr_segs calculation provided by bio_split_rw_at, and having
 428 * a good sanity check that the submitter built the bio correctly is nice to
 429 * have as well.
 430 */
 431struct bio *bio_split_zone_append(struct bio *bio,
 432		const struct queue_limits *lim, unsigned *nr_segs)
 433{
 434	int split_sectors;
 435
 436	split_sectors = bio_split_rw_at(bio, lim, nr_segs,
 437			lim->max_zone_append_sectors << SECTOR_SHIFT);
 438	if (WARN_ON_ONCE(split_sectors > 0))
 439		split_sectors = -EINVAL;
 440	return bio_submit_split(bio, split_sectors);
 441}
 442
 443struct bio *bio_split_write_zeroes(struct bio *bio,
 444		const struct queue_limits *lim, unsigned *nsegs)
 445{
 446	unsigned int max_sectors = get_max_io_size(bio, lim);
 447
 448	*nsegs = 0;
 449
 450	/*
 451	 * An unset limit should normally not happen, as bio submission is keyed
 452	 * off having a non-zero limit.  But SCSI can clear the limit in the
 453	 * I/O completion handler, and we can race and see this.  Splitting to a
 454	 * zero limit obviously doesn't make sense, so band-aid it here.
 455	 */
 456	if (!max_sectors)
 457		return bio;
 458	if (bio_sectors(bio) <= max_sectors)
 459		return bio;
 460	return bio_submit_split(bio, max_sectors);
 461}
 462
 463/**
 464 * bio_split_to_limits - split a bio to fit the queue limits
 465 * @bio:     bio to be split
 466 *
 467 * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
 468 * if so split off a bio fitting the limits from the beginning of @bio and
 469 * return it.  @bio is shortened to the remainder and re-submitted.
 470 *
 471 * The split bio is allocated from @q->bio_split, which is provided by the
 472 * block layer.
 473 */
 474struct bio *bio_split_to_limits(struct bio *bio)
 475{
 476	unsigned int nr_segs;
 477
 478	return __bio_split_to_limits(bio, bdev_limits(bio->bi_bdev), &nr_segs);
 479}
 480EXPORT_SYMBOL(bio_split_to_limits);
 481
 482unsigned int blk_recalc_rq_segments(struct request *rq)
 483{
 484	unsigned int nr_phys_segs = 0;
 485	unsigned int bytes = 0;
 486	struct req_iterator iter;
 487	struct bio_vec bv;
 488
 489	if (!rq->bio)
 490		return 0;
 491
 492	switch (bio_op(rq->bio)) {
 493	case REQ_OP_DISCARD:
 494	case REQ_OP_SECURE_ERASE:
 495		if (queue_max_discard_segments(rq->q) > 1) {
 496			struct bio *bio = rq->bio;
 497
 498			for_each_bio(bio)
 499				nr_phys_segs++;
 500			return nr_phys_segs;
 501		}
 502		return 1;
 503	case REQ_OP_WRITE_ZEROES:
 504		return 0;
 505	default:
 506		break;
 507	}
 508
 509	rq_for_each_bvec(bv, rq, iter)
 510		bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
 511				UINT_MAX, UINT_MAX);
 512	return nr_phys_segs;
 513}
 514
 515static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
 516						  sector_t offset)
 517{
 518	struct request_queue *q = rq->q;
 519	struct queue_limits *lim = &q->limits;
 520	unsigned int max_sectors, boundary_sectors;
 521	bool is_atomic = rq->cmd_flags & REQ_ATOMIC;
 522
 523	if (blk_rq_is_passthrough(rq))
 524		return q->limits.max_hw_sectors;
 525
 526	boundary_sectors = blk_boundary_sectors(lim, is_atomic);
 527	max_sectors = blk_queue_get_max_sectors(rq);
 528
 529	if (!boundary_sectors ||
 530	    req_op(rq) == REQ_OP_DISCARD ||
 531	    req_op(rq) == REQ_OP_SECURE_ERASE)
 532		return max_sectors;
 533	return min(max_sectors,
 534		   blk_boundary_sectors_left(offset, boundary_sectors));
 535}
 536
 537static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
 538		unsigned int nr_phys_segs)
 539{
 540	if (!blk_cgroup_mergeable(req, bio))
 541		goto no_merge;
 542
 543	if (blk_integrity_merge_bio(req->q, req, bio) == false)
 544		goto no_merge;
 545
 546	/* discard request merge won't add new segment */
 547	if (req_op(req) == REQ_OP_DISCARD)
 548		return 1;
 549
 550	if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))
 551		goto no_merge;
 552
 553	/*
 554	 * This will form the start of a new hw segment.  Bump both
 555	 * counters.
 556	 */
 557	req->nr_phys_segments += nr_phys_segs;
 558	if (bio_integrity(bio))
 559		req->nr_integrity_segments += blk_rq_count_integrity_sg(req->q,
 560									bio);
 561	return 1;
 562
 563no_merge:
 564	req_set_nomerge(req->q, req);
 565	return 0;
 566}
 567
 568int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
 569{
 570	if (req_gap_back_merge(req, bio))
 571		return 0;
 572	if (blk_integrity_rq(req) &&
 573	    integrity_req_gap_back_merge(req, bio))
 574		return 0;
 575	if (!bio_crypt_ctx_back_mergeable(req, bio))
 576		return 0;
 577	if (blk_rq_sectors(req) + bio_sectors(bio) >
 578	    blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
 579		req_set_nomerge(req->q, req);
 580		return 0;
 581	}
 582
 583	return ll_new_hw_segment(req, bio, nr_segs);
 584}
 585
 586static int ll_front_merge_fn(struct request *req, struct bio *bio,
 587		unsigned int nr_segs)
 588{
 589	if (req_gap_front_merge(req, bio))
 590		return 0;
 591	if (blk_integrity_rq(req) &&
 592	    integrity_req_gap_front_merge(req, bio))
 593		return 0;
 594	if (!bio_crypt_ctx_front_mergeable(req, bio))
 595		return 0;
 596	if (blk_rq_sectors(req) + bio_sectors(bio) >
 597	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
 598		req_set_nomerge(req->q, req);
 599		return 0;
 600	}
 601
 602	return ll_new_hw_segment(req, bio, nr_segs);
 603}
 604
 605static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
 606		struct request *next)
 607{
 608	unsigned short segments = blk_rq_nr_discard_segments(req);
 609
 610	if (segments >= queue_max_discard_segments(q))
 611		goto no_merge;
 612	if (blk_rq_sectors(req) + bio_sectors(next->bio) >
 613	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
 614		goto no_merge;
 615
 616	req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
 617	return true;
 618no_merge:
 619	req_set_nomerge(q, req);
 620	return false;
 621}
 622
 623static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
 624				struct request *next)
 625{
 626	int total_phys_segments;
 627
 628	if (req_gap_back_merge(req, next->bio))
 629		return 0;
 630
 631	/*
 632	 * Will it become too large?
 633	 */
 634	if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
 635	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
 636		return 0;
 637
 638	total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
 639	if (total_phys_segments > blk_rq_get_max_segments(req))
 640		return 0;
 641
 642	if (!blk_cgroup_mergeable(req, next->bio))
 643		return 0;
 644
 645	if (blk_integrity_merge_rq(q, req, next) == false)
 646		return 0;
 647
 648	if (!bio_crypt_ctx_merge_rq(req, next))
 649		return 0;
 650
 651	/* Merge is OK... */
 652	req->nr_phys_segments = total_phys_segments;
 653	req->nr_integrity_segments += next->nr_integrity_segments;
 654	return 1;
 655}
 656
 657/**
 658 * blk_rq_set_mixed_merge - mark a request as mixed merge
 659 * @rq: request to mark as mixed merge
 660 *
 661 * Description:
 662 *     @rq is about to be mixed merged.  Make sure the attributes
 663 *     which can be mixed are set in each bio and mark @rq as mixed
 664 *     merged.
 665 */
 666static void blk_rq_set_mixed_merge(struct request *rq)
 667{
 668	blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
 669	struct bio *bio;
 670
 671	if (rq->rq_flags & RQF_MIXED_MERGE)
 672		return;
 673
 674	/*
 675	 * @rq will no longer represent mixable attributes for all the
 676	 * contained bios.  It will just track those of the first one.
 677	 * Distributes the attributs to each bio.
 678	 */
 679	for (bio = rq->bio; bio; bio = bio->bi_next) {
 680		WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
 681			     (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
 682		bio->bi_opf |= ff;
 683	}
 684	rq->rq_flags |= RQF_MIXED_MERGE;
 685}
 686
 687static inline blk_opf_t bio_failfast(const struct bio *bio)
 688{
 689	if (bio->bi_opf & REQ_RAHEAD)
 690		return REQ_FAILFAST_MASK;
 691
 692	return bio->bi_opf & REQ_FAILFAST_MASK;
 693}
 694
 695/*
 696 * After we are marked as MIXED_MERGE, any new RA bio has to be updated
 697 * as failfast, and request's failfast has to be updated in case of
 698 * front merge.
 699 */
 700static inline void blk_update_mixed_merge(struct request *req,
 701		struct bio *bio, bool front_merge)
 702{
 703	if (req->rq_flags & RQF_MIXED_MERGE) {
 704		if (bio->bi_opf & REQ_RAHEAD)
 705			bio->bi_opf |= REQ_FAILFAST_MASK;
 706
 707		if (front_merge) {
 708			req->cmd_flags &= ~REQ_FAILFAST_MASK;
 709			req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK;
 710		}
 711	}
 712}
 713
 714static void blk_account_io_merge_request(struct request *req)
 715{
 716	if (req->rq_flags & RQF_IO_STAT) {
 717		part_stat_lock();
 718		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
 719		part_stat_local_dec(req->part,
 720				    in_flight[op_is_write(req_op(req))]);
 721		part_stat_unlock();
 722	}
 723}
 724
 725static enum elv_merge blk_try_req_merge(struct request *req,
 726					struct request *next)
 727{
 728	if (blk_discard_mergable(req))
 729		return ELEVATOR_DISCARD_MERGE;
 730	else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
 731		return ELEVATOR_BACK_MERGE;
 732
 733	return ELEVATOR_NO_MERGE;
 734}
 735
 736static bool blk_atomic_write_mergeable_rq_bio(struct request *rq,
 737					      struct bio *bio)
 738{
 739	return (rq->cmd_flags & REQ_ATOMIC) == (bio->bi_opf & REQ_ATOMIC);
 740}
 741
 742static bool blk_atomic_write_mergeable_rqs(struct request *rq,
 743					   struct request *next)
 744{
 745	return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC);
 746}
 747
 748u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next,
 749	       u8 gaps_bit)
 750{
 751	struct bio_vec pb, nb;
 752
 753	if (!bio_has_data(prev))
 754		return 0;
 755
 756	gaps_bit = min_not_zero(gaps_bit, prev->bi_bvec_gap_bit);
 757	gaps_bit = min_not_zero(gaps_bit, next->bi_bvec_gap_bit);
 758
 759	bio_get_last_bvec(prev, &pb);
 760	bio_get_first_bvec(next, &nb);
 761	if (!biovec_phys_mergeable(q, &pb, &nb))
 762		gaps_bit = min_not_zero(gaps_bit, ffs(bvec_seg_gap(&pb, &nb)));
 763	return gaps_bit;
 764}
 765
 766/*
 767 * For non-mq, this has to be called with the request spinlock acquired.
 768 * For mq with scheduling, the appropriate queue wide lock should be held.
 769 */
 770static struct request *attempt_merge(struct request_queue *q,
 771				     struct request *req, struct request *next)
 772{
 773	if (!rq_mergeable(req) || !rq_mergeable(next))
 774		return NULL;
 775
 776	if (req_op(req) != req_op(next))
 777		return NULL;
 778
 779	if (req->bio->bi_write_hint != next->bio->bi_write_hint)
 780		return NULL;
 781	if (req->bio->bi_write_stream != next->bio->bi_write_stream)
 782		return NULL;
 783	if (req->bio->bi_ioprio != next->bio->bi_ioprio)
 784		return NULL;
 785	if (!blk_atomic_write_mergeable_rqs(req, next))
 786		return NULL;
 787
 788	/*
 789	 * If we are allowed to merge, then append bio list
 790	 * from next to rq and release next. merge_requests_fn
 791	 * will have updated segment counts, update sector
 792	 * counts here. Handle DISCARDs separately, as they
 793	 * have separate settings.
 794	 */
 795
 796	switch (blk_try_req_merge(req, next)) {
 797	case ELEVATOR_DISCARD_MERGE:
 798		if (!req_attempt_discard_merge(q, req, next))
 799			return NULL;
 800		break;
 801	case ELEVATOR_BACK_MERGE:
 802		if (!ll_merge_requests_fn(q, req, next))
 803			return NULL;
 804		break;
 805	default:
 806		return NULL;
 807	}
 808
 809	/*
 810	 * If failfast settings disagree or any of the two is already
 811	 * a mixed merge, mark both as mixed before proceeding.  This
 812	 * makes sure that all involved bios have mixable attributes
 813	 * set properly.
 814	 */
 815	if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
 816	    (req->cmd_flags & REQ_FAILFAST_MASK) !=
 817	    (next->cmd_flags & REQ_FAILFAST_MASK)) {
 818		blk_rq_set_mixed_merge(req);
 819		blk_rq_set_mixed_merge(next);
 820	}
 821
 822	/*
 823	 * At this point we have either done a back merge or front merge. We
 824	 * need the smaller start_time_ns of the merged requests to be the
 825	 * current request for accounting purposes.
 826	 */
 827	if (next->start_time_ns < req->start_time_ns)
 828		req->start_time_ns = next->start_time_ns;
 829
 830	req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, next->bio,
 831					min_not_zero(next->phys_gap_bit,
 832						     req->phys_gap_bit));
 833	req->biotail->bi_next = next->bio;
 834	req->biotail = next->biotail;
 835
 836	req->__data_len += blk_rq_bytes(next);
 837
 838	if (!blk_discard_mergable(req))
 839		elv_merge_requests(q, req, next);
 840
 841	blk_crypto_rq_put_keyslot(next);
 842
 843	/*
 844	 * 'next' is going away, so update stats accordingly
 845	 */
 846	blk_account_io_merge_request(next);
 847
 848	trace_block_rq_merge(next);
 849
 850	/*
 851	 * ownership of bio passed from next to req, return 'next' for
 852	 * the caller to free
 853	 */
 854	next->bio = NULL;
 855	return next;
 856}
 857
 858static struct request *attempt_back_merge(struct request_queue *q,
 859		struct request *rq)
 860{
 861	struct request *next = elv_latter_request(q, rq);
 862
 863	if (next)
 864		return attempt_merge(q, rq, next);
 865
 866	return NULL;
 867}
 868
 869static struct request *attempt_front_merge(struct request_queue *q,
 870		struct request *rq)
 871{
 872	struct request *prev = elv_former_request(q, rq);
 873
 874	if (prev)
 875		return attempt_merge(q, prev, rq);
 876
 877	return NULL;
 878}
 879
 880/*
 881 * Try to merge 'next' into 'rq'. Return true if the merge happened, false
 882 * otherwise. The caller is responsible for freeing 'next' if the merge
 883 * happened.
 884 */
 885bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
 886			   struct request *next)
 887{
 888	return attempt_merge(q, rq, next);
 889}
 890
 891bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
 892{
 893	if (!rq_mergeable(rq) || !bio_mergeable(bio))
 894		return false;
 895
 896	if (req_op(rq) != bio_op(bio))
 897		return false;
 898
 899	if (!blk_cgroup_mergeable(rq, bio))
 900		return false;
 901	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
 902		return false;
 903	if (!bio_crypt_rq_ctx_compatible(rq, bio))
 904		return false;
 905	if (rq->bio->bi_write_hint != bio->bi_write_hint)
 906		return false;
 907	if (rq->bio->bi_write_stream != bio->bi_write_stream)
 908		return false;
 909	if (rq->bio->bi_ioprio != bio->bi_ioprio)
 910		return false;
 911	if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false)
 912		return false;
 913
 914	return true;
 915}
 916
 917enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
 918{
 919	if (blk_discard_mergable(rq))
 920		return ELEVATOR_DISCARD_MERGE;
 921	else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
 922		return ELEVATOR_BACK_MERGE;
 923	else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
 924		return ELEVATOR_FRONT_MERGE;
 925	return ELEVATOR_NO_MERGE;
 926}
 927
 928static void blk_account_io_merge_bio(struct request *req)
 929{
 930	if (req->rq_flags & RQF_IO_STAT) {
 931		part_stat_lock();
 932		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
 933		part_stat_unlock();
 934	}
 935}
 936
 937enum bio_merge_status bio_attempt_back_merge(struct request *req,
 938		struct bio *bio, unsigned int nr_segs)
 939{
 940	const blk_opf_t ff = bio_failfast(bio);
 941
 942	if (!ll_back_merge_fn(req, bio, nr_segs))
 943		return BIO_MERGE_FAILED;
 944
 945	trace_block_bio_backmerge(bio);
 946	rq_qos_merge(req->q, req, bio);
 947
 948	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
 949		blk_rq_set_mixed_merge(req);
 950
 951	blk_update_mixed_merge(req, bio, false);
 952
 953	if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
 954		blk_zone_write_plug_bio_merged(bio);
 955
 956	req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, bio,
 957					req->phys_gap_bit);
 958	req->biotail->bi_next = bio;
 959	req->biotail = bio;
 960	req->__data_len += bio->bi_iter.bi_size;
 961
 962	bio_crypt_free_ctx(bio);
 963
 964	blk_account_io_merge_bio(req);
 965	return BIO_MERGE_OK;
 966}
 967
 968static enum bio_merge_status bio_attempt_front_merge(struct request *req,
 969		struct bio *bio, unsigned int nr_segs)
 970{
 971	const blk_opf_t ff = bio_failfast(bio);
 972
 973	/*
 974	 * A front merge for writes to sequential zones of a zoned block device
 975	 * can happen only if the user submitted writes out of order. Do not
 976	 * merge such write to let it fail.
 977	 */
 978	if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
 979		return BIO_MERGE_FAILED;
 980
 981	if (!ll_front_merge_fn(req, bio, nr_segs))
 982		return BIO_MERGE_FAILED;
 983
 984	trace_block_bio_frontmerge(bio);
 985	rq_qos_merge(req->q, req, bio);
 986
 987	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
 988		blk_rq_set_mixed_merge(req);
 989
 990	blk_update_mixed_merge(req, bio, true);
 991
 992	req->phys_gap_bit = bio_seg_gap(req->q, bio, req->bio,
 993					req->phys_gap_bit);
 994	bio->bi_next = req->bio;
 995	req->bio = bio;
 996
 997	req->__sector = bio->bi_iter.bi_sector;
 998	req->__data_len += bio->bi_iter.bi_size;
 999
1000	bio_crypt_do_front_merge(req, bio);
1001
1002	blk_account_io_merge_bio(req);
1003	return BIO_MERGE_OK;
1004}
1005
1006static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
1007		struct request *req, struct bio *bio)
1008{
1009	unsigned short segments = blk_rq_nr_discard_segments(req);
1010
1011	if (segments >= queue_max_discard_segments(q))
1012		goto no_merge;
1013	if (blk_rq_sectors(req) + bio_sectors(bio) >
1014	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1015		goto no_merge;
1016
1017	rq_qos_merge(q, req, bio);
1018
1019	req->biotail->bi_next = bio;
1020	req->biotail = bio;
1021	req->__data_len += bio->bi_iter.bi_size;
1022	req->nr_phys_segments = segments + 1;
1023
1024	blk_account_io_merge_bio(req);
1025	return BIO_MERGE_OK;
1026no_merge:
1027	req_set_nomerge(q, req);
1028	return BIO_MERGE_FAILED;
1029}
1030
1031static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
1032						   struct request *rq,
1033						   struct bio *bio,
1034						   unsigned int nr_segs,
1035						   bool sched_allow_merge)
1036{
1037	if (!blk_rq_merge_ok(rq, bio))
1038		return BIO_MERGE_NONE;
1039
1040	switch (blk_try_merge(rq, bio)) {
1041	case ELEVATOR_BACK_MERGE:
1042		if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1043			return bio_attempt_back_merge(rq, bio, nr_segs);
1044		break;
1045	case ELEVATOR_FRONT_MERGE:
1046		if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1047			return bio_attempt_front_merge(rq, bio, nr_segs);
1048		break;
1049	case ELEVATOR_DISCARD_MERGE:
1050		return bio_attempt_discard_merge(q, rq, bio);
1051	default:
1052		return BIO_MERGE_NONE;
1053	}
1054
1055	return BIO_MERGE_FAILED;
1056}
1057
1058/**
1059 * blk_attempt_plug_merge - try to merge with %current's plugged list
1060 * @q: request_queue new bio is being queued at
1061 * @bio: new bio being queued
1062 * @nr_segs: number of segments in @bio
1063 * from the passed in @q already in the plug list
1064 *
1065 * Determine whether @bio being queued on @q can be merged with the previous
1066 * request on %current's plugged list.  Returns %true if merge was successful,
1067 * otherwise %false.
1068 *
1069 * Plugging coalesces IOs from the same issuer for the same purpose without
1070 * going through @q->queue_lock.  As such it's more of an issuing mechanism
1071 * than scheduling, and the request, while may have elvpriv data, is not
1072 * added on the elevator at this point.  In addition, we don't have
1073 * reliable access to the elevator outside queue lock.  Only check basic
1074 * merging parameters without querying the elevator.
1075 *
1076 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1077 */
1078bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1079		unsigned int nr_segs)
1080{
1081	struct blk_plug *plug = current->plug;
1082	struct request *rq;
1083
1084	if (!plug || rq_list_empty(&plug->mq_list))
1085		return false;
1086
1087	rq = plug->mq_list.tail;
1088	if (rq->q == q)
1089		return blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1090			BIO_MERGE_OK;
1091	else if (!plug->multiple_queues)
1092		return false;
1093
1094	rq_list_for_each(&plug->mq_list, rq) {
1095		if (rq->q != q)
1096			continue;
1097		if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1098		    BIO_MERGE_OK)
1099			return true;
1100		break;
1101	}
1102	return false;
1103}
1104
1105/*
1106 * Iterate list of requests and see if we can merge this bio with any
1107 * of them.
1108 */
1109bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
1110			struct bio *bio, unsigned int nr_segs)
1111{
1112	struct request *rq;
1113	int checked = 8;
1114
1115	list_for_each_entry_reverse(rq, list, queuelist) {
1116		if (!checked--)
1117			break;
1118
1119		switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
1120		case BIO_MERGE_NONE:
1121			continue;
1122		case BIO_MERGE_OK:
1123			return true;
1124		case BIO_MERGE_FAILED:
1125			return false;
1126		}
1127
1128	}
1129
1130	return false;
1131}
1132EXPORT_SYMBOL_GPL(blk_bio_list_merge);
1133
1134bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
1135		unsigned int nr_segs, struct request **merged_request)
1136{
1137	struct request *rq;
1138
1139	switch (elv_merge(q, &rq, bio)) {
1140	case ELEVATOR_BACK_MERGE:
1141		if (!blk_mq_sched_allow_merge(q, rq, bio))
1142			return false;
1143		if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1144			return false;
1145		*merged_request = attempt_back_merge(q, rq);
1146		if (!*merged_request)
1147			elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
1148		return true;
1149	case ELEVATOR_FRONT_MERGE:
1150		if (!blk_mq_sched_allow_merge(q, rq, bio))
1151			return false;
1152		if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1153			return false;
1154		*merged_request = attempt_front_merge(q, rq);
1155		if (!*merged_request)
1156			elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
1157		return true;
1158	case ELEVATOR_DISCARD_MERGE:
1159		return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
1160	default:
1161		return false;
1162	}
1163}
1164EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);