blk.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef BLK_INTERNAL_H
  3#define BLK_INTERNAL_H
  4
  5#include <linux/bio-integrity.h>
  6#include <linux/blk-crypto.h>
  7#include <linux/lockdep.h>
  8#include <linux/memblock.h>	/* for max_pfn/max_low_pfn */
  9#include <linux/sched/sysctl.h>
 10#include <linux/timekeeping.h>
 11#include <xen/xen.h>
 12#include "blk-crypto-internal.h"
 13
 14struct elv_change_ctx;
 15
 16/*
 17 * Default upper limit for the software max_sectors limit used for regular I/Os.
 18 * This can be increased through sysfs.
 19 *
 20 * This should not be confused with the max_hw_sector limit that is entirely
 21 * controlled by the block device driver, usually based on hardware limits.
 22 */
 23#define BLK_DEF_MAX_SECTORS_CAP	(SZ_4M >> SECTOR_SHIFT)
 24
 25#define	BLK_DEV_MAX_SECTORS	(LLONG_MAX >> 9)
 26#define	BLK_MIN_SEGMENT_SIZE	4096
 27
 28/* Max future timer expiry for timeouts */
 29#define BLK_MAX_TIMEOUT		(5 * HZ)
 30
 31extern const struct kobj_type blk_queue_ktype;
 32extern struct dentry *blk_debugfs_root;
 33
 34struct blk_flush_queue {
 35	spinlock_t		mq_flush_lock;
 36	unsigned int		flush_pending_idx:1;
 37	unsigned int		flush_running_idx:1;
 38	blk_status_t 		rq_status;
 39	unsigned long		flush_pending_since;
 40	struct list_head	flush_queue[2];
 41	unsigned long		flush_data_in_flight;
 42	struct request		*flush_rq;
 43	struct rcu_head		rcu_head;
 44};
 45
 46bool is_flush_rq(struct request *req);
 47
 48struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
 49					      gfp_t flags);
 50void blk_free_flush_queue(struct blk_flush_queue *q);
 51
 52bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
 53bool blk_queue_start_drain(struct request_queue *q);
 54bool __blk_freeze_queue_start(struct request_queue *q,
 55			      struct task_struct *owner);
 56int __bio_queue_enter(struct request_queue *q, struct bio *bio);
 57void submit_bio_noacct_nocheck(struct bio *bio, bool split);
 58void bio_await_chain(struct bio *bio);
 59
 60static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
 61{
 62	rcu_read_lock();
 63	if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter))
 64		goto fail;
 65
 66	/*
 67	 * The code that increments the pm_only counter must ensure that the
 68	 * counter is globally visible before the queue is unfrozen.
 69	 */
 70	if (blk_queue_pm_only(q) &&
 71	    (!pm || queue_rpm_status(q) == RPM_SUSPENDED))
 72		goto fail_put;
 73
 74	rcu_read_unlock();
 75	return true;
 76
 77fail_put:
 78	blk_queue_exit(q);
 79fail:
 80	rcu_read_unlock();
 81	return false;
 82}
 83
 84static inline int bio_queue_enter(struct bio *bio)
 85{
 86	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 87
 88	if (blk_try_enter_queue(q, false)) {
 89		rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);
 90		rwsem_release(&q->io_lockdep_map, _RET_IP_);
 91		return 0;
 92	}
 93	return __bio_queue_enter(q, bio);
 94}
 95
 96static inline void blk_wait_io(struct completion *done)
 97{
 98	/* Prevent hang_check timer from firing at us during very long I/O */
 99	unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
100
101	if (timeout)
102		while (!wait_for_completion_io_timeout(done, timeout))
103			;
104	else
105		wait_for_completion_io(done);
106}
107
108struct block_device *blkdev_get_no_open(dev_t dev, bool autoload);
109void blkdev_put_no_open(struct block_device *bdev);
110
111#define BIO_INLINE_VECS 4
112struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
113		gfp_t gfp_mask);
114void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
115
116bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
117		struct page *page, unsigned len, unsigned offset);
118
119static inline bool biovec_phys_mergeable(struct request_queue *q,
120		struct bio_vec *vec1, struct bio_vec *vec2)
121{
122	unsigned long mask = queue_segment_boundary(q);
123	phys_addr_t addr1 = bvec_phys(vec1);
124	phys_addr_t addr2 = bvec_phys(vec2);
125
126	/*
127	 * Merging adjacent physical pages may not work correctly under KMSAN
128	 * if their metadata pages aren't adjacent. Just disable merging.
129	 */
130	if (IS_ENABLED(CONFIG_KMSAN))
131		return false;
132
133	if (addr1 + vec1->bv_len != addr2)
134		return false;
135	if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
136		return false;
137	if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask))
138		return false;
139	return true;
140}
141
142static inline bool __bvec_gap_to_prev(const struct queue_limits *lim,
143		struct bio_vec *bprv, unsigned int offset)
144{
145	return (offset & lim->virt_boundary_mask) ||
146		((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask);
147}
148
149/*
150 * Check if adding a bio_vec after bprv with offset would create a gap in
151 * the SG list. Most drivers don't care about this, but some do.
152 */
153static inline bool bvec_gap_to_prev(const struct queue_limits *lim,
154		struct bio_vec *bprv, unsigned int offset)
155{
156	if (!lim->virt_boundary_mask)
157		return false;
158	return __bvec_gap_to_prev(lim, bprv, offset);
159}
160
161static inline bool rq_mergeable(struct request *rq)
162{
163	if (blk_rq_is_passthrough(rq))
164		return false;
165
166	if (req_op(rq) == REQ_OP_FLUSH)
167		return false;
168
169	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
170		return false;
171
172	if (req_op(rq) == REQ_OP_ZONE_APPEND)
173		return false;
174
175	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
176		return false;
177	if (rq->rq_flags & RQF_NOMERGE_FLAGS)
178		return false;
179
180	return true;
181}
182
183/*
184 * There are two different ways to handle DISCARD merges:
185 *  1) If max_discard_segments > 1, the driver treats every bio as a range and
186 *     send the bios to controller together. The ranges don't need to be
187 *     contiguous.
188 *  2) Otherwise, the request will be normal read/write requests.  The ranges
189 *     need to be contiguous.
190 */
191static inline bool blk_discard_mergable(struct request *req)
192{
193	if (req_op(req) == REQ_OP_DISCARD &&
194	    queue_max_discard_segments(req->q) > 1)
195		return true;
196	return false;
197}
198
199static inline unsigned int blk_rq_get_max_segments(struct request *rq)
200{
201	if (req_op(rq) == REQ_OP_DISCARD)
202		return queue_max_discard_segments(rq->q);
203	return queue_max_segments(rq->q);
204}
205
206static inline unsigned int blk_queue_get_max_sectors(struct request *rq)
207{
208	struct request_queue *q = rq->q;
209	enum req_op op = req_op(rq);
210
211	if (unlikely(op == REQ_OP_DISCARD))
212		return min(q->limits.max_discard_sectors,
213			   UINT_MAX >> SECTOR_SHIFT);
214
215	if (unlikely(op == REQ_OP_SECURE_ERASE))
216		return min(q->limits.max_secure_erase_sectors,
217			   UINT_MAX >> SECTOR_SHIFT);
218
219	if (unlikely(op == REQ_OP_WRITE_ZEROES))
220		return q->limits.max_write_zeroes_sectors;
221
222	if (rq->cmd_flags & REQ_ATOMIC)
223		return q->limits.atomic_write_max_sectors;
224
225	return q->limits.max_sectors;
226}
227
228#ifdef CONFIG_BLK_DEV_INTEGRITY
229void blk_flush_integrity(void);
230void bio_integrity_free(struct bio *bio);
231
232/*
233 * Integrity payloads can either be owned by the submitter, in which case
234 * bio_uninit will free them, or owned and generated by the block layer,
235 * in which case we'll verify them here (for reads) and free them before
236 * the bio is handed back to the submitted.
237 */
238bool __bio_integrity_endio(struct bio *bio);
239static inline bool bio_integrity_endio(struct bio *bio)
240{
241	struct bio_integrity_payload *bip = bio_integrity(bio);
242
243	if (bip && (bip->bip_flags & BIP_BLOCK_INTEGRITY))
244		return __bio_integrity_endio(bio);
245	return true;
246}
247
248bool blk_integrity_merge_rq(struct request_queue *, struct request *,
249		struct request *);
250bool blk_integrity_merge_bio(struct request_queue *, struct request *,
251		struct bio *);
252
253static inline bool integrity_req_gap_back_merge(struct request *req,
254		struct bio *next)
255{
256	struct bio_integrity_payload *bip = bio_integrity(req->bio);
257	struct bio_integrity_payload *bip_next = bio_integrity(next);
258
259	return bvec_gap_to_prev(&req->q->limits,
260				&bip->bip_vec[bip->bip_vcnt - 1],
261				bip_next->bip_vec[0].bv_offset);
262}
263
264static inline bool integrity_req_gap_front_merge(struct request *req,
265		struct bio *bio)
266{
267	struct bio_integrity_payload *bip = bio_integrity(bio);
268	struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
269
270	return bvec_gap_to_prev(&req->q->limits,
271				&bip->bip_vec[bip->bip_vcnt - 1],
272				bip_next->bip_vec[0].bv_offset);
273}
274
275extern const struct attribute_group blk_integrity_attr_group;
276#else /* CONFIG_BLK_DEV_INTEGRITY */
277static inline bool blk_integrity_merge_rq(struct request_queue *rq,
278		struct request *r1, struct request *r2)
279{
280	return true;
281}
282static inline bool blk_integrity_merge_bio(struct request_queue *rq,
283		struct request *r, struct bio *b)
284{
285	return true;
286}
287static inline bool integrity_req_gap_back_merge(struct request *req,
288		struct bio *next)
289{
290	return false;
291}
292static inline bool integrity_req_gap_front_merge(struct request *req,
293		struct bio *bio)
294{
295	return false;
296}
297
298static inline void blk_flush_integrity(void)
299{
300}
301static inline bool bio_integrity_endio(struct bio *bio)
302{
303	return true;
304}
305static inline void bio_integrity_free(struct bio *bio)
306{
307}
308#endif /* CONFIG_BLK_DEV_INTEGRITY */
309
310unsigned long blk_rq_timeout(unsigned long timeout);
311void blk_add_timer(struct request *req);
312
313enum bio_merge_status {
314	BIO_MERGE_OK,
315	BIO_MERGE_NONE,
316	BIO_MERGE_FAILED,
317};
318
319enum bio_merge_status bio_attempt_back_merge(struct request *req,
320		struct bio *bio, unsigned int nr_segs);
321bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
322		unsigned int nr_segs);
323bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
324			struct bio *bio, unsigned int nr_segs);
325
326/*
327 * Plug flush limits
328 */
329#define BLK_MAX_REQUEST_COUNT	32
330#define BLK_PLUG_FLUSH_SIZE	(128 * 1024)
331
332/*
333 * Internal elevator interface
334 */
335#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
336
337bool blk_insert_flush(struct request *rq);
338
339void elv_update_nr_hw_queues(struct request_queue *q,
340		struct elv_change_ctx *ctx);
341void elevator_set_default(struct request_queue *q);
342void elevator_set_none(struct request_queue *q);
343
344ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
345		char *buf);
346ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
347		char *buf);
348ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,
349		char *buf);
350ssize_t part_fail_show(struct device *dev, struct device_attribute *attr,
351		char *buf);
352ssize_t part_fail_store(struct device *dev, struct device_attribute *attr,
353		const char *buf, size_t count);
354ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
355ssize_t part_timeout_store(struct device *, struct device_attribute *,
356				const char *, size_t);
357
358struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
359		unsigned *nsegs);
360struct bio *bio_split_write_zeroes(struct bio *bio,
361		const struct queue_limits *lim, unsigned *nsegs);
362struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
363		unsigned *nr_segs);
364struct bio *bio_split_zone_append(struct bio *bio,
365		const struct queue_limits *lim, unsigned *nr_segs);
366
367/*
368 * All drivers must accept single-segments bios that are smaller than PAGE_SIZE.
369 *
370 * This is a quick and dirty check that relies on the fact that bi_io_vec[0] is
371 * always valid if a bio has data.  The check might lead to occasional false
372 * positives when bios are cloned, but compared to the performance impact of
373 * cloned bios themselves the loop below doesn't matter anyway.
374 */
375static inline bool bio_may_need_split(struct bio *bio,
376		const struct queue_limits *lim)
377{
378	if (lim->chunk_sectors)
379		return true;
380	if (bio->bi_vcnt != 1)
381		return true;
382	return bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset >
383		lim->max_fast_segment_size;
384}
385
386/**
387 * __bio_split_to_limits - split a bio to fit the queue limits
388 * @bio:     bio to be split
389 * @lim:     queue limits to split based on
390 * @nr_segs: returns the number of segments in the returned bio
391 *
392 * Check if @bio needs splitting based on the queue limits, and if so split off
393 * a bio fitting the limits from the beginning of @bio and return it.  @bio is
394 * shortened to the remainder and re-submitted.
395 *
396 * The split bio is allocated from @q->bio_split, which is provided by the
397 * block layer.
398 */
399static inline struct bio *__bio_split_to_limits(struct bio *bio,
400		const struct queue_limits *lim, unsigned int *nr_segs)
401{
402	switch (bio_op(bio)) {
403	case REQ_OP_READ:
404	case REQ_OP_WRITE:
405		if (bio_may_need_split(bio, lim))
406			return bio_split_rw(bio, lim, nr_segs);
407		*nr_segs = 1;
408		return bio;
409	case REQ_OP_ZONE_APPEND:
410		return bio_split_zone_append(bio, lim, nr_segs);
411	case REQ_OP_DISCARD:
412	case REQ_OP_SECURE_ERASE:
413		return bio_split_discard(bio, lim, nr_segs);
414	case REQ_OP_WRITE_ZEROES:
415		return bio_split_write_zeroes(bio, lim, nr_segs);
416	default:
417		/* other operations can't be split */
418		*nr_segs = 0;
419		return bio;
420	}
421}
422
423/**
424 * get_max_segment_size() - maximum number of bytes to add as a single segment
425 * @lim: Request queue limits.
426 * @paddr: address of the range to add
427 * @len: maximum length available to add at @paddr
428 *
429 * Returns the maximum number of bytes of the range starting at @paddr that can
430 * be added to a single segment.
431 */
432static inline unsigned get_max_segment_size(const struct queue_limits *lim,
433		phys_addr_t paddr, unsigned int len)
434{
435	/*
436	 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1
437	 * after having calculated the minimum.
438	 */
439	return min_t(unsigned long, len,
440		min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr),
441		    (unsigned long)lim->max_segment_size - 1) + 1);
442}
443
444int ll_back_merge_fn(struct request *req, struct bio *bio,
445		unsigned int nr_segs);
446bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
447				struct request *next);
448unsigned int blk_recalc_rq_segments(struct request *rq);
449bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
450enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
451
452int blk_set_default_limits(struct queue_limits *lim);
453void blk_apply_bdi_limits(struct backing_dev_info *bdi,
454		struct queue_limits *lim);
455int blk_dev_init(void);
456
457void update_io_ticks(struct block_device *part, unsigned long now, bool end);
458
459static inline void req_set_nomerge(struct request_queue *q, struct request *req)
460{
461	req->cmd_flags |= REQ_NOMERGE;
462	if (req == q->last_merge)
463		q->last_merge = NULL;
464}
465
466/*
467 * Internal io_context interface
468 */
469struct io_cq *ioc_find_get_icq(struct request_queue *q);
470struct io_cq *ioc_lookup_icq(struct request_queue *q);
471#ifdef CONFIG_BLK_ICQ
472void ioc_clear_queue(struct request_queue *q);
473#else
474static inline void ioc_clear_queue(struct request_queue *q)
475{
476}
477#endif /* CONFIG_BLK_ICQ */
478
479#ifdef CONFIG_BLK_DEV_ZONED
480void disk_init_zone_resources(struct gendisk *disk);
481void disk_free_zone_resources(struct gendisk *disk);
482static inline bool bio_zone_write_plugging(struct bio *bio)
483{
484	return bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING);
485}
486static inline bool blk_req_bio_is_zone_append(struct request *rq,
487					      struct bio *bio)
488{
489	return req_op(rq) == REQ_OP_ZONE_APPEND ||
490	       bio_flagged(bio, BIO_EMULATES_ZONE_APPEND);
491}
492void blk_zone_write_plug_bio_merged(struct bio *bio);
493void blk_zone_write_plug_init_request(struct request *rq);
494void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio);
495void blk_zone_mgmt_bio_endio(struct bio *bio);
496void blk_zone_write_plug_bio_endio(struct bio *bio);
497static inline void blk_zone_bio_endio(struct bio *bio)
498{
499	/*
500	 * Zone management BIOs may impact zone write plugs (e.g. a zone reset
501	 * changes a zone write plug zone write pointer offset), but these
502	 * operation do not go through zone write plugging as they may operate
503	 * on zones that do not have a zone write
504	 * plug. blk_zone_mgmt_bio_endio() handles the potential changes to zone
505	 * write plugs that are present.
506	 */
507	if (op_is_zone_mgmt(bio_op(bio))) {
508		blk_zone_mgmt_bio_endio(bio);
509		return;
510	}
511
512	/*
513	 * For write BIOs to zoned devices, signal the completion of the BIO so
514	 * that the next write BIO can be submitted by zone write plugging.
515	 */
516	if (bio_zone_write_plugging(bio))
517		blk_zone_write_plug_bio_endio(bio);
518}
519
520void blk_zone_write_plug_finish_request(struct request *rq);
521static inline void blk_zone_finish_request(struct request *rq)
522{
523	if (rq->rq_flags & RQF_ZONE_WRITE_PLUGGING)
524		blk_zone_write_plug_finish_request(rq);
525}
526int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd,
527		unsigned long arg);
528int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
529		unsigned int cmd, unsigned long arg);
530#else /* CONFIG_BLK_DEV_ZONED */
531static inline void disk_init_zone_resources(struct gendisk *disk)
532{
533}
534static inline void disk_free_zone_resources(struct gendisk *disk)
535{
536}
537static inline bool bio_zone_write_plugging(struct bio *bio)
538{
539	return false;
540}
541static inline bool blk_req_bio_is_zone_append(struct request *req,
542					      struct bio *bio)
543{
544	return false;
545}
546static inline void blk_zone_write_plug_bio_merged(struct bio *bio)
547{
548}
549static inline void blk_zone_write_plug_init_request(struct request *rq)
550{
551}
552static inline void blk_zone_append_update_request_bio(struct request *rq,
553						      struct bio *bio)
554{
555}
556static inline void blk_zone_bio_endio(struct bio *bio)
557{
558}
559static inline void blk_zone_finish_request(struct request *rq)
560{
561}
562static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
563		unsigned int cmd, unsigned long arg)
564{
565	return -ENOTTY;
566}
567static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
568		blk_mode_t mode, unsigned int cmd, unsigned long arg)
569{
570	return -ENOTTY;
571}
572#endif /* CONFIG_BLK_DEV_ZONED */
573
574struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
575void bdev_add(struct block_device *bdev, dev_t dev);
576void bdev_unhash(struct block_device *bdev);
577void bdev_drop(struct block_device *bdev);
578
579int blk_alloc_ext_minor(void);
580void blk_free_ext_minor(unsigned int minor);
581#define ADDPART_FLAG_NONE	0
582#define ADDPART_FLAG_RAID	1
583#define ADDPART_FLAG_WHOLEDISK	2
584#define ADDPART_FLAG_READONLY	4
585int bdev_add_partition(struct gendisk *disk, int partno, sector_t start,
586		sector_t length);
587int bdev_del_partition(struct gendisk *disk, int partno);
588int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start,
589		sector_t length);
590void drop_partition(struct block_device *part);
591
592void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors);
593
594struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
595		struct lock_class_key *lkclass);
596
597/*
598 * Clean up a page appropriately, where the page may be pinned, may have a
599 * ref taken on it or neither.
600 */
601static inline void bio_release_page(struct bio *bio, struct page *page)
602{
603	if (bio_flagged(bio, BIO_PAGE_PINNED))
604		unpin_user_page(page);
605}
606
607struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id);
608
609int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode);
610
611int disk_alloc_events(struct gendisk *disk);
612void disk_add_events(struct gendisk *disk);
613void disk_del_events(struct gendisk *disk);
614void disk_release_events(struct gendisk *disk);
615void disk_block_events(struct gendisk *disk);
616void disk_unblock_events(struct gendisk *disk);
617void disk_flush_events(struct gendisk *disk, unsigned int mask);
618extern struct device_attribute dev_attr_events;
619extern struct device_attribute dev_attr_events_async;
620extern struct device_attribute dev_attr_events_poll_msecs;
621
622extern struct attribute_group blk_trace_attr_group;
623
624blk_mode_t file_to_blk_mode(struct file *file);
625int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,
626		loff_t lstart, loff_t lend);
627long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
628int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);
629long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
630
631extern const struct address_space_operations def_blk_aops;
632
633int disk_register_independent_access_ranges(struct gendisk *disk);
634void disk_unregister_independent_access_ranges(struct gendisk *disk);
635
636int should_fail_bio(struct bio *bio);
637#ifdef CONFIG_FAIL_MAKE_REQUEST
638bool should_fail_request(struct block_device *part, unsigned int bytes);
639#else /* CONFIG_FAIL_MAKE_REQUEST */
640static inline bool should_fail_request(struct block_device *part,
641					unsigned int bytes)
642{
643	return false;
644}
645#endif /* CONFIG_FAIL_MAKE_REQUEST */
646
647/*
648 * Optimized request reference counting. Ideally we'd make timeouts be more
649 * clever, as that's the only reason we need references at all... But until
650 * this happens, this is faster than using refcount_t. Also see:
651 *
652 * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count")
653 */
654#define req_ref_zero_or_close_to_overflow(req)	\
655	((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u)
656
657static inline bool req_ref_inc_not_zero(struct request *req)
658{
659	return atomic_inc_not_zero(&req->ref);
660}
661
662static inline bool req_ref_put_and_test(struct request *req)
663{
664	WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
665	return atomic_dec_and_test(&req->ref);
666}
667
668static inline void req_ref_set(struct request *req, int value)
669{
670	atomic_set(&req->ref, value);
671}
672
673static inline int req_ref_read(struct request *req)
674{
675	return atomic_read(&req->ref);
676}
677
678static inline u64 blk_time_get_ns(void)
679{
680	struct blk_plug *plug = current->plug;
681
682	if (!plug || !in_task())
683		return ktime_get_ns();
684
685	/*
686	 * 0 could very well be a valid time, but rather than flag "this is
687	 * a valid timestamp" separately, just accept that we'll do an extra
688	 * ktime_get_ns() if we just happen to get 0 as the current time.
689	 */
690	if (!plug->cur_ktime) {
691		plug->cur_ktime = ktime_get_ns();
692		current->flags |= PF_BLOCK_TS;
693	}
694	return plug->cur_ktime;
695}
696
697static inline ktime_t blk_time_get(void)
698{
699	return ns_to_ktime(blk_time_get_ns());
700}
701
702void bdev_release(struct file *bdev_file);
703int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,
704	      const struct blk_holder_ops *hops, struct file *bdev_file);
705int bdev_permission(dev_t dev, blk_mode_t mode, void *holder);
706
707void blk_integrity_generate(struct bio *bio);
708void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter);
709void blk_integrity_prepare(struct request *rq);
710void blk_integrity_complete(struct request *rq, unsigned int nr_bytes);
711
712#ifdef CONFIG_LOCKDEP
713static inline void blk_freeze_acquire_lock(struct request_queue *q)
714{
715	if (!q->mq_freeze_disk_dead)
716		rwsem_acquire(&q->io_lockdep_map, 0, 1, _RET_IP_);
717	if (!q->mq_freeze_queue_dying)
718		rwsem_acquire(&q->q_lockdep_map, 0, 1, _RET_IP_);
719}
720
721static inline void blk_unfreeze_release_lock(struct request_queue *q)
722{
723	if (!q->mq_freeze_queue_dying)
724		rwsem_release(&q->q_lockdep_map, _RET_IP_);
725	if (!q->mq_freeze_disk_dead)
726		rwsem_release(&q->io_lockdep_map, _RET_IP_);
727}
728#else
729static inline void blk_freeze_acquire_lock(struct request_queue *q)
730{
731}
732static inline void blk_unfreeze_release_lock(struct request_queue *q)
733{
734}
735#endif
736
737#endif /* BLK_INTERNAL_H */