blk-wbt.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * buffered writeback throttling. loosely based on CoDel. We can't drop
  4 * packets for IO scheduling, so the logic is something like this:
  5 *
  6 * - Monitor latencies in a defined window of time.
  7 * - If the minimum latency in the above window exceeds some target, increment
  8 *   scaling step and scale down queue depth by a factor of 2x. The monitoring
  9 *   window is then shrunk to 100 / sqrt(scaling step + 1).
 10 * - For any window where we don't have solid data on what the latencies
 11 *   look like, retain status quo.
 12 * - If latencies look good, decrement scaling step.
 13 * - If we're only doing writes, allow the scaling step to go negative. This
 14 *   will temporarily boost write performance, snapping back to a stable
 15 *   scaling step of 0 if reads show up or the heavy writers finish. Unlike
 16 *   positive scaling steps where we shrink the monitoring window, a negative
 17 *   scaling step retains the default step==0 window size.
 18 *
 19 * Copyright (C) 2016 Jens Axboe
 20 *
 21 */
 22#include <linux/kernel.h>
 23#include <linux/blk_types.h>
 24#include <linux/slab.h>
 25#include <linux/backing-dev.h>
 26#include <linux/swap.h>
 27
 28#include "blk-stat.h"
 29#include "blk-wbt.h"
 30#include "blk-rq-qos.h"
 31#include "elevator.h"
 32#include "blk.h"
 33
 34#define CREATE_TRACE_POINTS
 35#include <trace/events/wbt.h>
 36
 37enum wbt_flags {
 38	WBT_TRACKED		= 1,	/* write, tracked for throttling */
 39	WBT_READ		= 2,	/* read */
 40	WBT_SWAP		= 4,	/* write, from swap_writeout() */
 41	WBT_DISCARD		= 8,	/* discard */
 42
 43	WBT_NR_BITS		= 4,	/* number of bits */
 44};
 45
 46enum {
 47	WBT_RWQ_BG		= 0,
 48	WBT_RWQ_SWAP,
 49	WBT_RWQ_DISCARD,
 50	WBT_NUM_RWQ,
 51};
 52
 53/*
 54 * If current state is WBT_STATE_ON/OFF_DEFAULT, it can be covered to any other
 55 * state, if current state is WBT_STATE_ON/OFF_MANUAL, it can only be covered
 56 * to WBT_STATE_OFF/ON_MANUAL.
 57 */
 58enum {
 59	WBT_STATE_ON_DEFAULT	= 1,	/* on by default */
 60	WBT_STATE_ON_MANUAL	= 2,	/* on manually by sysfs */
 61	WBT_STATE_OFF_DEFAULT	= 3,	/* off by default */
 62	WBT_STATE_OFF_MANUAL	= 4,	/* off manually by sysfs */
 63};
 64
 65struct rq_wb {
 66	/*
 67	 * Settings that govern how we throttle
 68	 */
 69	unsigned int wb_background;		/* background writeback */
 70	unsigned int wb_normal;			/* normal writeback */
 71
 72	short enable_state;			/* WBT_STATE_* */
 73
 74	/*
 75	 * Number of consecutive periods where we don't have enough
 76	 * information to make a firm scale up/down decision.
 77	 */
 78	unsigned int unknown_cnt;
 79
 80	u64 win_nsec;				/* default window size */
 81	u64 cur_win_nsec;			/* current window size */
 82
 83	struct blk_stat_callback *cb;
 84
 85	u64 sync_issue;
 86	void *sync_cookie;
 87
 88	unsigned long last_issue;	/* issue time of last read rq */
 89	unsigned long last_comp;	/* completion time of last read rq */
 90	unsigned long min_lat_nsec;
 91	struct rq_qos rqos;
 92	struct rq_wait rq_wait[WBT_NUM_RWQ];
 93	struct rq_depth rq_depth;
 94};
 95
 96static inline struct rq_wb *RQWB(struct rq_qos *rqos)
 97{
 98	return container_of(rqos, struct rq_wb, rqos);
 99}
100
101static inline void wbt_clear_state(struct request *rq)
102{
103	rq->wbt_flags = 0;
104}
105
106static inline enum wbt_flags wbt_flags(struct request *rq)
107{
108	return rq->wbt_flags;
109}
110
111static inline bool wbt_is_tracked(struct request *rq)
112{
113	return rq->wbt_flags & WBT_TRACKED;
114}
115
116static inline bool wbt_is_read(struct request *rq)
117{
118	return rq->wbt_flags & WBT_READ;
119}
120
121enum {
122	/*
123	 * Default setting, we'll scale up (to 75% of QD max) or down (min 1)
124	 * from here depending on device stats
125	 */
126	RWB_DEF_DEPTH	= 16,
127
128	/*
129	 * 100msec window
130	 */
131	RWB_WINDOW_NSEC		= 100 * 1000 * 1000ULL,
132
133	/*
134	 * Disregard stats, if we don't meet this minimum
135	 */
136	RWB_MIN_WRITE_SAMPLES	= 3,
137
138	/*
139	 * If we have this number of consecutive windows without enough
140	 * information to scale up or down, slowly return to center state
141	 * (step == 0).
142	 */
143	RWB_UNKNOWN_BUMP	= 5,
144};
145
146static inline bool rwb_enabled(struct rq_wb *rwb)
147{
148	return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT &&
149		      rwb->enable_state != WBT_STATE_OFF_MANUAL;
150}
151
152static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)
153{
154	if (rwb_enabled(rwb)) {
155		const unsigned long cur = jiffies;
156
157		if (cur != *var)
158			*var = cur;
159	}
160}
161
162/*
163 * If a task was rate throttled in balance_dirty_pages() within the last
164 * second or so, use that to indicate a higher cleaning rate.
165 */
166static bool wb_recent_wait(struct rq_wb *rwb)
167{
168	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;
169
170	return time_before(jiffies, bdi->last_bdp_sleep + HZ);
171}
172
173static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,
174					  enum wbt_flags wb_acct)
175{
176	if (wb_acct & WBT_SWAP)
177		return &rwb->rq_wait[WBT_RWQ_SWAP];
178	else if (wb_acct & WBT_DISCARD)
179		return &rwb->rq_wait[WBT_RWQ_DISCARD];
180
181	return &rwb->rq_wait[WBT_RWQ_BG];
182}
183
184static void rwb_wake_all(struct rq_wb *rwb)
185{
186	int i;
187
188	for (i = 0; i < WBT_NUM_RWQ; i++) {
189		struct rq_wait *rqw = &rwb->rq_wait[i];
190
191		if (wq_has_sleeper(&rqw->wait))
192			wake_up_all(&rqw->wait);
193	}
194}
195
196static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,
197			 enum wbt_flags wb_acct)
198{
199	int inflight, limit;
200
201	inflight = atomic_dec_return(&rqw->inflight);
202
203	/*
204	 * For discards, our limit is always the background. For writes, if
205	 * the device does write back caching, drop further down before we
206	 * wake people up.
207	 */
208	if (wb_acct & WBT_DISCARD)
209		limit = rwb->wb_background;
210	else if (blk_queue_write_cache(rwb->rqos.disk->queue) &&
211		 !wb_recent_wait(rwb))
212		limit = 0;
213	else
214		limit = rwb->wb_normal;
215
216	/*
217	 * Don't wake anyone up if we are above the normal limit.
218	 */
219	if (inflight && inflight >= limit)
220		return;
221
222	if (wq_has_sleeper(&rqw->wait)) {
223		int diff = limit - inflight;
224
225		if (!inflight || diff >= rwb->wb_background / 2)
226			wake_up_all(&rqw->wait);
227	}
228}
229
230static void __wbt_done(struct rq_qos *rqos, enum wbt_flags wb_acct)
231{
232	struct rq_wb *rwb = RQWB(rqos);
233	struct rq_wait *rqw;
234
235	if (!(wb_acct & WBT_TRACKED))
236		return;
237
238	rqw = get_rq_wait(rwb, wb_acct);
239	wbt_rqw_done(rwb, rqw, wb_acct);
240}
241
242/*
243 * Called on completion of a request. Note that it's also called when
244 * a request is merged, when the request gets freed.
245 */
246static void wbt_done(struct rq_qos *rqos, struct request *rq)
247{
248	struct rq_wb *rwb = RQWB(rqos);
249
250	if (!wbt_is_tracked(rq)) {
251		if (wbt_is_read(rq)) {
252			if (rwb->sync_cookie == rq) {
253				rwb->sync_issue = 0;
254				rwb->sync_cookie = NULL;
255			}
256
257			wb_timestamp(rwb, &rwb->last_comp);
258		}
259	} else {
260		WARN_ON_ONCE(rq == rwb->sync_cookie);
261		__wbt_done(rqos, wbt_flags(rq));
262	}
263	wbt_clear_state(rq);
264}
265
266static inline bool stat_sample_valid(struct blk_rq_stat *stat)
267{
268	/*
269	 * We need at least one read sample, and a minimum of
270	 * RWB_MIN_WRITE_SAMPLES. We require some write samples to know
271	 * that it's writes impacting us, and not just some sole read on
272	 * a device that is in a lower power state.
273	 */
274	return (stat[READ].nr_samples >= 1 &&
275		stat[WRITE].nr_samples >= RWB_MIN_WRITE_SAMPLES);
276}
277
278static u64 rwb_sync_issue_lat(struct rq_wb *rwb)
279{
280	u64 issue = READ_ONCE(rwb->sync_issue);
281
282	if (!issue || !rwb->sync_cookie)
283		return 0;
284
285	return blk_time_get_ns() - issue;
286}
287
288static inline unsigned int wbt_inflight(struct rq_wb *rwb)
289{
290	unsigned int i, ret = 0;
291
292	for (i = 0; i < WBT_NUM_RWQ; i++)
293		ret += atomic_read(&rwb->rq_wait[i].inflight);
294
295	return ret;
296}
297
298enum {
299	LAT_OK = 1,
300	LAT_UNKNOWN,
301	LAT_UNKNOWN_WRITES,
302	LAT_EXCEEDED,
303};
304
305static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)
306{
307	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;
308	struct rq_depth *rqd = &rwb->rq_depth;
309	u64 thislat;
310
311	/*
312	 * If our stored sync issue exceeds the window size, or it
313	 * exceeds our min target AND we haven't logged any entries,
314	 * flag the latency as exceeded. wbt works off completion latencies,
315	 * but for a flooded device, a single sync IO can take a long time
316	 * to complete after being issued. If this time exceeds our
317	 * monitoring window AND we didn't see any other completions in that
318	 * window, then count that sync IO as a violation of the latency.
319	 */
320	thislat = rwb_sync_issue_lat(rwb);
321	if (thislat > rwb->cur_win_nsec ||
322	    (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {
323		trace_wbt_lat(bdi, thislat);
324		return LAT_EXCEEDED;
325	}
326
327	/*
328	 * No read/write mix, if stat isn't valid
329	 */
330	if (!stat_sample_valid(stat)) {
331		/*
332		 * If we had writes in this stat window and the window is
333		 * current, we're only doing writes. If a task recently
334		 * waited or still has writes in flights, consider us doing
335		 * just writes as well.
336		 */
337		if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||
338		    wbt_inflight(rwb))
339			return LAT_UNKNOWN_WRITES;
340		return LAT_UNKNOWN;
341	}
342
343	/*
344	 * If the 'min' latency exceeds our target, step down.
345	 */
346	if (stat[READ].min > rwb->min_lat_nsec) {
347		trace_wbt_lat(bdi, stat[READ].min);
348		trace_wbt_stat(bdi, stat);
349		return LAT_EXCEEDED;
350	}
351
352	if (rqd->scale_step)
353		trace_wbt_stat(bdi, stat);
354
355	return LAT_OK;
356}
357
358static void rwb_trace_step(struct rq_wb *rwb, const char *msg)
359{
360	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;
361	struct rq_depth *rqd = &rwb->rq_depth;
362
363	trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,
364			rwb->wb_background, rwb->wb_normal, rqd->max_depth);
365}
366
367static void calc_wb_limits(struct rq_wb *rwb)
368{
369	if (rwb->min_lat_nsec == 0) {
370		rwb->wb_normal = rwb->wb_background = 0;
371	} else if (rwb->rq_depth.max_depth <= 2) {
372		rwb->wb_normal = rwb->rq_depth.max_depth;
373		rwb->wb_background = 1;
374	} else {
375		rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;
376		rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;
377	}
378}
379
380static void scale_up(struct rq_wb *rwb)
381{
382	if (!rq_depth_scale_up(&rwb->rq_depth))
383		return;
384	calc_wb_limits(rwb);
385	rwb->unknown_cnt = 0;
386	rwb_wake_all(rwb);
387	rwb_trace_step(rwb, tracepoint_string("scale up"));
388}
389
390static void scale_down(struct rq_wb *rwb, bool hard_throttle)
391{
392	if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle))
393		return;
394	calc_wb_limits(rwb);
395	rwb->unknown_cnt = 0;
396	rwb_trace_step(rwb, tracepoint_string("scale down"));
397}
398
399static void rwb_arm_timer(struct rq_wb *rwb)
400{
401	struct rq_depth *rqd = &rwb->rq_depth;
402
403	if (rqd->scale_step > 0) {
404		/*
405		 * We should speed this up, using some variant of a fast
406		 * integer inverse square root calculation. Since we only do
407		 * this for every window expiration, it's not a huge deal,
408		 * though.
409		 */
410		rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,
411					int_sqrt((rqd->scale_step + 1) << 8));
412	} else {
413		/*
414		 * For step < 0, we don't want to increase/decrease the
415		 * window size.
416		 */
417		rwb->cur_win_nsec = rwb->win_nsec;
418	}
419
420	blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);
421}
422
423static void wb_timer_fn(struct blk_stat_callback *cb)
424{
425	struct rq_wb *rwb = cb->data;
426	struct rq_depth *rqd = &rwb->rq_depth;
427	unsigned int inflight = wbt_inflight(rwb);
428	int status;
429
430	if (!rwb->rqos.disk)
431		return;
432
433	status = latency_exceeded(rwb, cb->stat);
434
435	trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight);
436
437	/*
438	 * If we exceeded the latency target, step down. If we did not,
439	 * step one level up. If we don't know enough to say either exceeded
440	 * or ok, then don't do anything.
441	 */
442	switch (status) {
443	case LAT_EXCEEDED:
444		scale_down(rwb, true);
445		break;
446	case LAT_OK:
447		scale_up(rwb);
448		break;
449	case LAT_UNKNOWN_WRITES:
450		/*
451		 * We don't have a valid read/write sample, but we do have
452		 * writes going on. Allow step to go negative, to increase
453		 * write performance.
454		 */
455		scale_up(rwb);
456		break;
457	case LAT_UNKNOWN:
458		if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)
459			break;
460		/*
461		 * We get here when previously scaled reduced depth, and we
462		 * currently don't have a valid read/write sample. For that
463		 * case, slowly return to center state (step == 0).
464		 */
465		if (rqd->scale_step > 0)
466			scale_up(rwb);
467		else if (rqd->scale_step < 0)
468			scale_down(rwb, false);
469		break;
470	default:
471		break;
472	}
473
474	/*
475	 * Re-arm timer, if we have IO in flight
476	 */
477	if (rqd->scale_step || inflight)
478		rwb_arm_timer(rwb);
479}
480
481static void wbt_update_limits(struct rq_wb *rwb)
482{
483	struct rq_depth *rqd = &rwb->rq_depth;
484
485	rqd->scale_step = 0;
486	rqd->scaled_max = false;
487
488	rq_depth_calc_max_depth(rqd);
489	calc_wb_limits(rwb);
490
491	rwb_wake_all(rwb);
492}
493
494bool wbt_disabled(struct request_queue *q)
495{
496	struct rq_qos *rqos = wbt_rq_qos(q);
497
498	return !rqos || !rwb_enabled(RQWB(rqos));
499}
500
501u64 wbt_get_min_lat(struct request_queue *q)
502{
503	struct rq_qos *rqos = wbt_rq_qos(q);
504	if (!rqos)
505		return 0;
506	return RQWB(rqos)->min_lat_nsec;
507}
508
509void wbt_set_min_lat(struct request_queue *q, u64 val)
510{
511	struct rq_qos *rqos = wbt_rq_qos(q);
512	if (!rqos)
513		return;
514
515	RQWB(rqos)->min_lat_nsec = val;
516	if (val)
517		RQWB(rqos)->enable_state = WBT_STATE_ON_MANUAL;
518	else
519		RQWB(rqos)->enable_state = WBT_STATE_OFF_MANUAL;
520
521	wbt_update_limits(RQWB(rqos));
522}
523
524
525static bool close_io(struct rq_wb *rwb)
526{
527	const unsigned long now = jiffies;
528
529	return time_before(now, rwb->last_issue + HZ / 10) ||
530		time_before(now, rwb->last_comp + HZ / 10);
531}
532
533#define REQ_HIPRIO	(REQ_SYNC | REQ_META | REQ_PRIO | REQ_SWAP)
534
535static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf)
536{
537	unsigned int limit;
538
539	if ((opf & REQ_OP_MASK) == REQ_OP_DISCARD)
540		return rwb->wb_background;
541
542	/*
543	 * At this point we know it's a buffered write. If this is
544	 * swap trying to free memory, or REQ_SYNC is set, then
545	 * it's WB_SYNC_ALL writeback, and we'll use the max limit for
546	 * that. If the write is marked as a background write, then use
547	 * the idle limit, or go to normal if we haven't had competing
548	 * IO for a bit.
549	 */
550	if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb))
551		limit = rwb->rq_depth.max_depth;
552	else if ((opf & REQ_BACKGROUND) || close_io(rwb)) {
553		/*
554		 * If less than 100ms since we completed unrelated IO,
555		 * limit us to half the depth for background writeback.
556		 */
557		limit = rwb->wb_background;
558	} else
559		limit = rwb->wb_normal;
560
561	return limit;
562}
563
564struct wbt_wait_data {
565	struct rq_wb *rwb;
566	enum wbt_flags wb_acct;
567	blk_opf_t opf;
568};
569
570static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data)
571{
572	struct wbt_wait_data *data = private_data;
573	return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf));
574}
575
576static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data)
577{
578	struct wbt_wait_data *data = private_data;
579	wbt_rqw_done(data->rwb, rqw, data->wb_acct);
580}
581
582/*
583 * Block if we will exceed our limit, or if we are currently waiting for
584 * the timer to kick off queuing again.
585 */
586static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
587		       blk_opf_t opf)
588{
589	struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
590	struct wbt_wait_data data = {
591		.rwb = rwb,
592		.wb_acct = wb_acct,
593		.opf = opf,
594	};
595
596	rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);
597}
598
599static inline bool wbt_should_throttle(struct bio *bio)
600{
601	switch (bio_op(bio)) {
602	case REQ_OP_WRITE:
603		/*
604		 * Don't throttle WRITE_ODIRECT
605		 */
606		if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) ==
607		    (REQ_SYNC | REQ_IDLE))
608			return false;
609		fallthrough;
610	case REQ_OP_DISCARD:
611		return true;
612	default:
613		return false;
614	}
615}
616
617static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)
618{
619	enum wbt_flags flags = 0;
620
621	if (!rwb_enabled(rwb))
622		return 0;
623
624	if (bio_op(bio) == REQ_OP_READ) {
625		flags = WBT_READ;
626	} else if (wbt_should_throttle(bio)) {
627		if (bio->bi_opf & REQ_SWAP)
628			flags |= WBT_SWAP;
629		if (bio_op(bio) == REQ_OP_DISCARD)
630			flags |= WBT_DISCARD;
631		flags |= WBT_TRACKED;
632	}
633	return flags;
634}
635
636static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio)
637{
638	struct rq_wb *rwb = RQWB(rqos);
639	enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);
640	__wbt_done(rqos, flags);
641}
642
643/* May sleep, if we have exceeded the writeback limits. */
644static void wbt_wait(struct rq_qos *rqos, struct bio *bio)
645{
646	struct rq_wb *rwb = RQWB(rqos);
647	enum wbt_flags flags;
648
649	flags = bio_to_wbt_flags(rwb, bio);
650	if (!(flags & WBT_TRACKED)) {
651		if (flags & WBT_READ)
652			wb_timestamp(rwb, &rwb->last_issue);
653		return;
654	}
655
656	__wbt_wait(rwb, flags, bio->bi_opf);
657
658	if (!blk_stat_is_active(rwb->cb))
659		rwb_arm_timer(rwb);
660}
661
662static void wbt_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
663{
664	struct rq_wb *rwb = RQWB(rqos);
665	rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);
666}
667
668static void wbt_issue(struct rq_qos *rqos, struct request *rq)
669{
670	struct rq_wb *rwb = RQWB(rqos);
671
672	if (!rwb_enabled(rwb))
673		return;
674
675	/*
676	 * Track sync issue, in case it takes a long time to complete. Allows us
677	 * to react quicker, if a sync IO takes a long time to complete. Note
678	 * that this is just a hint. The request can go away when it completes,
679	 * so it's important we never dereference it. We only use the address to
680	 * compare with, which is why we store the sync_issue time locally.
681	 */
682	if (wbt_is_read(rq) && !rwb->sync_issue) {
683		rwb->sync_cookie = rq;
684		rwb->sync_issue = rq->io_start_time_ns;
685	}
686}
687
688static void wbt_requeue(struct rq_qos *rqos, struct request *rq)
689{
690	struct rq_wb *rwb = RQWB(rqos);
691	if (!rwb_enabled(rwb))
692		return;
693	if (rq == rwb->sync_cookie) {
694		rwb->sync_issue = 0;
695		rwb->sync_cookie = NULL;
696	}
697}
698
699/*
700 * Enable wbt if defaults are configured that way
701 */
702static bool __wbt_enable_default(struct gendisk *disk)
703{
704	struct request_queue *q = disk->queue;
705	struct rq_qos *rqos;
706	bool enable = IS_ENABLED(CONFIG_BLK_WBT_MQ);
707
708	mutex_lock(&disk->rqos_state_mutex);
709
710	if (blk_queue_disable_wbt(q))
711		enable = false;
712
713	/* Throttling already enabled? */
714	rqos = wbt_rq_qos(q);
715	if (rqos) {
716		if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)
717			RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;
718		mutex_unlock(&disk->rqos_state_mutex);
719		return false;
720	}
721	mutex_unlock(&disk->rqos_state_mutex);
722
723	/* Queue not registered? Maybe shutting down... */
724	if (!blk_queue_registered(q))
725		return false;
726
727	if (queue_is_mq(q) && enable)
728		return true;
729	return false;
730}
731
732void wbt_enable_default(struct gendisk *disk)
733{
734	__wbt_enable_default(disk);
735}
736EXPORT_SYMBOL_GPL(wbt_enable_default);
737
738void wbt_init_enable_default(struct gendisk *disk)
739{
740	if (__wbt_enable_default(disk))
741		WARN_ON_ONCE(wbt_init(disk));
742}
743
744u64 wbt_default_latency_nsec(struct request_queue *q)
745{
746	/*
747	 * We default to 2msec for non-rotational storage, and 75msec
748	 * for rotational storage.
749	 */
750	if (blk_queue_nonrot(q))
751		return 2000000ULL;
752	else
753		return 75000000ULL;
754}
755
756static int wbt_data_dir(const struct request *rq)
757{
758	const enum req_op op = req_op(rq);
759
760	if (op == REQ_OP_READ)
761		return READ;
762	else if (op_is_write(op))
763		return WRITE;
764
765	/* don't account */
766	return -1;
767}
768
769static void wbt_queue_depth_changed(struct rq_qos *rqos)
770{
771	RQWB(rqos)->rq_depth.queue_depth = blk_queue_depth(rqos->disk->queue);
772	wbt_update_limits(RQWB(rqos));
773}
774
775static void wbt_exit(struct rq_qos *rqos)
776{
777	struct rq_wb *rwb = RQWB(rqos);
778
779	blk_stat_remove_callback(rqos->disk->queue, rwb->cb);
780	blk_stat_free_callback(rwb->cb);
781	kfree(rwb);
782}
783
784/*
785 * Disable wbt, if enabled by default.
786 */
787void wbt_disable_default(struct gendisk *disk)
788{
789	struct rq_qos *rqos = wbt_rq_qos(disk->queue);
790	struct rq_wb *rwb;
791	if (!rqos)
792		return;
793	mutex_lock(&disk->rqos_state_mutex);
794	rwb = RQWB(rqos);
795	if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {
796		blk_stat_deactivate(rwb->cb);
797		rwb->enable_state = WBT_STATE_OFF_DEFAULT;
798	}
799	mutex_unlock(&disk->rqos_state_mutex);
800}
801EXPORT_SYMBOL_GPL(wbt_disable_default);
802
803#ifdef CONFIG_BLK_DEBUG_FS
804static int wbt_curr_win_nsec_show(void *data, struct seq_file *m)
805{
806	struct rq_qos *rqos = data;
807	struct rq_wb *rwb = RQWB(rqos);
808
809	seq_printf(m, "%llu\n", rwb->cur_win_nsec);
810	return 0;
811}
812
813static int wbt_enabled_show(void *data, struct seq_file *m)
814{
815	struct rq_qos *rqos = data;
816	struct rq_wb *rwb = RQWB(rqos);
817
818	seq_printf(m, "%d\n", rwb->enable_state);
819	return 0;
820}
821
822static int wbt_id_show(void *data, struct seq_file *m)
823{
824	struct rq_qos *rqos = data;
825
826	seq_printf(m, "%u\n", rqos->id);
827	return 0;
828}
829
830static int wbt_inflight_show(void *data, struct seq_file *m)
831{
832	struct rq_qos *rqos = data;
833	struct rq_wb *rwb = RQWB(rqos);
834	int i;
835
836	for (i = 0; i < WBT_NUM_RWQ; i++)
837		seq_printf(m, "%d: inflight %d\n", i,
838			   atomic_read(&rwb->rq_wait[i].inflight));
839	return 0;
840}
841
842static int wbt_min_lat_nsec_show(void *data, struct seq_file *m)
843{
844	struct rq_qos *rqos = data;
845	struct rq_wb *rwb = RQWB(rqos);
846
847	seq_printf(m, "%lu\n", rwb->min_lat_nsec);
848	return 0;
849}
850
851static int wbt_unknown_cnt_show(void *data, struct seq_file *m)
852{
853	struct rq_qos *rqos = data;
854	struct rq_wb *rwb = RQWB(rqos);
855
856	seq_printf(m, "%u\n", rwb->unknown_cnt);
857	return 0;
858}
859
860static int wbt_normal_show(void *data, struct seq_file *m)
861{
862	struct rq_qos *rqos = data;
863	struct rq_wb *rwb = RQWB(rqos);
864
865	seq_printf(m, "%u\n", rwb->wb_normal);
866	return 0;
867}
868
869static int wbt_background_show(void *data, struct seq_file *m)
870{
871	struct rq_qos *rqos = data;
872	struct rq_wb *rwb = RQWB(rqos);
873
874	seq_printf(m, "%u\n", rwb->wb_background);
875	return 0;
876}
877
878static const struct blk_mq_debugfs_attr wbt_debugfs_attrs[] = {
879	{"curr_win_nsec", 0400, wbt_curr_win_nsec_show},
880	{"enabled", 0400, wbt_enabled_show},
881	{"id", 0400, wbt_id_show},
882	{"inflight", 0400, wbt_inflight_show},
883	{"min_lat_nsec", 0400, wbt_min_lat_nsec_show},
884	{"unknown_cnt", 0400, wbt_unknown_cnt_show},
885	{"wb_normal", 0400, wbt_normal_show},
886	{"wb_background", 0400, wbt_background_show},
887	{},
888};
889#endif
890
891static const struct rq_qos_ops wbt_rqos_ops = {
892	.throttle = wbt_wait,
893	.issue = wbt_issue,
894	.track = wbt_track,
895	.requeue = wbt_requeue,
896	.done = wbt_done,
897	.cleanup = wbt_cleanup,
898	.queue_depth_changed = wbt_queue_depth_changed,
899	.exit = wbt_exit,
900#ifdef CONFIG_BLK_DEBUG_FS
901	.debugfs_attrs = wbt_debugfs_attrs,
902#endif
903};
904
905int wbt_init(struct gendisk *disk)
906{
907	struct request_queue *q = disk->queue;
908	struct rq_wb *rwb;
909	int i;
910	int ret;
911
912	rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
913	if (!rwb)
914		return -ENOMEM;
915
916	rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);
917	if (!rwb->cb) {
918		kfree(rwb);
919		return -ENOMEM;
920	}
921
922	for (i = 0; i < WBT_NUM_RWQ; i++)
923		rq_wait_init(&rwb->rq_wait[i]);
924
925	rwb->last_comp = rwb->last_issue = jiffies;
926	rwb->win_nsec = RWB_WINDOW_NSEC;
927	rwb->enable_state = WBT_STATE_ON_DEFAULT;
928	rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
929	rwb->min_lat_nsec = wbt_default_latency_nsec(q);
930	rwb->rq_depth.queue_depth = blk_queue_depth(q);
931	wbt_update_limits(rwb);
932
933	/*
934	 * Assign rwb and add the stats callback.
935	 */
936	mutex_lock(&q->rq_qos_mutex);
937	ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);
938	mutex_unlock(&q->rq_qos_mutex);
939	if (ret)
940		goto err_free;
941
942	blk_stat_add_callback(q, rwb->cb);
943
944	return 0;
945
946err_free:
947	blk_stat_free_callback(rwb->cb);
948	kfree(rwb);
949	return ret;
950
951}