pelt.h

  1// SPDX-License-Identifier: GPL-2.0
  2#ifndef _KERNEL_SCHED_PELT_H
  3#define _KERNEL_SCHED_PELT_H
  4#include "sched.h"
  5
  6#include "sched-pelt.h"
  7
  8int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
  9int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
 10int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
 11int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
 12int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
 13bool update_other_load_avgs(struct rq *rq);
 14
 15#ifdef CONFIG_SCHED_HW_PRESSURE
 16int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity);
 17
 18static inline u64 hw_load_avg(struct rq *rq)
 19{
 20	return READ_ONCE(rq->avg_hw.load_avg);
 21}
 22#else /* !CONFIG_SCHED_HW_PRESSURE: */
 23static inline int
 24update_hw_load_avg(u64 now, struct rq *rq, u64 capacity)
 25{
 26	return 0;
 27}
 28
 29static inline u64 hw_load_avg(struct rq *rq)
 30{
 31	return 0;
 32}
 33#endif /* !CONFIG_SCHED_HW_PRESSURE */
 34
 35#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
 36int update_irq_load_avg(struct rq *rq, u64 running);
 37#else
 38static inline int
 39update_irq_load_avg(struct rq *rq, u64 running)
 40{
 41	return 0;
 42}
 43#endif
 44
 45#define PELT_MIN_DIVIDER	(LOAD_AVG_MAX - 1024)
 46
 47static inline u32 get_pelt_divider(struct sched_avg *avg)
 48{
 49	return PELT_MIN_DIVIDER + avg->period_contrib;
 50}
 51
 52static inline void cfs_se_util_change(struct sched_avg *avg)
 53{
 54	unsigned int enqueued;
 55
 56	if (!sched_feat(UTIL_EST))
 57		return;
 58
 59	/* Avoid store if the flag has been already reset */
 60	enqueued = avg->util_est;
 61	if (!(enqueued & UTIL_AVG_UNCHANGED))
 62		return;
 63
 64	/* Reset flag to report util_avg has been updated */
 65	enqueued &= ~UTIL_AVG_UNCHANGED;
 66	WRITE_ONCE(avg->util_est, enqueued);
 67}
 68
 69static inline u64 rq_clock_pelt(struct rq *rq)
 70{
 71	lockdep_assert_rq_held(rq);
 72	assert_clock_updated(rq);
 73
 74	return rq->clock_pelt - rq->lost_idle_time;
 75}
 76
 77/* The rq is idle, we can sync to clock_task */
 78static inline void _update_idle_rq_clock_pelt(struct rq *rq)
 79{
 80	rq->clock_pelt  = rq_clock_task(rq);
 81
 82	u64_u32_store(rq->clock_idle, rq_clock(rq));
 83	/* Paired with smp_rmb in migrate_se_pelt_lag() */
 84	smp_wmb();
 85	u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq));
 86}
 87
 88/*
 89 * The clock_pelt scales the time to reflect the effective amount of
 90 * computation done during the running delta time but then sync back to
 91 * clock_task when rq is idle.
 92 *
 93 *
 94 * absolute time   | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
 95 * @ max capacity  ------******---------------******---------------
 96 * @ half capacity ------************---------************---------
 97 * clock pelt      | 1| 2|    3|    4| 7| 8| 9|   10|   11|14|15|16
 98 *
 99 */
100static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
101{
102	if (unlikely(is_idle_task(rq->curr))) {
103		_update_idle_rq_clock_pelt(rq);
104		return;
105	}
106
107	/*
108	 * When a rq runs at a lower compute capacity, it will need
109	 * more time to do the same amount of work than at max
110	 * capacity. In order to be invariant, we scale the delta to
111	 * reflect how much work has been really done.
112	 * Running longer results in stealing idle time that will
113	 * disturb the load signal compared to max capacity. This
114	 * stolen idle time will be automatically reflected when the
115	 * rq will be idle and the clock will be synced with
116	 * rq_clock_task.
117	 */
118
119	/*
120	 * Scale the elapsed time to reflect the real amount of
121	 * computation
122	 */
123	delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
124	delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
125
126	rq->clock_pelt += delta;
127}
128
129/*
130 * When rq becomes idle, we have to check if it has lost idle time
131 * because it was fully busy. A rq is fully used when the /Sum util_sum
132 * is greater or equal to:
133 * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
134 * For optimization and computing rounding purpose, we don't take into account
135 * the position in the current window (period_contrib) and we use the higher
136 * bound of util_sum to decide.
137 */
138static inline void update_idle_rq_clock_pelt(struct rq *rq)
139{
140	u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
141	u32 util_sum = rq->cfs.avg.util_sum;
142	util_sum += rq->avg_rt.util_sum;
143	util_sum += rq->avg_dl.util_sum;
144
145	/*
146	 * Reflecting stolen time makes sense only if the idle
147	 * phase would be present at max capacity. As soon as the
148	 * utilization of a rq has reached the maximum value, it is
149	 * considered as an always running rq without idle time to
150	 * steal. This potential idle time is considered as lost in
151	 * this case. We keep track of this lost idle time compare to
152	 * rq's clock_task.
153	 */
154	if (util_sum >= divider)
155		rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
156
157	_update_idle_rq_clock_pelt(rq);
158}
159
160#ifdef CONFIG_CFS_BANDWIDTH
161static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
162{
163	u64 throttled;
164
165	if (unlikely(cfs_rq->pelt_clock_throttled))
166		throttled = U64_MAX;
167	else
168		throttled = cfs_rq->throttled_clock_pelt_time;
169
170	u64_u32_store(cfs_rq->throttled_pelt_idle, throttled);
171}
172
173/* rq->task_clock normalized against any time this cfs_rq has spent throttled */
174static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
175{
176	if (unlikely(cfs_rq->pelt_clock_throttled))
177		return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
178
179	return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
180}
181#else /* !CONFIG_CFS_BANDWIDTH: */
182static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { }
183static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
184{
185	return rq_clock_pelt(rq_of(cfs_rq));
186}
187#endif /* !CONFIG_CFS_BANDWIDTH */
188
189#endif /* _KERNEL_SCHED_PELT_H */