delayacct.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/* delayacct.h - per-task delay accounting
  3 *
  4 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
  5 */
  6
  7#ifndef _LINUX_DELAYACCT_H
  8#define _LINUX_DELAYACCT_H
  9
 10#include <uapi/linux/taskstats.h>
 11
 12#ifdef CONFIG_TASK_DELAY_ACCT
 13struct task_delay_info {
 14	raw_spinlock_t	lock;
 15
 16	/* For each stat XXX, add following, aligned appropriately
 17	 *
 18	 * struct timespec XXX_start, XXX_end;
 19	 * u64 XXX_delay;
 20	 * u32 XXX_count;
 21	 *
 22	 * Atomicity of updates to XXX_delay, XXX_count protected by
 23	 * single lock above (split into XXX_lock if contention is an issue).
 24	 */
 25
 26	/*
 27	 * XXX_count is incremented on every XXX operation, the delay
 28	 * associated with the operation is added to XXX_delay.
 29	 * XXX_delay contains the accumulated delay time in nanoseconds.
 30	 */
 31	u64 blkio_start;
 32	u64 blkio_delay_max;
 33	u64 blkio_delay_min;
 34	u64 blkio_delay;	/* wait for sync block io completion */
 35	u64 swapin_start;
 36	u64 swapin_delay_max;
 37	u64 swapin_delay_min;
 38	u64 swapin_delay;	/* wait for swapin */
 39	u32 blkio_count;	/* total count of the number of sync block */
 40				/* io operations performed */
 41	u32 swapin_count;	/* total count of swapin */
 42
 43	u64 freepages_start;
 44	u64 freepages_delay_max;
 45	u64 freepages_delay_min;
 46	u64 freepages_delay;	/* wait for memory reclaim */
 47
 48	u64 thrashing_start;
 49	u64 thrashing_delay_max;
 50	u64 thrashing_delay_min;
 51	u64 thrashing_delay;	/* wait for thrashing page */
 52
 53	u64 compact_start;
 54	u64 compact_delay_max;
 55	u64 compact_delay_min;
 56	u64 compact_delay;	/* wait for memory compact */
 57
 58	u64 wpcopy_start;
 59	u64 wpcopy_delay_max;
 60	u64 wpcopy_delay_min;
 61	u64 wpcopy_delay;	/* wait for write-protect copy */
 62
 63	u64 irq_delay_max;
 64	u64 irq_delay_min;
 65	u64 irq_delay;	/* wait for IRQ/SOFTIRQ */
 66
 67	u32 freepages_count;	/* total count of memory reclaim */
 68	u32 thrashing_count;	/* total count of thrash waits */
 69	u32 compact_count;	/* total count of memory compact */
 70	u32 wpcopy_count;	/* total count of write-protect copy */
 71	u32 irq_count;	/* total count of IRQ/SOFTIRQ */
 72};
 73#endif
 74
 75#include <linux/sched.h>
 76#include <linux/slab.h>
 77#include <linux/jump_label.h>
 78
 79#ifdef CONFIG_TASK_DELAY_ACCT
 80DECLARE_STATIC_KEY_FALSE(delayacct_key);
 81extern int delayacct_on;	/* Delay accounting turned on/off */
 82extern struct kmem_cache *delayacct_cache;
 83extern void delayacct_init(void);
 84
 85extern void __delayacct_tsk_init(struct task_struct *);
 86extern void __delayacct_tsk_exit(struct task_struct *);
 87extern void __delayacct_blkio_start(void);
 88extern void __delayacct_blkio_end(struct task_struct *);
 89extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
 90extern __u64 __delayacct_blkio_ticks(struct task_struct *);
 91extern void __delayacct_freepages_start(void);
 92extern void __delayacct_freepages_end(void);
 93extern void __delayacct_thrashing_start(bool *in_thrashing);
 94extern void __delayacct_thrashing_end(bool *in_thrashing);
 95extern void __delayacct_swapin_start(void);
 96extern void __delayacct_swapin_end(void);
 97extern void __delayacct_compact_start(void);
 98extern void __delayacct_compact_end(void);
 99extern void __delayacct_wpcopy_start(void);
100extern void __delayacct_wpcopy_end(void);
101extern void __delayacct_irq(struct task_struct *task, u32 delta);
102
103static inline void delayacct_tsk_init(struct task_struct *tsk)
104{
105	/* reinitialize in case parent's non-null pointer was dup'ed*/
106	tsk->delays = NULL;
107	if (delayacct_on)
108		__delayacct_tsk_init(tsk);
109}
110
111/* Free tsk->delays. Called from bad fork and __put_task_struct
112 * where there's no risk of tsk->delays being accessed elsewhere
113 */
114static inline void delayacct_tsk_free(struct task_struct *tsk)
115{
116	if (tsk->delays)
117		kmem_cache_free(delayacct_cache, tsk->delays);
118	tsk->delays = NULL;
119}
120
121static inline void delayacct_blkio_start(void)
122{
123	if (!static_branch_unlikely(&delayacct_key))
124		return;
125
126	if (current->delays)
127		__delayacct_blkio_start();
128}
129
130static inline void delayacct_blkio_end(struct task_struct *p)
131{
132	if (!static_branch_unlikely(&delayacct_key))
133		return;
134
135	if (p->delays)
136		__delayacct_blkio_end(p);
137}
138
139static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
140{
141	if (tsk->delays)
142		return __delayacct_blkio_ticks(tsk);
143	return 0;
144}
145
146static inline void delayacct_freepages_start(void)
147{
148	if (!static_branch_unlikely(&delayacct_key))
149		return;
150
151	if (current->delays)
152		__delayacct_freepages_start();
153}
154
155static inline void delayacct_freepages_end(void)
156{
157	if (!static_branch_unlikely(&delayacct_key))
158		return;
159
160	if (current->delays)
161		__delayacct_freepages_end();
162}
163
164static inline void delayacct_thrashing_start(bool *in_thrashing)
165{
166	if (!static_branch_unlikely(&delayacct_key))
167		return;
168
169	if (current->delays)
170		__delayacct_thrashing_start(in_thrashing);
171}
172
173static inline void delayacct_thrashing_end(bool *in_thrashing)
174{
175	if (!static_branch_unlikely(&delayacct_key))
176		return;
177
178	if (current->delays)
179		__delayacct_thrashing_end(in_thrashing);
180}
181
182static inline void delayacct_swapin_start(void)
183{
184	if (!static_branch_unlikely(&delayacct_key))
185		return;
186
187	if (current->delays)
188		__delayacct_swapin_start();
189}
190
191static inline void delayacct_swapin_end(void)
192{
193	if (!static_branch_unlikely(&delayacct_key))
194		return;
195
196	if (current->delays)
197		__delayacct_swapin_end();
198}
199
200static inline void delayacct_compact_start(void)
201{
202	if (!static_branch_unlikely(&delayacct_key))
203		return;
204
205	if (current->delays)
206		__delayacct_compact_start();
207}
208
209static inline void delayacct_compact_end(void)
210{
211	if (!static_branch_unlikely(&delayacct_key))
212		return;
213
214	if (current->delays)
215		__delayacct_compact_end();
216}
217
218static inline void delayacct_wpcopy_start(void)
219{
220	if (!static_branch_unlikely(&delayacct_key))
221		return;
222
223	if (current->delays)
224		__delayacct_wpcopy_start();
225}
226
227static inline void delayacct_wpcopy_end(void)
228{
229	if (!static_branch_unlikely(&delayacct_key))
230		return;
231
232	if (current->delays)
233		__delayacct_wpcopy_end();
234}
235
236static inline void delayacct_irq(struct task_struct *task, u32 delta)
237{
238	if (!static_branch_unlikely(&delayacct_key))
239		return;
240
241	if (task->delays)
242		__delayacct_irq(task, delta);
243}
244
245#else
246static inline void delayacct_init(void)
247{}
248static inline void delayacct_tsk_init(struct task_struct *tsk)
249{}
250static inline void delayacct_tsk_free(struct task_struct *tsk)
251{}
252static inline void delayacct_blkio_start(void)
253{}
254static inline void delayacct_blkio_end(struct task_struct *p)
255{}
256static inline int delayacct_add_tsk(struct taskstats *d,
257					struct task_struct *tsk)
258{ return 0; }
259static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
260{ return 0; }
261static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
262{ return 0; }
263static inline void delayacct_freepages_start(void)
264{}
265static inline void delayacct_freepages_end(void)
266{}
267static inline void delayacct_thrashing_start(bool *in_thrashing)
268{}
269static inline void delayacct_thrashing_end(bool *in_thrashing)
270{}
271static inline void delayacct_swapin_start(void)
272{}
273static inline void delayacct_swapin_end(void)
274{}
275static inline void delayacct_compact_start(void)
276{}
277static inline void delayacct_compact_end(void)
278{}
279static inline void delayacct_wpcopy_start(void)
280{}
281static inline void delayacct_wpcopy_end(void)
282{}
283static inline void delayacct_irq(struct task_struct *task, u32 delta)
284{}
285
286#endif /* CONFIG_TASK_DELAY_ACCT */
287
288#endif