psi_types.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _LINUX_PSI_TYPES_H
  3#define _LINUX_PSI_TYPES_H
  4
  5#include <linux/kthread.h>
  6#include <linux/seqlock.h>
  7#include <linux/types.h>
  8#include <linux/kref.h>
  9#include <linux/wait.h>
 10
 11#ifdef CONFIG_PSI
 12
 13/* Tracked task states */
 14enum psi_task_count {
 15	NR_IOWAIT,
 16	NR_MEMSTALL,
 17	NR_RUNNING,
 18	/*
 19	 * For IO and CPU stalls the presence of running/oncpu tasks
 20	 * in the domain means a partial rather than a full stall.
 21	 * For memory it's not so simple because of page reclaimers:
 22	 * they are running/oncpu while representing a stall. To tell
 23	 * whether a domain has productivity left or not, we need to
 24	 * distinguish between regular running (i.e. productive)
 25	 * threads and memstall ones.
 26	 */
 27	NR_MEMSTALL_RUNNING,
 28	NR_PSI_TASK_COUNTS = 4,
 29};
 30
 31/* Task state bitmasks */
 32#define TSK_IOWAIT	(1 << NR_IOWAIT)
 33#define TSK_MEMSTALL	(1 << NR_MEMSTALL)
 34#define TSK_RUNNING	(1 << NR_RUNNING)
 35#define TSK_MEMSTALL_RUNNING	(1 << NR_MEMSTALL_RUNNING)
 36
 37/* Only one task can be scheduled, no corresponding task count */
 38#define TSK_ONCPU	(1 << NR_PSI_TASK_COUNTS)
 39
 40/* Resources that workloads could be stalled on */
 41enum psi_res {
 42	PSI_IO,
 43	PSI_MEM,
 44	PSI_CPU,
 45#ifdef CONFIG_IRQ_TIME_ACCOUNTING
 46	PSI_IRQ,
 47#endif
 48	NR_PSI_RESOURCES,
 49};
 50
 51/*
 52 * Pressure states for each resource:
 53 *
 54 * SOME: Stalled tasks & working tasks
 55 * FULL: Stalled tasks & no working tasks
 56 */
 57enum psi_states {
 58	PSI_IO_SOME,
 59	PSI_IO_FULL,
 60	PSI_MEM_SOME,
 61	PSI_MEM_FULL,
 62	PSI_CPU_SOME,
 63	PSI_CPU_FULL,
 64#ifdef CONFIG_IRQ_TIME_ACCOUNTING
 65	PSI_IRQ_FULL,
 66#endif
 67	/* Only per-CPU, to weigh the CPU in the global average: */
 68	PSI_NONIDLE,
 69	NR_PSI_STATES,
 70};
 71
 72/* Use one bit in the state mask to track TSK_ONCPU */
 73#define PSI_ONCPU	(1 << NR_PSI_STATES)
 74
 75/* Flag whether to re-arm avgs_work, see details in get_recent_times() */
 76#define PSI_STATE_RESCHEDULE	(1 << (NR_PSI_STATES + 1))
 77
 78enum psi_aggregators {
 79	PSI_AVGS = 0,
 80	PSI_POLL,
 81	NR_PSI_AGGREGATORS,
 82};
 83
 84struct psi_group_cpu {
 85	/* 1st cacheline updated by the scheduler */
 86
 87	/* States of the tasks belonging to this group */
 88	unsigned int tasks[NR_PSI_TASK_COUNTS]
 89			____cacheline_aligned_in_smp;
 90
 91	/* Aggregate pressure state derived from the tasks */
 92	u32 state_mask;
 93
 94	/* Period time sampling buckets for each state of interest (ns) */
 95	u32 times[NR_PSI_STATES];
 96
 97	/* Time of last task change in this group (rq_clock) */
 98	u64 state_start;
 99
100	/* 2nd cacheline updated by the aggregator */
101
102	/* Delta detection against the sampling buckets */
103	u32 times_prev[NR_PSI_AGGREGATORS][NR_PSI_STATES]
104			____cacheline_aligned_in_smp;
105};
106
107/* PSI growth tracking window */
108struct psi_window {
109	/* Window size in ns */
110	u64 size;
111
112	/* Start time of the current window in ns */
113	u64 start_time;
114
115	/* Value at the start of the window */
116	u64 start_value;
117
118	/* Value growth in the previous window */
119	u64 prev_growth;
120};
121
122struct psi_trigger {
123	/* PSI state being monitored by the trigger */
124	enum psi_states state;
125
126	/* User-spacified threshold in ns */
127	u64 threshold;
128
129	/* List node inside triggers list */
130	struct list_head node;
131
132	/* Backpointer needed during trigger destruction */
133	struct psi_group *group;
134
135	/* Wait queue for polling */
136	wait_queue_head_t event_wait;
137
138	/* Kernfs file for cgroup triggers */
139	struct kernfs_open_file *of;
140
141	/* Pending event flag */
142	int event;
143
144	/* Tracking window */
145	struct psi_window win;
146
147	/*
148	 * Time last event was generated. Used for rate-limiting
149	 * events to one per window
150	 */
151	u64 last_event_time;
152
153	/* Deferred event(s) from previous ratelimit window */
154	bool pending_event;
155
156	/* Trigger type - PSI_AVGS for unprivileged, PSI_POLL for RT */
157	enum psi_aggregators aggregator;
158};
159
160struct psi_group {
161	struct psi_group *parent;
162	bool enabled;
163
164	/* Protects data used by the aggregator */
165	struct mutex avgs_lock;
166
167	/* Per-cpu task state & time tracking */
168	struct psi_group_cpu __percpu *pcpu;
169
170	/* Running pressure averages */
171	u64 avg_total[NR_PSI_STATES - 1];
172	u64 avg_last_update;
173	u64 avg_next_update;
174
175	/* Aggregator work control */
176	struct delayed_work avgs_work;
177
178	/* Unprivileged triggers against N*PSI_FREQ windows */
179	struct list_head avg_triggers;
180	u32 avg_nr_triggers[NR_PSI_STATES - 1];
181
182	/* Total stall times and sampled pressure averages */
183	u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1];
184	unsigned long avg[NR_PSI_STATES - 1][3];
185
186	/* Monitor RT polling work control */
187	struct task_struct __rcu *rtpoll_task;
188	struct timer_list rtpoll_timer;
189	wait_queue_head_t rtpoll_wait;
190	atomic_t rtpoll_wakeup;
191	atomic_t rtpoll_scheduled;
192
193	/* Protects data used by the monitor */
194	struct mutex rtpoll_trigger_lock;
195
196	/* Configured RT polling triggers */
197	struct list_head rtpoll_triggers;
198	u32 rtpoll_nr_triggers[NR_PSI_STATES - 1];
199	u32 rtpoll_states;
200	u64 rtpoll_min_period;
201
202	/* Total stall times at the start of RT polling monitor activation */
203	u64 rtpoll_total[NR_PSI_STATES - 1];
204	u64 rtpoll_next_update;
205	u64 rtpoll_until;
206};
207
208#else /* CONFIG_PSI */
209
210#define NR_PSI_RESOURCES	0
211
212struct psi_group { };
213
214#endif /* CONFIG_PSI */
215
216#endif /* _LINUX_PSI_TYPES_H */