kwork_trace.bpf.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
  2// Copyright (c) 2022, Huawei
  3
  4#include "vmlinux.h"
  5#include <bpf/bpf_helpers.h>
  6#include <bpf/bpf_tracing.h>
  7
  8#define KWORK_COUNT 100
  9#define MAX_KWORKNAME 128
 10
 11/*
 12 * This should be in sync with "util/kwork.h"
 13 */
 14enum kwork_class_type {
 15	KWORK_CLASS_IRQ,
 16	KWORK_CLASS_SOFTIRQ,
 17	KWORK_CLASS_WORKQUEUE,
 18	KWORK_CLASS_MAX,
 19};
 20
 21struct work_key {
 22	__u32 type;
 23	__u32 cpu;
 24	__u64 id;
 25};
 26
 27struct report_data {
 28	__u64 nr;
 29	__u64 total_time;
 30	__u64 max_time;
 31	__u64 max_time_start;
 32	__u64 max_time_end;
 33};
 34
 35struct {
 36	__uint(type, BPF_MAP_TYPE_HASH);
 37	__uint(key_size, sizeof(struct work_key));
 38	__uint(value_size, MAX_KWORKNAME);
 39	__uint(max_entries, KWORK_COUNT);
 40} perf_kwork_names SEC(".maps");
 41
 42struct {
 43	__uint(type, BPF_MAP_TYPE_HASH);
 44	__uint(key_size, sizeof(struct work_key));
 45	__uint(value_size, sizeof(__u64));
 46	__uint(max_entries, KWORK_COUNT);
 47} perf_kwork_time SEC(".maps");
 48
 49struct {
 50	__uint(type, BPF_MAP_TYPE_HASH);
 51	__uint(key_size, sizeof(struct work_key));
 52	__uint(value_size, sizeof(struct report_data));
 53	__uint(max_entries, KWORK_COUNT);
 54} perf_kwork_report SEC(".maps");
 55
 56struct {
 57	__uint(type, BPF_MAP_TYPE_HASH);
 58	__uint(key_size, sizeof(__u32));
 59	__uint(value_size, sizeof(__u8));
 60	__uint(max_entries, 1);
 61} perf_kwork_cpu_filter SEC(".maps");
 62
 63struct {
 64	__uint(type, BPF_MAP_TYPE_ARRAY);
 65	__uint(key_size, sizeof(__u32));
 66	__uint(value_size, MAX_KWORKNAME);
 67	__uint(max_entries, 1);
 68} perf_kwork_name_filter SEC(".maps");
 69
 70int enabled = 0;
 71
 72const volatile int has_cpu_filter = 0;
 73const volatile int has_name_filter = 0;
 74
 75static __always_inline int local_strncmp(const char *s1,
 76					 unsigned int sz, const char *s2)
 77{
 78	int ret = 0;
 79	unsigned int i;
 80
 81	for (i = 0; i < sz; i++) {
 82		ret = (unsigned char)s1[i] - (unsigned char)s2[i];
 83		if (ret || !s1[i])
 84			break;
 85	}
 86
 87	return ret;
 88}
 89
 90static __always_inline int trace_event_match(struct work_key *key, char *name)
 91{
 92	__u8 *cpu_val;
 93	char *name_val;
 94	__u32 zero = 0;
 95	__u32 cpu = bpf_get_smp_processor_id();
 96
 97	if (!enabled)
 98		return 0;
 99
100	if (has_cpu_filter) {
101		cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu);
102		if (!cpu_val)
103			return 0;
104	}
105
106	if (has_name_filter && (name != NULL)) {
107		name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero);
108		if (name_val &&
109		    (local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) {
110			return 0;
111		}
112	}
113
114	return 1;
115}
116
117static __always_inline void do_update_time(void *map, struct work_key *key,
118					   __u64 time_start, __u64 time_end)
119{
120	struct report_data zero, *data;
121	__s64 delta = time_end - time_start;
122
123	if (delta < 0)
124		return;
125
126	data = bpf_map_lookup_elem(map, key);
127	if (!data) {
128		__builtin_memset(&zero, 0, sizeof(zero));
129		bpf_map_update_elem(map, key, &zero, BPF_NOEXIST);
130		data = bpf_map_lookup_elem(map, key);
131		if (!data)
132			return;
133	}
134
135	if ((delta > data->max_time) ||
136	    (data->max_time == 0)) {
137		data->max_time       = delta;
138		data->max_time_start = time_start;
139		data->max_time_end   = time_end;
140	}
141
142	data->total_time += delta;
143	data->nr++;
144}
145
146static __always_inline void do_update_timestart(void *map, struct work_key *key)
147{
148	__u64 ts = bpf_ktime_get_ns();
149
150	bpf_map_update_elem(map, key, &ts, BPF_ANY);
151}
152
153static __always_inline void do_update_timeend(void *report_map, void *time_map,
154					      struct work_key *key)
155{
156	__u64 *time = bpf_map_lookup_elem(time_map, key);
157
158	if (time) {
159		bpf_map_delete_elem(time_map, key);
160		do_update_time(report_map, key, *time, bpf_ktime_get_ns());
161	}
162}
163
164static __always_inline void do_update_name(void *map,
165					   struct work_key *key, char *name)
166{
167	if (!bpf_map_lookup_elem(map, key))
168		bpf_map_update_elem(map, key, name, BPF_ANY);
169}
170
171static __always_inline int update_timestart(void *map, struct work_key *key)
172{
173	if (!trace_event_match(key, NULL))
174		return 0;
175
176	do_update_timestart(map, key);
177	return 0;
178}
179
180static __always_inline int update_timestart_and_name(void *time_map,
181						     void *names_map,
182						     struct work_key *key,
183						     char *name)
184{
185	if (!trace_event_match(key, name))
186		return 0;
187
188	do_update_timestart(time_map, key);
189	do_update_name(names_map, key, name);
190
191	return 0;
192}
193
194static __always_inline int update_timeend(void *report_map,
195					  void *time_map, struct work_key *key)
196{
197	if (!trace_event_match(key, NULL))
198		return 0;
199
200	do_update_timeend(report_map, time_map, key);
201
202	return 0;
203}
204
205static __always_inline int update_timeend_and_name(void *report_map,
206						   void *time_map,
207						   void *names_map,
208						   struct work_key *key,
209						   char *name)
210{
211	if (!trace_event_match(key, name))
212		return 0;
213
214	do_update_timeend(report_map, time_map, key);
215	do_update_name(names_map, key, name);
216
217	return 0;
218}
219
220SEC("tracepoint/irq/irq_handler_entry")
221int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx)
222{
223	char name[MAX_KWORKNAME];
224	struct work_key key = {
225		.type = KWORK_CLASS_IRQ,
226		.cpu  = bpf_get_smp_processor_id(),
227		.id   = (__u64)ctx->irq,
228	};
229	void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff);
230
231	bpf_probe_read_kernel_str(name, sizeof(name), name_addr);
232
233	return update_timestart_and_name(&perf_kwork_time,
234					 &perf_kwork_names, &key, name);
235}
236
237SEC("tracepoint/irq/irq_handler_exit")
238int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx)
239{
240	struct work_key key = {
241		.type = KWORK_CLASS_IRQ,
242		.cpu  = bpf_get_smp_processor_id(),
243		.id   = (__u64)ctx->irq,
244	};
245
246	return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
247}
248
249static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = {
250	{ "HI"       },
251	{ "TIMER"    },
252	{ "NET_TX"   },
253	{ "NET_RX"   },
254	{ "BLOCK"    },
255	{ "IRQ_POLL" },
256	{ "TASKLET"  },
257	{ "SCHED"    },
258	{ "HRTIMER"  },
259	{ "RCU"      },
260};
261
262SEC("tracepoint/irq/softirq_entry")
263int report_softirq_entry(struct trace_event_raw_softirq *ctx)
264{
265	unsigned int vec = ctx->vec;
266	struct work_key key = {
267		.type = KWORK_CLASS_SOFTIRQ,
268		.cpu  = bpf_get_smp_processor_id(),
269		.id   = (__u64)vec,
270	};
271
272	if (vec < NR_SOFTIRQS) {
273		return update_timestart_and_name(&perf_kwork_time,
274						 &perf_kwork_names, &key,
275						 softirq_name_list[vec]);
276	}
277
278	return 0;
279}
280
281SEC("tracepoint/irq/softirq_exit")
282int report_softirq_exit(struct trace_event_raw_softirq *ctx)
283{
284	struct work_key key = {
285		.type = KWORK_CLASS_SOFTIRQ,
286		.cpu  = bpf_get_smp_processor_id(),
287		.id   = (__u64)ctx->vec,
288	};
289
290	return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
291}
292
293SEC("tracepoint/irq/softirq_raise")
294int latency_softirq_raise(struct trace_event_raw_softirq *ctx)
295{
296	unsigned int vec = ctx->vec;
297	struct work_key key = {
298		.type = KWORK_CLASS_SOFTIRQ,
299		.cpu  = bpf_get_smp_processor_id(),
300		.id   = (__u64)vec,
301	};
302
303	if (vec < NR_SOFTIRQS) {
304		return update_timestart_and_name(&perf_kwork_time,
305						 &perf_kwork_names, &key,
306						 softirq_name_list[vec]);
307	}
308
309	return 0;
310}
311
312SEC("tracepoint/irq/softirq_entry")
313int latency_softirq_entry(struct trace_event_raw_softirq *ctx)
314{
315	struct work_key key = {
316		.type = KWORK_CLASS_SOFTIRQ,
317		.cpu  = bpf_get_smp_processor_id(),
318		.id   = (__u64)ctx->vec,
319	};
320
321	return update_timeend(&perf_kwork_report, &perf_kwork_time, &key);
322}
323
324SEC("tracepoint/workqueue/workqueue_execute_start")
325int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
326{
327	struct work_key key = {
328		.type = KWORK_CLASS_WORKQUEUE,
329		.cpu  = bpf_get_smp_processor_id(),
330		.id   = (__u64)ctx->work,
331	};
332
333	return update_timestart(&perf_kwork_time, &key);
334}
335
336SEC("tracepoint/workqueue/workqueue_execute_end")
337int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx)
338{
339	char name[MAX_KWORKNAME];
340	struct work_key key = {
341		.type = KWORK_CLASS_WORKQUEUE,
342		.cpu  = bpf_get_smp_processor_id(),
343		.id   = (__u64)ctx->work,
344	};
345	unsigned long long func_addr = (unsigned long long)ctx->function;
346
347	__builtin_memset(name, 0, sizeof(name));
348	bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
349
350	return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
351				       &perf_kwork_names, &key, name);
352}
353
354SEC("tracepoint/workqueue/workqueue_activate_work")
355int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx)
356{
357	struct work_key key = {
358		.type = KWORK_CLASS_WORKQUEUE,
359		.cpu  = bpf_get_smp_processor_id(),
360		.id   = (__u64)ctx->work,
361	};
362
363	return update_timestart(&perf_kwork_time, &key);
364}
365
366SEC("tracepoint/workqueue/workqueue_execute_start")
367int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx)
368{
369	char name[MAX_KWORKNAME];
370	struct work_key key = {
371		.type = KWORK_CLASS_WORKQUEUE,
372		.cpu  = bpf_get_smp_processor_id(),
373		.id   = (__u64)ctx->work,
374	};
375	unsigned long long func_addr = (unsigned long long)ctx->function;
376
377	__builtin_memset(name, 0, sizeof(name));
378	bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr));
379
380	return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time,
381				       &perf_kwork_names, &key, name);
382}
383
384char LICENSE[] SEC("license") = "Dual BSD/GPL";