func_latency.bpf.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
  2// Copyright (c) 2021 Google
  3#include "vmlinux.h"
  4#include <bpf/bpf_helpers.h>
  5#include <bpf/bpf_tracing.h>
  6
  7// This should be in sync with "util/ftrace.h"
  8#define NUM_BUCKET  22
  9
 10struct {
 11	__uint(type, BPF_MAP_TYPE_HASH);
 12	__uint(key_size, sizeof(__u64));
 13	__uint(value_size, sizeof(__u64));
 14	__uint(max_entries, 10000);
 15} functime SEC(".maps");
 16
 17struct {
 18	__uint(type, BPF_MAP_TYPE_HASH);
 19	__uint(key_size, sizeof(__u32));
 20	__uint(value_size, sizeof(__u8));
 21	__uint(max_entries, 1);
 22} cpu_filter SEC(".maps");
 23
 24struct {
 25	__uint(type, BPF_MAP_TYPE_HASH);
 26	__uint(key_size, sizeof(__u32));
 27	__uint(value_size, sizeof(__u8));
 28	__uint(max_entries, 1);
 29} task_filter SEC(".maps");
 30
 31struct {
 32	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
 33	__uint(key_size, sizeof(__u32));
 34	__uint(value_size, sizeof(__u64));
 35	__uint(max_entries, NUM_BUCKET);
 36} latency SEC(".maps");
 37
 38
 39int enabled = 0;
 40
 41// stats
 42__s64 total;
 43__s64 count;
 44__s64 max;
 45__s64 min;
 46
 47const volatile int has_cpu = 0;
 48const volatile int has_task = 0;
 49const volatile int use_nsec = 0;
 50const volatile unsigned int bucket_range;
 51const volatile unsigned int min_latency;
 52const volatile unsigned int max_latency;
 53const volatile unsigned int bucket_num = NUM_BUCKET;
 54
 55static bool can_record(void)
 56{
 57	if (has_cpu) {
 58		__u32 cpu = bpf_get_smp_processor_id();
 59		__u8 *ok;
 60
 61		ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
 62		if (!ok)
 63			return false;
 64	}
 65
 66	if (has_task) {
 67		__u32 pid = bpf_get_current_pid_tgid();
 68		__u8 *ok;
 69
 70		ok = bpf_map_lookup_elem(&task_filter, &pid);
 71		if (!ok)
 72			return false;
 73	}
 74	return true;
 75}
 76
 77static void update_latency(__s64 delta)
 78{
 79	__u64 val = delta;
 80	__u32 key = 0;
 81	__u64 *hist;
 82	__u64 cmp_base = use_nsec ? 1 : 1000;
 83
 84	if (delta < 0)
 85		return;
 86
 87	if (bucket_range != 0) {
 88		val = delta / cmp_base;
 89
 90		if (min_latency > 0) {
 91			if (val > min_latency)
 92				val -= min_latency;
 93			else
 94				goto do_lookup;
 95		}
 96
 97		// Less than 1 unit (ms or ns), or, in the future,
 98		// than the min latency desired.
 99		if (val > 0) { // 1st entry: [ 1 unit .. bucket_range units )
100			key = val / bucket_range + 1;
101			if (key >= bucket_num)
102				key = bucket_num - 1;
103		}
104
105		goto do_lookup;
106	}
107	// calculate index using delta
108	for (key = 0; key < (bucket_num - 1); key++) {
109		if (delta < (cmp_base << key))
110			break;
111	}
112
113do_lookup:
114	hist = bpf_map_lookup_elem(&latency, &key);
115	if (!hist)
116		return;
117
118	__sync_fetch_and_add(hist, 1);
119
120	__sync_fetch_and_add(&total, delta); // always in nsec
121	__sync_fetch_and_add(&count, 1);
122
123	if (delta > max)
124		max = delta;
125	if (delta < min)
126		min = delta;
127}
128
129SEC("kprobe/func")
130int BPF_PROG(func_begin)
131{
132	__u64 key, now;
133
134	if (!enabled || !can_record())
135		return 0;
136
137	key = bpf_get_current_pid_tgid();
138	now = bpf_ktime_get_ns();
139
140	// overwrite timestamp for nested functions
141	bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
142	return 0;
143}
144
145SEC("kretprobe/func")
146int BPF_PROG(func_end)
147{
148	__u64 tid;
149	__u64 *start;
150
151	if (!enabled)
152		return 0;
153
154	tid = bpf_get_current_pid_tgid();
155
156	start = bpf_map_lookup_elem(&functime, &tid);
157	if (start) {
158		update_latency(bpf_ktime_get_ns() - *start);
159		bpf_map_delete_elem(&functime, &tid);
160	}
161
162	return 0;
163}
164
165SEC("raw_tp")
166int BPF_PROG(event_begin)
167{
168	__u64 key, now;
169
170	if (!enabled || !can_record())
171		return 0;
172
173	key = bpf_get_current_pid_tgid();
174	now = bpf_ktime_get_ns();
175
176	// overwrite timestamp for nested events
177	bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
178	return 0;
179}
180
181SEC("raw_tp")
182int BPF_PROG(event_end)
183{
184	__u64 tid;
185	__u64 *start;
186
187	if (!enabled)
188		return 0;
189
190	tid = bpf_get_current_pid_tgid();
191
192	start = bpf_map_lookup_elem(&functime, &tid);
193	if (start) {
194		update_latency(bpf_ktime_get_ns() - *start);
195		bpf_map_delete_elem(&functime, &tid);
196	}
197
198	return 0;
199}