rcu_read_lock.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
  3
  4#include "vmlinux.h"
  5#include <bpf/bpf_helpers.h>
  6#include <bpf/bpf_tracing.h>
  7#include "bpf_tracing_net.h"
  8#include "bpf_misc.h"
  9
 10char _license[] SEC("license") = "GPL";
 11
 12struct {
 13	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
 14	__uint(map_flags, BPF_F_NO_PREALLOC);
 15	__type(key, int);
 16	__type(value, long);
 17} map_a SEC(".maps");
 18
 19__u32 user_data, target_pid;
 20__s32 key_serial;
 21__u64 flags, task_storage_val, cgroup_id;
 22
 23struct bpf_key *bpf_lookup_user_key(__s32 serial, __u64 flags) __ksym;
 24void bpf_key_put(struct bpf_key *key) __ksym;
 25void bpf_rcu_read_lock(void) __ksym;
 26void bpf_rcu_read_unlock(void) __ksym;
 27struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
 28void bpf_task_release(struct task_struct *p) __ksym;
 29
 30SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
 31int get_cgroup_id(void *ctx)
 32{
 33	struct task_struct *task;
 34	struct css_set *cgroups;
 35
 36	task = bpf_get_current_task_btf();
 37	if (task->pid != target_pid)
 38		return 0;
 39
 40	/* simulate bpf_get_current_cgroup_id() helper */
 41	bpf_rcu_read_lock();
 42	cgroups = task->cgroups;
 43	if (!cgroups)
 44		goto unlock;
 45	cgroup_id = cgroups->dfl_cgrp->kn->id;
 46unlock:
 47	bpf_rcu_read_unlock();
 48	return 0;
 49}
 50
 51SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
 52int task_succ(void *ctx)
 53{
 54	struct task_struct *task, *real_parent;
 55	long init_val = 2;
 56	long *ptr;
 57
 58	task = bpf_get_current_task_btf();
 59	if (task->pid != target_pid)
 60		return 0;
 61
 62	bpf_rcu_read_lock();
 63	/* region including helper using rcu ptr real_parent */
 64	real_parent = task->real_parent;
 65	if (!real_parent)
 66		goto out;
 67	ptr = bpf_task_storage_get(&map_a, real_parent, &init_val,
 68				   BPF_LOCAL_STORAGE_GET_F_CREATE);
 69	if (!ptr)
 70		goto out;
 71	ptr = bpf_task_storage_get(&map_a, real_parent, 0, 0);
 72	if (!ptr)
 73		goto out;
 74	task_storage_val = *ptr;
 75out:
 76	bpf_rcu_read_unlock();
 77	return 0;
 78}
 79
 80SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
 81int no_lock(void *ctx)
 82{
 83	struct task_struct *task, *real_parent;
 84
 85	/* old style ptr_to_btf_id is not allowed in sleepable */
 86	task = bpf_get_current_task_btf();
 87	real_parent = task->real_parent;
 88	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
 89	return 0;
 90}
 91
 92SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
 93int two_regions(void *ctx)
 94{
 95	struct task_struct *task, *real_parent;
 96
 97	/* two regions */
 98	task = bpf_get_current_task_btf();
 99	bpf_rcu_read_lock();
100	bpf_rcu_read_unlock();
101	bpf_rcu_read_lock();
102	real_parent = task->real_parent;
103	if (!real_parent)
104		goto out;
105	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
106out:
107	bpf_rcu_read_unlock();
108	return 0;
109}
110
111SEC("?fentry/" SYS_PREFIX "sys_getpgid")
112int non_sleepable_1(void *ctx)
113{
114	struct task_struct *task, *real_parent;
115
116	task = bpf_get_current_task_btf();
117	bpf_rcu_read_lock();
118	real_parent = task->real_parent;
119	if (!real_parent)
120		goto out;
121	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
122out:
123	bpf_rcu_read_unlock();
124	return 0;
125}
126
127SEC("?fentry/" SYS_PREFIX "sys_getpgid")
128int non_sleepable_2(void *ctx)
129{
130	struct task_struct *task, *real_parent;
131
132	bpf_rcu_read_lock();
133	task = bpf_get_current_task_btf();
134	bpf_rcu_read_unlock();
135
136	bpf_rcu_read_lock();
137	real_parent = task->real_parent;
138	if (!real_parent)
139		goto out;
140	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
141out:
142	bpf_rcu_read_unlock();
143	return 0;
144}
145
146SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
147int task_acquire(void *ctx)
148{
149	struct task_struct *task, *real_parent, *gparent;
150
151	task = bpf_get_current_task_btf();
152	bpf_rcu_read_lock();
153	real_parent = task->real_parent;
154	if (!real_parent)
155		goto out;
156
157	/* rcu_ptr->rcu_field */
158	gparent = real_parent->real_parent;
159	if (!gparent)
160		goto out;
161
162	/* acquire a reference which can be used outside rcu read lock region */
163	gparent = bpf_task_acquire(gparent);
164	if (!gparent)
165		goto out;
166
167	(void)bpf_task_storage_get(&map_a, gparent, 0, 0);
168	bpf_task_release(gparent);
169out:
170	bpf_rcu_read_unlock();
171	return 0;
172}
173
174SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
175int miss_lock(void *ctx)
176{
177	struct task_struct *task;
178
179	/* missing bpf_rcu_read_lock() */
180	task = bpf_get_current_task_btf();
181	bpf_rcu_read_lock();
182	(void)bpf_task_storage_get(&map_a, task, 0, 0);
183	bpf_rcu_read_unlock();
184	bpf_rcu_read_unlock();
185	return 0;
186}
187
188SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
189int miss_unlock(void *ctx)
190{
191	struct task_struct *task;
192
193	/* missing bpf_rcu_read_unlock() */
194	task = bpf_get_current_task_btf();
195	bpf_rcu_read_lock();
196	(void)bpf_task_storage_get(&map_a, task, 0, 0);
197	return 0;
198}
199
200SEC("?fentry/" SYS_PREFIX "sys_getpgid")
201int non_sleepable_rcu_mismatch(void *ctx)
202{
203	struct task_struct *task, *real_parent;
204
205	task = bpf_get_current_task_btf();
206	/* non-sleepable: missing bpf_rcu_read_unlock() in one path */
207	bpf_rcu_read_lock();
208	real_parent = task->real_parent;
209	if (!real_parent)
210		goto out;
211	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
212	if (real_parent)
213		bpf_rcu_read_unlock();
214out:
215	return 0;
216}
217
218SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
219int inproper_sleepable_helper(void *ctx)
220{
221	struct task_struct *task, *real_parent;
222	struct pt_regs *regs;
223	__u32 value = 0;
224	void *ptr;
225
226	task = bpf_get_current_task_btf();
227	/* sleepable helper in rcu read lock region */
228	bpf_rcu_read_lock();
229	real_parent = task->real_parent;
230	if (!real_parent)
231		goto out;
232	regs = (struct pt_regs *)bpf_task_pt_regs(real_parent);
233	if (!regs)
234		goto out;
235
236	ptr = (void *)PT_REGS_IP(regs);
237	(void)bpf_copy_from_user_task(&value, sizeof(uint32_t), ptr, task, 0);
238	user_data = value;
239	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
240out:
241	bpf_rcu_read_unlock();
242	return 0;
243}
244
245SEC("?lsm.s/bpf")
246int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size,
247	     bool kernel)
248{
249	struct bpf_key *bkey;
250
251	/* sleepable kfunc in rcu read lock region */
252	bpf_rcu_read_lock();
253	bkey = bpf_lookup_user_key(key_serial, flags);
254	bpf_rcu_read_unlock();
255	if (!bkey)
256		return -1;
257	bpf_key_put(bkey);
258
259	return 0;
260}
261
262SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
263int nested_rcu_region(void *ctx)
264{
265	struct task_struct *task, *real_parent;
266
267	/* nested rcu read lock regions */
268	task = bpf_get_current_task_btf();
269	bpf_rcu_read_lock();
270	bpf_rcu_read_lock();
271	real_parent = task->real_parent;
272	if (!real_parent)
273		goto out;
274	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
275out:
276	bpf_rcu_read_unlock();
277	bpf_rcu_read_unlock();
278	return 0;
279}
280
281SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
282int nested_rcu_region_unbalanced_1(void *ctx)
283{
284	struct task_struct *task, *real_parent;
285
286	/* nested rcu read lock regions */
287	task = bpf_get_current_task_btf();
288	bpf_rcu_read_lock();
289	bpf_rcu_read_lock();
290	real_parent = task->real_parent;
291	if (!real_parent)
292		goto out;
293	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
294out:
295	bpf_rcu_read_unlock();
296	bpf_rcu_read_unlock();
297	bpf_rcu_read_unlock();
298	return 0;
299}
300
301SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
302int nested_rcu_region_unbalanced_2(void *ctx)
303{
304	struct task_struct *task, *real_parent;
305
306	/* nested rcu read lock regions */
307	task = bpf_get_current_task_btf();
308	bpf_rcu_read_lock();
309	bpf_rcu_read_lock();
310	bpf_rcu_read_lock();
311	real_parent = task->real_parent;
312	if (!real_parent)
313		goto out;
314	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
315out:
316	bpf_rcu_read_unlock();
317	bpf_rcu_read_unlock();
318	return 0;
319}
320
321SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
322int task_trusted_non_rcuptr(void *ctx)
323{
324	struct task_struct *task, *group_leader;
325
326	task = bpf_get_current_task_btf();
327	bpf_rcu_read_lock();
328	/* the pointer group_leader is explicitly marked as trusted */
329	group_leader = task->real_parent->group_leader;
330	(void)bpf_task_storage_get(&map_a, group_leader, 0, 0);
331	bpf_rcu_read_unlock();
332	return 0;
333}
334
335SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
336int task_untrusted_rcuptr(void *ctx)
337{
338	struct task_struct *task, *real_parent;
339
340	task = bpf_get_current_task_btf();
341	bpf_rcu_read_lock();
342	real_parent = task->real_parent;
343	bpf_rcu_read_unlock();
344	/* helper use of rcu ptr outside the rcu read lock region */
345	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
346	return 0;
347}
348
349SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
350int cross_rcu_region(void *ctx)
351{
352	struct task_struct *task, *real_parent;
353
354	/* rcu ptr define/use in different regions */
355	task = bpf_get_current_task_btf();
356	bpf_rcu_read_lock();
357	real_parent = task->real_parent;
358	bpf_rcu_read_unlock();
359	bpf_rcu_read_lock();
360	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
361	bpf_rcu_read_unlock();
362	return 0;
363}
364
365__noinline
366static int static_subprog(void *ctx)
367{
368	volatile int ret = 0;
369
370	if (bpf_get_prandom_u32())
371		return ret + 42;
372	return ret + bpf_get_prandom_u32();
373}
374
375__noinline
376int global_subprog(u64 a)
377{
378	volatile int ret = a;
379
380	return ret + static_subprog(NULL);
381}
382
383__noinline
384static int static_subprog_lock(void *ctx)
385{
386	volatile int ret = 0;
387
388	bpf_rcu_read_lock();
389	if (bpf_get_prandom_u32())
390		return ret + 42;
391	return ret + bpf_get_prandom_u32();
392}
393
394__noinline
395int global_subprog_lock(u64 a)
396{
397	volatile int ret = a;
398
399	return ret + static_subprog_lock(NULL);
400}
401
402__noinline
403static int static_subprog_unlock(void *ctx)
404{
405	volatile int ret = 0;
406
407	bpf_rcu_read_unlock();
408	if (bpf_get_prandom_u32())
409		return ret + 42;
410	return ret + bpf_get_prandom_u32();
411}
412
413__noinline
414int global_subprog_unlock(u64 a)
415{
416	volatile int ret = a;
417
418	return ret + static_subprog_unlock(NULL);
419}
420
421SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
422int rcu_read_lock_subprog(void *ctx)
423{
424	volatile int ret = 0;
425
426	bpf_rcu_read_lock();
427	if (bpf_get_prandom_u32())
428		ret += static_subprog(ctx);
429	bpf_rcu_read_unlock();
430	return 0;
431}
432
433SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
434int rcu_read_lock_global_subprog(void *ctx)
435{
436	volatile int ret = 0;
437
438	bpf_rcu_read_lock();
439	if (bpf_get_prandom_u32())
440		ret += global_subprog(ret);
441	bpf_rcu_read_unlock();
442	return 0;
443}
444
445SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
446int rcu_read_lock_subprog_lock(void *ctx)
447{
448	volatile int ret = 0;
449
450	ret += static_subprog_lock(ctx);
451	bpf_rcu_read_unlock();
452	return 0;
453}
454
455SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
456int rcu_read_lock_global_subprog_lock(void *ctx)
457{
458	volatile int ret = 0;
459
460	ret += global_subprog_lock(ret);
461	bpf_rcu_read_unlock();
462	return 0;
463}
464
465SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
466int rcu_read_lock_subprog_unlock(void *ctx)
467{
468	volatile int ret = 0;
469
470	bpf_rcu_read_lock();
471	ret += static_subprog_unlock(ctx);
472	return 0;
473}
474
475SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
476int rcu_read_lock_global_subprog_unlock(void *ctx)
477{
478	volatile int ret = 0;
479
480	bpf_rcu_read_lock();
481	ret += global_subprog_unlock(ret);
482	return 0;
483}
484
485int __noinline
486global_sleepable_helper_subprog(int i)
487{
488	if (i)
489		bpf_copy_from_user(&i, sizeof(i), NULL);
490	return i;
491}
492
493int __noinline
494global_sleepable_kfunc_subprog(int i)
495{
496	if (i)
497		bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
498	global_subprog(i);
499	return i;
500}
501
502int __noinline
503global_subprog_calling_sleepable_global(int i)
504{
505	if (!i)
506		global_sleepable_kfunc_subprog(i);
507	return i;
508}
509
510SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
511int rcu_read_lock_sleepable_helper_global_subprog(void *ctx)
512{
513	volatile int ret = 0;
514
515	bpf_rcu_read_lock();
516	ret += global_sleepable_helper_subprog(ret);
517	bpf_rcu_read_unlock();
518	return 0;
519}
520
521SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
522int rcu_read_lock_sleepable_kfunc_global_subprog(void *ctx)
523{
524	volatile int ret = 0;
525
526	bpf_rcu_read_lock();
527	ret += global_sleepable_kfunc_subprog(ret);
528	bpf_rcu_read_unlock();
529	return 0;
530}
531
532SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
533int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx)
534{
535	volatile int ret = 0;
536
537	bpf_rcu_read_lock();
538	ret += global_subprog_calling_sleepable_global(ret);
539	bpf_rcu_read_unlock();
540	return 0;
541}