irq.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
  3#include <vmlinux.h>
  4#include <bpf/bpf_helpers.h>
  5#include "bpf_misc.h"
  6#include "bpf_experimental.h"
  7
  8unsigned long global_flags;
  9
 10extern void bpf_local_irq_save(unsigned long *) __weak __ksym;
 11extern void bpf_local_irq_restore(unsigned long *) __weak __ksym;
 12extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym;
 13
 14struct bpf_res_spin_lock lockA __hidden SEC(".data.A");
 15struct bpf_res_spin_lock lockB __hidden SEC(".data.B");
 16
 17SEC("?tc")
 18__failure __msg("arg#0 doesn't point to an irq flag on stack")
 19int irq_save_bad_arg(struct __sk_buff *ctx)
 20{
 21	bpf_local_irq_save(&global_flags);
 22	return 0;
 23}
 24
 25SEC("?tc")
 26__failure __msg("arg#0 doesn't point to an irq flag on stack")
 27int irq_restore_bad_arg(struct __sk_buff *ctx)
 28{
 29	bpf_local_irq_restore(&global_flags);
 30	return 0;
 31}
 32
 33SEC("?tc")
 34__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
 35int irq_restore_missing_2(struct __sk_buff *ctx)
 36{
 37	unsigned long flags1;
 38	unsigned long flags2;
 39
 40	bpf_local_irq_save(&flags1);
 41	bpf_local_irq_save(&flags2);
 42	return 0;
 43}
 44
 45SEC("?tc")
 46__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
 47int irq_restore_missing_3(struct __sk_buff *ctx)
 48{
 49	unsigned long flags1;
 50	unsigned long flags2;
 51	unsigned long flags3;
 52
 53	bpf_local_irq_save(&flags1);
 54	bpf_local_irq_save(&flags2);
 55	bpf_local_irq_save(&flags3);
 56	return 0;
 57}
 58
 59SEC("?tc")
 60__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
 61int irq_restore_missing_3_minus_2(struct __sk_buff *ctx)
 62{
 63	unsigned long flags1;
 64	unsigned long flags2;
 65	unsigned long flags3;
 66
 67	bpf_local_irq_save(&flags1);
 68	bpf_local_irq_save(&flags2);
 69	bpf_local_irq_save(&flags3);
 70	bpf_local_irq_restore(&flags3);
 71	bpf_local_irq_restore(&flags2);
 72	return 0;
 73}
 74
 75static __noinline void local_irq_save(unsigned long *flags)
 76{
 77	bpf_local_irq_save(flags);
 78}
 79
 80static __noinline void local_irq_restore(unsigned long *flags)
 81{
 82	bpf_local_irq_restore(flags);
 83}
 84
 85SEC("?tc")
 86__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
 87int irq_restore_missing_1_subprog(struct __sk_buff *ctx)
 88{
 89	unsigned long flags;
 90
 91	local_irq_save(&flags);
 92	return 0;
 93}
 94
 95SEC("?tc")
 96__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
 97int irq_restore_missing_2_subprog(struct __sk_buff *ctx)
 98{
 99	unsigned long flags1;
100	unsigned long flags2;
101
102	local_irq_save(&flags1);
103	local_irq_save(&flags2);
104	return 0;
105}
106
107SEC("?tc")
108__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
109int irq_restore_missing_3_subprog(struct __sk_buff *ctx)
110{
111	unsigned long flags1;
112	unsigned long flags2;
113	unsigned long flags3;
114
115	local_irq_save(&flags1);
116	local_irq_save(&flags2);
117	local_irq_save(&flags3);
118	return 0;
119}
120
121SEC("?tc")
122__failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
123int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx)
124{
125	unsigned long flags1;
126	unsigned long flags2;
127	unsigned long flags3;
128
129	local_irq_save(&flags1);
130	local_irq_save(&flags2);
131	local_irq_save(&flags3);
132	local_irq_restore(&flags3);
133	local_irq_restore(&flags2);
134	return 0;
135}
136
137SEC("?tc")
138__success
139int irq_balance(struct __sk_buff *ctx)
140{
141	unsigned long flags;
142
143	local_irq_save(&flags);
144	local_irq_restore(&flags);
145	return 0;
146}
147
148SEC("?tc")
149__success
150int irq_balance_n(struct __sk_buff *ctx)
151{
152	unsigned long flags1;
153	unsigned long flags2;
154	unsigned long flags3;
155
156	local_irq_save(&flags1);
157	local_irq_save(&flags2);
158	local_irq_save(&flags3);
159	local_irq_restore(&flags3);
160	local_irq_restore(&flags2);
161	local_irq_restore(&flags1);
162	return 0;
163}
164
165static __noinline void local_irq_balance(void)
166{
167	unsigned long flags;
168
169	local_irq_save(&flags);
170	local_irq_restore(&flags);
171}
172
173static __noinline void local_irq_balance_n(void)
174{
175	unsigned long flags1;
176	unsigned long flags2;
177	unsigned long flags3;
178
179	local_irq_save(&flags1);
180	local_irq_save(&flags2);
181	local_irq_save(&flags3);
182	local_irq_restore(&flags3);
183	local_irq_restore(&flags2);
184	local_irq_restore(&flags1);
185}
186
187SEC("?tc")
188__success
189int irq_balance_subprog(struct __sk_buff *ctx)
190{
191	local_irq_balance();
192	return 0;
193}
194
195SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
196__failure __msg("sleepable helper bpf_copy_from_user#")
197int irq_sleepable_helper(void *ctx)
198{
199	unsigned long flags;
200	u32 data;
201
202	local_irq_save(&flags);
203	bpf_copy_from_user(&data, sizeof(data), NULL);
204	local_irq_restore(&flags);
205	return 0;
206}
207
208SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
209__failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region")
210int irq_sleepable_kfunc(void *ctx)
211{
212	unsigned long flags;
213	u32 data;
214
215	local_irq_save(&flags);
216	bpf_copy_from_user_str(&data, sizeof(data), NULL, 0);
217	local_irq_restore(&flags);
218	return 0;
219}
220
221int __noinline global_local_irq_balance(void)
222{
223	local_irq_balance_n();
224	return 0;
225}
226
227SEC("?tc")
228__success
229int irq_global_subprog(struct __sk_buff *ctx)
230{
231	unsigned long flags;
232
233	bpf_local_irq_save(&flags);
234	global_local_irq_balance();
235	bpf_local_irq_restore(&flags);
236	return 0;
237}
238
239SEC("?tc")
240__failure __msg("cannot restore irq state out of order")
241int irq_restore_ooo(struct __sk_buff *ctx)
242{
243	unsigned long flags1;
244	unsigned long flags2;
245
246	bpf_local_irq_save(&flags1);
247	bpf_local_irq_save(&flags2);
248	bpf_local_irq_restore(&flags1);
249	bpf_local_irq_restore(&flags2);
250	return 0;
251}
252
253SEC("?tc")
254__failure __msg("cannot restore irq state out of order")
255int irq_restore_ooo_3(struct __sk_buff *ctx)
256{
257	unsigned long flags1;
258	unsigned long flags2;
259	unsigned long flags3;
260
261	bpf_local_irq_save(&flags1);
262	bpf_local_irq_save(&flags2);
263	bpf_local_irq_restore(&flags2);
264	bpf_local_irq_save(&flags3);
265	bpf_local_irq_restore(&flags1);
266	bpf_local_irq_restore(&flags3);
267	return 0;
268}
269
270static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2,
271					unsigned long *flags3)
272{
273	local_irq_save(flags1);
274	local_irq_save(flags2);
275	local_irq_save(flags3);
276}
277
278SEC("?tc")
279__success
280int irq_restore_3_subprog(struct __sk_buff *ctx)
281{
282	unsigned long flags1;
283	unsigned long flags2;
284	unsigned long flags3;
285
286	local_irq_save_3(&flags1, &flags2, &flags3);
287	bpf_local_irq_restore(&flags3);
288	bpf_local_irq_restore(&flags2);
289	bpf_local_irq_restore(&flags1);
290	return 0;
291}
292
293SEC("?tc")
294__failure __msg("cannot restore irq state out of order")
295int irq_restore_4_subprog(struct __sk_buff *ctx)
296{
297	unsigned long flags1;
298	unsigned long flags2;
299	unsigned long flags3;
300	unsigned long flags4;
301
302	local_irq_save_3(&flags1, &flags2, &flags3);
303	bpf_local_irq_restore(&flags3);
304	bpf_local_irq_save(&flags4);
305	bpf_local_irq_restore(&flags4);
306	bpf_local_irq_restore(&flags1);
307	return 0;
308}
309
310SEC("?tc")
311__failure __msg("cannot restore irq state out of order")
312int irq_restore_ooo_3_subprog(struct __sk_buff *ctx)
313{
314	unsigned long flags1;
315	unsigned long flags2;
316	unsigned long flags3;
317
318	local_irq_save_3(&flags1, &flags2, &flags3);
319	bpf_local_irq_restore(&flags3);
320	bpf_local_irq_restore(&flags2);
321	bpf_local_irq_save(&flags3);
322	bpf_local_irq_restore(&flags1);
323	return 0;
324}
325
326SEC("?tc")
327__failure __msg("expected an initialized")
328int irq_restore_invalid(struct __sk_buff *ctx)
329{
330	unsigned long flags1;
331	unsigned long flags = 0xfaceb00c;
332
333	bpf_local_irq_save(&flags1);
334	bpf_local_irq_restore(&flags);
335	return 0;
336}
337
338SEC("?tc")
339__failure __msg("expected uninitialized")
340int irq_save_invalid(struct __sk_buff *ctx)
341{
342	unsigned long flags1;
343
344	bpf_local_irq_save(&flags1);
345	bpf_local_irq_save(&flags1);
346	return 0;
347}
348
349SEC("?tc")
350__failure __msg("expected an initialized")
351int irq_restore_iter(struct __sk_buff *ctx)
352{
353	struct bpf_iter_num it;
354
355	bpf_iter_num_new(&it, 0, 42);
356	bpf_local_irq_restore((unsigned long *)&it);
357	return 0;
358}
359
360SEC("?tc")
361__failure __msg("Unreleased reference id=1")
362int irq_save_iter(struct __sk_buff *ctx)
363{
364	struct bpf_iter_num it;
365
366	/* Ensure same sized slot has st->ref_obj_id set, so we reject based on
367	 * slot_type != STACK_IRQ_FLAG...
368	 */
369	_Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size");
370
371	bpf_iter_num_new(&it, 0, 42);
372	bpf_local_irq_save((unsigned long *)&it);
373	bpf_local_irq_restore((unsigned long *)&it);
374	return 0;
375}
376
377SEC("?tc")
378__failure __msg("expected an initialized")
379int irq_flag_overwrite(struct __sk_buff *ctx)
380{
381	unsigned long flags;
382
383	bpf_local_irq_save(&flags);
384	flags = 0xdeadbeef;
385	bpf_local_irq_restore(&flags);
386	return 0;
387}
388
389SEC("?tc")
390__failure __msg("expected an initialized")
391int irq_flag_overwrite_partial(struct __sk_buff *ctx)
392{
393	unsigned long flags;
394
395	bpf_local_irq_save(&flags);
396	*(((char *)&flags) + 1) = 0xff;
397	bpf_local_irq_restore(&flags);
398	return 0;
399}
400
401SEC("?tc")
402__failure __msg("cannot restore irq state out of order")
403int irq_ooo_refs_array(struct __sk_buff *ctx)
404{
405	unsigned long flags[4];
406	struct { int i; } *p;
407
408	/* refs=1 */
409	bpf_local_irq_save(&flags[0]);
410
411	/* refs=1,2 */
412	p = bpf_obj_new(typeof(*p));
413	if (!p) {
414		bpf_local_irq_restore(&flags[0]);
415		return 0;
416	}
417
418	/* refs=1,2,3 */
419	bpf_local_irq_save(&flags[1]);
420
421	/* refs=1,2,3,4 */
422	bpf_local_irq_save(&flags[2]);
423
424	/* Now when we remove ref=2, the verifier must not break the ordering in
425	 * the refs array between 1,3,4. With an older implementation, the
426	 * verifier would swap the last element with the removed element, but to
427	 * maintain the stack property we need to use memmove.
428	 */
429	bpf_obj_drop(p);
430
431	/* Save and restore to reset active_irq_id to 3, as the ordering is now
432	 * refs=1,4,3. When restoring the linear scan will find prev_id in order
433	 * as 3 instead of 4.
434	 */
435	bpf_local_irq_save(&flags[3]);
436	bpf_local_irq_restore(&flags[3]);
437
438	/* With the incorrect implementation, we can release flags[1], flags[2],
439	 * and flags[0], i.e. in the wrong order.
440	 */
441	bpf_local_irq_restore(&flags[1]);
442	bpf_local_irq_restore(&flags[2]);
443	bpf_local_irq_restore(&flags[0]);
444	return 0;
445}
446
447int __noinline
448global_subprog(int i)
449{
450	if (i)
451		bpf_printk("%p", &i);
452	return i;
453}
454
455int __noinline
456global_sleepable_helper_subprog(int i)
457{
458	if (i)
459		bpf_copy_from_user(&i, sizeof(i), NULL);
460	return i;
461}
462
463int __noinline
464global_sleepable_kfunc_subprog(int i)
465{
466	if (i)
467		bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
468	global_subprog(i);
469	return i;
470}
471
472int __noinline
473global_subprog_calling_sleepable_global(int i)
474{
475	if (!i)
476		global_sleepable_kfunc_subprog(i);
477	return i;
478}
479
480SEC("?syscall")
481__success
482int irq_non_sleepable_global_subprog(void *ctx)
483{
484	unsigned long flags;
485
486	bpf_local_irq_save(&flags);
487	global_subprog(0);
488	bpf_local_irq_restore(&flags);
489	return 0;
490}
491
492SEC("?syscall")
493__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
494int irq_sleepable_helper_global_subprog(void *ctx)
495{
496	unsigned long flags;
497
498	bpf_local_irq_save(&flags);
499	global_sleepable_helper_subprog(0);
500	bpf_local_irq_restore(&flags);
501	return 0;
502}
503
504SEC("?syscall")
505__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
506int irq_sleepable_global_subprog_indirect(void *ctx)
507{
508	unsigned long flags;
509
510	bpf_local_irq_save(&flags);
511	global_subprog_calling_sleepable_global(0);
512	bpf_local_irq_restore(&flags);
513	return 0;
514}
515
516SEC("?tc")
517__failure __msg("cannot restore irq state out of order")
518int irq_ooo_lock_cond_inv(struct __sk_buff *ctx)
519{
520	unsigned long flags1, flags2;
521
522	if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
523		return 0;
524	if (bpf_res_spin_lock_irqsave(&lockB, &flags2)) {
525		bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
526		return 0;
527	}
528
529	bpf_res_spin_unlock_irqrestore(&lockB, &flags1);
530	bpf_res_spin_unlock_irqrestore(&lockA, &flags2);
531	return 0;
532}
533
534SEC("?tc")
535__failure __msg("function calls are not allowed")
536int irq_wrong_kfunc_class_1(struct __sk_buff *ctx)
537{
538	unsigned long flags1;
539
540	if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
541		return 0;
542	/* For now, bpf_local_irq_restore is not allowed in critical section,
543	 * but this test ensures error will be caught with kfunc_class when it's
544	 * opened up. Tested by temporarily permitting this kfunc in critical
545	 * section.
546	 */
547	bpf_local_irq_restore(&flags1);
548	bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
549	return 0;
550}
551
552SEC("?tc")
553__failure __msg("function calls are not allowed")
554int irq_wrong_kfunc_class_2(struct __sk_buff *ctx)
555{
556	unsigned long flags1, flags2;
557
558	bpf_local_irq_save(&flags1);
559	if (bpf_res_spin_lock_irqsave(&lockA, &flags2))
560		return 0;
561	bpf_local_irq_restore(&flags2);
562	bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
563	return 0;
564}
565
566char _license[] SEC("license") = "GPL";