verifier_bpf_fastcall.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <linux/bpf.h>
  4#include <bpf/bpf_helpers.h>
  5#include <bpf/bpf_core_read.h>
  6#include "../../../include/linux/filter.h"
  7#include "bpf_misc.h"
  8#include <stdbool.h>
  9#include "bpf_kfuncs.h"
 10
 11SEC("raw_tp")
 12__arch_x86_64
 13__log_level(4) __msg("stack depth 8")
 14__xlated("4: r5 = 5")
 15__xlated("5: r0 = ")
 16__xlated("6: r0 = &(void __percpu *)(r0)")
 17__xlated("7: r0 = *(u32 *)(r0 +0)")
 18__xlated("8: exit")
 19__success
 20__naked void simple(void)
 21{
 22	asm volatile (
 23	"r1 = 1;"
 24	"r2 = 2;"
 25	"r3 = 3;"
 26	"r4 = 4;"
 27	"r5 = 5;"
 28	"*(u64 *)(r10 - 16) = r1;"
 29	"*(u64 *)(r10 - 24) = r2;"
 30	"*(u64 *)(r10 - 32) = r3;"
 31	"*(u64 *)(r10 - 40) = r4;"
 32	"*(u64 *)(r10 - 48) = r5;"
 33	"call %[bpf_get_smp_processor_id];"
 34	"r5 = *(u64 *)(r10 - 48);"
 35	"r4 = *(u64 *)(r10 - 40);"
 36	"r3 = *(u64 *)(r10 - 32);"
 37	"r2 = *(u64 *)(r10 - 24);"
 38	"r1 = *(u64 *)(r10 - 16);"
 39	"exit;"
 40	:
 41	: __imm(bpf_get_smp_processor_id)
 42	: __clobber_all);
 43}
 44
 45/* The logic for detecting and verifying bpf_fastcall pattern is the same for
 46 * any arch, however x86 differs from arm64 or riscv64 in a way
 47 * bpf_get_smp_processor_id is rewritten:
 48 * - on x86 it is done by verifier
 49 * - on arm64 and riscv64 it is done by jit
 50 *
 51 * Which leads to different xlated patterns for different archs:
 52 * - on x86 the call is expanded as 3 instructions
 53 * - on arm64 and riscv64 the call remains as is
 54 *   (but spills/fills are still removed)
 55 *
 56 * It is really desirable to check instruction indexes in the xlated
 57 * patterns, so add this canary test to check that function rewrite by
 58 * jit is correctly processed by bpf_fastcall logic, keep the rest of the
 59 * tests as x86.
 60 */
 61SEC("raw_tp")
 62__arch_arm64
 63__arch_riscv64
 64__xlated("0: r1 = 1")
 65__xlated("1: call bpf_get_smp_processor_id")
 66__xlated("2: exit")
 67__success
 68__naked void canary_arm64_riscv64(void)
 69{
 70	asm volatile (
 71	"r1 = 1;"
 72	"*(u64 *)(r10 - 16) = r1;"
 73	"call %[bpf_get_smp_processor_id];"
 74	"r1 = *(u64 *)(r10 - 16);"
 75	"exit;"
 76	:
 77	: __imm(bpf_get_smp_processor_id)
 78	: __clobber_all);
 79}
 80
 81SEC("raw_tp")
 82__arch_x86_64
 83__xlated("1: r0 = &(void __percpu *)(r0)")
 84__xlated("...")
 85__xlated("3: exit")
 86__success
 87__naked void canary_zero_spills(void)
 88{
 89	asm volatile (
 90	"call %[bpf_get_smp_processor_id];"
 91	"exit;"
 92	:
 93	: __imm(bpf_get_smp_processor_id)
 94	: __clobber_all);
 95}
 96
 97SEC("raw_tp")
 98__arch_x86_64
 99__log_level(4) __msg("stack depth 16")
100__xlated("1: *(u64 *)(r10 -16) = r1")
101__xlated("...")
102__xlated("3: r0 = &(void __percpu *)(r0)")
103__xlated("...")
104__xlated("5: r2 = *(u64 *)(r10 -16)")
105__success
106__naked void wrong_reg_in_pattern1(void)
107{
108	asm volatile (
109	"r1 = 1;"
110	"*(u64 *)(r10 - 16) = r1;"
111	"call %[bpf_get_smp_processor_id];"
112	"r2 = *(u64 *)(r10 - 16);"
113	"exit;"
114	:
115	: __imm(bpf_get_smp_processor_id)
116	: __clobber_all);
117}
118
119SEC("raw_tp")
120__arch_x86_64
121__xlated("1: *(u64 *)(r10 -16) = r6")
122__xlated("...")
123__xlated("3: r0 = &(void __percpu *)(r0)")
124__xlated("...")
125__xlated("5: r6 = *(u64 *)(r10 -16)")
126__success
127__naked void wrong_reg_in_pattern2(void)
128{
129	asm volatile (
130	"r6 = 1;"
131	"*(u64 *)(r10 - 16) = r6;"
132	"call %[bpf_get_smp_processor_id];"
133	"r6 = *(u64 *)(r10 - 16);"
134	"exit;"
135	:
136	: __imm(bpf_get_smp_processor_id)
137	: __clobber_all);
138}
139
140SEC("raw_tp")
141__arch_x86_64
142__xlated("1: *(u64 *)(r10 -16) = r0")
143__xlated("...")
144__xlated("3: r0 = &(void __percpu *)(r0)")
145__xlated("...")
146__xlated("5: r0 = *(u64 *)(r10 -16)")
147__success
148__naked void wrong_reg_in_pattern3(void)
149{
150	asm volatile (
151	"r0 = 1;"
152	"*(u64 *)(r10 - 16) = r0;"
153	"call %[bpf_get_smp_processor_id];"
154	"r0 = *(u64 *)(r10 - 16);"
155	"exit;"
156	:
157	: __imm(bpf_get_smp_processor_id)
158	: __clobber_all);
159}
160
161SEC("raw_tp")
162__arch_x86_64
163__xlated("2: *(u64 *)(r2 -16) = r1")
164__xlated("...")
165__xlated("4: r0 = &(void __percpu *)(r0)")
166__xlated("...")
167__xlated("6: r1 = *(u64 *)(r10 -16)")
168__success
169__naked void wrong_base_in_pattern(void)
170{
171	asm volatile (
172	"r1 = 1;"
173	"r2 = r10;"
174	"*(u64 *)(r2 - 16) = r1;"
175	"call %[bpf_get_smp_processor_id];"
176	"r1 = *(u64 *)(r10 - 16);"
177	"exit;"
178	:
179	: __imm(bpf_get_smp_processor_id)
180	: __clobber_all);
181}
182
183SEC("raw_tp")
184__arch_x86_64
185__xlated("1: *(u64 *)(r10 -16) = r1")
186__xlated("...")
187__xlated("3: r0 = &(void __percpu *)(r0)")
188__xlated("...")
189__xlated("5: r2 = 1")
190__success
191__naked void wrong_insn_in_pattern(void)
192{
193	asm volatile (
194	"r1 = 1;"
195	"*(u64 *)(r10 - 16) = r1;"
196	"call %[bpf_get_smp_processor_id];"
197	"r2 = 1;"
198	"r1 = *(u64 *)(r10 - 16);"
199	"exit;"
200	:
201	: __imm(bpf_get_smp_processor_id)
202	: __clobber_all);
203}
204
205SEC("raw_tp")
206__arch_x86_64
207__xlated("2: *(u64 *)(r10 -16) = r1")
208__xlated("...")
209__xlated("4: r0 = &(void __percpu *)(r0)")
210__xlated("...")
211__xlated("6: r1 = *(u64 *)(r10 -8)")
212__success
213__naked void wrong_off_in_pattern1(void)
214{
215	asm volatile (
216	"r1 = 1;"
217	"*(u64 *)(r10 - 8) = r1;"
218	"*(u64 *)(r10 - 16) = r1;"
219	"call %[bpf_get_smp_processor_id];"
220	"r1 = *(u64 *)(r10 - 8);"
221	"exit;"
222	:
223	: __imm(bpf_get_smp_processor_id)
224	: __clobber_all);
225}
226
227SEC("raw_tp")
228__arch_x86_64
229__xlated("1: *(u32 *)(r10 -4) = r1")
230__xlated("...")
231__xlated("3: r0 = &(void __percpu *)(r0)")
232__xlated("...")
233__xlated("5: r1 = *(u32 *)(r10 -4)")
234__success
235__naked void wrong_off_in_pattern2(void)
236{
237	asm volatile (
238	"r1 = 1;"
239	"*(u32 *)(r10 - 4) = r1;"
240	"call %[bpf_get_smp_processor_id];"
241	"r1 = *(u32 *)(r10 - 4);"
242	"exit;"
243	:
244	: __imm(bpf_get_smp_processor_id)
245	: __clobber_all);
246}
247
248SEC("raw_tp")
249__arch_x86_64
250__xlated("1: *(u32 *)(r10 -16) = r1")
251__xlated("...")
252__xlated("3: r0 = &(void __percpu *)(r0)")
253__xlated("...")
254__xlated("5: r1 = *(u32 *)(r10 -16)")
255__success
256__naked void wrong_size_in_pattern(void)
257{
258	asm volatile (
259	"r1 = 1;"
260	"*(u32 *)(r10 - 16) = r1;"
261	"call %[bpf_get_smp_processor_id];"
262	"r1 = *(u32 *)(r10 - 16);"
263	"exit;"
264	:
265	: __imm(bpf_get_smp_processor_id)
266	: __clobber_all);
267}
268
269SEC("raw_tp")
270__arch_x86_64
271__xlated("2: *(u32 *)(r10 -8) = r1")
272__xlated("...")
273__xlated("4: r0 = &(void __percpu *)(r0)")
274__xlated("...")
275__xlated("6: r1 = *(u32 *)(r10 -8)")
276__success
277__naked void partial_pattern(void)
278{
279	asm volatile (
280	"r1 = 1;"
281	"r2 = 2;"
282	"*(u32 *)(r10 - 8) = r1;"
283	"*(u64 *)(r10 - 16) = r2;"
284	"call %[bpf_get_smp_processor_id];"
285	"r2 = *(u64 *)(r10 - 16);"
286	"r1 = *(u32 *)(r10 - 8);"
287	"exit;"
288	:
289	: __imm(bpf_get_smp_processor_id)
290	: __clobber_all);
291}
292
293SEC("raw_tp")
294__arch_x86_64
295__xlated("0: r1 = 1")
296__xlated("1: r2 = 2")
297/* not patched, spills for -8, -16 not removed */
298__xlated("2: *(u64 *)(r10 -8) = r1")
299__xlated("3: *(u64 *)(r10 -16) = r2")
300__xlated("...")
301__xlated("5: r0 = &(void __percpu *)(r0)")
302__xlated("...")
303__xlated("7: r2 = *(u64 *)(r10 -16)")
304__xlated("8: r1 = *(u64 *)(r10 -8)")
305/* patched, spills for -24, -32 removed */
306__xlated("...")
307__xlated("10: r0 = &(void __percpu *)(r0)")
308__xlated("...")
309__xlated("12: exit")
310__success
311__naked void min_stack_offset(void)
312{
313	asm volatile (
314	"r1 = 1;"
315	"r2 = 2;"
316	/* this call won't be patched */
317	"*(u64 *)(r10 - 8) = r1;"
318	"*(u64 *)(r10 - 16) = r2;"
319	"call %[bpf_get_smp_processor_id];"
320	"r2 = *(u64 *)(r10 - 16);"
321	"r1 = *(u64 *)(r10 - 8);"
322	/* this call would be patched */
323	"*(u64 *)(r10 - 24) = r1;"
324	"*(u64 *)(r10 - 32) = r2;"
325	"call %[bpf_get_smp_processor_id];"
326	"r2 = *(u64 *)(r10 - 32);"
327	"r1 = *(u64 *)(r10 - 24);"
328	"exit;"
329	:
330	: __imm(bpf_get_smp_processor_id)
331	: __clobber_all);
332}
333
334SEC("raw_tp")
335__arch_x86_64
336__xlated("1: *(u64 *)(r10 -8) = r1")
337__xlated("...")
338__xlated("3: r0 = &(void __percpu *)(r0)")
339__xlated("...")
340__xlated("5: r1 = *(u64 *)(r10 -8)")
341__success
342__naked void bad_fixed_read(void)
343{
344	asm volatile (
345	"r1 = 1;"
346	"*(u64 *)(r10 - 8) = r1;"
347	"call %[bpf_get_smp_processor_id];"
348	"r1 = *(u64 *)(r10 - 8);"
349	"r1 = r10;"
350	"r1 += -8;"
351	"r1 = *(u64 *)(r1 - 0);"
352	"exit;"
353	:
354	: __imm(bpf_get_smp_processor_id)
355	: __clobber_all);
356}
357
358SEC("raw_tp")
359__arch_x86_64
360__xlated("1: *(u64 *)(r10 -8) = r1")
361__xlated("...")
362__xlated("3: r0 = &(void __percpu *)(r0)")
363__xlated("...")
364__xlated("5: r1 = *(u64 *)(r10 -8)")
365__success
366__naked void bad_fixed_write(void)
367{
368	asm volatile (
369	"r1 = 1;"
370	"*(u64 *)(r10 - 8) = r1;"
371	"call %[bpf_get_smp_processor_id];"
372	"r1 = *(u64 *)(r10 - 8);"
373	"r1 = r10;"
374	"r1 += -8;"
375	"*(u64 *)(r1 - 0) = r1;"
376	"exit;"
377	:
378	: __imm(bpf_get_smp_processor_id)
379	: __clobber_all);
380}
381
382SEC("raw_tp")
383__arch_x86_64
384__xlated("6: *(u64 *)(r10 -16) = r1")
385__xlated("...")
386__xlated("8: r0 = &(void __percpu *)(r0)")
387__xlated("...")
388__xlated("10: r1 = *(u64 *)(r10 -16)")
389__success
390__naked void bad_varying_read(void)
391{
392	asm volatile (
393	"r6 = *(u64 *)(r1 + 0);" /* random scalar value */
394	"r6 &= 0x7;"		 /* r6 range [0..7] */
395	"r6 += 0x2;"		 /* r6 range [2..9] */
396	"r7 = 0;"
397	"r7 -= r6;"		 /* r7 range [-9..-2] */
398	"r1 = 1;"
399	"*(u64 *)(r10 - 16) = r1;"
400	"call %[bpf_get_smp_processor_id];"
401	"r1 = *(u64 *)(r10 - 16);"
402	"r1 = r10;"
403	"r1 += r7;"
404	"r1 = *(u8 *)(r1 - 0);" /* touches slot [-16..-9] where spills are stored */
405	"exit;"
406	:
407	: __imm(bpf_get_smp_processor_id)
408	: __clobber_all);
409}
410
411SEC("raw_tp")
412__arch_x86_64
413__xlated("6: *(u64 *)(r10 -16) = r1")
414__xlated("...")
415__xlated("8: r0 = &(void __percpu *)(r0)")
416__xlated("...")
417__xlated("10: r1 = *(u64 *)(r10 -16)")
418__success
419__naked void bad_varying_write(void)
420{
421	asm volatile (
422	"r6 = *(u64 *)(r1 + 0);" /* random scalar value */
423	"r6 &= 0x7;"		 /* r6 range [0..7] */
424	"r6 += 0x2;"		 /* r6 range [2..9] */
425	"r7 = 0;"
426	"r7 -= r6;"		 /* r7 range [-9..-2] */
427	"r1 = 1;"
428	"*(u64 *)(r10 - 16) = r1;"
429	"call %[bpf_get_smp_processor_id];"
430	"r1 = *(u64 *)(r10 - 16);"
431	"r1 = r10;"
432	"r1 += r7;"
433	"*(u8 *)(r1 - 0) = r7;" /* touches slot [-16..-9] where spills are stored */
434	"exit;"
435	:
436	: __imm(bpf_get_smp_processor_id)
437	: __clobber_all);
438}
439
440SEC("raw_tp")
441__arch_x86_64
442__xlated("1: *(u64 *)(r10 -8) = r1")
443__xlated("...")
444__xlated("3: r0 = &(void __percpu *)(r0)")
445__xlated("...")
446__xlated("5: r1 = *(u64 *)(r10 -8)")
447__success
448__naked void bad_write_in_subprog(void)
449{
450	asm volatile (
451	"r1 = 1;"
452	"*(u64 *)(r10 - 8) = r1;"
453	"call %[bpf_get_smp_processor_id];"
454	"r1 = *(u64 *)(r10 - 8);"
455	"r1 = r10;"
456	"r1 += -8;"
457	"call bad_write_in_subprog_aux;"
458	"exit;"
459	:
460	: __imm(bpf_get_smp_processor_id)
461	: __clobber_all);
462}
463
464__used
465__naked static void bad_write_in_subprog_aux(void)
466{
467	asm volatile (
468	"r0 = 1;"
469	"*(u64 *)(r1 - 0) = r0;"	/* invalidates bpf_fastcall contract for caller: */
470	"exit;"				/* caller stack at -8 used outside of the pattern */
471	::: __clobber_all);
472}
473
474SEC("raw_tp")
475__arch_x86_64
476__xlated("1: *(u64 *)(r10 -8) = r1")
477__xlated("...")
478__xlated("3: r0 = &(void __percpu *)(r0)")
479__xlated("...")
480__xlated("5: r1 = *(u64 *)(r10 -8)")
481__success
482__naked void bad_helper_write(void)
483{
484	asm volatile (
485	"r1 = 1;"
486	/* bpf_fastcall pattern with stack offset -8 */
487	"*(u64 *)(r10 - 8) = r1;"
488	"call %[bpf_get_smp_processor_id];"
489	"r1 = *(u64 *)(r10 - 8);"
490	"r1 = r10;"
491	"r1 += -8;"
492	"r2 = 1;"
493	"r3 = 42;"
494	/* read dst is fp[-8], thus bpf_fastcall rewrite not applied */
495	"call %[bpf_probe_read_kernel];"
496	"exit;"
497	:
498	: __imm(bpf_get_smp_processor_id),
499	  __imm(bpf_probe_read_kernel)
500	: __clobber_all);
501}
502
503SEC("raw_tp")
504__arch_x86_64
505/* main, not patched */
506__xlated("1: *(u64 *)(r10 -8) = r1")
507__xlated("...")
508__xlated("3: r0 = &(void __percpu *)(r0)")
509__xlated("...")
510__xlated("5: r1 = *(u64 *)(r10 -8)")
511__xlated("...")
512__xlated("9: call pc+1")
513__xlated("...")
514__xlated("10: exit")
515/* subprogram, patched */
516__xlated("11: r1 = 1")
517__xlated("...")
518__xlated("13: r0 = &(void __percpu *)(r0)")
519__xlated("...")
520__xlated("15: exit")
521__success
522__naked void invalidate_one_subprog(void)
523{
524	asm volatile (
525	"r1 = 1;"
526	"*(u64 *)(r10 - 8) = r1;"
527	"call %[bpf_get_smp_processor_id];"
528	"r1 = *(u64 *)(r10 - 8);"
529	"r1 = r10;"
530	"r1 += -8;"
531	"r1 = *(u64 *)(r1 - 0);"
532	"call invalidate_one_subprog_aux;"
533	"exit;"
534	:
535	: __imm(bpf_get_smp_processor_id)
536	: __clobber_all);
537}
538
539__used
540__naked static void invalidate_one_subprog_aux(void)
541{
542	asm volatile (
543	"r1 = 1;"
544	"*(u64 *)(r10 - 8) = r1;"
545	"call %[bpf_get_smp_processor_id];"
546	"r1 = *(u64 *)(r10 - 8);"
547	"exit;"
548	:
549	: __imm(bpf_get_smp_processor_id)
550	: __clobber_all);
551}
552
553SEC("raw_tp")
554__arch_x86_64
555/* main */
556__xlated("0: r1 = 1")
557__xlated("...")
558__xlated("2: r0 = &(void __percpu *)(r0)")
559__xlated("...")
560__xlated("4: call pc+1")
561__xlated("5: exit")
562/* subprogram */
563__xlated("6: r1 = 1")
564__xlated("...")
565__xlated("8: r0 = &(void __percpu *)(r0)")
566__xlated("...")
567__xlated("10: *(u64 *)(r10 -16) = r1")
568__xlated("11: exit")
569__success
570__naked void subprogs_use_independent_offsets(void)
571{
572	asm volatile (
573	"r1 = 1;"
574	"*(u64 *)(r10 - 16) = r1;"
575	"call %[bpf_get_smp_processor_id];"
576	"r1 = *(u64 *)(r10 - 16);"
577	"call subprogs_use_independent_offsets_aux;"
578	"exit;"
579	:
580	: __imm(bpf_get_smp_processor_id)
581	: __clobber_all);
582}
583
584__used
585__naked static void subprogs_use_independent_offsets_aux(void)
586{
587	asm volatile (
588	"r1 = 1;"
589	"*(u64 *)(r10 - 24) = r1;"
590	"call %[bpf_get_smp_processor_id];"
591	"r1 = *(u64 *)(r10 - 24);"
592	"*(u64 *)(r10 - 16) = r1;"
593	"exit;"
594	:
595	: __imm(bpf_get_smp_processor_id)
596	: __clobber_all);
597}
598
599SEC("raw_tp")
600__arch_x86_64
601__log_level(4) __msg("stack depth 8")
602__xlated("2: r0 = &(void __percpu *)(r0)")
603__success
604__naked void helper_call_does_not_prevent_bpf_fastcall(void)
605{
606	asm volatile (
607	"r1 = 1;"
608	"*(u64 *)(r10 - 8) = r1;"
609	"call %[bpf_get_smp_processor_id];"
610	"r1 = *(u64 *)(r10 - 8);"
611	"*(u64 *)(r10 - 8) = r1;"
612	"call %[bpf_get_prandom_u32];"
613	"r1 = *(u64 *)(r10 - 8);"
614	"exit;"
615	:
616	: __imm(bpf_get_smp_processor_id),
617	  __imm(bpf_get_prandom_u32)
618	: __clobber_all);
619}
620
621SEC("raw_tp")
622__arch_x86_64
623__log_level(4) __msg("stack depth 24")
624/* may_goto counter at -24 */
625__xlated("0: *(u64 *)(r10 -24) =")
626/* may_goto timestamp at -16 */
627__xlated("1: *(u64 *)(r10 -16) =")
628__xlated("2: r1 = 1")
629__xlated("...")
630__xlated("4: r0 = &(void __percpu *)(r0)")
631__xlated("...")
632/* may_goto expansion starts */
633__xlated("6: r11 = *(u64 *)(r10 -24)")
634__xlated("7: if r11 == 0x0 goto pc+6")
635__xlated("8: r11 -= 1")
636__xlated("9: if r11 != 0x0 goto pc+2")
637__xlated("10: r11 = -24")
638__xlated("11: call unknown")
639__xlated("12: *(u64 *)(r10 -24) = r11")
640/* may_goto expansion ends */
641__xlated("13: *(u64 *)(r10 -8) = r1")
642__xlated("14: exit")
643__success
644__naked void may_goto_interaction_x86_64(void)
645{
646	asm volatile (
647	"r1 = 1;"
648	"*(u64 *)(r10 - 16) = r1;"
649	"call %[bpf_get_smp_processor_id];"
650	"r1 = *(u64 *)(r10 - 16);"
651	".8byte %[may_goto];"
652	/* just touch some stack at -8 */
653	"*(u64 *)(r10 - 8) = r1;"
654	"exit;"
655	:
656	: __imm(bpf_get_smp_processor_id),
657	  __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, +1 /* offset */, 0))
658	: __clobber_all);
659}
660
661SEC("raw_tp")
662__arch_arm64
663__log_level(4) __msg("stack depth 24")
664/* may_goto counter at -24 */
665__xlated("0: *(u64 *)(r10 -24) =")
666/* may_goto timestamp at -16 */
667__xlated("1: *(u64 *)(r10 -16) =")
668__xlated("2: r1 = 1")
669__xlated("3: call bpf_get_smp_processor_id")
670/* may_goto expansion starts */
671__xlated("4: r11 = *(u64 *)(r10 -24)")
672__xlated("5: if r11 == 0x0 goto pc+6")
673__xlated("6: r11 -= 1")
674__xlated("7: if r11 != 0x0 goto pc+2")
675__xlated("8: r11 = -24")
676__xlated("9: call unknown")
677__xlated("10: *(u64 *)(r10 -24) = r11")
678/* may_goto expansion ends */
679__xlated("11: *(u64 *)(r10 -8) = r1")
680__xlated("12: exit")
681__success
682__naked void may_goto_interaction_arm64(void)
683{
684	asm volatile (
685	"r1 = 1;"
686	"*(u64 *)(r10 - 16) = r1;"
687	"call %[bpf_get_smp_processor_id];"
688	"r1 = *(u64 *)(r10 - 16);"
689	".8byte %[may_goto];"
690	/* just touch some stack at -8 */
691	"*(u64 *)(r10 - 8) = r1;"
692	"exit;"
693	:
694	: __imm(bpf_get_smp_processor_id),
695	  __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, +1 /* offset */, 0))
696	: __clobber_all);
697}
698
699__used
700__naked static void dummy_loop_callback(void)
701{
702	asm volatile (
703	"r0 = 0;"
704	"exit;"
705	::: __clobber_all);
706}
707
708SEC("raw_tp")
709__arch_x86_64
710__log_level(4) __msg("stack depth 32+0")
711__xlated("2: r1 = 1")
712__xlated("3: r0 =")
713__xlated("4: r0 = &(void __percpu *)(r0)")
714__xlated("5: r0 = *(u32 *)(r0 +0)")
715/* bpf_loop params setup */
716__xlated("6: r2 =")
717__xlated("7: r3 = 0")
718__xlated("8: r4 = 0")
719__xlated("...")
720/* ... part of the inlined bpf_loop */
721__xlated("12: *(u64 *)(r10 -32) = r6")
722__xlated("13: *(u64 *)(r10 -24) = r7")
723__xlated("14: *(u64 *)(r10 -16) = r8")
724__xlated("...")
725__xlated("21: call pc+8") /* dummy_loop_callback */
726/* ... last insns of the bpf_loop_interaction1 */
727__xlated("...")
728__xlated("28: r0 = 0")
729__xlated("29: exit")
730/* dummy_loop_callback */
731__xlated("30: r0 = 0")
732__xlated("31: exit")
733__success
734__naked int bpf_loop_interaction1(void)
735{
736	asm volatile (
737	"r1 = 1;"
738	/* bpf_fastcall stack region at -16, but could be removed */
739	"*(u64 *)(r10 - 16) = r1;"
740	"call %[bpf_get_smp_processor_id];"
741	"r1 = *(u64 *)(r10 - 16);"
742	"r2 = %[dummy_loop_callback];"
743	"r3 = 0;"
744	"r4 = 0;"
745	"call %[bpf_loop];"
746	"r0 = 0;"
747	"exit;"
748	:
749	: __imm_ptr(dummy_loop_callback),
750	  __imm(bpf_get_smp_processor_id),
751	  __imm(bpf_loop)
752	: __clobber_common
753	);
754}
755
756SEC("raw_tp")
757__arch_x86_64
758__log_level(4) __msg("stack depth 40+0")
759/* call bpf_get_smp_processor_id */
760__xlated("2: r1 = 42")
761__xlated("3: r0 =")
762__xlated("4: r0 = &(void __percpu *)(r0)")
763__xlated("5: r0 = *(u32 *)(r0 +0)")
764/* call bpf_get_prandom_u32 */
765__xlated("6: *(u64 *)(r10 -16) = r1")
766__xlated("7: call")
767__xlated("8: r1 = *(u64 *)(r10 -16)")
768__xlated("...")
769/* ... part of the inlined bpf_loop */
770__xlated("15: *(u64 *)(r10 -40) = r6")
771__xlated("16: *(u64 *)(r10 -32) = r7")
772__xlated("17: *(u64 *)(r10 -24) = r8")
773__success
774__naked int bpf_loop_interaction2(void)
775{
776	asm volatile (
777	"r1 = 42;"
778	/* bpf_fastcall stack region at -16, cannot be removed */
779	"*(u64 *)(r10 - 16) = r1;"
780	"call %[bpf_get_smp_processor_id];"
781	"r1 = *(u64 *)(r10 - 16);"
782	"*(u64 *)(r10 - 16) = r1;"
783	"call %[bpf_get_prandom_u32];"
784	"r1 = *(u64 *)(r10 - 16);"
785	"r2 = %[dummy_loop_callback];"
786	"r3 = 0;"
787	"r4 = 0;"
788	"call %[bpf_loop];"
789	"r0 = 0;"
790	"exit;"
791	:
792	: __imm_ptr(dummy_loop_callback),
793	  __imm(bpf_get_smp_processor_id),
794	  __imm(bpf_get_prandom_u32),
795	  __imm(bpf_loop)
796	: __clobber_common
797	);
798}
799
800SEC("raw_tp")
801__arch_x86_64
802__log_level(4)
803__msg("stack depth 512+0")
804/* just to print xlated version when debugging */
805__xlated("r0 = &(void __percpu *)(r0)")
806__success
807/* cumulative_stack_depth() stack usage is MAX_BPF_STACK,
808 * called subprogram uses an additional slot for bpf_fastcall spill/fill,
809 * since bpf_fastcall spill/fill could be removed the program still fits
810 * in MAX_BPF_STACK and should be accepted.
811 */
812__naked int cumulative_stack_depth(void)
813{
814	asm volatile(
815	"r1 = 42;"
816	"*(u64 *)(r10 - %[max_bpf_stack]) = r1;"
817	"call cumulative_stack_depth_subprog;"
818	"exit;"
819	:
820	: __imm_const(max_bpf_stack, MAX_BPF_STACK)
821	: __clobber_all
822	);
823}
824
825__used
826__naked static void cumulative_stack_depth_subprog(void)
827{
828	asm volatile (
829	"*(u64 *)(r10 - 8) = r1;"
830	"call %[bpf_get_smp_processor_id];"
831	"r1 = *(u64 *)(r10 - 8);"
832	"exit;"
833	:: __imm(bpf_get_smp_processor_id) : __clobber_all);
834}
835
836SEC("cgroup/getsockname_unix")
837__xlated("0: r2 = 1")
838/* bpf_cast_to_kern_ctx is replaced by a single assignment */
839__xlated("1: r0 = r1")
840__xlated("2: r0 = r2")
841__xlated("3: exit")
842__success
843__naked void kfunc_bpf_cast_to_kern_ctx(void)
844{
845	asm volatile (
846	"r2 = 1;"
847	"*(u64 *)(r10 - 32) = r2;"
848	"call %[bpf_cast_to_kern_ctx];"
849	"r2 = *(u64 *)(r10 - 32);"
850	"r0 = r2;"
851	"exit;"
852	:
853	: __imm(bpf_cast_to_kern_ctx)
854	: __clobber_all);
855}
856
857SEC("raw_tp")
858__xlated("3: r3 = 1")
859/* bpf_rdonly_cast is replaced by a single assignment */
860__xlated("4: r0 = r1")
861__xlated("5: r0 = r3")
862void kfunc_bpf_rdonly_cast(void)
863{
864	asm volatile (
865	"r2 = %[btf_id];"
866	"r3 = 1;"
867	"*(u64 *)(r10 - 32) = r3;"
868	"call %[bpf_rdonly_cast];"
869	"r3 = *(u64 *)(r10 - 32);"
870	"r0 = r3;"
871	:
872	: __imm(bpf_rdonly_cast),
873	 [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr))
874	: __clobber_common);
875}
876
877/* BTF FUNC records are not generated for kfuncs referenced
878 * from inline assembly. These records are necessary for
879 * libbpf to link the program. The function below is a hack
880 * to ensure that BTF FUNC records are generated.
881 */
882void kfunc_root(void)
883{
884	bpf_cast_to_kern_ctx(0);
885	bpf_rdonly_cast(0, 0);
886}
887
888char _license[] SEC("license") = "GPL";