verifier_load_acquire.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2025 Google LLC. */
  3
  4#include <linux/bpf.h>
  5#include <bpf/bpf_helpers.h>
  6#include "../../../include/linux/filter.h"
  7#include "bpf_misc.h"
  8
  9#ifdef CAN_USE_LOAD_ACQ_STORE_REL
 10
 11SEC("socket")
 12__description("load-acquire, 8-bit")
 13__success __success_unpriv __retval(0)
 14__naked void load_acquire_8(void)
 15{
 16	asm volatile (
 17	"r0 = 0;"
 18	"w1 = 0xfe;"
 19	"*(u8 *)(r10 - 1) = w1;"
 20	".8byte %[load_acquire_insn];" // w2 = load_acquire((u8 *)(r10 - 1));
 21	"if r2 == r1 goto 1f;"
 22	"r0 = 1;"
 23"1:"
 24	"exit;"
 25	:
 26	: __imm_insn(load_acquire_insn,
 27		     BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -1))
 28	: __clobber_all);
 29}
 30
 31SEC("socket")
 32__description("load-acquire, 16-bit")
 33__success __success_unpriv __retval(0)
 34__naked void load_acquire_16(void)
 35{
 36	asm volatile (
 37	"r0 = 0;"
 38	"w1 = 0xfedc;"
 39	"*(u16 *)(r10 - 2) = w1;"
 40	".8byte %[load_acquire_insn];" // w2 = load_acquire((u16 *)(r10 - 2));
 41	"if r2 == r1 goto 1f;"
 42	"r0 = 1;"
 43"1:"
 44	"exit;"
 45	:
 46	: __imm_insn(load_acquire_insn,
 47		     BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -2))
 48	: __clobber_all);
 49}
 50
 51SEC("socket")
 52__description("load-acquire, 32-bit")
 53__success __success_unpriv __retval(0)
 54__naked void load_acquire_32(void)
 55{
 56	asm volatile (
 57	"r0 = 0;"
 58	"w1 = 0xfedcba09;"
 59	"*(u32 *)(r10 - 4) = w1;"
 60	".8byte %[load_acquire_insn];" // w2 = load_acquire((u32 *)(r10 - 4));
 61	"if r2 == r1 goto 1f;"
 62	"r0 = 1;"
 63"1:"
 64	"exit;"
 65	:
 66	: __imm_insn(load_acquire_insn,
 67		     BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -4))
 68	: __clobber_all);
 69}
 70
 71SEC("socket")
 72__description("load-acquire, 64-bit")
 73__success __success_unpriv __retval(0)
 74__naked void load_acquire_64(void)
 75{
 76	asm volatile (
 77	"r0 = 0;"
 78	"r1 = 0xfedcba0987654321 ll;"
 79	"*(u64 *)(r10 - 8) = r1;"
 80	".8byte %[load_acquire_insn];" // r2 = load_acquire((u64 *)(r10 - 8));
 81	"if r2 == r1 goto 1f;"
 82	"r0 = 1;"
 83"1:"
 84	"exit;"
 85	:
 86	: __imm_insn(load_acquire_insn,
 87		     BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
 88	: __clobber_all);
 89}
 90
 91SEC("socket")
 92__description("load-acquire with uninitialized src_reg")
 93__failure __failure_unpriv __msg("R2 !read_ok")
 94__naked void load_acquire_with_uninitialized_src_reg(void)
 95{
 96	asm volatile (
 97	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0));
 98	"exit;"
 99	:
100	: __imm_insn(load_acquire_insn,
101		     BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
102	: __clobber_all);
103}
104
105SEC("socket")
106__description("load-acquire with non-pointer src_reg")
107__failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
108__naked void load_acquire_with_non_pointer_src_reg(void)
109{
110	asm volatile (
111	"r1 = 0;"
112	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0));
113	"exit;"
114	:
115	: __imm_insn(load_acquire_insn,
116		     BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
117	: __clobber_all);
118}
119
120SEC("socket")
121__description("misaligned load-acquire")
122__failure __failure_unpriv __msg("misaligned stack access off")
123__flag(BPF_F_ANY_ALIGNMENT)
124__naked void load_acquire_misaligned(void)
125{
126	asm volatile (
127	"r1 = 0;"
128	"*(u64 *)(r10 - 8) = r1;"
129	".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5));
130	"exit;"
131	:
132	: __imm_insn(load_acquire_insn,
133		     BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5))
134	: __clobber_all);
135}
136
137SEC("socket")
138__description("load-acquire from ctx pointer")
139__failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed")
140__naked void load_acquire_from_ctx_pointer(void)
141{
142	asm volatile (
143	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0));
144	"exit;"
145	:
146	: __imm_insn(load_acquire_insn,
147		     BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
148	: __clobber_all);
149}
150
151SEC("xdp")
152__description("load-acquire from pkt pointer")
153__failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed")
154__naked void load_acquire_from_pkt_pointer(void)
155{
156	asm volatile (
157	"r2 = *(u32 *)(r1 + %[xdp_md_data]);"
158	"r3 = *(u32 *)(r1 + %[xdp_md_data_end]);"
159	"r1 = r2;"
160	"r1 += 8;"
161	"if r1 >= r3 goto l0_%=;"
162	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
163"l0_%=:  r0 = 0;"
164	"exit;"
165	:
166	: __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
167	  __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
168	  __imm_insn(load_acquire_insn,
169		     BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
170	: __clobber_all);
171}
172
173SEC("flow_dissector")
174__description("load-acquire from flow_keys pointer")
175__failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed")
176__naked void load_acquire_from_flow_keys_pointer(void)
177{
178	asm volatile (
179	"r2 = *(u64 *)(r1 + %[__sk_buff_flow_keys]);"
180	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
181	"exit;"
182	:
183	: __imm_const(__sk_buff_flow_keys,
184		      offsetof(struct __sk_buff, flow_keys)),
185	  __imm_insn(load_acquire_insn,
186		     BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
187	: __clobber_all);
188}
189
190SEC("sk_reuseport")
191__description("load-acquire from sock pointer")
192__failure __msg("BPF_ATOMIC loads from R2 sock is not allowed")
193__naked void load_acquire_from_sock_pointer(void)
194{
195	asm volatile (
196	"r2 = *(u64 *)(r1 + %[sk_reuseport_md_sk]);"
197	// w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family)));
198	".8byte %[load_acquire_insn];"
199	"exit;"
200	:
201	: __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
202	  __imm_insn(load_acquire_insn,
203		     BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2,
204				   offsetof(struct bpf_sock, family)))
205	: __clobber_all);
206}
207
208SEC("socket")
209__description("load-acquire with invalid register R15")
210__failure __failure_unpriv __msg("R15 is invalid")
211__naked void load_acquire_with_invalid_reg(void)
212{
213	asm volatile (
214	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0));
215	"exit;"
216	:
217	: __imm_insn(load_acquire_insn,
218		     BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0))
219	: __clobber_all);
220}
221
222#else /* CAN_USE_LOAD_ACQ_STORE_REL */
223
224SEC("socket")
225__description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test")
226__success
227int dummy_test(void)
228{
229	return 0;
230}
231
232#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
233
234char _license[] SEC("license") = "GPL";