FILE
bpf_jit_comp32.c
File preview with syntax highlighting.
linux-6.19.14/arch/powerpc/net/bpf_jit_comp32.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * eBPF JIT compiler for PPC32 4 * 5 * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu> 6 * CS GROUP France 7 * 8 * Based on PPC64 eBPF JIT compiler by Naveen N. Rao 9 */ 10#include <linux/moduleloader.h> 11#include <asm/cacheflush.h> 12#include <asm/asm-compat.h> 13#include <linux/netdevice.h> 14#include <linux/filter.h> 15#include <linux/if_vlan.h> 16#include <asm/kprobes.h> 17#include <linux/bpf.h> 18 19#include "bpf_jit.h" 20 21/* 22 * Stack layout: 23 * 24 * [ prev sp ] <------------- 25 * [ nv gpr save area ] 16 * 4 | 26 * fp (r31) --> [ ebpf stack space ] upto 512 | 27 * [ frame header ] 16 | 28 * sp (r1) ---> [ stack pointer ] -------------- 29 */ 30 31/* for gpr non volatile registers r17 to r31 (14) + tail call */ 32#define BPF_PPC_STACK_SAVE (15 * 4 + 4) 33/* stack frame, ensure this is quadword aligned */ 34#define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size) 35 36#define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0)) 37 38/* PPC NVR range -- update this if we ever use NVRs below r17 */ 39#define BPF_PPC_NVR_MIN _R17 40#define BPF_PPC_TC _R16 41 42/* BPF register usage */ 43#define TMP_REG (MAX_BPF_JIT_REG + 0) 44 45/* BPF to ppc register mappings */ 46void bpf_jit_init_reg_mapping(struct codegen_context *ctx) 47{ 48 /* function return value */ 49 ctx->b2p[BPF_REG_0] = _R12; 50 /* function arguments */ 51 ctx->b2p[BPF_REG_1] = _R4; 52 ctx->b2p[BPF_REG_2] = _R6; 53 ctx->b2p[BPF_REG_3] = _R8; 54 ctx->b2p[BPF_REG_4] = _R10; 55 ctx->b2p[BPF_REG_5] = _R22; 56 /* non volatile registers */ 57 ctx->b2p[BPF_REG_6] = _R24; 58 ctx->b2p[BPF_REG_7] = _R26; 59 ctx->b2p[BPF_REG_8] = _R28; 60 ctx->b2p[BPF_REG_9] = _R30; 61 /* frame pointer aka BPF_REG_10 */ 62 ctx->b2p[BPF_REG_FP] = _R18; 63 /* eBPF jit internal registers */ 64 ctx->b2p[BPF_REG_AX] = _R20; 65 ctx->b2p[TMP_REG] = _R31; /* 32 bits */ 66} 67 68static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 69{ 70 if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC) 71 return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg); 72 73 WARN(true, "BPF JIT is asking about unknown registers, will crash the stack"); 74 /* Use the hole we have left for alignment */ 75 return BPF_PPC_STACKFRAME(ctx) - 4; 76} 77 78#define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */ 79#define SEEN_NVREG_FULL_MASK 0x0003ffff /* Non volatile registers r14-r31 */ 80#define SEEN_NVREG_TEMP_MASK 0x00001e01 /* BPF_REG_5, BPF_REG_AX, TMP_REG */ 81 82static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 83{ 84 /* 85 * We only need a stack frame if: 86 * - we call other functions (kernel helpers), or 87 * - we use non volatile registers, or 88 * - we use tail call counter 89 * - the bpf program uses its stack area 90 * The latter condition is deduced from the usage of BPF_REG_FP 91 */ 92 return ctx->seen & (SEEN_FUNC | SEEN_TAILCALL | SEEN_NVREG_FULL_MASK) || 93 bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)); 94} 95 96void bpf_jit_realloc_regs(struct codegen_context *ctx) 97{ 98 unsigned int nvreg_mask; 99 100 if (ctx->seen & SEEN_FUNC) 101 nvreg_mask = SEEN_NVREG_TEMP_MASK; 102 else 103 nvreg_mask = SEEN_NVREG_FULL_MASK; 104 105 while (ctx->seen & nvreg_mask && 106 (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) { 107 int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab)); 108 int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa)); 109 int i; 110 111 for (i = BPF_REG_0; i <= TMP_REG; i++) { 112 if (ctx->b2p[i] != old) 113 continue; 114 ctx->b2p[i] = new; 115 bpf_set_seen_register(ctx, new); 116 bpf_clear_seen_register(ctx, old); 117 if (i != TMP_REG) { 118 bpf_set_seen_register(ctx, new - 1); 119 bpf_clear_seen_register(ctx, old - 1); 120 } 121 break; 122 } 123 } 124} 125 126void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 127{ 128 int i; 129 130 /* Instruction for trampoline attach */ 131 EMIT(PPC_RAW_NOP()); 132 133 /* Initialize tail_call_cnt, to be skipped if we do tail calls. */ 134 if (ctx->seen & SEEN_TAILCALL) 135 EMIT(PPC_RAW_LI(_R4, 0)); 136 else 137 EMIT(PPC_RAW_NOP()); 138 139#define BPF_TAILCALL_PROLOGUE_SIZE 8 140 141 if (bpf_has_stack_frame(ctx)) 142 EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx))); 143 144 if (ctx->seen & SEEN_TAILCALL) 145 EMIT(PPC_RAW_STW(_R4, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 146 147 /* First arg comes in as a 32 bits pointer. */ 148 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3)); 149 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0)); 150 151 /* 152 * We need a stack frame, but we don't necessarily need to 153 * save/restore LR unless we call other functions 154 */ 155 if (ctx->seen & SEEN_FUNC) 156 EMIT(PPC_RAW_MFLR(_R0)); 157 158 /* 159 * Back up non-volatile regs -- registers r18-r31 160 */ 161 for (i = BPF_PPC_NVR_MIN; i <= 31; i++) 162 if (bpf_is_seen_register(ctx, i)) 163 EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i))); 164 165 /* Setup frame pointer to point to the bpf stack area */ 166 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) { 167 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0)); 168 EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, 169 STACK_FRAME_MIN_SIZE + ctx->stack_size)); 170 } 171 172 if (ctx->seen & SEEN_FUNC) 173 EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 174} 175 176static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 177{ 178 int i; 179 180 /* Restore NVRs */ 181 for (i = BPF_PPC_NVR_MIN; i <= 31; i++) 182 if (bpf_is_seen_register(ctx, i)) 183 EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i))); 184 185 if (ctx->seen & SEEN_FUNC) 186 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 187 188 /* Tear down our stack frame */ 189 if (bpf_has_stack_frame(ctx)) 190 EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx))); 191 192 if (ctx->seen & SEEN_FUNC) 193 EMIT(PPC_RAW_MTLR(_R0)); 194 195} 196 197void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 198{ 199 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0))); 200 201 bpf_jit_emit_common_epilogue(image, ctx); 202 203 EMIT(PPC_RAW_BLR()); 204 205 bpf_jit_build_fentry_stubs(image, ctx); 206} 207 208/* Relative offset needs to be calculated based on final image location */ 209int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) 210{ 211 s32 rel = (s32)func - (s32)(fimage + ctx->idx); 212 213 if (image && rel < 0x2000000 && rel >= -0x2000000) { 214 EMIT(PPC_RAW_BL(rel)); 215 } else { 216 /* Load function address into r0 */ 217 EMIT(PPC_RAW_LIS(_R0, IMM_H(func))); 218 EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func))); 219 EMIT(PPC_RAW_MTCTR(_R0)); 220 EMIT(PPC_RAW_BCTRL()); 221 } 222 223 return 0; 224} 225 226static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 227{ 228 /* 229 * By now, the eBPF program has already setup parameters in r3-r6 230 * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 231 * r5-r6/BPF_REG_2 - pointer to bpf_array 232 * r7-r8/BPF_REG_3 - index in bpf_array 233 */ 234 int b2p_bpf_array = bpf_to_ppc(BPF_REG_2); 235 int b2p_index = bpf_to_ppc(BPF_REG_3); 236 237 /* 238 * if (index >= array->map.max_entries) 239 * goto out; 240 */ 241 EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries))); 242 EMIT(PPC_RAW_CMPLW(b2p_index, _R0)); 243 EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 244 PPC_BCC_SHORT(COND_GE, out); 245 246 /* 247 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) 248 * goto out; 249 */ 250 EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT)); 251 /* tail_call_cnt++; */ 252 EMIT(PPC_RAW_ADDIC(_R0, _R0, 1)); 253 PPC_BCC_SHORT(COND_GE, out); 254 255 /* prog = array->ptrs[index]; */ 256 EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29)); 257 EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array)); 258 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs))); 259 260 /* 261 * if (prog == NULL) 262 * goto out; 263 */ 264 EMIT(PPC_RAW_CMPLWI(_R3, 0)); 265 PPC_BCC_SHORT(COND_EQ, out); 266 267 /* goto *(prog->bpf_func + prologue_size); */ 268 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func))); 269 EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE)); 270 EMIT(PPC_RAW_MTCTR(_R3)); 271 272 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1))); 273 274 /* Put tail_call_cnt in r4 */ 275 EMIT(PPC_RAW_MR(_R4, _R0)); 276 277 /* tear restore NVRs, ... */ 278 bpf_jit_emit_common_epilogue(image, ctx); 279 280 EMIT(PPC_RAW_BCTR()); 281 282 /* out: */ 283 return 0; 284} 285 286/* Assemble the body code between the prologue & epilogue */ 287int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, 288 u32 *addrs, int pass, bool extra_pass) 289{ 290 const struct bpf_insn *insn = fp->insnsi; 291 int flen = fp->len; 292 int i, ret; 293 294 /* Start of epilogue code - will only be valid 2nd pass onwards */ 295 u32 exit_addr = addrs[flen]; 296 297 for (i = 0; i < flen; i++) { 298 u32 code = insn[i].code; 299 u32 prevcode = i ? insn[i - 1].code : 0; 300 u32 dst_reg = bpf_to_ppc(insn[i].dst_reg); 301 u32 dst_reg_h = dst_reg - 1; 302 u32 src_reg = bpf_to_ppc(insn[i].src_reg); 303 u32 src_reg_h = src_reg - 1; 304 u32 src2_reg = dst_reg; 305 u32 src2_reg_h = dst_reg_h; 306 u32 ax_reg = bpf_to_ppc(BPF_REG_AX); 307 u32 tmp_reg = bpf_to_ppc(TMP_REG); 308 u32 size = BPF_SIZE(code); 309 u32 save_reg, ret_reg; 310 s16 off = insn[i].off; 311 s32 imm = insn[i].imm; 312 bool func_addr_fixed; 313 u64 func_addr; 314 u32 true_cond; 315 u32 tmp_idx; 316 317 if (i && (BPF_CLASS(code) == BPF_ALU64 || BPF_CLASS(code) == BPF_ALU) && 318 (BPF_CLASS(prevcode) == BPF_ALU64 || BPF_CLASS(prevcode) == BPF_ALU) && 319 BPF_OP(prevcode) == BPF_MOV && BPF_SRC(prevcode) == BPF_X && 320 insn[i - 1].dst_reg == insn[i].dst_reg && insn[i - 1].imm != 1) { 321 src2_reg = bpf_to_ppc(insn[i - 1].src_reg); 322 src2_reg_h = src2_reg - 1; 323 ctx->idx = addrs[i - 1] / 4; 324 } 325 326 /* 327 * addrs[] maps a BPF bytecode address into a real offset from 328 * the start of the body code. 329 */ 330 addrs[i] = ctx->idx * 4; 331 332 /* 333 * As an optimization, we note down which registers 334 * are used so that we can only save/restore those in our 335 * prologue and epilogue. We do this here regardless of whether 336 * the actual BPF instruction uses src/dst registers or not 337 * (for instance, BPF_CALL does not use them). The expectation 338 * is that those instructions will have src_reg/dst_reg set to 339 * 0. Even otherwise, we just lose some prologue/epilogue 340 * optimization but everything else should work without 341 * any issues. 342 */ 343 if (dst_reg >= 3 && dst_reg < 32) { 344 bpf_set_seen_register(ctx, dst_reg); 345 bpf_set_seen_register(ctx, dst_reg_h); 346 } 347 348 if (src_reg >= 3 && src_reg < 32) { 349 bpf_set_seen_register(ctx, src_reg); 350 bpf_set_seen_register(ctx, src_reg_h); 351 } 352 353 switch (code) { 354 /* 355 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 356 */ 357 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 358 EMIT(PPC_RAW_ADD(dst_reg, src2_reg, src_reg)); 359 break; 360 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 361 EMIT(PPC_RAW_ADDC(dst_reg, src2_reg, src_reg)); 362 EMIT(PPC_RAW_ADDE(dst_reg_h, src2_reg_h, src_reg_h)); 363 break; 364 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 365 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, src_reg)); 366 break; 367 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 368 EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, src2_reg)); 369 EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, src2_reg_h)); 370 break; 371 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 372 imm = -imm; 373 fallthrough; 374 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 375 if (!imm) { 376 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 377 } else if (IMM_HA(imm) & 0xffff) { 378 EMIT(PPC_RAW_ADDIS(dst_reg, src2_reg, IMM_HA(imm))); 379 src2_reg = dst_reg; 380 } 381 if (IMM_L(imm)) 382 EMIT(PPC_RAW_ADDI(dst_reg, src2_reg, IMM_L(imm))); 383 break; 384 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 385 imm = -imm; 386 fallthrough; 387 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 388 if (!imm) { 389 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 390 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h)); 391 break; 392 } 393 if (imm >= -32768 && imm < 32768) { 394 EMIT(PPC_RAW_ADDIC(dst_reg, src2_reg, imm)); 395 } else { 396 PPC_LI32(_R0, imm); 397 EMIT(PPC_RAW_ADDC(dst_reg, src2_reg, _R0)); 398 } 399 if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000)) 400 EMIT(PPC_RAW_ADDZE(dst_reg_h, src2_reg_h)); 401 else 402 EMIT(PPC_RAW_ADDME(dst_reg_h, src2_reg_h)); 403 break; 404 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 405 bpf_set_seen_register(ctx, tmp_reg); 406 EMIT(PPC_RAW_MULW(_R0, src2_reg, src_reg_h)); 407 EMIT(PPC_RAW_MULW(dst_reg_h, src2_reg_h, src_reg)); 408 EMIT(PPC_RAW_MULHWU(tmp_reg, src2_reg, src_reg)); 409 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, src_reg)); 410 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0)); 411 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg)); 412 break; 413 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 414 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, src_reg)); 415 break; 416 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 417 if (imm == 1) { 418 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 419 } else if (imm == -1) { 420 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0)); 421 } else if (is_power_of_2((u32)imm)) { 422 EMIT(PPC_RAW_SLWI(dst_reg, src2_reg, ilog2(imm))); 423 } else if (imm >= -32768 && imm < 32768) { 424 EMIT(PPC_RAW_MULI(dst_reg, src2_reg, imm)); 425 } else { 426 PPC_LI32(_R0, imm); 427 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, _R0)); 428 } 429 break; 430 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 431 if (!imm) { 432 PPC_LI32(dst_reg, 0); 433 PPC_LI32(dst_reg_h, 0); 434 } else if (imm == 1) { 435 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 436 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h)); 437 } else if (imm == -1) { 438 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0)); 439 EMIT(PPC_RAW_SUBFZE(dst_reg_h, src2_reg_h)); 440 } else if (imm > 0 && is_power_of_2(imm)) { 441 imm = ilog2(imm); 442 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg_h, imm, 0, 31 - imm)); 443 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31)); 444 EMIT(PPC_RAW_SLWI(dst_reg, src2_reg, imm)); 445 } else { 446 bpf_set_seen_register(ctx, tmp_reg); 447 PPC_LI32(tmp_reg, imm); 448 EMIT(PPC_RAW_MULW(dst_reg_h, src2_reg_h, tmp_reg)); 449 if (imm < 0) 450 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, src2_reg)); 451 EMIT(PPC_RAW_MULHWU(_R0, src2_reg, tmp_reg)); 452 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, tmp_reg)); 453 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0)); 454 } 455 break; 456 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 457 if (off) 458 EMIT(PPC_RAW_DIVW(dst_reg, src2_reg, src_reg)); 459 else 460 EMIT(PPC_RAW_DIVWU(dst_reg, src2_reg, src_reg)); 461 break; 462 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 463 if (off) 464 EMIT(PPC_RAW_DIVW(_R0, src2_reg, src_reg)); 465 else 466 EMIT(PPC_RAW_DIVWU(_R0, src2_reg, src_reg)); 467 EMIT(PPC_RAW_MULW(_R0, src_reg, _R0)); 468 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, _R0)); 469 break; 470 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 471 return -EOPNOTSUPP; 472 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 473 return -EOPNOTSUPP; 474 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 475 if (!imm) 476 return -EINVAL; 477 if (imm == 1) { 478 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 479 } else if (is_power_of_2((u32)imm)) { 480 if (off) 481 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg, ilog2(imm))); 482 else 483 EMIT(PPC_RAW_SRWI(dst_reg, src2_reg, ilog2(imm))); 484 } else { 485 PPC_LI32(_R0, imm); 486 if (off) 487 EMIT(PPC_RAW_DIVW(dst_reg, src2_reg, _R0)); 488 else 489 EMIT(PPC_RAW_DIVWU(dst_reg, src2_reg, _R0)); 490 } 491 break; 492 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 493 if (!imm) 494 return -EINVAL; 495 496 if (!is_power_of_2((u32)imm)) { 497 bpf_set_seen_register(ctx, tmp_reg); 498 PPC_LI32(tmp_reg, imm); 499 if (off) 500 EMIT(PPC_RAW_DIVW(_R0, src2_reg, tmp_reg)); 501 else 502 EMIT(PPC_RAW_DIVWU(_R0, src2_reg, tmp_reg)); 503 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0)); 504 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, _R0)); 505 } else if (imm == 1) { 506 EMIT(PPC_RAW_LI(dst_reg, 0)); 507 } else if (off) { 508 EMIT(PPC_RAW_SRAWI(_R0, src2_reg, ilog2(imm))); 509 EMIT(PPC_RAW_ADDZE(_R0, _R0)); 510 EMIT(PPC_RAW_SLWI(_R0, _R0, ilog2(imm))); 511 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, _R0)); 512 } else { 513 imm = ilog2((u32)imm); 514 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 32 - imm, 31)); 515 } 516 break; 517 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 518 if (!imm) 519 return -EINVAL; 520 if (imm < 0) 521 imm = -imm; 522 if (!is_power_of_2(imm)) 523 return -EOPNOTSUPP; 524 if (imm == 1) { 525 EMIT(PPC_RAW_LI(dst_reg, 0)); 526 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 527 } else if (off) { 528 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, 31)); 529 EMIT(PPC_RAW_XOR(dst_reg, src2_reg, dst_reg_h)); 530 EMIT(PPC_RAW_SUBFC(dst_reg, dst_reg_h, dst_reg)); 531 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31)); 532 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, dst_reg_h)); 533 EMIT(PPC_RAW_SUBFC(dst_reg, dst_reg_h, dst_reg)); 534 EMIT(PPC_RAW_SUBFE(dst_reg_h, dst_reg_h, dst_reg_h)); 535 } else { 536 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 32 - ilog2(imm), 31)); 537 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 538 } 539 break; 540 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 541 if (!imm) 542 return -EINVAL; 543 if (!is_power_of_2(abs(imm))) 544 return -EOPNOTSUPP; 545 546 if (imm < 0) { 547 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0)); 548 EMIT(PPC_RAW_SUBFZE(dst_reg_h, src2_reg_h)); 549 imm = -imm; 550 src2_reg = dst_reg; 551 } 552 if (imm == 1) { 553 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 554 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h)); 555 } else { 556 imm = ilog2(imm); 557 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 32 - imm, imm, 31)); 558 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg_h, 32 - imm, 0, imm - 1)); 559 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, imm)); 560 } 561 break; 562 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 563 EMIT(PPC_RAW_NEG(dst_reg, src2_reg)); 564 break; 565 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 566 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0)); 567 EMIT(PPC_RAW_SUBFZE(dst_reg_h, src2_reg_h)); 568 break; 569 570 /* 571 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 572 */ 573 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 574 EMIT(PPC_RAW_AND(dst_reg, src2_reg, src_reg)); 575 EMIT(PPC_RAW_AND(dst_reg_h, src2_reg_h, src_reg_h)); 576 break; 577 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 578 EMIT(PPC_RAW_AND(dst_reg, src2_reg, src_reg)); 579 break; 580 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 581 if (imm >= 0) 582 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 583 fallthrough; 584 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 585 if (!IMM_H(imm)) { 586 EMIT(PPC_RAW_ANDI(dst_reg, src2_reg, IMM_L(imm))); 587 } else if (!IMM_L(imm)) { 588 EMIT(PPC_RAW_ANDIS(dst_reg, src2_reg, IMM_H(imm))); 589 } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) { 590 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 591 32 - fls(imm), 32 - ffs(imm))); 592 } else { 593 PPC_LI32(_R0, imm); 594 EMIT(PPC_RAW_AND(dst_reg, src2_reg, _R0)); 595 } 596 break; 597 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 598 EMIT(PPC_RAW_OR(dst_reg, src2_reg, src_reg)); 599 EMIT(PPC_RAW_OR(dst_reg_h, src2_reg_h, src_reg_h)); 600 break; 601 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 602 EMIT(PPC_RAW_OR(dst_reg, src2_reg, src_reg)); 603 break; 604 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 605 /* Sign-extended */ 606 if (imm < 0) 607 EMIT(PPC_RAW_LI(dst_reg_h, -1)); 608 fallthrough; 609 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 610 if (IMM_L(imm)) { 611 EMIT(PPC_RAW_ORI(dst_reg, src2_reg, IMM_L(imm))); 612 src2_reg = dst_reg; 613 } 614 if (IMM_H(imm)) 615 EMIT(PPC_RAW_ORIS(dst_reg, src2_reg, IMM_H(imm))); 616 break; 617 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 618 if (dst_reg == src_reg) { 619 EMIT(PPC_RAW_LI(dst_reg, 0)); 620 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 621 } else { 622 EMIT(PPC_RAW_XOR(dst_reg, src2_reg, src_reg)); 623 EMIT(PPC_RAW_XOR(dst_reg_h, src2_reg_h, src_reg_h)); 624 } 625 break; 626 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 627 if (dst_reg == src_reg) 628 EMIT(PPC_RAW_LI(dst_reg, 0)); 629 else 630 EMIT(PPC_RAW_XOR(dst_reg, src2_reg, src_reg)); 631 break; 632 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 633 if (imm < 0) 634 EMIT(PPC_RAW_NOR(dst_reg_h, src2_reg_h, src2_reg_h)); 635 fallthrough; 636 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 637 if (IMM_L(imm)) { 638 EMIT(PPC_RAW_XORI(dst_reg, src2_reg, IMM_L(imm))); 639 src2_reg = dst_reg; 640 } 641 if (IMM_H(imm)) 642 EMIT(PPC_RAW_XORIS(dst_reg, src2_reg, IMM_H(imm))); 643 break; 644 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 645 EMIT(PPC_RAW_SLW(dst_reg, src2_reg, src_reg)); 646 break; 647 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 648 bpf_set_seen_register(ctx, tmp_reg); 649 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32)); 650 EMIT(PPC_RAW_SLW(dst_reg_h, src2_reg_h, src_reg)); 651 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 652 EMIT(PPC_RAW_SRW(_R0, src2_reg, _R0)); 653 EMIT(PPC_RAW_SLW(tmp_reg, src2_reg, tmp_reg)); 654 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0)); 655 EMIT(PPC_RAW_SLW(dst_reg, src2_reg, src_reg)); 656 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg)); 657 break; 658 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */ 659 if (imm) 660 EMIT(PPC_RAW_SLWI(dst_reg, src2_reg, imm)); 661 else 662 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 663 break; 664 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */ 665 if (imm < 0) 666 return -EINVAL; 667 if (!imm) { 668 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 669 } else if (imm < 32) { 670 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg_h, imm, 0, 31 - imm)); 671 EMIT(PPC_RAW_RLWIMI(dst_reg_h, src2_reg, imm, 32 - imm, 31)); 672 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, imm, 0, 31 - imm)); 673 } else if (imm < 64) { 674 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg, imm, 0, 31 - imm)); 675 EMIT(PPC_RAW_LI(dst_reg, 0)); 676 } else { 677 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 678 EMIT(PPC_RAW_LI(dst_reg, 0)); 679 } 680 break; 681 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 682 EMIT(PPC_RAW_SRW(dst_reg, src2_reg, src_reg)); 683 break; 684 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 685 bpf_set_seen_register(ctx, tmp_reg); 686 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32)); 687 EMIT(PPC_RAW_SRW(dst_reg, src2_reg, src_reg)); 688 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 689 EMIT(PPC_RAW_SLW(_R0, src2_reg_h, _R0)); 690 EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg)); 691 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0)); 692 EMIT(PPC_RAW_SRW(dst_reg_h, src2_reg_h, src_reg)); 693 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg)); 694 break; 695 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 696 if (imm) 697 EMIT(PPC_RAW_SRWI(dst_reg, src2_reg, imm)); 698 else 699 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 700 break; 701 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 702 if (imm < 0) 703 return -EINVAL; 704 if (!imm) { 705 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 706 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h)); 707 } else if (imm < 32) { 708 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 32 - imm, imm, 31)); 709 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg_h, 32 - imm, 0, imm - 1)); 710 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg_h, 32 - imm, imm, 31)); 711 } else if (imm < 64) { 712 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg_h, 64 - imm, imm - 32, 31)); 713 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 714 } else { 715 EMIT(PPC_RAW_LI(dst_reg, 0)); 716 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 717 } 718 break; 719 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ 720 EMIT(PPC_RAW_SRAW(dst_reg, src2_reg, src_reg)); 721 break; 722 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 723 bpf_set_seen_register(ctx, tmp_reg); 724 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32)); 725 EMIT(PPC_RAW_SRW(dst_reg, src2_reg, src_reg)); 726 EMIT(PPC_RAW_SLW(_R0, src2_reg_h, _R0)); 727 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 728 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0)); 729 EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26)); 730 EMIT(PPC_RAW_SRAW(tmp_reg, src2_reg_h, tmp_reg)); 731 EMIT(PPC_RAW_SRAW(dst_reg_h, src2_reg_h, src_reg)); 732 EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0)); 733 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg)); 734 break; 735 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ 736 if (imm) 737 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg, imm)); 738 else 739 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 740 break; 741 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 742 if (imm < 0) 743 return -EINVAL; 744 if (!imm) { 745 EMIT(PPC_RAW_MR(dst_reg, src2_reg)); 746 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h)); 747 } else if (imm < 32) { 748 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 32 - imm, imm, 31)); 749 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg_h, 32 - imm, 0, imm - 1)); 750 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, imm)); 751 } else if (imm < 64) { 752 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg_h, imm - 32)); 753 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, 31)); 754 } else { 755 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg_h, 31)); 756 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, 31)); 757 } 758 break; 759 760 /* 761 * MOV 762 */ 763 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 764 if (off == 8) { 765 EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); 766 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg, 31)); 767 } else if (off == 16) { 768 EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); 769 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg, 31)); 770 } else if (off == 32 && dst_reg == src_reg) { 771 EMIT(PPC_RAW_SRAWI(dst_reg_h, src_reg, 31)); 772 } else if (off == 32) { 773 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 774 EMIT(PPC_RAW_SRAWI(dst_reg_h, src_reg, 31)); 775 } else if (dst_reg != src_reg) { 776 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 777 EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h)); 778 } 779 break; 780 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 781 /* special mov32 for zext */ 782 if (imm == 1) 783 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 784 else if (off == 8) 785 EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); 786 else if (off == 16) 787 EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); 788 else if (dst_reg != src_reg) 789 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 790 break; 791 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 792 PPC_LI32(dst_reg, imm); 793 PPC_EX32(dst_reg_h, imm); 794 break; 795 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 796 PPC_LI32(dst_reg, imm); 797 break; 798 799 /* 800 * BPF_FROM_BE/LE 801 */ 802 case BPF_ALU | BPF_END | BPF_FROM_LE: 803 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 804 switch (imm) { 805 case 16: 806 /* Copy 16 bits to upper part */ 807 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg, 16, 0, 15)); 808 /* Rotate 8 bits right & mask */ 809 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31)); 810 break; 811 case 32: 812 /* 813 * Rotate word left by 8 bits: 814 * 2 bytes are already in their final position 815 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 816 */ 817 EMIT(PPC_RAW_RLWINM(_R0, src2_reg, 8, 0, 31)); 818 /* Rotate 24 bits and insert byte 1 */ 819 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg, 24, 0, 7)); 820 /* Rotate 24 bits and insert byte 3 */ 821 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg, 24, 16, 23)); 822 EMIT(PPC_RAW_MR(dst_reg, _R0)); 823 break; 824 case 64: 825 bpf_set_seen_register(ctx, tmp_reg); 826 EMIT(PPC_RAW_RLWINM(tmp_reg, src2_reg, 8, 0, 31)); 827 EMIT(PPC_RAW_RLWINM(_R0, src2_reg_h, 8, 0, 31)); 828 /* Rotate 24 bits and insert byte 1 */ 829 EMIT(PPC_RAW_RLWIMI(tmp_reg, src2_reg, 24, 0, 7)); 830 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg_h, 24, 0, 7)); 831 /* Rotate 24 bits and insert byte 3 */ 832 EMIT(PPC_RAW_RLWIMI(tmp_reg, src2_reg, 24, 16, 23)); 833 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg_h, 24, 16, 23)); 834 EMIT(PPC_RAW_MR(dst_reg, _R0)); 835 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg)); 836 break; 837 } 838 if (BPF_CLASS(code) == BPF_ALU64 && imm != 64) 839 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 840 break; 841 case BPF_ALU | BPF_END | BPF_FROM_BE: 842 switch (imm) { 843 case 16: 844 /* zero-extend 16 bits into 32 bits */ 845 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 16, 31)); 846 break; 847 case 32: 848 case 64: 849 /* nop */ 850 break; 851 } 852 break; 853 854 /* 855 * BPF_ST NOSPEC (speculation barrier) 856 */ 857 case BPF_ST | BPF_NOSPEC: 858 break; 859 860 /* 861 * BPF_ST(X) 862 */ 863 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 864 EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); 865 break; 866 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 867 PPC_LI32(_R0, imm); 868 EMIT(PPC_RAW_STB(_R0, dst_reg, off)); 869 break; 870 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 871 EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); 872 break; 873 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 874 PPC_LI32(_R0, imm); 875 EMIT(PPC_RAW_STH(_R0, dst_reg, off)); 876 break; 877 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 878 EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); 879 break; 880 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 881 PPC_LI32(_R0, imm); 882 EMIT(PPC_RAW_STW(_R0, dst_reg, off)); 883 break; 884 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 885 EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off)); 886 EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4)); 887 break; 888 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 889 PPC_LI32(_R0, imm); 890 EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4)); 891 PPC_EX32(_R0, imm); 892 EMIT(PPC_RAW_STW(_R0, dst_reg, off)); 893 break; 894 895 /* 896 * BPF_STX ATOMIC (atomic ops) 897 */ 898 case BPF_STX | BPF_ATOMIC | BPF_W: 899 save_reg = _R0; 900 ret_reg = src_reg; 901 902 bpf_set_seen_register(ctx, tmp_reg); 903 bpf_set_seen_register(ctx, ax_reg); 904 905 /* Get offset into TMP_REG */ 906 EMIT(PPC_RAW_LI(tmp_reg, off)); 907 /* 908 * Enforce full ordering for operations with BPF_FETCH by emitting a 'sync' 909 * before and after the operation. 910 * 911 * This is a requirement in the Linux Kernel Memory Model. 912 * See __cmpxchg_u32() in asm/cmpxchg.h as an example. 913 */ 914 if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP)) 915 EMIT(PPC_RAW_SYNC()); 916 tmp_idx = ctx->idx * 4; 917 /* load value from memory into r0 */ 918 EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0)); 919 920 /* Save old value in BPF_REG_AX */ 921 if (imm & BPF_FETCH) 922 EMIT(PPC_RAW_MR(ax_reg, _R0)); 923 924 switch (imm) { 925 case BPF_ADD: 926 case BPF_ADD | BPF_FETCH: 927 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg)); 928 break; 929 case BPF_AND: 930 case BPF_AND | BPF_FETCH: 931 EMIT(PPC_RAW_AND(_R0, _R0, src_reg)); 932 break; 933 case BPF_OR: 934 case BPF_OR | BPF_FETCH: 935 EMIT(PPC_RAW_OR(_R0, _R0, src_reg)); 936 break; 937 case BPF_XOR: 938 case BPF_XOR | BPF_FETCH: 939 EMIT(PPC_RAW_XOR(_R0, _R0, src_reg)); 940 break; 941 case BPF_CMPXCHG: 942 /* 943 * Return old value in BPF_REG_0 for BPF_CMPXCHG & 944 * in src_reg for other cases. 945 */ 946 ret_reg = bpf_to_ppc(BPF_REG_0); 947 948 /* Compare with old value in BPF_REG_0 */ 949 EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), _R0)); 950 /* Don't set if different from old value */ 951 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4); 952 fallthrough; 953 case BPF_XCHG: 954 save_reg = src_reg; 955 break; 956 default: 957 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n", 958 code, i); 959 return -EOPNOTSUPP; 960 } 961 962 /* store new value */ 963 EMIT(PPC_RAW_STWCX(save_reg, tmp_reg, dst_reg)); 964 /* we're done if this succeeded */ 965 PPC_BCC_SHORT(COND_NE, tmp_idx); 966 967 /* For the BPF_FETCH variant, get old data into src_reg */ 968 if (imm & BPF_FETCH) { 969 /* Emit 'sync' to enforce full ordering */ 970 if (IS_ENABLED(CONFIG_SMP)) 971 EMIT(PPC_RAW_SYNC()); 972 EMIT(PPC_RAW_MR(ret_reg, ax_reg)); 973 if (!fp->aux->verifier_zext) 974 EMIT(PPC_RAW_LI(ret_reg - 1, 0)); /* higher 32-bit */ 975 } 976 break; 977 978 case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */ 979 return -EOPNOTSUPP; 980 981 /* 982 * BPF_LDX 983 */ 984 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 985 case BPF_LDX | BPF_MEMSX | BPF_B: 986 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 987 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 988 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 989 case BPF_LDX | BPF_MEMSX | BPF_H: 990 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 991 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 992 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 993 case BPF_LDX | BPF_MEMSX | BPF_W: 994 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 995 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 996 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 997 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 998 /* 999 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid 1000 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM 1001 * load only if addr is kernel address (see is_kernel_addr()), otherwise 1002 * set dst_reg=0 and move on. 1003 */ 1004 if (BPF_MODE(code) == BPF_PROBE_MEM || BPF_MODE(code) == BPF_PROBE_MEMSX) { 1005 PPC_LI32(_R0, TASK_SIZE - off); 1006 EMIT(PPC_RAW_CMPLW(src_reg, _R0)); 1007 PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4); 1008 EMIT(PPC_RAW_LI(dst_reg, 0)); 1009 /* 1010 * For BPF_DW case, "li reg_h,0" would be needed when 1011 * !fp->aux->verifier_zext. Emit NOP otherwise. 1012 * 1013 * Note that "li reg_h,0" is emitted for BPF_B/H/W case, 1014 * if necessary. So, jump there instead of emitting an 1015 * additional "li reg_h,0" instruction. 1016 */ 1017 if (size == BPF_DW && !fp->aux->verifier_zext) 1018 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 1019 else 1020 EMIT(PPC_RAW_NOP()); 1021 /* 1022 * Need to jump two instructions instead of one for BPF_DW case 1023 * as there are two load instructions for dst_reg_h & dst_reg 1024 * respectively. 1025 */ 1026 if (size == BPF_DW || 1027 (size == BPF_B && BPF_MODE(code) == BPF_PROBE_MEMSX)) 1028 PPC_JMP((ctx->idx + 3) * 4); 1029 else 1030 PPC_JMP((ctx->idx + 2) * 4); 1031 } 1032 1033 if (BPF_MODE(code) == BPF_MEMSX || BPF_MODE(code) == BPF_PROBE_MEMSX) { 1034 switch (size) { 1035 case BPF_B: 1036 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 1037 EMIT(PPC_RAW_EXTSB(dst_reg, dst_reg)); 1038 break; 1039 case BPF_H: 1040 EMIT(PPC_RAW_LHA(dst_reg, src_reg, off)); 1041 break; 1042 case BPF_W: 1043 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 1044 break; 1045 } 1046 if (!fp->aux->verifier_zext) 1047 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg, 31)); 1048 1049 } else { 1050 switch (size) { 1051 case BPF_B: 1052 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 1053 break; 1054 case BPF_H: 1055 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); 1056 break; 1057 case BPF_W: 1058 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 1059 break; 1060 case BPF_DW: 1061 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off)); 1062 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4)); 1063 break; 1064 } 1065 if (size != BPF_DW && !fp->aux->verifier_zext) 1066 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 1067 } 1068 1069 if (BPF_MODE(code) == BPF_PROBE_MEM) { 1070 int insn_idx = ctx->idx - 1; 1071 int jmp_off = 4; 1072 1073 /* 1074 * In case of BPF_DW, two lwz instructions are emitted, one 1075 * for higher 32-bit and another for lower 32-bit. So, set 1076 * ex->insn to the first of the two and jump over both 1077 * instructions in fixup. 1078 * 1079 * Similarly, with !verifier_zext, two instructions are 1080 * emitted for BPF_B/H/W case. So, set ex->insn to the 1081 * instruction that could fault and skip over both 1082 * instructions. 1083 */ 1084 if (size == BPF_DW || !fp->aux->verifier_zext) { 1085 insn_idx -= 1; 1086 jmp_off += 4; 1087 } 1088 1089 ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, insn_idx, 1090 jmp_off, dst_reg, code); 1091 if (ret) 1092 return ret; 1093 } 1094 break; 1095 1096 /* 1097 * Doubleword load 1098 * 16 byte instruction that uses two 'struct bpf_insn' 1099 */ 1100 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 1101 PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm); 1102 PPC_LI32(dst_reg, (u32)insn[i].imm); 1103 /* Adjust for two bpf instructions */ 1104 addrs[++i] = ctx->idx * 4; 1105 break; 1106 1107 /* 1108 * Return/Exit 1109 */ 1110 case BPF_JMP | BPF_EXIT: 1111 /* 1112 * If this isn't the very last instruction, branch to 1113 * the epilogue. If we _are_ the last instruction, 1114 * we'll just fall through to the epilogue. 1115 */ 1116 if (i != flen - 1) { 1117 ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr); 1118 if (ret) 1119 return ret; 1120 } 1121 /* else fall through to the epilogue */ 1122 break; 1123 1124 /* 1125 * Call kernel helper or bpf function 1126 */ 1127 case BPF_JMP | BPF_CALL: 1128 ctx->seen |= SEEN_FUNC; 1129 1130 ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, 1131 &func_addr, &func_addr_fixed); 1132 if (ret < 0) 1133 return ret; 1134 1135 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) { 1136 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8)); 1137 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12)); 1138 } 1139 1140 ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr); 1141 if (ret) 1142 return ret; 1143 1144 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3)); 1145 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4)); 1146 break; 1147 1148 /* 1149 * Jumps and branches 1150 */ 1151 case BPF_JMP | BPF_JA: 1152 PPC_JMP(addrs[i + 1 + off]); 1153 break; 1154 case BPF_JMP32 | BPF_JA: 1155 PPC_JMP(addrs[i + 1 + imm]); 1156 break; 1157 1158 case BPF_JMP | BPF_JGT | BPF_K: 1159 case BPF_JMP | BPF_JGT | BPF_X: 1160 case BPF_JMP | BPF_JSGT | BPF_K: 1161 case BPF_JMP | BPF_JSGT | BPF_X: 1162 case BPF_JMP32 | BPF_JGT | BPF_K: 1163 case BPF_JMP32 | BPF_JGT | BPF_X: 1164 case BPF_JMP32 | BPF_JSGT | BPF_K: 1165 case BPF_JMP32 | BPF_JSGT | BPF_X: 1166 true_cond = COND_GT; 1167 goto cond_branch; 1168 case BPF_JMP | BPF_JLT | BPF_K: 1169 case BPF_JMP | BPF_JLT | BPF_X: 1170 case BPF_JMP | BPF_JSLT | BPF_K: 1171 case BPF_JMP | BPF_JSLT | BPF_X: 1172 case BPF_JMP32 | BPF_JLT | BPF_K: 1173 case BPF_JMP32 | BPF_JLT | BPF_X: 1174 case BPF_JMP32 | BPF_JSLT | BPF_K: 1175 case BPF_JMP32 | BPF_JSLT | BPF_X: 1176 true_cond = COND_LT; 1177 goto cond_branch; 1178 case BPF_JMP | BPF_JGE | BPF_K: 1179 case BPF_JMP | BPF_JGE | BPF_X: 1180 case BPF_JMP | BPF_JSGE | BPF_K: 1181 case BPF_JMP | BPF_JSGE | BPF_X: 1182 case BPF_JMP32 | BPF_JGE | BPF_K: 1183 case BPF_JMP32 | BPF_JGE | BPF_X: 1184 case BPF_JMP32 | BPF_JSGE | BPF_K: 1185 case BPF_JMP32 | BPF_JSGE | BPF_X: 1186 true_cond = COND_GE; 1187 goto cond_branch; 1188 case BPF_JMP | BPF_JLE | BPF_K: 1189 case BPF_JMP | BPF_JLE | BPF_X: 1190 case BPF_JMP | BPF_JSLE | BPF_K: 1191 case BPF_JMP | BPF_JSLE | BPF_X: 1192 case BPF_JMP32 | BPF_JLE | BPF_K: 1193 case BPF_JMP32 | BPF_JLE | BPF_X: 1194 case BPF_JMP32 | BPF_JSLE | BPF_K: 1195 case BPF_JMP32 | BPF_JSLE | BPF_X: 1196 true_cond = COND_LE; 1197 goto cond_branch; 1198 case BPF_JMP | BPF_JEQ | BPF_K: 1199 case BPF_JMP | BPF_JEQ | BPF_X: 1200 case BPF_JMP32 | BPF_JEQ | BPF_K: 1201 case BPF_JMP32 | BPF_JEQ | BPF_X: 1202 true_cond = COND_EQ; 1203 goto cond_branch; 1204 case BPF_JMP | BPF_JNE | BPF_K: 1205 case BPF_JMP | BPF_JNE | BPF_X: 1206 case BPF_JMP32 | BPF_JNE | BPF_K: 1207 case BPF_JMP32 | BPF_JNE | BPF_X: 1208 true_cond = COND_NE; 1209 goto cond_branch; 1210 case BPF_JMP | BPF_JSET | BPF_K: 1211 case BPF_JMP | BPF_JSET | BPF_X: 1212 case BPF_JMP32 | BPF_JSET | BPF_K: 1213 case BPF_JMP32 | BPF_JSET | BPF_X: 1214 true_cond = COND_NE; 1215 /* fallthrough; */ 1216 1217cond_branch: 1218 switch (code) { 1219 case BPF_JMP | BPF_JGT | BPF_X: 1220 case BPF_JMP | BPF_JLT | BPF_X: 1221 case BPF_JMP | BPF_JGE | BPF_X: 1222 case BPF_JMP | BPF_JLE | BPF_X: 1223 case BPF_JMP | BPF_JEQ | BPF_X: 1224 case BPF_JMP | BPF_JNE | BPF_X: 1225 /* unsigned comparison */ 1226 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h)); 1227 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1228 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1229 break; 1230 case BPF_JMP32 | BPF_JGT | BPF_X: 1231 case BPF_JMP32 | BPF_JLT | BPF_X: 1232 case BPF_JMP32 | BPF_JGE | BPF_X: 1233 case BPF_JMP32 | BPF_JLE | BPF_X: 1234 case BPF_JMP32 | BPF_JEQ | BPF_X: 1235 case BPF_JMP32 | BPF_JNE | BPF_X: 1236 /* unsigned comparison */ 1237 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1238 break; 1239 case BPF_JMP | BPF_JSGT | BPF_X: 1240 case BPF_JMP | BPF_JSLT | BPF_X: 1241 case BPF_JMP | BPF_JSGE | BPF_X: 1242 case BPF_JMP | BPF_JSLE | BPF_X: 1243 /* signed comparison */ 1244 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h)); 1245 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1246 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1247 break; 1248 case BPF_JMP32 | BPF_JSGT | BPF_X: 1249 case BPF_JMP32 | BPF_JSLT | BPF_X: 1250 case BPF_JMP32 | BPF_JSGE | BPF_X: 1251 case BPF_JMP32 | BPF_JSLE | BPF_X: 1252 /* signed comparison */ 1253 EMIT(PPC_RAW_CMPW(dst_reg, src_reg)); 1254 break; 1255 case BPF_JMP | BPF_JSET | BPF_X: 1256 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h)); 1257 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1258 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg)); 1259 break; 1260 case BPF_JMP32 | BPF_JSET | BPF_X: { 1261 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg)); 1262 break; 1263 case BPF_JMP | BPF_JNE | BPF_K: 1264 case BPF_JMP | BPF_JEQ | BPF_K: 1265 case BPF_JMP | BPF_JGT | BPF_K: 1266 case BPF_JMP | BPF_JLT | BPF_K: 1267 case BPF_JMP | BPF_JGE | BPF_K: 1268 case BPF_JMP | BPF_JLE | BPF_K: 1269 /* 1270 * Need sign-extended load, so only positive 1271 * values can be used as imm in cmplwi 1272 */ 1273 if (imm >= 0 && imm < 32768) { 1274 EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0)); 1275 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1276 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1277 } else { 1278 /* sign-extending load ... but unsigned comparison */ 1279 PPC_EX32(_R0, imm); 1280 EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0)); 1281 PPC_LI32(_R0, imm); 1282 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1283 EMIT(PPC_RAW_CMPLW(dst_reg, _R0)); 1284 } 1285 break; 1286 case BPF_JMP32 | BPF_JNE | BPF_K: 1287 case BPF_JMP32 | BPF_JEQ | BPF_K: 1288 case BPF_JMP32 | BPF_JGT | BPF_K: 1289 case BPF_JMP32 | BPF_JLT | BPF_K: 1290 case BPF_JMP32 | BPF_JGE | BPF_K: 1291 case BPF_JMP32 | BPF_JLE | BPF_K: 1292 if (imm >= 0 && imm < 65536) { 1293 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1294 } else { 1295 PPC_LI32(_R0, imm); 1296 EMIT(PPC_RAW_CMPLW(dst_reg, _R0)); 1297 } 1298 break; 1299 } 1300 case BPF_JMP | BPF_JSGT | BPF_K: 1301 case BPF_JMP | BPF_JSLT | BPF_K: 1302 case BPF_JMP | BPF_JSGE | BPF_K: 1303 case BPF_JMP | BPF_JSLE | BPF_K: 1304 if (imm >= 0 && imm < 65536) { 1305 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0)); 1306 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1307 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1308 } else { 1309 /* sign-extending load */ 1310 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0)); 1311 PPC_LI32(_R0, imm); 1312 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1313 EMIT(PPC_RAW_CMPLW(dst_reg, _R0)); 1314 } 1315 break; 1316 case BPF_JMP32 | BPF_JSGT | BPF_K: 1317 case BPF_JMP32 | BPF_JSLT | BPF_K: 1318 case BPF_JMP32 | BPF_JSGE | BPF_K: 1319 case BPF_JMP32 | BPF_JSLE | BPF_K: 1320 /* 1321 * signed comparison, so any 16-bit value 1322 * can be used in cmpwi 1323 */ 1324 if (imm >= -32768 && imm < 32768) { 1325 EMIT(PPC_RAW_CMPWI(dst_reg, imm)); 1326 } else { 1327 /* sign-extending load */ 1328 PPC_LI32(_R0, imm); 1329 EMIT(PPC_RAW_CMPW(dst_reg, _R0)); 1330 } 1331 break; 1332 case BPF_JMP | BPF_JSET | BPF_K: 1333 /* andi does not sign-extend the immediate */ 1334 if (imm >= 0 && imm < 32768) { 1335 /* PPC_ANDI is _only/always_ dot-form */ 1336 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm)); 1337 } else { 1338 PPC_LI32(_R0, imm); 1339 if (imm < 0) { 1340 EMIT(PPC_RAW_CMPWI(dst_reg_h, 0)); 1341 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1342 } 1343 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0)); 1344 } 1345 break; 1346 case BPF_JMP32 | BPF_JSET | BPF_K: 1347 /* andi does not sign-extend the immediate */ 1348 if (imm >= 0 && imm < 32768) { 1349 /* PPC_ANDI is _only/always_ dot-form */ 1350 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm)); 1351 } else { 1352 PPC_LI32(_R0, imm); 1353 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0)); 1354 } 1355 break; 1356 } 1357 PPC_BCC(true_cond, addrs[i + 1 + off]); 1358 break; 1359 1360 /* 1361 * Tail call 1362 */ 1363 case BPF_JMP | BPF_TAIL_CALL: 1364 ctx->seen |= SEEN_TAILCALL; 1365 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 1366 if (ret < 0) 1367 return ret; 1368 break; 1369 1370 default: 1371 /* 1372 * The filter contains something cruel & unusual. 1373 * We don't handle it, but also there shouldn't be 1374 * anything missing from our list. 1375 */ 1376 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i); 1377 return -EOPNOTSUPP; 1378 } 1379 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext && 1380 !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64)) 1381 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 1382 } 1383 1384 /* Set end-of-body-code address for exit. */ 1385 addrs[i] = ctx->idx * 4; 1386 1387 return 0; 1388}
~/
Read-only file preview.