bpf_jit_comp64.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * bpf_jit_comp64.c: eBPF JIT compiler
   4 *
   5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
   6 *		  IBM Corporation
   7 *
   8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
   9 */
  10#include <linux/moduleloader.h>
  11#include <asm/cacheflush.h>
  12#include <asm/asm-compat.h>
  13#include <linux/netdevice.h>
  14#include <linux/filter.h>
  15#include <linux/if_vlan.h>
  16#include <asm/kprobes.h>
  17#include <linux/bpf.h>
  18#include <asm/security_features.h>
  19
  20#include "bpf_jit.h"
  21
  22/*
  23 * Stack layout:
  24 * Ensure the top half (upto local_tmp_var) stays consistent
  25 * with our redzone usage.
  26 *
  27 *		[	prev sp		] <-------------
  28 *		[   nv gpr save area	] 6*8		|
  29 *		[    tail_call_cnt	] 8		|
  30 *		[    local_tmp_var	] 24		|
  31 * fp (r31) -->	[   ebpf stack space	] upto 512	|
  32 *		[     frame header	] 32/112	|
  33 * sp (r1) --->	[    stack pointer	] --------------
  34 */
  35
  36/* for gpr non volatile registers BPG_REG_6 to 10 */
  37#define BPF_PPC_STACK_SAVE	(6*8)
  38/* for bpf JIT code internal usage */
  39#define BPF_PPC_STACK_LOCALS	32
  40/* stack frame excluding BPF stack, ensure this is quadword aligned */
  41#define BPF_PPC_STACKFRAME	(STACK_FRAME_MIN_SIZE + \
  42				 BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE)
  43
  44/* BPF register usage */
  45#define TMP_REG_1	(MAX_BPF_JIT_REG + 0)
  46#define TMP_REG_2	(MAX_BPF_JIT_REG + 1)
  47#define ARENA_VM_START  (MAX_BPF_JIT_REG + 2)
  48
  49/* BPF to ppc register mappings */
  50void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
  51{
  52	/* function return value */
  53	ctx->b2p[BPF_REG_0] = _R8;
  54	/* function arguments */
  55	ctx->b2p[BPF_REG_1] = _R3;
  56	ctx->b2p[BPF_REG_2] = _R4;
  57	ctx->b2p[BPF_REG_3] = _R5;
  58	ctx->b2p[BPF_REG_4] = _R6;
  59	ctx->b2p[BPF_REG_5] = _R7;
  60	/* non volatile registers */
  61	ctx->b2p[BPF_REG_6] = _R27;
  62	ctx->b2p[BPF_REG_7] = _R28;
  63	ctx->b2p[BPF_REG_8] = _R29;
  64	ctx->b2p[BPF_REG_9] = _R30;
  65	/* frame pointer aka BPF_REG_10 */
  66	ctx->b2p[BPF_REG_FP] = _R31;
  67	/* eBPF jit internal registers */
  68	ctx->b2p[BPF_REG_AX] = _R12;
  69	ctx->b2p[TMP_REG_1] = _R9;
  70	ctx->b2p[TMP_REG_2] = _R10;
  71	/* non volatile register for kern_vm_start address */
  72	ctx->b2p[ARENA_VM_START] = _R26;
  73}
  74
  75/* PPC NVR range -- update this if we ever use NVRs below r26 */
  76#define BPF_PPC_NVR_MIN		_R26
  77
  78static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
  79{
  80	/*
  81	 * We only need a stack frame if:
  82	 * - we call other functions (kernel helpers), or
  83	 * - the bpf program uses its stack area
  84	 * The latter condition is deduced from the usage of BPF_REG_FP
  85	 */
  86	return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP));
  87}
  88
  89/*
  90 * When not setting up our own stackframe, the redzone (288 bytes) usage is:
  91 *
  92 *		[	prev sp		] <-------------
  93 *		[	  ...       	] 		|
  94 * sp (r1) --->	[    stack pointer	] --------------
  95 *		[   nv gpr save area	] 6*8
  96 *		[    tail_call_cnt	] 8
  97 *		[    local_tmp_var	] 24
  98 *		[   unused red zone	] 224
  99 */
 100static int bpf_jit_stack_local(struct codegen_context *ctx)
 101{
 102	if (bpf_has_stack_frame(ctx))
 103		return STACK_FRAME_MIN_SIZE + ctx->stack_size;
 104	else
 105		return -(BPF_PPC_STACK_SAVE + 32);
 106}
 107
 108static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
 109{
 110	return bpf_jit_stack_local(ctx) + 24;
 111}
 112
 113static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
 114{
 115	if (reg >= BPF_PPC_NVR_MIN && reg < 32)
 116		return (bpf_has_stack_frame(ctx) ?
 117			(BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
 118				- (8 * (32 - reg));
 119
 120	pr_err("BPF JIT is asking about unknown registers");
 121	BUG();
 122}
 123
 124void bpf_jit_realloc_regs(struct codegen_context *ctx)
 125{
 126}
 127
 128void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
 129{
 130	int i;
 131
 132	/* Instruction for trampoline attach */
 133	EMIT(PPC_RAW_NOP());
 134
 135#ifndef CONFIG_PPC_KERNEL_PCREL
 136	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
 137		EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc)));
 138#endif
 139
 140	/*
 141	 * Initialize tail_call_cnt if we do tail calls.
 142	 * Otherwise, put in NOPs so that it can be skipped when we are
 143	 * invoked through a tail call.
 144	 */
 145	if (ctx->seen & SEEN_TAILCALL) {
 146		EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0));
 147		/* this goes in the redzone */
 148		EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_STACK_SAVE + 8)));
 149	} else {
 150		EMIT(PPC_RAW_NOP());
 151		EMIT(PPC_RAW_NOP());
 152	}
 153
 154	if (bpf_has_stack_frame(ctx)) {
 155		/*
 156		 * We need a stack frame, but we don't necessarily need to
 157		 * save/restore LR unless we call other functions
 158		 */
 159		if (ctx->seen & SEEN_FUNC) {
 160			EMIT(PPC_RAW_MFLR(_R0));
 161			EMIT(PPC_RAW_STD(_R0, _R1, PPC_LR_STKOFF));
 162		}
 163
 164		EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_STACKFRAME + ctx->stack_size)));
 165	}
 166
 167	/*
 168	 * Back up non-volatile regs -- BPF registers 6-10
 169	 * If we haven't created our own stack frame, we save these
 170	 * in the protected zone below the previous stack frame
 171	 */
 172	for (i = BPF_REG_6; i <= BPF_REG_10; i++)
 173		if (bpf_is_seen_register(ctx, bpf_to_ppc(i)))
 174			EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
 175
 176	if (ctx->arena_vm_start)
 177		EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1,
 178				 bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));
 179
 180	/* Setup frame pointer to point to the bpf stack area */
 181	if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)))
 182		EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
 183				STACK_FRAME_MIN_SIZE + ctx->stack_size));
 184
 185	if (ctx->arena_vm_start)
 186		PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start);
 187}
 188
 189static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
 190{
 191	int i;
 192
 193	/* Restore NVRs */
 194	for (i = BPF_REG_6; i <= BPF_REG_10; i++)
 195		if (bpf_is_seen_register(ctx, bpf_to_ppc(i)))
 196			EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
 197
 198	if (ctx->arena_vm_start)
 199		EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1,
 200				bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));
 201
 202	/* Tear down our stack frame */
 203	if (bpf_has_stack_frame(ctx)) {
 204		EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size));
 205		if (ctx->seen & SEEN_FUNC) {
 206			EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF));
 207			EMIT(PPC_RAW_MTLR(_R0));
 208		}
 209	}
 210}
 211
 212void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
 213{
 214	bpf_jit_emit_common_epilogue(image, ctx);
 215
 216	/* Move result to r3 */
 217	EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
 218
 219	EMIT(PPC_RAW_BLR());
 220
 221	bpf_jit_build_fentry_stubs(image, ctx);
 222}
 223
 224int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func)
 225{
 226	unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0;
 227	long reladdr;
 228
 229	/* bpf to bpf call, func is not known in the initial pass. Emit 5 nops as a placeholder */
 230	if (!func) {
 231		for (int i = 0; i < 5; i++)
 232			EMIT(PPC_RAW_NOP());
 233		/* elfv1 needs an additional instruction to load addr from descriptor */
 234		if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1))
 235			EMIT(PPC_RAW_NOP());
 236		EMIT(PPC_RAW_MTCTR(_R12));
 237		EMIT(PPC_RAW_BCTRL());
 238		return 0;
 239	}
 240
 241#ifdef CONFIG_PPC_KERNEL_PCREL
 242	reladdr = func_addr - local_paca->kernelbase;
 243
 244	/*
 245	 * If fimage is NULL (the initial pass to find image size),
 246	 * account for the maximum no. of instructions possible.
 247	 */
 248	if (!fimage) {
 249		ctx->idx += 7;
 250		return 0;
 251	} else if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) {
 252		EMIT(PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)));
 253		/* Align for subsequent prefix instruction */
 254		if (!IS_ALIGNED((unsigned long)fimage + CTX_NIA(ctx), 8))
 255			EMIT(PPC_RAW_NOP());
 256		/* paddi r12,r12,addr */
 257		EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(0) | IMM_H18(reladdr));
 258		EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12) | IMM_L(reladdr));
 259	} else {
 260		unsigned long pc = (unsigned long)fimage + CTX_NIA(ctx);
 261		bool alignment_needed = !IS_ALIGNED(pc, 8);
 262
 263		reladdr = func_addr - (alignment_needed ? pc + 4 :  pc);
 264
 265		if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) {
 266			if (alignment_needed)
 267				EMIT(PPC_RAW_NOP());
 268			/* pla r12,addr */
 269			EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(1) | IMM_H18(reladdr));
 270			EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | IMM_L(reladdr));
 271		} else {
 272			/* We can clobber r12 */
 273			PPC_LI64(_R12, func);
 274		}
 275	}
 276	EMIT(PPC_RAW_MTCTR(_R12));
 277	EMIT(PPC_RAW_BCTRL());
 278#else
 279	if (core_kernel_text(func_addr)) {
 280		reladdr = func_addr - kernel_toc_addr();
 281		if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
 282			pr_err("eBPF: address of %ps out of range of kernel_toc.\n", (void *)func);
 283			return -ERANGE;
 284		}
 285
 286		EMIT(PPC_RAW_ADDIS(_R12, _R2, PPC_HA(reladdr)));
 287		EMIT(PPC_RAW_ADDI(_R12, _R12, PPC_LO(reladdr)));
 288		EMIT(PPC_RAW_MTCTR(_R12));
 289		EMIT(PPC_RAW_BCTRL());
 290	} else {
 291		if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1)) {
 292			/* func points to the function descriptor */
 293			PPC_LI64(bpf_to_ppc(TMP_REG_2), func);
 294			/* Load actual entry point from function descriptor */
 295			EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 0));
 296			/* ... and move it to CTR */
 297			EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_1)));
 298			/*
 299			 * Load TOC from function descriptor at offset 8.
 300			 * We can clobber r2 since we get called through a
 301			 * function pointer (so caller will save/restore r2).
 302			 */
 303			if (is_module_text_address(func_addr))
 304				EMIT(PPC_RAW_LD(_R2, bpf_to_ppc(TMP_REG_2), 8));
 305		} else {
 306			PPC_LI64(_R12, func);
 307			EMIT(PPC_RAW_MTCTR(_R12));
 308		}
 309		EMIT(PPC_RAW_BCTRL());
 310		/*
 311		 * Load r2 with kernel TOC as kernel TOC is used if function address falls
 312		 * within core kernel text.
 313		 */
 314		if (is_module_text_address(func_addr))
 315			EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc)));
 316	}
 317#endif
 318
 319	return 0;
 320}
 321
 322static int zero_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size)
 323{
 324	switch (size) {
 325	case 1:
 326		 /* zero-extend 8 bits into 64 bits */
 327		EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 56));
 328		return 0;
 329	case 2:
 330		 /* zero-extend 16 bits into 64 bits */
 331		EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 48));
 332		return 0;
 333	case 4:
 334		 /* zero-extend 32 bits into 64 bits */
 335		EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 32));
 336		fallthrough;
 337	case 8:
 338		/* Nothing to do */
 339		return 0;
 340	default:
 341		return -1;
 342	}
 343}
 344
 345static int sign_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size)
 346{
 347	switch (size) {
 348	case 1:
 349		 /* sign-extend 8 bits into 64 bits */
 350		EMIT(PPC_RAW_EXTSB(dst_reg, src_reg));
 351		return 0;
 352	case 2:
 353		 /* sign-extend 16 bits into 64 bits */
 354		EMIT(PPC_RAW_EXTSH(dst_reg, src_reg));
 355		return 0;
 356	case 4:
 357		 /* sign-extend 32 bits into 64 bits */
 358		EMIT(PPC_RAW_EXTSW(dst_reg, src_reg));
 359		fallthrough;
 360	case 8:
 361		/* Nothing to do */
 362		return 0;
 363	default:
 364		return -1;
 365	}
 366}
 367
 368/*
 369 * Handle powerpc ABI expectations from caller:
 370 *   - Unsigned arguments are zero-extended.
 371 *   - Signed arguments are sign-extended.
 372 */
 373static int prepare_for_kfunc_call(const struct bpf_prog *fp, u32 *image,
 374				  struct codegen_context *ctx,
 375				  const struct bpf_insn *insn)
 376{
 377	const struct btf_func_model *m = bpf_jit_find_kfunc_model(fp, insn);
 378	int i;
 379
 380	if (!m)
 381		return -1;
 382
 383	for (i = 0; i < m->nr_args; i++) {
 384		/* Note that BPF ABI only allows up to 5 args for kfuncs */
 385		u32 reg = bpf_to_ppc(BPF_REG_1 + i), size = m->arg_size[i];
 386
 387		if (!(m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG)) {
 388			if (zero_extend(image, ctx, reg, reg, size))
 389				return -1;
 390		} else {
 391			if (sign_extend(image, ctx, reg, reg, size))
 392				return -1;
 393		}
 394	}
 395
 396	return 0;
 397}
 398
 399static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
 400{
 401	/*
 402	 * By now, the eBPF program has already setup parameters in r3, r4 and r5
 403	 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
 404	 * r4/BPF_REG_2 - pointer to bpf_array
 405	 * r5/BPF_REG_3 - index in bpf_array
 406	 */
 407	int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
 408	int b2p_index = bpf_to_ppc(BPF_REG_3);
 409	int bpf_tailcall_prologue_size = 12;
 410
 411	if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL) && IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
 412		bpf_tailcall_prologue_size += 4; /* skip past the toc load */
 413
 414	/*
 415	 * if (index >= array->map.max_entries)
 416	 *   goto out;
 417	 */
 418	EMIT(PPC_RAW_LWZ(bpf_to_ppc(TMP_REG_1), b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
 419	EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31));
 420	EMIT(PPC_RAW_CMPLW(b2p_index, bpf_to_ppc(TMP_REG_1)));
 421	PPC_BCC_SHORT(COND_GE, out);
 422
 423	/*
 424	 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
 425	 *   goto out;
 426	 */
 427	EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx)));
 428	EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT));
 429	PPC_BCC_SHORT(COND_GE, out);
 430
 431	/*
 432	 * tail_call_cnt++;
 433	 * Writeback this updated value only if tailcall succeeds.
 434	 */
 435	EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1));
 436
 437	/* prog = array->ptrs[index]; */
 438	EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_2), b2p_index, 8));
 439	EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), b2p_bpf_array));
 440	EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2),
 441			offsetof(struct bpf_array, ptrs)));
 442
 443	/*
 444	 * if (prog == NULL)
 445	 *   goto out;
 446	 */
 447	EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_2), 0));
 448	PPC_BCC_SHORT(COND_EQ, out);
 449
 450	/* goto *(prog->bpf_func + prologue_size); */
 451	EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2),
 452			offsetof(struct bpf_prog, bpf_func)));
 453	EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2),
 454			  FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size));
 455	EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_2)));
 456
 457	/* Writeback updated tailcall count */
 458	EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx)));
 459
 460	/* tear down stack, restore NVRs, ... */
 461	bpf_jit_emit_common_epilogue(image, ctx);
 462
 463	EMIT(PPC_RAW_BCTR());
 464
 465	/* out: */
 466	return 0;
 467}
 468
 469bool bpf_jit_bypass_spec_v1(void)
 470{
 471#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64)
 472	return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
 473		 security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR));
 474#else
 475	return true;
 476#endif
 477}
 478
 479bool bpf_jit_bypass_spec_v4(void)
 480{
 481	return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
 482		 security_ftr_enabled(SEC_FTR_STF_BARRIER) &&
 483		 stf_barrier_type_get() != STF_BARRIER_NONE);
 484}
 485
 486/*
 487 * We spill into the redzone always, even if the bpf program has its own stackframe.
 488 * Offsets hardcoded based on BPF_PPC_STACK_SAVE -- see bpf_jit_stack_local()
 489 */
 490void bpf_stf_barrier(void);
 491
 492asm (
 493"		.global bpf_stf_barrier		;"
 494"	bpf_stf_barrier:			;"
 495"		std	21,-80(1)		;"
 496"		std	22,-72(1)		;"
 497"		sync				;"
 498"		ld	21,-80(1)		;"
 499"		ld	22,-72(1)		;"
 500"		ori	31,31,0			;"
 501"		.rept 14			;"
 502"		b	1f			;"
 503"	1:					;"
 504"		.endr				;"
 505"		blr				;"
 506);
 507
 508static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx,
 509				   const struct bpf_insn *insn, u32 *jmp_off,
 510				   u32 *tmp_idx, u32 *addrp)
 511{
 512	u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
 513	u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
 514	u32 size = BPF_SIZE(insn->code);
 515	u32 src_reg = bpf_to_ppc(insn->src_reg);
 516	u32 dst_reg = bpf_to_ppc(insn->dst_reg);
 517	s32 imm = insn->imm;
 518
 519	u32 save_reg = tmp2_reg;
 520	u32 ret_reg = src_reg;
 521	u32 fixup_idx;
 522
 523	/* Get offset into TMP_REG_1 */
 524	EMIT(PPC_RAW_LI(tmp1_reg, insn->off));
 525       /*
 526	* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
 527	* before and after the operation.
 528	*
 529	* This is a requirement in the Linux Kernel Memory Model.
 530	* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
 531	*/
 532	if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
 533		EMIT(PPC_RAW_SYNC());
 534
 535	*tmp_idx = ctx->idx;
 536
 537	/* load value from memory into TMP_REG_2 */
 538	if (size == BPF_DW)
 539		EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
 540	else
 541		EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
 542	/* Save old value in _R0 */
 543	if (imm & BPF_FETCH)
 544		EMIT(PPC_RAW_MR(_R0, tmp2_reg));
 545
 546	switch (imm) {
 547	case BPF_ADD:
 548	case BPF_ADD | BPF_FETCH:
 549		EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
 550		break;
 551	case BPF_AND:
 552	case BPF_AND | BPF_FETCH:
 553		EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
 554		break;
 555	case BPF_OR:
 556	case BPF_OR | BPF_FETCH:
 557		EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
 558		break;
 559	case BPF_XOR:
 560	case BPF_XOR | BPF_FETCH:
 561		EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
 562		break;
 563	case BPF_CMPXCHG:
 564	       /*
 565		* Return old value in BPF_REG_0 for BPF_CMPXCHG &
 566		* in src_reg for other cases.
 567		*/
 568		ret_reg = bpf_to_ppc(BPF_REG_0);
 569
 570		/* Compare with old value in BPF_R0 */
 571		if (size == BPF_DW)
 572			EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
 573		else
 574			EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
 575		/* Don't set if different from old value */
 576		PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
 577		fallthrough;
 578	case BPF_XCHG:
 579		save_reg = src_reg;
 580		break;
 581	default:
 582		return -EOPNOTSUPP;
 583	}
 584
 585	/* store new value */
 586	if (size == BPF_DW)
 587		EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
 588	else
 589		EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
 590	/* we're done if this succeeded */
 591	PPC_BCC_SHORT(COND_NE, *tmp_idx * 4);
 592	fixup_idx = ctx->idx;
 593
 594	if (imm & BPF_FETCH) {
 595		/* Emit 'sync' to enforce full ordering */
 596		if (IS_ENABLED(CONFIG_SMP))
 597			EMIT(PPC_RAW_SYNC());
 598		EMIT(PPC_RAW_MR(ret_reg, _R0));
 599		/*
 600		 * Skip unnecessary zero-extension for 32-bit cmpxchg.
 601		 * For context, see commit 39491867ace5.
 602		 */
 603		if (size != BPF_DW && imm == BPF_CMPXCHG &&
 604		    insn_is_zext(insn + 1))
 605			*addrp = ctx->idx * 4;
 606	}
 607
 608	*jmp_off = (fixup_idx - *tmp_idx) * 4;
 609
 610	return 0;
 611}
 612
 613static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off,
 614					u32 code, u32 *image)
 615{
 616	u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
 617	u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
 618
 619	switch (BPF_SIZE(code)) {
 620	case BPF_B:
 621		EMIT(PPC_RAW_STB(src_reg, tmp1_reg, off));
 622		break;
 623	case BPF_H:
 624		EMIT(PPC_RAW_STH(src_reg, tmp1_reg, off));
 625		break;
 626	case BPF_W:
 627		EMIT(PPC_RAW_STW(src_reg, tmp1_reg, off));
 628		break;
 629	case BPF_DW:
 630		if (off % 4) {
 631			EMIT(PPC_RAW_LI(tmp2_reg, off));
 632			EMIT(PPC_RAW_STDX(src_reg, tmp1_reg, tmp2_reg));
 633		} else {
 634			EMIT(PPC_RAW_STD(src_reg, tmp1_reg, off));
 635		}
 636		break;
 637	default:
 638		return -EINVAL;
 639	}
 640	return 0;
 641}
 642
 643static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image)
 644{
 645	u32 code = insn.code;
 646	u32 dst_reg = bpf_to_ppc(insn.dst_reg);
 647	u32 src_reg = bpf_to_ppc(insn.src_reg);
 648	u32 size = BPF_SIZE(code);
 649	u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
 650	u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
 651	s16 off = insn.off;
 652	s32 imm = insn.imm;
 653
 654	switch (imm) {
 655	case BPF_LOAD_ACQ:
 656		switch (size) {
 657		case BPF_B:
 658			EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
 659			break;
 660		case BPF_H:
 661			EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
 662			break;
 663		case BPF_W:
 664			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
 665			break;
 666		case BPF_DW:
 667			if (off % 4) {
 668				EMIT(PPC_RAW_LI(tmp1_reg, off));
 669				EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg));
 670			} else {
 671				EMIT(PPC_RAW_LD(dst_reg, src_reg, off));
 672			}
 673			break;
 674		}
 675		EMIT(PPC_RAW_LWSYNC());
 676		break;
 677	case BPF_STORE_REL:
 678		EMIT(PPC_RAW_LWSYNC());
 679		switch (size) {
 680		case BPF_B:
 681			EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
 682			break;
 683		case BPF_H:
 684			EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
 685			break;
 686		case BPF_W:
 687			EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
 688			break;
 689		case BPF_DW:
 690			if (off % 4) {
 691				EMIT(PPC_RAW_LI(tmp2_reg, off));
 692				EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg));
 693			} else {
 694				EMIT(PPC_RAW_STD(src_reg, dst_reg, off));
 695			}
 696			break;
 697		}
 698		break;
 699	default:
 700		pr_err_ratelimited("unexpected atomic load/store op code %02x\n",
 701				   imm);
 702		return -EINVAL;
 703	}
 704
 705	return 0;
 706}
 707
 708/* Assemble the body code between the prologue & epilogue */
 709int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx,
 710		       u32 *addrs, int pass, bool extra_pass)
 711{
 712	enum stf_barrier_type stf_barrier = stf_barrier_type_get();
 713	bool sync_emitted, ori31_emitted;
 714	const struct bpf_insn *insn = fp->insnsi;
 715	int flen = fp->len;
 716	int i, ret;
 717
 718	/* Start of epilogue code - will only be valid 2nd pass onwards */
 719	u32 exit_addr = addrs[flen];
 720
 721	for (i = 0; i < flen; i++) {
 722		u32 code = insn[i].code;
 723		u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
 724		u32 src_reg = bpf_to_ppc(insn[i].src_reg);
 725		u32 size = BPF_SIZE(code);
 726		u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
 727		u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
 728		s16 off = insn[i].off;
 729		s32 imm = insn[i].imm;
 730		bool func_addr_fixed;
 731		u64 func_addr;
 732		u64 imm64;
 733		u32 true_cond;
 734		u32 tmp_idx;
 735		u32 jmp_off;
 736
 737		/*
 738		 * addrs[] maps a BPF bytecode address into a real offset from
 739		 * the start of the body code.
 740		 */
 741		addrs[i] = ctx->idx * 4;
 742
 743		/*
 744		 * As an optimization, we note down which non-volatile registers
 745		 * are used so that we can only save/restore those in our
 746		 * prologue and epilogue. We do this here regardless of whether
 747		 * the actual BPF instruction uses src/dst registers or not
 748		 * (for instance, BPF_CALL does not use them). The expectation
 749		 * is that those instructions will have src_reg/dst_reg set to
 750		 * 0. Even otherwise, we just lose some prologue/epilogue
 751		 * optimization but everything else should work without
 752		 * any issues.
 753		 */
 754		if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
 755			bpf_set_seen_register(ctx, dst_reg);
 756		if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
 757			bpf_set_seen_register(ctx, src_reg);
 758
 759		switch (code) {
 760		/*
 761		 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
 762		 */
 763		case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
 764		case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
 765			EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
 766			goto bpf_alu32_trunc;
 767		case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
 768		case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
 769			EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
 770			goto bpf_alu32_trunc;
 771		case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
 772		case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
 773			if (!imm) {
 774				goto bpf_alu32_trunc;
 775			} else if (imm >= -32768 && imm < 32768) {
 776				EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
 777			} else {
 778				PPC_LI32(tmp1_reg, imm);
 779				EMIT(PPC_RAW_ADD(dst_reg, dst_reg, tmp1_reg));
 780			}
 781			goto bpf_alu32_trunc;
 782		case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
 783		case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
 784			if (!imm) {
 785				goto bpf_alu32_trunc;
 786			} else if (imm > -32768 && imm <= 32768) {
 787				EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm)));
 788			} else {
 789				PPC_LI32(tmp1_reg, imm);
 790				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
 791			}
 792			goto bpf_alu32_trunc;
 793		case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
 794		case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
 795			if (BPF_CLASS(code) == BPF_ALU)
 796				EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
 797			else
 798				EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg));
 799			goto bpf_alu32_trunc;
 800		case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
 801		case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
 802			if (imm >= -32768 && imm < 32768)
 803				EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
 804			else {
 805				PPC_LI32(tmp1_reg, imm);
 806				if (BPF_CLASS(code) == BPF_ALU)
 807					EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp1_reg));
 808				else
 809					EMIT(PPC_RAW_MULD(dst_reg, dst_reg, tmp1_reg));
 810			}
 811			goto bpf_alu32_trunc;
 812		case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
 813		case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
 814			if (BPF_OP(code) == BPF_MOD) {
 815				if (off)
 816					EMIT(PPC_RAW_DIVW(tmp1_reg, dst_reg, src_reg));
 817				else
 818					EMIT(PPC_RAW_DIVWU(tmp1_reg, dst_reg, src_reg));
 819
 820				EMIT(PPC_RAW_MULW(tmp1_reg, src_reg, tmp1_reg));
 821				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
 822			} else
 823				if (off)
 824					EMIT(PPC_RAW_DIVW(dst_reg, dst_reg, src_reg));
 825				else
 826					EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
 827			goto bpf_alu32_trunc;
 828		case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
 829		case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
 830			if (BPF_OP(code) == BPF_MOD) {
 831				if (off)
 832					EMIT(PPC_RAW_DIVD(tmp1_reg, dst_reg, src_reg));
 833				else
 834					EMIT(PPC_RAW_DIVDU(tmp1_reg, dst_reg, src_reg));
 835				EMIT(PPC_RAW_MULD(tmp1_reg, src_reg, tmp1_reg));
 836				EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
 837			} else
 838				if (off)
 839					EMIT(PPC_RAW_DIVD(dst_reg, dst_reg, src_reg));
 840				else
 841					EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
 842			break;
 843		case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
 844		case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
 845		case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
 846		case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
 847			if (imm == 0)
 848				return -EINVAL;
 849			if (imm == 1) {
 850				if (BPF_OP(code) == BPF_DIV) {
 851					goto bpf_alu32_trunc;
 852				} else {
 853					EMIT(PPC_RAW_LI(dst_reg, 0));
 854					break;
 855				}
 856			}
 857
 858			PPC_LI32(tmp1_reg, imm);
 859			switch (BPF_CLASS(code)) {
 860			case BPF_ALU:
 861				if (BPF_OP(code) == BPF_MOD) {
 862					if (off)
 863						EMIT(PPC_RAW_DIVW(tmp2_reg, dst_reg, tmp1_reg));
 864					else
 865						EMIT(PPC_RAW_DIVWU(tmp2_reg, dst_reg, tmp1_reg));
 866					EMIT(PPC_RAW_MULW(tmp1_reg, tmp1_reg, tmp2_reg));
 867					EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
 868				} else
 869					if (off)
 870						EMIT(PPC_RAW_DIVW(dst_reg, dst_reg, tmp1_reg));
 871					else
 872						EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, tmp1_reg));
 873				break;
 874			case BPF_ALU64:
 875				if (BPF_OP(code) == BPF_MOD) {
 876					if (off)
 877						EMIT(PPC_RAW_DIVD(tmp2_reg, dst_reg, tmp1_reg));
 878					else
 879						EMIT(PPC_RAW_DIVDU(tmp2_reg, dst_reg, tmp1_reg));
 880					EMIT(PPC_RAW_MULD(tmp1_reg, tmp1_reg, tmp2_reg));
 881					EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
 882				} else
 883					if (off)
 884						EMIT(PPC_RAW_DIVD(dst_reg, dst_reg, tmp1_reg));
 885					else
 886						EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, tmp1_reg));
 887				break;
 888			}
 889			goto bpf_alu32_trunc;
 890		case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
 891		case BPF_ALU64 | BPF_NEG: /* dst = -dst */
 892			EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
 893			goto bpf_alu32_trunc;
 894
 895		/*
 896		 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
 897		 */
 898		case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
 899		case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
 900			EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
 901			goto bpf_alu32_trunc;
 902		case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
 903		case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
 904			if (!IMM_H(imm))
 905				EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
 906			else {
 907				/* Sign-extended */
 908				PPC_LI32(tmp1_reg, imm);
 909				EMIT(PPC_RAW_AND(dst_reg, dst_reg, tmp1_reg));
 910			}
 911			goto bpf_alu32_trunc;
 912		case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
 913		case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
 914			EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
 915			goto bpf_alu32_trunc;
 916		case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
 917		case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
 918			if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
 919				/* Sign-extended */
 920				PPC_LI32(tmp1_reg, imm);
 921				EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp1_reg));
 922			} else {
 923				if (IMM_L(imm))
 924					EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
 925				if (IMM_H(imm))
 926					EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
 927			}
 928			goto bpf_alu32_trunc;
 929		case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
 930		case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
 931			EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
 932			goto bpf_alu32_trunc;
 933		case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
 934		case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
 935			if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
 936				/* Sign-extended */
 937				PPC_LI32(tmp1_reg, imm);
 938				EMIT(PPC_RAW_XOR(dst_reg, dst_reg, tmp1_reg));
 939			} else {
 940				if (IMM_L(imm))
 941					EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
 942				if (IMM_H(imm))
 943					EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
 944			}
 945			goto bpf_alu32_trunc;
 946		case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
 947			/* slw clears top 32 bits */
 948			EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
 949			/* skip zero extension move, but set address map. */
 950			if (insn_is_zext(&insn[i + 1]))
 951				addrs[++i] = ctx->idx * 4;
 952			break;
 953		case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
 954			EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg));
 955			break;
 956		case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
 957			/* with imm 0, we still need to clear top 32 bits */
 958			EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
 959			if (insn_is_zext(&insn[i + 1]))
 960				addrs[++i] = ctx->idx * 4;
 961			break;
 962		case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
 963			if (imm != 0)
 964				EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm));
 965			break;
 966		case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
 967			EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
 968			if (insn_is_zext(&insn[i + 1]))
 969				addrs[++i] = ctx->idx * 4;
 970			break;
 971		case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
 972			EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg));
 973			break;
 974		case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
 975			EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
 976			if (insn_is_zext(&insn[i + 1]))
 977				addrs[++i] = ctx->idx * 4;
 978			break;
 979		case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
 980			if (imm != 0)
 981				EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm));
 982			break;
 983		case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
 984			EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg));
 985			goto bpf_alu32_trunc;
 986		case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
 987			EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg));
 988			break;
 989		case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
 990			EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
 991			goto bpf_alu32_trunc;
 992		case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
 993			if (imm != 0)
 994				EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm));
 995			break;
 996
 997		/*
 998		 * MOV
 999		 */
1000		case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
1001		case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
1002
1003			if (insn_is_cast_user(&insn[i])) {
1004				EMIT(PPC_RAW_RLDICL_DOT(tmp1_reg, src_reg, 0, 32));
1005				PPC_LI64(dst_reg, (ctx->user_vm_start & 0xffffffff00000000UL));
1006				PPC_BCC_SHORT(COND_EQ, (ctx->idx + 2) * 4);
1007				EMIT(PPC_RAW_OR(tmp1_reg, dst_reg, tmp1_reg));
1008				EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
1009				break;
1010			}
1011
1012			if (imm == 1) {
1013				/* special mov32 for zext */
1014				EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
1015				break;
1016			}
1017			if (off == 0) {
1018				/* MOV */
1019				if (dst_reg != src_reg)
1020					EMIT(PPC_RAW_MR(dst_reg, src_reg));
1021			} else {
1022				/* MOVSX: dst = (s8,s16,s32)src (off = 8,16,32) */
1023				if (sign_extend(image, ctx, src_reg, dst_reg, off / 8))
1024					return -1;
1025			}
1026			goto bpf_alu32_trunc;
1027		case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
1028		case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
1029			PPC_LI32(dst_reg, imm);
1030			if (imm < 0)
1031				goto bpf_alu32_trunc;
1032			else if (insn_is_zext(&insn[i + 1]))
1033				addrs[++i] = ctx->idx * 4;
1034			break;
1035
1036bpf_alu32_trunc:
1037		/* Truncate to 32-bits */
1038		if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
1039			EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
1040		break;
1041
1042		/*
1043		 * BPF_FROM_BE/LE
1044		 */
1045		case BPF_ALU | BPF_END | BPF_FROM_LE:
1046		case BPF_ALU | BPF_END | BPF_FROM_BE:
1047		case BPF_ALU64 | BPF_END | BPF_FROM_LE:
1048#ifdef __BIG_ENDIAN__
1049			if (BPF_SRC(code) == BPF_FROM_BE)
1050				goto emit_clear;
1051#else /* !__BIG_ENDIAN__ */
1052			if (BPF_CLASS(code) == BPF_ALU && BPF_SRC(code) == BPF_FROM_LE)
1053				goto emit_clear;
1054#endif
1055			switch (imm) {
1056			case 16:
1057				/* Rotate 8 bits left & mask with 0x0000ff00 */
1058				EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 16, 23));
1059				/* Rotate 8 bits right & insert LSB to reg */
1060				EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 24, 31));
1061				/* Move result back to dst_reg */
1062				EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
1063				break;
1064			case 32:
1065				/*
1066				 * Rotate word left by 8 bits:
1067				 * 2 bytes are already in their final position
1068				 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
1069				 */
1070				EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 0, 31));
1071				/* Rotate 24 bits and insert byte 1 */
1072				EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 0, 7));
1073				/* Rotate 24 bits and insert byte 3 */
1074				EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 16, 23));
1075				EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
1076				break;
1077			case 64:
1078				/* Store the value to stack and then use byte-reverse loads */
1079				EMIT(PPC_RAW_STD(dst_reg, _R1, bpf_jit_stack_local(ctx)));
1080				EMIT(PPC_RAW_ADDI(tmp1_reg, _R1, bpf_jit_stack_local(ctx)));
1081				if (cpu_has_feature(CPU_FTR_ARCH_206)) {
1082					EMIT(PPC_RAW_LDBRX(dst_reg, 0, tmp1_reg));
1083				} else {
1084					EMIT(PPC_RAW_LWBRX(dst_reg, 0, tmp1_reg));
1085					if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
1086						EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32));
1087					EMIT(PPC_RAW_LI(tmp2_reg, 4));
1088					EMIT(PPC_RAW_LWBRX(tmp2_reg, tmp2_reg, tmp1_reg));
1089					if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1090						EMIT(PPC_RAW_SLDI(tmp2_reg, tmp2_reg, 32));
1091					EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp2_reg));
1092				}
1093				break;
1094			}
1095			break;
1096
1097emit_clear:
1098			switch (imm) {
1099			case 16:
1100				/* zero-extend 16 bits into 64 bits */
1101				EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48));
1102				if (insn_is_zext(&insn[i + 1]))
1103					addrs[++i] = ctx->idx * 4;
1104				break;
1105			case 32:
1106				if (!fp->aux->verifier_zext)
1107					/* zero-extend 32 bits into 64 bits */
1108					EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32));
1109				break;
1110			case 64:
1111				/* nop */
1112				break;
1113			}
1114			break;
1115
1116		/*
1117		 * BPF_ST NOSPEC (speculation barrier)
1118		 *
1119		 * The following must act as a barrier against both Spectre v1
1120		 * and v4 if we requested both mitigations. Therefore, also emit
1121		 * 'isync; sync' on E500 or 'ori31' on BOOK3S_64 in addition to
1122		 * the insns needed for a Spectre v4 barrier.
1123		 *
1124		 * If we requested only !bypass_spec_v1 OR only !bypass_spec_v4,
1125		 * we can skip the respective other barrier type as an
1126		 * optimization.
1127		 */
1128		case BPF_ST | BPF_NOSPEC:
1129			sync_emitted = false;
1130			ori31_emitted = false;
1131			if (IS_ENABLED(CONFIG_PPC_E500) &&
1132			    !bpf_jit_bypass_spec_v1()) {
1133				EMIT(PPC_RAW_ISYNC());
1134				EMIT(PPC_RAW_SYNC());
1135				sync_emitted = true;
1136			}
1137			if (!bpf_jit_bypass_spec_v4()) {
1138				switch (stf_barrier) {
1139				case STF_BARRIER_EIEIO:
1140					EMIT(PPC_RAW_EIEIO() | 0x02000000);
1141					break;
1142				case STF_BARRIER_SYNC_ORI:
1143					if (!sync_emitted)
1144						EMIT(PPC_RAW_SYNC());
1145					EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0));
1146					EMIT(PPC_RAW_ORI(_R31, _R31, 0));
1147					ori31_emitted = true;
1148					break;
1149				case STF_BARRIER_FALLBACK:
1150					ctx->seen |= SEEN_FUNC;
1151					PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier));
1152					EMIT(PPC_RAW_MTCTR(_R12));
1153					EMIT(PPC_RAW_BCTRL());
1154					break;
1155				case STF_BARRIER_NONE:
1156					break;
1157				}
1158			}
1159			if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
1160			    !bpf_jit_bypass_spec_v1() &&
1161			    !ori31_emitted)
1162				EMIT(PPC_RAW_ORI(_R31, _R31, 0));
1163			break;
1164
1165		/*
1166		 * BPF_ST(X)
1167		 */
1168		case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
1169		case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
1170			if (BPF_CLASS(code) == BPF_ST) {
1171				EMIT(PPC_RAW_LI(tmp1_reg, imm));
1172				src_reg = tmp1_reg;
1173			}
1174			EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
1175			break;
1176		case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
1177		case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
1178			if (BPF_CLASS(code) == BPF_ST) {
1179				EMIT(PPC_RAW_LI(tmp1_reg, imm));
1180				src_reg = tmp1_reg;
1181			}
1182			EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
1183			break;
1184		case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
1185		case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
1186			if (BPF_CLASS(code) == BPF_ST) {
1187				PPC_LI32(tmp1_reg, imm);
1188				src_reg = tmp1_reg;
1189			}
1190			EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
1191			break;
1192		case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
1193		case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
1194			if (BPF_CLASS(code) == BPF_ST) {
1195				PPC_LI32(tmp1_reg, imm);
1196				src_reg = tmp1_reg;
1197			}
1198			if (off % 4) {
1199				EMIT(PPC_RAW_LI(tmp2_reg, off));
1200				EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg));
1201			} else {
1202				EMIT(PPC_RAW_STD(src_reg, dst_reg, off));
1203			}
1204			break;
1205
1206		case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
1207		case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
1208		case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
1209		case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
1210
1211			EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START)));
1212
1213			ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image);
1214			if (ret)
1215				return ret;
1216
1217			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,
1218						    ctx->idx - 1, 4, -1, code);
1219			if (ret)
1220				return ret;
1221
1222			break;
1223
1224		case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
1225		case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
1226		case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
1227		case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
1228
1229			EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START)));
1230
1231			if (BPF_SIZE(code) == BPF_W || BPF_SIZE(code) == BPF_DW) {
1232				PPC_LI32(tmp2_reg, imm);
1233				src_reg = tmp2_reg;
1234			} else {
1235				EMIT(PPC_RAW_LI(tmp2_reg, imm));
1236				src_reg = tmp2_reg;
1237			}
1238
1239			ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image);
1240			if (ret)
1241				return ret;
1242
1243			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,
1244						    ctx->idx - 1, 4, -1, code);
1245			if (ret)
1246				return ret;
1247
1248			break;
1249
1250		/*
1251		 * BPF_STX PROBE_ATOMIC (arena atomic ops)
1252		 */
1253		case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
1254		case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
1255			EMIT(PPC_RAW_ADD(dst_reg, dst_reg, bpf_to_ppc(ARENA_VM_START)));
1256			ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],
1257						      &jmp_off, &tmp_idx, &addrs[i + 1]);
1258			if (ret) {
1259				if (ret == -EOPNOTSUPP) {
1260					pr_err_ratelimited(
1261						"eBPF filter atomic op code %02x (@%d) unsupported\n",
1262						code, i);
1263				}
1264				return ret;
1265			}
1266			/* LDARX/LWARX should land here on exception. */
1267			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,
1268						    tmp_idx, jmp_off, dst_reg, code);
1269			if (ret)
1270				return ret;
1271
1272			/* Retrieve the dst_reg */
1273			EMIT(PPC_RAW_SUB(dst_reg, dst_reg, bpf_to_ppc(ARENA_VM_START)));
1274			break;
1275
1276		/*
1277		 * BPF_STX ATOMIC (atomic ops)
1278		 */
1279		case BPF_STX | BPF_ATOMIC | BPF_B:
1280		case BPF_STX | BPF_ATOMIC | BPF_H:
1281		case BPF_STX | BPF_ATOMIC | BPF_W:
1282		case BPF_STX | BPF_ATOMIC | BPF_DW:
1283			if (bpf_atomic_is_load_store(&insn[i])) {
1284				ret = emit_atomic_ld_st(insn[i], ctx, image);
1285				if (ret)
1286					return ret;
1287
1288				if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
1289					addrs[++i] = ctx->idx * 4;
1290				break;
1291			} else if (size == BPF_B || size == BPF_H) {
1292				pr_err_ratelimited(
1293					"eBPF filter atomic op code %02x (@%d) unsupported\n",
1294					code, i);
1295				return -EOPNOTSUPP;
1296			}
1297
1298			ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],
1299						      &jmp_off, &tmp_idx, &addrs[i + 1]);
1300			if (ret) {
1301				if (ret == -EOPNOTSUPP) {
1302					pr_err_ratelimited(
1303						"eBPF filter atomic op code %02x (@%d) unsupported\n",
1304						code, i);
1305				}
1306				return ret;
1307			}
1308			break;
1309
1310		/*
1311		 * BPF_LDX
1312		 */
1313		/* dst = *(u8 *)(ul) (src + off) */
1314		case BPF_LDX | BPF_MEM | BPF_B:
1315		case BPF_LDX | BPF_MEMSX | BPF_B:
1316		case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1317		case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1318		/* dst = *(u16 *)(ul) (src + off) */
1319		case BPF_LDX | BPF_MEM | BPF_H:
1320		case BPF_LDX | BPF_MEMSX | BPF_H:
1321		case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1322		case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1323		/* dst = *(u32 *)(ul) (src + off) */
1324		case BPF_LDX | BPF_MEM | BPF_W:
1325		case BPF_LDX | BPF_MEMSX | BPF_W:
1326		case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1327		case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1328		/* dst = *(u64 *)(ul) (src + off) */
1329		case BPF_LDX | BPF_MEM | BPF_DW:
1330		case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1331			/*
1332			 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
1333			 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
1334			 * load only if addr is kernel address (see is_kernel_addr()), otherwise
1335			 * set dst_reg=0 and move on.
1336			 */
1337			if (BPF_MODE(code) == BPF_PROBE_MEM || BPF_MODE(code) == BPF_PROBE_MEMSX) {
1338				EMIT(PPC_RAW_ADDI(tmp1_reg, src_reg, off));
1339				if (IS_ENABLED(CONFIG_PPC_BOOK3E_64))
1340					PPC_LI64(tmp2_reg, 0x8000000000000000ul);
1341				else /* BOOK3S_64 */
1342					PPC_LI64(tmp2_reg, PAGE_OFFSET);
1343				EMIT(PPC_RAW_CMPLD(tmp1_reg, tmp2_reg));
1344				PPC_BCC_SHORT(COND_GT, (ctx->idx + 3) * 4);
1345				EMIT(PPC_RAW_LI(dst_reg, 0));
1346				/*
1347				 * Check if 'off' is word aligned for BPF_DW, because
1348				 * we might generate two instructions.
1349				 */
1350				if ((BPF_SIZE(code) == BPF_DW && (off & 3)) ||
1351				    (BPF_SIZE(code) == BPF_B &&
1352				     BPF_MODE(code) == BPF_PROBE_MEMSX) ||
1353				    (BPF_SIZE(code) == BPF_B && BPF_MODE(code) == BPF_MEMSX))
1354					PPC_JMP((ctx->idx + 3) * 4);
1355				else
1356					PPC_JMP((ctx->idx + 2) * 4);
1357			}
1358
1359			if (BPF_MODE(code) == BPF_MEMSX || BPF_MODE(code) == BPF_PROBE_MEMSX) {
1360				switch (size) {
1361				case BPF_B:
1362					EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
1363					EMIT(PPC_RAW_EXTSB(dst_reg, dst_reg));
1364					break;
1365				case BPF_H:
1366					EMIT(PPC_RAW_LHA(dst_reg, src_reg, off));
1367					break;
1368				case BPF_W:
1369					EMIT(PPC_RAW_LWA(dst_reg, src_reg, off));
1370					break;
1371				}
1372			} else {
1373				switch (size) {
1374				case BPF_B:
1375					EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
1376					break;
1377				case BPF_H:
1378					EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
1379					break;
1380				case BPF_W:
1381					EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
1382					break;
1383				case BPF_DW:
1384					if (off % 4) {
1385						EMIT(PPC_RAW_LI(tmp1_reg, off));
1386						EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg));
1387					} else {
1388						EMIT(PPC_RAW_LD(dst_reg, src_reg, off));
1389					}
1390					break;
1391				}
1392			}
1393
1394			if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
1395				addrs[++i] = ctx->idx * 4;
1396
1397			if (BPF_MODE(code) == BPF_PROBE_MEM) {
1398				ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,
1399							    ctx->idx - 1, 4, dst_reg, code);
1400				if (ret)
1401					return ret;
1402			}
1403			break;
1404
1405		/* dst = *(u64 *)(ul) (src + ARENA_VM_START + off) */
1406		case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
1407		case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
1408		case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
1409		case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
1410
1411			EMIT(PPC_RAW_ADD(tmp1_reg, src_reg, bpf_to_ppc(ARENA_VM_START)));
1412
1413			switch (size) {
1414			case BPF_B:
1415				EMIT(PPC_RAW_LBZ(dst_reg, tmp1_reg, off));
1416				break;
1417			case BPF_H:
1418				EMIT(PPC_RAW_LHZ(dst_reg, tmp1_reg, off));
1419				break;
1420			case BPF_W:
1421				EMIT(PPC_RAW_LWZ(dst_reg, tmp1_reg, off));
1422				break;
1423			case BPF_DW:
1424				if (off % 4) {
1425					EMIT(PPC_RAW_LI(tmp2_reg, off));
1426					EMIT(PPC_RAW_LDX(dst_reg, tmp1_reg, tmp2_reg));
1427				} else {
1428					EMIT(PPC_RAW_LD(dst_reg, tmp1_reg, off));
1429				}
1430				break;
1431			}
1432
1433			if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
1434				addrs[++i] = ctx->idx * 4;
1435
1436			ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx,
1437						    ctx->idx - 1, 4, dst_reg, code);
1438			if (ret)
1439				return ret;
1440			break;
1441
1442		/*
1443		 * Doubleword load
1444		 * 16 byte instruction that uses two 'struct bpf_insn'
1445		 */
1446		case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
1447			imm64 = ((u64)(u32) insn[i].imm) |
1448				    (((u64)(u32) insn[i+1].imm) << 32);
1449			PPC_LI64(dst_reg, imm64);
1450			/* Adjust for two bpf instructions */
1451			addrs[++i] = ctx->idx * 4;
1452			break;
1453
1454		/*
1455		 * Return/Exit
1456		 */
1457		case BPF_JMP | BPF_EXIT:
1458			/*
1459			 * If this isn't the very last instruction, branch to
1460			 * the epilogue. If we _are_ the last instruction,
1461			 * we'll just fall through to the epilogue.
1462			 */
1463			if (i != flen - 1) {
1464				ret = bpf_jit_emit_exit_insn(image, ctx, tmp1_reg, exit_addr);
1465				if (ret)
1466					return ret;
1467			}
1468			/* else fall through to the epilogue */
1469			break;
1470
1471		/*
1472		 * Call kernel helper or bpf function
1473		 */
1474		case BPF_JMP | BPF_CALL:
1475			ctx->seen |= SEEN_FUNC;
1476
1477			ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
1478						    &func_addr, &func_addr_fixed);
1479			if (ret < 0)
1480				return ret;
1481
1482			/* Take care of powerpc ABI requirements before kfunc call */
1483			if (insn[i].src_reg == BPF_PSEUDO_KFUNC_CALL) {
1484				if (prepare_for_kfunc_call(fp, image, ctx, &insn[i]))
1485					return -1;
1486			}
1487
1488			ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr);
1489			if (ret)
1490				return ret;
1491
1492			/* move return value from r3 to BPF_REG_0 */
1493			EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R3));
1494			break;
1495
1496		/*
1497		 * Jumps and branches
1498		 */
1499		case BPF_JMP | BPF_JA:
1500			PPC_JMP(addrs[i + 1 + off]);
1501			break;
1502		case BPF_JMP32 | BPF_JA:
1503			PPC_JMP(addrs[i + 1 + imm]);
1504			break;
1505
1506		case BPF_JMP | BPF_JGT | BPF_K:
1507		case BPF_JMP | BPF_JGT | BPF_X:
1508		case BPF_JMP | BPF_JSGT | BPF_K:
1509		case BPF_JMP | BPF_JSGT | BPF_X:
1510		case BPF_JMP32 | BPF_JGT | BPF_K:
1511		case BPF_JMP32 | BPF_JGT | BPF_X:
1512		case BPF_JMP32 | BPF_JSGT | BPF_K:
1513		case BPF_JMP32 | BPF_JSGT | BPF_X:
1514			true_cond = COND_GT;
1515			goto cond_branch;
1516		case BPF_JMP | BPF_JLT | BPF_K:
1517		case BPF_JMP | BPF_JLT | BPF_X:
1518		case BPF_JMP | BPF_JSLT | BPF_K:
1519		case BPF_JMP | BPF_JSLT | BPF_X:
1520		case BPF_JMP32 | BPF_JLT | BPF_K:
1521		case BPF_JMP32 | BPF_JLT | BPF_X:
1522		case BPF_JMP32 | BPF_JSLT | BPF_K:
1523		case BPF_JMP32 | BPF_JSLT | BPF_X:
1524			true_cond = COND_LT;
1525			goto cond_branch;
1526		case BPF_JMP | BPF_JGE | BPF_K:
1527		case BPF_JMP | BPF_JGE | BPF_X:
1528		case BPF_JMP | BPF_JSGE | BPF_K:
1529		case BPF_JMP | BPF_JSGE | BPF_X:
1530		case BPF_JMP32 | BPF_JGE | BPF_K:
1531		case BPF_JMP32 | BPF_JGE | BPF_X:
1532		case BPF_JMP32 | BPF_JSGE | BPF_K:
1533		case BPF_JMP32 | BPF_JSGE | BPF_X:
1534			true_cond = COND_GE;
1535			goto cond_branch;
1536		case BPF_JMP | BPF_JLE | BPF_K:
1537		case BPF_JMP | BPF_JLE | BPF_X:
1538		case BPF_JMP | BPF_JSLE | BPF_K:
1539		case BPF_JMP | BPF_JSLE | BPF_X:
1540		case BPF_JMP32 | BPF_JLE | BPF_K:
1541		case BPF_JMP32 | BPF_JLE | BPF_X:
1542		case BPF_JMP32 | BPF_JSLE | BPF_K:
1543		case BPF_JMP32 | BPF_JSLE | BPF_X:
1544			true_cond = COND_LE;
1545			goto cond_branch;
1546		case BPF_JMP | BPF_JEQ | BPF_K:
1547		case BPF_JMP | BPF_JEQ | BPF_X:
1548		case BPF_JMP32 | BPF_JEQ | BPF_K:
1549		case BPF_JMP32 | BPF_JEQ | BPF_X:
1550			true_cond = COND_EQ;
1551			goto cond_branch;
1552		case BPF_JMP | BPF_JNE | BPF_K:
1553		case BPF_JMP | BPF_JNE | BPF_X:
1554		case BPF_JMP32 | BPF_JNE | BPF_K:
1555		case BPF_JMP32 | BPF_JNE | BPF_X:
1556			true_cond = COND_NE;
1557			goto cond_branch;
1558		case BPF_JMP | BPF_JSET | BPF_K:
1559		case BPF_JMP | BPF_JSET | BPF_X:
1560		case BPF_JMP32 | BPF_JSET | BPF_K:
1561		case BPF_JMP32 | BPF_JSET | BPF_X:
1562			true_cond = COND_NE;
1563			/* Fall through */
1564
1565cond_branch:
1566			switch (code) {
1567			case BPF_JMP | BPF_JGT | BPF_X:
1568			case BPF_JMP | BPF_JLT | BPF_X:
1569			case BPF_JMP | BPF_JGE | BPF_X:
1570			case BPF_JMP | BPF_JLE | BPF_X:
1571			case BPF_JMP | BPF_JEQ | BPF_X:
1572			case BPF_JMP | BPF_JNE | BPF_X:
1573			case BPF_JMP32 | BPF_JGT | BPF_X:
1574			case BPF_JMP32 | BPF_JLT | BPF_X:
1575			case BPF_JMP32 | BPF_JGE | BPF_X:
1576			case BPF_JMP32 | BPF_JLE | BPF_X:
1577			case BPF_JMP32 | BPF_JEQ | BPF_X:
1578			case BPF_JMP32 | BPF_JNE | BPF_X:
1579				/* unsigned comparison */
1580				if (BPF_CLASS(code) == BPF_JMP32)
1581					EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1582				else
1583					EMIT(PPC_RAW_CMPLD(dst_reg, src_reg));
1584				break;
1585			case BPF_JMP | BPF_JSGT | BPF_X:
1586			case BPF_JMP | BPF_JSLT | BPF_X:
1587			case BPF_JMP | BPF_JSGE | BPF_X:
1588			case BPF_JMP | BPF_JSLE | BPF_X:
1589			case BPF_JMP32 | BPF_JSGT | BPF_X:
1590			case BPF_JMP32 | BPF_JSLT | BPF_X:
1591			case BPF_JMP32 | BPF_JSGE | BPF_X:
1592			case BPF_JMP32 | BPF_JSLE | BPF_X:
1593				/* signed comparison */
1594				if (BPF_CLASS(code) == BPF_JMP32)
1595					EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1596				else
1597					EMIT(PPC_RAW_CMPD(dst_reg, src_reg));
1598				break;
1599			case BPF_JMP | BPF_JSET | BPF_X:
1600			case BPF_JMP32 | BPF_JSET | BPF_X:
1601				if (BPF_CLASS(code) == BPF_JMP) {
1602					EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg, src_reg));
1603				} else {
1604					EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, src_reg));
1605					EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg, 0, 0, 31));
1606				}
1607				break;
1608			case BPF_JMP | BPF_JNE | BPF_K:
1609			case BPF_JMP | BPF_JEQ | BPF_K:
1610			case BPF_JMP | BPF_JGT | BPF_K:
1611			case BPF_JMP | BPF_JLT | BPF_K:
1612			case BPF_JMP | BPF_JGE | BPF_K:
1613			case BPF_JMP | BPF_JLE | BPF_K:
1614			case BPF_JMP32 | BPF_JNE | BPF_K:
1615			case BPF_JMP32 | BPF_JEQ | BPF_K:
1616			case BPF_JMP32 | BPF_JGT | BPF_K:
1617			case BPF_JMP32 | BPF_JLT | BPF_K:
1618			case BPF_JMP32 | BPF_JGE | BPF_K:
1619			case BPF_JMP32 | BPF_JLE | BPF_K:
1620			{
1621				bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
1622
1623				/*
1624				 * Need sign-extended load, so only positive
1625				 * values can be used as imm in cmpldi
1626				 */
1627				if (imm >= 0 && imm < 32768) {
1628					if (is_jmp32)
1629						EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1630					else
1631						EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
1632				} else {
1633					/* sign-extending load */
1634					PPC_LI32(tmp1_reg, imm);
1635					/* ... but unsigned comparison */
1636					if (is_jmp32)
1637						EMIT(PPC_RAW_CMPLW(dst_reg, tmp1_reg));
1638					else
1639						EMIT(PPC_RAW_CMPLD(dst_reg, tmp1_reg));
1640				}
1641				break;
1642			}
1643			case BPF_JMP | BPF_JSGT | BPF_K:
1644			case BPF_JMP | BPF_JSLT | BPF_K:
1645			case BPF_JMP | BPF_JSGE | BPF_K:
1646			case BPF_JMP | BPF_JSLE | BPF_K:
1647			case BPF_JMP32 | BPF_JSGT | BPF_K:
1648			case BPF_JMP32 | BPF_JSLT | BPF_K:
1649			case BPF_JMP32 | BPF_JSGE | BPF_K:
1650			case BPF_JMP32 | BPF_JSLE | BPF_K:
1651			{
1652				bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32;
1653
1654				/*
1655				 * signed comparison, so any 16-bit value
1656				 * can be used in cmpdi
1657				 */
1658				if (imm >= -32768 && imm < 32768) {
1659					if (is_jmp32)
1660						EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1661					else
1662						EMIT(PPC_RAW_CMPDI(dst_reg, imm));
1663				} else {
1664					PPC_LI32(tmp1_reg, imm);
1665					if (is_jmp32)
1666						EMIT(PPC_RAW_CMPW(dst_reg, tmp1_reg));
1667					else
1668						EMIT(PPC_RAW_CMPD(dst_reg, tmp1_reg));
1669				}
1670				break;
1671			}
1672			case BPF_JMP | BPF_JSET | BPF_K:
1673			case BPF_JMP32 | BPF_JSET | BPF_K:
1674				/* andi does not sign-extend the immediate */
1675				if (imm >= 0 && imm < 32768)
1676					/* PPC_ANDI is _only/always_ dot-form */
1677					EMIT(PPC_RAW_ANDI(tmp1_reg, dst_reg, imm));
1678				else {
1679					PPC_LI32(tmp1_reg, imm);
1680					if (BPF_CLASS(code) == BPF_JMP) {
1681						EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg,
1682								     tmp1_reg));
1683					} else {
1684						EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, tmp1_reg));
1685						EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg,
1686									0, 0, 31));
1687					}
1688				}
1689				break;
1690			}
1691			PPC_BCC(true_cond, addrs[i + 1 + off]);
1692			break;
1693
1694		/*
1695		 * Tail call
1696		 */
1697		case BPF_JMP | BPF_TAIL_CALL:
1698			ctx->seen |= SEEN_TAILCALL;
1699			ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1700			if (ret < 0)
1701				return ret;
1702			break;
1703
1704		default:
1705			/*
1706			 * The filter contains something cruel & unusual.
1707			 * We don't handle it, but also there shouldn't be
1708			 * anything missing from our list.
1709			 */
1710			pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
1711					code, i);
1712			return -ENOTSUPP;
1713		}
1714	}
1715
1716	/* Set end-of-body-code address for exit. */
1717	addrs[i] = ctx->idx * 4;
1718
1719	return 0;
1720}