bpf_jit_comp.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * eBPF JIT compiler
   4 *
   5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
   6 *		  IBM Corporation
   7 *
   8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
   9 */
  10#include <linux/moduleloader.h>
  11#include <asm/cacheflush.h>
  12#include <asm/asm-compat.h>
  13#include <linux/netdevice.h>
  14#include <linux/filter.h>
  15#include <linux/if_vlan.h>
  16#include <linux/kernel.h>
  17#include <linux/memory.h>
  18#include <linux/bpf.h>
  19
  20#include <asm/kprobes.h>
  21#include <asm/text-patching.h>
  22
  23#include "bpf_jit.h"
  24
  25/* These offsets are from bpf prog end and stay the same across progs */
  26static int bpf_jit_ool_stub, bpf_jit_long_branch_stub;
  27
  28static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
  29{
  30	memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
  31}
  32
  33void dummy_tramp(void);
  34
  35asm (
  36"	.pushsection .text, \"ax\", @progbits	;"
  37"	.global dummy_tramp			;"
  38"	.type dummy_tramp, @function		;"
  39"dummy_tramp:					;"
  40#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
  41"	blr					;"
  42#else
  43/* LR is always in r11, so we don't need a 'mflr r11' here */
  44"	mtctr	11				;"
  45"	mtlr	0				;"
  46"	bctr					;"
  47#endif
  48"	.size dummy_tramp, .-dummy_tramp	;"
  49"	.popsection				;"
  50);
  51
  52void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)
  53{
  54	int ool_stub_idx, long_branch_stub_idx;
  55
  56	/*
  57	 * Out-of-line stub:
  58	 *	mflr	r0
  59	 *	[b|bl]	tramp
  60	 *	mtlr	r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE
  61	 *	b	bpf_func + 4
  62	 */
  63	ool_stub_idx = ctx->idx;
  64	EMIT(PPC_RAW_MFLR(_R0));
  65	EMIT(PPC_RAW_NOP());
  66	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
  67		EMIT(PPC_RAW_MTLR(_R0));
  68	WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4));
  69	EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4));
  70
  71	/*
  72	 * Long branch stub:
  73	 *	.long	<dummy_tramp_addr>
  74	 *	mflr	r11
  75	 *	bcl	20,31,$+4
  76	 *	mflr	r12
  77	 *	ld	r12, -8-SZL(r12)
  78	 *	mtctr	r12
  79	 *	mtlr	r11 // needed to retain ftrace ABI
  80	 *	bctr
  81	 */
  82	if (image)
  83		*((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp;
  84	ctx->idx += SZL / 4;
  85	long_branch_stub_idx = ctx->idx;
  86	EMIT(PPC_RAW_MFLR(_R11));
  87	EMIT(PPC_RAW_BCL4());
  88	EMIT(PPC_RAW_MFLR(_R12));
  89	EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL));
  90	EMIT(PPC_RAW_MTCTR(_R12));
  91	EMIT(PPC_RAW_MTLR(_R11));
  92	EMIT(PPC_RAW_BCTR());
  93
  94	if (!bpf_jit_ool_stub) {
  95		bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4;
  96		bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4;
  97	}
  98}
  99
 100int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
 101{
 102	if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
 103		PPC_JMP(exit_addr);
 104	} else if (ctx->alt_exit_addr) {
 105		if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
 106			return -1;
 107		PPC_JMP(ctx->alt_exit_addr);
 108	} else {
 109		ctx->alt_exit_addr = ctx->idx * 4;
 110		bpf_jit_build_epilogue(image, ctx);
 111	}
 112
 113	return 0;
 114}
 115
 116struct powerpc_jit_data {
 117	/* address of rw header */
 118	struct bpf_binary_header *hdr;
 119	/* address of ro final header */
 120	struct bpf_binary_header *fhdr;
 121	u32 *addrs;
 122	u8 *fimage;
 123	u32 proglen;
 124	struct codegen_context ctx;
 125};
 126
 127bool bpf_jit_needs_zext(void)
 128{
 129	return true;
 130}
 131
 132struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 133{
 134	u32 proglen;
 135	u32 alloclen;
 136	u8 *image = NULL;
 137	u32 *code_base;
 138	u32 *addrs;
 139	struct powerpc_jit_data *jit_data;
 140	struct codegen_context cgctx;
 141	int pass;
 142	int flen;
 143	struct bpf_binary_header *fhdr = NULL;
 144	struct bpf_binary_header *hdr = NULL;
 145	struct bpf_prog *org_fp = fp;
 146	struct bpf_prog *tmp_fp;
 147	bool bpf_blinded = false;
 148	bool extra_pass = false;
 149	u8 *fimage = NULL;
 150	u32 *fcode_base;
 151	u32 extable_len;
 152	u32 fixup_len;
 153
 154	if (!fp->jit_requested)
 155		return org_fp;
 156
 157	tmp_fp = bpf_jit_blind_constants(org_fp);
 158	if (IS_ERR(tmp_fp))
 159		return org_fp;
 160
 161	if (tmp_fp != org_fp) {
 162		bpf_blinded = true;
 163		fp = tmp_fp;
 164	}
 165
 166	jit_data = fp->aux->jit_data;
 167	if (!jit_data) {
 168		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
 169		if (!jit_data) {
 170			fp = org_fp;
 171			goto out;
 172		}
 173		fp->aux->jit_data = jit_data;
 174	}
 175
 176	flen = fp->len;
 177	addrs = jit_data->addrs;
 178	if (addrs) {
 179		cgctx = jit_data->ctx;
 180		/*
 181		 * JIT compiled to a writable location (image/code_base) first.
 182		 * It is then moved to the readonly final location (fimage/fcode_base)
 183		 * using instruction patching.
 184		 */
 185		fimage = jit_data->fimage;
 186		fhdr = jit_data->fhdr;
 187		proglen = jit_data->proglen;
 188		hdr = jit_data->hdr;
 189		image = (void *)hdr + ((void *)fimage - (void *)fhdr);
 190		extra_pass = true;
 191		/* During extra pass, ensure index is reset before repopulating extable entries */
 192		cgctx.exentry_idx = 0;
 193		goto skip_init_ctx;
 194	}
 195
 196	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
 197	if (addrs == NULL) {
 198		fp = org_fp;
 199		goto out_addrs;
 200	}
 201
 202	memset(&cgctx, 0, sizeof(struct codegen_context));
 203	bpf_jit_init_reg_mapping(&cgctx);
 204
 205	/* Make sure that the stack is quadword aligned. */
 206	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
 207	cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
 208	cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
 209
 210	/* Scouting faux-generate pass 0 */
 211	if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
 212		/* We hit something illegal or unsupported. */
 213		fp = org_fp;
 214		goto out_addrs;
 215	}
 216
 217	/*
 218	 * If we have seen a tail call, we need a second pass.
 219	 * This is because bpf_jit_emit_common_epilogue() is called
 220	 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
 221	 * We also need a second pass if we ended up with too large
 222	 * a program so as to ensure BPF_EXIT branches are in range.
 223	 */
 224	if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
 225		cgctx.idx = 0;
 226		if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
 227			fp = org_fp;
 228			goto out_addrs;
 229		}
 230	}
 231
 232	bpf_jit_realloc_regs(&cgctx);
 233	/*
 234	 * Pretend to build prologue, given the features we've seen.  This will
 235	 * update ctgtx.idx as it pretends to output instructions, then we can
 236	 * calculate total size from idx.
 237	 */
 238	bpf_jit_build_prologue(NULL, &cgctx);
 239	addrs[fp->len] = cgctx.idx * 4;
 240	bpf_jit_build_epilogue(NULL, &cgctx);
 241
 242	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
 243	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
 244
 245	proglen = cgctx.idx * 4;
 246	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
 247
 248	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
 249					      bpf_jit_fill_ill_insns);
 250	if (!fhdr) {
 251		fp = org_fp;
 252		goto out_addrs;
 253	}
 254
 255	if (extable_len)
 256		fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
 257
 258skip_init_ctx:
 259	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
 260	fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
 261
 262	/* Code generation passes 1-2 */
 263	for (pass = 1; pass < 3; pass++) {
 264		/* Now build the prologue, body code & epilogue for real. */
 265		cgctx.idx = 0;
 266		cgctx.alt_exit_addr = 0;
 267		bpf_jit_build_prologue(code_base, &cgctx);
 268		if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
 269				       extra_pass)) {
 270			bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
 271			bpf_jit_binary_pack_free(fhdr, hdr);
 272			fp = org_fp;
 273			goto out_addrs;
 274		}
 275		bpf_jit_build_epilogue(code_base, &cgctx);
 276
 277		if (bpf_jit_enable > 1)
 278			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
 279				proglen - (cgctx.idx * 4), cgctx.seen);
 280	}
 281
 282	if (bpf_jit_enable > 1)
 283		/*
 284		 * Note that we output the base address of the code_base
 285		 * rather than image, since opcodes are in code_base.
 286		 */
 287		bpf_jit_dump(flen, proglen, pass, code_base);
 288
 289#ifdef CONFIG_PPC64_ELF_ABI_V1
 290	/* Function descriptor nastiness: Address + TOC */
 291	((u64 *)image)[0] = (u64)fcode_base;
 292	((u64 *)image)[1] = local_paca->kernel_toc;
 293#endif
 294
 295	fp->bpf_func = (void *)fimage;
 296	fp->jited = 1;
 297	fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;
 298
 299	if (!fp->is_func || extra_pass) {
 300		if (bpf_jit_binary_pack_finalize(fhdr, hdr)) {
 301			fp = org_fp;
 302			goto out_addrs;
 303		}
 304		bpf_prog_fill_jited_linfo(fp, addrs);
 305out_addrs:
 306		kfree(addrs);
 307		kfree(jit_data);
 308		fp->aux->jit_data = NULL;
 309	} else {
 310		jit_data->addrs = addrs;
 311		jit_data->ctx = cgctx;
 312		jit_data->proglen = proglen;
 313		jit_data->fimage = fimage;
 314		jit_data->fhdr = fhdr;
 315		jit_data->hdr = hdr;
 316	}
 317
 318out:
 319	if (bpf_blinded)
 320		bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
 321
 322	return fp;
 323}
 324
 325/*
 326 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
 327 * this function, as this only applies to BPF_PROBE_MEM, for now.
 328 */
 329int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
 330			  struct codegen_context *ctx, int insn_idx, int jmp_off,
 331			  int dst_reg, u32 code)
 332{
 333	off_t offset;
 334	unsigned long pc;
 335	struct exception_table_entry *ex, *ex_entry;
 336	u32 *fixup;
 337
 338	/* Populate extable entries only in the last pass */
 339	if (pass != 2)
 340		return 0;
 341
 342	if (!fp->aux->extable ||
 343	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
 344		return -EINVAL;
 345
 346	/*
 347	 * Program is first written to image before copying to the
 348	 * final location (fimage). Accordingly, update in the image first.
 349	 * As all offsets used are relative, copying as is to the
 350	 * final location should be alright.
 351	 */
 352	pc = (unsigned long)&image[insn_idx];
 353	ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
 354
 355	fixup = (void *)ex -
 356		(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
 357		(ctx->exentry_idx * BPF_FIXUP_LEN * 4);
 358
 359	fixup[0] = PPC_RAW_LI(dst_reg, 0);
 360	if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX)
 361		fixup[0] = PPC_RAW_NOP();
 362
 363	if (IS_ENABLED(CONFIG_PPC32))
 364		fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
 365
 366	fixup[BPF_FIXUP_LEN - 1] =
 367		PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
 368
 369	ex_entry = &ex[ctx->exentry_idx];
 370
 371	offset = pc - (long)&ex_entry->insn;
 372	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
 373		return -ERANGE;
 374	ex_entry->insn = offset;
 375
 376	offset = (long)fixup - (long)&ex_entry->fixup;
 377	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
 378		return -ERANGE;
 379	ex_entry->fixup = offset;
 380
 381	ctx->exentry_idx++;
 382	return 0;
 383}
 384
 385void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 386{
 387	int err;
 388
 389	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
 390		return ERR_PTR(-EINVAL);
 391
 392	mutex_lock(&text_mutex);
 393	err = patch_instructions(dst, src, len, false);
 394	mutex_unlock(&text_mutex);
 395
 396	return err ? ERR_PTR(err) : dst;
 397}
 398
 399int bpf_arch_text_invalidate(void *dst, size_t len)
 400{
 401	u32 insn = BREAKPOINT_INSTRUCTION;
 402	int ret;
 403
 404	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
 405		return -EINVAL;
 406
 407	mutex_lock(&text_mutex);
 408	ret = patch_instructions(dst, &insn, len, true);
 409	mutex_unlock(&text_mutex);
 410
 411	return ret;
 412}
 413
 414void bpf_jit_free(struct bpf_prog *fp)
 415{
 416	if (fp->jited) {
 417		struct powerpc_jit_data *jit_data = fp->aux->jit_data;
 418		struct bpf_binary_header *hdr;
 419
 420		/*
 421		 * If we fail the final pass of JIT (from jit_subprogs),
 422		 * the program may not be finalized yet. Call finalize here
 423		 * before freeing it.
 424		 */
 425		if (jit_data) {
 426			bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr);
 427			kvfree(jit_data->addrs);
 428			kfree(jit_data);
 429		}
 430		hdr = bpf_jit_binary_pack_hdr(fp);
 431		bpf_jit_binary_pack_free(hdr, NULL);
 432		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
 433	}
 434
 435	bpf_prog_unlock_free(fp);
 436}
 437
 438bool bpf_jit_supports_kfunc_call(void)
 439{
 440	return IS_ENABLED(CONFIG_PPC64);
 441}
 442
 443bool bpf_jit_supports_arena(void)
 444{
 445	return IS_ENABLED(CONFIG_PPC64);
 446}
 447
 448bool bpf_jit_supports_far_kfunc_call(void)
 449{
 450	return IS_ENABLED(CONFIG_PPC64);
 451}
 452
 453bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
 454{
 455	if (!in_arena)
 456		return true;
 457	switch (insn->code) {
 458	case BPF_STX | BPF_ATOMIC | BPF_H:
 459	case BPF_STX | BPF_ATOMIC | BPF_B:
 460	case BPF_STX | BPF_ATOMIC | BPF_W:
 461	case BPF_STX | BPF_ATOMIC | BPF_DW:
 462		if (bpf_atomic_is_load_store(insn))
 463			return false;
 464		return IS_ENABLED(CONFIG_PPC64);
 465	}
 466	return true;
 467}
 468
 469void *arch_alloc_bpf_trampoline(unsigned int size)
 470{
 471	return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns);
 472}
 473
 474void arch_free_bpf_trampoline(void *image, unsigned int size)
 475{
 476	bpf_prog_pack_free(image, size);
 477}
 478
 479int arch_protect_bpf_trampoline(void *image, unsigned int size)
 480{
 481	return 0;
 482}
 483
 484static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx,
 485			   struct bpf_tramp_link *l, int regs_off, int retval_off,
 486			   int run_ctx_off, bool save_ret)
 487{
 488	struct bpf_prog *p = l->link.prog;
 489	ppc_inst_t branch_insn;
 490	u32 jmp_idx;
 491	int ret = 0;
 492
 493	/* Save cookie */
 494	if (IS_ENABLED(CONFIG_PPC64)) {
 495		PPC_LI64(_R3, l->cookie);
 496		EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx,
 497				 bpf_cookie)));
 498	} else {
 499		PPC_LI32(_R3, l->cookie >> 32);
 500		PPC_LI32(_R4, l->cookie);
 501		EMIT(PPC_RAW_STW(_R3, _R1,
 502				 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie)));
 503		EMIT(PPC_RAW_STW(_R4, _R1,
 504				 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4));
 505	}
 506
 507	/* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */
 508	PPC_LI_ADDR(_R3, p);
 509	EMIT(PPC_RAW_MR(_R25, _R3));
 510	EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off));
 511	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
 512					 (unsigned long)bpf_trampoline_enter(p));
 513	if (ret)
 514		return ret;
 515
 516	/* Remember prog start time returned by __bpf_prog_enter */
 517	EMIT(PPC_RAW_MR(_R26, _R3));
 518
 519	/*
 520	 * if (__bpf_prog_enter(p) == 0)
 521	 *	goto skip_exec_of_prog;
 522	 *
 523	 * Emit a nop to be later patched with conditional branch, once offset is known
 524	 */
 525	EMIT(PPC_RAW_CMPLI(_R3, 0));
 526	jmp_idx = ctx->idx;
 527	EMIT(PPC_RAW_NOP());
 528
 529	/* p->bpf_func(ctx) */
 530	EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off));
 531	if (!p->jited)
 532		PPC_LI_ADDR(_R4, (unsigned long)p->insnsi);
 533	/* Account for max possible instructions during dummy pass for size calculation */
 534	if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx],
 535				    (unsigned long)p->bpf_func,
 536				    BRANCH_SET_LINK)) {
 537		image[ctx->idx] = ppc_inst_val(branch_insn);
 538		ctx->idx++;
 539	} else {
 540		EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func)));
 541		EMIT(PPC_RAW_MTCTR(_R12));
 542		EMIT(PPC_RAW_BCTRL());
 543	}
 544
 545	if (save_ret)
 546		EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
 547
 548	/* Fix up branch */
 549	if (image) {
 550		if (create_cond_branch(&branch_insn, &image[jmp_idx],
 551				       (unsigned long)&image[ctx->idx], COND_EQ << 16))
 552			return -EINVAL;
 553		image[jmp_idx] = ppc_inst_val(branch_insn);
 554	}
 555
 556	/* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */
 557	EMIT(PPC_RAW_MR(_R3, _R25));
 558	EMIT(PPC_RAW_MR(_R4, _R26));
 559	EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off));
 560	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
 561					 (unsigned long)bpf_trampoline_exit(p));
 562
 563	return ret;
 564}
 565
 566static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx,
 567			      struct bpf_tramp_links *tl, int regs_off, int retval_off,
 568			      int run_ctx_off, u32 *branches)
 569{
 570	int i;
 571
 572	/*
 573	 * The first fmod_ret program will receive a garbage return value.
 574	 * Set this to 0 to avoid confusing the program.
 575	 */
 576	EMIT(PPC_RAW_LI(_R3, 0));
 577	EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
 578	for (i = 0; i < tl->nr_links; i++) {
 579		if (invoke_bpf_prog(image, ro_image, ctx, tl->links[i], regs_off, retval_off,
 580				    run_ctx_off, true))
 581			return -EINVAL;
 582
 583		/*
 584		 * mod_ret prog stored return value after prog ctx. Emit:
 585		 * if (*(u64 *)(ret_val) !=  0)
 586		 *	goto do_fexit;
 587		 */
 588		EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
 589		EMIT(PPC_RAW_CMPLI(_R3, 0));
 590
 591		/*
 592		 * Save the location of the branch and generate a nop, which is
 593		 * replaced with a conditional jump once do_fexit (i.e. the
 594		 * start of the fexit invocation) is finalized.
 595		 */
 596		branches[i] = ctx->idx;
 597		EMIT(PPC_RAW_NOP());
 598	}
 599
 600	return 0;
 601}
 602
 603static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_context *ctx,
 604					       int func_frame_offset, int r4_off)
 605{
 606	if (IS_ENABLED(CONFIG_PPC64)) {
 607		/* See bpf_jit_stack_tailcallcnt() */
 608		int tailcallcnt_offset = 7 * 8;
 609
 610		EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset));
 611		EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset));
 612	} else {
 613		/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */
 614		EMIT(PPC_RAW_LL(_R4, _R1, r4_off));
 615	}
 616}
 617
 618static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx,
 619						 int func_frame_offset, int r4_off)
 620{
 621	if (IS_ENABLED(CONFIG_PPC64)) {
 622		/* See bpf_jit_stack_tailcallcnt() */
 623		int tailcallcnt_offset = 7 * 8;
 624
 625		EMIT(PPC_RAW_LL(_R3, _R1, -tailcallcnt_offset));
 626		EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset - tailcallcnt_offset));
 627	} else {
 628		/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */
 629		EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
 630	}
 631}
 632
 633static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx, int func_frame_offset,
 634				     int nr_regs, int regs_off)
 635{
 636	int param_save_area_offset;
 637
 638	param_save_area_offset = func_frame_offset; /* the two frames we alloted */
 639	param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */
 640
 641	for (int i = 0; i < nr_regs; i++) {
 642		if (i < 8) {
 643			EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL));
 644		} else {
 645			EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
 646			EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL));
 647		}
 648	}
 649}
 650
 651/* Used when restoring just the register parameters when returning back */
 652static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx,
 653					     int nr_regs, int regs_off)
 654{
 655	for (int i = 0; i < nr_regs && i < 8; i++)
 656		EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL));
 657}
 658
 659/* Used when we call into the traced function. Replicate parameter save area */
 660static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx,
 661					      int func_frame_offset, int nr_regs, int regs_off)
 662{
 663	int param_save_area_offset;
 664
 665	param_save_area_offset = func_frame_offset; /* the two frames we alloted */
 666	param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */
 667
 668	for (int i = 8; i < nr_regs; i++) {
 669		EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
 670		EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL));
 671	}
 672	bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
 673}
 674
 675static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image,
 676					 void *rw_image_end, void *ro_image,
 677					 const struct btf_func_model *m, u32 flags,
 678					 struct bpf_tramp_links *tlinks,
 679					 void *func_addr)
 680{
 681	int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off = 0;
 682	int i, ret, nr_regs, bpf_frame_size = 0, bpf_dummy_frame_size = 0, func_frame_offset;
 683	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
 684	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
 685	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
 686	struct codegen_context codegen_ctx, *ctx;
 687	u32 *image = (u32 *)rw_image;
 688	ppc_inst_t branch_insn;
 689	u32 *branches = NULL;
 690	bool save_ret;
 691
 692	if (IS_ENABLED(CONFIG_PPC32))
 693		return -EOPNOTSUPP;
 694
 695	nr_regs = m->nr_args;
 696	/* Extra registers for struct arguments */
 697	for (i = 0; i < m->nr_args; i++)
 698		if (m->arg_size[i] > SZL)
 699			nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1;
 700
 701	if (nr_regs > MAX_BPF_FUNC_ARGS)
 702		return -EOPNOTSUPP;
 703
 704	ctx = &codegen_ctx;
 705	memset(ctx, 0, sizeof(*ctx));
 706
 707	/*
 708	 * Generated stack layout:
 709	 *
 710	 * func prev back chain         [ back chain        ]
 711	 *                              [                   ]
 712	 * bpf prog redzone/tailcallcnt [ ...               ] 64 bytes (64-bit powerpc)
 713	 *                              [                   ] --
 714	 * LR save area                 [ r0 save (64-bit)  ]   | header
 715	 *                              [ r0 save (32-bit)  ]   |
 716	 * dummy frame for unwind       [ back chain 1      ] --
 717	 *                              [ padding           ] align stack frame
 718	 *       r4_off                 [ r4 (tailcallcnt)  ] optional - 32-bit powerpc
 719	 *       alt_lr_off             [ real lr (ool stub)] optional - actual lr
 720	 *                              [ r26               ]
 721	 *       nvr_off                [ r25               ] nvr save area
 722	 *       retval_off             [ return value      ]
 723	 *                              [ reg argN          ]
 724	 *                              [ ...               ]
 725	 *       regs_off               [ reg_arg1          ] prog_ctx
 726	 *       nregs_off              [ args count        ] ((u64 *)prog_ctx)[-1]
 727	 *       ip_off                 [ traced function   ] ((u64 *)prog_ctx)[-2]
 728	 *                              [ ...               ]
 729	 *       run_ctx_off            [ bpf_tramp_run_ctx ]
 730	 *                              [ reg argN          ]
 731	 *                              [ ...               ]
 732	 *       param_save_area        [ reg_arg1          ] min 8 doublewords, per ABI
 733	 *                              [ TOC save (64-bit) ] --
 734	 *                              [ LR save (64-bit)  ]   | header
 735	 *                              [ LR save (32-bit)  ]   |
 736	 * bpf trampoline frame	        [ back chain 2      ] --
 737	 *
 738	 */
 739
 740	/* Minimum stack frame header */
 741	bpf_frame_size = STACK_FRAME_MIN_SIZE;
 742
 743	/*
 744	 * Room for parameter save area.
 745	 *
 746	 * As per the ABI, this is required if we call into the traced
 747	 * function (BPF_TRAMP_F_CALL_ORIG):
 748	 * - if the function takes more than 8 arguments for the rest to spill onto the stack
 749	 * - or, if the function has variadic arguments
 750	 * - or, if this functions's prototype was not available to the caller
 751	 *
 752	 * Reserve space for at least 8 registers for now. This can be optimized later.
 753	 */
 754	bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL;
 755
 756	/* Room for struct bpf_tramp_run_ctx */
 757	run_ctx_off = bpf_frame_size;
 758	bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL);
 759
 760	/* Room for IP address argument */
 761	ip_off = bpf_frame_size;
 762	if (flags & BPF_TRAMP_F_IP_ARG)
 763		bpf_frame_size += SZL;
 764
 765	/* Room for args count */
 766	nregs_off = bpf_frame_size;
 767	bpf_frame_size += SZL;
 768
 769	/* Room for args */
 770	regs_off = bpf_frame_size;
 771	bpf_frame_size += nr_regs * SZL;
 772
 773	/* Room for return value of func_addr or fentry prog */
 774	retval_off = bpf_frame_size;
 775	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
 776	if (save_ret)
 777		bpf_frame_size += SZL;
 778
 779	/* Room for nvr save area */
 780	nvr_off = bpf_frame_size;
 781	bpf_frame_size += 2 * SZL;
 782
 783	/* Optional save area for actual LR in case of ool ftrace */
 784	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
 785		alt_lr_off = bpf_frame_size;
 786		bpf_frame_size += SZL;
 787	}
 788
 789	if (IS_ENABLED(CONFIG_PPC32)) {
 790		if (nr_regs < 2) {
 791			r4_off = bpf_frame_size;
 792			bpf_frame_size += SZL;
 793		} else {
 794			r4_off = regs_off + SZL;
 795		}
 796	}
 797
 798	/* Padding to align stack frame, if any */
 799	bpf_frame_size = round_up(bpf_frame_size, SZL * 2);
 800
 801	/* Dummy frame size for proper unwind - includes 64-bytes red zone for 64-bit powerpc */
 802	bpf_dummy_frame_size = STACK_FRAME_MIN_SIZE + 64;
 803
 804	/* Offset to the traced function's stack frame */
 805	func_frame_offset = bpf_dummy_frame_size + bpf_frame_size;
 806
 807	/* Create dummy frame for unwind, store original return value */
 808	EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF));
 809	/* Protect red zone where tail call count goes */
 810	EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_dummy_frame_size));
 811
 812	/* Create our stack frame */
 813	EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size));
 814
 815	/* 64-bit: Save TOC and load kernel TOC */
 816	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
 817		EMIT(PPC_RAW_STD(_R2, _R1, 24));
 818		PPC64_LOAD_PACA();
 819	}
 820
 821	/* 32-bit: save tail call count in r4 */
 822	if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2)
 823		EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
 824
 825	bpf_trampoline_save_args(image, ctx, func_frame_offset, nr_regs, regs_off);
 826
 827	/* Save our LR/return address */
 828	EMIT(PPC_RAW_MFLR(_R3));
 829	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
 830		EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off));
 831	else
 832		EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF));
 833
 834	/*
 835	 * Derive IP address of the traced function.
 836	 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction
 837	 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and
 838	 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively.
 839	 * Relevant stub sequence:
 840	 *
 841	 *               bl <tramp>
 842	 *   LR (R3) =>  mtlr r0
 843	 *               b <func_addr+4>
 844	 *
 845	 * Recover kernel function/bpf program address from the unconditional
 846	 * branch instruction at the end of OOL stub.
 847	 */
 848	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) {
 849		EMIT(PPC_RAW_LWZ(_R4, _R3, 4));
 850		EMIT(PPC_RAW_SLWI(_R4, _R4, 6));
 851		EMIT(PPC_RAW_SRAWI(_R4, _R4, 6));
 852		EMIT(PPC_RAW_ADD(_R3, _R3, _R4));
 853	}
 854
 855	if (flags & BPF_TRAMP_F_IP_ARG)
 856		EMIT(PPC_RAW_STL(_R3, _R1, ip_off));
 857
 858	if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
 859		/* Fake our LR for unwind */
 860		EMIT(PPC_RAW_ADDI(_R3, _R3, 4));
 861		EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF));
 862	}
 863
 864	/* Save function arg count -- see bpf_get_func_arg_cnt() */
 865	EMIT(PPC_RAW_LI(_R3, nr_regs));
 866	EMIT(PPC_RAW_STL(_R3, _R1, nregs_off));
 867
 868	/* Save nv regs */
 869	EMIT(PPC_RAW_STL(_R25, _R1, nvr_off));
 870	EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL));
 871
 872	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 873		PPC_LI_ADDR(_R3, (unsigned long)im);
 874		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
 875						 (unsigned long)__bpf_tramp_enter);
 876		if (ret)
 877			return ret;
 878	}
 879
 880	for (i = 0; i < fentry->nr_links; i++)
 881		if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off,
 882				    run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
 883			return -EINVAL;
 884
 885	if (fmod_ret->nr_links) {
 886		branches = kcalloc(fmod_ret->nr_links, sizeof(u32), GFP_KERNEL);
 887		if (!branches)
 888			return -ENOMEM;
 889
 890		if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off,
 891				       run_ctx_off, branches)) {
 892			ret = -EINVAL;
 893			goto cleanup;
 894		}
 895	}
 896
 897	/* Call the traced function */
 898	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 899		/*
 900		 * The address in LR save area points to the correct point in the original function
 901		 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction
 902		 * sequence
 903		 */
 904		EMIT(PPC_RAW_LL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF));
 905		EMIT(PPC_RAW_MTCTR(_R3));
 906
 907		/* Replicate tail_call_cnt before calling the original BPF prog */
 908		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
 909			bpf_trampoline_setup_tail_call_cnt(image, ctx, func_frame_offset, r4_off);
 910
 911		/* Restore args */
 912		bpf_trampoline_restore_args_stack(image, ctx, func_frame_offset, nr_regs, regs_off);
 913
 914		/* Restore TOC for 64-bit */
 915		if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
 916			EMIT(PPC_RAW_LD(_R2, _R1, 24));
 917		EMIT(PPC_RAW_BCTRL());
 918		if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
 919			PPC64_LOAD_PACA();
 920
 921		/* Store return value for bpf prog to access */
 922		EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
 923
 924		/* Restore updated tail_call_cnt */
 925		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
 926			bpf_trampoline_restore_tail_call_cnt(image, ctx, func_frame_offset, r4_off);
 927
 928		/* Reserve space to patch branch instruction to skip fexit progs */
 929		if (ro_image) /* image is NULL for dummy pass */
 930			im->ip_after_call = &((u32 *)ro_image)[ctx->idx];
 931		EMIT(PPC_RAW_NOP());
 932	}
 933
 934	/* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */
 935	for (i = 0; i < fmod_ret->nr_links && image; i++) {
 936		if (create_cond_branch(&branch_insn, &image[branches[i]],
 937				       (unsigned long)&image[ctx->idx], COND_NE << 16)) {
 938			ret = -EINVAL;
 939			goto cleanup;
 940		}
 941
 942		image[branches[i]] = ppc_inst_val(branch_insn);
 943	}
 944
 945	for (i = 0; i < fexit->nr_links; i++)
 946		if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off,
 947				    run_ctx_off, false)) {
 948			ret = -EINVAL;
 949			goto cleanup;
 950		}
 951
 952	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 953		if (ro_image) /* image is NULL for dummy pass */
 954			im->ip_epilogue = &((u32 *)ro_image)[ctx->idx];
 955		PPC_LI_ADDR(_R3, im);
 956		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
 957						 (unsigned long)__bpf_tramp_exit);
 958		if (ret)
 959			goto cleanup;
 960	}
 961
 962	if (flags & BPF_TRAMP_F_RESTORE_REGS)
 963		bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
 964
 965	/* Restore return value of func_addr or fentry prog */
 966	if (save_ret)
 967		EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
 968
 969	/* Restore nv regs */
 970	EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL));
 971	EMIT(PPC_RAW_LL(_R25, _R1, nvr_off));
 972
 973	/* Epilogue */
 974	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
 975		EMIT(PPC_RAW_LD(_R2, _R1, 24));
 976	if (flags & BPF_TRAMP_F_SKIP_FRAME) {
 977		/* Skip the traced function and return to parent */
 978		EMIT(PPC_RAW_ADDI(_R1, _R1, func_frame_offset));
 979		EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
 980		EMIT(PPC_RAW_MTLR(_R0));
 981		EMIT(PPC_RAW_BLR());
 982	} else {
 983		if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
 984			EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off));
 985			EMIT(PPC_RAW_MTLR(_R0));
 986			EMIT(PPC_RAW_ADDI(_R1, _R1, func_frame_offset));
 987			EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
 988			EMIT(PPC_RAW_BLR());
 989		} else {
 990			EMIT(PPC_RAW_LL(_R0, _R1, bpf_frame_size + PPC_LR_STKOFF));
 991			EMIT(PPC_RAW_MTCTR(_R0));
 992			EMIT(PPC_RAW_ADDI(_R1, _R1, func_frame_offset));
 993			EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
 994			EMIT(PPC_RAW_MTLR(_R0));
 995			EMIT(PPC_RAW_BCTR());
 996		}
 997	}
 998
 999	/* Make sure the trampoline generation logic doesn't overflow */
1000	if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) {
1001		ret = -EFAULT;
1002		goto cleanup;
1003	}
1004	ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4;
1005
1006cleanup:
1007	kfree(branches);
1008	return ret;
1009}
1010
1011int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1012			     struct bpf_tramp_links *tlinks, void *func_addr)
1013{
1014	struct bpf_tramp_image im;
1015	int ret;
1016
1017	ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tlinks, func_addr);
1018	return ret;
1019}
1020
1021int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1022				const struct btf_func_model *m, u32 flags,
1023				struct bpf_tramp_links *tlinks,
1024				void *func_addr)
1025{
1026	u32 size = image_end - image;
1027	void *rw_image, *tmp;
1028	int ret;
1029
1030	/*
1031	 * rw_image doesn't need to be in module memory range, so we can
1032	 * use kvmalloc.
1033	 */
1034	rw_image = kvmalloc(size, GFP_KERNEL);
1035	if (!rw_image)
1036		return -ENOMEM;
1037
1038	ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m,
1039					    flags, tlinks, func_addr);
1040	if (ret < 0)
1041		goto out;
1042
1043	if (bpf_jit_enable > 1)
1044		bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image);
1045
1046	tmp = bpf_arch_text_copy(image, rw_image, size);
1047	if (IS_ERR(tmp))
1048		ret = PTR_ERR(tmp);
1049
1050out:
1051	kvfree(rw_image);
1052	return ret;
1053}
1054
1055static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst)
1056{
1057	ppc_inst_t org_inst;
1058
1059	if (copy_inst_from_kernel_nofault(&org_inst, ip)) {
1060		pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip);
1061		return -EFAULT;
1062	}
1063
1064	if (!ppc_inst_equal(org_inst, old_inst)) {
1065		pr_err("0x%lx: expected (%08lx) != found (%08lx)\n",
1066		       (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst));
1067		return -EINVAL;
1068	}
1069
1070	if (ppc_inst_equal(old_inst, new_inst))
1071		return 0;
1072
1073	return patch_instruction(ip, new_inst);
1074}
1075
1076static void do_isync(void *info __maybe_unused)
1077{
1078	isync();
1079}
1080
1081/*
1082 * A 3-step process for bpf prog entry:
1083 * 1. At bpf prog entry, a single nop/b:
1084 * bpf_func:
1085 *	[nop|b]	ool_stub
1086 * 2. Out-of-line stub:
1087 * ool_stub:
1088 *	mflr	r0
1089 *	[b|bl]	<bpf_prog>/<long_branch_stub>
1090 *	mtlr	r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only
1091 *	b	bpf_func + 4
1092 * 3. Long branch stub:
1093 * long_branch_stub:
1094 *	.long	<branch_addr>/<dummy_tramp>
1095 *	mflr	r11
1096 *	bcl	20,31,$+4
1097 *	mflr	r12
1098 *	ld	r12, -16(r12)
1099 *	mtctr	r12
1100 *	mtlr	r11 // needed to retain ftrace ABI
1101 *	bctr
1102 *
1103 * dummy_tramp is used to reduce synchronization requirements.
1104 *
1105 * When attaching a bpf trampoline to a bpf prog, we do not need any
1106 * synchronization here since we always have a valid branch target regardless
1107 * of the order in which the above stores are seen. dummy_tramp ensures that
1108 * the long_branch stub goes to a valid destination on other cpus, even when
1109 * the branch to the long_branch stub is seen before the updated trampoline
1110 * address.
1111 *
1112 * However, when detaching a bpf trampoline from a bpf prog, or if changing
1113 * the bpf trampoline address, we need synchronization to ensure that other
1114 * cpus can no longer branch into the older trampoline so that it can be
1115 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus
1116 * make forward progress, but we still need to ensure that other cpus
1117 * execute isync (or some CSI) so that they don't go back into the
1118 * trampoline again.
1119 */
1120int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
1121		       enum bpf_text_poke_type new_t, void *old_addr,
1122		       void *new_addr)
1123{
1124	unsigned long bpf_func, bpf_func_end, size, offset;
1125	ppc_inst_t old_inst, new_inst;
1126	int ret = 0, branch_flags;
1127	char name[KSYM_NAME_LEN];
1128
1129	if (IS_ENABLED(CONFIG_PPC32))
1130		return -EOPNOTSUPP;
1131
1132	bpf_func = (unsigned long)ip;
1133
1134	/* We currently only support poking bpf programs */
1135	if (!bpf_address_lookup(bpf_func, &size, &offset, name)) {
1136		pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func);
1137		return -EOPNOTSUPP;
1138	}
1139
1140	/*
1141	 * If we are not poking at bpf prog entry, then we are simply patching in/out
1142	 * an unconditional branch instruction at im->ip_after_call
1143	 */
1144	if (offset) {
1145		if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) {
1146			pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__,
1147			       bpf_func);
1148			return -EOPNOTSUPP;
1149		}
1150		old_inst = ppc_inst(PPC_RAW_NOP());
1151		if (old_addr)
1152			if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0))
1153				return -ERANGE;
1154		new_inst = ppc_inst(PPC_RAW_NOP());
1155		if (new_addr)
1156			if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0))
1157				return -ERANGE;
1158		mutex_lock(&text_mutex);
1159		ret = bpf_modify_inst(ip, old_inst, new_inst);
1160		mutex_unlock(&text_mutex);
1161
1162		/* Make sure all cpus see the new instruction */
1163		smp_call_function(do_isync, NULL, 1);
1164		return ret;
1165	}
1166
1167	bpf_func_end = bpf_func + size;
1168
1169	/* Address of the jmp/call instruction in the out-of-line stub */
1170	ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4);
1171
1172	if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) {
1173		pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__,
1174		       bpf_func);
1175		return -ERANGE;
1176	}
1177
1178	old_inst = ppc_inst(PPC_RAW_NOP());
1179	branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
1180	if (old_addr) {
1181		if (is_offset_in_branch_range(ip - old_addr))
1182			create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags);
1183		else
1184			create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
1185				      branch_flags);
1186	}
1187	new_inst = ppc_inst(PPC_RAW_NOP());
1188	branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
1189	if (new_addr) {
1190		if (is_offset_in_branch_range(ip - new_addr))
1191			create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags);
1192		else
1193			create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
1194				      branch_flags);
1195	}
1196
1197	mutex_lock(&text_mutex);
1198
1199	/*
1200	 * 1. Update the address in the long branch stub:
1201	 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr
1202	 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here.
1203	 */
1204	if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) ||
1205	    (old_addr && !is_offset_in_branch_range(old_addr - ip)))
1206		ret = patch_ulong((void *)(bpf_func_end - bpf_jit_long_branch_stub - SZL),
1207				  (new_addr && !is_offset_in_branch_range(new_addr - ip)) ?
1208				  (unsigned long)new_addr : (unsigned long)dummy_tramp);
1209	if (ret)
1210		goto out;
1211
1212	/* 2. Update the branch/call in the out-of-line stub */
1213	ret = bpf_modify_inst(ip, old_inst, new_inst);
1214	if (ret)
1215		goto out;
1216
1217	/* 3. Update instruction at bpf prog entry */
1218	ip = (void *)bpf_func;
1219	if (!old_addr || !new_addr) {
1220		if (!old_addr) {
1221			old_inst = ppc_inst(PPC_RAW_NOP());
1222			create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
1223		} else {
1224			new_inst = ppc_inst(PPC_RAW_NOP());
1225			create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
1226		}
1227		ret = bpf_modify_inst(ip, old_inst, new_inst);
1228	}
1229
1230out:
1231	mutex_unlock(&text_mutex);
1232
1233	/*
1234	 * Sync only if we are not attaching a trampoline to a bpf prog so the older
1235	 * trampoline can be freed safely.
1236	 */
1237	if (old_addr)
1238		smp_call_function(do_isync, NULL, 1);
1239
1240	return ret;
1241}