bpf_jit_comp64.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* BPF JIT compiler for RV64G
   3 *
   4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   5 *
   6 */
   7
   8#include <linux/bitfield.h>
   9#include <linux/bpf.h>
  10#include <linux/filter.h>
  11#include <linux/memory.h>
  12#include <linux/stop_machine.h>
  13#include <asm/text-patching.h>
  14#include <asm/cfi.h>
  15#include <asm/percpu.h>
  16#include "bpf_jit.h"
  17
  18#define RV_MAX_REG_ARGS 8
  19#define RV_FENTRY_NINSNS 2
  20#define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4)
  21#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0)
  22/* imm that allows emit_imm to emit max count insns */
  23#define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF
  24
  25#define RV_REG_TCC RV_REG_A6
  26#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
  27#define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */
  28
  29static const int regmap[] = {
  30	[BPF_REG_0] =	RV_REG_A5,
  31	[BPF_REG_1] =	RV_REG_A0,
  32	[BPF_REG_2] =	RV_REG_A1,
  33	[BPF_REG_3] =	RV_REG_A2,
  34	[BPF_REG_4] =	RV_REG_A3,
  35	[BPF_REG_5] =	RV_REG_A4,
  36	[BPF_REG_6] =	RV_REG_S1,
  37	[BPF_REG_7] =	RV_REG_S2,
  38	[BPF_REG_8] =	RV_REG_S3,
  39	[BPF_REG_9] =	RV_REG_S4,
  40	[BPF_REG_FP] =	RV_REG_S5,
  41	[BPF_REG_AX] =	RV_REG_T0,
  42};
  43
  44static const int pt_regmap[] = {
  45	[RV_REG_A0] = offsetof(struct pt_regs, a0),
  46	[RV_REG_A1] = offsetof(struct pt_regs, a1),
  47	[RV_REG_A2] = offsetof(struct pt_regs, a2),
  48	[RV_REG_A3] = offsetof(struct pt_regs, a3),
  49	[RV_REG_A4] = offsetof(struct pt_regs, a4),
  50	[RV_REG_A5] = offsetof(struct pt_regs, a5),
  51	[RV_REG_S1] = offsetof(struct pt_regs, s1),
  52	[RV_REG_S2] = offsetof(struct pt_regs, s2),
  53	[RV_REG_S3] = offsetof(struct pt_regs, s3),
  54	[RV_REG_S4] = offsetof(struct pt_regs, s4),
  55	[RV_REG_S5] = offsetof(struct pt_regs, s5),
  56	[RV_REG_T0] = offsetof(struct pt_regs, t0),
  57};
  58
  59enum {
  60	RV_CTX_F_SEEN_TAIL_CALL =	0,
  61	RV_CTX_F_SEEN_CALL =		RV_REG_RA,
  62	RV_CTX_F_SEEN_S1 =		RV_REG_S1,
  63	RV_CTX_F_SEEN_S2 =		RV_REG_S2,
  64	RV_CTX_F_SEEN_S3 =		RV_REG_S3,
  65	RV_CTX_F_SEEN_S4 =		RV_REG_S4,
  66	RV_CTX_F_SEEN_S5 =		RV_REG_S5,
  67	RV_CTX_F_SEEN_S6 =		RV_REG_S6,
  68};
  69
  70static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
  71{
  72	u8 reg = regmap[bpf_reg];
  73
  74	switch (reg) {
  75	case RV_CTX_F_SEEN_S1:
  76	case RV_CTX_F_SEEN_S2:
  77	case RV_CTX_F_SEEN_S3:
  78	case RV_CTX_F_SEEN_S4:
  79	case RV_CTX_F_SEEN_S5:
  80	case RV_CTX_F_SEEN_S6:
  81		__set_bit(reg, &ctx->flags);
  82	}
  83	return reg;
  84};
  85
  86static bool seen_reg(int reg, struct rv_jit_context *ctx)
  87{
  88	switch (reg) {
  89	case RV_CTX_F_SEEN_CALL:
  90	case RV_CTX_F_SEEN_S1:
  91	case RV_CTX_F_SEEN_S2:
  92	case RV_CTX_F_SEEN_S3:
  93	case RV_CTX_F_SEEN_S4:
  94	case RV_CTX_F_SEEN_S5:
  95	case RV_CTX_F_SEEN_S6:
  96		return test_bit(reg, &ctx->flags);
  97	}
  98	return false;
  99}
 100
 101static void mark_fp(struct rv_jit_context *ctx)
 102{
 103	__set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
 104}
 105
 106static void mark_call(struct rv_jit_context *ctx)
 107{
 108	__set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
 109}
 110
 111static bool seen_call(struct rv_jit_context *ctx)
 112{
 113	return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
 114}
 115
 116static void mark_tail_call(struct rv_jit_context *ctx)
 117{
 118	__set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
 119}
 120
 121static bool seen_tail_call(struct rv_jit_context *ctx)
 122{
 123	return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
 124}
 125
 126static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
 127{
 128	mark_tail_call(ctx);
 129
 130	if (seen_call(ctx)) {
 131		__set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
 132		return RV_REG_S6;
 133	}
 134	return RV_REG_A6;
 135}
 136
 137static bool is_32b_int(s64 val)
 138{
 139	return -(1L << 31) <= val && val < (1L << 31);
 140}
 141
 142static bool in_auipc_jalr_range(s64 val)
 143{
 144	/*
 145	 * auipc+jalr can reach any signed PC-relative offset in the range
 146	 * [-2^31 - 2^11, 2^31 - 2^11).
 147	 */
 148	return (-(1L << 31) - (1L << 11)) <= val &&
 149		val < ((1L << 31) - (1L << 11));
 150}
 151
 152/* Modify rd pointer to alternate reg to avoid corrupting original reg */
 153static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx)
 154{
 155	emit_sextw(ra, *rd, ctx);
 156	*rd = ra;
 157}
 158
 159static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx)
 160{
 161	emit_zextw(ra, *rd, ctx);
 162	*rd = ra;
 163}
 164
 165/* Emit fixed-length instructions for address */
 166static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx)
 167{
 168	/*
 169	 * Use the ro_insns(RX) to calculate the offset as the BPF program will
 170	 * finally run from this memory region.
 171	 */
 172	u64 ip = (u64)(ctx->ro_insns + ctx->ninsns);
 173	s64 off = addr - ip;
 174	s64 upper = (off + (1 << 11)) >> 12;
 175	s64 lower = off & 0xfff;
 176
 177	if (extra_pass && !in_auipc_jalr_range(off)) {
 178		pr_err("bpf-jit: target offset 0x%llx is out of range\n", off);
 179		return -ERANGE;
 180	}
 181
 182	emit(rv_auipc(rd, upper), ctx);
 183	emit(rv_addi(rd, rd, lower), ctx);
 184	return 0;
 185}
 186
 187/* Emit variable-length instructions for 32-bit and 64-bit imm */
 188static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
 189{
 190	/* Note that the immediate from the add is sign-extended,
 191	 * which means that we need to compensate this by adding 2^12,
 192	 * when the 12th bit is set. A simpler way of doing this, and
 193	 * getting rid of the check, is to just add 2**11 before the
 194	 * shift. The "Loading a 32-Bit constant" example from the
 195	 * "Computer Organization and Design, RISC-V edition" book by
 196	 * Patterson/Hennessy highlights this fact.
 197	 *
 198	 * This also means that we need to process LSB to MSB.
 199	 */
 200	s64 upper = (val + (1 << 11)) >> 12;
 201	/* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw,
 202	 * and addi are signed and RVC checks will perform signed comparisons.
 203	 */
 204	s64 lower = ((val & 0xfff) << 52) >> 52;
 205	int shift;
 206
 207	if (is_32b_int(val)) {
 208		if (upper)
 209			emit_lui(rd, upper, ctx);
 210
 211		if (!upper) {
 212			emit_li(rd, lower, ctx);
 213			return;
 214		}
 215
 216		emit_addiw(rd, rd, lower, ctx);
 217		return;
 218	}
 219
 220	shift = __ffs(upper);
 221	upper >>= shift;
 222	shift += 12;
 223
 224	emit_imm(rd, upper, ctx);
 225
 226	emit_slli(rd, rd, shift, ctx);
 227	if (lower)
 228		emit_addi(rd, rd, lower, ctx);
 229}
 230
 231static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
 232{
 233	int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
 234
 235	if (seen_reg(RV_REG_RA, ctx)) {
 236		emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx);
 237		store_offset -= 8;
 238	}
 239	emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx);
 240	store_offset -= 8;
 241	if (seen_reg(RV_REG_S1, ctx)) {
 242		emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx);
 243		store_offset -= 8;
 244	}
 245	if (seen_reg(RV_REG_S2, ctx)) {
 246		emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx);
 247		store_offset -= 8;
 248	}
 249	if (seen_reg(RV_REG_S3, ctx)) {
 250		emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx);
 251		store_offset -= 8;
 252	}
 253	if (seen_reg(RV_REG_S4, ctx)) {
 254		emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx);
 255		store_offset -= 8;
 256	}
 257	if (seen_reg(RV_REG_S5, ctx)) {
 258		emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx);
 259		store_offset -= 8;
 260	}
 261	if (seen_reg(RV_REG_S6, ctx)) {
 262		emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx);
 263		store_offset -= 8;
 264	}
 265	if (ctx->arena_vm_start) {
 266		emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx);
 267		store_offset -= 8;
 268	}
 269
 270	emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx);
 271	/* Set return value. */
 272	if (!is_tail_call)
 273		emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx);
 274	emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
 275		  /* kcfi, fentry and TCC init insns will be skipped on tailcall */
 276		  is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0,
 277		  ctx);
 278}
 279
 280static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
 281		     struct rv_jit_context *ctx)
 282{
 283	switch (cond) {
 284	case BPF_JEQ:
 285		emit(rv_beq(rd, rs, rvoff >> 1), ctx);
 286		return;
 287	case BPF_JGT:
 288		emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
 289		return;
 290	case BPF_JLT:
 291		emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
 292		return;
 293	case BPF_JGE:
 294		emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
 295		return;
 296	case BPF_JLE:
 297		emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
 298		return;
 299	case BPF_JNE:
 300		emit(rv_bne(rd, rs, rvoff >> 1), ctx);
 301		return;
 302	case BPF_JSGT:
 303		emit(rv_blt(rs, rd, rvoff >> 1), ctx);
 304		return;
 305	case BPF_JSLT:
 306		emit(rv_blt(rd, rs, rvoff >> 1), ctx);
 307		return;
 308	case BPF_JSGE:
 309		emit(rv_bge(rd, rs, rvoff >> 1), ctx);
 310		return;
 311	case BPF_JSLE:
 312		emit(rv_bge(rs, rd, rvoff >> 1), ctx);
 313	}
 314}
 315
 316static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
 317			struct rv_jit_context *ctx)
 318{
 319	s64 upper, lower;
 320
 321	if (is_13b_int(rvoff)) {
 322		emit_bcc(cond, rd, rs, rvoff, ctx);
 323		return;
 324	}
 325
 326	/* Adjust for jal */
 327	rvoff -= 4;
 328
 329	/* Transform, e.g.:
 330	 *   bne rd,rs,foo
 331	 * to
 332	 *   beq rd,rs,<.L1>
 333	 *   (auipc foo)
 334	 *   jal(r) foo
 335	 * .L1
 336	 */
 337	cond = invert_bpf_cond(cond);
 338	if (is_21b_int(rvoff)) {
 339		emit_bcc(cond, rd, rs, 8, ctx);
 340		emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
 341		return;
 342	}
 343
 344	/* 32b No need for an additional rvoff adjustment, since we
 345	 * get that from the auipc at PC', where PC = PC' + 4.
 346	 */
 347	upper = (rvoff + (1 << 11)) >> 12;
 348	lower = rvoff & 0xfff;
 349
 350	emit_bcc(cond, rd, rs, 12, ctx);
 351	emit(rv_auipc(RV_REG_T1, upper), ctx);
 352	emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
 353}
 354
 355static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
 356{
 357	int tc_ninsn, off, start_insn = ctx->ninsns;
 358	u8 tcc = rv_tail_call_reg(ctx);
 359
 360	/* a0: &ctx
 361	 * a1: &array
 362	 * a2: index
 363	 *
 364	 * if (index >= array->map.max_entries)
 365	 *	goto out;
 366	 */
 367	tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
 368		   ctx->offset[0];
 369	emit_zextw(RV_REG_A2, RV_REG_A2, ctx);
 370
 371	off = offsetof(struct bpf_array, map.max_entries);
 372	if (is_12b_check(off, insn))
 373		return -1;
 374	emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
 375	off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
 376	emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
 377
 378	/* if (--TCC < 0)
 379	 *     goto out;
 380	 */
 381	emit_addi(RV_REG_TCC, tcc, -1, ctx);
 382	off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
 383	emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx);
 384
 385	/* prog = array->ptrs[index];
 386	 * if (!prog)
 387	 *     goto out;
 388	 */
 389	emit_sh3add(RV_REG_T2, RV_REG_A2, RV_REG_A1, ctx);
 390	off = offsetof(struct bpf_array, ptrs);
 391	if (is_12b_check(off, insn))
 392		return -1;
 393	emit_ld(RV_REG_T2, off, RV_REG_T2, ctx);
 394	off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
 395	emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
 396
 397	/* goto *(prog->bpf_func + 4); */
 398	off = offsetof(struct bpf_prog, bpf_func);
 399	if (is_12b_check(off, insn))
 400		return -1;
 401	emit_ld(RV_REG_T3, off, RV_REG_T2, ctx);
 402	__build_epilogue(true, ctx);
 403	return 0;
 404}
 405
 406static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
 407		      struct rv_jit_context *ctx)
 408{
 409	u8 code = insn->code;
 410
 411	switch (code) {
 412	case BPF_JMP | BPF_JA:
 413	case BPF_JMP | BPF_CALL:
 414	case BPF_JMP | BPF_EXIT:
 415	case BPF_JMP | BPF_TAIL_CALL:
 416		break;
 417	default:
 418		*rd = bpf_to_rv_reg(insn->dst_reg, ctx);
 419	}
 420
 421	if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
 422	    code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
 423	    code & BPF_LDX || code & BPF_STX)
 424		*rs = bpf_to_rv_reg(insn->src_reg, ctx);
 425}
 426
 427static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr,
 428			      struct rv_jit_context *ctx)
 429{
 430	s64 upper, lower;
 431
 432	if (rvoff && fixed_addr && is_21b_int(rvoff)) {
 433		emit(rv_jal(rd, rvoff >> 1), ctx);
 434		return 0;
 435	} else if (in_auipc_jalr_range(rvoff)) {
 436		upper = (rvoff + (1 << 11)) >> 12;
 437		lower = rvoff & 0xfff;
 438		emit(rv_auipc(RV_REG_T1, upper), ctx);
 439		emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
 440		return 0;
 441	}
 442
 443	pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff);
 444	return -ERANGE;
 445}
 446
 447static bool is_signed_bpf_cond(u8 cond)
 448{
 449	return cond == BPF_JSGT || cond == BPF_JSLT ||
 450		cond == BPF_JSGE || cond == BPF_JSLE;
 451}
 452
 453static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx)
 454{
 455	s64 off = 0;
 456	u64 ip;
 457
 458	if (addr && ctx->insns && ctx->ro_insns) {
 459		/*
 460		 * Use the ro_insns(RX) to calculate the offset as the BPF
 461		 * program will finally run from this memory region.
 462		 */
 463		ip = (u64)(long)(ctx->ro_insns + ctx->ninsns);
 464		off = addr - ip;
 465	}
 466
 467	return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx);
 468}
 469
 470static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx)
 471{
 472	if (IS_ENABLED(CONFIG_CFI))
 473		emit(hash, ctx);
 474}
 475
 476static void emit_ldx_insn(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext,
 477			  struct rv_jit_context *ctx)
 478{
 479	switch (size) {
 480	case BPF_B:
 481		emit(sign_ext ? rv_lb(rd, off, rs) : rv_lbu(rd, off, rs), ctx);
 482		break;
 483	case BPF_H:
 484		emit(sign_ext ? rv_lh(rd, off, rs) : rv_lhu(rd, off, rs), ctx);
 485		break;
 486	case BPF_W:
 487		emit(sign_ext ? rv_lw(rd, off, rs) : rv_lwu(rd, off, rs), ctx);
 488		break;
 489	case BPF_DW:
 490		emit_ld(rd, off, rs, ctx);
 491		break;
 492	}
 493
 494}
 495
 496static void emit_stx_insn(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
 497{
 498	switch (size) {
 499	case BPF_B:
 500		emit(rv_sb(rd, off, rs), ctx);
 501		break;
 502	case BPF_H:
 503		emit(rv_sh(rd, off, rs), ctx);
 504		break;
 505	case BPF_W:
 506		emit_sw(rd, off, rs, ctx);
 507		break;
 508	case BPF_DW:
 509		emit_sd(rd, off, rs, ctx);
 510		break;
 511	}
 512}
 513
 514static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext,
 515		    struct rv_jit_context *ctx)
 516{
 517	if (is_12b_int(off)) {
 518		ctx->ex_insn_off = ctx->ninsns;
 519		emit_ldx_insn(rd, off, rs, size, sign_ext, ctx);
 520		ctx->ex_jmp_off = ctx->ninsns;
 521		return;
 522	}
 523
 524	emit_imm(RV_REG_T1, off, ctx);
 525	emit_add(RV_REG_T1, RV_REG_T1, rs, ctx);
 526	ctx->ex_insn_off = ctx->ninsns;
 527	emit_ldx_insn(rd, 0, RV_REG_T1, size, sign_ext, ctx);
 528	ctx->ex_jmp_off = ctx->ninsns;
 529}
 530
 531static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx)
 532{
 533	emit_imm(RV_REG_T1, imm, ctx);
 534	if (is_12b_int(off)) {
 535		ctx->ex_insn_off = ctx->ninsns;
 536		emit_stx_insn(rd, off, RV_REG_T1, size, ctx);
 537		ctx->ex_jmp_off = ctx->ninsns;
 538		return;
 539	}
 540
 541	emit_imm(RV_REG_T2, off, ctx);
 542	emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);
 543	ctx->ex_insn_off = ctx->ninsns;
 544	emit_stx_insn(RV_REG_T2, 0, RV_REG_T1, size, ctx);
 545	ctx->ex_jmp_off = ctx->ninsns;
 546}
 547
 548static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
 549{
 550	if (is_12b_int(off)) {
 551		ctx->ex_insn_off = ctx->ninsns;
 552		emit_stx_insn(rd, off, rs, size, ctx);
 553		ctx->ex_jmp_off = ctx->ninsns;
 554		return;
 555	}
 556
 557	emit_imm(RV_REG_T1, off, ctx);
 558	emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
 559	ctx->ex_insn_off = ctx->ninsns;
 560	emit_stx_insn(RV_REG_T1, 0, rs, size, ctx);
 561	ctx->ex_jmp_off = ctx->ninsns;
 562}
 563
 564static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
 565			     struct rv_jit_context *ctx)
 566{
 567	u8 code = insn->code;
 568	s32 imm = insn->imm;
 569	s16 off = insn->off;
 570
 571	switch (imm) {
 572	/* dst_reg = load_acquire(src_reg + off16) */
 573	case BPF_LOAD_ACQ:
 574		if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
 575			emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx);
 576			rs = RV_REG_T2;
 577		}
 578
 579		emit_ldx(rd, off, rs, BPF_SIZE(code), false, ctx);
 580		emit_fence_r_rw(ctx);
 581
 582		/* If our next insn is a redundant zext, return 1 to tell
 583		 * build_body() to skip it.
 584		 */
 585		if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))
 586			return 1;
 587		break;
 588	/* store_release(dst_reg + off16, src_reg) */
 589	case BPF_STORE_REL:
 590		if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
 591			emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx);
 592			rd = RV_REG_T2;
 593		}
 594
 595		emit_fence_rw_w(ctx);
 596		emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
 597		break;
 598	default:
 599		pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm);
 600		return -EINVAL;
 601	}
 602
 603	return 0;
 604}
 605
 606static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
 607			   struct rv_jit_context *ctx)
 608{
 609	u8 code = insn->code;
 610	s16 off = insn->off;
 611	s32 imm = insn->imm;
 612	bool is64 = BPF_SIZE(code) == BPF_DW;
 613
 614	if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
 615		pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n");
 616		return -EINVAL;
 617	}
 618
 619	if (off) {
 620		if (is_12b_int(off)) {
 621			emit_addi(RV_REG_T1, rd, off, ctx);
 622		} else {
 623			emit_imm(RV_REG_T1, off, ctx);
 624			emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);
 625		}
 626		rd = RV_REG_T1;
 627	}
 628
 629	if (BPF_MODE(code) == BPF_PROBE_ATOMIC) {
 630		emit_add(RV_REG_T1, rd, RV_REG_ARENA, ctx);
 631		rd = RV_REG_T1;
 632	}
 633
 634	switch (imm) {
 635	/* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */
 636	case BPF_ADD:
 637		ctx->ex_insn_off = ctx->ninsns;
 638		emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) :
 639		     rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
 640		ctx->ex_jmp_off = ctx->ninsns;
 641		break;
 642	case BPF_AND:
 643		ctx->ex_insn_off = ctx->ninsns;
 644		emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) :
 645		     rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
 646		ctx->ex_jmp_off = ctx->ninsns;
 647		break;
 648	case BPF_OR:
 649		ctx->ex_insn_off = ctx->ninsns;
 650		emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) :
 651		     rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
 652		ctx->ex_jmp_off = ctx->ninsns;
 653		break;
 654	case BPF_XOR:
 655		ctx->ex_insn_off = ctx->ninsns;
 656		emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) :
 657		     rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx);
 658		ctx->ex_jmp_off = ctx->ninsns;
 659		break;
 660	/* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */
 661	case BPF_ADD | BPF_FETCH:
 662		ctx->ex_insn_off = ctx->ninsns;
 663		emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) :
 664		     rv_amoadd_w(rs, rs, rd, 1, 1), ctx);
 665		ctx->ex_jmp_off = ctx->ninsns;
 666		if (!is64)
 667			emit_zextw(rs, rs, ctx);
 668		break;
 669	case BPF_AND | BPF_FETCH:
 670		ctx->ex_insn_off = ctx->ninsns;
 671		emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) :
 672		     rv_amoand_w(rs, rs, rd, 1, 1), ctx);
 673		ctx->ex_jmp_off = ctx->ninsns;
 674		if (!is64)
 675			emit_zextw(rs, rs, ctx);
 676		break;
 677	case BPF_OR | BPF_FETCH:
 678		ctx->ex_insn_off = ctx->ninsns;
 679		emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) :
 680		     rv_amoor_w(rs, rs, rd, 1, 1), ctx);
 681		ctx->ex_jmp_off = ctx->ninsns;
 682		if (!is64)
 683			emit_zextw(rs, rs, ctx);
 684		break;
 685	case BPF_XOR | BPF_FETCH:
 686		ctx->ex_insn_off = ctx->ninsns;
 687		emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) :
 688		     rv_amoxor_w(rs, rs, rd, 1, 1), ctx);
 689		ctx->ex_jmp_off = ctx->ninsns;
 690		if (!is64)
 691			emit_zextw(rs, rs, ctx);
 692		break;
 693	/* src_reg = atomic_xchg(dst_reg + off16, src_reg); */
 694	case BPF_XCHG:
 695		ctx->ex_insn_off = ctx->ninsns;
 696		emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) :
 697		     rv_amoswap_w(rs, rs, rd, 1, 1), ctx);
 698		ctx->ex_jmp_off = ctx->ninsns;
 699		if (!is64)
 700			emit_zextw(rs, rs, ctx);
 701		break;
 702	/* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
 703	case BPF_CMPXCHG:
 704		emit_cmpxchg(rd, rs, regmap[BPF_REG_0], is64, ctx);
 705		break;
 706	default:
 707		pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);
 708		return -EINVAL;
 709	}
 710
 711	return 0;
 712}
 713
 714/*
 715 * Sign-extend the register if necessary
 716 */
 717static int sign_extend(u8 rd, u8 rs, u8 sz, bool sign, struct rv_jit_context *ctx)
 718{
 719	if (!sign && (sz == 1 || sz == 2)) {
 720		if (rd != rs)
 721			emit_mv(rd, rs, ctx);
 722		return 0;
 723	}
 724
 725	switch (sz) {
 726	case 1:
 727		emit_sextb(rd, rs, ctx);
 728		break;
 729	case 2:
 730		emit_sexth(rd, rs, ctx);
 731		break;
 732	case 4:
 733		emit_sextw(rd, rs, ctx);
 734		break;
 735	case 8:
 736		if (rd != rs)
 737			emit_mv(rd, rs, ctx);
 738		break;
 739	default:
 740		pr_err("bpf-jit: invalid size %d for sign_extend\n", sz);
 741		return -EINVAL;
 742	}
 743
 744	return 0;
 745}
 746
 747#define BPF_FIXUP_OFFSET_MASK   GENMASK(26, 0)
 748#define BPF_FIXUP_REG_MASK      GENMASK(31, 27)
 749#define REG_DONT_CLEAR_MARKER	0	/* RV_REG_ZERO unused in pt_regmap */
 750
 751bool ex_handler_bpf(const struct exception_table_entry *ex,
 752		    struct pt_regs *regs)
 753{
 754	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
 755	int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
 756
 757	if (regs_offset != REG_DONT_CLEAR_MARKER)
 758		*(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0;
 759	regs->epc = (unsigned long)&ex->fixup - offset;
 760
 761	return true;
 762}
 763
 764/* For accesses to BTF pointers, add an entry to the exception table */
 765static int add_exception_handler(const struct bpf_insn *insn, int dst_reg,
 766				 struct rv_jit_context *ctx)
 767{
 768	struct exception_table_entry *ex;
 769	unsigned long pc;
 770	off_t ins_offset;
 771	off_t fixup_offset;
 772
 773	if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable ||
 774	    ctx->ex_insn_off <= 0 || ctx->ex_jmp_off <= 0)
 775		return 0;
 776
 777	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
 778	    BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
 779	    BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
 780	    BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
 781		return 0;
 782
 783	if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries))
 784		return -EINVAL;
 785
 786	if (WARN_ON_ONCE(ctx->ex_insn_off > ctx->ninsns || ctx->ex_jmp_off > ctx->ninsns))
 787		return -EINVAL;
 788
 789	ex = &ctx->prog->aux->extable[ctx->nexentries];
 790	pc = (unsigned long)&ctx->ro_insns[ctx->ex_insn_off];
 791
 792	/*
 793	 * This is the relative offset of the instruction that may fault from
 794	 * the exception table itself. This will be written to the exception
 795	 * table and if this instruction faults, the destination register will
 796	 * be set to '0' and the execution will jump to the next instruction.
 797	 */
 798	ins_offset = pc - (long)&ex->insn;
 799	if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
 800		return -ERANGE;
 801
 802	/*
 803	 * Since the extable follows the program, the fixup offset is always
 804	 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
 805	 * to keep things simple, and put the destination register in the upper
 806	 * bits. We don't need to worry about buildtime or runtime sort
 807	 * modifying the upper bits because the table is already sorted, and
 808	 * isn't part of the main exception table.
 809	 *
 810	 * The fixup_offset is set to the next instruction from the instruction
 811	 * that may fault. The execution will jump to this after handling the
 812	 * fault.
 813	 */
 814	fixup_offset = (long)&ex->fixup - (long)&ctx->ro_insns[ctx->ex_jmp_off];
 815	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
 816		return -ERANGE;
 817
 818	/*
 819	 * The offsets above have been calculated using the RO buffer but we
 820	 * need to use the R/W buffer for writes.
 821	 * switch ex to rw buffer for writing.
 822	 */
 823	ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns);
 824
 825	ex->insn = ins_offset;
 826
 827	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) |
 828		FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
 829	ex->type = EX_TYPE_BPF;
 830
 831	ctx->ex_insn_off = 0;
 832	ctx->ex_jmp_off = 0;
 833	ctx->nexentries++;
 834	return 0;
 835}
 836
 837static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
 838{
 839	s64 rvoff;
 840	struct rv_jit_context ctx;
 841
 842	ctx.ninsns = 0;
 843	ctx.insns = (u16 *)insns;
 844
 845	if (!target) {
 846		emit(rv_nop(), &ctx);
 847		emit(rv_nop(), &ctx);
 848		return 0;
 849	}
 850
 851	rvoff = (s64)(target - ip);
 852	return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx);
 853}
 854
 855int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 856		       enum bpf_text_poke_type new_t, void *old_addr,
 857		       void *new_addr)
 858{
 859	u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS];
 860	bool is_call;
 861	int ret;
 862
 863	if (!is_kernel_text((unsigned long)ip) &&
 864	    !is_bpf_text_address((unsigned long)ip))
 865		return -ENOTSUPP;
 866
 867	is_call = old_t == BPF_MOD_CALL;
 868	ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call);
 869	if (ret)
 870		return ret;
 871
 872	if (memcmp(ip, old_insns, RV_FENTRY_NBYTES))
 873		return -EFAULT;
 874
 875	is_call = new_t == BPF_MOD_CALL;
 876	ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call);
 877	if (ret)
 878		return ret;
 879
 880	cpus_read_lock();
 881	mutex_lock(&text_mutex);
 882	if (memcmp(ip, new_insns, RV_FENTRY_NBYTES))
 883		ret = patch_text(ip, new_insns, RV_FENTRY_NBYTES);
 884	mutex_unlock(&text_mutex);
 885	cpus_read_unlock();
 886
 887	return ret;
 888}
 889
 890static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx)
 891{
 892	int i;
 893
 894	for (i = 0; i < nr_arg_slots; i++) {
 895		if (i < RV_MAX_REG_ARGS) {
 896			emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx);
 897		} else {
 898			/* skip slots for T0 and FP of traced function */
 899			emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx);
 900			emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx);
 901		}
 902		args_off -= 8;
 903	}
 904}
 905
 906static void restore_args(int nr_reg_args, int args_off, struct rv_jit_context *ctx)
 907{
 908	int i;
 909
 910	for (i = 0; i < nr_reg_args; i++) {
 911		emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx);
 912		args_off -= 8;
 913	}
 914}
 915
 916static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off,
 917			       struct rv_jit_context *ctx)
 918{
 919	int i;
 920
 921	for (i = 0; i < nr_stack_args; i++) {
 922		emit_ld(RV_REG_T1, -(args_off - RV_MAX_REG_ARGS * 8), RV_REG_FP, ctx);
 923		emit_sd(RV_REG_FP, -stk_arg_off, RV_REG_T1, ctx);
 924		args_off -= 8;
 925		stk_arg_off -= 8;
 926	}
 927}
 928
 929static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off,
 930			   int run_ctx_off, bool save_ret, struct rv_jit_context *ctx)
 931{
 932	int ret, branch_off;
 933	struct bpf_prog *p = l->link.prog;
 934	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 935
 936	if (l->cookie) {
 937		emit_imm(RV_REG_T1, l->cookie, ctx);
 938		emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx);
 939	} else {
 940		emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx);
 941	}
 942
 943	/* arg1: prog */
 944	emit_imm(RV_REG_A0, (const s64)p, ctx);
 945	/* arg2: &run_ctx */
 946	emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx);
 947	ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx);
 948	if (ret)
 949		return ret;
 950
 951	/* store prog start time */
 952	emit_mv(RV_REG_S1, RV_REG_A0, ctx);
 953
 954	/* if (__bpf_prog_enter(prog) == 0)
 955	 *	goto skip_exec_of_prog;
 956	 */
 957	branch_off = ctx->ninsns;
 958	/* nop reserved for conditional jump */
 959	emit(rv_nop(), ctx);
 960
 961	/* arg1: &args_off */
 962	emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx);
 963	if (!p->jited)
 964		/* arg2: progs[i]->insnsi for interpreter */
 965		emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx);
 966	ret = emit_call((const u64)p->bpf_func, true, ctx);
 967	if (ret)
 968		return ret;
 969
 970	if (save_ret) {
 971		emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
 972		emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
 973	}
 974
 975	/* update branch with beqz */
 976	if (ctx->insns) {
 977		int offset = ninsns_rvoff(ctx->ninsns - branch_off);
 978		u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1);
 979		*(u32 *)(ctx->insns + branch_off) = insn;
 980	}
 981
 982	/* arg1: prog */
 983	emit_imm(RV_REG_A0, (const s64)p, ctx);
 984	/* arg2: prog start time */
 985	emit_mv(RV_REG_A1, RV_REG_S1, ctx);
 986	/* arg3: &run_ctx */
 987	emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx);
 988	ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx);
 989
 990	return ret;
 991}
 992
 993static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 994					 const struct btf_func_model *m,
 995					 struct bpf_tramp_links *tlinks,
 996					 void *func_addr, u32 flags,
 997					 struct rv_jit_context *ctx)
 998{
 999	int i, ret, offset;
1000	int *branches_off = NULL;
1001	int stack_size = 0, nr_arg_slots = 0;
1002	int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off;
1003	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
1004	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
1005	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
1006	bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
1007	void *orig_call = func_addr;
1008	bool save_ret;
1009	u32 insn;
1010
1011	/* Two types of generated trampoline stack layout:
1012	 *
1013	 * 1. trampoline called from function entry
1014	 * --------------------------------------
1015	 * FP + 8	    [ RA to parent func	] return address to parent
1016	 *					  function
1017	 * FP + 0	    [ FP of parent func ] frame pointer of parent
1018	 *					  function
1019	 * FP - 8           [ T0 to traced func ] return address of traced
1020	 *					  function
1021	 * FP - 16	    [ FP of traced func ] frame pointer of traced
1022	 *					  function
1023	 * --------------------------------------
1024	 *
1025	 * 2. trampoline called directly
1026	 * --------------------------------------
1027	 * FP - 8	    [ RA to caller func ] return address to caller
1028	 *					  function
1029	 * FP - 16	    [ FP of caller func	] frame pointer of caller
1030	 *					  function
1031	 * --------------------------------------
1032	 *
1033	 * FP - retval_off  [ return value      ] BPF_TRAMP_F_CALL_ORIG or
1034	 *					  BPF_TRAMP_F_RET_FENTRY_RET
1035	 *                  [ argN              ]
1036	 *                  [ ...               ]
1037	 * FP - args_off    [ arg1              ]
1038	 *
1039	 * FP - nregs_off   [ regs count        ]
1040	 *
1041	 * FP - ip_off      [ traced func	] BPF_TRAMP_F_IP_ARG
1042	 *
1043	 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
1044	 *
1045	 * FP - sreg_off    [ callee saved reg	]
1046	 *
1047	 *		    [ pads              ] pads for 16 bytes alignment
1048	 *
1049	 *		    [ stack_argN        ]
1050	 *		    [ ...               ]
1051	 * FP - stk_arg_off [ stack_arg1        ] BPF_TRAMP_F_CALL_ORIG
1052	 */
1053
1054	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
1055		return -ENOTSUPP;
1056
1057	if (m->nr_args > MAX_BPF_FUNC_ARGS)
1058		return -ENOTSUPP;
1059
1060	for (i = 0; i < m->nr_args; i++)
1061		nr_arg_slots += round_up(m->arg_size[i], 8) / 8;
1062
1063	/* room of trampoline frame to store return address and frame pointer */
1064	stack_size += 16;
1065
1066	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
1067	if (save_ret)
1068		stack_size += 16; /* Save both A5 (BPF R0) and A0 */
1069	retval_off = stack_size;
1070
1071	stack_size += nr_arg_slots * 8;
1072	args_off = stack_size;
1073
1074	stack_size += 8;
1075	nregs_off = stack_size;
1076
1077	if (flags & BPF_TRAMP_F_IP_ARG) {
1078		stack_size += 8;
1079		ip_off = stack_size;
1080	}
1081
1082	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
1083	run_ctx_off = stack_size;
1084
1085	stack_size += 8;
1086	sreg_off = stack_size;
1087
1088	if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0))
1089		stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8;
1090
1091	stack_size = round_up(stack_size, STACK_ALIGN);
1092
1093	/* room for args on stack must be at the top of stack */
1094	stk_arg_off = stack_size;
1095
1096	if (!is_struct_ops) {
1097		/* For the trampoline called from function entry,
1098		 * the frame of traced function and the frame of
1099		 * trampoline need to be considered.
1100		 */
1101		emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx);
1102		emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx);
1103		emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx);
1104		emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx);
1105
1106		emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx);
1107		emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx);
1108		emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
1109		emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
1110	} else {
1111		/* emit kcfi hash */
1112		emit_kcfi(cfi_get_func_hash(func_addr), ctx);
1113		/* For the trampoline called directly, just handle
1114		 * the frame of trampoline.
1115		 */
1116		emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx);
1117		emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx);
1118		emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx);
1119		emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx);
1120	}
1121
1122	/* callee saved register S1 to pass start time */
1123	emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx);
1124
1125	/* store ip address of the traced function */
1126	if (flags & BPF_TRAMP_F_IP_ARG) {
1127		emit_imm(RV_REG_T1, (const s64)func_addr, ctx);
1128		emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx);
1129	}
1130
1131	emit_li(RV_REG_T1, nr_arg_slots, ctx);
1132	emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx);
1133
1134	store_args(nr_arg_slots, args_off, ctx);
1135
1136	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1137		emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
1138		ret = emit_call((const u64)__bpf_tramp_enter, true, ctx);
1139		if (ret)
1140			return ret;
1141	}
1142
1143	for (i = 0; i < fentry->nr_links; i++) {
1144		ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off,
1145				      flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx);
1146		if (ret)
1147			return ret;
1148	}
1149
1150	if (fmod_ret->nr_links) {
1151		branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL);
1152		if (!branches_off)
1153			return -ENOMEM;
1154
1155		/* cleanup to avoid garbage return value confusion */
1156		emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
1157		for (i = 0; i < fmod_ret->nr_links; i++) {
1158			ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off,
1159					      run_ctx_off, true, ctx);
1160			if (ret)
1161				goto out;
1162			emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx);
1163			branches_off[i] = ctx->ninsns;
1164			/* nop reserved for conditional jump */
1165			emit(rv_nop(), ctx);
1166		}
1167	}
1168
1169	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1170		/* skip to actual body of traced function */
1171		orig_call += RV_FENTRY_NINSNS * 4;
1172		restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
1173		restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx);
1174		ret = emit_call((const u64)orig_call, true, ctx);
1175		if (ret)
1176			goto out;
1177		emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
1178		emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
1179		im->ip_after_call = ctx->ro_insns + ctx->ninsns;
1180		/* 2 nops reserved for auipc+jalr pair */
1181		emit(rv_nop(), ctx);
1182		emit(rv_nop(), ctx);
1183	}
1184
1185	/* update branches saved in invoke_bpf_mod_ret with bnez */
1186	for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) {
1187		offset = ninsns_rvoff(ctx->ninsns - branches_off[i]);
1188		insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1);
1189		*(u32 *)(ctx->insns + branches_off[i]) = insn;
1190	}
1191
1192	for (i = 0; i < fexit->nr_links; i++) {
1193		ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off,
1194				      run_ctx_off, false, ctx);
1195		if (ret)
1196			goto out;
1197	}
1198
1199	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1200		im->ip_epilogue = ctx->ro_insns + ctx->ninsns;
1201		emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
1202		ret = emit_call((const u64)__bpf_tramp_exit, true, ctx);
1203		if (ret)
1204			goto out;
1205	}
1206
1207	if (flags & BPF_TRAMP_F_RESTORE_REGS)
1208		restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
1209
1210	if (save_ret) {
1211		emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx);
1212		if (is_struct_ops) {
1213			ret = sign_extend(RV_REG_A0, regmap[BPF_REG_0], m->ret_size,
1214					  m->ret_flags & BTF_FMODEL_SIGNED_ARG, ctx);
1215			if (ret)
1216				goto out;
1217		} else {
1218			emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx);
1219		}
1220	}
1221
1222	emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx);
1223
1224	if (!is_struct_ops) {
1225		/* trampoline called from function entry */
1226		emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx);
1227		emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);
1228		emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx);
1229
1230		emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx);
1231		emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx);
1232		emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx);
1233
1234		if (flags & BPF_TRAMP_F_SKIP_FRAME)
1235			/* return to parent function */
1236			emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx);
1237		else
1238			/* return to traced function */
1239			emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx);
1240	} else {
1241		/* trampoline called directly */
1242		emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx);
1243		emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx);
1244		emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx);
1245
1246		emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx);
1247	}
1248
1249	ret = ctx->ninsns;
1250out:
1251	kfree(branches_off);
1252	return ret;
1253}
1254
1255int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1256			     struct bpf_tramp_links *tlinks, void *func_addr)
1257{
1258	struct bpf_tramp_image im;
1259	struct rv_jit_context ctx;
1260	int ret;
1261
1262	ctx.ninsns = 0;
1263	ctx.insns = NULL;
1264	ctx.ro_insns = NULL;
1265	ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx);
1266
1267	return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns);
1268}
1269
1270void *arch_alloc_bpf_trampoline(unsigned int size)
1271{
1272	return bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
1273}
1274
1275void arch_free_bpf_trampoline(void *image, unsigned int size)
1276{
1277	bpf_prog_pack_free(image, size);
1278}
1279
1280int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
1281				void *ro_image_end, const struct btf_func_model *m,
1282				u32 flags, struct bpf_tramp_links *tlinks,
1283				void *func_addr)
1284{
1285	int ret;
1286	void *image, *res;
1287	struct rv_jit_context ctx;
1288	u32 size = ro_image_end - ro_image;
1289
1290	image = kvmalloc(size, GFP_KERNEL);
1291	if (!image)
1292		return -ENOMEM;
1293
1294	ctx.ninsns = 0;
1295	ctx.insns = image;
1296	ctx.ro_insns = ro_image;
1297	ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
1298	if (ret < 0)
1299		goto out;
1300
1301	if (WARN_ON(size < ninsns_rvoff(ctx.ninsns))) {
1302		ret = -E2BIG;
1303		goto out;
1304	}
1305
1306	res = bpf_arch_text_copy(ro_image, image, size);
1307	if (IS_ERR(res)) {
1308		ret = PTR_ERR(res);
1309		goto out;
1310	}
1311
1312out:
1313	kvfree(image);
1314	return ret < 0 ? ret : size;
1315}
1316
1317int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1318		      bool extra_pass)
1319{
1320	bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
1321		    BPF_CLASS(insn->code) == BPF_JMP;
1322	int s, e, rvoff, ret, i = insn - ctx->prog->insnsi;
1323	struct bpf_prog_aux *aux = ctx->prog->aux;
1324	u8 rd = -1, rs = -1, code = insn->code;
1325	s16 off = insn->off;
1326	s32 imm = insn->imm;
1327
1328	init_regs(&rd, &rs, insn, ctx);
1329
1330	switch (code) {
1331	/* dst = src */
1332	case BPF_ALU | BPF_MOV | BPF_X:
1333	case BPF_ALU64 | BPF_MOV | BPF_X:
1334		if (insn_is_cast_user(insn)) {
1335			emit_mv(RV_REG_T1, rs, ctx);
1336			emit_zextw(RV_REG_T1, RV_REG_T1, ctx);
1337			emit_imm(rd, (ctx->user_vm_start >> 32) << 32, ctx);
1338			emit(rv_beq(RV_REG_T1, RV_REG_ZERO, 4), ctx);
1339			emit_or(RV_REG_T1, rd, RV_REG_T1, ctx);
1340			emit_mv(rd, RV_REG_T1, ctx);
1341			break;
1342		} else if (insn_is_mov_percpu_addr(insn)) {
1343			if (rd != rs)
1344				emit_mv(rd, rs, ctx);
1345#ifdef CONFIG_SMP
1346			/* Load current CPU number in T1 */
1347			emit_lw(RV_REG_T1, offsetof(struct thread_info, cpu),
1348				RV_REG_TP, ctx);
1349			/* Load address of __per_cpu_offset array in T2 */
1350			emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx);
1351			/* Get address of __per_cpu_offset[cpu] in T1 */
1352			emit_sh3add(RV_REG_T1, RV_REG_T1, RV_REG_T2, ctx);
1353			/* Load __per_cpu_offset[cpu] in T1 */
1354			emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx);
1355			/* Add the offset to Rd */
1356			emit_add(rd, rd, RV_REG_T1, ctx);
1357#endif
1358		}
1359		if (imm == 1) {
1360			/* Special mov32 for zext */
1361			emit_zextw(rd, rd, ctx);
1362			break;
1363		}
1364		switch (insn->off) {
1365		case 0:
1366			emit_mv(rd, rs, ctx);
1367			break;
1368		case 8:
1369			emit_sextb(rd, rs, ctx);
1370			break;
1371		case 16:
1372			emit_sexth(rd, rs, ctx);
1373			break;
1374		case 32:
1375			emit_sextw(rd, rs, ctx);
1376			break;
1377		}
1378		if (!is64 && !aux->verifier_zext)
1379			emit_zextw(rd, rd, ctx);
1380		break;
1381
1382	/* dst = dst OP src */
1383	case BPF_ALU | BPF_ADD | BPF_X:
1384	case BPF_ALU64 | BPF_ADD | BPF_X:
1385		emit_add(rd, rd, rs, ctx);
1386		if (!is64 && !aux->verifier_zext)
1387			emit_zextw(rd, rd, ctx);
1388		break;
1389	case BPF_ALU | BPF_SUB | BPF_X:
1390	case BPF_ALU64 | BPF_SUB | BPF_X:
1391		if (is64)
1392			emit_sub(rd, rd, rs, ctx);
1393		else
1394			emit_subw(rd, rd, rs, ctx);
1395
1396		if (!is64 && !aux->verifier_zext)
1397			emit_zextw(rd, rd, ctx);
1398		break;
1399	case BPF_ALU | BPF_AND | BPF_X:
1400	case BPF_ALU64 | BPF_AND | BPF_X:
1401		emit_and(rd, rd, rs, ctx);
1402		if (!is64 && !aux->verifier_zext)
1403			emit_zextw(rd, rd, ctx);
1404		break;
1405	case BPF_ALU | BPF_OR | BPF_X:
1406	case BPF_ALU64 | BPF_OR | BPF_X:
1407		emit_or(rd, rd, rs, ctx);
1408		if (!is64 && !aux->verifier_zext)
1409			emit_zextw(rd, rd, ctx);
1410		break;
1411	case BPF_ALU | BPF_XOR | BPF_X:
1412	case BPF_ALU64 | BPF_XOR | BPF_X:
1413		emit_xor(rd, rd, rs, ctx);
1414		if (!is64 && !aux->verifier_zext)
1415			emit_zextw(rd, rd, ctx);
1416		break;
1417	case BPF_ALU | BPF_MUL | BPF_X:
1418	case BPF_ALU64 | BPF_MUL | BPF_X:
1419		emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
1420		if (!is64 && !aux->verifier_zext)
1421			emit_zextw(rd, rd, ctx);
1422		break;
1423	case BPF_ALU | BPF_DIV | BPF_X:
1424	case BPF_ALU64 | BPF_DIV | BPF_X:
1425		if (off)
1426			emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx);
1427		else
1428			emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
1429		if (!is64 && !aux->verifier_zext)
1430			emit_zextw(rd, rd, ctx);
1431		break;
1432	case BPF_ALU | BPF_MOD | BPF_X:
1433	case BPF_ALU64 | BPF_MOD | BPF_X:
1434		if (off)
1435			emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx);
1436		else
1437			emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
1438		if (!is64 && !aux->verifier_zext)
1439			emit_zextw(rd, rd, ctx);
1440		break;
1441	case BPF_ALU | BPF_LSH | BPF_X:
1442	case BPF_ALU64 | BPF_LSH | BPF_X:
1443		emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
1444		if (!is64 && !aux->verifier_zext)
1445			emit_zextw(rd, rd, ctx);
1446		break;
1447	case BPF_ALU | BPF_RSH | BPF_X:
1448	case BPF_ALU64 | BPF_RSH | BPF_X:
1449		emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
1450		if (!is64 && !aux->verifier_zext)
1451			emit_zextw(rd, rd, ctx);
1452		break;
1453	case BPF_ALU | BPF_ARSH | BPF_X:
1454	case BPF_ALU64 | BPF_ARSH | BPF_X:
1455		emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
1456		if (!is64 && !aux->verifier_zext)
1457			emit_zextw(rd, rd, ctx);
1458		break;
1459
1460	/* dst = -dst */
1461	case BPF_ALU | BPF_NEG:
1462	case BPF_ALU64 | BPF_NEG:
1463		emit_sub(rd, RV_REG_ZERO, rd, ctx);
1464		if (!is64 && !aux->verifier_zext)
1465			emit_zextw(rd, rd, ctx);
1466		break;
1467
1468	/* dst = BSWAP##imm(dst) */
1469	case BPF_ALU | BPF_END | BPF_FROM_LE:
1470		switch (imm) {
1471		case 16:
1472			emit_zexth(rd, rd, ctx);
1473			break;
1474		case 32:
1475			if (!aux->verifier_zext)
1476				emit_zextw(rd, rd, ctx);
1477			break;
1478		case 64:
1479			/* Do nothing */
1480			break;
1481		}
1482		break;
1483	case BPF_ALU | BPF_END | BPF_FROM_BE:
1484	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
1485		emit_bswap(rd, imm, ctx);
1486		break;
1487
1488	/* dst = imm */
1489	case BPF_ALU | BPF_MOV | BPF_K:
1490	case BPF_ALU64 | BPF_MOV | BPF_K:
1491		emit_imm(rd, imm, ctx);
1492		if (!is64 && !aux->verifier_zext)
1493			emit_zextw(rd, rd, ctx);
1494		break;
1495
1496	/* dst = dst OP imm */
1497	case BPF_ALU | BPF_ADD | BPF_K:
1498	case BPF_ALU64 | BPF_ADD | BPF_K:
1499		if (is_12b_int(imm)) {
1500			emit_addi(rd, rd, imm, ctx);
1501		} else {
1502			emit_imm(RV_REG_T1, imm, ctx);
1503			emit_add(rd, rd, RV_REG_T1, ctx);
1504		}
1505		if (!is64 && !aux->verifier_zext)
1506			emit_zextw(rd, rd, ctx);
1507		break;
1508	case BPF_ALU | BPF_SUB | BPF_K:
1509	case BPF_ALU64 | BPF_SUB | BPF_K:
1510		if (is_12b_int(-imm)) {
1511			emit_addi(rd, rd, -imm, ctx);
1512		} else {
1513			emit_imm(RV_REG_T1, imm, ctx);
1514			emit_sub(rd, rd, RV_REG_T1, ctx);
1515		}
1516		if (!is64 && !aux->verifier_zext)
1517			emit_zextw(rd, rd, ctx);
1518		break;
1519	case BPF_ALU | BPF_AND | BPF_K:
1520	case BPF_ALU64 | BPF_AND | BPF_K:
1521		if (is_12b_int(imm)) {
1522			emit_andi(rd, rd, imm, ctx);
1523		} else {
1524			emit_imm(RV_REG_T1, imm, ctx);
1525			emit_and(rd, rd, RV_REG_T1, ctx);
1526		}
1527		if (!is64 && !aux->verifier_zext)
1528			emit_zextw(rd, rd, ctx);
1529		break;
1530	case BPF_ALU | BPF_OR | BPF_K:
1531	case BPF_ALU64 | BPF_OR | BPF_K:
1532		if (is_12b_int(imm)) {
1533			emit(rv_ori(rd, rd, imm), ctx);
1534		} else {
1535			emit_imm(RV_REG_T1, imm, ctx);
1536			emit_or(rd, rd, RV_REG_T1, ctx);
1537		}
1538		if (!is64 && !aux->verifier_zext)
1539			emit_zextw(rd, rd, ctx);
1540		break;
1541	case BPF_ALU | BPF_XOR | BPF_K:
1542	case BPF_ALU64 | BPF_XOR | BPF_K:
1543		if (is_12b_int(imm)) {
1544			emit(rv_xori(rd, rd, imm), ctx);
1545		} else {
1546			emit_imm(RV_REG_T1, imm, ctx);
1547			emit_xor(rd, rd, RV_REG_T1, ctx);
1548		}
1549		if (!is64 && !aux->verifier_zext)
1550			emit_zextw(rd, rd, ctx);
1551		break;
1552	case BPF_ALU | BPF_MUL | BPF_K:
1553	case BPF_ALU64 | BPF_MUL | BPF_K:
1554		emit_imm(RV_REG_T1, imm, ctx);
1555		emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
1556		     rv_mulw(rd, rd, RV_REG_T1), ctx);
1557		if (!is64 && !aux->verifier_zext)
1558			emit_zextw(rd, rd, ctx);
1559		break;
1560	case BPF_ALU | BPF_DIV | BPF_K:
1561	case BPF_ALU64 | BPF_DIV | BPF_K:
1562		emit_imm(RV_REG_T1, imm, ctx);
1563		if (off)
1564			emit(is64 ? rv_div(rd, rd, RV_REG_T1) :
1565			     rv_divw(rd, rd, RV_REG_T1), ctx);
1566		else
1567			emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
1568			     rv_divuw(rd, rd, RV_REG_T1), ctx);
1569		if (!is64 && !aux->verifier_zext)
1570			emit_zextw(rd, rd, ctx);
1571		break;
1572	case BPF_ALU | BPF_MOD | BPF_K:
1573	case BPF_ALU64 | BPF_MOD | BPF_K:
1574		emit_imm(RV_REG_T1, imm, ctx);
1575		if (off)
1576			emit(is64 ? rv_rem(rd, rd, RV_REG_T1) :
1577			     rv_remw(rd, rd, RV_REG_T1), ctx);
1578		else
1579			emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
1580			     rv_remuw(rd, rd, RV_REG_T1), ctx);
1581		if (!is64 && !aux->verifier_zext)
1582			emit_zextw(rd, rd, ctx);
1583		break;
1584	case BPF_ALU | BPF_LSH | BPF_K:
1585	case BPF_ALU64 | BPF_LSH | BPF_K:
1586		emit_slli(rd, rd, imm, ctx);
1587
1588		if (!is64 && !aux->verifier_zext)
1589			emit_zextw(rd, rd, ctx);
1590		break;
1591	case BPF_ALU | BPF_RSH | BPF_K:
1592	case BPF_ALU64 | BPF_RSH | BPF_K:
1593		if (is64)
1594			emit_srli(rd, rd, imm, ctx);
1595		else
1596			emit(rv_srliw(rd, rd, imm), ctx);
1597
1598		if (!is64 && !aux->verifier_zext)
1599			emit_zextw(rd, rd, ctx);
1600		break;
1601	case BPF_ALU | BPF_ARSH | BPF_K:
1602	case BPF_ALU64 | BPF_ARSH | BPF_K:
1603		if (is64)
1604			emit_srai(rd, rd, imm, ctx);
1605		else
1606			emit(rv_sraiw(rd, rd, imm), ctx);
1607
1608		if (!is64 && !aux->verifier_zext)
1609			emit_zextw(rd, rd, ctx);
1610		break;
1611
1612	/* JUMP off */
1613	case BPF_JMP | BPF_JA:
1614	case BPF_JMP32 | BPF_JA:
1615		if (BPF_CLASS(code) == BPF_JMP)
1616			rvoff = rv_offset(i, off, ctx);
1617		else
1618			rvoff = rv_offset(i, imm, ctx);
1619		ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
1620		if (ret)
1621			return ret;
1622		break;
1623
1624	/* IF (dst COND src) JUMP off */
1625	case BPF_JMP | BPF_JEQ | BPF_X:
1626	case BPF_JMP32 | BPF_JEQ | BPF_X:
1627	case BPF_JMP | BPF_JGT | BPF_X:
1628	case BPF_JMP32 | BPF_JGT | BPF_X:
1629	case BPF_JMP | BPF_JLT | BPF_X:
1630	case BPF_JMP32 | BPF_JLT | BPF_X:
1631	case BPF_JMP | BPF_JGE | BPF_X:
1632	case BPF_JMP32 | BPF_JGE | BPF_X:
1633	case BPF_JMP | BPF_JLE | BPF_X:
1634	case BPF_JMP32 | BPF_JLE | BPF_X:
1635	case BPF_JMP | BPF_JNE | BPF_X:
1636	case BPF_JMP32 | BPF_JNE | BPF_X:
1637	case BPF_JMP | BPF_JSGT | BPF_X:
1638	case BPF_JMP32 | BPF_JSGT | BPF_X:
1639	case BPF_JMP | BPF_JSLT | BPF_X:
1640	case BPF_JMP32 | BPF_JSLT | BPF_X:
1641	case BPF_JMP | BPF_JSGE | BPF_X:
1642	case BPF_JMP32 | BPF_JSGE | BPF_X:
1643	case BPF_JMP | BPF_JSLE | BPF_X:
1644	case BPF_JMP32 | BPF_JSLE | BPF_X:
1645	case BPF_JMP | BPF_JSET | BPF_X:
1646	case BPF_JMP32 | BPF_JSET | BPF_X:
1647		rvoff = rv_offset(i, off, ctx);
1648		if (!is64) {
1649			s = ctx->ninsns;
1650			if (is_signed_bpf_cond(BPF_OP(code))) {
1651				emit_sextw_alt(&rs, RV_REG_T1, ctx);
1652				emit_sextw_alt(&rd, RV_REG_T2, ctx);
1653			} else {
1654				emit_zextw_alt(&rs, RV_REG_T1, ctx);
1655				emit_zextw_alt(&rd, RV_REG_T2, ctx);
1656			}
1657			e = ctx->ninsns;
1658
1659			/* Adjust for extra insns */
1660			rvoff -= ninsns_rvoff(e - s);
1661		}
1662
1663		if (BPF_OP(code) == BPF_JSET) {
1664			/* Adjust for and */
1665			rvoff -= 4;
1666			emit_and(RV_REG_T1, rd, rs, ctx);
1667			emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);
1668		} else {
1669			emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
1670		}
1671		break;
1672
1673	/* IF (dst COND imm) JUMP off */
1674	case BPF_JMP | BPF_JEQ | BPF_K:
1675	case BPF_JMP32 | BPF_JEQ | BPF_K:
1676	case BPF_JMP | BPF_JGT | BPF_K:
1677	case BPF_JMP32 | BPF_JGT | BPF_K:
1678	case BPF_JMP | BPF_JLT | BPF_K:
1679	case BPF_JMP32 | BPF_JLT | BPF_K:
1680	case BPF_JMP | BPF_JGE | BPF_K:
1681	case BPF_JMP32 | BPF_JGE | BPF_K:
1682	case BPF_JMP | BPF_JLE | BPF_K:
1683	case BPF_JMP32 | BPF_JLE | BPF_K:
1684	case BPF_JMP | BPF_JNE | BPF_K:
1685	case BPF_JMP32 | BPF_JNE | BPF_K:
1686	case BPF_JMP | BPF_JSGT | BPF_K:
1687	case BPF_JMP32 | BPF_JSGT | BPF_K:
1688	case BPF_JMP | BPF_JSLT | BPF_K:
1689	case BPF_JMP32 | BPF_JSLT | BPF_K:
1690	case BPF_JMP | BPF_JSGE | BPF_K:
1691	case BPF_JMP32 | BPF_JSGE | BPF_K:
1692	case BPF_JMP | BPF_JSLE | BPF_K:
1693	case BPF_JMP32 | BPF_JSLE | BPF_K:
1694		rvoff = rv_offset(i, off, ctx);
1695		s = ctx->ninsns;
1696		if (imm)
1697			emit_imm(RV_REG_T1, imm, ctx);
1698		rs = imm ? RV_REG_T1 : RV_REG_ZERO;
1699		if (!is64) {
1700			if (is_signed_bpf_cond(BPF_OP(code))) {
1701				emit_sextw_alt(&rd, RV_REG_T2, ctx);
1702				/* rs has been sign extended */
1703			} else {
1704				emit_zextw_alt(&rd, RV_REG_T2, ctx);
1705				if (imm)
1706					emit_zextw(rs, rs, ctx);
1707			}
1708		}
1709		e = ctx->ninsns;
1710
1711		/* Adjust for extra insns */
1712		rvoff -= ninsns_rvoff(e - s);
1713		emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
1714		break;
1715
1716	case BPF_JMP | BPF_JSET | BPF_K:
1717	case BPF_JMP32 | BPF_JSET | BPF_K:
1718		rvoff = rv_offset(i, off, ctx);
1719		s = ctx->ninsns;
1720		if (is_12b_int(imm)) {
1721			emit_andi(RV_REG_T1, rd, imm, ctx);
1722		} else {
1723			emit_imm(RV_REG_T1, imm, ctx);
1724			emit_and(RV_REG_T1, rd, RV_REG_T1, ctx);
1725		}
1726		/* For jset32, we should clear the upper 32 bits of t1, but
1727		 * sign-extension is sufficient here and saves one instruction,
1728		 * as t1 is used only in comparison against zero.
1729		 */
1730		if (!is64 && imm < 0)
1731			emit_sextw(RV_REG_T1, RV_REG_T1, ctx);
1732		e = ctx->ninsns;
1733		rvoff -= ninsns_rvoff(e - s);
1734		emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx);
1735		break;
1736
1737	/* function call */
1738	case BPF_JMP | BPF_CALL:
1739	{
1740		bool fixed_addr;
1741		u64 addr;
1742
1743		/* Inline calls to bpf_get_smp_processor_id()
1744		 *
1745		 * RV_REG_TP holds the address of the current CPU's task_struct and thread_info is
1746		 * at offset 0 in task_struct.
1747		 * Load cpu from thread_info:
1748		 *     Set R0 to ((struct thread_info *)(RV_REG_TP))->cpu
1749		 *
1750		 * This replicates the implementation of raw_smp_processor_id() on RISCV
1751		 */
1752		if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) {
1753			/* Load current CPU number in R0 */
1754			emit_lw(bpf_to_rv_reg(BPF_REG_0, ctx), offsetof(struct thread_info, cpu),
1755				RV_REG_TP, ctx);
1756			break;
1757		}
1758
1759		mark_call(ctx);
1760		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
1761					    &addr, &fixed_addr);
1762		if (ret < 0)
1763			return ret;
1764
1765		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
1766			const struct btf_func_model *fm;
1767			int idx;
1768
1769			fm = bpf_jit_find_kfunc_model(ctx->prog, insn);
1770			if (!fm)
1771				return -EINVAL;
1772
1773			for (idx = 0; idx < fm->nr_args; idx++) {
1774				u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx);
1775
1776				if (fm->arg_size[idx] == sizeof(int))
1777					emit_sextw(reg, reg, ctx);
1778			}
1779		}
1780
1781		ret = emit_call(addr, fixed_addr, ctx);
1782		if (ret)
1783			return ret;
1784
1785		if (insn->src_reg != BPF_PSEUDO_CALL)
1786			emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx);
1787		break;
1788	}
1789	/* tail call */
1790	case BPF_JMP | BPF_TAIL_CALL:
1791		if (emit_bpf_tail_call(i, ctx))
1792			return -1;
1793		break;
1794
1795	/* function return */
1796	case BPF_JMP | BPF_EXIT:
1797		if (i == ctx->prog->len - 1)
1798			break;
1799
1800		rvoff = epilogue_offset(ctx);
1801		ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
1802		if (ret)
1803			return ret;
1804		break;
1805
1806	/* dst = imm64 */
1807	case BPF_LD | BPF_IMM | BPF_DW:
1808	{
1809		struct bpf_insn insn1 = insn[1];
1810		u64 imm64;
1811
1812		imm64 = (u64)insn1.imm << 32 | (u32)imm;
1813		if (bpf_pseudo_func(insn)) {
1814			/* fixed-length insns for extra jit pass */
1815			ret = emit_addr(rd, imm64, extra_pass, ctx);
1816			if (ret)
1817				return ret;
1818		} else {
1819			emit_imm(rd, imm64, ctx);
1820		}
1821
1822		return 1;
1823	}
1824
1825	/* LDX: dst = *(unsigned size *)(src + off) */
1826	case BPF_LDX | BPF_MEM | BPF_B:
1827	case BPF_LDX | BPF_MEM | BPF_H:
1828	case BPF_LDX | BPF_MEM | BPF_W:
1829	case BPF_LDX | BPF_MEM | BPF_DW:
1830	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1831	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1832	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1833	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1834	/* LDSX: dst = *(signed size *)(src + off) */
1835	case BPF_LDX | BPF_MEMSX | BPF_B:
1836	case BPF_LDX | BPF_MEMSX | BPF_H:
1837	case BPF_LDX | BPF_MEMSX | BPF_W:
1838	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1839	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1840	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1841	/* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + RV_REG_ARENA + off) */
1842	case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
1843	case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
1844	case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
1845	case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
1846	{
1847		bool sign_ext;
1848
1849		sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
1850			   BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
1851
1852		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
1853			emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx);
1854			rs = RV_REG_T2;
1855		}
1856
1857		emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
1858
1859		ret = add_exception_handler(insn, rd, ctx);
1860		if (ret)
1861			return ret;
1862
1863		if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1]))
1864			return 1;
1865		break;
1866	}
1867
1868	/* speculation barrier */
1869	case BPF_ST | BPF_NOSPEC:
1870		break;
1871
1872	/* ST: *(size *)(dst + off) = imm */
1873	case BPF_ST | BPF_MEM | BPF_B:
1874	case BPF_ST | BPF_MEM | BPF_H:
1875	case BPF_ST | BPF_MEM | BPF_W:
1876	case BPF_ST | BPF_MEM | BPF_DW:
1877	/* ST | PROBE_MEM32: *(size *)(dst + RV_REG_ARENA + off) = imm */
1878	case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
1879	case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
1880	case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
1881	case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
1882		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
1883			emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx);
1884			rd = RV_REG_T3;
1885		}
1886
1887		emit_st(rd, off, imm, BPF_SIZE(code), ctx);
1888
1889		ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
1890		if (ret)
1891			return ret;
1892		break;
1893
1894	/* STX: *(size *)(dst + off) = src */
1895	case BPF_STX | BPF_MEM | BPF_B:
1896	case BPF_STX | BPF_MEM | BPF_H:
1897	case BPF_STX | BPF_MEM | BPF_W:
1898	case BPF_STX | BPF_MEM | BPF_DW:
1899	/* STX | PROBE_MEM32: *(size *)(dst + RV_REG_ARENA + off) = src */
1900	case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
1901	case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
1902	case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
1903	case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
1904		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
1905			emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx);
1906			rd = RV_REG_T2;
1907		}
1908
1909		emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
1910
1911		ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
1912		if (ret)
1913			return ret;
1914		break;
1915
1916	/* Atomics */
1917	case BPF_STX | BPF_ATOMIC | BPF_B:
1918	case BPF_STX | BPF_ATOMIC | BPF_H:
1919	case BPF_STX | BPF_ATOMIC | BPF_W:
1920	case BPF_STX | BPF_ATOMIC | BPF_DW:
1921	case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
1922	case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
1923	case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
1924	case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
1925		if (bpf_atomic_is_load_store(insn))
1926			ret = emit_atomic_ld_st(rd, rs, insn, ctx);
1927		else
1928			ret = emit_atomic_rmw(rd, rs, insn, ctx);
1929
1930		ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
1931		if (ret)
1932			return ret;
1933		break;
1934
1935	default:
1936		pr_err("bpf-jit: unknown opcode %02x\n", code);
1937		return -EINVAL;
1938	}
1939
1940	return 0;
1941}
1942
1943void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
1944{
1945	int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
1946
1947	bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
1948	if (bpf_stack_adjust)
1949		mark_fp(ctx);
1950
1951	if (seen_reg(RV_REG_RA, ctx))
1952		stack_adjust += 8;
1953	stack_adjust += 8; /* RV_REG_FP */
1954	if (seen_reg(RV_REG_S1, ctx))
1955		stack_adjust += 8;
1956	if (seen_reg(RV_REG_S2, ctx))
1957		stack_adjust += 8;
1958	if (seen_reg(RV_REG_S3, ctx))
1959		stack_adjust += 8;
1960	if (seen_reg(RV_REG_S4, ctx))
1961		stack_adjust += 8;
1962	if (seen_reg(RV_REG_S5, ctx))
1963		stack_adjust += 8;
1964	if (seen_reg(RV_REG_S6, ctx))
1965		stack_adjust += 8;
1966	if (ctx->arena_vm_start)
1967		stack_adjust += 8;
1968
1969	stack_adjust = round_up(stack_adjust, STACK_ALIGN);
1970	stack_adjust += bpf_stack_adjust;
1971
1972	store_offset = stack_adjust - 8;
1973
1974	/* emit kcfi type preamble immediately before the  first insn */
1975	emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx);
1976
1977	/* nops reserved for auipc+jalr pair */
1978	for (i = 0; i < RV_FENTRY_NINSNS; i++)
1979		emit(rv_nop(), ctx);
1980
1981	/* First instruction is always setting the tail-call-counter
1982	 * (TCC) register. This instruction is skipped for tail calls.
1983	 * Force using a 4-byte (non-compressed) instruction.
1984	 */
1985	emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1986
1987	emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx);
1988
1989	if (seen_reg(RV_REG_RA, ctx)) {
1990		emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx);
1991		store_offset -= 8;
1992	}
1993	emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx);
1994	store_offset -= 8;
1995	if (seen_reg(RV_REG_S1, ctx)) {
1996		emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx);
1997		store_offset -= 8;
1998	}
1999	if (seen_reg(RV_REG_S2, ctx)) {
2000		emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx);
2001		store_offset -= 8;
2002	}
2003	if (seen_reg(RV_REG_S3, ctx)) {
2004		emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx);
2005		store_offset -= 8;
2006	}
2007	if (seen_reg(RV_REG_S4, ctx)) {
2008		emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx);
2009		store_offset -= 8;
2010	}
2011	if (seen_reg(RV_REG_S5, ctx)) {
2012		emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx);
2013		store_offset -= 8;
2014	}
2015	if (seen_reg(RV_REG_S6, ctx)) {
2016		emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx);
2017		store_offset -= 8;
2018	}
2019	if (ctx->arena_vm_start) {
2020		emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx);
2021		store_offset -= 8;
2022	}
2023
2024	emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx);
2025
2026	if (bpf_stack_adjust)
2027		emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx);
2028
2029	/* Program contains calls and tail calls, so RV_REG_TCC need
2030	 * to be saved across calls.
2031	 */
2032	if (seen_tail_call(ctx) && seen_call(ctx))
2033		emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx);
2034
2035	ctx->stack_size = stack_adjust;
2036
2037	if (ctx->arena_vm_start)
2038		emit_imm(RV_REG_ARENA, ctx->arena_vm_start, ctx);
2039}
2040
2041void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
2042{
2043	__build_epilogue(false, ctx);
2044}
2045
2046bool bpf_jit_supports_kfunc_call(void)
2047{
2048	return true;
2049}
2050
2051bool bpf_jit_supports_ptr_xchg(void)
2052{
2053	return true;
2054}
2055
2056bool bpf_jit_supports_arena(void)
2057{
2058	return true;
2059}
2060
2061bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
2062{
2063	if (in_arena) {
2064		switch (insn->code) {
2065		case BPF_STX | BPF_ATOMIC | BPF_W:
2066		case BPF_STX | BPF_ATOMIC | BPF_DW:
2067			if (insn->imm == BPF_CMPXCHG)
2068				return rv_ext_enabled(ZACAS);
2069			break;
2070		case BPF_LDX | BPF_MEMSX | BPF_B:
2071		case BPF_LDX | BPF_MEMSX | BPF_H:
2072		case BPF_LDX | BPF_MEMSX | BPF_W:
2073			return false;
2074		}
2075	}
2076
2077	return true;
2078}
2079
2080bool bpf_jit_supports_percpu_insn(void)
2081{
2082	return true;
2083}
2084
2085bool bpf_jit_inlines_helper_call(s32 imm)
2086{
2087	switch (imm) {
2088	case BPF_FUNC_get_smp_processor_id:
2089		return true;
2090	default:
2091		return false;
2092	}
2093}