bpf_jit.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * BPF JIT compiler for LoongArch
   4 *
   5 * Copyright (C) 2022 Loongson Technology Corporation Limited
   6 */
   7#include <linux/memory.h>
   8#include "bpf_jit.h"
   9
  10#define LOONGARCH_MAX_REG_ARGS 8
  11
  12#define LOONGARCH_LONG_JUMP_NINSNS 5
  13#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
  14
  15#define LOONGARCH_FENTRY_NINSNS 2
  16#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
  17#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
  18
  19#define REG_TCC		LOONGARCH_GPR_A6
  20#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80)
  21
  22static const int regmap[] = {
  23	/* return value from in-kernel function, and exit value for eBPF program */
  24	[BPF_REG_0] = LOONGARCH_GPR_A5,
  25	/* arguments from eBPF program to in-kernel function */
  26	[BPF_REG_1] = LOONGARCH_GPR_A0,
  27	[BPF_REG_2] = LOONGARCH_GPR_A1,
  28	[BPF_REG_3] = LOONGARCH_GPR_A2,
  29	[BPF_REG_4] = LOONGARCH_GPR_A3,
  30	[BPF_REG_5] = LOONGARCH_GPR_A4,
  31	/* callee saved registers that in-kernel function will preserve */
  32	[BPF_REG_6] = LOONGARCH_GPR_S0,
  33	[BPF_REG_7] = LOONGARCH_GPR_S1,
  34	[BPF_REG_8] = LOONGARCH_GPR_S2,
  35	[BPF_REG_9] = LOONGARCH_GPR_S3,
  36	/* read-only frame pointer to access stack */
  37	[BPF_REG_FP] = LOONGARCH_GPR_S4,
  38	/* temporary register for blinding constants */
  39	[BPF_REG_AX] = LOONGARCH_GPR_T0,
  40};
  41
  42static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
  43{
  44	const struct bpf_prog *prog = ctx->prog;
  45	const bool is_main_prog = !bpf_is_subprog(prog);
  46
  47	if (is_main_prog) {
  48		/*
  49		 * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
  50		 * if (REG_TCC > T3 )
  51		 *	std REG_TCC -> LOONGARCH_GPR_SP + store_offset
  52		 * else
  53		 *	std REG_TCC -> LOONGARCH_GPR_SP + store_offset
  54		 *	REG_TCC = LOONGARCH_GPR_SP + store_offset
  55		 *
  56		 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset
  57		 *
  58		 * The purpose of this code is to first push the TCC into stack,
  59		 * and then push the address of TCC into stack.
  60		 * In cases where bpf2bpf and tailcall are used in combination,
  61		 * the value in REG_TCC may be a count or an address,
  62		 * these two cases need to be judged and handled separately.
  63		 */
  64		emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
  65		*store_offset -= sizeof(long);
  66
  67		emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4);
  68
  69		/*
  70		 * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
  71		 * push tcc into stack
  72		 */
  73		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
  74
  75		/* Push the address of TCC into the REG_TCC */
  76		emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
  77
  78		emit_uncond_jmp(ctx, 2);
  79
  80		/*
  81		 * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
  82		 * push tcc_ptr into stack
  83		 */
  84		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
  85	} else {
  86		*store_offset -= sizeof(long);
  87		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
  88	}
  89
  90	/* Push tcc_ptr into stack */
  91	*store_offset -= sizeof(long);
  92	emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
  93}
  94
  95/*
  96 * eBPF prog stack layout:
  97 *
  98 *                                        high
  99 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
 100 *                            |           $ra           |
 101 *                            +-------------------------+
 102 *                            |           $fp           |
 103 *                            +-------------------------+
 104 *                            |           $s0           |
 105 *                            +-------------------------+
 106 *                            |           $s1           |
 107 *                            +-------------------------+
 108 *                            |           $s2           |
 109 *                            +-------------------------+
 110 *                            |           $s3           |
 111 *                            +-------------------------+
 112 *                            |           $s4           |
 113 *                            +-------------------------+
 114 *                            |           $s5           |
 115 *                            +-------------------------+
 116 *                            |           tcc           |
 117 *                            +-------------------------+
 118 *                            |           tcc_ptr       |
 119 *                            +-------------------------+ <--BPF_REG_FP
 120 *                            |  prog->aux->stack_depth |
 121 *                            |        (optional)       |
 122 * current $sp -------------> +-------------------------+
 123 *                                        low
 124 */
 125static void build_prologue(struct jit_ctx *ctx)
 126{
 127	int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
 128	const struct bpf_prog *prog = ctx->prog;
 129	const bool is_main_prog = !bpf_is_subprog(prog);
 130
 131	bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
 132
 133	/* To store ra, fp, s0, s1, s2, s3, s4, s5 */
 134	stack_adjust += sizeof(long) * 8;
 135
 136	/* To store tcc and tcc_ptr */
 137	stack_adjust += sizeof(long) * 2;
 138
 139	stack_adjust = round_up(stack_adjust, 16);
 140	stack_adjust += bpf_stack_adjust;
 141
 142	move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA);
 143	/* Reserve space for the move_imm + jirl instruction */
 144	for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
 145		emit_insn(ctx, nop);
 146
 147	/*
 148	 * First instruction initializes the tail call count (TCC)
 149	 * register to zero. On tail call we skip this instruction,
 150	 * and the TCC is passed in REG_TCC from the caller.
 151	 */
 152	if (is_main_prog)
 153		emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0);
 154
 155	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
 156
 157	store_offset = stack_adjust - sizeof(long);
 158	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
 159
 160	store_offset -= sizeof(long);
 161	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
 162
 163	store_offset -= sizeof(long);
 164	emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
 165
 166	store_offset -= sizeof(long);
 167	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
 168
 169	store_offset -= sizeof(long);
 170	emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
 171
 172	store_offset -= sizeof(long);
 173	emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
 174
 175	store_offset -= sizeof(long);
 176	emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
 177
 178	store_offset -= sizeof(long);
 179	emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
 180
 181	prepare_bpf_tail_call_cnt(ctx, &store_offset);
 182
 183	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
 184
 185	if (bpf_stack_adjust)
 186		emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
 187
 188	ctx->stack_size = stack_adjust;
 189}
 190
 191static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
 192{
 193	int stack_adjust = ctx->stack_size;
 194	int load_offset;
 195
 196	load_offset = stack_adjust - sizeof(long);
 197	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
 198
 199	load_offset -= sizeof(long);
 200	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
 201
 202	load_offset -= sizeof(long);
 203	emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
 204
 205	load_offset -= sizeof(long);
 206	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
 207
 208	load_offset -= sizeof(long);
 209	emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
 210
 211	load_offset -= sizeof(long);
 212	emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
 213
 214	load_offset -= sizeof(long);
 215	emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
 216
 217	load_offset -= sizeof(long);
 218	emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
 219
 220	/*
 221	 * When push into the stack, follow the order of tcc then tcc_ptr.
 222	 * When pop from the stack, first pop tcc_ptr then followed by tcc.
 223	 */
 224	load_offset -= 2 * sizeof(long);
 225	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
 226
 227	load_offset += sizeof(long);
 228	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
 229
 230	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
 231
 232	if (!is_tail_call) {
 233		/* Set return value */
 234		emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0);
 235		/* Return to the caller */
 236		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
 237	} else {
 238		/*
 239		 * Call the next bpf prog and skip the first instruction
 240		 * of TCC initialization.
 241		 */
 242		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7);
 243	}
 244}
 245
 246static void build_epilogue(struct jit_ctx *ctx)
 247{
 248	__build_epilogue(ctx, false);
 249}
 250
 251bool bpf_jit_supports_kfunc_call(void)
 252{
 253	return true;
 254}
 255
 256bool bpf_jit_supports_far_kfunc_call(void)
 257{
 258	return true;
 259}
 260
 261static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
 262{
 263	int off, tc_ninsn = 0;
 264	int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
 265	u8 a1 = LOONGARCH_GPR_A1;
 266	u8 a2 = LOONGARCH_GPR_A2;
 267	u8 t1 = LOONGARCH_GPR_T1;
 268	u8 t2 = LOONGARCH_GPR_T2;
 269	u8 t3 = LOONGARCH_GPR_T3;
 270	const int idx0 = ctx->idx;
 271
 272#define cur_offset (ctx->idx - idx0)
 273#define jmp_offset (tc_ninsn - (cur_offset))
 274
 275	/*
 276	 * a0: &ctx
 277	 * a1: &array
 278	 * a2: index
 279	 *
 280	 * if (index >= array->map.max_entries)
 281	 *	 goto out;
 282	 */
 283	tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
 284	emit_zext_32(ctx, a2, true);
 285
 286	off = offsetof(struct bpf_array, map.max_entries);
 287	emit_insn(ctx, ldwu, t1, a1, off);
 288	/* bgeu $a2, $t1, jmp_offset */
 289	if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
 290		goto toofar;
 291
 292	/*
 293	 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
 294	 *      goto out;
 295	 */
 296	emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
 297	emit_insn(ctx, ldd, t3, REG_TCC, 0);
 298	emit_insn(ctx, addid, t3, t3, 1);
 299	emit_insn(ctx, std, t3, REG_TCC, 0);
 300	emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
 301	if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0)
 302		goto toofar;
 303
 304	/*
 305	 * prog = array->ptrs[index];
 306	 * if (!prog)
 307	 *	 goto out;
 308	 */
 309	emit_insn(ctx, alsld, t2, a2, a1, 2);
 310	off = offsetof(struct bpf_array, ptrs);
 311	emit_insn(ctx, ldd, t2, t2, off);
 312	/* beq $t2, $zero, jmp_offset */
 313	if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
 314		goto toofar;
 315
 316	/* goto *(prog->bpf_func + 4); */
 317	off = offsetof(struct bpf_prog, bpf_func);
 318	emit_insn(ctx, ldd, t3, t2, off);
 319	__build_epilogue(ctx, true);
 320
 321	return 0;
 322
 323toofar:
 324	pr_info_once("tail_call: jump too far\n");
 325	return -1;
 326#undef cur_offset
 327#undef jmp_offset
 328}
 329
 330static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
 331{
 332	const u8 t1 = LOONGARCH_GPR_T1;
 333	const u8 t2 = LOONGARCH_GPR_T2;
 334	const u8 t3 = LOONGARCH_GPR_T3;
 335	const u8 r0 = regmap[BPF_REG_0];
 336	const u8 src = regmap[insn->src_reg];
 337	const u8 dst = regmap[insn->dst_reg];
 338	const s16 off = insn->off;
 339	const s32 imm = insn->imm;
 340	const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
 341
 342	move_imm(ctx, t1, off, false);
 343	emit_insn(ctx, addd, t1, dst, t1);
 344	move_reg(ctx, t3, src);
 345
 346	switch (imm) {
 347	/* lock *(size *)(dst + off) <op>= src */
 348	case BPF_ADD:
 349		if (isdw)
 350			emit_insn(ctx, amaddd, t2, t1, src);
 351		else
 352			emit_insn(ctx, amaddw, t2, t1, src);
 353		break;
 354	case BPF_AND:
 355		if (isdw)
 356			emit_insn(ctx, amandd, t2, t1, src);
 357		else
 358			emit_insn(ctx, amandw, t2, t1, src);
 359		break;
 360	case BPF_OR:
 361		if (isdw)
 362			emit_insn(ctx, amord, t2, t1, src);
 363		else
 364			emit_insn(ctx, amorw, t2, t1, src);
 365		break;
 366	case BPF_XOR:
 367		if (isdw)
 368			emit_insn(ctx, amxord, t2, t1, src);
 369		else
 370			emit_insn(ctx, amxorw, t2, t1, src);
 371		break;
 372	/* src = atomic_fetch_<op>(dst + off, src) */
 373	case BPF_ADD | BPF_FETCH:
 374		if (isdw) {
 375			emit_insn(ctx, amaddd, src, t1, t3);
 376		} else {
 377			emit_insn(ctx, amaddw, src, t1, t3);
 378			emit_zext_32(ctx, src, true);
 379		}
 380		break;
 381	case BPF_AND | BPF_FETCH:
 382		if (isdw) {
 383			emit_insn(ctx, amandd, src, t1, t3);
 384		} else {
 385			emit_insn(ctx, amandw, src, t1, t3);
 386			emit_zext_32(ctx, src, true);
 387		}
 388		break;
 389	case BPF_OR | BPF_FETCH:
 390		if (isdw) {
 391			emit_insn(ctx, amord, src, t1, t3);
 392		} else {
 393			emit_insn(ctx, amorw, src, t1, t3);
 394			emit_zext_32(ctx, src, true);
 395		}
 396		break;
 397	case BPF_XOR | BPF_FETCH:
 398		if (isdw) {
 399			emit_insn(ctx, amxord, src, t1, t3);
 400		} else {
 401			emit_insn(ctx, amxorw, src, t1, t3);
 402			emit_zext_32(ctx, src, true);
 403		}
 404		break;
 405	/* src = atomic_xchg(dst + off, src); */
 406	case BPF_XCHG:
 407		if (isdw) {
 408			emit_insn(ctx, amswapd, src, t1, t3);
 409		} else {
 410			emit_insn(ctx, amswapw, src, t1, t3);
 411			emit_zext_32(ctx, src, true);
 412		}
 413		break;
 414	/* r0 = atomic_cmpxchg(dst + off, r0, src); */
 415	case BPF_CMPXCHG:
 416		move_reg(ctx, t2, r0);
 417		if (isdw) {
 418			emit_insn(ctx, lld, r0, t1, 0);
 419			emit_insn(ctx, bne, t2, r0, 4);
 420			move_reg(ctx, t3, src);
 421			emit_insn(ctx, scd, t3, t1, 0);
 422			emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
 423		} else {
 424			emit_insn(ctx, llw, r0, t1, 0);
 425			emit_zext_32(ctx, t2, true);
 426			emit_zext_32(ctx, r0, true);
 427			emit_insn(ctx, bne, t2, r0, 4);
 428			move_reg(ctx, t3, src);
 429			emit_insn(ctx, scw, t3, t1, 0);
 430			emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
 431			emit_zext_32(ctx, r0, true);
 432		}
 433		break;
 434	}
 435}
 436
 437static bool is_signed_bpf_cond(u8 cond)
 438{
 439	return cond == BPF_JSGT || cond == BPF_JSLT ||
 440	       cond == BPF_JSGE || cond == BPF_JSLE;
 441}
 442
 443#define BPF_FIXUP_REG_MASK	GENMASK(31, 27)
 444#define BPF_FIXUP_OFFSET_MASK	GENMASK(26, 0)
 445
 446bool ex_handler_bpf(const struct exception_table_entry *ex,
 447		    struct pt_regs *regs)
 448{
 449	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
 450	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
 451
 452	regs->regs[dst_reg] = 0;
 453	regs->csr_era = (unsigned long)&ex->fixup - offset;
 454
 455	return true;
 456}
 457
 458/* For accesses to BTF pointers, add an entry to the exception table */
 459static int add_exception_handler(const struct bpf_insn *insn,
 460				 struct jit_ctx *ctx,
 461				 int dst_reg)
 462{
 463	unsigned long pc;
 464	off_t offset;
 465	struct exception_table_entry *ex;
 466
 467	if (!ctx->image || !ctx->prog->aux->extable)
 468		return 0;
 469
 470	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
 471	    BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
 472		return 0;
 473
 474	if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
 475		return -EINVAL;
 476
 477	ex = &ctx->prog->aux->extable[ctx->num_exentries];
 478	pc = (unsigned long)&ctx->image[ctx->idx - 1];
 479
 480	offset = pc - (long)&ex->insn;
 481	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
 482		return -ERANGE;
 483
 484	ex->insn = offset;
 485
 486	/*
 487	 * Since the extable follows the program, the fixup offset is always
 488	 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
 489	 * to keep things simple, and put the destination register in the upper
 490	 * bits. We don't need to worry about buildtime or runtime sort
 491	 * modifying the upper bits because the table is already sorted, and
 492	 * isn't part of the main exception table.
 493	 */
 494	offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
 495	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
 496		return -ERANGE;
 497
 498	ex->type = EX_TYPE_BPF;
 499	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
 500
 501	ctx->num_exentries++;
 502
 503	return 0;
 504}
 505
 506static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
 507{
 508	u8 tm = -1;
 509	u64 func_addr;
 510	bool func_addr_fixed, sign_extend;
 511	int i = insn - ctx->prog->insnsi;
 512	int ret, jmp_offset, tcc_ptr_off;
 513	const u8 code = insn->code;
 514	const u8 cond = BPF_OP(code);
 515	const u8 t1 = LOONGARCH_GPR_T1;
 516	const u8 t2 = LOONGARCH_GPR_T2;
 517	const u8 src = regmap[insn->src_reg];
 518	const u8 dst = regmap[insn->dst_reg];
 519	const s16 off = insn->off;
 520	const s32 imm = insn->imm;
 521	const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
 522
 523	switch (code) {
 524	/* dst = src */
 525	case BPF_ALU | BPF_MOV | BPF_X:
 526	case BPF_ALU64 | BPF_MOV | BPF_X:
 527		switch (off) {
 528		case 0:
 529			move_reg(ctx, dst, src);
 530			emit_zext_32(ctx, dst, is32);
 531			break;
 532		case 8:
 533			emit_insn(ctx, extwb, dst, src);
 534			emit_zext_32(ctx, dst, is32);
 535			break;
 536		case 16:
 537			emit_insn(ctx, extwh, dst, src);
 538			emit_zext_32(ctx, dst, is32);
 539			break;
 540		case 32:
 541			emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
 542			break;
 543		}
 544		break;
 545
 546	/* dst = imm */
 547	case BPF_ALU | BPF_MOV | BPF_K:
 548	case BPF_ALU64 | BPF_MOV | BPF_K:
 549		move_imm(ctx, dst, imm, is32);
 550		break;
 551
 552	/* dst = dst + src */
 553	case BPF_ALU | BPF_ADD | BPF_X:
 554	case BPF_ALU64 | BPF_ADD | BPF_X:
 555		emit_insn(ctx, addd, dst, dst, src);
 556		emit_zext_32(ctx, dst, is32);
 557		break;
 558
 559	/* dst = dst + imm */
 560	case BPF_ALU | BPF_ADD | BPF_K:
 561	case BPF_ALU64 | BPF_ADD | BPF_K:
 562		if (is_signed_imm12(imm)) {
 563			emit_insn(ctx, addid, dst, dst, imm);
 564		} else {
 565			move_imm(ctx, t1, imm, is32);
 566			emit_insn(ctx, addd, dst, dst, t1);
 567		}
 568		emit_zext_32(ctx, dst, is32);
 569		break;
 570
 571	/* dst = dst - src */
 572	case BPF_ALU | BPF_SUB | BPF_X:
 573	case BPF_ALU64 | BPF_SUB | BPF_X:
 574		emit_insn(ctx, subd, dst, dst, src);
 575		emit_zext_32(ctx, dst, is32);
 576		break;
 577
 578	/* dst = dst - imm */
 579	case BPF_ALU | BPF_SUB | BPF_K:
 580	case BPF_ALU64 | BPF_SUB | BPF_K:
 581		if (is_signed_imm12(-imm)) {
 582			emit_insn(ctx, addid, dst, dst, -imm);
 583		} else {
 584			move_imm(ctx, t1, imm, is32);
 585			emit_insn(ctx, subd, dst, dst, t1);
 586		}
 587		emit_zext_32(ctx, dst, is32);
 588		break;
 589
 590	/* dst = dst * src */
 591	case BPF_ALU | BPF_MUL | BPF_X:
 592	case BPF_ALU64 | BPF_MUL | BPF_X:
 593		emit_insn(ctx, muld, dst, dst, src);
 594		emit_zext_32(ctx, dst, is32);
 595		break;
 596
 597	/* dst = dst * imm */
 598	case BPF_ALU | BPF_MUL | BPF_K:
 599	case BPF_ALU64 | BPF_MUL | BPF_K:
 600		move_imm(ctx, t1, imm, is32);
 601		emit_insn(ctx, muld, dst, dst, t1);
 602		emit_zext_32(ctx, dst, is32);
 603		break;
 604
 605	/* dst = dst / src */
 606	case BPF_ALU | BPF_DIV | BPF_X:
 607	case BPF_ALU64 | BPF_DIV | BPF_X:
 608		if (!off) {
 609			emit_zext_32(ctx, dst, is32);
 610			move_reg(ctx, t1, src);
 611			emit_zext_32(ctx, t1, is32);
 612			emit_insn(ctx, divdu, dst, dst, t1);
 613			emit_zext_32(ctx, dst, is32);
 614		} else {
 615			emit_sext_32(ctx, dst, is32);
 616			move_reg(ctx, t1, src);
 617			emit_sext_32(ctx, t1, is32);
 618			emit_insn(ctx, divd, dst, dst, t1);
 619			emit_sext_32(ctx, dst, is32);
 620		}
 621		break;
 622
 623	/* dst = dst / imm */
 624	case BPF_ALU | BPF_DIV | BPF_K:
 625	case BPF_ALU64 | BPF_DIV | BPF_K:
 626		if (!off) {
 627			move_imm(ctx, t1, imm, is32);
 628			emit_zext_32(ctx, dst, is32);
 629			emit_insn(ctx, divdu, dst, dst, t1);
 630			emit_zext_32(ctx, dst, is32);
 631		} else {
 632			move_imm(ctx, t1, imm, false);
 633			emit_sext_32(ctx, t1, is32);
 634			emit_sext_32(ctx, dst, is32);
 635			emit_insn(ctx, divd, dst, dst, t1);
 636			emit_sext_32(ctx, dst, is32);
 637		}
 638		break;
 639
 640	/* dst = dst % src */
 641	case BPF_ALU | BPF_MOD | BPF_X:
 642	case BPF_ALU64 | BPF_MOD | BPF_X:
 643		if (!off) {
 644			emit_zext_32(ctx, dst, is32);
 645			move_reg(ctx, t1, src);
 646			emit_zext_32(ctx, t1, is32);
 647			emit_insn(ctx, moddu, dst, dst, t1);
 648			emit_zext_32(ctx, dst, is32);
 649		} else {
 650			emit_sext_32(ctx, dst, is32);
 651			move_reg(ctx, t1, src);
 652			emit_sext_32(ctx, t1, is32);
 653			emit_insn(ctx, modd, dst, dst, t1);
 654			emit_sext_32(ctx, dst, is32);
 655		}
 656		break;
 657
 658	/* dst = dst % imm */
 659	case BPF_ALU | BPF_MOD | BPF_K:
 660	case BPF_ALU64 | BPF_MOD | BPF_K:
 661		if (!off) {
 662			move_imm(ctx, t1, imm, is32);
 663			emit_zext_32(ctx, dst, is32);
 664			emit_insn(ctx, moddu, dst, dst, t1);
 665			emit_zext_32(ctx, dst, is32);
 666		} else {
 667			move_imm(ctx, t1, imm, false);
 668			emit_sext_32(ctx, t1, is32);
 669			emit_sext_32(ctx, dst, is32);
 670			emit_insn(ctx, modd, dst, dst, t1);
 671			emit_sext_32(ctx, dst, is32);
 672		}
 673		break;
 674
 675	/* dst = -dst */
 676	case BPF_ALU | BPF_NEG:
 677	case BPF_ALU64 | BPF_NEG:
 678		move_imm(ctx, t1, imm, is32);
 679		emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
 680		emit_zext_32(ctx, dst, is32);
 681		break;
 682
 683	/* dst = dst & src */
 684	case BPF_ALU | BPF_AND | BPF_X:
 685	case BPF_ALU64 | BPF_AND | BPF_X:
 686		emit_insn(ctx, and, dst, dst, src);
 687		emit_zext_32(ctx, dst, is32);
 688		break;
 689
 690	/* dst = dst & imm */
 691	case BPF_ALU | BPF_AND | BPF_K:
 692	case BPF_ALU64 | BPF_AND | BPF_K:
 693		if (is_unsigned_imm12(imm)) {
 694			emit_insn(ctx, andi, dst, dst, imm);
 695		} else {
 696			move_imm(ctx, t1, imm, is32);
 697			emit_insn(ctx, and, dst, dst, t1);
 698		}
 699		emit_zext_32(ctx, dst, is32);
 700		break;
 701
 702	/* dst = dst | src */
 703	case BPF_ALU | BPF_OR | BPF_X:
 704	case BPF_ALU64 | BPF_OR | BPF_X:
 705		emit_insn(ctx, or, dst, dst, src);
 706		emit_zext_32(ctx, dst, is32);
 707		break;
 708
 709	/* dst = dst | imm */
 710	case BPF_ALU | BPF_OR | BPF_K:
 711	case BPF_ALU64 | BPF_OR | BPF_K:
 712		if (is_unsigned_imm12(imm)) {
 713			emit_insn(ctx, ori, dst, dst, imm);
 714		} else {
 715			move_imm(ctx, t1, imm, is32);
 716			emit_insn(ctx, or, dst, dst, t1);
 717		}
 718		emit_zext_32(ctx, dst, is32);
 719		break;
 720
 721	/* dst = dst ^ src */
 722	case BPF_ALU | BPF_XOR | BPF_X:
 723	case BPF_ALU64 | BPF_XOR | BPF_X:
 724		emit_insn(ctx, xor, dst, dst, src);
 725		emit_zext_32(ctx, dst, is32);
 726		break;
 727
 728	/* dst = dst ^ imm */
 729	case BPF_ALU | BPF_XOR | BPF_K:
 730	case BPF_ALU64 | BPF_XOR | BPF_K:
 731		if (is_unsigned_imm12(imm)) {
 732			emit_insn(ctx, xori, dst, dst, imm);
 733		} else {
 734			move_imm(ctx, t1, imm, is32);
 735			emit_insn(ctx, xor, dst, dst, t1);
 736		}
 737		emit_zext_32(ctx, dst, is32);
 738		break;
 739
 740	/* dst = dst << src (logical) */
 741	case BPF_ALU | BPF_LSH | BPF_X:
 742		emit_insn(ctx, sllw, dst, dst, src);
 743		emit_zext_32(ctx, dst, is32);
 744		break;
 745
 746	case BPF_ALU64 | BPF_LSH | BPF_X:
 747		emit_insn(ctx, slld, dst, dst, src);
 748		break;
 749
 750	/* dst = dst << imm (logical) */
 751	case BPF_ALU | BPF_LSH | BPF_K:
 752		emit_insn(ctx, slliw, dst, dst, imm);
 753		emit_zext_32(ctx, dst, is32);
 754		break;
 755
 756	case BPF_ALU64 | BPF_LSH | BPF_K:
 757		emit_insn(ctx, sllid, dst, dst, imm);
 758		break;
 759
 760	/* dst = dst >> src (logical) */
 761	case BPF_ALU | BPF_RSH | BPF_X:
 762		emit_insn(ctx, srlw, dst, dst, src);
 763		emit_zext_32(ctx, dst, is32);
 764		break;
 765
 766	case BPF_ALU64 | BPF_RSH | BPF_X:
 767		emit_insn(ctx, srld, dst, dst, src);
 768		break;
 769
 770	/* dst = dst >> imm (logical) */
 771	case BPF_ALU | BPF_RSH | BPF_K:
 772		emit_insn(ctx, srliw, dst, dst, imm);
 773		emit_zext_32(ctx, dst, is32);
 774		break;
 775
 776	case BPF_ALU64 | BPF_RSH | BPF_K:
 777		emit_insn(ctx, srlid, dst, dst, imm);
 778		break;
 779
 780	/* dst = dst >> src (arithmetic) */
 781	case BPF_ALU | BPF_ARSH | BPF_X:
 782		emit_insn(ctx, sraw, dst, dst, src);
 783		emit_zext_32(ctx, dst, is32);
 784		break;
 785
 786	case BPF_ALU64 | BPF_ARSH | BPF_X:
 787		emit_insn(ctx, srad, dst, dst, src);
 788		break;
 789
 790	/* dst = dst >> imm (arithmetic) */
 791	case BPF_ALU | BPF_ARSH | BPF_K:
 792		emit_insn(ctx, sraiw, dst, dst, imm);
 793		emit_zext_32(ctx, dst, is32);
 794		break;
 795
 796	case BPF_ALU64 | BPF_ARSH | BPF_K:
 797		emit_insn(ctx, sraid, dst, dst, imm);
 798		break;
 799
 800	/* dst = BSWAP##imm(dst) */
 801	case BPF_ALU | BPF_END | BPF_FROM_LE:
 802		switch (imm) {
 803		case 16:
 804			/* zero-extend 16 bits into 64 bits */
 805			emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
 806			break;
 807		case 32:
 808			/* zero-extend 32 bits into 64 bits */
 809			emit_zext_32(ctx, dst, is32);
 810			break;
 811		case 64:
 812			/* do nothing */
 813			break;
 814		}
 815		break;
 816
 817	case BPF_ALU | BPF_END | BPF_FROM_BE:
 818	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
 819		switch (imm) {
 820		case 16:
 821			emit_insn(ctx, revb2h, dst, dst);
 822			/* zero-extend 16 bits into 64 bits */
 823			emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
 824			break;
 825		case 32:
 826			emit_insn(ctx, revb2w, dst, dst);
 827			/* clear the upper 32 bits */
 828			emit_zext_32(ctx, dst, true);
 829			break;
 830		case 64:
 831			emit_insn(ctx, revbd, dst, dst);
 832			break;
 833		}
 834		break;
 835
 836	/* PC += off if dst cond src */
 837	case BPF_JMP | BPF_JEQ | BPF_X:
 838	case BPF_JMP | BPF_JNE | BPF_X:
 839	case BPF_JMP | BPF_JGT | BPF_X:
 840	case BPF_JMP | BPF_JGE | BPF_X:
 841	case BPF_JMP | BPF_JLT | BPF_X:
 842	case BPF_JMP | BPF_JLE | BPF_X:
 843	case BPF_JMP | BPF_JSGT | BPF_X:
 844	case BPF_JMP | BPF_JSGE | BPF_X:
 845	case BPF_JMP | BPF_JSLT | BPF_X:
 846	case BPF_JMP | BPF_JSLE | BPF_X:
 847	case BPF_JMP32 | BPF_JEQ | BPF_X:
 848	case BPF_JMP32 | BPF_JNE | BPF_X:
 849	case BPF_JMP32 | BPF_JGT | BPF_X:
 850	case BPF_JMP32 | BPF_JGE | BPF_X:
 851	case BPF_JMP32 | BPF_JLT | BPF_X:
 852	case BPF_JMP32 | BPF_JLE | BPF_X:
 853	case BPF_JMP32 | BPF_JSGT | BPF_X:
 854	case BPF_JMP32 | BPF_JSGE | BPF_X:
 855	case BPF_JMP32 | BPF_JSLT | BPF_X:
 856	case BPF_JMP32 | BPF_JSLE | BPF_X:
 857		jmp_offset = bpf2la_offset(i, off, ctx);
 858		move_reg(ctx, t1, dst);
 859		move_reg(ctx, t2, src);
 860		if (is_signed_bpf_cond(BPF_OP(code))) {
 861			emit_sext_32(ctx, t1, is32);
 862			emit_sext_32(ctx, t2, is32);
 863		} else {
 864			emit_zext_32(ctx, t1, is32);
 865			emit_zext_32(ctx, t2, is32);
 866		}
 867		if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
 868			goto toofar;
 869		break;
 870
 871	/* PC += off if dst cond imm */
 872	case BPF_JMP | BPF_JEQ | BPF_K:
 873	case BPF_JMP | BPF_JNE | BPF_K:
 874	case BPF_JMP | BPF_JGT | BPF_K:
 875	case BPF_JMP | BPF_JGE | BPF_K:
 876	case BPF_JMP | BPF_JLT | BPF_K:
 877	case BPF_JMP | BPF_JLE | BPF_K:
 878	case BPF_JMP | BPF_JSGT | BPF_K:
 879	case BPF_JMP | BPF_JSGE | BPF_K:
 880	case BPF_JMP | BPF_JSLT | BPF_K:
 881	case BPF_JMP | BPF_JSLE | BPF_K:
 882	case BPF_JMP32 | BPF_JEQ | BPF_K:
 883	case BPF_JMP32 | BPF_JNE | BPF_K:
 884	case BPF_JMP32 | BPF_JGT | BPF_K:
 885	case BPF_JMP32 | BPF_JGE | BPF_K:
 886	case BPF_JMP32 | BPF_JLT | BPF_K:
 887	case BPF_JMP32 | BPF_JLE | BPF_K:
 888	case BPF_JMP32 | BPF_JSGT | BPF_K:
 889	case BPF_JMP32 | BPF_JSGE | BPF_K:
 890	case BPF_JMP32 | BPF_JSLT | BPF_K:
 891	case BPF_JMP32 | BPF_JSLE | BPF_K:
 892		jmp_offset = bpf2la_offset(i, off, ctx);
 893		if (imm) {
 894			move_imm(ctx, t1, imm, false);
 895			tm = t1;
 896		} else {
 897			/* If imm is 0, simply use zero register. */
 898			tm = LOONGARCH_GPR_ZERO;
 899		}
 900		move_reg(ctx, t2, dst);
 901		if (is_signed_bpf_cond(BPF_OP(code))) {
 902			emit_sext_32(ctx, tm, is32);
 903			emit_sext_32(ctx, t2, is32);
 904		} else {
 905			emit_zext_32(ctx, tm, is32);
 906			emit_zext_32(ctx, t2, is32);
 907		}
 908		if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
 909			goto toofar;
 910		break;
 911
 912	/* PC += off if dst & src */
 913	case BPF_JMP | BPF_JSET | BPF_X:
 914	case BPF_JMP32 | BPF_JSET | BPF_X:
 915		jmp_offset = bpf2la_offset(i, off, ctx);
 916		emit_insn(ctx, and, t1, dst, src);
 917		emit_zext_32(ctx, t1, is32);
 918		if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
 919			goto toofar;
 920		break;
 921
 922	/* PC += off if dst & imm */
 923	case BPF_JMP | BPF_JSET | BPF_K:
 924	case BPF_JMP32 | BPF_JSET | BPF_K:
 925		jmp_offset = bpf2la_offset(i, off, ctx);
 926		move_imm(ctx, t1, imm, is32);
 927		emit_insn(ctx, and, t1, dst, t1);
 928		emit_zext_32(ctx, t1, is32);
 929		if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
 930			goto toofar;
 931		break;
 932
 933	/* PC += off */
 934	case BPF_JMP | BPF_JA:
 935	case BPF_JMP32 | BPF_JA:
 936		if (BPF_CLASS(code) == BPF_JMP)
 937			jmp_offset = bpf2la_offset(i, off, ctx);
 938		else
 939			jmp_offset = bpf2la_offset(i, imm, ctx);
 940		if (emit_uncond_jmp(ctx, jmp_offset) < 0)
 941			goto toofar;
 942		break;
 943
 944	/* function call */
 945	case BPF_JMP | BPF_CALL:
 946		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
 947					    &func_addr, &func_addr_fixed);
 948		if (ret < 0)
 949			return ret;
 950
 951		if (insn->src_reg == BPF_PSEUDO_CALL) {
 952			tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
 953			emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
 954		}
 955
 956		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
 957			const struct btf_func_model *m;
 958			int i;
 959
 960			m = bpf_jit_find_kfunc_model(ctx->prog, insn);
 961			if (!m)
 962				return -EINVAL;
 963
 964			for (i = 0; i < m->nr_args; i++) {
 965				u8 reg = regmap[BPF_REG_1 + i];
 966				bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG;
 967
 968				emit_abi_ext(ctx, reg, m->arg_size[i], sign);
 969			}
 970		}
 971
 972		move_addr(ctx, t1, func_addr);
 973		emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
 974
 975		if (insn->src_reg != BPF_PSEUDO_CALL)
 976			move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
 977
 978		break;
 979
 980	/* tail call */
 981	case BPF_JMP | BPF_TAIL_CALL:
 982		if (emit_bpf_tail_call(ctx, i) < 0)
 983			return -EINVAL;
 984		break;
 985
 986	/* function return */
 987	case BPF_JMP | BPF_EXIT:
 988		if (i == ctx->prog->len - 1)
 989			break;
 990
 991		jmp_offset = epilogue_offset(ctx);
 992		if (emit_uncond_jmp(ctx, jmp_offset) < 0)
 993			goto toofar;
 994		break;
 995
 996	/* dst = imm64 */
 997	case BPF_LD | BPF_IMM | BPF_DW:
 998	{
 999		const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
1000
1001		if (bpf_pseudo_func(insn))
1002			move_addr(ctx, dst, imm64);
1003		else
1004			move_imm(ctx, dst, imm64, is32);
1005		return 1;
1006	}
1007
1008	/* dst = *(size *)(src + off) */
1009	case BPF_LDX | BPF_MEM | BPF_B:
1010	case BPF_LDX | BPF_MEM | BPF_H:
1011	case BPF_LDX | BPF_MEM | BPF_W:
1012	case BPF_LDX | BPF_MEM | BPF_DW:
1013	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1014	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1015	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1016	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1017	/* dst_reg = (s64)*(signed size *)(src_reg + off) */
1018	case BPF_LDX | BPF_MEMSX | BPF_B:
1019	case BPF_LDX | BPF_MEMSX | BPF_H:
1020	case BPF_LDX | BPF_MEMSX | BPF_W:
1021	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1022	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1023	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1024		sign_extend = BPF_MODE(insn->code) == BPF_MEMSX ||
1025			      BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
1026		switch (BPF_SIZE(code)) {
1027		case BPF_B:
1028			if (is_signed_imm12(off)) {
1029				if (sign_extend)
1030					emit_insn(ctx, ldb, dst, src, off);
1031				else
1032					emit_insn(ctx, ldbu, dst, src, off);
1033			} else {
1034				move_imm(ctx, t1, off, is32);
1035				if (sign_extend)
1036					emit_insn(ctx, ldxb, dst, src, t1);
1037				else
1038					emit_insn(ctx, ldxbu, dst, src, t1);
1039			}
1040			break;
1041		case BPF_H:
1042			if (is_signed_imm12(off)) {
1043				if (sign_extend)
1044					emit_insn(ctx, ldh, dst, src, off);
1045				else
1046					emit_insn(ctx, ldhu, dst, src, off);
1047			} else {
1048				move_imm(ctx, t1, off, is32);
1049				if (sign_extend)
1050					emit_insn(ctx, ldxh, dst, src, t1);
1051				else
1052					emit_insn(ctx, ldxhu, dst, src, t1);
1053			}
1054			break;
1055		case BPF_W:
1056			if (is_signed_imm12(off)) {
1057				if (sign_extend)
1058					emit_insn(ctx, ldw, dst, src, off);
1059				else
1060					emit_insn(ctx, ldwu, dst, src, off);
1061			} else {
1062				move_imm(ctx, t1, off, is32);
1063				if (sign_extend)
1064					emit_insn(ctx, ldxw, dst, src, t1);
1065				else
1066					emit_insn(ctx, ldxwu, dst, src, t1);
1067			}
1068			break;
1069		case BPF_DW:
1070			move_imm(ctx, t1, off, is32);
1071			emit_insn(ctx, ldxd, dst, src, t1);
1072			break;
1073		}
1074
1075		ret = add_exception_handler(insn, ctx, dst);
1076		if (ret)
1077			return ret;
1078		break;
1079
1080	/* *(size *)(dst + off) = imm */
1081	case BPF_ST | BPF_MEM | BPF_B:
1082	case BPF_ST | BPF_MEM | BPF_H:
1083	case BPF_ST | BPF_MEM | BPF_W:
1084	case BPF_ST | BPF_MEM | BPF_DW:
1085		switch (BPF_SIZE(code)) {
1086		case BPF_B:
1087			move_imm(ctx, t1, imm, is32);
1088			if (is_signed_imm12(off)) {
1089				emit_insn(ctx, stb, t1, dst, off);
1090			} else {
1091				move_imm(ctx, t2, off, is32);
1092				emit_insn(ctx, stxb, t1, dst, t2);
1093			}
1094			break;
1095		case BPF_H:
1096			move_imm(ctx, t1, imm, is32);
1097			if (is_signed_imm12(off)) {
1098				emit_insn(ctx, sth, t1, dst, off);
1099			} else {
1100				move_imm(ctx, t2, off, is32);
1101				emit_insn(ctx, stxh, t1, dst, t2);
1102			}
1103			break;
1104		case BPF_W:
1105			move_imm(ctx, t1, imm, is32);
1106			if (is_signed_imm12(off)) {
1107				emit_insn(ctx, stw, t1, dst, off);
1108			} else if (is_signed_imm14(off)) {
1109				emit_insn(ctx, stptrw, t1, dst, off);
1110			} else {
1111				move_imm(ctx, t2, off, is32);
1112				emit_insn(ctx, stxw, t1, dst, t2);
1113			}
1114			break;
1115		case BPF_DW:
1116			move_imm(ctx, t1, imm, is32);
1117			if (is_signed_imm12(off)) {
1118				emit_insn(ctx, std, t1, dst, off);
1119			} else if (is_signed_imm14(off)) {
1120				emit_insn(ctx, stptrd, t1, dst, off);
1121			} else {
1122				move_imm(ctx, t2, off, is32);
1123				emit_insn(ctx, stxd, t1, dst, t2);
1124			}
1125			break;
1126		}
1127		break;
1128
1129	/* *(size *)(dst + off) = src */
1130	case BPF_STX | BPF_MEM | BPF_B:
1131	case BPF_STX | BPF_MEM | BPF_H:
1132	case BPF_STX | BPF_MEM | BPF_W:
1133	case BPF_STX | BPF_MEM | BPF_DW:
1134		switch (BPF_SIZE(code)) {
1135		case BPF_B:
1136			if (is_signed_imm12(off)) {
1137				emit_insn(ctx, stb, src, dst, off);
1138			} else {
1139				move_imm(ctx, t1, off, is32);
1140				emit_insn(ctx, stxb, src, dst, t1);
1141			}
1142			break;
1143		case BPF_H:
1144			if (is_signed_imm12(off)) {
1145				emit_insn(ctx, sth, src, dst, off);
1146			} else {
1147				move_imm(ctx, t1, off, is32);
1148				emit_insn(ctx, stxh, src, dst, t1);
1149			}
1150			break;
1151		case BPF_W:
1152			if (is_signed_imm12(off)) {
1153				emit_insn(ctx, stw, src, dst, off);
1154			} else if (is_signed_imm14(off)) {
1155				emit_insn(ctx, stptrw, src, dst, off);
1156			} else {
1157				move_imm(ctx, t1, off, is32);
1158				emit_insn(ctx, stxw, src, dst, t1);
1159			}
1160			break;
1161		case BPF_DW:
1162			if (is_signed_imm12(off)) {
1163				emit_insn(ctx, std, src, dst, off);
1164			} else if (is_signed_imm14(off)) {
1165				emit_insn(ctx, stptrd, src, dst, off);
1166			} else {
1167				move_imm(ctx, t1, off, is32);
1168				emit_insn(ctx, stxd, src, dst, t1);
1169			}
1170			break;
1171		}
1172		break;
1173
1174	case BPF_STX | BPF_ATOMIC | BPF_W:
1175	case BPF_STX | BPF_ATOMIC | BPF_DW:
1176		emit_atomic(insn, ctx);
1177		break;
1178
1179	/* Speculation barrier */
1180	case BPF_ST | BPF_NOSPEC:
1181		break;
1182
1183	default:
1184		pr_err("bpf_jit: unknown opcode %02x\n", code);
1185		return -EINVAL;
1186	}
1187
1188	return 0;
1189
1190toofar:
1191	pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1192	return -E2BIG;
1193}
1194
1195static int build_body(struct jit_ctx *ctx, bool extra_pass)
1196{
1197	int i;
1198	const struct bpf_prog *prog = ctx->prog;
1199
1200	for (i = 0; i < prog->len; i++) {
1201		const struct bpf_insn *insn = &prog->insnsi[i];
1202		int ret;
1203
1204		if (ctx->image == NULL)
1205			ctx->offset[i] = ctx->idx;
1206
1207		ret = build_insn(insn, ctx, extra_pass);
1208		if (ret > 0) {
1209			i++;
1210			if (ctx->image == NULL)
1211				ctx->offset[i] = ctx->idx;
1212			continue;
1213		}
1214		if (ret)
1215			return ret;
1216	}
1217
1218	if (ctx->image == NULL)
1219		ctx->offset[i] = ctx->idx;
1220
1221	return 0;
1222}
1223
1224/* Fill space with break instructions */
1225static void jit_fill_hole(void *area, unsigned int size)
1226{
1227	u32 *ptr;
1228
1229	/* We are guaranteed to have aligned memory */
1230	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1231		*ptr++ = INSN_BREAK;
1232}
1233
1234static int validate_code(struct jit_ctx *ctx)
1235{
1236	int i;
1237	union loongarch_instruction insn;
1238
1239	for (i = 0; i < ctx->idx; i++) {
1240		insn = ctx->image[i];
1241		/* Check INSN_BREAK */
1242		if (insn.word == INSN_BREAK)
1243			return -1;
1244	}
1245
1246	return 0;
1247}
1248
1249static int validate_ctx(struct jit_ctx *ctx)
1250{
1251	if (validate_code(ctx))
1252		return -1;
1253
1254	if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1255		return -1;
1256
1257	return 0;
1258}
1259
1260static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target)
1261{
1262	if (!target) {
1263		pr_err("bpf_jit: jump target address is error\n");
1264		return -EFAULT;
1265	}
1266
1267	move_imm(ctx, LOONGARCH_GPR_T1, target, false);
1268	emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
1269
1270	return 0;
1271}
1272
1273static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
1274{
1275	int i;
1276	struct jit_ctx ctx;
1277
1278	ctx.idx = 0;
1279	ctx.image = (union loongarch_instruction *)insns;
1280
1281	if (!target) {
1282		for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1283			emit_insn((&ctx), nop);
1284		return 0;
1285	}
1286
1287	return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target);
1288}
1289
1290static int emit_call(struct jit_ctx *ctx, u64 addr)
1291{
1292	return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr);
1293}
1294
1295void *bpf_arch_text_copy(void *dst, void *src, size_t len)
1296{
1297	int ret;
1298
1299	mutex_lock(&text_mutex);
1300	ret = larch_insn_text_copy(dst, src, len);
1301	mutex_unlock(&text_mutex);
1302
1303	return ret ? ERR_PTR(-EINVAL) : dst;
1304}
1305
1306int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
1307		       enum bpf_text_poke_type new_t, void *old_addr,
1308		       void *new_addr)
1309{
1310	int ret;
1311	bool is_call;
1312	unsigned long size = 0;
1313	unsigned long offset = 0;
1314	void *image = NULL;
1315	char namebuf[KSYM_NAME_LEN];
1316	u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1317	u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1318
1319	/* Only poking bpf text is supported. Since kernel function entry
1320	 * is set up by ftrace, we rely on ftrace to poke kernel functions.
1321	 */
1322	if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf))
1323		return -ENOTSUPP;
1324
1325	image = ip - offset;
1326
1327	/* zero offset means we're poking bpf prog entry */
1328	if (offset == 0) {
1329		/* skip to the nop instruction in bpf prog entry:
1330		 * move t0, ra
1331		 * nop
1332		 */
1333		ip = image + LOONGARCH_INSN_SIZE;
1334	}
1335
1336	is_call = old_t == BPF_MOD_CALL;
1337	ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call);
1338	if (ret)
1339		return ret;
1340
1341	if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES))
1342		return -EFAULT;
1343
1344	is_call = new_t == BPF_MOD_CALL;
1345	ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call);
1346	if (ret)
1347		return ret;
1348
1349	mutex_lock(&text_mutex);
1350	if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES))
1351		ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
1352	mutex_unlock(&text_mutex);
1353
1354	return ret;
1355}
1356
1357int bpf_arch_text_invalidate(void *dst, size_t len)
1358{
1359	int i;
1360	int ret = 0;
1361	u32 *inst;
1362
1363	inst = kvmalloc(len, GFP_KERNEL);
1364	if (!inst)
1365		return -ENOMEM;
1366
1367	for (i = 0; i < (len / sizeof(u32)); i++)
1368		inst[i] = INSN_BREAK;
1369
1370	mutex_lock(&text_mutex);
1371	if (larch_insn_text_copy(dst, inst, len))
1372		ret = -EINVAL;
1373	mutex_unlock(&text_mutex);
1374
1375	kvfree(inst);
1376
1377	return ret;
1378}
1379
1380static void store_args(struct jit_ctx *ctx, int nargs, int args_off)
1381{
1382	int i;
1383
1384	for (i = 0; i < nargs; i++) {
1385		emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1386		args_off -= 8;
1387	}
1388}
1389
1390static void restore_args(struct jit_ctx *ctx, int nargs, int args_off)
1391{
1392	int i;
1393
1394	for (i = 0; i < nargs; i++) {
1395		emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1396		args_off -= 8;
1397	}
1398}
1399
1400static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
1401			   int args_off, int retval_off, int run_ctx_off, bool save_ret)
1402{
1403	int ret;
1404	u32 *branch;
1405	struct bpf_prog *p = l->link.prog;
1406	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
1407
1408	if (l->cookie) {
1409		move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false);
1410		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1411	} else {
1412		emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1413	}
1414
1415	/* arg1: prog */
1416	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1417	/* arg2: &run_ctx */
1418	emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
1419	ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
1420	if (ret)
1421		return ret;
1422
1423	/* store prog start time */
1424	move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
1425
1426	/*
1427	 * if (__bpf_prog_enter(prog) == 0)
1428	 *      goto skip_exec_of_prog;
1429	 */
1430	branch = (u32 *)ctx->image + ctx->idx;
1431	/* nop reserved for conditional jump */
1432	emit_insn(ctx, nop);
1433
1434	/* arg1: &args_off */
1435	emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
1436	if (!p->jited)
1437		move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
1438	ret = emit_call(ctx, (const u64)p->bpf_func);
1439	if (ret)
1440		return ret;
1441
1442	if (save_ret) {
1443		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1444		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1445	}
1446
1447	/* update branch with beqz */
1448	if (ctx->image) {
1449		int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch;
1450		*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
1451	}
1452
1453	/* arg1: prog */
1454	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1455	/* arg2: prog start time */
1456	move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
1457	/* arg3: &run_ctx */
1458	emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
1459	ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
1460
1461	return ret;
1462}
1463
1464static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
1465			       int args_off, int retval_off, int run_ctx_off, u32 **branches)
1466{
1467	int i;
1468
1469	emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
1470	for (i = 0; i < tl->nr_links; i++) {
1471		invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true);
1472		emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
1473		branches[i] = (u32 *)ctx->image + ctx->idx;
1474		emit_insn(ctx, nop);
1475	}
1476}
1477
1478void *arch_alloc_bpf_trampoline(unsigned int size)
1479{
1480	return bpf_prog_pack_alloc(size, jit_fill_hole);
1481}
1482
1483void arch_free_bpf_trampoline(void *image, unsigned int size)
1484{
1485	bpf_prog_pack_free(image, size);
1486}
1487
1488/*
1489 * Sign-extend the register if necessary
1490 */
1491static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign)
1492{
1493	/* ABI requires unsigned char/short to be zero-extended */
1494	if (!sign && (size == 1 || size == 2)) {
1495		if (rd != rj)
1496			move_reg(ctx, rd, rj);
1497		return;
1498	}
1499
1500	switch (size) {
1501	case 1:
1502		emit_insn(ctx, extwb, rd, rj);
1503		break;
1504	case 2:
1505		emit_insn(ctx, extwh, rd, rj);
1506		break;
1507	case 4:
1508		emit_insn(ctx, addiw, rd, rj, 0);
1509		break;
1510	case 8:
1511		if (rd != rj)
1512			move_reg(ctx, rd, rj);
1513		break;
1514	default:
1515		pr_warn("bpf_jit: invalid size %d for sign_extend\n", size);
1516	}
1517}
1518
1519static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
1520					 const struct btf_func_model *m, struct bpf_tramp_links *tlinks,
1521					 void *func_addr, u32 flags)
1522{
1523	int i, ret, save_ret;
1524	int stack_size, nargs;
1525	int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;
1526	bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
1527	void *orig_call = func_addr;
1528	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
1529	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
1530	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
1531	u32 **branches = NULL;
1532
1533	/*
1534	 * FP + 8       [ RA to parent func ] return address to parent
1535	 *                    function
1536	 * FP + 0       [ FP of parent func ] frame pointer of parent
1537	 *                    function
1538	 * FP - 8       [ T0 to traced func ] return address of traced
1539	 *                    function
1540	 * FP - 16      [ FP of traced func ] frame pointer of traced
1541	 *                    function
1542	 *
1543	 * FP - retval_off  [ return value      ] BPF_TRAMP_F_CALL_ORIG or
1544	 *                    BPF_TRAMP_F_RET_FENTRY_RET
1545	 *                  [ argN              ]
1546	 *                  [ ...               ]
1547	 * FP - args_off    [ arg1              ]
1548	 *
1549	 * FP - nargs_off   [ regs count        ]
1550	 *
1551	 * FP - ip_off      [ traced func   ] BPF_TRAMP_F_IP_ARG
1552	 *
1553	 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
1554	 *
1555	 * FP - sreg_off    [ callee saved reg  ]
1556	 *
1557	 * FP - tcc_ptr_off [ tail_call_cnt_ptr ]
1558	 */
1559
1560	if (m->nr_args > LOONGARCH_MAX_REG_ARGS)
1561		return -ENOTSUPP;
1562
1563	/* FIXME: No support of struct argument */
1564	for (i = 0; i < m->nr_args; i++) {
1565		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
1566			return -ENOTSUPP;
1567	}
1568
1569	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
1570		return -ENOTSUPP;
1571
1572	/* Room of trampoline frame to store return address and frame pointer */
1573	stack_size = 16;
1574
1575	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
1576	if (save_ret)
1577		stack_size += 16; /* Save BPF R0 and A0 */
1578
1579	retval_off = stack_size;
1580
1581	/* Room of trampoline frame to store args */
1582	nargs = m->nr_args;
1583	stack_size += nargs * 8;
1584	args_off = stack_size;
1585
1586	/* Room of trampoline frame to store args number */
1587	stack_size += 8;
1588	nargs_off = stack_size;
1589
1590	/* Room of trampoline frame to store ip address */
1591	if (flags & BPF_TRAMP_F_IP_ARG) {
1592		stack_size += 8;
1593		ip_off = stack_size;
1594	}
1595
1596	/* Room of trampoline frame to store struct bpf_tramp_run_ctx */
1597	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
1598	run_ctx_off = stack_size;
1599
1600	stack_size += 8;
1601	sreg_off = stack_size;
1602
1603	/* Room of trampoline frame to store tail_call_cnt_ptr */
1604	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
1605		stack_size += 8;
1606		tcc_ptr_off = stack_size;
1607	}
1608
1609	stack_size = round_up(stack_size, 16);
1610
1611	if (is_struct_ops) {
1612		/*
1613		 * For the trampoline called directly, just handle
1614		 * the frame of trampoline.
1615		 */
1616		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1617		emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1618		emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1619		emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1620	} else {
1621		/*
1622		 * For the trampoline called from function entry,
1623		 * the frame of traced function and the frame of
1624		 * trampoline need to be considered.
1625		 */
1626		/* RA and FP for parent function */
1627		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
1628		emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1629		emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1630		emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
1631
1632		/* RA and FP for traced function */
1633		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1634		emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1635		emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1636		emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1637	}
1638
1639	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1640		emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1641
1642	/* callee saved register S1 to pass start time */
1643	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1644
1645	/* store ip address of the traced function */
1646	if (flags & BPF_TRAMP_F_IP_ARG) {
1647		move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false);
1648		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off);
1649	}
1650
1651	/* store nargs number */
1652	move_imm(ctx, LOONGARCH_GPR_T1, nargs, false);
1653	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off);
1654
1655	store_args(ctx, nargs, args_off);
1656
1657	/* To traced function */
1658	/* Ftrace jump skips 2 NOP instructions */
1659	if (is_kernel_text((unsigned long)orig_call) ||
1660	    is_module_text_address((unsigned long)orig_call))
1661		orig_call += LOONGARCH_FENTRY_NBYTES;
1662	/* Direct jump skips 5 NOP instructions */
1663	else if (is_bpf_text_address((unsigned long)orig_call))
1664		orig_call += LOONGARCH_BPF_FENTRY_NBYTES;
1665
1666	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1667		move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
1668		ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
1669		if (ret)
1670			return ret;
1671	}
1672
1673	for (i = 0; i < fentry->nr_links; i++) {
1674		ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off,
1675				      run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET);
1676		if (ret)
1677			return ret;
1678	}
1679	if (fmod_ret->nr_links) {
1680		branches  = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
1681		if (!branches)
1682			return -ENOMEM;
1683
1684		invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches);
1685	}
1686
1687	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1688		restore_args(ctx, m->nr_args, args_off);
1689
1690		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1691			emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1692
1693		ret = emit_call(ctx, (const u64)orig_call);
1694		if (ret)
1695			goto out;
1696		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1697		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1698		im->ip_after_call = ctx->ro_image + ctx->idx;
1699		/* Reserve space for the move_imm + jirl instruction */
1700		for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1701			emit_insn(ctx, nop);
1702	}
1703
1704	for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
1705		int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i];
1706		*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
1707	}
1708
1709	for (i = 0; i < fexit->nr_links; i++) {
1710		ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false);
1711		if (ret)
1712			goto out;
1713	}
1714
1715	if (flags & BPF_TRAMP_F_CALL_ORIG) {
1716		im->ip_epilogue = ctx->ro_image + ctx->idx;
1717		move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
1718		ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
1719		if (ret)
1720			goto out;
1721	}
1722
1723	if (flags & BPF_TRAMP_F_RESTORE_REGS)
1724		restore_args(ctx, m->nr_args, args_off);
1725
1726	if (save_ret) {
1727		emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1728		if (is_struct_ops)
1729			sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0],
1730				    m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG);
1731		else
1732			emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1733	}
1734
1735	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1736
1737	if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1738		emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1739
1740	if (is_struct_ops) {
1741		/* trampoline called directly */
1742		emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1743		emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1744		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
1745
1746		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
1747	} else {
1748		/* trampoline called from function entry */
1749		emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1750		emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1751		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
1752
1753		emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1754		emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1755		emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
1756
1757		if (flags & BPF_TRAMP_F_SKIP_FRAME) {
1758			/* return to parent function */
1759			move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
1760			emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
1761		} else {
1762			/* return to traced function */
1763			move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA);
1764			move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
1765			emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0);
1766		}
1767	}
1768
1769	ret = ctx->idx;
1770out:
1771	kfree(branches);
1772
1773	return ret;
1774}
1775
1776int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
1777				void *ro_image_end, const struct btf_func_model *m,
1778				u32 flags, struct bpf_tramp_links *tlinks, void *func_addr)
1779{
1780	int ret, size;
1781	void *image, *tmp;
1782	struct jit_ctx ctx;
1783
1784	size = ro_image_end - ro_image;
1785	image = kvmalloc(size, GFP_KERNEL);
1786	if (!image)
1787		return -ENOMEM;
1788
1789	ctx.image = (union loongarch_instruction *)image;
1790	ctx.ro_image = (union loongarch_instruction *)ro_image;
1791	ctx.idx = 0;
1792
1793	jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
1794	ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags);
1795	if (ret < 0)
1796		goto out;
1797
1798	if (validate_code(&ctx) < 0) {
1799		ret = -EINVAL;
1800		goto out;
1801	}
1802
1803	tmp = bpf_arch_text_copy(ro_image, image, size);
1804	if (IS_ERR(tmp)) {
1805		ret = PTR_ERR(tmp);
1806		goto out;
1807	}
1808
1809out:
1810	kvfree(image);
1811	return ret < 0 ? ret : size;
1812}
1813
1814int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1815			     struct bpf_tramp_links *tlinks, void *func_addr)
1816{
1817	int ret;
1818	struct jit_ctx ctx;
1819	struct bpf_tramp_image im;
1820
1821	ctx.image = NULL;
1822	ctx.idx = 0;
1823
1824	ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags);
1825
1826	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
1827}
1828
1829struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1830{
1831	bool tmp_blinded = false, extra_pass = false;
1832	u8 *image_ptr;
1833	int image_size, prog_size, extable_size;
1834	struct jit_ctx ctx;
1835	struct jit_data *jit_data;
1836	struct bpf_binary_header *header;
1837	struct bpf_prog *tmp, *orig_prog = prog;
1838
1839	/*
1840	 * If BPF JIT was not enabled then we must fall back to
1841	 * the interpreter.
1842	 */
1843	if (!prog->jit_requested)
1844		return orig_prog;
1845
1846	tmp = bpf_jit_blind_constants(prog);
1847	/*
1848	 * If blinding was requested and we failed during blinding,
1849	 * we must fall back to the interpreter. Otherwise, we save
1850	 * the new JITed code.
1851	 */
1852	if (IS_ERR(tmp))
1853		return orig_prog;
1854
1855	if (tmp != prog) {
1856		tmp_blinded = true;
1857		prog = tmp;
1858	}
1859
1860	jit_data = prog->aux->jit_data;
1861	if (!jit_data) {
1862		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1863		if (!jit_data) {
1864			prog = orig_prog;
1865			goto out;
1866		}
1867		prog->aux->jit_data = jit_data;
1868	}
1869	if (jit_data->ctx.offset) {
1870		ctx = jit_data->ctx;
1871		image_ptr = jit_data->image;
1872		header = jit_data->header;
1873		extra_pass = true;
1874		prog_size = sizeof(u32) * ctx.idx;
1875		goto skip_init_ctx;
1876	}
1877
1878	memset(&ctx, 0, sizeof(ctx));
1879	ctx.prog = prog;
1880
1881	ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1882	if (ctx.offset == NULL) {
1883		prog = orig_prog;
1884		goto out_offset;
1885	}
1886
1887	/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1888	build_prologue(&ctx);
1889	if (build_body(&ctx, extra_pass)) {
1890		prog = orig_prog;
1891		goto out_offset;
1892	}
1893	ctx.epilogue_offset = ctx.idx;
1894	build_epilogue(&ctx);
1895
1896	extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
1897
1898	/* Now we know the actual image size.
1899	 * As each LoongArch instruction is of length 32bit,
1900	 * we are translating number of JITed intructions into
1901	 * the size required to store these JITed code.
1902	 */
1903	prog_size = sizeof(u32) * ctx.idx;
1904	image_size = prog_size + extable_size;
1905	/* Now we know the size of the structure to make */
1906	header = bpf_jit_binary_alloc(image_size, &image_ptr,
1907				      sizeof(u32), jit_fill_hole);
1908	if (header == NULL) {
1909		prog = orig_prog;
1910		goto out_offset;
1911	}
1912
1913	/* 2. Now, the actual pass to generate final JIT code */
1914	ctx.image = (union loongarch_instruction *)image_ptr;
1915	if (extable_size)
1916		prog->aux->extable = (void *)image_ptr + prog_size;
1917
1918skip_init_ctx:
1919	ctx.idx = 0;
1920	ctx.num_exentries = 0;
1921
1922	build_prologue(&ctx);
1923	if (build_body(&ctx, extra_pass)) {
1924		bpf_jit_binary_free(header);
1925		prog = orig_prog;
1926		goto out_offset;
1927	}
1928	build_epilogue(&ctx);
1929
1930	/* 3. Extra pass to validate JITed code */
1931	if (validate_ctx(&ctx)) {
1932		bpf_jit_binary_free(header);
1933		prog = orig_prog;
1934		goto out_offset;
1935	}
1936
1937	/* And we're done */
1938	if (bpf_jit_enable > 1)
1939		bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1940
1941	/* Update the icache */
1942	flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
1943
1944	if (!prog->is_func || extra_pass) {
1945		int err;
1946
1947		if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1948			pr_err_once("multi-func JIT bug %d != %d\n",
1949				    ctx.idx, jit_data->ctx.idx);
1950			goto out_free;
1951		}
1952		err = bpf_jit_binary_lock_ro(header);
1953		if (err) {
1954			pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
1955				    err);
1956			goto out_free;
1957		}
1958	} else {
1959		jit_data->ctx = ctx;
1960		jit_data->image = image_ptr;
1961		jit_data->header = header;
1962	}
1963	prog->jited = 1;
1964	prog->jited_len = prog_size;
1965	prog->bpf_func = (void *)ctx.image;
1966
1967	if (!prog->is_func || extra_pass) {
1968		int i;
1969
1970		/* offset[prog->len] is the size of program */
1971		for (i = 0; i <= prog->len; i++)
1972			ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1973		bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1974
1975out_offset:
1976		kvfree(ctx.offset);
1977		kfree(jit_data);
1978		prog->aux->jit_data = NULL;
1979	}
1980
1981out:
1982	if (tmp_blinded)
1983		bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
1984
1985
1986	return prog;
1987
1988out_free:
1989	bpf_jit_binary_free(header);
1990	prog->bpf_func = NULL;
1991	prog->jited = 0;
1992	prog->jited_len = 0;
1993	goto out_offset;
1994}
1995
1996bool bpf_jit_bypass_spec_v1(void)
1997{
1998	return true;
1999}
2000
2001bool bpf_jit_bypass_spec_v4(void)
2002{
2003	return true;
2004}
2005
2006/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
2007bool bpf_jit_supports_subprog_tailcalls(void)
2008{
2009	return true;
2010}