bpf_jit.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * Common functionality for RV32 and RV64 BPF JIT compilers
   4 *
   5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   6 *
   7 */
   8
   9#ifndef _BPF_JIT_H
  10#define _BPF_JIT_H
  11
  12#include <linux/bpf.h>
  13#include <linux/filter.h>
  14#include <asm/cacheflush.h>
  15
  16/* verify runtime detection extension status */
  17#define rv_ext_enabled(ext) \
  18	(IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext))
  19
  20static inline bool rvc_enabled(void)
  21{
  22	return IS_ENABLED(CONFIG_RISCV_ISA_C);
  23}
  24
  25enum {
  26	RV_REG_ZERO =	0,	/* The constant value 0 */
  27	RV_REG_RA =	1,	/* Return address */
  28	RV_REG_SP =	2,	/* Stack pointer */
  29	RV_REG_GP =	3,	/* Global pointer */
  30	RV_REG_TP =	4,	/* Thread pointer */
  31	RV_REG_T0 =	5,	/* Temporaries */
  32	RV_REG_T1 =	6,
  33	RV_REG_T2 =	7,
  34	RV_REG_FP =	8,	/* Saved register/frame pointer */
  35	RV_REG_S1 =	9,	/* Saved register */
  36	RV_REG_A0 =	10,	/* Function argument/return values */
  37	RV_REG_A1 =	11,	/* Function arguments */
  38	RV_REG_A2 =	12,
  39	RV_REG_A3 =	13,
  40	RV_REG_A4 =	14,
  41	RV_REG_A5 =	15,
  42	RV_REG_A6 =	16,
  43	RV_REG_A7 =	17,
  44	RV_REG_S2 =	18,	/* Saved registers */
  45	RV_REG_S3 =	19,
  46	RV_REG_S4 =	20,
  47	RV_REG_S5 =	21,
  48	RV_REG_S6 =	22,
  49	RV_REG_S7 =	23,
  50	RV_REG_S8 =	24,
  51	RV_REG_S9 =	25,
  52	RV_REG_S10 =	26,
  53	RV_REG_S11 =	27,
  54	RV_REG_T3 =	28,	/* Temporaries */
  55	RV_REG_T4 =	29,
  56	RV_REG_T5 =	30,
  57	RV_REG_T6 =	31,
  58};
  59
  60static inline bool is_creg(u8 reg)
  61{
  62	return (1 << reg) & (BIT(RV_REG_FP) |
  63			     BIT(RV_REG_S1) |
  64			     BIT(RV_REG_A0) |
  65			     BIT(RV_REG_A1) |
  66			     BIT(RV_REG_A2) |
  67			     BIT(RV_REG_A3) |
  68			     BIT(RV_REG_A4) |
  69			     BIT(RV_REG_A5));
  70}
  71
  72struct rv_jit_context {
  73	struct bpf_prog *prog;
  74	u16 *insns;		/* RV insns */
  75	u16 *ro_insns;
  76	int ninsns;
  77	int prologue_len;
  78	int epilogue_offset;
  79	int *offset;		/* BPF to RV */
  80	int nexentries;
  81	int ex_insn_off;
  82	int ex_jmp_off;
  83	unsigned long flags;
  84	int stack_size;
  85	u64 arena_vm_start;
  86	u64 user_vm_start;
  87};
  88
  89/* Convert from ninsns to bytes. */
  90static inline int ninsns_rvoff(int ninsns)
  91{
  92	return ninsns << 1;
  93}
  94
  95struct rv_jit_data {
  96	struct bpf_binary_header *header;
  97	struct bpf_binary_header *ro_header;
  98	u8 *image;
  99	u8 *ro_image;
 100	struct rv_jit_context ctx;
 101};
 102
 103static inline void bpf_fill_ill_insns(void *area, unsigned int size)
 104{
 105	memset(area, 0, size);
 106}
 107
 108static inline void bpf_flush_icache(void *start, void *end)
 109{
 110	flush_icache_range((unsigned long)start, (unsigned long)end);
 111}
 112
 113/* Emit a 4-byte riscv instruction. */
 114static inline void emit(const u32 insn, struct rv_jit_context *ctx)
 115{
 116	if (ctx->insns) {
 117		ctx->insns[ctx->ninsns] = insn;
 118		ctx->insns[ctx->ninsns + 1] = (insn >> 16);
 119	}
 120
 121	ctx->ninsns += 2;
 122}
 123
 124/* Emit a 2-byte riscv compressed instruction. */
 125static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
 126{
 127	BUILD_BUG_ON(!rvc_enabled());
 128
 129	if (ctx->insns)
 130		ctx->insns[ctx->ninsns] = insn;
 131
 132	ctx->ninsns++;
 133}
 134
 135static inline int epilogue_offset(struct rv_jit_context *ctx)
 136{
 137	int to = ctx->epilogue_offset, from = ctx->ninsns;
 138
 139	return ninsns_rvoff(to - from);
 140}
 141
 142/* Return -1 or inverted cond. */
 143static inline int invert_bpf_cond(u8 cond)
 144{
 145	switch (cond) {
 146	case BPF_JEQ:
 147		return BPF_JNE;
 148	case BPF_JGT:
 149		return BPF_JLE;
 150	case BPF_JLT:
 151		return BPF_JGE;
 152	case BPF_JGE:
 153		return BPF_JLT;
 154	case BPF_JLE:
 155		return BPF_JGT;
 156	case BPF_JNE:
 157		return BPF_JEQ;
 158	case BPF_JSGT:
 159		return BPF_JSLE;
 160	case BPF_JSLT:
 161		return BPF_JSGE;
 162	case BPF_JSGE:
 163		return BPF_JSLT;
 164	case BPF_JSLE:
 165		return BPF_JSGT;
 166	}
 167	return -1;
 168}
 169
 170static inline bool is_6b_int(long val)
 171{
 172	return -(1L << 5) <= val && val < (1L << 5);
 173}
 174
 175static inline bool is_7b_uint(unsigned long val)
 176{
 177	return val < (1UL << 7);
 178}
 179
 180static inline bool is_8b_uint(unsigned long val)
 181{
 182	return val < (1UL << 8);
 183}
 184
 185static inline bool is_9b_uint(unsigned long val)
 186{
 187	return val < (1UL << 9);
 188}
 189
 190static inline bool is_10b_int(long val)
 191{
 192	return -(1L << 9) <= val && val < (1L << 9);
 193}
 194
 195static inline bool is_10b_uint(unsigned long val)
 196{
 197	return val < (1UL << 10);
 198}
 199
 200static inline bool is_12b_int(long val)
 201{
 202	return -(1L << 11) <= val && val < (1L << 11);
 203}
 204
 205static inline int is_12b_check(int off, int insn)
 206{
 207	if (!is_12b_int(off)) {
 208		pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
 209		       insn, (int)off);
 210		return -1;
 211	}
 212	return 0;
 213}
 214
 215static inline bool is_13b_int(long val)
 216{
 217	return -(1L << 12) <= val && val < (1L << 12);
 218}
 219
 220static inline bool is_21b_int(long val)
 221{
 222	return -(1L << 20) <= val && val < (1L << 20);
 223}
 224
 225static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
 226{
 227	int from, to;
 228
 229	off++; /* BPF branch is from PC+1, RV is from PC */
 230	from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len;
 231	to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len;
 232	return ninsns_rvoff(to - from);
 233}
 234
 235/* Instruction formats. */
 236
 237static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
 238			    u8 opcode)
 239{
 240	return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 241		(rd << 7) | opcode;
 242}
 243
 244static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
 245{
 246	return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
 247		opcode;
 248}
 249
 250static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
 251{
 252	u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
 253
 254	return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 255		(imm4_0 << 7) | opcode;
 256}
 257
 258static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
 259{
 260	u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
 261	u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
 262
 263	return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 264		(imm4_1 << 7) | opcode;
 265}
 266
 267static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
 268{
 269	return (imm31_12 << 12) | (rd << 7) | opcode;
 270}
 271
 272static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
 273{
 274	u32 imm;
 275
 276	imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
 277		((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
 278
 279	return (imm << 12) | (rd << 7) | opcode;
 280}
 281
 282static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
 283			      u8 funct3, u8 rd, u8 opcode)
 284{
 285	u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
 286
 287	return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
 288}
 289
 290/* RISC-V compressed instruction formats. */
 291
 292static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
 293{
 294	return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
 295}
 296
 297static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
 298{
 299	u32 imm;
 300
 301	imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
 302	return (funct3 << 13) | (rd << 7) | op | imm;
 303}
 304
 305static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
 306{
 307	return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
 308}
 309
 310static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
 311{
 312	return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
 313}
 314
 315static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
 316			     u8 op)
 317{
 318	return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
 319		(imm_lo << 5) | ((rd & 0x7) << 2) | op;
 320}
 321
 322static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
 323			     u8 op)
 324{
 325	return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
 326		(imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
 327}
 328
 329static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
 330{
 331	return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
 332		((rs2 & 0x7) << 2) | op;
 333}
 334
 335static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
 336{
 337	u32 imm;
 338
 339	imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
 340	return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
 341}
 342
 343/* Instructions shared by both RV32 and RV64. */
 344
 345static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
 346{
 347	return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
 348}
 349
 350static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
 351{
 352	return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
 353}
 354
 355static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
 356{
 357	return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
 358}
 359
 360static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
 361{
 362	return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
 363}
 364
 365static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
 366{
 367	return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
 368}
 369
 370static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
 371{
 372	return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
 373}
 374
 375static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
 376{
 377	return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
 378}
 379
 380static inline u32 rv_lui(u8 rd, u32 imm31_12)
 381{
 382	return rv_u_insn(imm31_12, rd, 0x37);
 383}
 384
 385static inline u32 rv_auipc(u8 rd, u32 imm31_12)
 386{
 387	return rv_u_insn(imm31_12, rd, 0x17);
 388}
 389
 390static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
 391{
 392	return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
 393}
 394
 395static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
 396{
 397	return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
 398}
 399
 400static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
 401{
 402	return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
 403}
 404
 405static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
 406{
 407	return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
 408}
 409
 410static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
 411{
 412	return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
 413}
 414
 415static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
 416{
 417	return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
 418}
 419
 420static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
 421{
 422	return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
 423}
 424
 425static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
 426{
 427	return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
 428}
 429
 430static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
 431{
 432	return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
 433}
 434
 435static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
 436{
 437	return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
 438}
 439
 440static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
 441{
 442	return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
 443}
 444
 445static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
 446{
 447	return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);
 448}
 449
 450static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
 451{
 452	return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
 453}
 454
 455static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
 456{
 457	return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);
 458}
 459
 460static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
 461{
 462	return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
 463}
 464
 465static inline u32 rv_jal(u8 rd, u32 imm20_1)
 466{
 467	return rv_j_insn(imm20_1, rd, 0x6f);
 468}
 469
 470static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
 471{
 472	return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
 473}
 474
 475static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
 476{
 477	return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
 478}
 479
 480static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
 481{
 482	return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
 483}
 484
 485static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
 486{
 487	return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
 488}
 489
 490static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
 491{
 492	return rv_bltu(rs2, rs1, imm12_1);
 493}
 494
 495static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
 496{
 497	return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
 498}
 499
 500static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
 501{
 502	return rv_bgeu(rs2, rs1, imm12_1);
 503}
 504
 505static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
 506{
 507	return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
 508}
 509
 510static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
 511{
 512	return rv_blt(rs2, rs1, imm12_1);
 513}
 514
 515static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
 516{
 517	return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
 518}
 519
 520static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
 521{
 522	return rv_bge(rs2, rs1, imm12_1);
 523}
 524
 525static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
 526{
 527	return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);
 528}
 529
 530static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
 531{
 532	return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);
 533}
 534
 535static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
 536{
 537	return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
 538}
 539
 540static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
 541{
 542	return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
 543}
 544
 545static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
 546{
 547	return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
 548}
 549
 550static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
 551{
 552	return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
 553}
 554
 555static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
 556{
 557	return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
 558}
 559
 560static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
 561{
 562	return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
 563}
 564
 565static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 566{
 567	return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
 568}
 569
 570static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 571{
 572	return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);
 573}
 574
 575static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 576{
 577	return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);
 578}
 579
 580static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 581{
 582	return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);
 583}
 584
 585static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 586{
 587	return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);
 588}
 589
 590static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 591{
 592	return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);
 593}
 594
 595static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 596{
 597	return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);
 598}
 599
 600static inline u32 rv_fence(u8 pred, u8 succ)
 601{
 602	u16 imm11_0 = pred << 4 | succ;
 603
 604	return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
 605}
 606
 607static inline void emit_fence_r_rw(struct rv_jit_context *ctx)
 608{
 609	emit(rv_fence(0x2, 0x3), ctx);
 610}
 611
 612static inline void emit_fence_rw_w(struct rv_jit_context *ctx)
 613{
 614	emit(rv_fence(0x3, 0x1), ctx);
 615}
 616
 617static inline void emit_fence_rw_rw(struct rv_jit_context *ctx)
 618{
 619	emit(rv_fence(0x3, 0x3), ctx);
 620}
 621
 622static inline u32 rv_nop(void)
 623{
 624	return rv_i_insn(0, 0, 0, 0, 0x13);
 625}
 626
 627/* RVC instructions. */
 628
 629static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
 630{
 631	u32 imm;
 632
 633	imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
 634		((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
 635	return rv_ciw_insn(0x0, imm, rd, 0x0);
 636}
 637
 638static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
 639{
 640	u32 imm_hi, imm_lo;
 641
 642	imm_hi = (imm7 & 0x38) >> 3;
 643	imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
 644	return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
 645}
 646
 647static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
 648{
 649	u32 imm_hi, imm_lo;
 650
 651	imm_hi = (imm7 & 0x38) >> 3;
 652	imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
 653	return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
 654}
 655
 656static inline u16 rvc_addi(u8 rd, u32 imm6)
 657{
 658	return rv_ci_insn(0, imm6, rd, 0x1);
 659}
 660
 661static inline u16 rvc_li(u8 rd, u32 imm6)
 662{
 663	return rv_ci_insn(0x2, imm6, rd, 0x1);
 664}
 665
 666static inline u16 rvc_addi16sp(u32 imm10)
 667{
 668	u32 imm;
 669
 670	imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
 671		((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
 672	return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
 673}
 674
 675static inline u16 rvc_lui(u8 rd, u32 imm6)
 676{
 677	return rv_ci_insn(0x3, imm6, rd, 0x1);
 678}
 679
 680static inline u16 rvc_srli(u8 rd, u32 imm6)
 681{
 682	return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
 683}
 684
 685static inline u16 rvc_srai(u8 rd, u32 imm6)
 686{
 687	return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
 688}
 689
 690static inline u16 rvc_andi(u8 rd, u32 imm6)
 691{
 692	return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
 693}
 694
 695static inline u16 rvc_sub(u8 rd, u8 rs)
 696{
 697	return rv_ca_insn(0x23, rd, 0, rs, 0x1);
 698}
 699
 700static inline u16 rvc_xor(u8 rd, u8 rs)
 701{
 702	return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
 703}
 704
 705static inline u16 rvc_or(u8 rd, u8 rs)
 706{
 707	return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
 708}
 709
 710static inline u16 rvc_and(u8 rd, u8 rs)
 711{
 712	return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
 713}
 714
 715static inline u16 rvc_slli(u8 rd, u32 imm6)
 716{
 717	return rv_ci_insn(0, imm6, rd, 0x2);
 718}
 719
 720static inline u16 rvc_lwsp(u8 rd, u32 imm8)
 721{
 722	u32 imm;
 723
 724	imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
 725	return rv_ci_insn(0x2, imm, rd, 0x2);
 726}
 727
 728static inline u16 rvc_jr(u8 rs1)
 729{
 730	return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
 731}
 732
 733static inline u16 rvc_mv(u8 rd, u8 rs)
 734{
 735	return rv_cr_insn(0x8, rd, rs, 0x2);
 736}
 737
 738static inline u16 rvc_jalr(u8 rs1)
 739{
 740	return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
 741}
 742
 743static inline u16 rvc_add(u8 rd, u8 rs)
 744{
 745	return rv_cr_insn(0x9, rd, rs, 0x2);
 746}
 747
 748static inline u16 rvc_swsp(u32 imm8, u8 rs2)
 749{
 750	u32 imm;
 751
 752	imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
 753	return rv_css_insn(0x6, imm, rs2, 0x2);
 754}
 755
 756/* RVZACAS instructions. */
 757static inline u32 rvzacas_amocas_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 758{
 759	return rv_amo_insn(0x5, aq, rl, rs2, rs1, 2, rd, 0x2f);
 760}
 761
 762static inline u32 rvzacas_amocas_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 763{
 764	return rv_amo_insn(0x5, aq, rl, rs2, rs1, 3, rd, 0x2f);
 765}
 766
 767/* RVZBA instructions. */
 768static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2)
 769{
 770	return rv_r_insn(0x10, rs2, rs1, 0x4, rd, 0x33);
 771}
 772
 773static inline u32 rvzba_sh3add(u8 rd, u8 rs1, u8 rs2)
 774{
 775	return rv_r_insn(0x10, rs2, rs1, 0x6, rd, 0x33);
 776}
 777
 778/* RVZBB instructions. */
 779static inline u32 rvzbb_sextb(u8 rd, u8 rs1)
 780{
 781	return rv_i_insn(0x604, rs1, 1, rd, 0x13);
 782}
 783
 784static inline u32 rvzbb_sexth(u8 rd, u8 rs1)
 785{
 786	return rv_i_insn(0x605, rs1, 1, rd, 0x13);
 787}
 788
 789static inline u32 rvzbb_zexth(u8 rd, u8 rs)
 790{
 791	if (IS_ENABLED(CONFIG_64BIT))
 792		return rv_i_insn(0x80, rs, 4, rd, 0x3b);
 793
 794	return rv_i_insn(0x80, rs, 4, rd, 0x33);
 795}
 796
 797static inline u32 rvzbb_rev8(u8 rd, u8 rs)
 798{
 799	if (IS_ENABLED(CONFIG_64BIT))
 800		return rv_i_insn(0x6b8, rs, 5, rd, 0x13);
 801
 802	return rv_i_insn(0x698, rs, 5, rd, 0x13);
 803}
 804
 805/*
 806 * RV64-only instructions.
 807 *
 808 * These instructions are not available on RV32.  Wrap them below a #if to
 809 * ensure that the RV32 JIT doesn't emit any of these instructions.
 810 */
 811
 812#if __riscv_xlen == 64
 813
 814static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
 815{
 816	return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
 817}
 818
 819static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
 820{
 821	return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
 822}
 823
 824static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
 825{
 826	return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
 827}
 828
 829static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
 830{
 831	return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
 832}
 833
 834static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
 835{
 836	return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
 837}
 838
 839static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
 840{
 841	return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
 842}
 843
 844static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
 845{
 846	return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
 847}
 848
 849static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
 850{
 851	return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
 852}
 853
 854static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
 855{
 856	return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
 857}
 858
 859static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
 860{
 861	return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
 862}
 863
 864static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
 865{
 866	return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
 867}
 868
 869static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
 870{
 871	return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
 872}
 873
 874static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
 875{
 876	return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
 877}
 878
 879static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
 880{
 881	return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
 882}
 883
 884static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
 885{
 886	return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
 887}
 888
 889static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
 890{
 891	return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
 892}
 893
 894static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
 895{
 896	return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
 897}
 898
 899static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 900{
 901	return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
 902}
 903
 904static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 905{
 906	return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
 907}
 908
 909static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 910{
 911	return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
 912}
 913
 914static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 915{
 916	return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
 917}
 918
 919static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 920{
 921	return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
 922}
 923
 924static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 925{
 926	return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
 927}
 928
 929static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 930{
 931	return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
 932}
 933
 934/* RV64-only RVC instructions. */
 935
 936static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
 937{
 938	u32 imm_hi, imm_lo;
 939
 940	imm_hi = (imm8 & 0x38) >> 3;
 941	imm_lo = (imm8 & 0xc0) >> 6;
 942	return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
 943}
 944
 945static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
 946{
 947	u32 imm_hi, imm_lo;
 948
 949	imm_hi = (imm8 & 0x38) >> 3;
 950	imm_lo = (imm8 & 0xc0) >> 6;
 951	return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
 952}
 953
 954static inline u16 rvc_subw(u8 rd, u8 rs)
 955{
 956	return rv_ca_insn(0x27, rd, 0, rs, 0x1);
 957}
 958
 959static inline u16 rvc_addiw(u8 rd, u32 imm6)
 960{
 961	return rv_ci_insn(0x1, imm6, rd, 0x1);
 962}
 963
 964static inline u16 rvc_ldsp(u8 rd, u32 imm9)
 965{
 966	u32 imm;
 967
 968	imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
 969	return rv_ci_insn(0x3, imm, rd, 0x2);
 970}
 971
 972static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
 973{
 974	u32 imm;
 975
 976	imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
 977	return rv_css_insn(0x7, imm, rs2, 0x2);
 978}
 979
 980/* RV64-only ZBA instructions. */
 981
 982static inline u32 rvzba_zextw(u8 rd, u8 rs1)
 983{
 984	/* add.uw rd, rs1, ZERO */
 985	return rv_r_insn(0x04, RV_REG_ZERO, rs1, 0, rd, 0x3b);
 986}
 987
 988#endif /* __riscv_xlen == 64 */
 989
 990/* Helper functions that emit RVC instructions when possible. */
 991
 992static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
 993{
 994	if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
 995		emitc(rvc_jalr(rs), ctx);
 996	else if (rvc_enabled() && !rd && rs && !imm)
 997		emitc(rvc_jr(rs), ctx);
 998	else
 999		emit(rv_jalr(rd, rs, imm), ctx);
1000}
1001
1002static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
1003{
1004	if (rvc_enabled() && rd && rs)
1005		emitc(rvc_mv(rd, rs), ctx);
1006	else
1007		emit(rv_addi(rd, rs, 0), ctx);
1008}
1009
1010static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1011{
1012	if (rvc_enabled() && rd && rd == rs1 && rs2)
1013		emitc(rvc_add(rd, rs2), ctx);
1014	else
1015		emit(rv_add(rd, rs1, rs2), ctx);
1016}
1017
1018static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1019{
1020	if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
1021		emitc(rvc_addi16sp(imm), ctx);
1022	else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
1023		 !(imm & 0x3) && imm)
1024		emitc(rvc_addi4spn(rd, imm), ctx);
1025	else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
1026		emitc(rvc_addi(rd, imm), ctx);
1027	else
1028		emit(rv_addi(rd, rs, imm), ctx);
1029}
1030
1031static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
1032{
1033	if (rvc_enabled() && rd && is_6b_int(imm))
1034		emitc(rvc_li(rd, imm), ctx);
1035	else
1036		emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
1037}
1038
1039static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
1040{
1041	if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
1042		emitc(rvc_lui(rd, imm), ctx);
1043	else
1044		emit(rv_lui(rd, imm), ctx);
1045}
1046
1047static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1048{
1049	if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
1050		emitc(rvc_slli(rd, imm), ctx);
1051	else
1052		emit(rv_slli(rd, rs, imm), ctx);
1053}
1054
1055static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1056{
1057	if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
1058		emitc(rvc_andi(rd, imm), ctx);
1059	else
1060		emit(rv_andi(rd, rs, imm), ctx);
1061}
1062
1063static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1064{
1065	if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
1066		emitc(rvc_srli(rd, imm), ctx);
1067	else
1068		emit(rv_srli(rd, rs, imm), ctx);
1069}
1070
1071static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1072{
1073	if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
1074		emitc(rvc_srai(rd, imm), ctx);
1075	else
1076		emit(rv_srai(rd, rs, imm), ctx);
1077}
1078
1079static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1080{
1081	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1082		emitc(rvc_sub(rd, rs2), ctx);
1083	else
1084		emit(rv_sub(rd, rs1, rs2), ctx);
1085}
1086
1087static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1088{
1089	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1090		emitc(rvc_or(rd, rs2), ctx);
1091	else
1092		emit(rv_or(rd, rs1, rs2), ctx);
1093}
1094
1095static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1096{
1097	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1098		emitc(rvc_and(rd, rs2), ctx);
1099	else
1100		emit(rv_and(rd, rs1, rs2), ctx);
1101}
1102
1103static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1104{
1105	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1106		emitc(rvc_xor(rd, rs2), ctx);
1107	else
1108		emit(rv_xor(rd, rs1, rs2), ctx);
1109}
1110
1111static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1112{
1113	if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
1114		emitc(rvc_lwsp(rd, off), ctx);
1115	else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
1116		emitc(rvc_lw(rd, off, rs1), ctx);
1117	else
1118		emit(rv_lw(rd, off, rs1), ctx);
1119}
1120
1121static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1122{
1123	if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
1124		emitc(rvc_swsp(off, rs2), ctx);
1125	else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
1126		emitc(rvc_sw(rs1, off, rs2), ctx);
1127	else
1128		emit(rv_sw(rs1, off, rs2), ctx);
1129}
1130
1131static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1132{
1133	if (rv_ext_enabled(ZBA)) {
1134		emit(rvzba_sh2add(rd, rs1, rs2), ctx);
1135		return;
1136	}
1137
1138	emit_slli(rd, rs1, 2, ctx);
1139	emit_add(rd, rd, rs2, ctx);
1140}
1141
1142static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1143{
1144	if (rv_ext_enabled(ZBA)) {
1145		emit(rvzba_sh3add(rd, rs1, rs2), ctx);
1146		return;
1147	}
1148
1149	emit_slli(rd, rs1, 3, ctx);
1150	emit_add(rd, rd, rs2, ctx);
1151}
1152
1153/* RV64-only helper functions. */
1154#if __riscv_xlen == 64
1155
1156static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1157{
1158	if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1159		emitc(rvc_addiw(rd, imm), ctx);
1160	else
1161		emit(rv_addiw(rd, rs, imm), ctx);
1162}
1163
1164static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1165{
1166	if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1167		emitc(rvc_ldsp(rd, off), ctx);
1168	else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1169		emitc(rvc_ld(rd, off, rs1), ctx);
1170	else
1171		emit(rv_ld(rd, off, rs1), ctx);
1172}
1173
1174static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1175{
1176	if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1177		emitc(rvc_sdsp(off, rs2), ctx);
1178	else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1179		emitc(rvc_sd(rs1, off, rs2), ctx);
1180	else
1181		emit(rv_sd(rs1, off, rs2), ctx);
1182}
1183
1184static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1185{
1186	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1187		emitc(rvc_subw(rd, rs2), ctx);
1188	else
1189		emit(rv_subw(rd, rs1, rs2), ctx);
1190}
1191
1192static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx)
1193{
1194	if (rv_ext_enabled(ZBB)) {
1195		emit(rvzbb_sextb(rd, rs), ctx);
1196		return;
1197	}
1198
1199	emit_slli(rd, rs, 56, ctx);
1200	emit_srai(rd, rd, 56, ctx);
1201}
1202
1203static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
1204{
1205	if (rv_ext_enabled(ZBB)) {
1206		emit(rvzbb_sexth(rd, rs), ctx);
1207		return;
1208	}
1209
1210	emit_slli(rd, rs, 48, ctx);
1211	emit_srai(rd, rd, 48, ctx);
1212}
1213
1214static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
1215{
1216	emit_addiw(rd, rs, 0, ctx);
1217}
1218
1219static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
1220{
1221	if (rv_ext_enabled(ZBB)) {
1222		emit(rvzbb_zexth(rd, rs), ctx);
1223		return;
1224	}
1225
1226	emit_slli(rd, rs, 48, ctx);
1227	emit_srli(rd, rd, 48, ctx);
1228}
1229
1230static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
1231{
1232	if (rv_ext_enabled(ZBA)) {
1233		emit(rvzba_zextw(rd, rs), ctx);
1234		return;
1235	}
1236
1237	emit_slli(rd, rs, 32, ctx);
1238	emit_srli(rd, rd, 32, ctx);
1239}
1240
1241static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
1242{
1243	if (rv_ext_enabled(ZBB)) {
1244		int bits = 64 - imm;
1245
1246		emit(rvzbb_rev8(rd, rd), ctx);
1247		if (bits)
1248			emit_srli(rd, rd, bits, ctx);
1249		return;
1250	}
1251
1252	emit_li(RV_REG_T2, 0, ctx);
1253
1254	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1255	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1256	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1257	emit_srli(rd, rd, 8, ctx);
1258	if (imm == 16)
1259		goto out_be;
1260
1261	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1262	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1263	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1264	emit_srli(rd, rd, 8, ctx);
1265
1266	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1267	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1268	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1269	emit_srli(rd, rd, 8, ctx);
1270	if (imm == 32)
1271		goto out_be;
1272
1273	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1274	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1275	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1276	emit_srli(rd, rd, 8, ctx);
1277
1278	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1279	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1280	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1281	emit_srli(rd, rd, 8, ctx);
1282
1283	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1284	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1285	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1286	emit_srli(rd, rd, 8, ctx);
1287
1288	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1289	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1290	emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
1291	emit_srli(rd, rd, 8, ctx);
1292out_be:
1293	emit_andi(RV_REG_T1, rd, 0xff, ctx);
1294	emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
1295
1296	emit_mv(rd, RV_REG_T2, ctx);
1297}
1298
1299static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx)
1300{
1301	int jmp_offset;
1302
1303	if (rv_ext_enabled(ZACAS)) {
1304		ctx->ex_insn_off = ctx->ninsns;
1305		emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) :
1306		     rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx);
1307		ctx->ex_jmp_off = ctx->ninsns;
1308		if (!is64)
1309			emit_zextw(r0, r0, ctx);
1310		return;
1311	}
1312
1313	if (is64)
1314		emit_mv(RV_REG_T2, r0, ctx);
1315	else
1316		emit_addiw(RV_REG_T2, r0, 0, ctx);
1317	emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
1318	     rv_lr_w(r0, 0, rd, 0, 0), ctx);
1319	jmp_offset = ninsns_rvoff(8);
1320	emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
1321	emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
1322	     rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
1323	jmp_offset = ninsns_rvoff(-6);
1324	emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
1325	emit_fence_rw_rw(ctx);
1326}
1327
1328#endif /* __riscv_xlen == 64 */
1329
1330void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);
1331void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1332
1333int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1334		      bool extra_pass);
1335
1336#endif /* _BPF_JIT_H */