bpf_jit_core.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Common functionality for HPPA32 and HPPA64 BPF JIT compilers
  4 *
  5 * Copyright (c) 2023 Helge Deller <deller@gmx.de>
  6 *
  7 */
  8
  9#include <linux/bpf.h>
 10#include <linux/filter.h>
 11#include "bpf_jit.h"
 12
 13/* Number of iterations to try until offsets converge. */
 14#define NR_JIT_ITERATIONS	35
 15
 16static int build_body(struct hppa_jit_context *ctx, bool extra_pass, int *offset)
 17{
 18	const struct bpf_prog *prog = ctx->prog;
 19	int i;
 20
 21	ctx->reg_seen_collect = true;
 22	for (i = 0; i < prog->len; i++) {
 23		const struct bpf_insn *insn = &prog->insnsi[i];
 24		int ret;
 25
 26		ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
 27		/* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
 28		if (ret > 0)
 29			i++;
 30		if (offset)
 31			offset[i] = ctx->ninsns;
 32		if (ret < 0)
 33			return ret;
 34	}
 35	ctx->reg_seen_collect = false;
 36	return 0;
 37}
 38
 39bool bpf_jit_needs_zext(void)
 40{
 41	return true;
 42}
 43
 44struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 45{
 46	unsigned int prog_size = 0, extable_size = 0;
 47	bool tmp_blinded = false, extra_pass = false;
 48	struct bpf_prog *tmp, *orig_prog = prog;
 49	int pass = 0, prev_ninsns = 0, prologue_len, i;
 50	struct hppa_jit_data *jit_data;
 51	struct hppa_jit_context *ctx;
 52
 53	if (!prog->jit_requested)
 54		return orig_prog;
 55
 56	tmp = bpf_jit_blind_constants(prog);
 57	if (IS_ERR(tmp))
 58		return orig_prog;
 59	if (tmp != prog) {
 60		tmp_blinded = true;
 61		prog = tmp;
 62	}
 63
 64	jit_data = prog->aux->jit_data;
 65	if (!jit_data) {
 66		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
 67		if (!jit_data) {
 68			prog = orig_prog;
 69			goto out;
 70		}
 71		prog->aux->jit_data = jit_data;
 72	}
 73
 74	ctx = &jit_data->ctx;
 75
 76	if (ctx->offset) {
 77		extra_pass = true;
 78		prog_size = sizeof(*ctx->insns) * ctx->ninsns;
 79		goto skip_init_ctx;
 80	}
 81
 82	ctx->prog = prog;
 83	ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
 84	if (!ctx->offset) {
 85		prog = orig_prog;
 86		goto out_offset;
 87	}
 88	for (i = 0; i < prog->len; i++) {
 89		prev_ninsns += 20;
 90		ctx->offset[i] = prev_ninsns;
 91	}
 92
 93	for (i = 0; i < NR_JIT_ITERATIONS; i++) {
 94		pass++;
 95		ctx->ninsns = 0;
 96		if (build_body(ctx, extra_pass, ctx->offset)) {
 97			prog = orig_prog;
 98			goto out_offset;
 99		}
100		ctx->body_len = ctx->ninsns;
101		bpf_jit_build_prologue(ctx);
102		ctx->prologue_len = ctx->ninsns - ctx->body_len;
103		ctx->epilogue_offset = ctx->ninsns;
104		bpf_jit_build_epilogue(ctx);
105
106		if (ctx->ninsns == prev_ninsns) {
107			if (jit_data->header)
108				break;
109			/* obtain the actual image size */
110			extable_size = prog->aux->num_exentries *
111				sizeof(struct exception_table_entry);
112			prog_size = sizeof(*ctx->insns) * ctx->ninsns;
113
114			jit_data->header =
115				bpf_jit_binary_alloc(prog_size + extable_size,
116						     &jit_data->image,
117						     sizeof(long),
118						     bpf_fill_ill_insns);
119			if (!jit_data->header) {
120				prog = orig_prog;
121				goto out_offset;
122			}
123
124			ctx->insns = (u32 *)jit_data->image;
125			/*
126			 * Now, when the image is allocated, the image can
127			 * potentially shrink more (auipc/jalr -> jal).
128			 */
129		}
130		prev_ninsns = ctx->ninsns;
131	}
132
133	if (i == NR_JIT_ITERATIONS) {
134		pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
135		if (jit_data->header)
136			bpf_jit_binary_free(jit_data->header);
137		prog = orig_prog;
138		goto out_offset;
139	}
140
141	if (extable_size)
142		prog->aux->extable = (void *)ctx->insns + prog_size;
143
144skip_init_ctx:
145	pass++;
146	ctx->ninsns = 0;
147
148	bpf_jit_build_prologue(ctx);
149	if (build_body(ctx, extra_pass, NULL)) {
150		bpf_jit_binary_free(jit_data->header);
151		prog = orig_prog;
152		goto out_offset;
153	}
154	bpf_jit_build_epilogue(ctx);
155
156	if (HPPA_JIT_DEBUG || bpf_jit_enable > 1) {
157		if (HPPA_JIT_DUMP)
158			bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
159		if (HPPA_JIT_REBOOT)
160			{ extern int machine_restart(char *); machine_restart(""); }
161	}
162
163	prog->bpf_func = (void *)ctx->insns;
164	prog->jited = 1;
165	prog->jited_len = prog_size;
166
167	bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
168
169	if (!prog->is_func || extra_pass) {
170		if (bpf_jit_binary_lock_ro(jit_data->header)) {
171			bpf_jit_binary_free(jit_data->header);
172			prog->bpf_func = NULL;
173			prog->jited = 0;
174			prog->jited_len = 0;
175			goto out_offset;
176		}
177		prologue_len = ctx->epilogue_offset - ctx->body_len;
178		for (i = 0; i < prog->len; i++)
179			ctx->offset[i] += prologue_len;
180		bpf_prog_fill_jited_linfo(prog, ctx->offset);
181out_offset:
182		kfree(ctx->offset);
183		kfree(jit_data);
184		prog->aux->jit_data = NULL;
185	}
186out:
187	if (HPPA_JIT_REBOOT)
188		{ extern int machine_restart(char *); machine_restart(""); }
189
190	if (tmp_blinded)
191		bpf_jit_prog_release_other(prog, prog == orig_prog ?
192					   tmp : orig_prog);
193	return prog;
194}
195
196u64 hppa_div64(u64 div, u64 divisor)
197{
198	div = div64_u64(div, divisor);
199	return div;
200}
201
202u64 hppa_div64_rem(u64 div, u64 divisor)
203{
204	u64 rem;
205	div64_u64_rem(div, divisor, &rem);
206	return rem;
207}