vector.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Copyright (C) 2023 SiFive
  4 * Author: Andy Chiu <andy.chiu@sifive.com>
  5 */
  6#include <linux/export.h>
  7#include <linux/sched/signal.h>
  8#include <linux/types.h>
  9#include <linux/slab.h>
 10#include <linux/sched.h>
 11#include <linux/uaccess.h>
 12#include <linux/prctl.h>
 13
 14#include <asm/thread_info.h>
 15#include <asm/processor.h>
 16#include <asm/insn.h>
 17#include <asm/vector.h>
 18#include <asm/csr.h>
 19#include <asm/elf.h>
 20#include <asm/ptrace.h>
 21#include <asm/bug.h>
 22
 23static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
 24static struct kmem_cache *riscv_v_user_cachep;
 25#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
 26static struct kmem_cache *riscv_v_kernel_cachep;
 27#endif
 28
 29unsigned long riscv_v_vsize __read_mostly;
 30EXPORT_SYMBOL_GPL(riscv_v_vsize);
 31
 32int riscv_v_setup_vsize(void)
 33{
 34	unsigned long this_vsize;
 35
 36	/*
 37	 * There are 32 vector registers with vlenb length.
 38	 *
 39	 * If the thead,vlenb property was provided by the firmware, use that
 40	 * instead of probing the CSRs.
 41	 */
 42	if (thead_vlenb_of) {
 43		riscv_v_vsize = thead_vlenb_of * 32;
 44		return 0;
 45	}
 46
 47	riscv_v_enable();
 48	this_vsize = csr_read(CSR_VLENB) * 32;
 49	riscv_v_disable();
 50
 51	if (!riscv_v_vsize) {
 52		riscv_v_vsize = this_vsize;
 53		return 0;
 54	}
 55
 56	if (riscv_v_vsize != this_vsize) {
 57		WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
 58		return -EOPNOTSUPP;
 59	}
 60
 61	return 0;
 62}
 63
 64void __init riscv_v_setup_ctx_cache(void)
 65{
 66	if (!(has_vector() || has_xtheadvector()))
 67		return;
 68
 69	update_regset_vector_info(riscv_v_vsize);
 70
 71	riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx",
 72							 riscv_v_vsize, 16, SLAB_PANIC,
 73							 0, riscv_v_vsize, NULL);
 74#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
 75	riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx",
 76						  riscv_v_vsize, 16,
 77						  SLAB_PANIC, NULL);
 78#endif
 79}
 80
 81bool insn_is_vector(u32 insn_buf)
 82{
 83	u32 opcode = insn_buf & __INSN_OPCODE_MASK;
 84	u32 width, csr;
 85
 86	/*
 87	 * All V-related instructions, including CSR operations are 4-Byte. So,
 88	 * do not handle if the instruction length is not 4-Byte.
 89	 */
 90	if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
 91		return false;
 92
 93	switch (opcode) {
 94	case RVV_OPCODE_VECTOR:
 95		return true;
 96	case RVV_OPCODE_VL:
 97	case RVV_OPCODE_VS:
 98		width = RVV_EXTRACT_VL_VS_WIDTH(insn_buf);
 99		if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
100		    width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
101			return true;
102
103		break;
104	case RVG_OPCODE_SYSTEM:
105		csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
106		if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
107		    (csr >= CSR_VL && csr <= CSR_VLENB))
108			return true;
109	}
110
111	return false;
112}
113
114static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache,
115				    struct __riscv_v_ext_state *ctx)
116{
117	void *datap;
118
119	datap = kmem_cache_zalloc(cache, GFP_KERNEL);
120	if (!datap)
121		return -ENOMEM;
122
123	ctx->datap = datap;
124	memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap));
125	ctx->vlenb = riscv_v_vsize / 32;
126
127	return 0;
128}
129
130void riscv_v_thread_alloc(struct task_struct *tsk)
131{
132#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
133	riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);
134#endif
135}
136
137void riscv_v_thread_free(struct task_struct *tsk)
138{
139	if (tsk->thread.vstate.datap)
140		kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap);
141#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
142	if (tsk->thread.kernel_vstate.datap)
143		kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);
144#endif
145}
146
147#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
148#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
149#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
150#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
151static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
152{
153	return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
154}
155
156static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
157{
158	return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
159}
160
161static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
162{
163	return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
164}
165
166static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
167				    bool inherit)
168{
169	unsigned long ctrl;
170
171	ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
172	ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
173	if (inherit)
174		ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
175	tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK;
176	tsk->thread.vstate_ctrl |= ctrl;
177}
178
179bool riscv_v_vstate_ctrl_user_allowed(void)
180{
181	return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
182}
183EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
184
185bool riscv_v_first_use_handler(struct pt_regs *regs)
186{
187	u32 __user *epc = (u32 __user *)regs->epc;
188	u32 insn = (u32)regs->badaddr;
189
190	if (!(has_vector() || has_xtheadvector()))
191		return false;
192
193	/* Do not handle if V is not supported, or disabled */
194	if (!riscv_v_vstate_ctrl_user_allowed())
195		return false;
196
197	/* If V has been enabled then it is not the first-use trap */
198	if (riscv_v_vstate_query(regs))
199		return false;
200
201	/* Get the instruction */
202	if (!insn) {
203		if (__get_user(insn, epc))
204			return false;
205	}
206
207	/* Filter out non-V instructions */
208	if (!insn_is_vector(insn))
209		return false;
210
211	/* Sanity check. datap should be null by the time of the first-use trap */
212	WARN_ON(current->thread.vstate.datap);
213
214	/*
215	 * Now we sure that this is a V instruction. And it executes in the
216	 * context where VS has been off. So, try to allocate the user's V
217	 * context and resume execution.
218	 */
219	if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, &current->thread.vstate)) {
220		force_sig(SIGBUS);
221		return true;
222	}
223
224	riscv_v_vstate_on(regs);
225	riscv_v_vstate_set_restore(current, regs);
226
227	return true;
228}
229
230void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
231{
232	bool inherit;
233	int cur, next;
234
235	if (!(has_vector() || has_xtheadvector()))
236		return;
237
238	next = riscv_v_ctrl_get_next(tsk);
239	if (!next) {
240		if (READ_ONCE(riscv_v_implicit_uacc))
241			cur = PR_RISCV_V_VSTATE_CTRL_ON;
242		else
243			cur = PR_RISCV_V_VSTATE_CTRL_OFF;
244	} else {
245		cur = next;
246	}
247	/* Clear next mask if inherit-bit is not set */
248	inherit = riscv_v_ctrl_test_inherit(tsk);
249	if (!inherit)
250		next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
251
252	riscv_v_ctrl_set(tsk, cur, next, inherit);
253}
254
255long riscv_v_vstate_ctrl_get_current(void)
256{
257	if (!(has_vector() || has_xtheadvector()))
258		return -EINVAL;
259
260	return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
261}
262
263long riscv_v_vstate_ctrl_set_current(unsigned long arg)
264{
265	bool inherit;
266	int cur, next;
267
268	if (!(has_vector() || has_xtheadvector()))
269		return -EINVAL;
270
271	if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
272		return -EINVAL;
273
274	cur = VSTATE_CTRL_GET_CUR(arg);
275	switch (cur) {
276	case PR_RISCV_V_VSTATE_CTRL_OFF:
277		/* Do not allow user to turn off V if current is not off */
278		if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
279			return -EPERM;
280
281		break;
282	case PR_RISCV_V_VSTATE_CTRL_ON:
283		break;
284	case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
285		cur = riscv_v_ctrl_get_cur(current);
286		break;
287	default:
288		return -EINVAL;
289	}
290
291	next = VSTATE_CTRL_GET_NEXT(arg);
292	inherit = VSTATE_CTRL_GET_INHERIT(arg);
293	switch (next) {
294	case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
295	case PR_RISCV_V_VSTATE_CTRL_OFF:
296	case PR_RISCV_V_VSTATE_CTRL_ON:
297		riscv_v_ctrl_set(current, cur, next, inherit);
298		return 0;
299	}
300
301	return -EINVAL;
302}
303
304#ifdef CONFIG_SYSCTL
305
306static const struct ctl_table riscv_v_default_vstate_table[] = {
307	{
308		.procname	= "riscv_v_default_allow",
309		.data		= &riscv_v_implicit_uacc,
310		.maxlen		= sizeof(riscv_v_implicit_uacc),
311		.mode		= 0644,
312		.proc_handler	= proc_dobool,
313	},
314};
315
316static int __init riscv_v_sysctl_init(void)
317{
318	if (has_vector() || has_xtheadvector())
319		if (!register_sysctl("abi", riscv_v_default_vstate_table))
320			return -EINVAL;
321	return 0;
322}
323
324#else /* ! CONFIG_SYSCTL */
325static int __init riscv_v_sysctl_init(void) { return 0; }
326#endif /* ! CONFIG_SYSCTL */
327
328static int __init riscv_v_init(void)
329{
330	return riscv_v_sysctl_init();
331}
332core_initcall(riscv_v_init);