vector.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/*
  3 * Copyright (C) 2020 SiFive
  4 */
  5
  6#ifndef __ASM_RISCV_VECTOR_H
  7#define __ASM_RISCV_VECTOR_H
  8
  9#include <linux/types.h>
 10#include <uapi/asm-generic/errno.h>
 11
 12#ifdef CONFIG_RISCV_ISA_V
 13
 14#include <linux/stringify.h>
 15#include <linux/sched.h>
 16#include <linux/sched/task_stack.h>
 17#include <asm/ptrace.h>
 18#include <asm/cpufeature.h>
 19#include <asm/csr.h>
 20#include <asm/asm.h>
 21#include <asm/vendorid_list.h>
 22#include <asm/vendor_extensions.h>
 23#include <asm/vendor_extensions/thead.h>
 24
 25#define __riscv_v_vstate_or(_val, TYPE) ({				\
 26	typeof(_val) _res = _val;					\
 27	if (has_xtheadvector()) \
 28		_res = (_res & ~SR_VS_THEAD) | SR_VS_##TYPE##_THEAD;	\
 29	else								\
 30		_res = (_res & ~SR_VS) | SR_VS_##TYPE;			\
 31	_res;								\
 32})
 33
 34#define __riscv_v_vstate_check(_val, TYPE) ({				\
 35	bool _res;							\
 36	if (has_xtheadvector()) \
 37		_res = ((_val) & SR_VS_THEAD) == SR_VS_##TYPE##_THEAD;	\
 38	else								\
 39		_res = ((_val) & SR_VS) == SR_VS_##TYPE;		\
 40	_res;								\
 41})
 42
 43extern unsigned long riscv_v_vsize;
 44int riscv_v_setup_vsize(void);
 45bool insn_is_vector(u32 insn_buf);
 46bool riscv_v_first_use_handler(struct pt_regs *regs);
 47void kernel_vector_begin(void);
 48void kernel_vector_end(void);
 49void get_cpu_vector_context(void);
 50void put_cpu_vector_context(void);
 51void riscv_v_thread_free(struct task_struct *tsk);
 52void __init riscv_v_setup_ctx_cache(void);
 53void riscv_v_thread_alloc(struct task_struct *tsk);
 54void __init update_regset_vector_info(unsigned long size);
 55
 56static inline u32 riscv_v_flags(void)
 57{
 58	return READ_ONCE(current->thread.riscv_v_flags);
 59}
 60
 61static __always_inline bool has_vector(void)
 62{
 63	return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE32X);
 64}
 65
 66static __always_inline bool has_xtheadvector_no_alternatives(void)
 67{
 68	if (IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR))
 69		return riscv_isa_vendor_extension_available(THEAD_VENDOR_ID, XTHEADVECTOR);
 70	else
 71		return false;
 72}
 73
 74static __always_inline bool has_xtheadvector(void)
 75{
 76	if (IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR))
 77		return riscv_has_vendor_extension_unlikely(THEAD_VENDOR_ID,
 78							   RISCV_ISA_VENDOR_EXT_XTHEADVECTOR);
 79	else
 80		return false;
 81}
 82
 83static inline void __riscv_v_vstate_clean(struct pt_regs *regs)
 84{
 85	regs->status = __riscv_v_vstate_or(regs->status, CLEAN);
 86}
 87
 88static inline void __riscv_v_vstate_dirty(struct pt_regs *regs)
 89{
 90	regs->status = __riscv_v_vstate_or(regs->status, DIRTY);
 91}
 92
 93static inline void riscv_v_vstate_off(struct pt_regs *regs)
 94{
 95	regs->status = __riscv_v_vstate_or(regs->status, OFF);
 96}
 97
 98static inline void riscv_v_vstate_on(struct pt_regs *regs)
 99{
100	regs->status = __riscv_v_vstate_or(regs->status, INITIAL);
101}
102
103static inline bool riscv_v_vstate_query(struct pt_regs *regs)
104{
105	return !__riscv_v_vstate_check(regs->status, OFF);
106}
107
108static __always_inline void riscv_v_enable(void)
109{
110	if (has_xtheadvector())
111		csr_set(CSR_SSTATUS, SR_VS_THEAD);
112	else
113		csr_set(CSR_SSTATUS, SR_VS);
114}
115
116static __always_inline void riscv_v_disable(void)
117{
118	if (has_xtheadvector())
119		csr_clear(CSR_SSTATUS, SR_VS_THEAD);
120	else
121		csr_clear(CSR_SSTATUS, SR_VS);
122}
123
124static __always_inline bool riscv_v_is_on(void)
125{
126	return !!(csr_read(CSR_SSTATUS) & SR_VS);
127}
128
129static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest)
130{
131	asm volatile (
132		"csrr	%0, " __stringify(CSR_VSTART) "\n\t"
133		"csrr	%1, " __stringify(CSR_VTYPE) "\n\t"
134		"csrr	%2, " __stringify(CSR_VL) "\n\t"
135		: "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl),
136		"=r" (dest->vcsr) : :);
137
138	if (has_xtheadvector()) {
139		unsigned long status;
140
141		/*
142		 * CSR_VCSR is defined as
143		 * [2:1] - vxrm[1:0]
144		 * [0] - vxsat
145		 * The earlier vector spec implemented by T-Head uses separate
146		 * registers for the same bit-elements, so just combine those
147		 * into the existing output field.
148		 *
149		 * Additionally T-Head cores need FS to be enabled when accessing
150		 * the VXRM and VXSAT CSRs, otherwise ending in illegal instructions.
151		 * Though the cores do not implement the VXRM and VXSAT fields in the
152		 * FCSR CSR that vector-0.7.1 specifies.
153		 */
154		status = csr_read_set(CSR_STATUS, SR_FS_DIRTY);
155		dest->vcsr = csr_read(CSR_VXSAT) | csr_read(CSR_VXRM) << CSR_VXRM_SHIFT;
156
157		dest->vlenb = riscv_v_vsize / 32;
158
159		if ((status & SR_FS) != SR_FS_DIRTY)
160			csr_write(CSR_STATUS, status);
161	} else {
162		dest->vcsr = csr_read(CSR_VCSR);
163		dest->vlenb = csr_read(CSR_VLENB);
164	}
165}
166
167static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src)
168{
169	asm volatile (
170		".option push\n\t"
171		".option arch, +zve32x\n\t"
172		"vsetvl	 x0, %2, %1\n\t"
173		".option pop\n\t"
174		"csrw	" __stringify(CSR_VSTART) ", %0\n\t"
175		: : "r" (src->vstart), "r" (src->vtype), "r" (src->vl));
176
177	if (has_xtheadvector()) {
178		unsigned long status = csr_read(CSR_SSTATUS);
179
180		/*
181		 * Similar to __vstate_csr_save above, restore values for the
182		 * separate VXRM and VXSAT CSRs from the vcsr variable.
183		 */
184		status = csr_read_set(CSR_STATUS, SR_FS_DIRTY);
185
186		csr_write(CSR_VXRM, (src->vcsr >> CSR_VXRM_SHIFT) & CSR_VXRM_MASK);
187		csr_write(CSR_VXSAT, src->vcsr & CSR_VXSAT_MASK);
188
189		if ((status & SR_FS) != SR_FS_DIRTY)
190			csr_write(CSR_STATUS, status);
191	} else {
192		csr_write(CSR_VCSR, src->vcsr);
193	}
194}
195
196static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
197					 void *datap)
198{
199	unsigned long vl;
200
201	riscv_v_enable();
202	__vstate_csr_save(save_to);
203	if (has_xtheadvector()) {
204		asm volatile (
205			"mv t0, %0\n\t"
206			THEAD_VSETVLI_T4X0E8M8D1
207			THEAD_VSB_V_V0T0
208			"add		t0, t0, t4\n\t"
209			THEAD_VSB_V_V8T0
210			"add		t0, t0, t4\n\t"
211			THEAD_VSB_V_V16T0
212			"add		t0, t0, t4\n\t"
213			THEAD_VSB_V_V24T0
214			: : "r" (datap) : "memory", "t0", "t4");
215	} else {
216		asm volatile (
217			".option push\n\t"
218			".option arch, +zve32x\n\t"
219			"vsetvli	%0, x0, e8, m8, ta, ma\n\t"
220			"vse8.v		v0, (%1)\n\t"
221			"add		%1, %1, %0\n\t"
222			"vse8.v		v8, (%1)\n\t"
223			"add		%1, %1, %0\n\t"
224			"vse8.v		v16, (%1)\n\t"
225			"add		%1, %1, %0\n\t"
226			"vse8.v		v24, (%1)\n\t"
227			".option pop\n\t"
228			: "=&r" (vl) : "r" (datap) : "memory");
229	}
230	riscv_v_disable();
231}
232
233static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from,
234					    void *datap)
235{
236	unsigned long vl;
237
238	riscv_v_enable();
239	if (has_xtheadvector()) {
240		asm volatile (
241			"mv t0, %0\n\t"
242			THEAD_VSETVLI_T4X0E8M8D1
243			THEAD_VLB_V_V0T0
244			"add		t0, t0, t4\n\t"
245			THEAD_VLB_V_V8T0
246			"add		t0, t0, t4\n\t"
247			THEAD_VLB_V_V16T0
248			"add		t0, t0, t4\n\t"
249			THEAD_VLB_V_V24T0
250			: : "r" (datap) : "memory", "t0", "t4");
251	} else {
252		asm volatile (
253			".option push\n\t"
254			".option arch, +zve32x\n\t"
255			"vsetvli	%0, x0, e8, m8, ta, ma\n\t"
256			"vle8.v		v0, (%1)\n\t"
257			"add		%1, %1, %0\n\t"
258			"vle8.v		v8, (%1)\n\t"
259			"add		%1, %1, %0\n\t"
260			"vle8.v		v16, (%1)\n\t"
261			"add		%1, %1, %0\n\t"
262			"vle8.v		v24, (%1)\n\t"
263			".option pop\n\t"
264			: "=&r" (vl) : "r" (datap) : "memory");
265	}
266	__vstate_csr_restore(restore_from);
267	riscv_v_disable();
268}
269
270static inline void __riscv_v_vstate_discard(void)
271{
272	unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1);
273
274	riscv_v_enable();
275	if (has_xtheadvector())
276		asm volatile (THEAD_VSETVLI_T4X0E8M8D1 : : : "t4");
277	else
278		asm volatile (
279			".option push\n\t"
280			".option arch, +zve32x\n\t"
281			"vsetvli	%0, x0, e8, m8, ta, ma\n\t"
282			".option pop\n\t": "=&r" (vl));
283
284	asm volatile (
285		".option push\n\t"
286		".option arch, +zve32x\n\t"
287		"vmv.v.i	v0, -1\n\t"
288		"vmv.v.i	v8, -1\n\t"
289		"vmv.v.i	v16, -1\n\t"
290		"vmv.v.i	v24, -1\n\t"
291		"vsetvl		%0, x0, %1\n\t"
292		".option pop\n\t"
293		: "=&r" (vl) : "r" (vtype_inval));
294
295	riscv_v_disable();
296}
297
298static inline void riscv_v_vstate_discard(struct pt_regs *regs)
299{
300	if (riscv_v_vstate_query(regs)) {
301		__riscv_v_vstate_discard();
302		__riscv_v_vstate_dirty(regs);
303	}
304}
305
306static inline void riscv_v_vstate_save(struct __riscv_v_ext_state *vstate,
307				       struct pt_regs *regs)
308{
309	if (__riscv_v_vstate_check(regs->status, DIRTY)) {
310		__riscv_v_vstate_save(vstate, vstate->datap);
311		__riscv_v_vstate_clean(regs);
312	}
313}
314
315static inline void riscv_v_vstate_restore(struct __riscv_v_ext_state *vstate,
316					  struct pt_regs *regs)
317{
318	if (riscv_v_vstate_query(regs)) {
319		__riscv_v_vstate_restore(vstate, vstate->datap);
320		__riscv_v_vstate_clean(regs);
321	}
322}
323
324static inline void riscv_v_vstate_set_restore(struct task_struct *task,
325					      struct pt_regs *regs)
326{
327	if (riscv_v_vstate_query(regs)) {
328		set_tsk_thread_flag(task, TIF_RISCV_V_DEFER_RESTORE);
329		riscv_v_vstate_on(regs);
330	}
331}
332
333#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
334static inline bool riscv_preempt_v_dirty(struct task_struct *task)
335{
336	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_DIRTY);
337}
338
339static inline bool riscv_preempt_v_restore(struct task_struct *task)
340{
341	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V_NEED_RESTORE);
342}
343
344static inline void riscv_preempt_v_clear_dirty(struct task_struct *task)
345{
346	barrier();
347	task->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_DIRTY;
348}
349
350static inline void riscv_preempt_v_set_restore(struct task_struct *task)
351{
352	barrier();
353	task->thread.riscv_v_flags |= RISCV_PREEMPT_V_NEED_RESTORE;
354}
355
356static inline bool riscv_preempt_v_started(struct task_struct *task)
357{
358	return !!(task->thread.riscv_v_flags & RISCV_PREEMPT_V);
359}
360
361#else /* !CONFIG_RISCV_ISA_V_PREEMPTIVE */
362static inline bool riscv_preempt_v_dirty(struct task_struct *task) { return false; }
363static inline bool riscv_preempt_v_restore(struct task_struct *task) { return false; }
364static inline bool riscv_preempt_v_started(struct task_struct *task) { return false; }
365#define riscv_preempt_v_clear_dirty(tsk)	do {} while (0)
366#define riscv_preempt_v_set_restore(tsk)	do {} while (0)
367#endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */
368
369static inline void __switch_to_vector(struct task_struct *prev,
370				      struct task_struct *next)
371{
372	struct pt_regs *regs;
373
374	if (riscv_preempt_v_started(prev)) {
375		if (riscv_v_is_on()) {
376			WARN_ON(prev->thread.riscv_v_flags & RISCV_V_CTX_DEPTH_MASK);
377			riscv_v_disable();
378			prev->thread.riscv_v_flags |= RISCV_PREEMPT_V_IN_SCHEDULE;
379		}
380		if (riscv_preempt_v_dirty(prev)) {
381			__riscv_v_vstate_save(&prev->thread.kernel_vstate,
382					      prev->thread.kernel_vstate.datap);
383			riscv_preempt_v_clear_dirty(prev);
384		}
385	} else {
386		regs = task_pt_regs(prev);
387		riscv_v_vstate_save(&prev->thread.vstate, regs);
388	}
389
390	if (riscv_preempt_v_started(next)) {
391		if (next->thread.riscv_v_flags & RISCV_PREEMPT_V_IN_SCHEDULE) {
392			next->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_IN_SCHEDULE;
393			riscv_v_enable();
394		} else {
395			riscv_preempt_v_set_restore(next);
396		}
397	} else {
398		riscv_v_vstate_set_restore(next, task_pt_regs(next));
399	}
400}
401
402void riscv_v_vstate_ctrl_init(struct task_struct *tsk);
403bool riscv_v_vstate_ctrl_user_allowed(void);
404
405#else /* ! CONFIG_RISCV_ISA_V  */
406
407struct pt_regs;
408
409static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }
410static __always_inline bool has_vector(void) { return false; }
411static __always_inline bool insn_is_vector(u32 insn_buf) { return false; }
412static __always_inline bool has_xtheadvector_no_alternatives(void) { return false; }
413static __always_inline bool has_xtheadvector(void) { return false; }
414static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; }
415static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; }
416static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }
417#define riscv_v_vsize (0)
418#define riscv_v_vstate_discard(regs)		do {} while (0)
419#define riscv_v_vstate_save(vstate, regs)	do {} while (0)
420#define riscv_v_vstate_restore(vstate, regs)	do {} while (0)
421#define __switch_to_vector(__prev, __next)	do {} while (0)
422#define riscv_v_vstate_off(regs)		do {} while (0)
423#define riscv_v_vstate_on(regs)			do {} while (0)
424#define riscv_v_thread_free(tsk)		do {} while (0)
425#define  riscv_v_setup_ctx_cache()		do {} while (0)
426#define riscv_v_thread_alloc(tsk)		do {} while (0)
427#define get_cpu_vector_context()		do {} while (0)
428#define put_cpu_vector_context()		do {} while (0)
429#define riscv_v_vstate_set_restore(task, regs)	do {} while (0)
430
431#endif /* CONFIG_RISCV_ISA_V */
432
433/*
434 * Return the implementation's vlen value.
435 *
436 * riscv_v_vsize contains the value of "32 vector registers with vlenb length"
437 * so rebuild the vlen value in bits from it.
438 */
439static inline int riscv_vector_vlen(void)
440{
441	return riscv_v_vsize / 32 * 8;
442}
443
444#endif /* ! __ASM_RISCV_VECTOR_H */