fpsimd.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2012 ARM Ltd.
  4 */
  5#ifndef __ASM_FP_H
  6#define __ASM_FP_H
  7
  8#include <asm/errno.h>
  9#include <asm/percpu.h>
 10#include <asm/ptrace.h>
 11#include <asm/processor.h>
 12#include <asm/sigcontext.h>
 13#include <asm/sysreg.h>
 14
 15#ifndef __ASSEMBLER__
 16
 17#include <linux/bitmap.h>
 18#include <linux/build_bug.h>
 19#include <linux/bug.h>
 20#include <linux/cache.h>
 21#include <linux/init.h>
 22#include <linux/stddef.h>
 23#include <linux/types.h>
 24
 25/* Masks for extracting the FPSR and FPCR from the FPSCR */
 26#define VFP_FPSCR_STAT_MASK	0xf800009f
 27#define VFP_FPSCR_CTRL_MASK	0x07f79f00
 28/*
 29 * The VFP state has 32x64-bit registers and a single 32-bit
 30 * control/status register.
 31 */
 32#define VFP_STATE_SIZE		((32 * 8) + 4)
 33
 34static inline unsigned long cpacr_save_enable_kernel_sve(void)
 35{
 36	unsigned long old = read_sysreg(cpacr_el1);
 37	unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN;
 38
 39	write_sysreg(old | set, cpacr_el1);
 40	isb();
 41	return old;
 42}
 43
 44static inline unsigned long cpacr_save_enable_kernel_sme(void)
 45{
 46	unsigned long old = read_sysreg(cpacr_el1);
 47	unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN;
 48
 49	write_sysreg(old | set, cpacr_el1);
 50	isb();
 51	return old;
 52}
 53
 54static inline void cpacr_restore(unsigned long cpacr)
 55{
 56	write_sysreg(cpacr, cpacr_el1);
 57	isb();
 58}
 59
 60/*
 61 * When we defined the maximum SVE vector length we defined the ABI so
 62 * that the maximum vector length included all the reserved for future
 63 * expansion bits in ZCR rather than those just currently defined by
 64 * the architecture.  Using this length to allocate worst size buffers
 65 * results in excessively large allocations, and this effect is even
 66 * more pronounced for SME due to ZA.  Define more suitable VLs for
 67 * these situations.
 68 */
 69#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
 70#define SME_VQ_MAX	((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
 71
 72struct task_struct;
 73
 74extern void fpsimd_save_state(struct user_fpsimd_state *state);
 75extern void fpsimd_load_state(struct user_fpsimd_state *state);
 76
 77extern void fpsimd_thread_switch(struct task_struct *next);
 78extern void fpsimd_flush_thread(void);
 79
 80extern void fpsimd_preserve_current_state(void);
 81extern void fpsimd_restore_current_state(void);
 82extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
 83
 84struct cpu_fp_state {
 85	struct user_fpsimd_state *st;
 86	void *sve_state;
 87	void *sme_state;
 88	u64 *svcr;
 89	u64 *fpmr;
 90	unsigned int sve_vl;
 91	unsigned int sme_vl;
 92	enum fp_type *fp_type;
 93	enum fp_type to_save;
 94};
 95
 96DECLARE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
 97
 98extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
 99
100extern void fpsimd_flush_task_state(struct task_struct *target);
101extern void fpsimd_save_and_flush_current_state(void);
102extern void fpsimd_save_and_flush_cpu_state(void);
103
104static inline bool thread_sm_enabled(struct thread_struct *thread)
105{
106	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
107}
108
109static inline bool thread_za_enabled(struct thread_struct *thread)
110{
111	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
112}
113
114extern void task_smstop_sm(struct task_struct *task);
115
116/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
117#define VL_ARCH_MAX 0x100
118
119/* Offset of FFR in the SVE register dump */
120static inline size_t sve_ffr_offset(int vl)
121{
122	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
123}
124
125static inline void *sve_pffr(struct thread_struct *thread)
126{
127	unsigned int vl;
128
129	if (system_supports_sme() && thread_sm_enabled(thread))
130		vl = thread_get_sme_vl(thread);
131	else
132		vl = thread_get_sve_vl(thread);
133
134	return (char *)thread->sve_state + sve_ffr_offset(vl);
135}
136
137static inline void *thread_zt_state(struct thread_struct *thread)
138{
139	/* The ZT register state is stored immediately after the ZA state */
140	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
141	return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
142}
143
144extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
145extern void sve_load_state(void const *state, u32 const *pfpsr,
146			   int restore_ffr);
147extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
148extern unsigned int sve_get_vl(void);
149extern void sve_set_vq(unsigned long vq_minus_1);
150extern void sme_set_vq(unsigned long vq_minus_1);
151extern void sme_save_state(void *state, int zt);
152extern void sme_load_state(void const *state, int zt);
153
154struct arm64_cpu_capabilities;
155extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
156extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
157extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
158extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
159extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
160extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
161
162/*
163 * Helpers to translate bit indices in sve_vq_map to VQ values (and
164 * vice versa).  This allows find_next_bit() to be used to find the
165 * _maximum_ VQ not exceeding a certain value.
166 */
167static inline unsigned int __vq_to_bit(unsigned int vq)
168{
169	return SVE_VQ_MAX - vq;
170}
171
172static inline unsigned int __bit_to_vq(unsigned int bit)
173{
174	return SVE_VQ_MAX - bit;
175}
176
177
178struct vl_info {
179	enum vec_type type;
180	const char *name;		/* For display purposes */
181
182	/* Minimum supported vector length across all CPUs */
183	int min_vl;
184
185	/* Maximum supported vector length across all CPUs */
186	int max_vl;
187	int max_virtualisable_vl;
188
189	/*
190	 * Set of available vector lengths,
191	 * where length vq encoded as bit __vq_to_bit(vq):
192	 */
193	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
194
195	/* Set of vector lengths present on at least one cpu: */
196	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
197};
198
199#ifdef CONFIG_ARM64_SVE
200
201extern void sve_alloc(struct task_struct *task, bool flush);
202extern void fpsimd_release_task(struct task_struct *task);
203extern void fpsimd_sync_from_effective_state(struct task_struct *task);
204extern void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task);
205
206extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
207				 unsigned long vl, unsigned long flags);
208
209extern int sve_set_current_vl(unsigned long arg);
210extern int sve_get_current_vl(void);
211
212static inline void sve_user_disable(void)
213{
214	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
215}
216
217static inline void sve_user_enable(void)
218{
219	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
220}
221
222#define sve_cond_update_zcr_vq(val, reg)		\
223	do {						\
224		u64 __zcr = read_sysreg_s((reg));	\
225		u64 __new = __zcr & ~ZCR_ELx_LEN_MASK;	\
226		__new |= (val) & ZCR_ELx_LEN_MASK;	\
227		if (__zcr != __new)			\
228			write_sysreg_s(__new, (reg));	\
229	} while (0)
230
231/*
232 * Probing and setup functions.
233 * Calls to these functions must be serialised with one another.
234 */
235enum vec_type;
236
237extern void __init vec_init_vq_map(enum vec_type type);
238extern void vec_update_vq_map(enum vec_type type);
239extern int vec_verify_vq_map(enum vec_type type);
240extern void __init sve_setup(void);
241
242extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
243
244static inline void write_vl(enum vec_type type, u64 val)
245{
246	u64 tmp;
247
248	switch (type) {
249#ifdef CONFIG_ARM64_SVE
250	case ARM64_VEC_SVE:
251		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
252		write_sysreg_s(tmp | val, SYS_ZCR_EL1);
253		break;
254#endif
255#ifdef CONFIG_ARM64_SME
256	case ARM64_VEC_SME:
257		tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
258		write_sysreg_s(tmp | val, SYS_SMCR_EL1);
259		break;
260#endif
261	default:
262		WARN_ON_ONCE(1);
263		break;
264	}
265}
266
267static inline int vec_max_vl(enum vec_type type)
268{
269	return vl_info[type].max_vl;
270}
271
272static inline int vec_max_virtualisable_vl(enum vec_type type)
273{
274	return vl_info[type].max_virtualisable_vl;
275}
276
277static inline int sve_max_vl(void)
278{
279	return vec_max_vl(ARM64_VEC_SVE);
280}
281
282static inline int sve_max_virtualisable_vl(void)
283{
284	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
285}
286
287/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
288static inline bool vq_available(enum vec_type type, unsigned int vq)
289{
290	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
291}
292
293static inline bool sve_vq_available(unsigned int vq)
294{
295	return vq_available(ARM64_VEC_SVE, vq);
296}
297
298static inline size_t __sve_state_size(unsigned int sve_vl, unsigned int sme_vl)
299{
300	unsigned int vl = max(sve_vl, sme_vl);
301	return SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl));
302}
303
304/*
305 * Return how many bytes of memory are required to store the full SVE
306 * state for task, given task's currently configured vector length.
307 */
308static inline size_t sve_state_size(struct task_struct const *task)
309{
310	unsigned int sve_vl = task_get_sve_vl(task);
311	unsigned int sme_vl = task_get_sme_vl(task);
312	return __sve_state_size(sve_vl, sme_vl);
313}
314
315#else /* ! CONFIG_ARM64_SVE */
316
317static inline void sve_alloc(struct task_struct *task, bool flush) { }
318static inline void fpsimd_release_task(struct task_struct *task) { }
319static inline void fpsimd_sync_from_effective_state(struct task_struct *task) { }
320static inline void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { }
321
322static inline int sve_max_virtualisable_vl(void)
323{
324	return 0;
325}
326
327static inline int sve_set_current_vl(unsigned long arg)
328{
329	return -EINVAL;
330}
331
332static inline int sve_get_current_vl(void)
333{
334	return -EINVAL;
335}
336
337static inline int sve_max_vl(void)
338{
339	return -EINVAL;
340}
341
342static inline bool sve_vq_available(unsigned int vq) { return false; }
343
344static inline void sve_user_disable(void) { BUILD_BUG(); }
345static inline void sve_user_enable(void) { BUILD_BUG(); }
346
347#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
348
349static inline void vec_init_vq_map(enum vec_type t) { }
350static inline void vec_update_vq_map(enum vec_type t) { }
351static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
352static inline void sve_setup(void) { }
353
354static inline size_t __sve_state_size(unsigned int sve_vl, unsigned int sme_vl)
355{
356	return 0;
357}
358
359static inline size_t sve_state_size(struct task_struct const *task)
360{
361	return 0;
362}
363
364#endif /* ! CONFIG_ARM64_SVE */
365
366#ifdef CONFIG_ARM64_SME
367
368static inline void sme_user_disable(void)
369{
370	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
371}
372
373static inline void sme_user_enable(void)
374{
375	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
376}
377
378static inline void sme_smstart_sm(void)
379{
380	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
381}
382
383static inline void sme_smstop_sm(void)
384{
385	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
386}
387
388static inline void sme_smstop(void)
389{
390	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
391}
392
393extern void __init sme_setup(void);
394
395static inline int sme_max_vl(void)
396{
397	return vec_max_vl(ARM64_VEC_SME);
398}
399
400static inline int sme_max_virtualisable_vl(void)
401{
402	return vec_max_virtualisable_vl(ARM64_VEC_SME);
403}
404
405extern void sme_alloc(struct task_struct *task, bool flush);
406extern unsigned int sme_get_vl(void);
407extern int sme_set_current_vl(unsigned long arg);
408extern int sme_get_current_vl(void);
409extern void sme_suspend_exit(void);
410
411static inline size_t __sme_state_size(unsigned int sme_vl)
412{
413	size_t size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(sme_vl));
414
415	if (system_supports_sme2())
416		size += ZT_SIG_REG_SIZE;
417
418	return size;
419}
420
421/*
422 * Return how many bytes of memory are required to store the full SME
423 * specific state for task, given task's currently configured vector
424 * length.
425 */
426static inline size_t sme_state_size(struct task_struct const *task)
427{
428	return __sme_state_size(task_get_sme_vl(task));
429}
430
431#else
432
433static inline void sme_user_disable(void) { BUILD_BUG(); }
434static inline void sme_user_enable(void) { BUILD_BUG(); }
435
436static inline void sme_smstart_sm(void) { }
437static inline void sme_smstop_sm(void) { }
438static inline void sme_smstop(void) { }
439
440static inline void sme_alloc(struct task_struct *task, bool flush) { }
441static inline void sme_setup(void) { }
442static inline unsigned int sme_get_vl(void) { return 0; }
443static inline int sme_max_vl(void) { return 0; }
444static inline int sme_max_virtualisable_vl(void) { return 0; }
445static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
446static inline int sme_get_current_vl(void) { return -EINVAL; }
447static inline void sme_suspend_exit(void) { }
448
449static inline size_t __sme_state_size(unsigned int sme_vl)
450{
451	return 0;
452}
453
454static inline size_t sme_state_size(struct task_struct const *task)
455{
456	return 0;
457}
458
459#endif /* ! CONFIG_ARM64_SME */
460
461/* For use by EFI runtime services calls only */
462extern void __efi_fpsimd_begin(void);
463extern void __efi_fpsimd_end(void);
464
465#endif
466
467#endif