fpu.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Author: Huacai Chen <chenhuacai@loongson.cn>
  4 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  5 */
  6#ifndef _ASM_FPU_H
  7#define _ASM_FPU_H
  8
  9#include <linux/sched.h>
 10#include <linux/sched/task_stack.h>
 11#include <linux/ptrace.h>
 12#include <linux/thread_info.h>
 13#include <linux/bitops.h>
 14
 15#include <asm/cpu.h>
 16#include <asm/cpu-features.h>
 17#include <asm/current.h>
 18#include <asm/loongarch.h>
 19#include <asm/processor.h>
 20#include <asm/ptrace.h>
 21
 22struct sigcontext;
 23
 24#define kernel_fpu_available() cpu_has_fpu
 25
 26void kernel_fpu_begin(void);
 27void kernel_fpu_end(void);
 28
 29asmlinkage void _init_fpu(unsigned int);
 30asmlinkage void _save_fp(struct loongarch_fpu *);
 31asmlinkage void _restore_fp(struct loongarch_fpu *);
 32asmlinkage int _save_fp_context(void __user *fpregs, void __user *fcc, void __user *csr);
 33asmlinkage int _restore_fp_context(void __user *fpregs, void __user *fcc, void __user *csr);
 34
 35asmlinkage void _save_lsx(struct loongarch_fpu *fpu);
 36asmlinkage void _restore_lsx(struct loongarch_fpu *fpu);
 37asmlinkage void _init_lsx_upper(void);
 38asmlinkage void _restore_lsx_upper(struct loongarch_fpu *fpu);
 39asmlinkage int _save_lsx_context(void __user *fpregs, void __user *fcc, void __user *fcsr);
 40asmlinkage int _restore_lsx_context(void __user *fpregs, void __user *fcc, void __user *fcsr);
 41
 42asmlinkage void _save_lasx(struct loongarch_fpu *fpu);
 43asmlinkage void _restore_lasx(struct loongarch_fpu *fpu);
 44asmlinkage void _init_lasx_upper(void);
 45asmlinkage void _restore_lasx_upper(struct loongarch_fpu *fpu);
 46asmlinkage int _save_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr);
 47asmlinkage int _restore_lasx_context(void __user *fpregs, void __user *fcc, void __user *fcsr);
 48
 49static inline void enable_lsx(void);
 50static inline void disable_lsx(void);
 51static inline void save_lsx(struct task_struct *t);
 52static inline void restore_lsx(struct task_struct *t);
 53
 54static inline void enable_lasx(void);
 55static inline void disable_lasx(void);
 56static inline void save_lasx(struct task_struct *t);
 57static inline void restore_lasx(struct task_struct *t);
 58
 59/*
 60 * Mask the FCSR Cause bits according to the Enable bits, observing
 61 * that Unimplemented is always enabled.
 62 */
 63static inline unsigned long mask_fcsr_x(unsigned long fcsr)
 64{
 65	return fcsr & ((fcsr & FPU_CSR_ALL_E) <<
 66			(ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E)));
 67}
 68
 69static inline int is_fp_enabled(void)
 70{
 71	return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_FPEN) ?
 72		1 : 0;
 73}
 74
 75static inline int is_lsx_enabled(void)
 76{
 77	if (!cpu_has_lsx)
 78		return 0;
 79
 80	return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ?
 81		1 : 0;
 82}
 83
 84static inline int is_lasx_enabled(void)
 85{
 86	if (!cpu_has_lasx)
 87		return 0;
 88
 89	return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ?
 90		1 : 0;
 91}
 92
 93static inline int is_simd_enabled(void)
 94{
 95	return is_lsx_enabled() | is_lasx_enabled();
 96}
 97
 98#define enable_fpu()		set_csr_euen(CSR_EUEN_FPEN)
 99
100#define disable_fpu()		clear_csr_euen(CSR_EUEN_FPEN)
101
102#define clear_fpu_owner()	clear_thread_flag(TIF_USEDFPU)
103
104static inline int is_fpu_owner(void)
105{
106	return test_thread_flag(TIF_USEDFPU);
107}
108
109static inline void __own_fpu(void)
110{
111	enable_fpu();
112	set_thread_flag(TIF_USEDFPU);
113	KSTK_EUEN(current) |= CSR_EUEN_FPEN;
114}
115
116static inline void own_fpu_inatomic(int restore)
117{
118	if (cpu_has_fpu && !is_fpu_owner()) {
119		__own_fpu();
120		if (restore)
121			_restore_fp(&current->thread.fpu);
122	}
123}
124
125static inline void own_fpu(int restore)
126{
127	preempt_disable();
128	own_fpu_inatomic(restore);
129	preempt_enable();
130}
131
132static inline void lose_fpu_inatomic(int save, struct task_struct *tsk)
133{
134	if (is_fpu_owner()) {
135		if (!is_simd_enabled()) {
136			if (save)
137				_save_fp(&tsk->thread.fpu);
138			disable_fpu();
139		} else {
140			if (save) {
141				if (!is_lasx_enabled())
142					save_lsx(tsk);
143				else
144					save_lasx(tsk);
145			}
146			disable_fpu();
147			disable_lsx();
148			disable_lasx();
149			clear_tsk_thread_flag(tsk, TIF_USEDSIMD);
150		}
151		clear_tsk_thread_flag(tsk, TIF_USEDFPU);
152	}
153	KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
154}
155
156static inline void lose_fpu(int save)
157{
158	preempt_disable();
159	lose_fpu_inatomic(save, current);
160	preempt_enable();
161}
162
163static inline void init_fpu(void)
164{
165	unsigned int fcsr = current->thread.fpu.fcsr;
166
167	__own_fpu();
168	_init_fpu(fcsr);
169	set_used_math();
170}
171
172static inline void save_fp(struct task_struct *tsk)
173{
174	if (cpu_has_fpu)
175		_save_fp(&tsk->thread.fpu);
176}
177
178static inline void restore_fp(struct task_struct *tsk)
179{
180	if (cpu_has_fpu)
181		_restore_fp(&tsk->thread.fpu);
182}
183
184static inline void save_fpu_regs(struct task_struct *tsk)
185{
186	unsigned int euen;
187
188	if (tsk == current) {
189		preempt_disable();
190
191		euen = csr_read32(LOONGARCH_CSR_EUEN);
192
193#ifdef CONFIG_CPU_HAS_LASX
194		if (euen & CSR_EUEN_LASXEN)
195			_save_lasx(&current->thread.fpu);
196		else
197#endif
198#ifdef CONFIG_CPU_HAS_LSX
199		if (euen & CSR_EUEN_LSXEN)
200			_save_lsx(&current->thread.fpu);
201		else
202#endif
203		if (euen & CSR_EUEN_FPEN)
204			_save_fp(&current->thread.fpu);
205
206		preempt_enable();
207	}
208}
209
210static inline int is_simd_owner(void)
211{
212	return test_thread_flag(TIF_USEDSIMD);
213}
214
215#ifdef CONFIG_CPU_HAS_LSX
216
217static inline void enable_lsx(void)
218{
219	if (cpu_has_lsx)
220		csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN);
221}
222
223static inline void disable_lsx(void)
224{
225	if (cpu_has_lsx)
226		csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN);
227}
228
229static inline void save_lsx(struct task_struct *t)
230{
231	if (cpu_has_lsx)
232		_save_lsx(&t->thread.fpu);
233}
234
235static inline void restore_lsx(struct task_struct *t)
236{
237	if (cpu_has_lsx)
238		_restore_lsx(&t->thread.fpu);
239}
240
241static inline void init_lsx_upper(void)
242{
243	if (cpu_has_lsx)
244		_init_lsx_upper();
245}
246
247static inline void restore_lsx_upper(struct task_struct *t)
248{
249	if (cpu_has_lsx)
250		_restore_lsx_upper(&t->thread.fpu);
251}
252
253#else
254static inline void enable_lsx(void) {}
255static inline void disable_lsx(void) {}
256static inline void save_lsx(struct task_struct *t) {}
257static inline void restore_lsx(struct task_struct *t) {}
258static inline void init_lsx_upper(void) {}
259static inline void restore_lsx_upper(struct task_struct *t) {}
260#endif
261
262#ifdef CONFIG_CPU_HAS_LASX
263
264static inline void enable_lasx(void)
265{
266
267	if (cpu_has_lasx)
268		csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN);
269}
270
271static inline void disable_lasx(void)
272{
273	if (cpu_has_lasx)
274		csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN);
275}
276
277static inline void save_lasx(struct task_struct *t)
278{
279	if (cpu_has_lasx)
280		_save_lasx(&t->thread.fpu);
281}
282
283static inline void restore_lasx(struct task_struct *t)
284{
285	if (cpu_has_lasx)
286		_restore_lasx(&t->thread.fpu);
287}
288
289static inline void init_lasx_upper(void)
290{
291	if (cpu_has_lasx)
292		_init_lasx_upper();
293}
294
295static inline void restore_lasx_upper(struct task_struct *t)
296{
297	if (cpu_has_lasx)
298		_restore_lasx_upper(&t->thread.fpu);
299}
300
301#else
302static inline void enable_lasx(void) {}
303static inline void disable_lasx(void) {}
304static inline void save_lasx(struct task_struct *t) {}
305static inline void restore_lasx(struct task_struct *t) {}
306static inline void init_lasx_upper(void) {}
307static inline void restore_lasx_upper(struct task_struct *t) {}
308#endif
309
310static inline int thread_lsx_context_live(void)
311{
312	if (!cpu_has_lsx)
313		return 0;
314
315	return test_thread_flag(TIF_LSX_CTX_LIVE);
316}
317
318static inline int thread_lasx_context_live(void)
319{
320	if (!cpu_has_lasx)
321		return 0;
322
323	return test_thread_flag(TIF_LASX_CTX_LIVE);
324}
325
326#endif /* _ASM_FPU_H */