kfpu.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2023 Loongson Technology Corporation Limited
  4 */
  5
  6#include <linux/cpu.h>
  7#include <linux/export.h>
  8#include <linux/init.h>
  9#include <asm/fpu.h>
 10#include <asm/smp.h>
 11
 12static unsigned int euen_mask = CSR_EUEN_FPEN;
 13
 14/*
 15 * The critical section between kernel_fpu_begin() and kernel_fpu_end()
 16 * is non-reentrant. It is the caller's responsibility to avoid reentrance.
 17 * See drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c as an example.
 18 */
 19static DEFINE_PER_CPU(bool, in_kernel_fpu);
 20static DEFINE_PER_CPU(unsigned int, euen_current);
 21
 22static inline void fpregs_lock(void)
 23{
 24	if (IS_ENABLED(CONFIG_PREEMPT_RT))
 25		preempt_disable();
 26	else
 27		local_bh_disable();
 28}
 29
 30static inline void fpregs_unlock(void)
 31{
 32	if (IS_ENABLED(CONFIG_PREEMPT_RT))
 33		preempt_enable();
 34	else
 35		local_bh_enable();
 36}
 37
 38void kernel_fpu_begin(void)
 39{
 40	unsigned int *euen_curr;
 41
 42	if (!irqs_disabled())
 43		fpregs_lock();
 44
 45	WARN_ON(this_cpu_read(in_kernel_fpu));
 46
 47	this_cpu_write(in_kernel_fpu, true);
 48	euen_curr = this_cpu_ptr(&euen_current);
 49
 50	*euen_curr = csr_xchg32(euen_mask, euen_mask, LOONGARCH_CSR_EUEN);
 51
 52#ifdef CONFIG_CPU_HAS_LASX
 53	if (*euen_curr & CSR_EUEN_LASXEN)
 54		_save_lasx(&current->thread.fpu);
 55	else
 56#endif
 57#ifdef CONFIG_CPU_HAS_LSX
 58	if (*euen_curr & CSR_EUEN_LSXEN)
 59		_save_lsx(&current->thread.fpu);
 60	else
 61#endif
 62	if (*euen_curr & CSR_EUEN_FPEN)
 63		_save_fp(&current->thread.fpu);
 64
 65	write_fcsr(LOONGARCH_FCSR0, 0);
 66}
 67EXPORT_SYMBOL_GPL(kernel_fpu_begin);
 68
 69void kernel_fpu_end(void)
 70{
 71	unsigned int *euen_curr;
 72
 73	WARN_ON(!this_cpu_read(in_kernel_fpu));
 74
 75	euen_curr = this_cpu_ptr(&euen_current);
 76
 77#ifdef CONFIG_CPU_HAS_LASX
 78	if (*euen_curr & CSR_EUEN_LASXEN)
 79		_restore_lasx(&current->thread.fpu);
 80	else
 81#endif
 82#ifdef CONFIG_CPU_HAS_LSX
 83	if (*euen_curr & CSR_EUEN_LSXEN)
 84		_restore_lsx(&current->thread.fpu);
 85	else
 86#endif
 87	if (*euen_curr & CSR_EUEN_FPEN)
 88		_restore_fp(&current->thread.fpu);
 89
 90	*euen_curr = csr_xchg32(*euen_curr, euen_mask, LOONGARCH_CSR_EUEN);
 91
 92	this_cpu_write(in_kernel_fpu, false);
 93
 94	if (!irqs_disabled())
 95		fpregs_unlock();
 96}
 97EXPORT_SYMBOL_GPL(kernel_fpu_end);
 98
 99static int __init init_euen_mask(void)
100{
101	if (cpu_has_lsx)
102		euen_mask |= CSR_EUEN_LSXEN;
103
104	if (cpu_has_lasx)
105		euen_mask |= CSR_EUEN_LASXEN;
106
107	return 0;
108}
109arch_initcall(init_euen_mask);