fpsimd.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * FP/SIMD context switching and fault handling
   4 *
   5 * Copyright (C) 2012 ARM Ltd.
   6 * Author: Catalin Marinas <catalin.marinas@arm.com>
   7 */
   8
   9#include <linux/bitmap.h>
  10#include <linux/bitops.h>
  11#include <linux/bottom_half.h>
  12#include <linux/bug.h>
  13#include <linux/cache.h>
  14#include <linux/compat.h>
  15#include <linux/compiler.h>
  16#include <linux/cpu.h>
  17#include <linux/cpu_pm.h>
  18#include <linux/ctype.h>
  19#include <linux/kernel.h>
  20#include <linux/linkage.h>
  21#include <linux/irqflags.h>
  22#include <linux/init.h>
  23#include <linux/percpu.h>
  24#include <linux/prctl.h>
  25#include <linux/preempt.h>
  26#include <linux/ptrace.h>
  27#include <linux/sched/signal.h>
  28#include <linux/sched/task_stack.h>
  29#include <linux/signal.h>
  30#include <linux/slab.h>
  31#include <linux/stddef.h>
  32#include <linux/sysctl.h>
  33#include <linux/swab.h>
  34
  35#include <asm/esr.h>
  36#include <asm/exception.h>
  37#include <asm/fpsimd.h>
  38#include <asm/cpufeature.h>
  39#include <asm/cputype.h>
  40#include <asm/neon.h>
  41#include <asm/processor.h>
  42#include <asm/simd.h>
  43#include <asm/sigcontext.h>
  44#include <asm/sysreg.h>
  45#include <asm/traps.h>
  46#include <asm/virt.h>
  47
  48#define FPEXC_IOF	(1 << 0)
  49#define FPEXC_DZF	(1 << 1)
  50#define FPEXC_OFF	(1 << 2)
  51#define FPEXC_UFF	(1 << 3)
  52#define FPEXC_IXF	(1 << 4)
  53#define FPEXC_IDF	(1 << 7)
  54
  55/*
  56 * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
  57 *
  58 * In order to reduce the number of times the FPSIMD state is needlessly saved
  59 * and restored, we need to keep track of two things:
  60 * (a) for each task, we need to remember which CPU was the last one to have
  61 *     the task's FPSIMD state loaded into its FPSIMD registers;
  62 * (b) for each CPU, we need to remember which task's userland FPSIMD state has
  63 *     been loaded into its FPSIMD registers most recently, or whether it has
  64 *     been used to perform kernel mode NEON in the meantime.
  65 *
  66 * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to
  67 * the id of the current CPU every time the state is loaded onto a CPU. For (b),
  68 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
  69 * address of the userland FPSIMD state of the task that was loaded onto the CPU
  70 * the most recently, or NULL if kernel mode NEON has been performed after that.
  71 *
  72 * With this in place, we no longer have to restore the next FPSIMD state right
  73 * when switching between tasks. Instead, we can defer this check to userland
  74 * resume, at which time we verify whether the CPU's fpsimd_last_state and the
  75 * task's fpsimd_cpu are still mutually in sync. If this is the case, we
  76 * can omit the FPSIMD restore.
  77 *
  78 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
  79 * indicate whether or not the userland FPSIMD state of the current task is
  80 * present in the registers. The flag is set unless the FPSIMD registers of this
  81 * CPU currently contain the most recent userland FPSIMD state of the current
  82 * task. If the task is behaving as a VMM, then this is will be managed by
  83 * KVM which will clear it to indicate that the vcpu FPSIMD state is currently
  84 * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
  85 * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and
  86 * flag the register state as invalid.
  87 *
  88 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may be
  89 * called from softirq context, which will save the task's FPSIMD context back
  90 * to task_struct. To prevent this from racing with the manipulation of the
  91 * task's FPSIMD state from task context and thereby corrupting the state, it
  92 * is necessary to protect any manipulation of a task's fpsimd_state or
  93 * TIF_FOREIGN_FPSTATE flag with get_cpu_fpsimd_context(), which will suspend
  94 * softirq servicing entirely until put_cpu_fpsimd_context() is called.
  95 *
  96 * For a certain task, the sequence may look something like this:
  97 * - the task gets scheduled in; if both the task's fpsimd_cpu field
  98 *   contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
  99 *   variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
 100 *   cleared, otherwise it is set;
 101 *
 102 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
 103 *   userland FPSIMD state is copied from memory to the registers, the task's
 104 *   fpsimd_cpu field is set to the id of the current CPU, the current
 105 *   CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
 106 *   TIF_FOREIGN_FPSTATE flag is cleared;
 107 *
 108 * - the task executes an ordinary syscall; upon return to userland, the
 109 *   TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
 110 *   restored;
 111 *
 112 * - the task executes a syscall which executes some NEON instructions; this is
 113 *   preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
 114 *   register contents to memory, clears the fpsimd_last_state per-cpu variable
 115 *   and sets the TIF_FOREIGN_FPSTATE flag;
 116 *
 117 * - the task gets preempted after kernel_neon_end() is called; as we have not
 118 *   returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
 119 *   whatever is in the FPSIMD registers is not saved to memory, but discarded.
 120 */
 121
 122DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
 123
 124__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = {
 125#ifdef CONFIG_ARM64_SVE
 126	[ARM64_VEC_SVE] = {
 127		.type			= ARM64_VEC_SVE,
 128		.name			= "SVE",
 129		.min_vl			= SVE_VL_MIN,
 130		.max_vl			= SVE_VL_MIN,
 131		.max_virtualisable_vl	= SVE_VL_MIN,
 132	},
 133#endif
 134#ifdef CONFIG_ARM64_SME
 135	[ARM64_VEC_SME] = {
 136		.type			= ARM64_VEC_SME,
 137		.name			= "SME",
 138	},
 139#endif
 140};
 141
 142static unsigned int vec_vl_inherit_flag(enum vec_type type)
 143{
 144	switch (type) {
 145	case ARM64_VEC_SVE:
 146		return TIF_SVE_VL_INHERIT;
 147	case ARM64_VEC_SME:
 148		return TIF_SME_VL_INHERIT;
 149	default:
 150		WARN_ON_ONCE(1);
 151		return 0;
 152	}
 153}
 154
 155struct vl_config {
 156	int __default_vl;		/* Default VL for tasks */
 157};
 158
 159static struct vl_config vl_config[ARM64_VEC_MAX];
 160
 161static inline int get_default_vl(enum vec_type type)
 162{
 163	return READ_ONCE(vl_config[type].__default_vl);
 164}
 165
 166#ifdef CONFIG_ARM64_SVE
 167
 168static inline int get_sve_default_vl(void)
 169{
 170	return get_default_vl(ARM64_VEC_SVE);
 171}
 172
 173static inline void set_default_vl(enum vec_type type, int val)
 174{
 175	WRITE_ONCE(vl_config[type].__default_vl, val);
 176}
 177
 178static inline void set_sve_default_vl(int val)
 179{
 180	set_default_vl(ARM64_VEC_SVE, val);
 181}
 182
 183#endif /* ! CONFIG_ARM64_SVE */
 184
 185#ifdef CONFIG_ARM64_SME
 186
 187static int get_sme_default_vl(void)
 188{
 189	return get_default_vl(ARM64_VEC_SME);
 190}
 191
 192static void set_sme_default_vl(int val)
 193{
 194	set_default_vl(ARM64_VEC_SME, val);
 195}
 196
 197static void sme_free(struct task_struct *);
 198
 199#else
 200
 201static inline void sme_free(struct task_struct *t) { }
 202
 203#endif
 204
 205static void fpsimd_bind_task_to_cpu(void);
 206
 207/*
 208 * Claim ownership of the CPU FPSIMD context for use by the calling context.
 209 *
 210 * The caller may freely manipulate the FPSIMD context metadata until
 211 * put_cpu_fpsimd_context() is called.
 212 *
 213 * On RT kernels local_bh_disable() is not sufficient because it only
 214 * serializes soft interrupt related sections via a local lock, but stays
 215 * preemptible. Disabling preemption is the right choice here as bottom
 216 * half processing is always in thread context on RT kernels so it
 217 * implicitly prevents bottom half processing as well.
 218 */
 219static void get_cpu_fpsimd_context(void)
 220{
 221	if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
 222		/*
 223		 * The softirq subsystem lacks a true unmask/mask API, and
 224		 * re-enabling softirq processing using local_bh_enable() will
 225		 * not only unmask softirqs, it will also result in immediate
 226		 * delivery of any pending softirqs.
 227		 * This is undesirable when running with IRQs disabled, but in
 228		 * that case, there is no need to mask softirqs in the first
 229		 * place, so only bother doing so when IRQs are enabled.
 230		 */
 231		if (!irqs_disabled())
 232			local_bh_disable();
 233	} else {
 234		preempt_disable();
 235	}
 236}
 237
 238/*
 239 * Release the CPU FPSIMD context.
 240 *
 241 * Must be called from a context in which get_cpu_fpsimd_context() was
 242 * previously called, with no call to put_cpu_fpsimd_context() in the
 243 * meantime.
 244 */
 245static void put_cpu_fpsimd_context(void)
 246{
 247	if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
 248		if (!irqs_disabled())
 249			local_bh_enable();
 250	} else {
 251		preempt_enable();
 252	}
 253}
 254
 255unsigned int task_get_vl(const struct task_struct *task, enum vec_type type)
 256{
 257	return task->thread.vl[type];
 258}
 259
 260void task_set_vl(struct task_struct *task, enum vec_type type,
 261		 unsigned long vl)
 262{
 263	task->thread.vl[type] = vl;
 264}
 265
 266unsigned int task_get_vl_onexec(const struct task_struct *task,
 267				enum vec_type type)
 268{
 269	return task->thread.vl_onexec[type];
 270}
 271
 272void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
 273			unsigned long vl)
 274{
 275	task->thread.vl_onexec[type] = vl;
 276}
 277
 278/*
 279 * TIF_SME controls whether a task can use SME without trapping while
 280 * in userspace, when TIF_SME is set then we must have storage
 281 * allocated in sve_state and sme_state to store the contents of both ZA
 282 * and the SVE registers for both streaming and non-streaming modes.
 283 *
 284 * If both SVCR.ZA and SVCR.SM are disabled then at any point we
 285 * may disable TIF_SME and reenable traps.
 286 */
 287
 288
 289/*
 290 * TIF_SVE controls whether a task can use SVE without trapping while
 291 * in userspace, and also (together with TIF_SME) the way a task's
 292 * FPSIMD/SVE state is stored in thread_struct.
 293 *
 294 * The kernel uses this flag to track whether a user task is actively
 295 * using SVE, and therefore whether full SVE register state needs to
 296 * be tracked.  If not, the cheaper FPSIMD context handling code can
 297 * be used instead of the more costly SVE equivalents.
 298 *
 299 *  * TIF_SVE or SVCR.SM set:
 300 *
 301 *    The task can execute SVE instructions while in userspace without
 302 *    trapping to the kernel.
 303 *
 304 *    During any syscall, the kernel may optionally clear TIF_SVE and
 305 *    discard the vector state except for the FPSIMD subset.
 306 *
 307 *  * TIF_SVE clear:
 308 *
 309 *    An attempt by the user task to execute an SVE instruction causes
 310 *    do_sve_acc() to be called, which does some preparation and then
 311 *    sets TIF_SVE.
 312 *
 313 * During any syscall, the kernel may optionally clear TIF_SVE and
 314 * discard the vector state except for the FPSIMD subset.
 315 *
 316 * The data will be stored in one of two formats:
 317 *
 318 *  * FPSIMD only - FP_STATE_FPSIMD:
 319 *
 320 *    When the FPSIMD only state stored task->thread.fp_type is set to
 321 *    FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
 322 *    task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
 323 *    logically zero but not stored anywhere; P0-P15 and FFR are not
 324 *    stored and have unspecified values from userspace's point of
 325 *    view.  For hygiene purposes, the kernel zeroes them on next use,
 326 *    but userspace is discouraged from relying on this.
 327 *
 328 *    task->thread.sve_state does not need to be non-NULL, valid or any
 329 *    particular size: it must not be dereferenced and any data stored
 330 *    there should be considered stale and not referenced.
 331 *
 332 *  * SVE state - FP_STATE_SVE:
 333 *
 334 *    When the full SVE state is stored task->thread.fp_type is set to
 335 *    FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
 336 *    corresponding Zn), P0-P15 and FFR are encoded in in
 337 *    task->thread.sve_state, formatted appropriately for vector
 338 *    length task->thread.sve_vl or, if SVCR.SM is set,
 339 *    task->thread.sme_vl. The storage for the vector registers in
 340 *    task->thread.uw.fpsimd_state should be ignored.
 341 *
 342 *    task->thread.sve_state must point to a valid buffer at least
 343 *    sve_state_size(task) bytes in size. The data stored in
 344 *    task->thread.uw.fpsimd_state.vregs should be considered stale
 345 *    and not referenced.
 346 *
 347 *  * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
 348 *    irrespective of whether TIF_SVE is clear or set, since these are
 349 *    not vector length dependent.
 350 */
 351
 352/*
 353 * Update current's FPSIMD/SVE registers from thread_struct.
 354 *
 355 * This function should be called only when the FPSIMD/SVE state in
 356 * thread_struct is known to be up to date, when preparing to enter
 357 * userspace.
 358 */
 359static void task_fpsimd_load(void)
 360{
 361	bool restore_sve_regs = false;
 362	bool restore_ffr;
 363
 364	WARN_ON(!system_supports_fpsimd());
 365	WARN_ON(preemptible());
 366	WARN_ON(test_thread_flag(TIF_KERNEL_FPSTATE));
 367
 368	if (system_supports_sve() || system_supports_sme()) {
 369		switch (current->thread.fp_type) {
 370		case FP_STATE_FPSIMD:
 371			/* Stop tracking SVE for this task until next use. */
 372			clear_thread_flag(TIF_SVE);
 373			break;
 374		case FP_STATE_SVE:
 375			if (!thread_sm_enabled(&current->thread))
 376				WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE));
 377
 378			if (test_thread_flag(TIF_SVE))
 379				sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
 380
 381			restore_sve_regs = true;
 382			restore_ffr = true;
 383			break;
 384		default:
 385			/*
 386			 * This indicates either a bug in
 387			 * fpsimd_save_user_state() or memory corruption, we
 388			 * should always record an explicit format
 389			 * when we save. We always at least have the
 390			 * memory allocated for FPSIMD registers so
 391			 * try that and hope for the best.
 392			 */
 393			WARN_ON_ONCE(1);
 394			clear_thread_flag(TIF_SVE);
 395			break;
 396		}
 397	}
 398
 399	/* Restore SME, override SVE register configuration if needed */
 400	if (system_supports_sme()) {
 401		unsigned long sme_vl = task_get_sme_vl(current);
 402
 403		/* Ensure VL is set up for restoring data */
 404		if (test_thread_flag(TIF_SME))
 405			sme_set_vq(sve_vq_from_vl(sme_vl) - 1);
 406
 407		write_sysreg_s(current->thread.svcr, SYS_SVCR);
 408
 409		if (thread_za_enabled(&current->thread))
 410			sme_load_state(current->thread.sme_state,
 411				       system_supports_sme2());
 412
 413		if (thread_sm_enabled(&current->thread))
 414			restore_ffr = system_supports_fa64();
 415	}
 416
 417	if (system_supports_fpmr())
 418		write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR);
 419
 420	if (restore_sve_regs) {
 421		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
 422		sve_load_state(sve_pffr(&current->thread),
 423			       &current->thread.uw.fpsimd_state.fpsr,
 424			       restore_ffr);
 425	} else {
 426		WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
 427		fpsimd_load_state(&current->thread.uw.fpsimd_state);
 428	}
 429}
 430
 431/*
 432 * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
 433 * date with respect to the CPU registers. Note carefully that the
 434 * current context is the context last bound to the CPU stored in
 435 * last, if KVM is involved this may be the guest VM context rather
 436 * than the host thread for the VM pointed to by current. This means
 437 * that we must always reference the state storage via last rather
 438 * than via current, if we are saving KVM state then it will have
 439 * ensured that the type of registers to save is set in last->to_save.
 440 */
 441static void fpsimd_save_user_state(void)
 442{
 443	struct cpu_fp_state const *last =
 444		this_cpu_ptr(&fpsimd_last_state);
 445	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
 446	bool save_sve_regs = false;
 447	bool save_ffr;
 448	unsigned int vl;
 449
 450	WARN_ON(!system_supports_fpsimd());
 451	WARN_ON(preemptible());
 452
 453	if (test_thread_flag(TIF_FOREIGN_FPSTATE))
 454		return;
 455
 456	if (system_supports_fpmr())
 457		*(last->fpmr) = read_sysreg_s(SYS_FPMR);
 458
 459	/*
 460	 * Save SVE state if it is live.
 461	 *
 462	 * The syscall ABI discards live SVE state at syscall entry. When
 463	 * entering a syscall, fpsimd_syscall_enter() sets to_save to
 464	 * FP_STATE_FPSIMD to allow the SVE state to be lazily discarded until
 465	 * either new SVE state is loaded+bound or fpsimd_syscall_exit() is
 466	 * called prior to a return to userspace.
 467	 */
 468	if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) ||
 469	    last->to_save == FP_STATE_SVE) {
 470		save_sve_regs = true;
 471		save_ffr = true;
 472		vl = last->sve_vl;
 473	}
 474
 475	if (system_supports_sme()) {
 476		u64 *svcr = last->svcr;
 477
 478		*svcr = read_sysreg_s(SYS_SVCR);
 479
 480		if (*svcr & SVCR_ZA_MASK)
 481			sme_save_state(last->sme_state,
 482				       system_supports_sme2());
 483
 484		/* If we are in streaming mode override regular SVE. */
 485		if (*svcr & SVCR_SM_MASK) {
 486			save_sve_regs = true;
 487			save_ffr = system_supports_fa64();
 488			vl = last->sme_vl;
 489		}
 490	}
 491
 492	if (IS_ENABLED(CONFIG_ARM64_SVE) && save_sve_regs) {
 493		/* Get the configured VL from RDVL, will account for SM */
 494		if (WARN_ON(sve_get_vl() != vl)) {
 495			/*
 496			 * Can't save the user regs, so current would
 497			 * re-enter user with corrupt state.
 498			 * There's no way to recover, so kill it:
 499			 */
 500			force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
 501			return;
 502		}
 503
 504		sve_save_state((char *)last->sve_state +
 505					sve_ffr_offset(vl),
 506			       &last->st->fpsr, save_ffr);
 507		*last->fp_type = FP_STATE_SVE;
 508	} else {
 509		fpsimd_save_state(last->st);
 510		*last->fp_type = FP_STATE_FPSIMD;
 511	}
 512}
 513
 514/*
 515 * All vector length selection from userspace comes through here.
 516 * We're on a slow path, so some sanity-checks are included.
 517 * If things go wrong there's a bug somewhere, but try to fall back to a
 518 * safe choice.
 519 */
 520static unsigned int find_supported_vector_length(enum vec_type type,
 521						 unsigned int vl)
 522{
 523	struct vl_info *info = &vl_info[type];
 524	int bit;
 525	int max_vl = info->max_vl;
 526
 527	if (WARN_ON(!sve_vl_valid(vl)))
 528		vl = info->min_vl;
 529
 530	if (WARN_ON(!sve_vl_valid(max_vl)))
 531		max_vl = info->min_vl;
 532
 533	if (vl > max_vl)
 534		vl = max_vl;
 535	if (vl < info->min_vl)
 536		vl = info->min_vl;
 537
 538	bit = find_next_bit(info->vq_map, SVE_VQ_MAX,
 539			    __vq_to_bit(sve_vq_from_vl(vl)));
 540	return sve_vl_from_vq(__bit_to_vq(bit));
 541}
 542
 543#if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
 544
 545static int vec_proc_do_default_vl(const struct ctl_table *table, int write,
 546				  void *buffer, size_t *lenp, loff_t *ppos)
 547{
 548	struct vl_info *info = table->extra1;
 549	enum vec_type type = info->type;
 550	int ret;
 551	int vl = get_default_vl(type);
 552	struct ctl_table tmp_table = {
 553		.data = &vl,
 554		.maxlen = sizeof(vl),
 555	};
 556
 557	ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
 558	if (ret || !write)
 559		return ret;
 560
 561	/* Writing -1 has the special meaning "set to max": */
 562	if (vl == -1)
 563		vl = info->max_vl;
 564
 565	if (!sve_vl_valid(vl))
 566		return -EINVAL;
 567
 568	set_default_vl(type, find_supported_vector_length(type, vl));
 569	return 0;
 570}
 571
 572static const struct ctl_table sve_default_vl_table[] = {
 573	{
 574		.procname	= "sve_default_vector_length",
 575		.mode		= 0644,
 576		.proc_handler	= vec_proc_do_default_vl,
 577		.extra1		= &vl_info[ARM64_VEC_SVE],
 578	},
 579};
 580
 581static int __init sve_sysctl_init(void)
 582{
 583	if (system_supports_sve())
 584		if (!register_sysctl("abi", sve_default_vl_table))
 585			return -EINVAL;
 586
 587	return 0;
 588}
 589
 590#else /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
 591static int __init sve_sysctl_init(void) { return 0; }
 592#endif /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
 593
 594#if defined(CONFIG_ARM64_SME) && defined(CONFIG_SYSCTL)
 595static const struct ctl_table sme_default_vl_table[] = {
 596	{
 597		.procname	= "sme_default_vector_length",
 598		.mode		= 0644,
 599		.proc_handler	= vec_proc_do_default_vl,
 600		.extra1		= &vl_info[ARM64_VEC_SME],
 601	},
 602};
 603
 604static int __init sme_sysctl_init(void)
 605{
 606	if (system_supports_sme())
 607		if (!register_sysctl("abi", sme_default_vl_table))
 608			return -EINVAL;
 609
 610	return 0;
 611}
 612
 613#else /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */
 614static int __init sme_sysctl_init(void) { return 0; }
 615#endif /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */
 616
 617#define ZREG(sve_state, vq, n) ((char *)(sve_state) +		\
 618	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
 619
 620#ifdef CONFIG_CPU_BIG_ENDIAN
 621static __uint128_t arm64_cpu_to_le128(__uint128_t x)
 622{
 623	u64 a = swab64(x);
 624	u64 b = swab64(x >> 64);
 625
 626	return ((__uint128_t)a << 64) | b;
 627}
 628#else
 629static __uint128_t arm64_cpu_to_le128(__uint128_t x)
 630{
 631	return x;
 632}
 633#endif
 634
 635#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
 636
 637static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
 638			    unsigned int vq)
 639{
 640	unsigned int i;
 641	__uint128_t *p;
 642
 643	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 644		p = (__uint128_t *)ZREG(sst, vq, i);
 645		*p = arm64_cpu_to_le128(fst->vregs[i]);
 646	}
 647}
 648
 649/*
 650 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
 651 * task->thread.sve_state.
 652 *
 653 * Task can be a non-runnable task, or current.  In the latter case,
 654 * the caller must have ownership of the cpu FPSIMD context before calling
 655 * this function.
 656 * task->thread.sve_state must point to at least sve_state_size(task)
 657 * bytes of allocated kernel memory.
 658 * task->thread.uw.fpsimd_state must be up to date before calling this
 659 * function.
 660 */
 661static inline void fpsimd_to_sve(struct task_struct *task)
 662{
 663	unsigned int vq;
 664	void *sst = task->thread.sve_state;
 665	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
 666
 667	if (!system_supports_sve() && !system_supports_sme())
 668		return;
 669
 670	vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));
 671	__fpsimd_to_sve(sst, fst, vq);
 672}
 673
 674/*
 675 * Transfer the SVE state in task->thread.sve_state to
 676 * task->thread.uw.fpsimd_state.
 677 *
 678 * Task can be a non-runnable task, or current.  In the latter case,
 679 * the caller must have ownership of the cpu FPSIMD context before calling
 680 * this function.
 681 * task->thread.sve_state must point to at least sve_state_size(task)
 682 * bytes of allocated kernel memory.
 683 * task->thread.sve_state must be up to date before calling this function.
 684 */
 685static inline void sve_to_fpsimd(struct task_struct *task)
 686{
 687	unsigned int vq, vl;
 688	void const *sst = task->thread.sve_state;
 689	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
 690	unsigned int i;
 691	__uint128_t const *p;
 692
 693	if (!system_supports_sve() && !system_supports_sme())
 694		return;
 695
 696	vl = thread_get_cur_vl(&task->thread);
 697	vq = sve_vq_from_vl(vl);
 698	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 699		p = (__uint128_t const *)ZREG(sst, vq, i);
 700		fst->vregs[i] = arm64_le128_to_cpu(*p);
 701	}
 702}
 703
 704static inline void __fpsimd_zero_vregs(struct user_fpsimd_state *fpsimd)
 705{
 706	memset(&fpsimd->vregs, 0, sizeof(fpsimd->vregs));
 707}
 708
 709/*
 710 * Simulate the effects of an SMSTOP SM instruction.
 711 */
 712void task_smstop_sm(struct task_struct *task)
 713{
 714	if (!thread_sm_enabled(&task->thread))
 715		return;
 716
 717	__fpsimd_zero_vregs(&task->thread.uw.fpsimd_state);
 718	task->thread.uw.fpsimd_state.fpsr = 0x0800009f;
 719	if (system_supports_fpmr())
 720		task->thread.uw.fpmr = 0;
 721
 722	task->thread.svcr &= ~SVCR_SM_MASK;
 723	task->thread.fp_type = FP_STATE_FPSIMD;
 724}
 725
 726void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p)
 727{
 728	write_sysreg_s(read_sysreg_s(SYS_SCTLR_EL1) | SCTLR_EL1_EnFPM_MASK,
 729		       SYS_SCTLR_EL1);
 730}
 731
 732#ifdef CONFIG_ARM64_SVE
 733static void sve_free(struct task_struct *task)
 734{
 735	kfree(task->thread.sve_state);
 736	task->thread.sve_state = NULL;
 737}
 738
 739/*
 740 * Ensure that task->thread.sve_state is allocated and sufficiently large.
 741 *
 742 * This function should be used only in preparation for replacing
 743 * task->thread.sve_state with new data.  The memory is always zeroed
 744 * here to prevent stale data from showing through: this is done in
 745 * the interest of testability and predictability: except in the
 746 * do_sve_acc() case, there is no ABI requirement to hide stale data
 747 * written previously be task.
 748 */
 749void sve_alloc(struct task_struct *task, bool flush)
 750{
 751	if (task->thread.sve_state) {
 752		if (flush)
 753			memset(task->thread.sve_state, 0,
 754			       sve_state_size(task));
 755		return;
 756	}
 757
 758	/* This is a small allocation (maximum ~8KB) and Should Not Fail. */
 759	task->thread.sve_state =
 760		kzalloc(sve_state_size(task), GFP_KERNEL);
 761}
 762
 763/*
 764 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the
 765 * task's currently effective FPSIMD/SVE state.
 766 *
 767 * The task's FPSIMD/SVE/SME state must not be subject to concurrent
 768 * manipulation.
 769 */
 770void fpsimd_sync_from_effective_state(struct task_struct *task)
 771{
 772	if (task->thread.fp_type == FP_STATE_SVE)
 773		sve_to_fpsimd(task);
 774}
 775
 776/*
 777 * Ensure that the task's currently effective FPSIMD/SVE state is up to date
 778 * with respect to task->thread.uw.fpsimd_state, zeroing any effective
 779 * non-FPSIMD (S)SVE state.
 780 *
 781 * The task's FPSIMD/SVE/SME state must not be subject to concurrent
 782 * manipulation.
 783 */
 784void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task)
 785{
 786	unsigned int vq;
 787	void *sst = task->thread.sve_state;
 788	struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
 789
 790	if (task->thread.fp_type != FP_STATE_SVE)
 791		return;
 792
 793	vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));
 794
 795	memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
 796	__fpsimd_to_sve(sst, fst, vq);
 797}
 798
 799static int change_live_vector_length(struct task_struct *task,
 800				     enum vec_type type,
 801				     unsigned long vl)
 802{
 803	unsigned int sve_vl = task_get_sve_vl(task);
 804	unsigned int sme_vl = task_get_sme_vl(task);
 805	void *sve_state = NULL, *sme_state = NULL;
 806
 807	if (type == ARM64_VEC_SME)
 808		sme_vl = vl;
 809	else
 810		sve_vl = vl;
 811
 812	/*
 813	 * Allocate the new sve_state and sme_state before freeing the old
 814	 * copies so that allocation failure can be handled without needing to
 815	 * mutate the task's state in any way.
 816	 *
 817	 * Changes to the SVE vector length must not discard live ZA state or
 818	 * clear PSTATE.ZA, as userspace code which is unaware of the AAPCS64
 819	 * ZA lazy saving scheme may attempt to change the SVE vector length
 820	 * while unsaved/dormant ZA state exists.
 821	 */
 822	sve_state = kzalloc(__sve_state_size(sve_vl, sme_vl), GFP_KERNEL);
 823	if (!sve_state)
 824		goto out_mem;
 825
 826	if (type == ARM64_VEC_SME) {
 827		sme_state = kzalloc(__sme_state_size(sme_vl), GFP_KERNEL);
 828		if (!sme_state)
 829			goto out_mem;
 830	}
 831
 832	if (task == current)
 833		fpsimd_save_and_flush_current_state();
 834	else
 835		fpsimd_flush_task_state(task);
 836
 837	/*
 838	 * Always preserve PSTATE.SM and the effective FPSIMD state, zeroing
 839	 * other SVE state.
 840	 */
 841	fpsimd_sync_from_effective_state(task);
 842	task_set_vl(task, type, vl);
 843	kfree(task->thread.sve_state);
 844	task->thread.sve_state = sve_state;
 845	fpsimd_sync_to_effective_state_zeropad(task);
 846
 847	if (type == ARM64_VEC_SME) {
 848		task->thread.svcr &= ~SVCR_ZA_MASK;
 849		kfree(task->thread.sme_state);
 850		task->thread.sme_state = sme_state;
 851	}
 852
 853	return 0;
 854
 855out_mem:
 856	kfree(sve_state);
 857	kfree(sme_state);
 858	return -ENOMEM;
 859}
 860
 861int vec_set_vector_length(struct task_struct *task, enum vec_type type,
 862			  unsigned long vl, unsigned long flags)
 863{
 864	bool onexec = flags & PR_SVE_SET_VL_ONEXEC;
 865	bool inherit = flags & PR_SVE_VL_INHERIT;
 866
 867	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
 868				     PR_SVE_SET_VL_ONEXEC))
 869		return -EINVAL;
 870
 871	if (!sve_vl_valid(vl))
 872		return -EINVAL;
 873
 874	/*
 875	 * Clamp to the maximum vector length that VL-agnostic code
 876	 * can work with.  A flag may be assigned in the future to
 877	 * allow setting of larger vector lengths without confusing
 878	 * older software.
 879	 */
 880	if (vl > VL_ARCH_MAX)
 881		vl = VL_ARCH_MAX;
 882
 883	vl = find_supported_vector_length(type, vl);
 884
 885	if (!onexec && vl != task_get_vl(task, type)) {
 886		if (change_live_vector_length(task, type, vl))
 887			return -ENOMEM;
 888	}
 889
 890	if (onexec || inherit)
 891		task_set_vl_onexec(task, type, vl);
 892	else
 893		/* Reset VL to system default on next exec: */
 894		task_set_vl_onexec(task, type, 0);
 895
 896	update_tsk_thread_flag(task, vec_vl_inherit_flag(type),
 897			       flags & PR_SVE_VL_INHERIT);
 898
 899	return 0;
 900}
 901
 902/*
 903 * Encode the current vector length and flags for return.
 904 * This is only required for prctl(): ptrace has separate fields.
 905 * SVE and SME use the same bits for _ONEXEC and _INHERIT.
 906 *
 907 * flags are as for vec_set_vector_length().
 908 */
 909static int vec_prctl_status(enum vec_type type, unsigned long flags)
 910{
 911	int ret;
 912
 913	if (flags & PR_SVE_SET_VL_ONEXEC)
 914		ret = task_get_vl_onexec(current, type);
 915	else
 916		ret = task_get_vl(current, type);
 917
 918	if (test_thread_flag(vec_vl_inherit_flag(type)))
 919		ret |= PR_SVE_VL_INHERIT;
 920
 921	return ret;
 922}
 923
 924/* PR_SVE_SET_VL */
 925int sve_set_current_vl(unsigned long arg)
 926{
 927	unsigned long vl, flags;
 928	int ret;
 929
 930	vl = arg & PR_SVE_VL_LEN_MASK;
 931	flags = arg & ~vl;
 932
 933	if (!system_supports_sve() || is_compat_task())
 934		return -EINVAL;
 935
 936	ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags);
 937	if (ret)
 938		return ret;
 939
 940	return vec_prctl_status(ARM64_VEC_SVE, flags);
 941}
 942
 943/* PR_SVE_GET_VL */
 944int sve_get_current_vl(void)
 945{
 946	if (!system_supports_sve() || is_compat_task())
 947		return -EINVAL;
 948
 949	return vec_prctl_status(ARM64_VEC_SVE, 0);
 950}
 951
 952#ifdef CONFIG_ARM64_SME
 953/* PR_SME_SET_VL */
 954int sme_set_current_vl(unsigned long arg)
 955{
 956	unsigned long vl, flags;
 957	int ret;
 958
 959	vl = arg & PR_SME_VL_LEN_MASK;
 960	flags = arg & ~vl;
 961
 962	if (!system_supports_sme() || is_compat_task())
 963		return -EINVAL;
 964
 965	ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags);
 966	if (ret)
 967		return ret;
 968
 969	return vec_prctl_status(ARM64_VEC_SME, flags);
 970}
 971
 972/* PR_SME_GET_VL */
 973int sme_get_current_vl(void)
 974{
 975	if (!system_supports_sme() || is_compat_task())
 976		return -EINVAL;
 977
 978	return vec_prctl_status(ARM64_VEC_SME, 0);
 979}
 980#endif /* CONFIG_ARM64_SME */
 981
 982static void vec_probe_vqs(struct vl_info *info,
 983			  DECLARE_BITMAP(map, SVE_VQ_MAX))
 984{
 985	unsigned int vq, vl;
 986
 987	bitmap_zero(map, SVE_VQ_MAX);
 988
 989	for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
 990		write_vl(info->type, vq - 1); /* self-syncing */
 991
 992		switch (info->type) {
 993		case ARM64_VEC_SVE:
 994			vl = sve_get_vl();
 995			break;
 996		case ARM64_VEC_SME:
 997			vl = sme_get_vl();
 998			break;
 999		default:
1000			vl = 0;
1001			break;
1002		}
1003
1004		/* Minimum VL identified? */
1005		if (sve_vq_from_vl(vl) > vq)
1006			break;
1007
1008		vq = sve_vq_from_vl(vl); /* skip intervening lengths */
1009		set_bit(__vq_to_bit(vq), map);
1010	}
1011}
1012
1013/*
1014 * Initialise the set of known supported VQs for the boot CPU.
1015 * This is called during kernel boot, before secondary CPUs are brought up.
1016 */
1017void __init vec_init_vq_map(enum vec_type type)
1018{
1019	struct vl_info *info = &vl_info[type];
1020	vec_probe_vqs(info, info->vq_map);
1021	bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX);
1022}
1023
1024/*
1025 * If we haven't committed to the set of supported VQs yet, filter out
1026 * those not supported by the current CPU.
1027 * This function is called during the bring-up of early secondary CPUs only.
1028 */
1029void vec_update_vq_map(enum vec_type type)
1030{
1031	struct vl_info *info = &vl_info[type];
1032	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1033
1034	vec_probe_vqs(info, tmp_map);
1035	bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX);
1036	bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map,
1037		  SVE_VQ_MAX);
1038}
1039
1040/*
1041 * Check whether the current CPU supports all VQs in the committed set.
1042 * This function is called during the bring-up of late secondary CPUs only.
1043 */
1044int vec_verify_vq_map(enum vec_type type)
1045{
1046	struct vl_info *info = &vl_info[type];
1047	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1048	unsigned long b;
1049
1050	vec_probe_vqs(info, tmp_map);
1051
1052	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
1053	if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) {
1054		pr_warn("%s: cpu%d: Required vector length(s) missing\n",
1055			info->name, smp_processor_id());
1056		return -EINVAL;
1057	}
1058
1059	if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
1060		return 0;
1061
1062	/*
1063	 * For KVM, it is necessary to ensure that this CPU doesn't
1064	 * support any vector length that guests may have probed as
1065	 * unsupported.
1066	 */
1067
1068	/* Recover the set of supported VQs: */
1069	bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
1070	/* Find VQs supported that are not globally supported: */
1071	bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX);
1072
1073	/* Find the lowest such VQ, if any: */
1074	b = find_last_bit(tmp_map, SVE_VQ_MAX);
1075	if (b >= SVE_VQ_MAX)
1076		return 0; /* no mismatches */
1077
1078	/*
1079	 * Mismatches above sve_max_virtualisable_vl are fine, since
1080	 * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
1081	 */
1082	if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) {
1083		pr_warn("%s: cpu%d: Unsupported vector length(s) present\n",
1084			info->name, smp_processor_id());
1085		return -EINVAL;
1086	}
1087
1088	return 0;
1089}
1090
1091void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p)
1092{
1093	write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
1094	isb();
1095
1096	write_sysreg_s(0, SYS_ZCR_EL1);
1097}
1098
1099void __init sve_setup(void)
1100{
1101	struct vl_info *info = &vl_info[ARM64_VEC_SVE];
1102	DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1103	unsigned long b;
1104	int max_bit;
1105
1106	if (!system_supports_sve())
1107		return;
1108
1109	/*
1110	 * The SVE architecture mandates support for 128-bit vectors,
1111	 * so sve_vq_map must have at least SVE_VQ_MIN set.
1112	 * If something went wrong, at least try to patch it up:
1113	 */
1114	if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map)))
1115		set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map);
1116
1117	max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX);
1118	info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit));
1119
1120	/*
1121	 * For the default VL, pick the maximum supported value <= 64.
1122	 * VL == 64 is guaranteed not to grow the signal frame.
1123	 */
1124	set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64));
1125
1126	bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map,
1127		      SVE_VQ_MAX);
1128
1129	b = find_last_bit(tmp_map, SVE_VQ_MAX);
1130	if (b >= SVE_VQ_MAX)
1131		/* No non-virtualisable VLs found */
1132		info->max_virtualisable_vl = SVE_VQ_MAX;
1133	else if (WARN_ON(b == SVE_VQ_MAX - 1))
1134		/* No virtualisable VLs?  This is architecturally forbidden. */
1135		info->max_virtualisable_vl = SVE_VQ_MIN;
1136	else /* b + 1 < SVE_VQ_MAX */
1137		info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
1138
1139	if (info->max_virtualisable_vl > info->max_vl)
1140		info->max_virtualisable_vl = info->max_vl;
1141
1142	pr_info("%s: maximum available vector length %u bytes per vector\n",
1143		info->name, info->max_vl);
1144	pr_info("%s: default vector length %u bytes per vector\n",
1145		info->name, get_sve_default_vl());
1146
1147	/* KVM decides whether to support mismatched systems. Just warn here: */
1148	if (sve_max_virtualisable_vl() < sve_max_vl())
1149		pr_warn("%s: unvirtualisable vector lengths present\n",
1150			info->name);
1151}
1152
1153/*
1154 * Called from the put_task_struct() path, which cannot get here
1155 * unless dead_task is really dead and not schedulable.
1156 */
1157void fpsimd_release_task(struct task_struct *dead_task)
1158{
1159	sve_free(dead_task);
1160	sme_free(dead_task);
1161}
1162
1163#endif /* CONFIG_ARM64_SVE */
1164
1165#ifdef CONFIG_ARM64_SME
1166
1167/*
1168 * Ensure that task->thread.sme_state is allocated and sufficiently large.
1169 *
1170 * This function should be used only in preparation for replacing
1171 * task->thread.sme_state with new data.  The memory is always zeroed
1172 * here to prevent stale data from showing through: this is done in
1173 * the interest of testability and predictability, the architecture
1174 * guarantees that when ZA is enabled it will be zeroed.
1175 */
1176void sme_alloc(struct task_struct *task, bool flush)
1177{
1178	if (task->thread.sme_state) {
1179		if (flush)
1180			memset(task->thread.sme_state, 0,
1181			       sme_state_size(task));
1182		return;
1183	}
1184
1185	/* This could potentially be up to 64K. */
1186	task->thread.sme_state =
1187		kzalloc(sme_state_size(task), GFP_KERNEL);
1188}
1189
1190static void sme_free(struct task_struct *task)
1191{
1192	kfree(task->thread.sme_state);
1193	task->thread.sme_state = NULL;
1194}
1195
1196void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p)
1197{
1198	/* Set priority for all PEs to architecturally defined minimum */
1199	write_sysreg_s(read_sysreg_s(SYS_SMPRI_EL1) & ~SMPRI_EL1_PRIORITY_MASK,
1200		       SYS_SMPRI_EL1);
1201
1202	/* Allow SME in kernel */
1203	write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_SMEN_EL1EN, CPACR_EL1);
1204	isb();
1205
1206	/* Ensure all bits in SMCR are set to known values */
1207	write_sysreg_s(0, SYS_SMCR_EL1);
1208
1209	/* Allow EL0 to access TPIDR2 */
1210	write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1);
1211	isb();
1212}
1213
1214void cpu_enable_sme2(const struct arm64_cpu_capabilities *__always_unused p)
1215{
1216	/* This must be enabled after SME */
1217	BUILD_BUG_ON(ARM64_SME2 <= ARM64_SME);
1218
1219	/* Allow use of ZT0 */
1220	write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_EZT0_MASK,
1221		       SYS_SMCR_EL1);
1222}
1223
1224void cpu_enable_fa64(const struct arm64_cpu_capabilities *__always_unused p)
1225{
1226	/* This must be enabled after SME */
1227	BUILD_BUG_ON(ARM64_SME_FA64 <= ARM64_SME);
1228
1229	/* Allow use of FA64 */
1230	write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_FA64_MASK,
1231		       SYS_SMCR_EL1);
1232}
1233
1234void __init sme_setup(void)
1235{
1236	struct vl_info *info = &vl_info[ARM64_VEC_SME];
1237	int min_bit, max_bit;
1238
1239	if (!system_supports_sme())
1240		return;
1241
1242	min_bit = find_last_bit(info->vq_map, SVE_VQ_MAX);
1243
1244	/*
1245	 * SME doesn't require any particular vector length be
1246	 * supported but it does require at least one.  We should have
1247	 * disabled the feature entirely while bringing up CPUs but
1248	 * let's double check here.  The bitmap is SVE_VQ_MAP sized for
1249	 * sharing with SVE.
1250	 */
1251	WARN_ON(min_bit >= SVE_VQ_MAX);
1252
1253	info->min_vl = sve_vl_from_vq(__bit_to_vq(min_bit));
1254
1255	max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX);
1256	info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit));
1257
1258	WARN_ON(info->min_vl > info->max_vl);
1259
1260	/*
1261	 * For the default VL, pick the maximum supported value <= 32
1262	 * (256 bits) if there is one since this is guaranteed not to
1263	 * grow the signal frame when in streaming mode, otherwise the
1264	 * minimum available VL will be used.
1265	 */
1266	set_sme_default_vl(find_supported_vector_length(ARM64_VEC_SME, 32));
1267
1268	pr_info("SME: minimum available vector length %u bytes per vector\n",
1269		info->min_vl);
1270	pr_info("SME: maximum available vector length %u bytes per vector\n",
1271		info->max_vl);
1272	pr_info("SME: default vector length %u bytes per vector\n",
1273		get_sme_default_vl());
1274}
1275
1276void sme_suspend_exit(void)
1277{
1278	u64 smcr = 0;
1279
1280	if (!system_supports_sme())
1281		return;
1282
1283	if (system_supports_fa64())
1284		smcr |= SMCR_ELx_FA64;
1285	if (system_supports_sme2())
1286		smcr |= SMCR_ELx_EZT0;
1287
1288	write_sysreg_s(smcr, SYS_SMCR_EL1);
1289	write_sysreg_s(0, SYS_SMPRI_EL1);
1290}
1291
1292#endif /* CONFIG_ARM64_SME */
1293
1294static void sve_init_regs(void)
1295{
1296	/*
1297	 * Convert the FPSIMD state to SVE, zeroing all the state that
1298	 * is not shared with FPSIMD. If (as is likely) the current
1299	 * state is live in the registers then do this there and
1300	 * update our metadata for the current task including
1301	 * disabling the trap, otherwise update our in-memory copy.
1302	 * We are guaranteed to not be in streaming mode, we can only
1303	 * take a SVE trap when not in streaming mode and we can't be
1304	 * in streaming mode when taking a SME trap.
1305	 */
1306	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
1307		unsigned long vq_minus_one =
1308			sve_vq_from_vl(task_get_sve_vl(current)) - 1;
1309		sve_set_vq(vq_minus_one);
1310		sve_flush_live(true, vq_minus_one);
1311		fpsimd_bind_task_to_cpu();
1312	} else {
1313		fpsimd_to_sve(current);
1314		current->thread.fp_type = FP_STATE_SVE;
1315		fpsimd_flush_task_state(current);
1316	}
1317}
1318
1319/*
1320 * Trapped SVE access
1321 *
1322 * Storage is allocated for the full SVE state, the current FPSIMD
1323 * register contents are migrated across, and the access trap is
1324 * disabled.
1325 *
1326 * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
1327 * would have disabled the SVE access trap for userspace during
1328 * ret_to_user, making an SVE access trap impossible in that case.
1329 */
1330void do_sve_acc(unsigned long esr, struct pt_regs *regs)
1331{
1332	/* Even if we chose not to use SVE, the hardware could still trap: */
1333	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
1334		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1335		return;
1336	}
1337
1338	sve_alloc(current, true);
1339	if (!current->thread.sve_state) {
1340		force_sig(SIGKILL);
1341		return;
1342	}
1343
1344	get_cpu_fpsimd_context();
1345
1346	if (test_and_set_thread_flag(TIF_SVE))
1347		WARN_ON(1); /* SVE access shouldn't have trapped */
1348
1349	/*
1350	 * Even if the task can have used streaming mode we can only
1351	 * generate SVE access traps in normal SVE mode and
1352	 * transitioning out of streaming mode may discard any
1353	 * streaming mode state.  Always clear the high bits to avoid
1354	 * any potential errors tracking what is properly initialised.
1355	 */
1356	sve_init_regs();
1357
1358	put_cpu_fpsimd_context();
1359}
1360
1361/*
1362 * Trapped SME access
1363 *
1364 * Storage is allocated for the full SVE and SME state, the current
1365 * FPSIMD register contents are migrated to SVE if SVE is not already
1366 * active, and the access trap is disabled.
1367 *
1368 * TIF_SME should be clear on entry: otherwise, fpsimd_restore_current_state()
1369 * would have disabled the SME access trap for userspace during
1370 * ret_to_user, making an SME access trap impossible in that case.
1371 */
1372void do_sme_acc(unsigned long esr, struct pt_regs *regs)
1373{
1374	/* Even if we chose not to use SME, the hardware could still trap: */
1375	if (unlikely(!system_supports_sme()) || WARN_ON(is_compat_task())) {
1376		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1377		return;
1378	}
1379
1380	/*
1381	 * If this not a trap due to SME being disabled then something
1382	 * is being used in the wrong mode, report as SIGILL.
1383	 */
1384	if (ESR_ELx_SME_ISS_SMTC(esr) != ESR_ELx_SME_ISS_SMTC_SME_DISABLED) {
1385		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1386		return;
1387	}
1388
1389	sve_alloc(current, false);
1390	sme_alloc(current, true);
1391	if (!current->thread.sve_state || !current->thread.sme_state) {
1392		force_sig(SIGKILL);
1393		return;
1394	}
1395
1396	get_cpu_fpsimd_context();
1397
1398	/* With TIF_SME userspace shouldn't generate any traps */
1399	if (test_and_set_thread_flag(TIF_SME))
1400		WARN_ON(1);
1401
1402	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
1403		unsigned long vq_minus_one =
1404			sve_vq_from_vl(task_get_sme_vl(current)) - 1;
1405		sme_set_vq(vq_minus_one);
1406
1407		fpsimd_bind_task_to_cpu();
1408	} else {
1409		fpsimd_flush_task_state(current);
1410	}
1411
1412	put_cpu_fpsimd_context();
1413}
1414
1415/*
1416 * Trapped FP/ASIMD access.
1417 */
1418void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs)
1419{
1420	/* Even if we chose not to use FPSIMD, the hardware could still trap: */
1421	if (!system_supports_fpsimd()) {
1422		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1423		return;
1424	}
1425
1426	/*
1427	 * When FPSIMD is enabled, we should never take a trap unless something
1428	 * has gone very wrong.
1429	 */
1430	BUG();
1431}
1432
1433/*
1434 * Raise a SIGFPE for the current process.
1435 */
1436void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs)
1437{
1438	unsigned int si_code = FPE_FLTUNK;
1439
1440	if (esr & ESR_ELx_FP_EXC_TFV) {
1441		if (esr & FPEXC_IOF)
1442			si_code = FPE_FLTINV;
1443		else if (esr & FPEXC_DZF)
1444			si_code = FPE_FLTDIV;
1445		else if (esr & FPEXC_OFF)
1446			si_code = FPE_FLTOVF;
1447		else if (esr & FPEXC_UFF)
1448			si_code = FPE_FLTUND;
1449		else if (esr & FPEXC_IXF)
1450			si_code = FPE_FLTRES;
1451	}
1452
1453	send_sig_fault(SIGFPE, si_code,
1454		       (void __user *)instruction_pointer(regs),
1455		       current);
1456}
1457
1458static void fpsimd_load_kernel_state(struct task_struct *task)
1459{
1460	struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1461
1462	/*
1463	 * Elide the load if this CPU holds the most recent kernel mode
1464	 * FPSIMD context of the current task.
1465	 */
1466	if (last->st == task->thread.kernel_fpsimd_state &&
1467	    task->thread.kernel_fpsimd_cpu == smp_processor_id())
1468		return;
1469
1470	fpsimd_load_state(task->thread.kernel_fpsimd_state);
1471}
1472
1473static void fpsimd_save_kernel_state(struct task_struct *task)
1474{
1475	struct cpu_fp_state cpu_fp_state = {
1476		.st		= task->thread.kernel_fpsimd_state,
1477		.to_save	= FP_STATE_FPSIMD,
1478	};
1479
1480	BUG_ON(!cpu_fp_state.st);
1481
1482	fpsimd_save_state(task->thread.kernel_fpsimd_state);
1483	fpsimd_bind_state_to_cpu(&cpu_fp_state);
1484
1485	task->thread.kernel_fpsimd_cpu = smp_processor_id();
1486}
1487
1488/*
1489 * Invalidate any task's FPSIMD state that is present on this cpu.
1490 * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1491 * before calling this function.
1492 */
1493static void fpsimd_flush_cpu_state(void)
1494{
1495	WARN_ON(!system_supports_fpsimd());
1496	__this_cpu_write(fpsimd_last_state.st, NULL);
1497
1498	/*
1499	 * Leaving streaming mode enabled will cause issues for any kernel
1500	 * NEON and leaving streaming mode or ZA enabled may increase power
1501	 * consumption.
1502	 */
1503	if (system_supports_sme())
1504		sme_smstop();
1505
1506	set_thread_flag(TIF_FOREIGN_FPSTATE);
1507}
1508
1509void fpsimd_thread_switch(struct task_struct *next)
1510{
1511	bool wrong_task, wrong_cpu;
1512
1513	if (!system_supports_fpsimd())
1514		return;
1515
1516	WARN_ON_ONCE(!irqs_disabled());
1517
1518	/* Save unsaved fpsimd state, if any: */
1519	if (test_thread_flag(TIF_KERNEL_FPSTATE))
1520		fpsimd_save_kernel_state(current);
1521	else
1522		fpsimd_save_user_state();
1523
1524	if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) {
1525		fpsimd_flush_cpu_state();
1526		fpsimd_load_kernel_state(next);
1527	} else {
1528		/*
1529		 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
1530		 * state.  For kernel threads, FPSIMD registers are never
1531		 * loaded with user mode FPSIMD state and so wrong_task and
1532		 * wrong_cpu will always be true.
1533		 */
1534		wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
1535			&next->thread.uw.fpsimd_state;
1536		wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
1537
1538		update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
1539				       wrong_task || wrong_cpu);
1540	}
1541}
1542
1543static void fpsimd_flush_thread_vl(enum vec_type type)
1544{
1545	int vl, supported_vl;
1546
1547	/*
1548	 * Reset the task vector length as required.  This is where we
1549	 * ensure that all user tasks have a valid vector length
1550	 * configured: no kernel task can become a user task without
1551	 * an exec and hence a call to this function.  By the time the
1552	 * first call to this function is made, all early hardware
1553	 * probing is complete, so __sve_default_vl should be valid.
1554	 * If a bug causes this to go wrong, we make some noise and
1555	 * try to fudge thread.sve_vl to a safe value here.
1556	 */
1557	vl = task_get_vl_onexec(current, type);
1558	if (!vl)
1559		vl = get_default_vl(type);
1560
1561	if (WARN_ON(!sve_vl_valid(vl)))
1562		vl = vl_info[type].min_vl;
1563
1564	supported_vl = find_supported_vector_length(type, vl);
1565	if (WARN_ON(supported_vl != vl))
1566		vl = supported_vl;
1567
1568	task_set_vl(current, type, vl);
1569
1570	/*
1571	 * If the task is not set to inherit, ensure that the vector
1572	 * length will be reset by a subsequent exec:
1573	 */
1574	if (!test_thread_flag(vec_vl_inherit_flag(type)))
1575		task_set_vl_onexec(current, type, 0);
1576}
1577
1578void fpsimd_flush_thread(void)
1579{
1580	void *sve_state = NULL;
1581	void *sme_state = NULL;
1582
1583	if (!system_supports_fpsimd())
1584		return;
1585
1586	get_cpu_fpsimd_context();
1587
1588	fpsimd_flush_task_state(current);
1589	memset(&current->thread.uw.fpsimd_state, 0,
1590	       sizeof(current->thread.uw.fpsimd_state));
1591
1592	if (system_supports_sve()) {
1593		clear_thread_flag(TIF_SVE);
1594
1595		/* Defer kfree() while in atomic context */
1596		sve_state = current->thread.sve_state;
1597		current->thread.sve_state = NULL;
1598
1599		fpsimd_flush_thread_vl(ARM64_VEC_SVE);
1600	}
1601
1602	if (system_supports_sme()) {
1603		clear_thread_flag(TIF_SME);
1604
1605		/* Defer kfree() while in atomic context */
1606		sme_state = current->thread.sme_state;
1607		current->thread.sme_state = NULL;
1608
1609		fpsimd_flush_thread_vl(ARM64_VEC_SME);
1610		current->thread.svcr = 0;
1611	}
1612
1613	if (system_supports_fpmr())
1614		current->thread.uw.fpmr = 0;
1615
1616	current->thread.fp_type = FP_STATE_FPSIMD;
1617
1618	put_cpu_fpsimd_context();
1619	kfree(sve_state);
1620	kfree(sme_state);
1621}
1622
1623/*
1624 * Save the userland FPSIMD state of 'current' to memory, but only if the state
1625 * currently held in the registers does in fact belong to 'current'
1626 */
1627void fpsimd_preserve_current_state(void)
1628{
1629	if (!system_supports_fpsimd())
1630		return;
1631
1632	get_cpu_fpsimd_context();
1633	fpsimd_save_user_state();
1634	put_cpu_fpsimd_context();
1635}
1636
1637/*
1638 * Associate current's FPSIMD context with this cpu
1639 * The caller must have ownership of the cpu FPSIMD context before calling
1640 * this function.
1641 */
1642static void fpsimd_bind_task_to_cpu(void)
1643{
1644	struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1645
1646	WARN_ON(!system_supports_fpsimd());
1647	last->st = &current->thread.uw.fpsimd_state;
1648	last->sve_state = current->thread.sve_state;
1649	last->sme_state = current->thread.sme_state;
1650	last->sve_vl = task_get_sve_vl(current);
1651	last->sme_vl = task_get_sme_vl(current);
1652	last->svcr = &current->thread.svcr;
1653	last->fpmr = &current->thread.uw.fpmr;
1654	last->fp_type = &current->thread.fp_type;
1655	last->to_save = FP_STATE_CURRENT;
1656	current->thread.fpsimd_cpu = smp_processor_id();
1657
1658	/*
1659	 * Toggle SVE and SME trapping for userspace if needed, these
1660	 * are serialsied by ret_to_user().
1661	 */
1662	if (system_supports_sme()) {
1663		if (test_thread_flag(TIF_SME))
1664			sme_user_enable();
1665		else
1666			sme_user_disable();
1667	}
1668
1669	if (system_supports_sve()) {
1670		if (test_thread_flag(TIF_SVE))
1671			sve_user_enable();
1672		else
1673			sve_user_disable();
1674	}
1675}
1676
1677void fpsimd_bind_state_to_cpu(struct cpu_fp_state *state)
1678{
1679	struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1680
1681	WARN_ON(!system_supports_fpsimd());
1682	WARN_ON(!in_softirq() && !irqs_disabled());
1683
1684	*last = *state;
1685}
1686
1687/*
1688 * Load the userland FPSIMD state of 'current' from memory, but only if the
1689 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
1690 * state of 'current'.  This is called when we are preparing to return to
1691 * userspace to ensure that userspace sees a good register state.
1692 */
1693void fpsimd_restore_current_state(void)
1694{
1695	/*
1696	 * TIF_FOREIGN_FPSTATE is set on the init task and copied by
1697	 * arch_dup_task_struct() regardless of whether FP/SIMD is detected.
1698	 * Thus user threads can have this set even when FP/SIMD hasn't been
1699	 * detected.
1700	 *
1701	 * When FP/SIMD is detected, begin_new_exec() will set
1702	 * TIF_FOREIGN_FPSTATE via flush_thread() -> fpsimd_flush_thread(),
1703	 * and fpsimd_thread_switch() will set TIF_FOREIGN_FPSTATE when
1704	 * switching tasks. We detect FP/SIMD before we exec the first user
1705	 * process, ensuring this has TIF_FOREIGN_FPSTATE set and
1706	 * do_notify_resume() will call fpsimd_restore_current_state() to
1707	 * install the user FP/SIMD context.
1708	 *
1709	 * When FP/SIMD is not detected, nothing else will clear or set
1710	 * TIF_FOREIGN_FPSTATE prior to the first return to userspace, and
1711	 * we must clear TIF_FOREIGN_FPSTATE to avoid do_notify_resume()
1712	 * looping forever calling fpsimd_restore_current_state().
1713	 */
1714	if (!system_supports_fpsimd()) {
1715		clear_thread_flag(TIF_FOREIGN_FPSTATE);
1716		return;
1717	}
1718
1719	get_cpu_fpsimd_context();
1720
1721	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1722		task_fpsimd_load();
1723		fpsimd_bind_task_to_cpu();
1724	}
1725
1726	put_cpu_fpsimd_context();
1727}
1728
1729void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1730{
1731	if (WARN_ON(!system_supports_fpsimd()))
1732		return;
1733
1734	current->thread.uw.fpsimd_state = *state;
1735	if (current->thread.fp_type == FP_STATE_SVE)
1736		fpsimd_to_sve(current);
1737}
1738
1739/*
1740 * Invalidate live CPU copies of task t's FPSIMD state
1741 *
1742 * This function may be called with preemption enabled.  The barrier()
1743 * ensures that the assignment to fpsimd_cpu is visible to any
1744 * preemption/softirq that could race with set_tsk_thread_flag(), so
1745 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1746 *
1747 * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1748 * subsequent code.
1749 */
1750void fpsimd_flush_task_state(struct task_struct *t)
1751{
1752	t->thread.fpsimd_cpu = NR_CPUS;
1753	t->thread.kernel_fpsimd_state = NULL;
1754	/*
1755	 * If we don't support fpsimd, bail out after we have
1756	 * reset the fpsimd_cpu for this task and clear the
1757	 * FPSTATE.
1758	 */
1759	if (!system_supports_fpsimd())
1760		return;
1761	barrier();
1762	set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1763
1764	barrier();
1765}
1766
1767void fpsimd_save_and_flush_current_state(void)
1768{
1769	if (!system_supports_fpsimd())
1770		return;
1771
1772	get_cpu_fpsimd_context();
1773	fpsimd_save_user_state();
1774	fpsimd_flush_task_state(current);
1775	put_cpu_fpsimd_context();
1776}
1777
1778/*
1779 * Save the FPSIMD state to memory and invalidate cpu view.
1780 * This function must be called with preemption disabled.
1781 */
1782void fpsimd_save_and_flush_cpu_state(void)
1783{
1784	unsigned long flags;
1785
1786	if (!system_supports_fpsimd())
1787		return;
1788	WARN_ON(preemptible());
1789	local_irq_save(flags);
1790	fpsimd_save_user_state();
1791	fpsimd_flush_cpu_state();
1792	local_irq_restore(flags);
1793}
1794
1795#ifdef CONFIG_KERNEL_MODE_NEON
1796
1797/*
1798 * Kernel-side NEON support functions
1799 */
1800
1801/*
1802 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
1803 * context
1804 *
1805 * Must not be called unless may_use_simd() returns true.
1806 * Task context in the FPSIMD registers is saved back to memory as necessary.
1807 *
1808 * A matching call to kernel_neon_end() must be made before returning from the
1809 * calling context.
1810 *
1811 * The caller may freely use the FPSIMD registers until kernel_neon_end() is
1812 * called.
1813 *
1814 * Unless called from non-preemptible task context, @state must point to a
1815 * caller provided buffer that will be used to preserve the task's kernel mode
1816 * FPSIMD context when it is scheduled out, or if it is interrupted by kernel
1817 * mode FPSIMD occurring in softirq context. May be %NULL otherwise.
1818 */
1819void kernel_neon_begin(struct user_fpsimd_state *state)
1820{
1821	if (WARN_ON(!system_supports_fpsimd()))
1822		return;
1823
1824	WARN_ON((preemptible() || in_serving_softirq()) && !state);
1825
1826	BUG_ON(!may_use_simd());
1827
1828	get_cpu_fpsimd_context();
1829
1830	/* Save unsaved fpsimd state, if any: */
1831	if (test_thread_flag(TIF_KERNEL_FPSTATE)) {
1832		BUG_ON(IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq());
1833		fpsimd_save_state(state);
1834	} else {
1835		fpsimd_save_user_state();
1836
1837		/*
1838		 * Set the thread flag so that the kernel mode FPSIMD state
1839		 * will be context switched along with the rest of the task
1840		 * state.
1841		 *
1842		 * On non-PREEMPT_RT, softirqs may interrupt task level kernel
1843		 * mode FPSIMD, but the task will not be preemptible so setting
1844		 * TIF_KERNEL_FPSTATE for those would be both wrong (as it
1845		 * would mark the task context FPSIMD state as requiring a
1846		 * context switch) and unnecessary.
1847		 *
1848		 * On PREEMPT_RT, softirqs are serviced from a separate thread,
1849		 * which is scheduled as usual, and this guarantees that these
1850		 * softirqs are not interrupting use of the FPSIMD in kernel
1851		 * mode in task context. So in this case, setting the flag here
1852		 * is always appropriate.
1853		 */
1854		if (IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq()) {
1855			/*
1856			 * Record the caller provided buffer as the kernel mode
1857			 * FP/SIMD buffer for this task, so that the state can
1858			 * be preserved and restored on a context switch.
1859			 */
1860			WARN_ON(current->thread.kernel_fpsimd_state != NULL);
1861			current->thread.kernel_fpsimd_state = state;
1862			set_thread_flag(TIF_KERNEL_FPSTATE);
1863		}
1864	}
1865
1866	/* Invalidate any task state remaining in the fpsimd regs: */
1867	fpsimd_flush_cpu_state();
1868
1869	put_cpu_fpsimd_context();
1870}
1871EXPORT_SYMBOL_GPL(kernel_neon_begin);
1872
1873/*
1874 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1875 *
1876 * Must be called from a context in which kernel_neon_begin() was previously
1877 * called, with no call to kernel_neon_end() in the meantime.
1878 *
1879 * The caller must not use the FPSIMD registers after this function is called,
1880 * unless kernel_neon_begin() is called again in the meantime.
1881 *
1882 * The value of @state must match the value passed to the preceding call to
1883 * kernel_neon_begin().
1884 */
1885void kernel_neon_end(struct user_fpsimd_state *state)
1886{
1887	if (!system_supports_fpsimd())
1888		return;
1889
1890	if (!test_thread_flag(TIF_KERNEL_FPSTATE))
1891		return;
1892
1893	/*
1894	 * If we are returning from a nested use of kernel mode FPSIMD, restore
1895	 * the task context kernel mode FPSIMD state. This can only happen when
1896	 * running in softirq context on non-PREEMPT_RT.
1897	 */
1898	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && in_serving_softirq()) {
1899		fpsimd_load_state(state);
1900	} else {
1901		clear_thread_flag(TIF_KERNEL_FPSTATE);
1902		WARN_ON(current->thread.kernel_fpsimd_state != state);
1903		current->thread.kernel_fpsimd_state = NULL;
1904	}
1905}
1906EXPORT_SYMBOL_GPL(kernel_neon_end);
1907
1908#ifdef CONFIG_EFI
1909
1910static struct user_fpsimd_state efi_fpsimd_state;
1911
1912/*
1913 * EFI runtime services support functions
1914 *
1915 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
1916 * This means that for EFI (and only for EFI), we have to assume that FPSIMD
1917 * is always used rather than being an optional accelerator.
1918 *
1919 * These functions provide the necessary support for ensuring FPSIMD
1920 * save/restore in the contexts from which EFI is used.
1921 *
1922 * Do not use them for any other purpose -- if tempted to do so, you are
1923 * either doing something wrong or you need to propose some refactoring.
1924 */
1925
1926/*
1927 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
1928 */
1929void __efi_fpsimd_begin(void)
1930{
1931	if (!system_supports_fpsimd())
1932		return;
1933
1934	if (may_use_simd()) {
1935		kernel_neon_begin(&efi_fpsimd_state);
1936	} else {
1937		/*
1938		 * We are running in hardirq or NMI context, and the only
1939		 * legitimate case where this might happen is when EFI pstore
1940		 * is attempting to record the system's dying gasps into EFI
1941		 * variables. This could be due to an oops, a panic or a call
1942		 * to emergency_restart(), and in none of those cases, we can
1943		 * expect the current task to ever return to user space again,
1944		 * or for the kernel to resume any normal execution, for that
1945		 * matter (an oops in hardirq context triggers a panic too).
1946		 *
1947		 * Therefore, there is no point in attempting to preserve any
1948		 * SVE/SME state here. On the off chance that we might have
1949		 * ended up here for a different reason inadvertently, kill the
1950		 * task and preserve/restore the base FP/SIMD state, which
1951		 * might belong to kernel mode FP/SIMD.
1952		 */
1953		pr_warn_ratelimited("Calling EFI runtime from %s context\n",
1954				    in_nmi() ? "NMI" : "hardirq");
1955		force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
1956		fpsimd_save_state(&efi_fpsimd_state);
1957	}
1958}
1959
1960/*
1961 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
1962 */
1963void __efi_fpsimd_end(void)
1964{
1965	if (!system_supports_fpsimd())
1966		return;
1967
1968	if (may_use_simd()) {
1969		kernel_neon_end(&efi_fpsimd_state);
1970	} else {
1971		fpsimd_load_state(&efi_fpsimd_state);
1972	}
1973}
1974
1975#endif /* CONFIG_EFI */
1976
1977#endif /* CONFIG_KERNEL_MODE_NEON */
1978
1979#ifdef CONFIG_CPU_PM
1980static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
1981				  unsigned long cmd, void *v)
1982{
1983	switch (cmd) {
1984	case CPU_PM_ENTER:
1985		fpsimd_save_and_flush_cpu_state();
1986		break;
1987	case CPU_PM_EXIT:
1988		break;
1989	case CPU_PM_ENTER_FAILED:
1990	default:
1991		return NOTIFY_DONE;
1992	}
1993	return NOTIFY_OK;
1994}
1995
1996static struct notifier_block fpsimd_cpu_pm_notifier_block = {
1997	.notifier_call = fpsimd_cpu_pm_notifier,
1998};
1999
2000static void __init fpsimd_pm_init(void)
2001{
2002	cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
2003}
2004
2005#else
2006static inline void fpsimd_pm_init(void) { }
2007#endif /* CONFIG_CPU_PM */
2008
2009#ifdef CONFIG_HOTPLUG_CPU
2010static int fpsimd_cpu_dead(unsigned int cpu)
2011{
2012	per_cpu(fpsimd_last_state.st, cpu) = NULL;
2013	return 0;
2014}
2015
2016static inline void fpsimd_hotplug_init(void)
2017{
2018	cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
2019				  NULL, fpsimd_cpu_dead);
2020}
2021
2022#else
2023static inline void fpsimd_hotplug_init(void) { }
2024#endif
2025
2026void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__always_unused p)
2027{
2028	unsigned long enable = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN;
2029	write_sysreg(read_sysreg(CPACR_EL1) | enable, CPACR_EL1);
2030	isb();
2031}
2032
2033/*
2034 * FP/SIMD support code initialisation.
2035 */
2036static int __init fpsimd_init(void)
2037{
2038	if (cpu_have_named_feature(FP)) {
2039		fpsimd_pm_init();
2040		fpsimd_hotplug_init();
2041	} else {
2042		pr_notice("Floating-point is not implemented\n");
2043	}
2044
2045	if (!cpu_have_named_feature(ASIMD))
2046		pr_notice("Advanced SIMD is not implemented\n");
2047
2048
2049	sve_sysctl_init();
2050	sme_sysctl_init();
2051
2052	return 0;
2053}
2054core_initcall(fpsimd_init);