entry-common.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Exception handling code
   4 *
   5 * Copyright (C) 2019 ARM Ltd.
   6 */
   7
   8#include <linux/context_tracking.h>
   9#include <linux/irq-entry-common.h>
  10#include <linux/kasan.h>
  11#include <linux/linkage.h>
  12#include <linux/livepatch.h>
  13#include <linux/lockdep.h>
  14#include <linux/ptrace.h>
  15#include <linux/resume_user_mode.h>
  16#include <linux/sched.h>
  17#include <linux/sched/debug.h>
  18#include <linux/thread_info.h>
  19
  20#include <asm/cpufeature.h>
  21#include <asm/daifflags.h>
  22#include <asm/esr.h>
  23#include <asm/exception.h>
  24#include <asm/irq_regs.h>
  25#include <asm/kprobes.h>
  26#include <asm/mmu.h>
  27#include <asm/processor.h>
  28#include <asm/sdei.h>
  29#include <asm/stacktrace.h>
  30#include <asm/sysreg.h>
  31#include <asm/system_misc.h>
  32
  33/*
  34 * Handle IRQ/context state management when entering from kernel mode.
  35 * Before this function is called it is not safe to call regular kernel code,
  36 * instrumentable code, or any code which may trigger an exception.
  37 */
  38static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs)
  39{
  40	irqentry_state_t state;
  41
  42	state = irqentry_enter(regs);
  43	mte_check_tfsr_entry();
  44	mte_disable_tco_entry(current);
  45
  46	return state;
  47}
  48
  49/*
  50 * Handle IRQ/context state management when exiting to kernel mode.
  51 * After this function returns it is not safe to call regular kernel code,
  52 * instrumentable code, or any code which may trigger an exception.
  53 */
  54static void noinstr exit_to_kernel_mode(struct pt_regs *regs,
  55					irqentry_state_t state)
  56{
  57	mte_check_tfsr_exit();
  58	irqentry_exit(regs, state);
  59}
  60
  61/*
  62 * Handle IRQ/context state management when entering from user mode.
  63 * Before this function is called it is not safe to call regular kernel code,
  64 * instrumentable code, or any code which may trigger an exception.
  65 */
  66static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs)
  67{
  68	enter_from_user_mode(regs);
  69	mte_disable_tco_entry(current);
  70}
  71
  72/*
  73 * Handle IRQ/context state management when exiting to user mode.
  74 * After this function returns it is not safe to call regular kernel code,
  75 * instrumentable code, or any code which may trigger an exception.
  76 */
  77
  78static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
  79{
  80	local_irq_disable();
  81	exit_to_user_mode_prepare_legacy(regs);
  82	local_daif_mask();
  83	mte_check_tfsr_exit();
  84	exit_to_user_mode();
  85}
  86
  87asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
  88{
  89	arm64_exit_to_user_mode(regs);
  90}
  91
  92/*
  93 * Handle IRQ/context state management when entering a debug exception from
  94 * kernel mode. Before this function is called it is not safe to call regular
  95 * kernel code, instrumentable code, or any code which may trigger an exception.
  96 */
  97static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs)
  98{
  99	irqentry_state_t state;
 100
 101	state.lockdep = lockdep_hardirqs_enabled();
 102
 103	lockdep_hardirqs_off(CALLER_ADDR0);
 104	ct_nmi_enter();
 105
 106	trace_hardirqs_off_finish();
 107
 108	return state;
 109}
 110
 111/*
 112 * Handle IRQ/context state management when exiting a debug exception from
 113 * kernel mode. After this function returns it is not safe to call regular
 114 * kernel code, instrumentable code, or any code which may trigger an exception.
 115 */
 116static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs,
 117				       irqentry_state_t state)
 118{
 119	if (state.lockdep) {
 120		trace_hardirqs_on_prepare();
 121		lockdep_hardirqs_on_prepare();
 122	}
 123
 124	ct_nmi_exit();
 125	if (state.lockdep)
 126		lockdep_hardirqs_on(CALLER_ADDR0);
 127}
 128
 129static void do_interrupt_handler(struct pt_regs *regs,
 130				 void (*handler)(struct pt_regs *))
 131{
 132	struct pt_regs *old_regs = set_irq_regs(regs);
 133
 134	if (on_thread_stack())
 135		call_on_irq_stack(regs, handler);
 136	else
 137		handler(regs);
 138
 139	set_irq_regs(old_regs);
 140}
 141
 142extern void (*handle_arch_irq)(struct pt_regs *);
 143extern void (*handle_arch_fiq)(struct pt_regs *);
 144
 145static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
 146				      unsigned long esr)
 147{
 148	irqentry_nmi_enter(regs);
 149
 150	console_verbose();
 151
 152	pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n",
 153		vector, smp_processor_id(), esr,
 154		esr_get_class_string(esr));
 155
 156	__show_regs(regs);
 157	panic("Unhandled exception");
 158}
 159
 160#define UNHANDLED(el, regsize, vector)							\
 161asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
 162{											\
 163	const char *desc = #regsize "-bit " #el " " #vector;				\
 164	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
 165}
 166
 167#ifdef CONFIG_ARM64_ERRATUM_1463225
 168static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
 169
 170static void cortex_a76_erratum_1463225_svc_handler(void)
 171{
 172	u64 reg, val;
 173
 174	if (!unlikely(test_thread_flag(TIF_SINGLESTEP)))
 175		return;
 176
 177	if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225)))
 178		return;
 179
 180	__this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1);
 181	reg = read_sysreg(mdscr_el1);
 182	val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE;
 183	write_sysreg(val, mdscr_el1);
 184	asm volatile("msr daifclr, #8");
 185	isb();
 186
 187	/* We will have taken a single-step exception by this point */
 188
 189	write_sysreg(reg, mdscr_el1);
 190	__this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0);
 191}
 192
 193static __always_inline bool
 194cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
 195{
 196	if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
 197		return false;
 198
 199	/*
 200	 * We've taken a dummy step exception from the kernel to ensure
 201	 * that interrupts are re-enabled on the syscall path. Return back
 202	 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
 203	 * masked so that we can safely restore the mdscr and get on with
 204	 * handling the syscall.
 205	 */
 206	regs->pstate |= PSR_D_BIT;
 207	return true;
 208}
 209#else /* CONFIG_ARM64_ERRATUM_1463225 */
 210static void cortex_a76_erratum_1463225_svc_handler(void) { }
 211static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
 212{
 213	return false;
 214}
 215#endif /* CONFIG_ARM64_ERRATUM_1463225 */
 216
 217/*
 218 * As per the ABI exit SME streaming mode and clear the SVE state not
 219 * shared with FPSIMD on syscall entry.
 220 */
 221static inline void fpsimd_syscall_enter(void)
 222{
 223	/* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */
 224	if (system_supports_sme())
 225		sme_smstop_sm();
 226
 227	/*
 228	 * The CPU is not in streaming mode. If non-streaming SVE is not
 229	 * supported, there is no SVE state that needs to be discarded.
 230	 */
 231	if (!system_supports_sve())
 232		return;
 233
 234	if (test_thread_flag(TIF_SVE)) {
 235		unsigned int sve_vq_minus_one;
 236
 237		sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
 238		sve_flush_live(true, sve_vq_minus_one);
 239	}
 240
 241	/*
 242	 * Any live non-FPSIMD SVE state has been zeroed. Allow
 243	 * fpsimd_save_user_state() to lazily discard SVE state until either
 244	 * the live state is unbound or fpsimd_syscall_exit() is called.
 245	 */
 246	__this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD);
 247}
 248
 249static __always_inline void fpsimd_syscall_exit(void)
 250{
 251	if (!system_supports_sve())
 252		return;
 253
 254	/*
 255	 * The current task's user FPSIMD/SVE/SME state is now bound to this
 256	 * CPU. The fpsimd_last_state.to_save value is either:
 257	 *
 258	 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU
 259	 *   since fpsimd_syscall_enter().
 260	 *
 261	 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at
 262	 *   any point.
 263	 *
 264	 * Reset this to FP_STATE_CURRENT to stop lazy discarding.
 265	 */
 266	__this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT);
 267}
 268
 269/*
 270 * In debug exception context, we explicitly disable preemption despite
 271 * having interrupts disabled.
 272 * This serves two purposes: it makes it much less likely that we would
 273 * accidentally schedule in exception context and it will force a warning
 274 * if we somehow manage to schedule by accident.
 275 */
 276static void debug_exception_enter(struct pt_regs *regs)
 277{
 278	preempt_disable();
 279
 280	/* This code is a bit fragile.  Test it. */
 281	RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
 282}
 283NOKPROBE_SYMBOL(debug_exception_enter);
 284
 285static void debug_exception_exit(struct pt_regs *regs)
 286{
 287	preempt_enable_no_resched();
 288}
 289NOKPROBE_SYMBOL(debug_exception_exit);
 290
 291UNHANDLED(el1t, 64, sync)
 292UNHANDLED(el1t, 64, irq)
 293UNHANDLED(el1t, 64, fiq)
 294UNHANDLED(el1t, 64, error)
 295
 296static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
 297{
 298	unsigned long far = read_sysreg(far_el1);
 299	irqentry_state_t state;
 300
 301	state = enter_from_kernel_mode(regs);
 302	local_daif_inherit(regs);
 303	do_mem_abort(far, esr, regs);
 304	local_daif_mask();
 305	exit_to_kernel_mode(regs, state);
 306}
 307
 308static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
 309{
 310	unsigned long far = read_sysreg(far_el1);
 311	irqentry_state_t state;
 312
 313	state = enter_from_kernel_mode(regs);
 314	local_daif_inherit(regs);
 315	do_sp_pc_abort(far, esr, regs);
 316	local_daif_mask();
 317	exit_to_kernel_mode(regs, state);
 318}
 319
 320static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
 321{
 322	irqentry_state_t state;
 323
 324	state = enter_from_kernel_mode(regs);
 325	local_daif_inherit(regs);
 326	do_el1_undef(regs, esr);
 327	local_daif_mask();
 328	exit_to_kernel_mode(regs, state);
 329}
 330
 331static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
 332{
 333	irqentry_state_t state;
 334
 335	state = enter_from_kernel_mode(regs);
 336	local_daif_inherit(regs);
 337	do_el1_bti(regs, esr);
 338	local_daif_mask();
 339	exit_to_kernel_mode(regs, state);
 340}
 341
 342static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr)
 343{
 344	irqentry_state_t state;
 345
 346	state = enter_from_kernel_mode(regs);
 347	local_daif_inherit(regs);
 348	do_el1_gcs(regs, esr);
 349	local_daif_mask();
 350	exit_to_kernel_mode(regs, state);
 351}
 352
 353static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr)
 354{
 355	irqentry_state_t state;
 356
 357	state = enter_from_kernel_mode(regs);
 358	local_daif_inherit(regs);
 359	do_el1_mops(regs, esr);
 360	local_daif_mask();
 361	exit_to_kernel_mode(regs, state);
 362}
 363
 364static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr)
 365{
 366	irqentry_state_t state;
 367
 368	state = arm64_enter_el1_dbg(regs);
 369	debug_exception_enter(regs);
 370	do_breakpoint(esr, regs);
 371	debug_exception_exit(regs);
 372	arm64_exit_el1_dbg(regs, state);
 373}
 374
 375static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)
 376{
 377	irqentry_state_t state;
 378
 379	state = arm64_enter_el1_dbg(regs);
 380	if (!cortex_a76_erratum_1463225_debug_handler(regs)) {
 381		debug_exception_enter(regs);
 382		/*
 383		 * After handling a breakpoint, we suspend the breakpoint
 384		 * and use single-step to move to the next instruction.
 385		 * If we are stepping a suspended breakpoint there's nothing more to do:
 386		 * the single-step is complete.
 387		 */
 388		if (!try_step_suspended_breakpoints(regs))
 389			do_el1_softstep(esr, regs);
 390		debug_exception_exit(regs);
 391	}
 392	arm64_exit_el1_dbg(regs, state);
 393}
 394
 395static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr)
 396{
 397	/* Watchpoints are the only debug exception to write FAR_EL1 */
 398	unsigned long far = read_sysreg(far_el1);
 399	irqentry_state_t state;
 400
 401	state = arm64_enter_el1_dbg(regs);
 402	debug_exception_enter(regs);
 403	do_watchpoint(far, esr, regs);
 404	debug_exception_exit(regs);
 405	arm64_exit_el1_dbg(regs, state);
 406}
 407
 408static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr)
 409{
 410	irqentry_state_t state;
 411
 412	state = arm64_enter_el1_dbg(regs);
 413	debug_exception_enter(regs);
 414	do_el1_brk64(esr, regs);
 415	debug_exception_exit(regs);
 416	arm64_exit_el1_dbg(regs, state);
 417}
 418
 419static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
 420{
 421	irqentry_state_t state;
 422
 423	state = enter_from_kernel_mode(regs);
 424	local_daif_inherit(regs);
 425	do_el1_fpac(regs, esr);
 426	local_daif_mask();
 427	exit_to_kernel_mode(regs, state);
 428}
 429
 430asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
 431{
 432	unsigned long esr = read_sysreg(esr_el1);
 433
 434	switch (ESR_ELx_EC(esr)) {
 435	case ESR_ELx_EC_DABT_CUR:
 436	case ESR_ELx_EC_IABT_CUR:
 437		el1_abort(regs, esr);
 438		break;
 439	/*
 440	 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
 441	 * recursive exception when trying to push the initial pt_regs.
 442	 */
 443	case ESR_ELx_EC_PC_ALIGN:
 444		el1_pc(regs, esr);
 445		break;
 446	case ESR_ELx_EC_SYS64:
 447	case ESR_ELx_EC_UNKNOWN:
 448		el1_undef(regs, esr);
 449		break;
 450	case ESR_ELx_EC_BTI:
 451		el1_bti(regs, esr);
 452		break;
 453	case ESR_ELx_EC_GCS:
 454		el1_gcs(regs, esr);
 455		break;
 456	case ESR_ELx_EC_MOPS:
 457		el1_mops(regs, esr);
 458		break;
 459	case ESR_ELx_EC_BREAKPT_CUR:
 460		el1_breakpt(regs, esr);
 461		break;
 462	case ESR_ELx_EC_SOFTSTP_CUR:
 463		el1_softstp(regs, esr);
 464		break;
 465	case ESR_ELx_EC_WATCHPT_CUR:
 466		el1_watchpt(regs, esr);
 467		break;
 468	case ESR_ELx_EC_BRK64:
 469		el1_brk64(regs, esr);
 470		break;
 471	case ESR_ELx_EC_FPAC:
 472		el1_fpac(regs, esr);
 473		break;
 474	default:
 475		__panic_unhandled(regs, "64-bit el1h sync", esr);
 476	}
 477}
 478
 479static __always_inline void __el1_pnmi(struct pt_regs *regs,
 480				       void (*handler)(struct pt_regs *))
 481{
 482	irqentry_state_t state;
 483
 484	state = irqentry_nmi_enter(regs);
 485	do_interrupt_handler(regs, handler);
 486	irqentry_nmi_exit(regs, state);
 487}
 488
 489static __always_inline void __el1_irq(struct pt_regs *regs,
 490				      void (*handler)(struct pt_regs *))
 491{
 492	irqentry_state_t state;
 493
 494	state = enter_from_kernel_mode(regs);
 495
 496	irq_enter_rcu();
 497	do_interrupt_handler(regs, handler);
 498	irq_exit_rcu();
 499
 500	exit_to_kernel_mode(regs, state);
 501}
 502static void noinstr el1_interrupt(struct pt_regs *regs,
 503				  void (*handler)(struct pt_regs *))
 504{
 505	write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
 506
 507	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
 508		__el1_pnmi(regs, handler);
 509	else
 510		__el1_irq(regs, handler);
 511}
 512
 513asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
 514{
 515	el1_interrupt(regs, handle_arch_irq);
 516}
 517
 518asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
 519{
 520	el1_interrupt(regs, handle_arch_fiq);
 521}
 522
 523asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
 524{
 525	unsigned long esr = read_sysreg(esr_el1);
 526	irqentry_state_t state;
 527
 528	local_daif_restore(DAIF_ERRCTX);
 529	state = irqentry_nmi_enter(regs);
 530	do_serror(regs, esr);
 531	irqentry_nmi_exit(regs, state);
 532}
 533
 534static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
 535{
 536	unsigned long far = read_sysreg(far_el1);
 537
 538	arm64_enter_from_user_mode(regs);
 539	local_daif_restore(DAIF_PROCCTX);
 540	do_mem_abort(far, esr, regs);
 541	arm64_exit_to_user_mode(regs);
 542}
 543
 544static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
 545{
 546	unsigned long far = read_sysreg(far_el1);
 547
 548	/*
 549	 * We've taken an instruction abort from userspace and not yet
 550	 * re-enabled IRQs. If the address is a kernel address, apply
 551	 * BP hardening prior to enabling IRQs and pre-emption.
 552	 */
 553	if (!is_ttbr0_addr(far))
 554		arm64_apply_bp_hardening();
 555
 556	arm64_enter_from_user_mode(regs);
 557	local_daif_restore(DAIF_PROCCTX);
 558	do_mem_abort(far, esr, regs);
 559	arm64_exit_to_user_mode(regs);
 560}
 561
 562static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
 563{
 564	arm64_enter_from_user_mode(regs);
 565	local_daif_restore(DAIF_PROCCTX);
 566	do_fpsimd_acc(esr, regs);
 567	arm64_exit_to_user_mode(regs);
 568}
 569
 570static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
 571{
 572	arm64_enter_from_user_mode(regs);
 573	local_daif_restore(DAIF_PROCCTX);
 574	do_sve_acc(esr, regs);
 575	arm64_exit_to_user_mode(regs);
 576}
 577
 578static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
 579{
 580	arm64_enter_from_user_mode(regs);
 581	local_daif_restore(DAIF_PROCCTX);
 582	do_sme_acc(esr, regs);
 583	arm64_exit_to_user_mode(regs);
 584}
 585
 586static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
 587{
 588	arm64_enter_from_user_mode(regs);
 589	local_daif_restore(DAIF_PROCCTX);
 590	do_fpsimd_exc(esr, regs);
 591	arm64_exit_to_user_mode(regs);
 592}
 593
 594static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
 595{
 596	arm64_enter_from_user_mode(regs);
 597	local_daif_restore(DAIF_PROCCTX);
 598	do_el0_sys(esr, regs);
 599	arm64_exit_to_user_mode(regs);
 600}
 601
 602static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
 603{
 604	unsigned long far = read_sysreg(far_el1);
 605
 606	if (!is_ttbr0_addr(instruction_pointer(regs)))
 607		arm64_apply_bp_hardening();
 608
 609	arm64_enter_from_user_mode(regs);
 610	local_daif_restore(DAIF_PROCCTX);
 611	do_sp_pc_abort(far, esr, regs);
 612	arm64_exit_to_user_mode(regs);
 613}
 614
 615static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
 616{
 617	arm64_enter_from_user_mode(regs);
 618	local_daif_restore(DAIF_PROCCTX);
 619	do_sp_pc_abort(regs->sp, esr, regs);
 620	arm64_exit_to_user_mode(regs);
 621}
 622
 623static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
 624{
 625	arm64_enter_from_user_mode(regs);
 626	local_daif_restore(DAIF_PROCCTX);
 627	do_el0_undef(regs, esr);
 628	arm64_exit_to_user_mode(regs);
 629}
 630
 631static void noinstr el0_bti(struct pt_regs *regs)
 632{
 633	arm64_enter_from_user_mode(regs);
 634	local_daif_restore(DAIF_PROCCTX);
 635	do_el0_bti(regs);
 636	arm64_exit_to_user_mode(regs);
 637}
 638
 639static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
 640{
 641	arm64_enter_from_user_mode(regs);
 642	local_daif_restore(DAIF_PROCCTX);
 643	do_el0_mops(regs, esr);
 644	arm64_exit_to_user_mode(regs);
 645}
 646
 647static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr)
 648{
 649	arm64_enter_from_user_mode(regs);
 650	local_daif_restore(DAIF_PROCCTX);
 651	do_el0_gcs(regs, esr);
 652	arm64_exit_to_user_mode(regs);
 653}
 654
 655static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
 656{
 657	arm64_enter_from_user_mode(regs);
 658	local_daif_restore(DAIF_PROCCTX);
 659	bad_el0_sync(regs, 0, esr);
 660	arm64_exit_to_user_mode(regs);
 661}
 662
 663static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr)
 664{
 665	if (!is_ttbr0_addr(regs->pc))
 666		arm64_apply_bp_hardening();
 667
 668	arm64_enter_from_user_mode(regs);
 669	debug_exception_enter(regs);
 670	do_breakpoint(esr, regs);
 671	debug_exception_exit(regs);
 672	local_daif_restore(DAIF_PROCCTX);
 673	arm64_exit_to_user_mode(regs);
 674}
 675
 676static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr)
 677{
 678	bool step_done;
 679
 680	if (!is_ttbr0_addr(regs->pc))
 681		arm64_apply_bp_hardening();
 682
 683	arm64_enter_from_user_mode(regs);
 684	/*
 685	 * After handling a breakpoint, we suspend the breakpoint
 686	 * and use single-step to move to the next instruction.
 687	 * If we are stepping a suspended breakpoint there's nothing more to do:
 688	 * the single-step is complete.
 689	 */
 690	step_done = try_step_suspended_breakpoints(regs);
 691	local_daif_restore(DAIF_PROCCTX);
 692	if (!step_done)
 693		do_el0_softstep(esr, regs);
 694	arm64_exit_to_user_mode(regs);
 695}
 696
 697static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr)
 698{
 699	/* Watchpoints are the only debug exception to write FAR_EL1 */
 700	unsigned long far = read_sysreg(far_el1);
 701
 702	arm64_enter_from_user_mode(regs);
 703	debug_exception_enter(regs);
 704	do_watchpoint(far, esr, regs);
 705	debug_exception_exit(regs);
 706	local_daif_restore(DAIF_PROCCTX);
 707	arm64_exit_to_user_mode(regs);
 708}
 709
 710static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
 711{
 712	arm64_enter_from_user_mode(regs);
 713	local_daif_restore(DAIF_PROCCTX);
 714	do_el0_brk64(esr, regs);
 715	arm64_exit_to_user_mode(regs);
 716}
 717
 718static void noinstr el0_svc(struct pt_regs *regs)
 719{
 720	arm64_enter_from_user_mode(regs);
 721	cortex_a76_erratum_1463225_svc_handler();
 722	fpsimd_syscall_enter();
 723	local_daif_restore(DAIF_PROCCTX);
 724	do_el0_svc(regs);
 725	arm64_exit_to_user_mode(regs);
 726	fpsimd_syscall_exit();
 727}
 728
 729static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
 730{
 731	arm64_enter_from_user_mode(regs);
 732	local_daif_restore(DAIF_PROCCTX);
 733	do_el0_fpac(regs, esr);
 734	arm64_exit_to_user_mode(regs);
 735}
 736
 737asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
 738{
 739	unsigned long esr = read_sysreg(esr_el1);
 740
 741	switch (ESR_ELx_EC(esr)) {
 742	case ESR_ELx_EC_SVC64:
 743		el0_svc(regs);
 744		break;
 745	case ESR_ELx_EC_DABT_LOW:
 746		el0_da(regs, esr);
 747		break;
 748	case ESR_ELx_EC_IABT_LOW:
 749		el0_ia(regs, esr);
 750		break;
 751	case ESR_ELx_EC_FP_ASIMD:
 752		el0_fpsimd_acc(regs, esr);
 753		break;
 754	case ESR_ELx_EC_SVE:
 755		el0_sve_acc(regs, esr);
 756		break;
 757	case ESR_ELx_EC_SME:
 758		el0_sme_acc(regs, esr);
 759		break;
 760	case ESR_ELx_EC_FP_EXC64:
 761		el0_fpsimd_exc(regs, esr);
 762		break;
 763	case ESR_ELx_EC_SYS64:
 764	case ESR_ELx_EC_WFx:
 765		el0_sys(regs, esr);
 766		break;
 767	case ESR_ELx_EC_SP_ALIGN:
 768		el0_sp(regs, esr);
 769		break;
 770	case ESR_ELx_EC_PC_ALIGN:
 771		el0_pc(regs, esr);
 772		break;
 773	case ESR_ELx_EC_UNKNOWN:
 774		el0_undef(regs, esr);
 775		break;
 776	case ESR_ELx_EC_BTI:
 777		el0_bti(regs);
 778		break;
 779	case ESR_ELx_EC_MOPS:
 780		el0_mops(regs, esr);
 781		break;
 782	case ESR_ELx_EC_GCS:
 783		el0_gcs(regs, esr);
 784		break;
 785	case ESR_ELx_EC_BREAKPT_LOW:
 786		el0_breakpt(regs, esr);
 787		break;
 788	case ESR_ELx_EC_SOFTSTP_LOW:
 789		el0_softstp(regs, esr);
 790		break;
 791	case ESR_ELx_EC_WATCHPT_LOW:
 792		el0_watchpt(regs, esr);
 793		break;
 794	case ESR_ELx_EC_BRK64:
 795		el0_brk64(regs, esr);
 796		break;
 797	case ESR_ELx_EC_FPAC:
 798		el0_fpac(regs, esr);
 799		break;
 800	default:
 801		el0_inv(regs, esr);
 802	}
 803}
 804
 805static void noinstr el0_interrupt(struct pt_regs *regs,
 806				  void (*handler)(struct pt_regs *))
 807{
 808	arm64_enter_from_user_mode(regs);
 809
 810	write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
 811
 812	if (regs->pc & BIT(55))
 813		arm64_apply_bp_hardening();
 814
 815	irq_enter_rcu();
 816	do_interrupt_handler(regs, handler);
 817	irq_exit_rcu();
 818
 819	arm64_exit_to_user_mode(regs);
 820}
 821
 822static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
 823{
 824	el0_interrupt(regs, handle_arch_irq);
 825}
 826
 827asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
 828{
 829	__el0_irq_handler_common(regs);
 830}
 831
 832static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
 833{
 834	el0_interrupt(regs, handle_arch_fiq);
 835}
 836
 837asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
 838{
 839	__el0_fiq_handler_common(regs);
 840}
 841
 842static void noinstr __el0_error_handler_common(struct pt_regs *regs)
 843{
 844	unsigned long esr = read_sysreg(esr_el1);
 845	irqentry_state_t state;
 846
 847	arm64_enter_from_user_mode(regs);
 848	local_daif_restore(DAIF_ERRCTX);
 849	state = irqentry_nmi_enter(regs);
 850	do_serror(regs, esr);
 851	irqentry_nmi_exit(regs, state);
 852	local_daif_restore(DAIF_PROCCTX);
 853	arm64_exit_to_user_mode(regs);
 854}
 855
 856asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
 857{
 858	__el0_error_handler_common(regs);
 859}
 860
 861#ifdef CONFIG_COMPAT
 862static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
 863{
 864	arm64_enter_from_user_mode(regs);
 865	local_daif_restore(DAIF_PROCCTX);
 866	do_el0_cp15(esr, regs);
 867	arm64_exit_to_user_mode(regs);
 868}
 869
 870static void noinstr el0_svc_compat(struct pt_regs *regs)
 871{
 872	arm64_enter_from_user_mode(regs);
 873	cortex_a76_erratum_1463225_svc_handler();
 874	local_daif_restore(DAIF_PROCCTX);
 875	do_el0_svc_compat(regs);
 876	arm64_exit_to_user_mode(regs);
 877}
 878
 879static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr)
 880{
 881	arm64_enter_from_user_mode(regs);
 882	local_daif_restore(DAIF_PROCCTX);
 883	do_bkpt32(esr, regs);
 884	arm64_exit_to_user_mode(regs);
 885}
 886
 887asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
 888{
 889	unsigned long esr = read_sysreg(esr_el1);
 890
 891	switch (ESR_ELx_EC(esr)) {
 892	case ESR_ELx_EC_SVC32:
 893		el0_svc_compat(regs);
 894		break;
 895	case ESR_ELx_EC_DABT_LOW:
 896		el0_da(regs, esr);
 897		break;
 898	case ESR_ELx_EC_IABT_LOW:
 899		el0_ia(regs, esr);
 900		break;
 901	case ESR_ELx_EC_FP_ASIMD:
 902		el0_fpsimd_acc(regs, esr);
 903		break;
 904	case ESR_ELx_EC_FP_EXC32:
 905		el0_fpsimd_exc(regs, esr);
 906		break;
 907	case ESR_ELx_EC_PC_ALIGN:
 908		el0_pc(regs, esr);
 909		break;
 910	case ESR_ELx_EC_UNKNOWN:
 911	case ESR_ELx_EC_CP14_MR:
 912	case ESR_ELx_EC_CP14_LS:
 913	case ESR_ELx_EC_CP14_64:
 914		el0_undef(regs, esr);
 915		break;
 916	case ESR_ELx_EC_CP15_32:
 917	case ESR_ELx_EC_CP15_64:
 918		el0_cp15(regs, esr);
 919		break;
 920	case ESR_ELx_EC_BREAKPT_LOW:
 921		el0_breakpt(regs, esr);
 922		break;
 923	case ESR_ELx_EC_SOFTSTP_LOW:
 924		el0_softstp(regs, esr);
 925		break;
 926	case ESR_ELx_EC_WATCHPT_LOW:
 927		el0_watchpt(regs, esr);
 928		break;
 929	case ESR_ELx_EC_BKPT32:
 930		el0_bkpt32(regs, esr);
 931		break;
 932	default:
 933		el0_inv(regs, esr);
 934	}
 935}
 936
 937asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
 938{
 939	__el0_irq_handler_common(regs);
 940}
 941
 942asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
 943{
 944	__el0_fiq_handler_common(regs);
 945}
 946
 947asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
 948{
 949	__el0_error_handler_common(regs);
 950}
 951#else /* CONFIG_COMPAT */
 952UNHANDLED(el0t, 32, sync)
 953UNHANDLED(el0t, 32, irq)
 954UNHANDLED(el0t, 32, fiq)
 955UNHANDLED(el0t, 32, error)
 956#endif /* CONFIG_COMPAT */
 957
 958asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
 959{
 960	unsigned long esr = read_sysreg(esr_el1);
 961	unsigned long far = read_sysreg(far_el1);
 962
 963	irqentry_nmi_enter(regs);
 964	panic_bad_stack(regs, esr, far);
 965}
 966
 967#ifdef CONFIG_ARM_SDE_INTERFACE
 968asmlinkage noinstr unsigned long
 969__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
 970{
 971	irqentry_state_t state;
 972	unsigned long ret;
 973
 974	/*
 975	 * We didn't take an exception to get here, so the HW hasn't
 976	 * set/cleared bits in PSTATE that we may rely on.
 977	 *
 978	 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
 979	 * whether PSTATE bits are inherited unchanged or generated from
 980	 * scratch, and the TF-A implementation always clears PAN and always
 981	 * clears UAO. There are no other known implementations.
 982	 *
 983	 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
 984	 * PSTATE is modified upon architectural exceptions, and so PAN is
 985	 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
 986	 * cleared.
 987	 *
 988	 * We must explicitly reset PAN to the expected state, including
 989	 * clearing it when the host isn't using it, in case a VM had it set.
 990	 */
 991	if (system_uses_hw_pan())
 992		set_pstate_pan(1);
 993	else if (cpu_has_pan())
 994		set_pstate_pan(0);
 995
 996	state = irqentry_nmi_enter(regs);
 997	ret = do_sdei_event(regs, arg);
 998	irqentry_nmi_exit(regs, state);
 999
1000	return ret;
1001}
1002#endif /* CONFIG_ARM_SDE_INTERFACE */