traps.c

   1/*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * Copyright (C) 1994 - 1999, 2000, 01, 06 Ralf Baechle
   7 * Copyright (C) 1995, 1996 Paul M. Antoine
   8 * Copyright (C) 1998 Ulf Carlsson
   9 * Copyright (C) 1999 Silicon Graphics, Inc.
  10 * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
  11 * Copyright (C) 2002, 2003, 2004, 2005, 2007  Maciej W. Rozycki
  12 * Copyright (C) 2000, 2001, 2012 MIPS Technologies, Inc.  All rights reserved.
  13 * Copyright (C) 2014, Imagination Technologies Ltd.
  14 */
  15#include <linux/bitops.h>
  16#include <linux/bug.h>
  17#include <linux/compiler.h>
  18#include <linux/context_tracking.h>
  19#include <linux/cpu_pm.h>
  20#include <linux/kexec.h>
  21#include <linux/init.h>
  22#include <linux/kernel.h>
  23#include <linux/module.h>
  24#include <linux/extable.h>
  25#include <linux/mm.h>
  26#include <linux/sched/mm.h>
  27#include <linux/sched/debug.h>
  28#include <linux/smp.h>
  29#include <linux/spinlock.h>
  30#include <linux/kallsyms.h>
  31#include <linux/memblock.h>
  32#include <linux/interrupt.h>
  33#include <linux/ptrace.h>
  34#include <linux/kgdb.h>
  35#include <linux/kdebug.h>
  36#include <linux/kprobes.h>
  37#include <linux/notifier.h>
  38#include <linux/kdb.h>
  39#include <linux/irq.h>
  40#include <linux/perf_event.h>
  41#include <linux/string_choices.h>
  42
  43#include <asm/addrspace.h>
  44#include <asm/bootinfo.h>
  45#include <asm/branch.h>
  46#include <asm/break.h>
  47#include <asm/cop2.h>
  48#include <asm/cpu.h>
  49#include <asm/cpu-type.h>
  50#include <asm/dsp.h>
  51#include <asm/fpu.h>
  52#include <asm/fpu_emulator.h>
  53#include <asm/idle.h>
  54#include <asm/isa-rev.h>
  55#include <asm/mips-cps.h>
  56#include <asm/mips-r2-to-r6-emul.h>
  57#include <asm/mipsregs.h>
  58#include <asm/mipsmtregs.h>
  59#include <asm/module.h>
  60#include <asm/msa.h>
  61#include <asm/ptrace.h>
  62#include <asm/regdef.h>
  63#include <asm/sections.h>
  64#include <asm/siginfo.h>
  65#include <asm/tlbdebug.h>
  66#include <asm/traps.h>
  67#include <linux/uaccess.h>
  68#include <asm/watch.h>
  69#include <asm/mmu_context.h>
  70#include <asm/types.h>
  71#include <asm/stacktrace.h>
  72#include <asm/tlbex.h>
  73#include <asm/uasm.h>
  74
  75#include <asm/mach-loongson64/cpucfg-emul.h>
  76
  77#include "access-helper.h"
  78
  79extern void check_wait(void);
  80extern asmlinkage void skipover_handle_int(void);
  81extern asmlinkage void handle_int(void);
  82extern asmlinkage void handle_adel(void);
  83extern asmlinkage void handle_ades(void);
  84extern asmlinkage void handle_ibe(void);
  85extern asmlinkage void handle_dbe(void);
  86extern asmlinkage void handle_sys(void);
  87extern asmlinkage void handle_bp(void);
  88extern asmlinkage void handle_ri(void);
  89extern asmlinkage void handle_ri_rdhwr_tlbp(void);
  90extern asmlinkage void handle_ri_rdhwr(void);
  91extern asmlinkage void handle_cpu(void);
  92extern asmlinkage void handle_ov(void);
  93extern asmlinkage void handle_tr(void);
  94extern asmlinkage void handle_msa_fpe(void);
  95extern asmlinkage void handle_fpe(void);
  96extern asmlinkage void handle_ftlb(void);
  97extern asmlinkage void handle_gsexc(void);
  98extern asmlinkage void handle_msa(void);
  99extern asmlinkage void handle_mdmx(void);
 100extern asmlinkage void handle_watch(void);
 101extern asmlinkage void handle_mt(void);
 102extern asmlinkage void handle_dsp(void);
 103extern asmlinkage void handle_mcheck(void);
 104extern asmlinkage void handle_reserved(void);
 105extern void tlb_do_page_fault_0(void);
 106
 107void (*board_be_init)(void);
 108static int (*board_be_handler)(struct pt_regs *regs, int is_fixup);
 109void (*board_nmi_handler_setup)(void);
 110void (*board_ejtag_handler_setup)(void);
 111void (*board_bind_eic_interrupt)(int irq, int regset);
 112void (*board_ebase_setup)(void);
 113void(*board_cache_error_setup)(void);
 114
 115void mips_set_be_handler(int (*handler)(struct pt_regs *regs, int is_fixup))
 116{
 117	board_be_handler = handler;
 118}
 119EXPORT_SYMBOL_GPL(mips_set_be_handler);
 120
 121static void show_raw_backtrace(unsigned long reg29, const char *loglvl,
 122			       bool user)
 123{
 124	unsigned long *sp = (unsigned long *)(reg29 & ~3);
 125	unsigned long addr;
 126
 127	printk("%sCall Trace:", loglvl);
 128#ifdef CONFIG_KALLSYMS
 129	printk("%s\n", loglvl);
 130#endif
 131	while (!kstack_end(sp)) {
 132		if (__get_addr(&addr, sp++, user)) {
 133			printk("%s (Bad stack address)", loglvl);
 134			break;
 135		}
 136		if (__kernel_text_address(addr))
 137			print_ip_sym(loglvl, addr);
 138	}
 139	printk("%s\n", loglvl);
 140}
 141
 142#ifdef CONFIG_KALLSYMS
 143int raw_show_trace;
 144static int __init set_raw_show_trace(char *str)
 145{
 146	raw_show_trace = 1;
 147	return 1;
 148}
 149__setup("raw_show_trace", set_raw_show_trace);
 150#endif
 151
 152static void show_backtrace(struct task_struct *task, const struct pt_regs *regs,
 153			   const char *loglvl, bool user)
 154{
 155	unsigned long sp = regs->regs[29];
 156	unsigned long ra = regs->regs[31];
 157	unsigned long pc = regs->cp0_epc;
 158
 159	if (!task)
 160		task = current;
 161
 162	if (raw_show_trace || user_mode(regs) || !__kernel_text_address(pc)) {
 163		show_raw_backtrace(sp, loglvl, user);
 164		return;
 165	}
 166	printk("%sCall Trace:\n", loglvl);
 167	do {
 168		print_ip_sym(loglvl, pc);
 169		pc = unwind_stack(task, &sp, pc, &ra);
 170	} while (pc);
 171	pr_cont("\n");
 172}
 173
 174/*
 175 * This routine abuses get_user()/put_user() to reference pointers
 176 * with at least a bit of error checking ...
 177 */
 178static void show_stacktrace(struct task_struct *task,
 179	const struct pt_regs *regs, const char *loglvl, bool user)
 180{
 181	const int field = 2 * sizeof(unsigned long);
 182	unsigned long stackdata;
 183	int i;
 184	unsigned long *sp = (unsigned long *)regs->regs[29];
 185
 186	printk("%sStack :", loglvl);
 187	i = 0;
 188	while ((unsigned long) sp & (PAGE_SIZE - 1)) {
 189		if (i && ((i % (64 / field)) == 0)) {
 190			pr_cont("\n");
 191			printk("%s       ", loglvl);
 192		}
 193		if (i > 39) {
 194			pr_cont(" ...");
 195			break;
 196		}
 197
 198		if (__get_addr(&stackdata, sp++, user)) {
 199			pr_cont(" (Bad stack address)");
 200			break;
 201		}
 202
 203		pr_cont(" %0*lx", field, stackdata);
 204		i++;
 205	}
 206	pr_cont("\n");
 207	show_backtrace(task, regs, loglvl, user);
 208}
 209
 210void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl)
 211{
 212	struct pt_regs regs;
 213
 214	regs.cp0_status = KSU_KERNEL;
 215	if (sp) {
 216		regs.regs[29] = (unsigned long)sp;
 217		regs.regs[31] = 0;
 218		regs.cp0_epc = 0;
 219	} else {
 220		if (task && task != current) {
 221			regs.regs[29] = task->thread.reg29;
 222			regs.regs[31] = 0;
 223			regs.cp0_epc = task->thread.reg31;
 224		} else {
 225			prepare_frametrace(&regs);
 226		}
 227	}
 228	show_stacktrace(task, &regs, loglvl, false);
 229}
 230
 231static void show_code(void *pc, bool user)
 232{
 233	long i;
 234	unsigned short *pc16 = NULL;
 235
 236	printk("Code:");
 237
 238	if ((unsigned long)pc & 1)
 239		pc16 = (u16 *)((unsigned long)pc & ~1);
 240
 241	for(i = -3 ; i < 6 ; i++) {
 242		if (pc16) {
 243			u16 insn16;
 244
 245			if (__get_inst16(&insn16, pc16 + i, user))
 246				goto bad_address;
 247
 248			pr_cont("%c%04x%c", (i?' ':'<'), insn16, (i?' ':'>'));
 249		} else {
 250			u32 insn32;
 251
 252			if (__get_inst32(&insn32, (u32 *)pc + i, user))
 253				goto bad_address;
 254
 255			pr_cont("%c%08x%c", (i?' ':'<'), insn32, (i?' ':'>'));
 256		}
 257	}
 258	pr_cont("\n");
 259	return;
 260
 261bad_address:
 262	pr_cont(" (Bad address in epc)\n\n");
 263}
 264
 265static void __show_regs(const struct pt_regs *regs)
 266{
 267	const int field = 2 * sizeof(unsigned long);
 268	unsigned int cause = regs->cp0_cause;
 269	unsigned int exccode;
 270	int i;
 271
 272	show_regs_print_info(KERN_DEFAULT);
 273
 274	/*
 275	 * Saved main processor registers
 276	 */
 277	for (i = 0; i < 32; ) {
 278		if ((i % 4) == 0)
 279			printk("$%2d   :", i);
 280		if (i == 0)
 281			pr_cont(" %0*lx", field, 0UL);
 282		else if (i == 26 || i == 27)
 283			pr_cont(" %*s", field, "");
 284		else
 285			pr_cont(" %0*lx", field, regs->regs[i]);
 286
 287		i++;
 288		if ((i % 4) == 0)
 289			pr_cont("\n");
 290	}
 291
 292#ifdef CONFIG_CPU_HAS_SMARTMIPS
 293	printk("Acx    : %0*lx\n", field, regs->acx);
 294#endif
 295	if (MIPS_ISA_REV < 6) {
 296		printk("Hi    : %0*lx\n", field, regs->hi);
 297		printk("Lo    : %0*lx\n", field, regs->lo);
 298	}
 299
 300	/*
 301	 * Saved cp0 registers
 302	 */
 303	printk("epc   : %0*lx %pS\n", field, regs->cp0_epc,
 304	       (void *) regs->cp0_epc);
 305	printk("ra    : %0*lx %pS\n", field, regs->regs[31],
 306	       (void *) regs->regs[31]);
 307
 308	printk("Status: %08x	", (uint32_t) regs->cp0_status);
 309
 310	if (cpu_has_3kex) {
 311		if (regs->cp0_status & ST0_KUO)
 312			pr_cont("KUo ");
 313		if (regs->cp0_status & ST0_IEO)
 314			pr_cont("IEo ");
 315		if (regs->cp0_status & ST0_KUP)
 316			pr_cont("KUp ");
 317		if (regs->cp0_status & ST0_IEP)
 318			pr_cont("IEp ");
 319		if (regs->cp0_status & ST0_KUC)
 320			pr_cont("KUc ");
 321		if (regs->cp0_status & ST0_IEC)
 322			pr_cont("IEc ");
 323	} else if (cpu_has_4kex) {
 324		if (regs->cp0_status & ST0_KX)
 325			pr_cont("KX ");
 326		if (regs->cp0_status & ST0_SX)
 327			pr_cont("SX ");
 328		if (regs->cp0_status & ST0_UX)
 329			pr_cont("UX ");
 330		switch (regs->cp0_status & ST0_KSU) {
 331		case KSU_USER:
 332			pr_cont("USER ");
 333			break;
 334		case KSU_SUPERVISOR:
 335			pr_cont("SUPERVISOR ");
 336			break;
 337		case KSU_KERNEL:
 338			pr_cont("KERNEL ");
 339			break;
 340		default:
 341			pr_cont("BAD_MODE ");
 342			break;
 343		}
 344		if (regs->cp0_status & ST0_ERL)
 345			pr_cont("ERL ");
 346		if (regs->cp0_status & ST0_EXL)
 347			pr_cont("EXL ");
 348		if (regs->cp0_status & ST0_IE)
 349			pr_cont("IE ");
 350	}
 351	pr_cont("\n");
 352
 353	exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE;
 354	printk("Cause : %08x (ExcCode %02x)\n", cause, exccode);
 355
 356	if (1 <= exccode && exccode <= 5)
 357		printk("BadVA : %0*lx\n", field, regs->cp0_badvaddr);
 358
 359	printk("PrId  : %08x (%s)\n", read_c0_prid(),
 360	       cpu_name_string());
 361}
 362
 363/*
 364 * FIXME: really the generic show_regs should take a const pointer argument.
 365 */
 366void show_regs(struct pt_regs *regs)
 367{
 368	__show_regs(regs);
 369	dump_stack();
 370}
 371
 372void show_registers(struct pt_regs *regs)
 373{
 374	const int field = 2 * sizeof(unsigned long);
 375
 376	__show_regs(regs);
 377	print_modules();
 378	printk("Process %s (pid: %d, threadinfo=%p, task=%p, tls=%0*lx)\n",
 379	       current->comm, current->pid, current_thread_info(), current,
 380	      field, current_thread_info()->tp_value);
 381	if (cpu_has_userlocal) {
 382		unsigned long tls;
 383
 384		tls = read_c0_userlocal();
 385		if (tls != current_thread_info()->tp_value)
 386			printk("*HwTLS: %0*lx\n", field, tls);
 387	}
 388
 389	show_stacktrace(current, regs, KERN_DEFAULT, user_mode(regs));
 390	show_code((void *)regs->cp0_epc, user_mode(regs));
 391	printk("\n");
 392}
 393
 394static DEFINE_RAW_SPINLOCK(die_lock);
 395
 396void __noreturn die(const char *str, struct pt_regs *regs)
 397{
 398	static int die_counter;
 399	int sig = SIGSEGV;
 400
 401	oops_enter();
 402
 403	if (notify_die(DIE_OOPS, str, regs, 0, current->thread.trap_nr,
 404		       SIGSEGV) == NOTIFY_STOP)
 405		sig = 0;
 406
 407	console_verbose();
 408	raw_spin_lock_irq(&die_lock);
 409	bust_spinlocks(1);
 410
 411	printk("%s[#%d]:\n", str, ++die_counter);
 412	show_registers(regs);
 413	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 414	raw_spin_unlock_irq(&die_lock);
 415
 416	oops_exit();
 417
 418	if (in_interrupt())
 419		panic("Fatal exception in interrupt");
 420
 421	if (panic_on_oops)
 422		panic("Fatal exception");
 423
 424	if (regs && kexec_should_crash(current))
 425		crash_kexec(regs);
 426
 427	make_task_dead(sig);
 428}
 429
 430extern struct exception_table_entry __start___dbe_table[];
 431extern struct exception_table_entry __stop___dbe_table[];
 432
 433__asm__(
 434"	.section	__dbe_table, \"a\"\n"
 435"	.previous			\n");
 436
 437/* Given an address, look for it in the exception tables. */
 438static const struct exception_table_entry *search_dbe_tables(unsigned long addr)
 439{
 440	const struct exception_table_entry *e;
 441
 442	e = search_extable(__start___dbe_table,
 443			   __stop___dbe_table - __start___dbe_table, addr);
 444	if (!e)
 445		e = search_module_dbetables(addr);
 446	return e;
 447}
 448
 449asmlinkage void do_be(struct pt_regs *regs)
 450{
 451	const int field = 2 * sizeof(unsigned long);
 452	const struct exception_table_entry *fixup = NULL;
 453	int data = regs->cp0_cause & 4;
 454	int action = MIPS_BE_FATAL;
 455	enum ctx_state prev_state;
 456
 457	prev_state = exception_enter();
 458	/* XXX For now.	 Fixme, this searches the wrong table ...  */
 459	if (data && !user_mode(regs))
 460		fixup = search_dbe_tables(exception_epc(regs));
 461
 462	if (fixup)
 463		action = MIPS_BE_FIXUP;
 464
 465	if (board_be_handler)
 466		action = board_be_handler(regs, fixup != NULL);
 467	else
 468		mips_cm_error_report();
 469
 470	switch (action) {
 471	case MIPS_BE_DISCARD:
 472		goto out;
 473	case MIPS_BE_FIXUP:
 474		if (fixup) {
 475			regs->cp0_epc = fixup->nextinsn;
 476			goto out;
 477		}
 478		break;
 479	default:
 480		break;
 481	}
 482
 483	/*
 484	 * Assume it would be too dangerous to continue ...
 485	 */
 486	printk(KERN_ALERT "%s bus error, epc == %0*lx, ra == %0*lx\n",
 487	       data ? "Data" : "Instruction",
 488	       field, regs->cp0_epc, field, regs->regs[31]);
 489	if (notify_die(DIE_OOPS, "bus error", regs, 0, current->thread.trap_nr,
 490		       SIGBUS) == NOTIFY_STOP)
 491		goto out;
 492
 493	die_if_kernel("Oops", regs);
 494	force_sig(SIGBUS);
 495
 496out:
 497	exception_exit(prev_state);
 498}
 499
 500/*
 501 * ll/sc, rdhwr, sync emulation
 502 */
 503
 504#define OPCODE 0xfc000000
 505#define BASE   0x03e00000
 506#define RT     0x001f0000
 507#define OFFSET 0x0000ffff
 508#define LL     0xc0000000
 509#define SC     0xe0000000
 510#define SPEC0  0x00000000
 511#define SPEC3  0x7c000000
 512#define RD     0x0000f800
 513#define FUNC   0x0000003f
 514#define SYNC   0x0000000f
 515#define RDHWR  0x0000003b
 516
 517/*  microMIPS definitions   */
 518#define MM_POOL32A_FUNC 0xfc00ffff
 519#define MM_RDHWR        0x00006b3c
 520#define MM_RS           0x001f0000
 521#define MM_RT           0x03e00000
 522
 523/*
 524 * The ll_bit is cleared by r*_switch.S
 525 */
 526
 527unsigned int ll_bit;
 528struct task_struct *ll_task;
 529
 530static inline int simulate_ll(struct pt_regs *regs, unsigned int opcode)
 531{
 532	unsigned long value, __user *vaddr;
 533	long offset;
 534
 535	/*
 536	 * analyse the ll instruction that just caused a ri exception
 537	 * and put the referenced address to addr.
 538	 */
 539
 540	/* sign extend offset */
 541	offset = opcode & OFFSET;
 542	offset <<= 16;
 543	offset >>= 16;
 544
 545	vaddr = (unsigned long __user *)
 546		((unsigned long)(regs->regs[(opcode & BASE) >> 21]) + offset);
 547
 548	if ((unsigned long)vaddr & 3)
 549		return SIGBUS;
 550	if (get_user(value, vaddr))
 551		return SIGSEGV;
 552
 553	preempt_disable();
 554
 555	if (ll_task == NULL || ll_task == current) {
 556		ll_bit = 1;
 557	} else {
 558		ll_bit = 0;
 559	}
 560	ll_task = current;
 561
 562	preempt_enable();
 563
 564	regs->regs[(opcode & RT) >> 16] = value;
 565
 566	return 0;
 567}
 568
 569static inline int simulate_sc(struct pt_regs *regs, unsigned int opcode)
 570{
 571	unsigned long __user *vaddr;
 572	unsigned long reg;
 573	long offset;
 574
 575	/*
 576	 * analyse the sc instruction that just caused a ri exception
 577	 * and put the referenced address to addr.
 578	 */
 579
 580	/* sign extend offset */
 581	offset = opcode & OFFSET;
 582	offset <<= 16;
 583	offset >>= 16;
 584
 585	vaddr = (unsigned long __user *)
 586		((unsigned long)(regs->regs[(opcode & BASE) >> 21]) + offset);
 587	reg = (opcode & RT) >> 16;
 588
 589	if ((unsigned long)vaddr & 3)
 590		return SIGBUS;
 591
 592	preempt_disable();
 593
 594	if (ll_bit == 0 || ll_task != current) {
 595		regs->regs[reg] = 0;
 596		preempt_enable();
 597		return 0;
 598	}
 599
 600	preempt_enable();
 601
 602	if (put_user(regs->regs[reg], vaddr))
 603		return SIGSEGV;
 604
 605	regs->regs[reg] = 1;
 606
 607	return 0;
 608}
 609
 610/*
 611 * ll uses the opcode of lwc0 and sc uses the opcode of swc0.  That is both
 612 * opcodes are supposed to result in coprocessor unusable exceptions if
 613 * executed on ll/sc-less processors.  That's the theory.  In practice a
 614 * few processors such as NEC's VR4100 throw reserved instruction exceptions
 615 * instead, so we're doing the emulation thing in both exception handlers.
 616 */
 617static int simulate_llsc(struct pt_regs *regs, unsigned int opcode)
 618{
 619	if ((opcode & OPCODE) == LL) {
 620		perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
 621				1, regs, 0);
 622		return simulate_ll(regs, opcode);
 623	}
 624	if ((opcode & OPCODE) == SC) {
 625		perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
 626				1, regs, 0);
 627		return simulate_sc(regs, opcode);
 628	}
 629
 630	return -1;			/* Must be something else ... */
 631}
 632
 633/*
 634 * Simulate trapping 'rdhwr' instructions to provide user accessible
 635 * registers not implemented in hardware.
 636 */
 637static int simulate_rdhwr(struct pt_regs *regs, int rd, int rt)
 638{
 639	struct thread_info *ti = task_thread_info(current);
 640
 641	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
 642			1, regs, 0);
 643	switch (rd) {
 644	case MIPS_HWR_CPUNUM:		/* CPU number */
 645		regs->regs[rt] = smp_processor_id();
 646		return 0;
 647	case MIPS_HWR_SYNCISTEP:	/* SYNCI length */
 648		regs->regs[rt] = min(current_cpu_data.dcache.linesz,
 649				     current_cpu_data.icache.linesz);
 650		return 0;
 651	case MIPS_HWR_CC:		/* Read count register */
 652		regs->regs[rt] = read_c0_count();
 653		return 0;
 654	case MIPS_HWR_CCRES:		/* Count register resolution */
 655		switch (current_cpu_type()) {
 656		case CPU_20KC:
 657		case CPU_25KF:
 658			regs->regs[rt] = 1;
 659			break;
 660		default:
 661			regs->regs[rt] = 2;
 662		}
 663		return 0;
 664	case MIPS_HWR_ULR:		/* Read UserLocal register */
 665		regs->regs[rt] = ti->tp_value;
 666		return 0;
 667	default:
 668		return -1;
 669	}
 670}
 671
 672static int simulate_rdhwr_normal(struct pt_regs *regs, unsigned int opcode)
 673{
 674	if ((opcode & OPCODE) == SPEC3 && (opcode & FUNC) == RDHWR) {
 675		int rd = (opcode & RD) >> 11;
 676		int rt = (opcode & RT) >> 16;
 677
 678		simulate_rdhwr(regs, rd, rt);
 679		return 0;
 680	}
 681
 682	/* Not ours.  */
 683	return -1;
 684}
 685
 686static int simulate_rdhwr_mm(struct pt_regs *regs, unsigned int opcode)
 687{
 688	if ((opcode & MM_POOL32A_FUNC) == MM_RDHWR) {
 689		int rd = (opcode & MM_RS) >> 16;
 690		int rt = (opcode & MM_RT) >> 21;
 691		simulate_rdhwr(regs, rd, rt);
 692		return 0;
 693	}
 694
 695	/* Not ours.  */
 696	return -1;
 697}
 698
 699static int simulate_sync(struct pt_regs *regs, unsigned int opcode)
 700{
 701	if ((opcode & OPCODE) == SPEC0 && (opcode & FUNC) == SYNC) {
 702		perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS,
 703				1, regs, 0);
 704		return 0;
 705	}
 706
 707	return -1;			/* Must be something else ... */
 708}
 709
 710/*
 711 * Loongson-3 CSR instructions emulation
 712 */
 713
 714#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION
 715
 716#define LWC2             0xc8000000
 717#define RS               BASE
 718#define CSR_OPCODE2      0x00000118
 719#define CSR_OPCODE2_MASK 0x000007ff
 720#define CSR_FUNC_MASK    RT
 721#define CSR_FUNC_CPUCFG  0x8
 722
 723static int simulate_loongson3_cpucfg(struct pt_regs *regs,
 724				     unsigned int opcode)
 725{
 726	int op = opcode & OPCODE;
 727	int op2 = opcode & CSR_OPCODE2_MASK;
 728	int csr_func = (opcode & CSR_FUNC_MASK) >> 16;
 729
 730	if (op == LWC2 && op2 == CSR_OPCODE2 && csr_func == CSR_FUNC_CPUCFG) {
 731		int rd = (opcode & RD) >> 11;
 732		int rs = (opcode & RS) >> 21;
 733		__u64 sel = regs->regs[rs];
 734
 735		perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 736
 737		/* Do not emulate on unsupported core models. */
 738		preempt_disable();
 739		if (!loongson3_cpucfg_emulation_enabled(&current_cpu_data)) {
 740			preempt_enable();
 741			return -1;
 742		}
 743		regs->regs[rd] = loongson3_cpucfg_read_synthesized(
 744			&current_cpu_data, sel);
 745		preempt_enable();
 746		return 0;
 747	}
 748
 749	/* Not ours.  */
 750	return -1;
 751}
 752#endif /* CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION */
 753
 754asmlinkage void do_ov(struct pt_regs *regs)
 755{
 756	enum ctx_state prev_state;
 757
 758	prev_state = exception_enter();
 759	die_if_kernel("Integer overflow", regs);
 760
 761	force_sig_fault(SIGFPE, FPE_INTOVF, (void __user *)regs->cp0_epc);
 762	exception_exit(prev_state);
 763}
 764
 765#ifdef CONFIG_MIPS_FP_SUPPORT
 766
 767/*
 768 * Send SIGFPE according to FCSR Cause bits, which must have already
 769 * been masked against Enable bits.  This is impotant as Inexact can
 770 * happen together with Overflow or Underflow, and `ptrace' can set
 771 * any bits.
 772 */
 773void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr,
 774		     struct task_struct *tsk)
 775{
 776	int si_code = FPE_FLTUNK;
 777
 778	if (fcr31 & FPU_CSR_INV_X)
 779		si_code = FPE_FLTINV;
 780	else if (fcr31 & FPU_CSR_DIV_X)
 781		si_code = FPE_FLTDIV;
 782	else if (fcr31 & FPU_CSR_OVF_X)
 783		si_code = FPE_FLTOVF;
 784	else if (fcr31 & FPU_CSR_UDF_X)
 785		si_code = FPE_FLTUND;
 786	else if (fcr31 & FPU_CSR_INE_X)
 787		si_code = FPE_FLTRES;
 788
 789	force_sig_fault_to_task(SIGFPE, si_code, fault_addr, tsk);
 790}
 791
 792int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31)
 793{
 794	int si_code;
 795
 796	switch (sig) {
 797	case 0:
 798		return 0;
 799
 800	case SIGFPE:
 801		force_fcr31_sig(fcr31, fault_addr, current);
 802		return 1;
 803
 804	case SIGBUS:
 805		force_sig_fault(SIGBUS, BUS_ADRERR, fault_addr);
 806		return 1;
 807
 808	case SIGSEGV:
 809		mmap_read_lock(current->mm);
 810		if (vma_lookup(current->mm, (unsigned long)fault_addr))
 811			si_code = SEGV_ACCERR;
 812		else
 813			si_code = SEGV_MAPERR;
 814		mmap_read_unlock(current->mm);
 815		force_sig_fault(SIGSEGV, si_code, fault_addr);
 816		return 1;
 817
 818	default:
 819		force_sig(sig);
 820		return 1;
 821	}
 822}
 823
 824static int simulate_fp(struct pt_regs *regs, unsigned int opcode,
 825		       unsigned long old_epc, unsigned long old_ra)
 826{
 827	union mips_instruction inst = { .word = opcode };
 828	void __user *fault_addr;
 829	unsigned long fcr31;
 830	int sig;
 831
 832	/* If it's obviously not an FP instruction, skip it */
 833	switch (inst.i_format.opcode) {
 834	case cop1_op:
 835	case cop1x_op:
 836	case lwc1_op:
 837	case ldc1_op:
 838	case swc1_op:
 839	case sdc1_op:
 840		break;
 841
 842	default:
 843		return -1;
 844	}
 845
 846	/*
 847	 * do_ri skipped over the instruction via compute_return_epc, undo
 848	 * that for the FPU emulator.
 849	 */
 850	regs->cp0_epc = old_epc;
 851	regs->regs[31] = old_ra;
 852
 853	/* Run the emulator */
 854	sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
 855				       &fault_addr);
 856
 857	/*
 858	 * We can't allow the emulated instruction to leave any
 859	 * enabled Cause bits set in $fcr31.
 860	 */
 861	fcr31 = mask_fcr31_x(current->thread.fpu.fcr31);
 862	current->thread.fpu.fcr31 &= ~fcr31;
 863
 864	/* Restore the hardware register state */
 865	own_fpu(1);
 866
 867	/* Send a signal if required.  */
 868	process_fpemu_return(sig, fault_addr, fcr31);
 869
 870	return 0;
 871}
 872
 873/*
 874 * XXX Delayed fp exceptions when doing a lazy ctx switch XXX
 875 */
 876asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
 877{
 878	enum ctx_state prev_state;
 879	void __user *fault_addr;
 880	int sig;
 881
 882	prev_state = exception_enter();
 883	if (notify_die(DIE_FP, "FP exception", regs, 0, current->thread.trap_nr,
 884		       SIGFPE) == NOTIFY_STOP)
 885		goto out;
 886
 887	/* Clear FCSR.Cause before enabling interrupts */
 888	write_32bit_cp1_register(CP1_STATUS, fcr31 & ~mask_fcr31_x(fcr31));
 889	local_irq_enable();
 890
 891	die_if_kernel("FP exception in kernel code", regs);
 892
 893	if (fcr31 & FPU_CSR_UNI_X) {
 894		/*
 895		 * Unimplemented operation exception.  If we've got the full
 896		 * software emulator on-board, let's use it...
 897		 *
 898		 * Force FPU to dump state into task/thread context.  We're
 899		 * moving a lot of data here for what is probably a single
 900		 * instruction, but the alternative is to pre-decode the FP
 901		 * register operands before invoking the emulator, which seems
 902		 * a bit extreme for what should be an infrequent event.
 903		 */
 904
 905		/* Run the emulator */
 906		sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
 907					       &fault_addr);
 908
 909		/*
 910		 * We can't allow the emulated instruction to leave any
 911		 * enabled Cause bits set in $fcr31.
 912		 */
 913		fcr31 = mask_fcr31_x(current->thread.fpu.fcr31);
 914		current->thread.fpu.fcr31 &= ~fcr31;
 915
 916		/* Restore the hardware register state */
 917		own_fpu(1);	/* Using the FPU again.	 */
 918	} else {
 919		sig = SIGFPE;
 920		fault_addr = (void __user *) regs->cp0_epc;
 921	}
 922
 923	/* Send a signal if required.  */
 924	process_fpemu_return(sig, fault_addr, fcr31);
 925
 926out:
 927	exception_exit(prev_state);
 928}
 929
 930/*
 931 * MIPS MT processors may have fewer FPU contexts than CPU threads. If we've
 932 * emulated more than some threshold number of instructions, force migration to
 933 * a "CPU" that has FP support.
 934 */
 935static void mt_ase_fp_affinity(void)
 936{
 937#ifdef CONFIG_MIPS_MT_FPAFF
 938	if (mt_fpemul_threshold > 0 &&
 939	     ((current->thread.emulated_fp++ > mt_fpemul_threshold))) {
 940		/*
 941		 * If there's no FPU present, or if the application has already
 942		 * restricted the allowed set to exclude any CPUs with FPUs,
 943		 * we'll skip the procedure.
 944		 */
 945		if (cpumask_intersects(&current->cpus_mask, &mt_fpu_cpumask)) {
 946			cpumask_t tmask;
 947
 948			current->thread.user_cpus_allowed
 949				= current->cpus_mask;
 950			cpumask_and(&tmask, &current->cpus_mask,
 951				    &mt_fpu_cpumask);
 952			set_cpus_allowed_ptr(current, &tmask);
 953			set_thread_flag(TIF_FPUBOUND);
 954		}
 955	}
 956#endif /* CONFIG_MIPS_MT_FPAFF */
 957}
 958
 959#else /* !CONFIG_MIPS_FP_SUPPORT */
 960
 961static int simulate_fp(struct pt_regs *regs, unsigned int opcode,
 962		       unsigned long old_epc, unsigned long old_ra)
 963{
 964	return -1;
 965}
 966
 967#endif /* !CONFIG_MIPS_FP_SUPPORT */
 968
 969void do_trap_or_bp(struct pt_regs *regs, unsigned int code, int si_code,
 970	const char *str)
 971{
 972	char b[40];
 973
 974#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
 975	if (kgdb_ll_trap(DIE_TRAP, str, regs, code, current->thread.trap_nr,
 976			 SIGTRAP) == NOTIFY_STOP)
 977		return;
 978#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
 979
 980	if (notify_die(DIE_TRAP, str, regs, code, current->thread.trap_nr,
 981		       SIGTRAP) == NOTIFY_STOP)
 982		return;
 983
 984	/*
 985	 * A short test says that IRIX 5.3 sends SIGTRAP for all trap
 986	 * insns, even for trap and break codes that indicate arithmetic
 987	 * failures.  Weird ...
 988	 * But should we continue the brokenness???  --macro
 989	 */
 990	switch (code) {
 991	case BRK_OVERFLOW:
 992	case BRK_DIVZERO:
 993		scnprintf(b, sizeof(b), "%s instruction in kernel code", str);
 994		die_if_kernel(b, regs);
 995		force_sig_fault(SIGFPE,
 996				code == BRK_DIVZERO ? FPE_INTDIV : FPE_INTOVF,
 997				(void __user *) regs->cp0_epc);
 998		break;
 999	case BRK_BUG:
1000		die_if_kernel("Kernel bug detected", regs);
1001		force_sig(SIGTRAP);
1002		break;
1003	case BRK_MEMU:
1004		/*
1005		 * This breakpoint code is used by the FPU emulator to retake
1006		 * control of the CPU after executing the instruction from the
1007		 * delay slot of an emulated branch.
1008		 *
1009		 * Terminate if exception was recognized as a delay slot return
1010		 * otherwise handle as normal.
1011		 */
1012		if (do_dsemulret(regs))
1013			return;
1014
1015		die_if_kernel("Math emu break/trap", regs);
1016		force_sig(SIGTRAP);
1017		break;
1018	default:
1019		scnprintf(b, sizeof(b), "%s instruction in kernel code", str);
1020		die_if_kernel(b, regs);
1021		if (si_code) {
1022			force_sig_fault(SIGTRAP, si_code, NULL);
1023		} else {
1024			force_sig(SIGTRAP);
1025		}
1026	}
1027}
1028
1029asmlinkage void do_bp(struct pt_regs *regs)
1030{
1031	unsigned long epc = msk_isa16_mode(exception_epc(regs));
1032	unsigned int opcode, bcode;
1033	enum ctx_state prev_state;
1034	bool user = user_mode(regs);
1035
1036	prev_state = exception_enter();
1037	current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
1038	if (get_isa16_mode(regs->cp0_epc)) {
1039		u16 instr[2];
1040
1041		if (__get_inst16(&instr[0], (u16 *)epc, user))
1042			goto out_sigsegv;
1043
1044		if (!cpu_has_mmips) {
1045			/* MIPS16e mode */
1046			bcode = (instr[0] >> 5) & 0x3f;
1047		} else if (mm_insn_16bit(instr[0])) {
1048			/* 16-bit microMIPS BREAK */
1049			bcode = instr[0] & 0xf;
1050		} else {
1051			/* 32-bit microMIPS BREAK */
1052			if (__get_inst16(&instr[1], (u16 *)(epc + 2), user))
1053				goto out_sigsegv;
1054			opcode = (instr[0] << 16) | instr[1];
1055			bcode = (opcode >> 6) & ((1 << 20) - 1);
1056		}
1057	} else {
1058		if (__get_inst32(&opcode, (u32 *)epc, user))
1059			goto out_sigsegv;
1060		bcode = (opcode >> 6) & ((1 << 20) - 1);
1061	}
1062
1063	/*
1064	 * There is the ancient bug in the MIPS assemblers that the break
1065	 * code starts left to bit 16 instead to bit 6 in the opcode.
1066	 * Gas is bug-compatible, but not always, grrr...
1067	 * We handle both cases with a simple heuristics.  --macro
1068	 */
1069	if (bcode >= (1 << 10))
1070		bcode = ((bcode & ((1 << 10) - 1)) << 10) | (bcode >> 10);
1071
1072	/*
1073	 * notify the kprobe handlers, if instruction is likely to
1074	 * pertain to them.
1075	 */
1076	switch (bcode) {
1077	case BRK_UPROBE:
1078		if (notify_die(DIE_UPROBE, "uprobe", regs, bcode,
1079			       current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
1080			goto out;
1081		else
1082			break;
1083	case BRK_UPROBE_XOL:
1084		if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, bcode,
1085			       current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
1086			goto out;
1087		else
1088			break;
1089	case BRK_KPROBE_BP:
1090		if (notify_die(DIE_BREAK, "debug", regs, bcode,
1091			       current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
1092			goto out;
1093		else
1094			break;
1095	case BRK_KPROBE_SSTEPBP:
1096		if (notify_die(DIE_SSTEPBP, "single_step", regs, bcode,
1097			       current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
1098			goto out;
1099		else
1100			break;
1101	default:
1102		break;
1103	}
1104
1105	do_trap_or_bp(regs, bcode, TRAP_BRKPT, "Break");
1106
1107out:
1108	exception_exit(prev_state);
1109	return;
1110
1111out_sigsegv:
1112	force_sig(SIGSEGV);
1113	goto out;
1114}
1115
1116asmlinkage void do_tr(struct pt_regs *regs)
1117{
1118	u32 opcode, tcode = 0;
1119	enum ctx_state prev_state;
1120	u16 instr[2];
1121	bool user = user_mode(regs);
1122	unsigned long epc = msk_isa16_mode(exception_epc(regs));
1123
1124	prev_state = exception_enter();
1125	current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
1126	if (get_isa16_mode(regs->cp0_epc)) {
1127		if (__get_inst16(&instr[0], (u16 *)(epc + 0), user) ||
1128		    __get_inst16(&instr[1], (u16 *)(epc + 2), user))
1129			goto out_sigsegv;
1130		opcode = (instr[0] << 16) | instr[1];
1131		/* Immediate versions don't provide a code.  */
1132		if (!(opcode & OPCODE))
1133			tcode = (opcode >> 12) & ((1 << 4) - 1);
1134	} else {
1135		if (__get_inst32(&opcode, (u32 *)epc, user))
1136			goto out_sigsegv;
1137		/* Immediate versions don't provide a code.  */
1138		if (!(opcode & OPCODE))
1139			tcode = (opcode >> 6) & ((1 << 10) - 1);
1140	}
1141
1142	do_trap_or_bp(regs, tcode, 0, "Trap");
1143
1144out:
1145	exception_exit(prev_state);
1146	return;
1147
1148out_sigsegv:
1149	force_sig(SIGSEGV);
1150	goto out;
1151}
1152
1153asmlinkage void do_ri(struct pt_regs *regs)
1154{
1155	unsigned int __user *epc = (unsigned int __user *)exception_epc(regs);
1156	unsigned long old_epc = regs->cp0_epc;
1157	unsigned long old31 = regs->regs[31];
1158	enum ctx_state prev_state;
1159	unsigned int opcode = 0;
1160	int status = -1;
1161
1162	/*
1163	 * Avoid any kernel code. Just emulate the R2 instruction
1164	 * as quickly as possible.
1165	 */
1166	if (mipsr2_emulation && cpu_has_mips_r6 &&
1167	    likely(user_mode(regs)) &&
1168	    likely(get_user(opcode, epc) >= 0)) {
1169		unsigned long fcr31 = 0;
1170
1171		status = mipsr2_decoder(regs, opcode, &fcr31);
1172		switch (status) {
1173		case 0:
1174		case SIGEMT:
1175			return;
1176		case SIGILL:
1177			goto no_r2_instr;
1178		default:
1179			process_fpemu_return(status,
1180					     &current->thread.cp0_baduaddr,
1181					     fcr31);
1182			return;
1183		}
1184	}
1185
1186no_r2_instr:
1187
1188	prev_state = exception_enter();
1189	current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
1190
1191	if (notify_die(DIE_RI, "RI Fault", regs, 0, current->thread.trap_nr,
1192		       SIGILL) == NOTIFY_STOP)
1193		goto out;
1194
1195	die_if_kernel("Reserved instruction in kernel code", regs);
1196
1197	if (unlikely(compute_return_epc(regs) < 0))
1198		goto out;
1199
1200	if (!get_isa16_mode(regs->cp0_epc)) {
1201		if (unlikely(get_user(opcode, epc) < 0))
1202			status = SIGSEGV;
1203
1204		if (!cpu_has_llsc && status < 0)
1205			status = simulate_llsc(regs, opcode);
1206
1207		if (status < 0)
1208			status = simulate_rdhwr_normal(regs, opcode);
1209
1210		if (status < 0)
1211			status = simulate_sync(regs, opcode);
1212
1213		if (status < 0)
1214			status = simulate_fp(regs, opcode, old_epc, old31);
1215
1216#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION
1217		if (status < 0)
1218			status = simulate_loongson3_cpucfg(regs, opcode);
1219#endif
1220	} else if (cpu_has_mmips) {
1221		unsigned short mmop[2] = { 0 };
1222
1223		if (unlikely(get_user(mmop[0], (u16 __user *)epc + 0) < 0))
1224			status = SIGSEGV;
1225		if (unlikely(get_user(mmop[1], (u16 __user *)epc + 1) < 0))
1226			status = SIGSEGV;
1227		opcode = mmop[0];
1228		opcode = (opcode << 16) | mmop[1];
1229
1230		if (status < 0)
1231			status = simulate_rdhwr_mm(regs, opcode);
1232	}
1233
1234	if (status < 0)
1235		status = SIGILL;
1236
1237	if (unlikely(status > 0)) {
1238		regs->cp0_epc = old_epc;		/* Undo skip-over.  */
1239		regs->regs[31] = old31;
1240		force_sig(status);
1241	}
1242
1243out:
1244	exception_exit(prev_state);
1245}
1246
1247/*
1248 * No lock; only written during early bootup by CPU 0.
1249 */
1250static RAW_NOTIFIER_HEAD(cu2_chain);
1251
1252int __ref register_cu2_notifier(struct notifier_block *nb)
1253{
1254	return raw_notifier_chain_register(&cu2_chain, nb);
1255}
1256
1257int cu2_notifier_call_chain(unsigned long val, void *v)
1258{
1259	return raw_notifier_call_chain(&cu2_chain, val, v);
1260}
1261
1262static int default_cu2_call(struct notifier_block *nfb, unsigned long action,
1263	void *data)
1264{
1265	struct pt_regs *regs = data;
1266
1267	die_if_kernel("COP2: Unhandled kernel unaligned access or invalid "
1268			      "instruction", regs);
1269	force_sig(SIGILL);
1270
1271	return NOTIFY_OK;
1272}
1273
1274#ifdef CONFIG_MIPS_FP_SUPPORT
1275
1276static int enable_restore_fp_context(int msa)
1277{
1278	int err, was_fpu_owner, prior_msa;
1279	bool first_fp;
1280
1281	/* Initialize context if it hasn't been used already */
1282	first_fp = init_fp_ctx(current);
1283
1284	if (first_fp) {
1285		preempt_disable();
1286		err = own_fpu_inatomic(1);
1287		if (msa && !err) {
1288			enable_msa();
1289			/*
1290			 * with MSA enabled, userspace can see MSACSR
1291			 * and MSA regs, but the values in them are from
1292			 * other task before current task, restore them
1293			 * from saved fp/msa context
1294			 */
1295			write_msa_csr(current->thread.fpu.msacsr);
1296			/*
1297			 * own_fpu_inatomic(1) just restore low 64bit,
1298			 * fix the high 64bit
1299			 */
1300			init_msa_upper();
1301			set_thread_flag(TIF_USEDMSA);
1302			set_thread_flag(TIF_MSA_CTX_LIVE);
1303		}
1304		preempt_enable();
1305		return err;
1306	}
1307
1308	/*
1309	 * This task has formerly used the FP context.
1310	 *
1311	 * If this thread has no live MSA vector context then we can simply
1312	 * restore the scalar FP context. If it has live MSA vector context
1313	 * (that is, it has or may have used MSA since last performing a
1314	 * function call) then we'll need to restore the vector context. This
1315	 * applies even if we're currently only executing a scalar FP
1316	 * instruction. This is because if we were to later execute an MSA
1317	 * instruction then we'd either have to:
1318	 *
1319	 *  - Restore the vector context & clobber any registers modified by
1320	 *    scalar FP instructions between now & then.
1321	 *
1322	 * or
1323	 *
1324	 *  - Not restore the vector context & lose the most significant bits
1325	 *    of all vector registers.
1326	 *
1327	 * Neither of those options is acceptable. We cannot restore the least
1328	 * significant bits of the registers now & only restore the most
1329	 * significant bits later because the most significant bits of any
1330	 * vector registers whose aliased FP register is modified now will have
1331	 * been zeroed. We'd have no way to know that when restoring the vector
1332	 * context & thus may load an outdated value for the most significant
1333	 * bits of a vector register.
1334	 */
1335	if (!msa && !thread_msa_context_live())
1336		return own_fpu(1);
1337
1338	/*
1339	 * This task is using or has previously used MSA. Thus we require
1340	 * that Status.FR == 1.
1341	 */
1342	preempt_disable();
1343	was_fpu_owner = is_fpu_owner();
1344	err = own_fpu_inatomic(0);
1345	if (err)
1346		goto out;
1347
1348	enable_msa();
1349	write_msa_csr(current->thread.fpu.msacsr);
1350	set_thread_flag(TIF_USEDMSA);
1351
1352	/*
1353	 * If this is the first time that the task is using MSA and it has
1354	 * previously used scalar FP in this time slice then we already nave
1355	 * FP context which we shouldn't clobber. We do however need to clear
1356	 * the upper 64b of each vector register so that this task has no
1357	 * opportunity to see data left behind by another.
1358	 */
1359	prior_msa = test_and_set_thread_flag(TIF_MSA_CTX_LIVE);
1360	if (!prior_msa && was_fpu_owner) {
1361		init_msa_upper();
1362
1363		goto out;
1364	}
1365
1366	if (!prior_msa) {
1367		/*
1368		 * Restore the least significant 64b of each vector register
1369		 * from the existing scalar FP context.
1370		 */
1371		_restore_fp(current);
1372
1373		/*
1374		 * The task has not formerly used MSA, so clear the upper 64b
1375		 * of each vector register such that it cannot see data left
1376		 * behind by another task.
1377		 */
1378		init_msa_upper();
1379	} else {
1380		/* We need to restore the vector context. */
1381		restore_msa(current);
1382
1383		/* Restore the scalar FP control & status register */
1384		if (!was_fpu_owner)
1385			write_32bit_cp1_register(CP1_STATUS,
1386						 current->thread.fpu.fcr31);
1387	}
1388
1389out:
1390	preempt_enable();
1391
1392	return 0;
1393}
1394
1395#else /* !CONFIG_MIPS_FP_SUPPORT */
1396
1397static int enable_restore_fp_context(int msa)
1398{
1399	return SIGILL;
1400}
1401
1402#endif /* CONFIG_MIPS_FP_SUPPORT */
1403
1404asmlinkage void do_cpu(struct pt_regs *regs)
1405{
1406	enum ctx_state prev_state;
1407	unsigned int __user *epc;
1408	unsigned long old_epc, old31;
1409	unsigned int opcode;
1410	unsigned int cpid;
1411	int status;
1412
1413	prev_state = exception_enter();
1414	cpid = (regs->cp0_cause >> CAUSEB_CE) & 3;
1415
1416	if (cpid != 2)
1417		die_if_kernel("do_cpu invoked from kernel context!", regs);
1418
1419	switch (cpid) {
1420	case 0:
1421		epc = (unsigned int __user *)exception_epc(regs);
1422		old_epc = regs->cp0_epc;
1423		old31 = regs->regs[31];
1424		opcode = 0;
1425		status = -1;
1426
1427		if (unlikely(compute_return_epc(regs) < 0))
1428			break;
1429
1430		if (!get_isa16_mode(regs->cp0_epc)) {
1431			if (unlikely(get_user(opcode, epc) < 0))
1432				status = SIGSEGV;
1433
1434			if (!cpu_has_llsc && status < 0)
1435				status = simulate_llsc(regs, opcode);
1436		}
1437
1438		if (status < 0)
1439			status = SIGILL;
1440
1441		if (unlikely(status > 0)) {
1442			regs->cp0_epc = old_epc;	/* Undo skip-over.  */
1443			regs->regs[31] = old31;
1444			force_sig(status);
1445		}
1446
1447		break;
1448
1449#ifdef CONFIG_MIPS_FP_SUPPORT
1450	case 3:
1451		/*
1452		 * The COP3 opcode space and consequently the CP0.Status.CU3
1453		 * bit and the CP0.Cause.CE=3 encoding have been removed as
1454		 * of the MIPS III ISA.  From the MIPS IV and MIPS32r2 ISAs
1455		 * up the space has been reused for COP1X instructions, that
1456		 * are enabled by the CP0.Status.CU1 bit and consequently
1457		 * use the CP0.Cause.CE=1 encoding for Coprocessor Unusable
1458		 * exceptions.  Some FPU-less processors that implement one
1459		 * of these ISAs however use this code erroneously for COP1X
1460		 * instructions.  Therefore we redirect this trap to the FP
1461		 * emulator too.
1462		 */
1463		if (raw_cpu_has_fpu || !cpu_has_mips_4_5_64_r2_r6) {
1464			force_sig(SIGILL);
1465			break;
1466		}
1467		fallthrough;
1468	case 1: {
1469		void __user *fault_addr;
1470		unsigned long fcr31;
1471		int err, sig;
1472
1473		err = enable_restore_fp_context(0);
1474
1475		if (raw_cpu_has_fpu && !err)
1476			break;
1477
1478		sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 0,
1479					       &fault_addr);
1480
1481		/*
1482		 * We can't allow the emulated instruction to leave
1483		 * any enabled Cause bits set in $fcr31.
1484		 */
1485		fcr31 = mask_fcr31_x(current->thread.fpu.fcr31);
1486		current->thread.fpu.fcr31 &= ~fcr31;
1487
1488		/* Send a signal if required.  */
1489		if (!process_fpemu_return(sig, fault_addr, fcr31) && !err)
1490			mt_ase_fp_affinity();
1491
1492		break;
1493	}
1494#else /* CONFIG_MIPS_FP_SUPPORT */
1495	case 1:
1496	case 3:
1497		force_sig(SIGILL);
1498		break;
1499#endif /* CONFIG_MIPS_FP_SUPPORT */
1500
1501	case 2:
1502		raw_notifier_call_chain(&cu2_chain, CU2_EXCEPTION, regs);
1503		break;
1504	}
1505
1506	exception_exit(prev_state);
1507}
1508
1509asmlinkage void do_msa_fpe(struct pt_regs *regs, unsigned int msacsr)
1510{
1511	enum ctx_state prev_state;
1512
1513	prev_state = exception_enter();
1514	current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
1515	if (notify_die(DIE_MSAFP, "MSA FP exception", regs, 0,
1516		       current->thread.trap_nr, SIGFPE) == NOTIFY_STOP)
1517		goto out;
1518
1519	/* Clear MSACSR.Cause before enabling interrupts */
1520	write_msa_csr(msacsr & ~MSA_CSR_CAUSEF);
1521	local_irq_enable();
1522
1523	die_if_kernel("do_msa_fpe invoked from kernel context!", regs);
1524	force_sig(SIGFPE);
1525out:
1526	exception_exit(prev_state);
1527}
1528
1529asmlinkage void do_msa(struct pt_regs *regs)
1530{
1531	enum ctx_state prev_state;
1532	int err;
1533
1534	prev_state = exception_enter();
1535
1536	if (!cpu_has_msa || test_thread_flag(TIF_32BIT_FPREGS)) {
1537		force_sig(SIGILL);
1538		goto out;
1539	}
1540
1541	die_if_kernel("do_msa invoked from kernel context!", regs);
1542
1543	err = enable_restore_fp_context(1);
1544	if (err)
1545		force_sig(SIGILL);
1546out:
1547	exception_exit(prev_state);
1548}
1549
1550asmlinkage void do_mdmx(struct pt_regs *regs)
1551{
1552	enum ctx_state prev_state;
1553
1554	prev_state = exception_enter();
1555	force_sig(SIGILL);
1556	exception_exit(prev_state);
1557}
1558
1559/*
1560 * Called with interrupts disabled.
1561 */
1562asmlinkage void do_watch(struct pt_regs *regs)
1563{
1564	enum ctx_state prev_state;
1565
1566	prev_state = exception_enter();
1567	/*
1568	 * Clear WP (bit 22) bit of cause register so we don't loop
1569	 * forever.
1570	 */
1571	clear_c0_cause(CAUSEF_WP);
1572
1573	/*
1574	 * If the current thread has the watch registers loaded, save
1575	 * their values and send SIGTRAP.  Otherwise another thread
1576	 * left the registers set, clear them and continue.
1577	 */
1578	if (test_tsk_thread_flag(current, TIF_LOAD_WATCH)) {
1579		mips_read_watch_registers();
1580		local_irq_enable();
1581		force_sig_fault(SIGTRAP, TRAP_HWBKPT, NULL);
1582	} else {
1583		mips_clear_watch_registers();
1584		local_irq_enable();
1585	}
1586	exception_exit(prev_state);
1587}
1588
1589asmlinkage void do_mcheck(struct pt_regs *regs)
1590{
1591	int multi_match = regs->cp0_status & ST0_TS;
1592	enum ctx_state prev_state;
1593
1594	prev_state = exception_enter();
1595	show_regs(regs);
1596
1597	if (multi_match) {
1598		dump_tlb_regs();
1599		pr_info("\n");
1600		dump_tlb_all();
1601	}
1602
1603	show_code((void *)regs->cp0_epc, user_mode(regs));
1604
1605	/*
1606	 * Some chips may have other causes of machine check (e.g. SB1
1607	 * graduation timer)
1608	 */
1609	panic("Caught Machine Check exception - %scaused by multiple "
1610	      "matching entries in the TLB.",
1611	      (multi_match) ? "" : "not ");
1612}
1613
1614asmlinkage void do_mt(struct pt_regs *regs)
1615{
1616	int subcode;
1617
1618	subcode = (read_vpe_c0_vpecontrol() & VPECONTROL_EXCPT)
1619			>> VPECONTROL_EXCPT_SHIFT;
1620	switch (subcode) {
1621	case 0:
1622		printk(KERN_DEBUG "Thread Underflow\n");
1623		break;
1624	case 1:
1625		printk(KERN_DEBUG "Thread Overflow\n");
1626		break;
1627	case 2:
1628		printk(KERN_DEBUG "Invalid YIELD Qualifier\n");
1629		break;
1630	case 3:
1631		printk(KERN_DEBUG "Gating Storage Exception\n");
1632		break;
1633	case 4:
1634		printk(KERN_DEBUG "YIELD Scheduler Exception\n");
1635		break;
1636	case 5:
1637		printk(KERN_DEBUG "Gating Storage Scheduler Exception\n");
1638		break;
1639	default:
1640		printk(KERN_DEBUG "*** UNKNOWN THREAD EXCEPTION %d ***\n",
1641			subcode);
1642		break;
1643	}
1644	die_if_kernel("MIPS MT Thread exception in kernel", regs);
1645
1646	force_sig(SIGILL);
1647}
1648
1649
1650asmlinkage void do_dsp(struct pt_regs *regs)
1651{
1652	if (cpu_has_dsp)
1653		panic("Unexpected DSP exception");
1654
1655	force_sig(SIGILL);
1656}
1657
1658asmlinkage void do_reserved(struct pt_regs *regs)
1659{
1660	/*
1661	 * Game over - no way to handle this if it ever occurs.	 Most probably
1662	 * caused by a new unknown cpu type or after another deadly
1663	 * hard/software error.
1664	 */
1665	show_regs(regs);
1666	panic("Caught reserved exception %ld - should not happen.",
1667	      (regs->cp0_cause & 0x7f) >> 2);
1668}
1669
1670static int __initdata l1parity = 1;
1671static int __init nol1parity(char *s)
1672{
1673	l1parity = 0;
1674	return 1;
1675}
1676__setup("nol1par", nol1parity);
1677static int __initdata l2parity = 1;
1678static int __init nol2parity(char *s)
1679{
1680	l2parity = 0;
1681	return 1;
1682}
1683__setup("nol2par", nol2parity);
1684
1685/*
1686 * Some MIPS CPUs can enable/disable for cache parity detection, but do
1687 * it different ways.
1688 */
1689static inline __init void parity_protection_init(void)
1690{
1691#define ERRCTL_PE	0x80000000
1692#define ERRCTL_L2P	0x00800000
1693
1694	if (mips_cm_revision() >= CM_REV_CM3) {
1695		ulong gcr_ectl, cp0_ectl;
1696
1697		/*
1698		 * With CM3 systems we need to ensure that the L1 & L2
1699		 * parity enables are set to the same value, since this
1700		 * is presumed by the hardware engineers.
1701		 *
1702		 * If the user disabled either of L1 or L2 ECC checking,
1703		 * disable both.
1704		 */
1705		l1parity &= l2parity;
1706		l2parity &= l1parity;
1707
1708		/* Probe L1 ECC support */
1709		cp0_ectl = read_c0_errctl();
1710		write_c0_errctl(cp0_ectl | ERRCTL_PE);
1711		back_to_back_c0_hazard();
1712		cp0_ectl = read_c0_errctl();
1713
1714		/* Probe L2 ECC support */
1715		gcr_ectl = read_gcr_err_control();
1716
1717		if (!(gcr_ectl & CM_GCR_ERR_CONTROL_L2_ECC_SUPPORT) ||
1718		    !(cp0_ectl & ERRCTL_PE)) {
1719			/*
1720			 * One of L1 or L2 ECC checking isn't supported,
1721			 * so we cannot enable either.
1722			 */
1723			l1parity = l2parity = 0;
1724		}
1725
1726		/* Configure L1 ECC checking */
1727		if (l1parity)
1728			cp0_ectl |= ERRCTL_PE;
1729		else
1730			cp0_ectl &= ~ERRCTL_PE;
1731		write_c0_errctl(cp0_ectl);
1732		back_to_back_c0_hazard();
1733		WARN_ON(!!(read_c0_errctl() & ERRCTL_PE) != l1parity);
1734
1735		/* Configure L2 ECC checking */
1736		if (l2parity)
1737			gcr_ectl |= CM_GCR_ERR_CONTROL_L2_ECC_EN;
1738		else
1739			gcr_ectl &= ~CM_GCR_ERR_CONTROL_L2_ECC_EN;
1740		write_gcr_err_control(gcr_ectl);
1741		gcr_ectl = read_gcr_err_control();
1742		gcr_ectl &= CM_GCR_ERR_CONTROL_L2_ECC_EN;
1743		WARN_ON(!!gcr_ectl != l2parity);
1744
1745		pr_info("Cache parity protection %s\n",
1746			str_enabled_disabled(l1parity));
1747		return;
1748	}
1749
1750	switch (current_cpu_type()) {
1751	case CPU_24K:
1752	case CPU_34K:
1753	case CPU_74K:
1754	case CPU_1004K:
1755	case CPU_1074K:
1756	case CPU_INTERAPTIV:
1757	case CPU_PROAPTIV:
1758	case CPU_P5600:
1759	case CPU_QEMU_GENERIC:
1760	case CPU_P6600:
1761		{
1762			unsigned long errctl;
1763			unsigned int l1parity_present, l2parity_present;
1764
1765			errctl = read_c0_errctl();
1766			errctl &= ~(ERRCTL_PE|ERRCTL_L2P);
1767
1768			/* probe L1 parity support */
1769			write_c0_errctl(errctl | ERRCTL_PE);
1770			back_to_back_c0_hazard();
1771			l1parity_present = (read_c0_errctl() & ERRCTL_PE);
1772
1773			/* probe L2 parity support */
1774			write_c0_errctl(errctl|ERRCTL_L2P);
1775			back_to_back_c0_hazard();
1776			l2parity_present = (read_c0_errctl() & ERRCTL_L2P);
1777
1778			if (l1parity_present && l2parity_present) {
1779				if (l1parity)
1780					errctl |= ERRCTL_PE;
1781				if (l1parity ^ l2parity)
1782					errctl |= ERRCTL_L2P;
1783			} else if (l1parity_present) {
1784				if (l1parity)
1785					errctl |= ERRCTL_PE;
1786			} else if (l2parity_present) {
1787				if (l2parity)
1788					errctl |= ERRCTL_L2P;
1789			} else {
1790				/* No parity available */
1791			}
1792
1793			printk(KERN_INFO "Writing ErrCtl register=%08lx\n", errctl);
1794
1795			write_c0_errctl(errctl);
1796			back_to_back_c0_hazard();
1797			errctl = read_c0_errctl();
1798			printk(KERN_INFO "Readback ErrCtl register=%08lx\n", errctl);
1799
1800			if (l1parity_present)
1801				pr_info("Cache parity protection %s\n",
1802					str_enabled_disabled(errctl & ERRCTL_PE));
1803
1804			if (l2parity_present) {
1805				if (l1parity_present && l1parity)
1806					errctl ^= ERRCTL_L2P;
1807				pr_info("L2 cache parity protection %s\n",
1808					str_enabled_disabled(errctl & ERRCTL_L2P));
1809			}
1810		}
1811		break;
1812
1813	case CPU_5KC:
1814	case CPU_5KE:
1815	case CPU_LOONGSON32:
1816		write_c0_errctl(0x80000000);
1817		back_to_back_c0_hazard();
1818		/* Set the PE bit (bit 31) in the c0_errctl register. */
1819		pr_info("Cache parity protection %s\n",
1820			str_enabled_disabled(read_c0_errctl() & 0x80000000));
1821		break;
1822	case CPU_20KC:
1823	case CPU_25KF:
1824		/* Clear the DE bit (bit 16) in the c0_status register. */
1825		printk(KERN_INFO "Enable cache parity protection for "
1826		       "MIPS 20KC/25KF CPUs.\n");
1827		clear_c0_status(ST0_DE);
1828		break;
1829	default:
1830		break;
1831	}
1832}
1833
1834asmlinkage void cache_parity_error(void)
1835{
1836	const int field = 2 * sizeof(unsigned long);
1837	unsigned int reg_val;
1838
1839	/* For the moment, report the problem and hang. */
1840	printk("Cache error exception:\n");
1841	printk("cp0_errorepc == %0*lx\n", field, read_c0_errorepc());
1842	reg_val = read_c0_cacheerr();
1843	printk("c0_cacheerr == %08x\n", reg_val);
1844
1845	printk("Decoded c0_cacheerr: %s cache fault in %s reference.\n",
1846	       reg_val & (1<<30) ? "secondary" : "primary",
1847	       reg_val & (1<<31) ? "data" : "insn");
1848	if ((cpu_has_mips_r2_r6) &&
1849	    ((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_MIPS)) {
1850		pr_err("Error bits: %s%s%s%s%s%s%s%s\n",
1851			reg_val & (1<<29) ? "ED " : "",
1852			reg_val & (1<<28) ? "ET " : "",
1853			reg_val & (1<<27) ? "ES " : "",
1854			reg_val & (1<<26) ? "EE " : "",
1855			reg_val & (1<<25) ? "EB " : "",
1856			reg_val & (1<<24) ? "EI " : "",
1857			reg_val & (1<<23) ? "E1 " : "",
1858			reg_val & (1<<22) ? "E0 " : "");
1859	} else {
1860		pr_err("Error bits: %s%s%s%s%s%s%s\n",
1861			reg_val & (1<<29) ? "ED " : "",
1862			reg_val & (1<<28) ? "ET " : "",
1863			reg_val & (1<<26) ? "EE " : "",
1864			reg_val & (1<<25) ? "EB " : "",
1865			reg_val & (1<<24) ? "EI " : "",
1866			reg_val & (1<<23) ? "E1 " : "",
1867			reg_val & (1<<22) ? "E0 " : "");
1868	}
1869	printk("IDX: 0x%08x\n", reg_val & ((1<<22)-1));
1870
1871#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64)
1872	if (reg_val & (1<<22))
1873		printk("DErrAddr0: 0x%0*lx\n", field, read_c0_derraddr0());
1874
1875	if (reg_val & (1<<23))
1876		printk("DErrAddr1: 0x%0*lx\n", field, read_c0_derraddr1());
1877#endif
1878
1879	panic("Can't handle the cache error!");
1880}
1881
1882asmlinkage void do_ftlb(void)
1883{
1884	const int field = 2 * sizeof(unsigned long);
1885	unsigned int reg_val;
1886
1887	/* For the moment, report the problem and hang. */
1888	if ((cpu_has_mips_r2_r6) &&
1889	    (((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_MIPS) ||
1890	    ((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_LOONGSON))) {
1891		pr_err("FTLB error exception, cp0_errctl=0x%08x:\n",
1892		       read_c0_errctl());
1893		pr_err("cp0_errorepc == %0*lx\n", field, read_c0_errorepc());
1894		reg_val = read_c0_cacheerr();
1895		pr_err("c0_cacheerr == %08x\n", reg_val);
1896
1897		if ((reg_val & 0xc0000000) == 0xc0000000) {
1898			pr_err("Decoded c0_cacheerr: FTLB parity error\n");
1899		} else {
1900			pr_err("Decoded c0_cacheerr: %s cache fault in %s reference.\n",
1901			       reg_val & (1<<30) ? "secondary" : "primary",
1902			       reg_val & (1<<31) ? "data" : "insn");
1903		}
1904	} else {
1905		pr_err("FTLB error exception\n");
1906	}
1907	/* Just print the cacheerr bits for now */
1908	cache_parity_error();
1909}
1910
1911asmlinkage void do_gsexc(struct pt_regs *regs, u32 diag1)
1912{
1913	u32 exccode = (diag1 & LOONGSON_DIAG1_EXCCODE) >>
1914			LOONGSON_DIAG1_EXCCODE_SHIFT;
1915	enum ctx_state prev_state;
1916
1917	prev_state = exception_enter();
1918
1919	switch (exccode) {
1920	case 0x08:
1921		/* Undocumented exception, will trigger on certain
1922		 * also-undocumented instructions accessible from userspace.
1923		 * Processor state is not otherwise corrupted, but currently
1924		 * we don't know how to proceed. Maybe there is some
1925		 * undocumented control flag to enable the instructions?
1926		 */
1927		force_sig(SIGILL);
1928		break;
1929
1930	default:
1931		/* None of the other exceptions, documented or not, have
1932		 * further details given; none are encountered in the wild
1933		 * either. Panic in case some of them turn out to be fatal.
1934		 */
1935		show_regs(regs);
1936		panic("Unhandled Loongson exception - GSCause = %08x", diag1);
1937	}
1938
1939	exception_exit(prev_state);
1940}
1941
1942/*
1943 * SDBBP EJTAG debug exception handler.
1944 * We skip the instruction and return to the next instruction.
1945 */
1946void ejtag_exception_handler(struct pt_regs *regs)
1947{
1948	const int field = 2 * sizeof(unsigned long);
1949	unsigned long depc, old_epc, old_ra;
1950	unsigned int debug;
1951
1952	printk(KERN_DEBUG "SDBBP EJTAG debug exception - not handled yet, just ignored!\n");
1953	depc = read_c0_depc();
1954	debug = read_c0_debug();
1955	printk(KERN_DEBUG "c0_depc = %0*lx, DEBUG = %08x\n", field, depc, debug);
1956	if (debug & 0x80000000) {
1957		/*
1958		 * In branch delay slot.
1959		 * We cheat a little bit here and use EPC to calculate the
1960		 * debug return address (DEPC). EPC is restored after the
1961		 * calculation.
1962		 */
1963		old_epc = regs->cp0_epc;
1964		old_ra = regs->regs[31];
1965		regs->cp0_epc = depc;
1966		compute_return_epc(regs);
1967		depc = regs->cp0_epc;
1968		regs->cp0_epc = old_epc;
1969		regs->regs[31] = old_ra;
1970	} else
1971		depc += 4;
1972	write_c0_depc(depc);
1973
1974#if 0
1975	printk(KERN_DEBUG "\n\n----- Enable EJTAG single stepping ----\n\n");
1976	write_c0_debug(debug | 0x100);
1977#endif
1978}
1979
1980/*
1981 * NMI exception handler.
1982 * No lock; only written during early bootup by CPU 0.
1983 */
1984static RAW_NOTIFIER_HEAD(nmi_chain);
1985
1986int register_nmi_notifier(struct notifier_block *nb)
1987{
1988	return raw_notifier_chain_register(&nmi_chain, nb);
1989}
1990
1991void __noreturn nmi_exception_handler(struct pt_regs *regs)
1992{
1993	char str[100];
1994
1995	nmi_enter();
1996	raw_notifier_call_chain(&nmi_chain, 0, regs);
1997	bust_spinlocks(1);
1998	snprintf(str, 100, "CPU%d NMI taken, CP0_EPC=%lx\n",
1999		 smp_processor_id(), regs->cp0_epc);
2000	regs->cp0_epc = read_c0_errorepc();
2001	die(str, regs);
2002	nmi_exit();
2003}
2004
2005unsigned long ebase;
2006EXPORT_SYMBOL_GPL(ebase);
2007unsigned long exception_handlers[32];
2008unsigned long vi_handlers[64];
2009
2010void reserve_exception_space(phys_addr_t addr, unsigned long size)
2011{
2012	/*
2013	 * reserve exception space on CPUs other than CPU0
2014	 * is too late, since memblock is unavailable when APs
2015	 * up
2016	 */
2017	if (smp_processor_id() == 0)
2018		memblock_reserve(addr, size);
2019}
2020
2021void __init *set_except_vector(int n, void *addr)
2022{
2023	unsigned long handler = (unsigned long) addr;
2024	unsigned long old_handler;
2025
2026#ifdef CONFIG_CPU_MICROMIPS
2027	/*
2028	 * Only the TLB handlers are cache aligned with an even
2029	 * address. All other handlers are on an odd address and
2030	 * require no modification. Otherwise, MIPS32 mode will
2031	 * be entered when handling any TLB exceptions. That
2032	 * would be bad...since we must stay in microMIPS mode.
2033	 */
2034	if (!(handler & 0x1))
2035		handler |= 1;
2036#endif
2037	old_handler = xchg(&exception_handlers[n], handler);
2038
2039	if (n == 0 && cpu_has_divec) {
2040#ifdef CONFIG_CPU_MICROMIPS
2041		unsigned long jump_mask = ~((1 << 27) - 1);
2042#else
2043		unsigned long jump_mask = ~((1 << 28) - 1);
2044#endif
2045		u32 *buf = (u32 *)(ebase + 0x200);
2046		if ((handler & jump_mask) == ((ebase + 0x200) & jump_mask)) {
2047			uasm_i_j(&buf, handler & ~jump_mask);
2048			uasm_i_nop(&buf);
2049		} else {
2050			UASM_i_LA(&buf, GPR_K0, handler);
2051			uasm_i_jr(&buf, GPR_K0);
2052			uasm_i_nop(&buf);
2053		}
2054		local_flush_icache_range(ebase + 0x200, (unsigned long)buf);
2055	}
2056	return (void *)old_handler;
2057}
2058
2059static void do_default_vi(void)
2060{
2061	show_regs(get_irq_regs());
2062	panic("Caught unexpected vectored interrupt.");
2063}
2064
2065void *set_vi_handler(int n, vi_handler_t addr)
2066{
2067	extern const u8 except_vec_vi[];
2068	extern const u8 except_vec_vi_ori[], except_vec_vi_end[];
2069	extern const u8 skipover_except_vec_vi[];
2070	unsigned long handler;
2071	unsigned long old_handler = vi_handlers[n];
2072	int srssets = current_cpu_data.srsets;
2073	u16 *h;
2074	unsigned char *b;
2075	const u8 *vec_start;
2076	int ori_offset;
2077	int handler_len;
2078
2079	BUG_ON(!cpu_has_veic && !cpu_has_vint);
2080
2081	if (addr == NULL) {
2082		handler = (unsigned long) do_default_vi;
2083	} else
2084		handler = (unsigned long) addr;
2085	vi_handlers[n] = handler;
2086
2087	b = (unsigned char *)(ebase + 0x200 + n*VECTORSPACING);
2088
2089	if (cpu_has_veic) {
2090		if (board_bind_eic_interrupt)
2091			board_bind_eic_interrupt(n, 0);
2092	} else if (cpu_has_vint) {
2093		/* SRSMap is only defined if shadow sets are implemented */
2094		if (srssets > 1)
2095			change_c0_srsmap(0xf << n*4, 0 << n*4);
2096	}
2097
2098	vec_start = using_skipover_handler() ? skipover_except_vec_vi :
2099					       except_vec_vi;
2100#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN)
2101	ori_offset = except_vec_vi_ori - vec_start + 2;
2102#else
2103	ori_offset = except_vec_vi_ori - vec_start;
2104#endif
2105	handler_len = except_vec_vi_end - vec_start;
2106
2107	if (handler_len > VECTORSPACING) {
2108		/*
2109		 * Sigh... panicing won't help as the console
2110		 * is probably not configured :(
2111		 */
2112		panic("VECTORSPACING too small");
2113	}
2114
2115	set_handler(((unsigned long)b - ebase), vec_start,
2116#ifdef CONFIG_CPU_MICROMIPS
2117			(handler_len - 1));
2118#else
2119			handler_len);
2120#endif
2121	/* insert offset into vi_handlers[] */
2122	h = (u16 *)(b + ori_offset);
2123	*h = n * sizeof(handler);
2124	local_flush_icache_range((unsigned long)b,
2125				 (unsigned long)(b+handler_len));
2126
2127	return (void *)old_handler;
2128}
2129
2130/*
2131 * Timer interrupt
2132 */
2133int cp0_compare_irq;
2134EXPORT_SYMBOL_GPL(cp0_compare_irq);
2135int cp0_compare_irq_shift;
2136
2137/*
2138 * Performance counter IRQ or -1 if shared with timer
2139 */
2140int cp0_perfcount_irq;
2141EXPORT_SYMBOL_GPL(cp0_perfcount_irq);
2142
2143/*
2144 * Fast debug channel IRQ or -1 if not present
2145 */
2146int cp0_fdc_irq;
2147EXPORT_SYMBOL_GPL(cp0_fdc_irq);
2148
2149static int noulri;
2150
2151static int __init ulri_disable(char *s)
2152{
2153	pr_info("Disabling ulri\n");
2154	noulri = 1;
2155
2156	return 1;
2157}
2158__setup("noulri", ulri_disable);
2159
2160/* configure STATUS register */
2161static void configure_status(void)
2162{
2163	/*
2164	 * Disable coprocessors and select 32-bit or 64-bit addressing
2165	 * and the 16/32 or 32/32 FPR register model.  Reset the BEV
2166	 * flag that some firmware may have left set and the TS bit (for
2167	 * IP27).  Set XX for ISA IV code to work.
2168	 */
2169	unsigned int status_set = ST0_KERNEL_CUMASK;
2170#ifdef CONFIG_64BIT
2171	status_set |= ST0_FR|ST0_KX|ST0_SX|ST0_UX;
2172#endif
2173	if (current_cpu_data.isa_level & MIPS_CPU_ISA_IV)
2174		status_set |= ST0_XX;
2175	if (cpu_has_dsp)
2176		status_set |= ST0_MX;
2177
2178	change_c0_status(ST0_CU|ST0_MX|ST0_RE|ST0_FR|ST0_BEV|ST0_TS|ST0_KX|ST0_SX|ST0_UX,
2179			 status_set);
2180	back_to_back_c0_hazard();
2181}
2182
2183unsigned int hwrena;
2184EXPORT_SYMBOL_GPL(hwrena);
2185
2186/* configure HWRENA register */
2187static void configure_hwrena(void)
2188{
2189	hwrena = cpu_hwrena_impl_bits;
2190
2191	if (cpu_has_mips_r2_r6)
2192		hwrena |= MIPS_HWRENA_CPUNUM |
2193			  MIPS_HWRENA_SYNCISTEP |
2194			  MIPS_HWRENA_CC |
2195			  MIPS_HWRENA_CCRES;
2196
2197	if (!noulri && cpu_has_userlocal)
2198		hwrena |= MIPS_HWRENA_ULR;
2199
2200	if (hwrena)
2201		write_c0_hwrena(hwrena);
2202}
2203
2204static void configure_exception_vector(void)
2205{
2206	if (cpu_has_mips_r2_r6) {
2207		unsigned long sr = set_c0_status(ST0_BEV);
2208		/* If available, use WG to set top bits of EBASE */
2209		if (cpu_has_ebase_wg) {
2210#ifdef CONFIG_64BIT
2211			write_c0_ebase_64(ebase | MIPS_EBASE_WG);
2212#else
2213			write_c0_ebase(ebase | MIPS_EBASE_WG);
2214#endif
2215		}
2216		write_c0_ebase(ebase);
2217		write_c0_status(sr);
2218	}
2219	if (cpu_has_veic || cpu_has_vint) {
2220		/* Setting vector spacing enables EI/VI mode  */
2221		change_c0_intctl(0x3e0, VECTORSPACING);
2222	}
2223	if (cpu_has_divec) {
2224		if (cpu_has_mipsmt) {
2225			unsigned int vpflags = dvpe();
2226			set_c0_cause(CAUSEF_IV);
2227			evpe(vpflags);
2228		} else
2229			set_c0_cause(CAUSEF_IV);
2230	}
2231}
2232
2233void per_cpu_trap_init(bool is_boot_cpu)
2234{
2235	unsigned int cpu = smp_processor_id();
2236
2237	configure_status();
2238	configure_hwrena();
2239
2240	configure_exception_vector();
2241
2242	/*
2243	 * Before R2 both interrupt numbers were fixed to 7, so on R2 only:
2244	 *
2245	 *  o read IntCtl.IPTI to determine the timer interrupt
2246	 *  o read IntCtl.IPPCI to determine the performance counter interrupt
2247	 *  o read IntCtl.IPFDC to determine the fast debug channel interrupt
2248	 */
2249	if (cpu_has_mips_r2_r6) {
2250		cp0_compare_irq_shift = CAUSEB_TI - CAUSEB_IP;
2251		cp0_compare_irq = (read_c0_intctl() >> INTCTLB_IPTI) & 7;
2252		cp0_perfcount_irq = (read_c0_intctl() >> INTCTLB_IPPCI) & 7;
2253		cp0_fdc_irq = (read_c0_intctl() >> INTCTLB_IPFDC) & 7;
2254		if (!cp0_fdc_irq)
2255			cp0_fdc_irq = -1;
2256
2257	} else {
2258		cp0_compare_irq = CP0_LEGACY_COMPARE_IRQ;
2259		cp0_compare_irq_shift = CP0_LEGACY_PERFCNT_IRQ;
2260		cp0_perfcount_irq = -1;
2261		cp0_fdc_irq = -1;
2262	}
2263
2264	if (cpu_has_mmid)
2265		cpu_data[cpu].asid_cache = 0;
2266	else if (!cpu_data[cpu].asid_cache)
2267		cpu_data[cpu].asid_cache = asid_first_version(cpu);
2268
2269	mmgrab(&init_mm);
2270	current->active_mm = &init_mm;
2271	BUG_ON(current->mm);
2272	enter_lazy_tlb(&init_mm, current);
2273
2274	/* Boot CPU's cache setup in setup_arch(). */
2275	if (!is_boot_cpu)
2276		cpu_cache_init();
2277	tlb_init();
2278	TLBMISS_HANDLER_SETUP();
2279}
2280
2281/* Install CPU exception handler */
2282void set_handler(unsigned long offset, const void *addr, unsigned long size)
2283{
2284#ifdef CONFIG_CPU_MICROMIPS
2285	memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size);
2286#else
2287	memcpy((void *)(ebase + offset), addr, size);
2288#endif
2289	local_flush_icache_range(ebase + offset, ebase + offset + size);
2290}
2291
2292static const char panic_null_cerr[] =
2293	"Trying to set NULL cache error exception handler\n";
2294
2295/*
2296 * Install uncached CPU exception handler.
2297 * This is suitable only for the cache error exception which is the only
2298 * exception handler that is being run uncached.
2299 */
2300void set_uncached_handler(unsigned long offset, void *addr,
2301	unsigned long size)
2302{
2303	unsigned long uncached_ebase = CKSEG1ADDR_OR_64BIT(__pa(ebase));
2304
2305	if (!addr)
2306		panic(panic_null_cerr);
2307
2308	memcpy((void *)(uncached_ebase + offset), addr, size);
2309}
2310
2311static int __initdata rdhwr_noopt;
2312static int __init set_rdhwr_noopt(char *str)
2313{
2314	rdhwr_noopt = 1;
2315	return 1;
2316}
2317
2318__setup("rdhwr_noopt", set_rdhwr_noopt);
2319
2320void __init trap_init(void)
2321{
2322	extern char except_vec3_generic;
2323	extern char except_vec4;
2324	extern char except_vec3_r4000;
2325	unsigned long i, vec_size;
2326	phys_addr_t ebase_pa;
2327
2328	check_wait();
2329
2330	if (!cpu_has_mips_r2_r6) {
2331		ebase = CAC_BASE;
2332		vec_size = 0x400;
2333	} else {
2334		if (cpu_has_veic || cpu_has_vint)
2335			vec_size = 0x200 + VECTORSPACING*64;
2336		else
2337			vec_size = PAGE_SIZE;
2338
2339		ebase_pa = memblock_phys_alloc(vec_size, 1 << fls(vec_size));
2340		if (!ebase_pa)
2341			panic("%s: Failed to allocate %lu bytes align=0x%x\n",
2342			      __func__, vec_size, 1 << fls(vec_size));
2343
2344		/*
2345		 * Try to ensure ebase resides in KSeg0 if possible.
2346		 *
2347		 * It shouldn't generally be in XKPhys on MIPS64 to avoid
2348		 * hitting a poorly defined exception base for Cache Errors.
2349		 * The allocation is likely to be in the low 512MB of physical,
2350		 * in which case we should be able to convert to KSeg0.
2351		 *
2352		 * EVA is special though as it allows segments to be rearranged
2353		 * and to become uncached during cache error handling.
2354		 */
2355		if (!IS_ENABLED(CONFIG_EVA) && ebase_pa < 0x20000000)
2356			ebase = CKSEG0ADDR(ebase_pa);
2357		else
2358			ebase = (unsigned long)phys_to_virt(ebase_pa);
2359		if (ebase_pa >= 0x20000000)
2360			pr_warn("ebase(%pa) should better be in KSeg0",
2361				&ebase_pa);
2362	}
2363
2364	if (cpu_has_mmips) {
2365		unsigned int config3 = read_c0_config3();
2366
2367		if (IS_ENABLED(CONFIG_CPU_MICROMIPS))
2368			write_c0_config3(config3 | MIPS_CONF3_ISA_OE);
2369		else
2370			write_c0_config3(config3 & ~MIPS_CONF3_ISA_OE);
2371	}
2372
2373	if (board_ebase_setup)
2374		board_ebase_setup();
2375	per_cpu_trap_init(true);
2376	memblock_set_bottom_up(false);
2377
2378	/*
2379	 * Copy the generic exception handlers to their final destination.
2380	 * This will be overridden later as suitable for a particular
2381	 * configuration.
2382	 */
2383	set_handler(0x180, &except_vec3_generic, 0x80);
2384
2385	/*
2386	 * Setup default vectors
2387	 */
2388	for (i = 0; i <= 31; i++)
2389		set_except_vector(i, handle_reserved);
2390
2391	/*
2392	 * Copy the EJTAG debug exception vector handler code to its final
2393	 * destination.
2394	 */
2395	if (cpu_has_ejtag && board_ejtag_handler_setup)
2396		board_ejtag_handler_setup();
2397
2398	/*
2399	 * Only some CPUs have the watch exceptions.
2400	 */
2401	if (cpu_has_watch)
2402		set_except_vector(EXCCODE_WATCH, handle_watch);
2403
2404	/*
2405	 * Initialise interrupt handlers
2406	 */
2407	if (cpu_has_veic || cpu_has_vint) {
2408		int nvec = cpu_has_veic ? 64 : 8;
2409		for (i = 0; i < nvec; i++)
2410			set_vi_handler(i, NULL);
2411	}
2412	else if (cpu_has_divec)
2413		set_handler(0x200, &except_vec4, 0x8);
2414
2415	/*
2416	 * Some CPUs can enable/disable for cache parity detection, but does
2417	 * it different ways.
2418	 */
2419	parity_protection_init();
2420
2421	/*
2422	 * The Data Bus Errors / Instruction Bus Errors are signaled
2423	 * by external hardware.  Therefore these two exceptions
2424	 * may have board specific handlers.
2425	 */
2426	if (board_be_init)
2427		board_be_init();
2428
2429	set_except_vector(EXCCODE_INT, using_skipover_handler() ?
2430					skipover_handle_int : handle_int);
2431	set_except_vector(EXCCODE_MOD, handle_tlbm);
2432	set_except_vector(EXCCODE_TLBL, handle_tlbl);
2433	set_except_vector(EXCCODE_TLBS, handle_tlbs);
2434
2435	set_except_vector(EXCCODE_ADEL, handle_adel);
2436	set_except_vector(EXCCODE_ADES, handle_ades);
2437
2438	set_except_vector(EXCCODE_IBE, handle_ibe);
2439	set_except_vector(EXCCODE_DBE, handle_dbe);
2440
2441	set_except_vector(EXCCODE_SYS, handle_sys);
2442	set_except_vector(EXCCODE_BP, handle_bp);
2443
2444	if (rdhwr_noopt)
2445		set_except_vector(EXCCODE_RI, handle_ri);
2446	else {
2447		if (cpu_has_vtag_icache)
2448			set_except_vector(EXCCODE_RI, handle_ri_rdhwr_tlbp);
2449		else if (current_cpu_type() == CPU_LOONGSON64)
2450			set_except_vector(EXCCODE_RI, handle_ri_rdhwr_tlbp);
2451		else
2452			set_except_vector(EXCCODE_RI, handle_ri_rdhwr);
2453	}
2454
2455	set_except_vector(EXCCODE_CPU, handle_cpu);
2456	set_except_vector(EXCCODE_OV, handle_ov);
2457	set_except_vector(EXCCODE_TR, handle_tr);
2458	set_except_vector(EXCCODE_MSAFPE, handle_msa_fpe);
2459
2460	if (board_nmi_handler_setup)
2461		board_nmi_handler_setup();
2462
2463	if (cpu_has_fpu && !cpu_has_nofpuex)
2464		set_except_vector(EXCCODE_FPE, handle_fpe);
2465
2466	if (cpu_has_ftlbparex)
2467		set_except_vector(MIPS_EXCCODE_TLBPAR, handle_ftlb);
2468
2469	if (cpu_has_gsexcex)
2470		set_except_vector(LOONGSON_EXCCODE_GSEXC, handle_gsexc);
2471
2472	if (cpu_has_rixiex) {
2473		set_except_vector(EXCCODE_TLBRI, tlb_do_page_fault_0);
2474		set_except_vector(EXCCODE_TLBXI, tlb_do_page_fault_0);
2475	}
2476
2477	set_except_vector(EXCCODE_MSADIS, handle_msa);
2478	set_except_vector(EXCCODE_MDMX, handle_mdmx);
2479
2480	if (cpu_has_mcheck)
2481		set_except_vector(EXCCODE_MCHECK, handle_mcheck);
2482
2483	if (cpu_has_mipsmt)
2484		set_except_vector(EXCCODE_THREAD, handle_mt);
2485
2486	set_except_vector(EXCCODE_DSPDIS, handle_dsp);
2487
2488	if (board_cache_error_setup)
2489		board_cache_error_setup();
2490
2491	if (cpu_has_vce)
2492		/* Special exception: R4[04]00 uses also the divec space. */
2493		set_handler(0x180, &except_vec3_r4000, 0x100);
2494	else if (cpu_has_4kex)
2495		set_handler(0x180, &except_vec3_generic, 0x80);
2496	else
2497		set_handler(0x080, &except_vec3_generic, 0x80);
2498
2499	local_flush_icache_range(ebase, ebase + vec_size);
2500
2501	sort_extable(__start___dbe_table, __stop___dbe_table);
2502
2503	cu2_notifier(default_cu2_call, 0x80000000);	/* Run last  */
2504}
2505
2506static int trap_pm_notifier(struct notifier_block *self, unsigned long cmd,
2507			    void *v)
2508{
2509	switch (cmd) {
2510	case CPU_PM_ENTER_FAILED:
2511	case CPU_PM_EXIT:
2512		configure_status();
2513		configure_hwrena();
2514		configure_exception_vector();
2515
2516		/* Restore register with CPU number for TLB handlers */
2517		TLBMISS_HANDLER_RESTORE();
2518
2519		break;
2520	}
2521
2522	return NOTIFY_OK;
2523}
2524
2525static struct notifier_block trap_pm_notifier_block = {
2526	.notifier_call = trap_pm_notifier,
2527};
2528
2529static int __init trap_pm_init(void)
2530{
2531	return cpu_pm_register_notifier(&trap_pm_notifier_block);
2532}
2533arch_initcall(trap_pm_init);