traps_misaligned.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
  4 */
  5#include <linux/kernel.h>
  6#include <linux/init.h>
  7#include <linux/mm.h>
  8#include <linux/module.h>
  9#include <linux/perf_event.h>
 10#include <linux/irq.h>
 11#include <linux/stringify.h>
 12
 13#include <asm/processor.h>
 14#include <asm/ptrace.h>
 15#include <asm/csr.h>
 16#include <asm/entry-common.h>
 17#include <asm/hwprobe.h>
 18#include <asm/cpufeature.h>
 19#include <asm/sbi.h>
 20#include <asm/vector.h>
 21#include <asm/insn.h>
 22
 23#ifdef CONFIG_FPU
 24
 25#define FP_GET_RD(insn)		(insn >> 7 & 0x1F)
 26
 27extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
 28
 29static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
 30		      unsigned long val)
 31{
 32	unsigned long fp_reg = FP_GET_RD(insn);
 33
 34	put_f32_reg(fp_reg, val);
 35	regs->status |= SR_FS_DIRTY;
 36
 37	return 0;
 38}
 39
 40extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
 41
 42static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
 43{
 44	unsigned long fp_reg = FP_GET_RD(insn);
 45	unsigned long value;
 46
 47#if __riscv_xlen == 32
 48	value = (unsigned long) &val;
 49#else
 50	value = val;
 51#endif
 52	put_f64_reg(fp_reg, value);
 53	regs->status |= SR_FS_DIRTY;
 54
 55	return 0;
 56}
 57
 58#if __riscv_xlen == 32
 59extern void get_f64_reg(unsigned long fp_reg, u64 *value);
 60
 61static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
 62		      struct pt_regs *regs)
 63{
 64	unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
 65	u64 val;
 66
 67	get_f64_reg(fp_reg, &val);
 68	regs->status |= SR_FS_DIRTY;
 69
 70	return val;
 71}
 72#else
 73
 74extern unsigned long get_f64_reg(unsigned long fp_reg);
 75
 76static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
 77				struct pt_regs *regs)
 78{
 79	unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
 80	unsigned long val;
 81
 82	val = get_f64_reg(fp_reg);
 83	regs->status |= SR_FS_DIRTY;
 84
 85	return val;
 86}
 87
 88#endif
 89
 90extern unsigned long get_f32_reg(unsigned long fp_reg);
 91
 92static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
 93				struct pt_regs *regs)
 94{
 95	unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
 96	unsigned long val;
 97
 98	val = get_f32_reg(fp_reg);
 99	regs->status |= SR_FS_DIRTY;
100
101	return val;
102}
103
104#else /* CONFIG_FPU */
105static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
106		       unsigned long val) {}
107
108static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
109
110static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
111				struct pt_regs *regs)
112{
113	return 0;
114}
115
116static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
117				struct pt_regs *regs)
118{
119	return 0;
120}
121
122#endif
123
124#define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
125#define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
126#define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
127
128#define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
129#define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
130#define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
131
132#define __read_insn(regs, insn, insn_addr, type)	\
133({							\
134	int __ret;					\
135							\
136	if (user_mode(regs)) {				\
137		__ret = get_user(insn, (type __user *) insn_addr); \
138	} else {					\
139		insn = *(type *)insn_addr;		\
140		__ret = 0;				\
141	}						\
142							\
143	__ret;						\
144})
145
146static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
147{
148	ulong insn = 0;
149
150	if (epc & 0x2) {
151		ulong tmp = 0;
152
153		if (__read_insn(regs, insn, epc, u16))
154			return -EFAULT;
155		/* __get_user() uses regular "lw" which sign extend the loaded
156		 * value make sure to clear higher order bits in case we "or" it
157		 * below with the upper 16 bits half.
158		 */
159		insn &= GENMASK(15, 0);
160		if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
161			*r_insn = insn;
162			return 0;
163		}
164		epc += sizeof(u16);
165		if (__read_insn(regs, tmp, epc, u16))
166			return -EFAULT;
167		*r_insn = (tmp << 16) | insn;
168
169		return 0;
170	} else {
171		if (__read_insn(regs, insn, epc, u32))
172			return -EFAULT;
173		if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
174			*r_insn = insn;
175			return 0;
176		}
177		insn &= GENMASK(15, 0);
178		*r_insn = insn;
179
180		return 0;
181	}
182}
183
184union reg_data {
185	u8 data_bytes[8];
186	ulong data_ulong;
187	u64 data_u64;
188};
189
190/* sysctl hooks */
191int unaligned_enabled __read_mostly = 1;	/* Enabled by default */
192
193#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
194static int handle_vector_misaligned_load(struct pt_regs *regs)
195{
196	unsigned long epc = regs->epc;
197	unsigned long insn;
198
199	if (get_insn(regs, epc, &insn))
200		return -1;
201
202	/* Only return 0 when in check_vector_unaligned_access_emulated */
203	if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
204		*this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
205		regs->epc = epc + INSN_LEN(insn);
206		return 0;
207	}
208
209	/* If vector instruction we don't emulate it yet */
210	regs->epc = epc;
211	return -1;
212}
213#else
214static int handle_vector_misaligned_load(struct pt_regs *regs)
215{
216	return -1;
217}
218#endif
219
220static int handle_scalar_misaligned_load(struct pt_regs *regs)
221{
222	union reg_data val;
223	unsigned long epc = regs->epc;
224	unsigned long insn;
225	unsigned long addr = regs->badaddr;
226	int fp = 0, shift = 0, len = 0;
227
228	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
229
230	*this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
231
232	if (!unaligned_enabled)
233		return -1;
234
235	if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
236		return -1;
237
238	if (get_insn(regs, epc, &insn))
239		return -1;
240
241	regs->epc = 0;
242
243	if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
244		len = 4;
245		shift = 8 * (sizeof(unsigned long) - len);
246#if defined(CONFIG_64BIT)
247	} else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
248		len = 8;
249		shift = 8 * (sizeof(unsigned long) - len);
250	} else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
251		len = 4;
252#endif
253	} else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
254		fp = 1;
255		len = 8;
256	} else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
257		fp = 1;
258		len = 4;
259	} else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
260		len = 2;
261		shift = 8 * (sizeof(unsigned long) - len);
262	} else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
263		len = 2;
264#if defined(CONFIG_64BIT)
265	} else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
266		len = 8;
267		shift = 8 * (sizeof(unsigned long) - len);
268		insn = RVC_RS2S(insn) << SH_RD;
269	} else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
270		   ((insn >> SH_RD) & 0x1f)) {
271		len = 8;
272		shift = 8 * (sizeof(unsigned long) - len);
273#endif
274	} else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
275		len = 4;
276		shift = 8 * (sizeof(unsigned long) - len);
277		insn = RVC_RS2S(insn) << SH_RD;
278	} else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
279		   ((insn >> SH_RD) & 0x1f)) {
280		len = 4;
281		shift = 8 * (sizeof(unsigned long) - len);
282	} else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
283		fp = 1;
284		len = 8;
285		insn = RVC_RS2S(insn) << SH_RD;
286	} else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
287		fp = 1;
288		len = 8;
289#if defined(CONFIG_32BIT)
290	} else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
291		fp = 1;
292		len = 4;
293		insn = RVC_RS2S(insn) << SH_RD;
294	} else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
295		fp = 1;
296		len = 4;
297#endif
298	} else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) {
299		len = 2;
300		insn = RVC_RS2S(insn) << SH_RD;
301	} else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) {
302		len = 2;
303		shift = 8 * (sizeof(ulong) - len);
304		insn = RVC_RS2S(insn) << SH_RD;
305	} else {
306		regs->epc = epc;
307		return -1;
308	}
309
310	if (!IS_ENABLED(CONFIG_FPU) && fp)
311		return -EOPNOTSUPP;
312
313	val.data_u64 = 0;
314	if (user_mode(regs)) {
315		if (copy_from_user(&val, (u8 __user *)addr, len))
316			return -1;
317	} else {
318		memcpy(&val, (u8 *)addr, len);
319	}
320
321	if (!fp)
322		SET_RD(insn, regs, (long)(val.data_ulong << shift) >> shift);
323	else if (len == 8)
324		set_f64_rd(insn, regs, val.data_u64);
325	else
326		set_f32_rd(insn, regs, val.data_ulong);
327
328	regs->epc = epc + INSN_LEN(insn);
329
330	return 0;
331}
332
333static int handle_scalar_misaligned_store(struct pt_regs *regs)
334{
335	union reg_data val;
336	unsigned long epc = regs->epc;
337	unsigned long insn;
338	unsigned long addr = regs->badaddr;
339	int len = 0, fp = 0;
340
341	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
342
343	if (!unaligned_enabled)
344		return -1;
345
346	if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
347		return -1;
348
349	if (get_insn(regs, epc, &insn))
350		return -1;
351
352	regs->epc = 0;
353
354	val.data_ulong = GET_RS2(insn, regs);
355
356	if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
357		len = 4;
358#if defined(CONFIG_64BIT)
359	} else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
360		len = 8;
361#endif
362	} else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
363		fp = 1;
364		len = 8;
365		val.data_u64 = GET_F64_RS2(insn, regs);
366	} else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
367		fp = 1;
368		len = 4;
369		val.data_ulong = GET_F32_RS2(insn, regs);
370	} else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
371		len = 2;
372#if defined(CONFIG_64BIT)
373	} else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
374		len = 8;
375		val.data_ulong = GET_RS2S(insn, regs);
376	} else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
377		len = 8;
378		val.data_ulong = GET_RS2C(insn, regs);
379#endif
380	} else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
381		len = 4;
382		val.data_ulong = GET_RS2S(insn, regs);
383	} else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
384		len = 4;
385		val.data_ulong = GET_RS2C(insn, regs);
386	} else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
387		fp = 1;
388		len = 8;
389		val.data_u64 = GET_F64_RS2S(insn, regs);
390	} else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
391		fp = 1;
392		len = 8;
393		val.data_u64 = GET_F64_RS2C(insn, regs);
394#if !defined(CONFIG_64BIT)
395	} else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
396		fp = 1;
397		len = 4;
398		val.data_ulong = GET_F32_RS2S(insn, regs);
399	} else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
400		fp = 1;
401		len = 4;
402		val.data_ulong = GET_F32_RS2C(insn, regs);
403#endif
404	} else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) {
405		len = 2;
406		val.data_ulong = GET_RS2S(insn, regs);
407	} else {
408		regs->epc = epc;
409		return -1;
410	}
411
412	if (!IS_ENABLED(CONFIG_FPU) && fp)
413		return -EOPNOTSUPP;
414
415	if (user_mode(regs)) {
416		if (copy_to_user((u8 __user *)addr, &val, len))
417			return -1;
418	} else {
419		memcpy((u8 *)addr, &val, len);
420	}
421
422	regs->epc = epc + INSN_LEN(insn);
423
424	return 0;
425}
426
427int handle_misaligned_load(struct pt_regs *regs)
428{
429	unsigned long epc = regs->epc;
430	unsigned long insn;
431
432	if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
433		if (get_insn(regs, epc, &insn))
434			return -1;
435
436		if (insn_is_vector(insn))
437			return handle_vector_misaligned_load(regs);
438	}
439
440	if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
441		return handle_scalar_misaligned_load(regs);
442
443	return -1;
444}
445
446int handle_misaligned_store(struct pt_regs *regs)
447{
448	if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
449		return handle_scalar_misaligned_store(regs);
450
451	return -1;
452}
453
454#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
455void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
456{
457	long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
458	unsigned long tmp_var;
459
460	*mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
461
462	kernel_vector_begin();
463	/*
464	 * In pre-13.0.0 versions of GCC, vector registers cannot appear in
465	 * the clobber list. This inline asm clobbers v0, but since we do not
466	 * currently build the kernel with V enabled, the v0 clobber arg is not
467	 * needed (as the compiler will not emit vector code itself). If the kernel
468	 * is changed to build with V enabled, the clobber arg will need to be
469	 * added here.
470	 */
471	__asm__ __volatile__ (
472		".balign 4\n\t"
473		".option push\n\t"
474		".option arch, +zve32x\n\t"
475		"       vsetivli zero, 1, e16, m1, ta, ma\n\t"	// Vectors of 16b
476		"       vle16.v v0, (%[ptr])\n\t"		// Load bytes
477		".option pop\n\t"
478		: : [ptr] "r" ((u8 *)&tmp_var + 1));
479	kernel_vector_end();
480}
481
482bool __init check_vector_unaligned_access_emulated_all_cpus(void)
483{
484	int cpu;
485
486	/*
487	 * While being documented as very slow, schedule_on_each_cpu() is used since
488	 * kernel_vector_begin() expects irqs to be enabled or it will panic()
489	 */
490	schedule_on_each_cpu(check_vector_unaligned_access_emulated);
491
492	for_each_online_cpu(cpu)
493		if (per_cpu(vector_misaligned_access, cpu)
494		    == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
495			return false;
496
497	return true;
498}
499#else
500bool __init check_vector_unaligned_access_emulated_all_cpus(void)
501{
502	return false;
503}
504#endif
505
506static bool all_cpus_unaligned_scalar_access_emulated(void)
507{
508	int cpu;
509
510	for_each_online_cpu(cpu)
511		if (per_cpu(misaligned_access_speed, cpu) !=
512		    RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
513			return false;
514
515	return true;
516}
517
518#ifdef CONFIG_RISCV_SCALAR_MISALIGNED
519
520static bool unaligned_ctl __read_mostly;
521
522static void check_unaligned_access_emulated(void *arg __always_unused)
523{
524	int cpu = smp_processor_id();
525	long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
526	unsigned long tmp_var, tmp_val;
527
528	*mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
529
530	__asm__ __volatile__ (
531		"       "REG_L" %[tmp], 1(%[ptr])\n"
532		: [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
533}
534
535static int cpu_online_check_unaligned_access_emulated(unsigned int cpu)
536{
537	long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
538
539	check_unaligned_access_emulated(NULL);
540
541	/*
542	 * If unaligned_ctl is already set, this means that we detected that all
543	 * CPUS uses emulated misaligned access at boot time. If that changed
544	 * when hotplugging the new cpu, this is something we don't handle.
545	 */
546	if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
547		pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
548		return -EINVAL;
549	}
550
551	return 0;
552}
553
554bool __init check_unaligned_access_emulated_all_cpus(void)
555{
556	/*
557	 * We can only support PR_UNALIGN controls if all CPUs have misaligned
558	 * accesses emulated since tasks requesting such control can run on any
559	 * CPU.
560	 */
561	on_each_cpu(check_unaligned_access_emulated, NULL, 1);
562
563	if (!all_cpus_unaligned_scalar_access_emulated())
564		return false;
565
566	unaligned_ctl = true;
567	return true;
568}
569
570bool unaligned_ctl_available(void)
571{
572	return unaligned_ctl;
573}
574#else
575bool __init check_unaligned_access_emulated_all_cpus(void)
576{
577	return false;
578}
579static int cpu_online_check_unaligned_access_emulated(unsigned int cpu)
580{
581	return 0;
582}
583#endif
584
585static bool misaligned_traps_delegated;
586
587#ifdef CONFIG_RISCV_SBI
588
589static int cpu_online_sbi_unaligned_setup(unsigned int cpu)
590{
591	if (sbi_fwft_set(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0) &&
592	    misaligned_traps_delegated) {
593		pr_crit("Misaligned trap delegation non homogeneous (expected delegated)");
594		return -EINVAL;
595	}
596
597	return 0;
598}
599
600void __init unaligned_access_init(void)
601{
602	int ret;
603
604	ret = sbi_fwft_set_online_cpus(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0);
605	if (ret)
606		return;
607
608	misaligned_traps_delegated = true;
609	pr_info("SBI misaligned access exception delegation ok\n");
610	/*
611	 * Note that we don't have to take any specific action here, if
612	 * the delegation is successful, then
613	 * check_unaligned_access_emulated() will verify that indeed the
614	 * platform traps on misaligned accesses.
615	 */
616}
617#else
618void __init unaligned_access_init(void) {}
619
620static int cpu_online_sbi_unaligned_setup(unsigned int cpu __always_unused)
621{
622	return 0;
623}
624
625#endif
626
627int cpu_online_unaligned_access_init(unsigned int cpu)
628{
629	int ret;
630
631	ret = cpu_online_sbi_unaligned_setup(cpu);
632	if (ret)
633		return ret;
634
635	return cpu_online_check_unaligned_access_emulated(cpu);
636}
637
638bool misaligned_traps_can_delegate(void)
639{
640	/*
641	 * Either we successfully requested misaligned traps delegation for all
642	 * CPUs, or the SBI does not implement the FWFT extension but delegated
643	 * the exception by default.
644	 */
645	return misaligned_traps_delegated ||
646	       all_cpus_unaligned_scalar_access_emulated();
647}
648EXPORT_SYMBOL_GPL(misaligned_traps_can_delegate);