uaccess.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 *  arch/arm/include/asm/uaccess.h
  4 */
  5#ifndef _ASMARM_UACCESS_H
  6#define _ASMARM_UACCESS_H
  7
  8/*
  9 * User space memory access functions
 10 */
 11#include <linux/kernel.h>
 12#include <linux/string.h>
 13#include <asm/page.h>
 14#include <asm/domain.h>
 15#include <linux/unaligned.h>
 16#include <asm/unified.h>
 17#include <asm/pgtable.h>
 18#include <asm/proc-fns.h>
 19#include <asm/compiler.h>
 20
 21#include <asm/extable.h>
 22
 23/*
 24 * These two functions allow hooking accesses to userspace to increase
 25 * system integrity by ensuring that the kernel can not inadvertantly
 26 * perform such accesses (eg, via list poison values) which could then
 27 * be exploited for priviledge escalation.
 28 */
 29#if defined(CONFIG_CPU_SW_DOMAIN_PAN)
 30
 31static __always_inline unsigned int uaccess_save_and_enable(void)
 32{
 33	unsigned int old_domain = get_domain();
 34
 35	/* Set the current domain access to permit user accesses */
 36	set_domain((old_domain & ~domain_mask(DOMAIN_USER)) |
 37		   domain_val(DOMAIN_USER, DOMAIN_CLIENT));
 38
 39	return old_domain;
 40}
 41
 42static __always_inline void uaccess_restore(unsigned int flags)
 43{
 44	/* Restore the user access mask */
 45	set_domain(flags);
 46}
 47
 48#elif defined(CONFIG_CPU_TTBR0_PAN)
 49
 50static __always_inline unsigned int uaccess_save_and_enable(void)
 51{
 52	unsigned int old_ttbcr = cpu_get_ttbcr();
 53
 54	/*
 55	 * Enable TTBR0 page table walks (T0SZ = 0, EDP0 = 0) and ASID from
 56	 * TTBR0 (A1 = 0).
 57	 */
 58	cpu_set_ttbcr(old_ttbcr & ~(TTBCR_A1 | TTBCR_EPD0 | TTBCR_T0SZ_MASK));
 59	isb();
 60
 61	return old_ttbcr;
 62}
 63
 64static inline void uaccess_restore(unsigned int flags)
 65{
 66	cpu_set_ttbcr(flags);
 67	isb();
 68}
 69
 70#else
 71
 72static inline unsigned int uaccess_save_and_enable(void)
 73{
 74	return 0;
 75}
 76
 77static inline void uaccess_restore(unsigned int flags)
 78{
 79}
 80
 81#endif
 82
 83/*
 84 * These two are intentionally not defined anywhere - if the kernel
 85 * code generates any references to them, that's a bug.
 86 */
 87extern int __get_user_bad(void);
 88extern int __put_user_bad(void);
 89
 90#ifdef CONFIG_MMU
 91
 92/*
 93 * This is a type: either unsigned long, if the argument fits into
 94 * that type, or otherwise unsigned long long.
 95 */
 96#define __inttype(x) \
 97	__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
 98
 99/*
100 * Sanitise a uaccess pointer such that it becomes NULL if addr+size
101 * is above the current addr_limit.
102 */
103#define uaccess_mask_range_ptr(ptr, size)			\
104	((__typeof__(ptr))__uaccess_mask_range_ptr(ptr, size))
105static inline void __user *__uaccess_mask_range_ptr(const void __user *ptr,
106						    size_t size)
107{
108	void __user *safe_ptr = (void __user *)ptr;
109	unsigned long tmp;
110
111	asm volatile(
112	"	.syntax unified\n"
113	"	sub	%1, %3, #1\n"
114	"	subs	%1, %1, %0\n"
115	"	addhs	%1, %1, #1\n"
116	"	subshs	%1, %1, %2\n"
117	"	movlo	%0, #0\n"
118	: "+r" (safe_ptr), "=&r" (tmp)
119	: "r" (size), "r" (TASK_SIZE)
120	: "cc");
121
122	csdb();
123	return safe_ptr;
124}
125
126/*
127 * Single-value transfer routines.  They automatically use the right
128 * size if we just have the right pointer type.  Note that the functions
129 * which read from user space (*get_*) need to take care not to leak
130 * kernel data even if the calling code is buggy and fails to check
131 * the return value.  This means zeroing out the destination variable
132 * or buffer on error.  Normally this is done out of line by the
133 * fixup code, but there are a few places where it intrudes on the
134 * main code path.  When we only write to user space, there is no
135 * problem.
136 */
137extern int __get_user_1(void *);
138extern int __get_user_2(void *);
139extern int __get_user_4(void *);
140extern int __get_user_32t_8(void *);
141extern int __get_user_8(void *);
142extern int __get_user_64t_1(void *);
143extern int __get_user_64t_2(void *);
144extern int __get_user_64t_4(void *);
145
146#define __get_user_x(__r2, __p, __e, __l, __s)				\
147	   __asm__ __volatile__ (					\
148		__asmeq("%0", "r0") __asmeq("%1", "r2")			\
149		__asmeq("%3", "r1")					\
150		"bl	__get_user_" #__s				\
151		: "=&r" (__e), "=r" (__r2)				\
152		: "0" (__p), "r" (__l)					\
153		: "ip", "lr", "cc")
154
155/* narrowing a double-word get into a single 32bit word register: */
156#ifdef __ARMEB__
157#define __get_user_x_32t(__r2, __p, __e, __l, __s)			\
158	__get_user_x(__r2, __p, __e, __l, 32t_8)
159#else
160#define __get_user_x_32t __get_user_x
161#endif
162
163/*
164 * storing result into proper least significant word of 64bit target var,
165 * different only for big endian case where 64 bit __r2 lsw is r3:
166 */
167#ifdef __ARMEB__
168#define __get_user_x_64t(__r2, __p, __e, __l, __s)		        \
169	   __asm__ __volatile__ (					\
170		__asmeq("%0", "r0") __asmeq("%1", "r2")			\
171		__asmeq("%3", "r1")					\
172		"bl	__get_user_64t_" #__s				\
173		: "=&r" (__e), "=r" (__r2)				\
174		: "0" (__p), "r" (__l)					\
175		: "ip", "lr", "cc")
176#else
177#define __get_user_x_64t __get_user_x
178#endif
179
180
181#define __get_user_check(x, p)						\
182	({								\
183		unsigned long __limit = TASK_SIZE - 1; \
184		register typeof(*(p)) __user *__p asm("r0") = (p);	\
185		register __inttype(x) __r2 asm("r2");			\
186		register unsigned long __l asm("r1") = __limit;		\
187		register int __e asm("r0");				\
188		unsigned int __ua_flags = uaccess_save_and_enable();	\
189		int __tmp_e;						\
190		switch (sizeof(*(__p))) {				\
191		case 1:							\
192			if (sizeof((x)) >= 8)				\
193				__get_user_x_64t(__r2, __p, __e, __l, 1); \
194			else						\
195				__get_user_x(__r2, __p, __e, __l, 1);	\
196			break;						\
197		case 2:							\
198			if (sizeof((x)) >= 8)				\
199				__get_user_x_64t(__r2, __p, __e, __l, 2); \
200			else						\
201				__get_user_x(__r2, __p, __e, __l, 2);	\
202			break;						\
203		case 4:							\
204			if (sizeof((x)) >= 8)				\
205				__get_user_x_64t(__r2, __p, __e, __l, 4); \
206			else						\
207				__get_user_x(__r2, __p, __e, __l, 4);	\
208			break;						\
209		case 8:							\
210			if (sizeof((x)) < 8)				\
211				__get_user_x_32t(__r2, __p, __e, __l, 4); \
212			else						\
213				__get_user_x(__r2, __p, __e, __l, 8);	\
214			break;						\
215		default: __e = __get_user_bad(); break;			\
216		}							\
217		__tmp_e = __e;						\
218		uaccess_restore(__ua_flags);				\
219		x = (typeof(*(p))) __r2;				\
220		__tmp_e;						\
221	})
222
223#define get_user(x, p)							\
224	({								\
225		might_fault();						\
226		__get_user_check(x, p);					\
227	 })
228
229extern int __put_user_1(void *, unsigned int);
230extern int __put_user_2(void *, unsigned int);
231extern int __put_user_4(void *, unsigned int);
232extern int __put_user_8(void *, unsigned long long);
233
234#define __put_user_check(__pu_val, __ptr, __err, __s)			\
235	({								\
236		unsigned long __limit = TASK_SIZE - 1; \
237		register typeof(__pu_val) __r2 asm("r2") = __pu_val;	\
238		register const void __user *__p asm("r0") = __ptr;	\
239		register unsigned long __l asm("r1") = __limit;		\
240		register int __e asm("r0");				\
241		__asm__ __volatile__ (					\
242			__asmeq("%0", "r0") __asmeq("%2", "r2")		\
243			__asmeq("%3", "r1")				\
244			"bl	__put_user_" #__s			\
245			: "=&r" (__e)					\
246			: "0" (__p), "r" (__r2), "r" (__l)		\
247			: "ip", "lr", "cc");				\
248		__err = __e;						\
249	})
250
251#else /* CONFIG_MMU */
252
253#define get_user(x, p)	__get_user(x, p)
254#define __put_user_check __put_user_nocheck
255
256#endif /* CONFIG_MMU */
257
258#include <asm-generic/access_ok.h>
259
260#ifdef CONFIG_CPU_SPECTRE
261/*
262 * When mitigating Spectre variant 1, it is not worth fixing the non-
263 * verifying accessors, because we need to add verification of the
264 * address space there.  Force these to use the standard get_user()
265 * version instead.
266 */
267#define __get_user(x, ptr) get_user(x, ptr)
268#else
269
270/*
271 * The "__xxx" versions of the user access functions do not verify the
272 * address space - it must have been done previously with a separate
273 * "access_ok()" call.
274 *
275 * The "xxx_error" versions set the third argument to EFAULT if an
276 * error occurs, and leave it unchanged on success.  Note that these
277 * versions are void (ie, don't return a value as such).
278 */
279#define __get_user(x, ptr)						\
280({									\
281	long __gu_err = 0;						\
282	__get_user_err((x), (ptr), __gu_err, TUSER());			\
283	__gu_err;							\
284})
285
286/*
287 * This is a type: either unsigned long, if the argument fits into
288 * that type, or otherwise unsigned long long.
289 */
290#define __long_type(x) \
291	__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
292
293#define __get_user_err(x, ptr, err, __t)				\
294do {									\
295	unsigned long __gu_addr = (unsigned long)(ptr);			\
296	__long_type(x) __gu_val;					\
297	unsigned int __ua_flags;					\
298	__chk_user_ptr(ptr);						\
299	might_fault();							\
300	__ua_flags = uaccess_save_and_enable();				\
301	switch (sizeof(*(ptr))) {					\
302	case 1:	__get_user_asm_byte(__gu_val, __gu_addr, err, __t); break;	\
303	case 2:	__get_user_asm_half(__gu_val, __gu_addr, err, __t); break;	\
304	case 4:	__get_user_asm_word(__gu_val, __gu_addr, err, __t); break;	\
305	case 8:	__get_user_asm_dword(__gu_val, __gu_addr, err, __t); break;	\
306	default: (__gu_val) = __get_user_bad();				\
307	}								\
308	uaccess_restore(__ua_flags);					\
309	(x) = (__typeof__(*(ptr)))__gu_val;				\
310} while (0)
311#endif
312
313#define __get_user_asm(x, addr, err, instr)			\
314	__asm__ __volatile__(					\
315	"1:	" instr " %1, [%2], #0\n"			\
316	"2:\n"							\
317	"	.pushsection .text.fixup,\"ax\"\n"		\
318	"	.align	2\n"					\
319	"3:	mov	%0, %3\n"				\
320	"	mov	%1, #0\n"				\
321	"	b	2b\n"					\
322	"	.popsection\n"					\
323	"	.pushsection __ex_table,\"a\"\n"		\
324	"	.align	3\n"					\
325	"	.long	1b, 3b\n"				\
326	"	.popsection"					\
327	: "+r" (err), "=&r" (x)					\
328	: "r" (addr), "i" (-EFAULT)				\
329	: "cc")
330
331#define __get_user_asm_byte(x, addr, err, __t)			\
332	__get_user_asm(x, addr, err, "ldrb" __t)
333
334#if __LINUX_ARM_ARCH__ >= 6
335
336#define __get_user_asm_half(x, addr, err, __t)			\
337	__get_user_asm(x, addr, err, "ldrh" __t)
338
339#else
340
341#ifndef __ARMEB__
342#define __get_user_asm_half(x, __gu_addr, err, __t)		\
343({								\
344	unsigned long __b1, __b2;				\
345	__get_user_asm_byte(__b1, __gu_addr, err, __t);		\
346	__get_user_asm_byte(__b2, __gu_addr + 1, err, __t);	\
347	(x) = __b1 | (__b2 << 8);				\
348})
349#else
350#define __get_user_asm_half(x, __gu_addr, err, __t)		\
351({								\
352	unsigned long __b1, __b2;				\
353	__get_user_asm_byte(__b1, __gu_addr, err, __t);		\
354	__get_user_asm_byte(__b2, __gu_addr + 1, err, __t);	\
355	(x) = (__b1 << 8) | __b2;				\
356})
357#endif
358
359#endif /* __LINUX_ARM_ARCH__ >= 6 */
360
361#define __get_user_asm_word(x, addr, err, __t)			\
362	__get_user_asm(x, addr, err, "ldr" __t)
363
364#ifdef __ARMEB__
365#define __WORD0_OFFS	4
366#define __WORD1_OFFS	0
367#else
368#define __WORD0_OFFS	0
369#define __WORD1_OFFS	4
370#endif
371
372#define __get_user_asm_dword(x, addr, err, __t)				\
373	({								\
374	unsigned long __w0, __w1;					\
375	__get_user_asm(__w0, addr + __WORD0_OFFS, err, "ldr" __t);	\
376	__get_user_asm(__w1, addr + __WORD1_OFFS, err, "ldr" __t);	\
377	(x) = ((u64)__w1 << 32) | (u64) __w0;				\
378})
379
380#define __put_user_switch(x, ptr, __err, __fn)				\
381	do {								\
382		const __typeof__(*(ptr)) __user *__pu_ptr = (ptr);	\
383		__typeof__(*(ptr)) __pu_val = (x);			\
384		unsigned int __ua_flags;				\
385		might_fault();						\
386		__ua_flags = uaccess_save_and_enable();			\
387		switch (sizeof(*(ptr))) {				\
388		case 1: __fn(__pu_val, __pu_ptr, __err, 1); break;	\
389		case 2:	__fn(__pu_val, __pu_ptr, __err, 2); break;	\
390		case 4:	__fn(__pu_val, __pu_ptr, __err, 4); break;	\
391		case 8:	__fn(__pu_val, __pu_ptr, __err, 8); break;	\
392		default: __err = __put_user_bad(); break;		\
393		}							\
394		uaccess_restore(__ua_flags);				\
395	} while (0)
396
397#define put_user(x, ptr)						\
398({									\
399	int __pu_err = 0;						\
400	__put_user_switch((x), (ptr), __pu_err, __put_user_check);	\
401	__pu_err;							\
402})
403
404#ifdef CONFIG_CPU_SPECTRE
405/*
406 * When mitigating Spectre variant 1.1, all accessors need to include
407 * verification of the address space.
408 */
409#define __put_user(x, ptr) put_user(x, ptr)
410
411#else
412#define __put_user(x, ptr)						\
413({									\
414	long __pu_err = 0;						\
415	__put_user_switch((x), (ptr), __pu_err, __put_user_nocheck);	\
416	__pu_err;							\
417})
418
419#define __put_user_nocheck(x, __pu_ptr, __err, __size)			\
420	do {								\
421		unsigned long __pu_addr = (unsigned long)__pu_ptr;	\
422		__put_user_nocheck_##__size(x, __pu_addr, __err, TUSER());\
423	} while (0)
424
425#define __put_user_nocheck_1 __put_user_asm_byte
426#define __put_user_nocheck_2 __put_user_asm_half
427#define __put_user_nocheck_4 __put_user_asm_word
428#define __put_user_nocheck_8 __put_user_asm_dword
429
430#endif /* !CONFIG_CPU_SPECTRE */
431
432#define __put_user_asm(x, __pu_addr, err, instr)		\
433	__asm__ __volatile__(					\
434	"1:	" instr " %1, [%2], #0\n"		\
435	"2:\n"							\
436	"	.pushsection .text.fixup,\"ax\"\n"		\
437	"	.align	2\n"					\
438	"3:	mov	%0, %3\n"				\
439	"	b	2b\n"					\
440	"	.popsection\n"					\
441	"	.pushsection __ex_table,\"a\"\n"		\
442	"	.align	3\n"					\
443	"	.long	1b, 3b\n"				\
444	"	.popsection"					\
445	: "+r" (err)						\
446	: "r" (x), "r" (__pu_addr), "i" (-EFAULT)		\
447	: "cc")
448
449#define __put_user_asm_byte(x, __pu_addr, err, __t)		\
450	__put_user_asm(x, __pu_addr, err, "strb" __t)
451
452#if __LINUX_ARM_ARCH__ >= 6
453
454#define __put_user_asm_half(x, __pu_addr, err, __t)		\
455	__put_user_asm(x, __pu_addr, err, "strh" __t)
456
457#else
458
459#ifndef __ARMEB__
460#define __put_user_asm_half(x, __pu_addr, err, __t)		\
461({								\
462	unsigned long __temp = (__force unsigned long)(x);	\
463	__put_user_asm_byte(__temp, __pu_addr, err, __t);	\
464	__put_user_asm_byte(__temp >> 8, __pu_addr + 1, err, __t);\
465})
466#else
467#define __put_user_asm_half(x, __pu_addr, err, __t)		\
468({								\
469	unsigned long __temp = (__force unsigned long)(x);	\
470	__put_user_asm_byte(__temp >> 8, __pu_addr, err, __t);	\
471	__put_user_asm_byte(__temp, __pu_addr + 1, err, __t);	\
472})
473#endif
474
475#endif /* __LINUX_ARM_ARCH__ >= 6 */
476
477#define __put_user_asm_word(x, __pu_addr, err, __t)		\
478	__put_user_asm(x, __pu_addr, err, "str" __t)
479
480#ifndef __ARMEB__
481#define	__reg_oper0	"%R2"
482#define	__reg_oper1	"%Q2"
483#else
484#define	__reg_oper0	"%Q2"
485#define	__reg_oper1	"%R2"
486#endif
487
488#define __put_user_asm_dword(x, __pu_addr, err, __t)		\
489	__asm__ __volatile__(					\
490 ARM(	"1:	str" __t "	" __reg_oper1 ", [%1], #4\n"  ) \
491 ARM(	"2:	str" __t "	" __reg_oper0 ", [%1]\n"      ) \
492 THUMB(	"1:	str" __t "	" __reg_oper1 ", [%1]\n"      ) \
493 THUMB(	"2:	str" __t "	" __reg_oper0 ", [%1, #4]\n"  ) \
494	"3:\n"							\
495	"	.pushsection .text.fixup,\"ax\"\n"		\
496	"	.align	2\n"					\
497	"4:	mov	%0, %3\n"				\
498	"	b	3b\n"					\
499	"	.popsection\n"					\
500	"	.pushsection __ex_table,\"a\"\n"		\
501	"	.align	3\n"					\
502	"	.long	1b, 4b\n"				\
503	"	.long	2b, 4b\n"				\
504	"	.popsection"					\
505	: "+r" (err), "+r" (__pu_addr)				\
506	: "r" (x), "i" (-EFAULT)				\
507	: "cc")
508
509#define __get_kernel_nofault(dst, src, type, err_label)			\
510do {									\
511	const type *__pk_ptr = (src);					\
512	unsigned long __src = (unsigned long)(__pk_ptr);		\
513	type __val;							\
514	int __err = 0;							\
515	switch (sizeof(type)) {						\
516	case 1:	__get_user_asm_byte(__val, __src, __err, ""); break;	\
517	case 2: __get_user_asm_half(__val, __src, __err, ""); break;	\
518	case 4: __get_user_asm_word(__val, __src, __err, ""); break;	\
519	case 8: {							\
520		u32 *__v32 = (u32*)&__val;				\
521		__get_user_asm_word(__v32[0], __src, __err, "");	\
522		if (__err)						\
523			break;						\
524		__get_user_asm_word(__v32[1], __src+4, __err, "");	\
525		break;							\
526	}								\
527	default: __err = __get_user_bad(); break;			\
528	}								\
529	if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))		\
530		put_unaligned(__val, (type *)(dst));			\
531	else								\
532		*(type *)(dst) = __val; /* aligned by caller */		\
533	if (__err)							\
534		goto err_label;						\
535} while (0)
536
537#define __put_kernel_nofault(dst, src, type, err_label)			\
538do {									\
539	const type *__pk_ptr = (dst);					\
540	unsigned long __dst = (unsigned long)__pk_ptr;			\
541	int __err = 0;							\
542	type __val = IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)	\
543		     ? get_unaligned((type *)(src))			\
544		     : *(type *)(src);	/* aligned by caller */		\
545	switch (sizeof(type)) {						\
546	case 1: __put_user_asm_byte(__val, __dst, __err, ""); break;	\
547	case 2:	__put_user_asm_half(__val, __dst, __err, ""); break;	\
548	case 4:	__put_user_asm_word(__val, __dst, __err, ""); break;	\
549	case 8:	__put_user_asm_dword(__val, __dst, __err, ""); break;	\
550	default: __err = __put_user_bad(); break;			\
551	}								\
552	if (__err)							\
553		goto err_label;						\
554} while (0)
555
556#ifdef CONFIG_MMU
557extern unsigned long __must_check
558arm_copy_from_user(void *to, const void __user *from, unsigned long n);
559
560static inline unsigned long __must_check
561raw_copy_from_user(void *to, const void __user *from, unsigned long n)
562{
563	unsigned int __ua_flags;
564
565	__ua_flags = uaccess_save_and_enable();
566	n = arm_copy_from_user(to, from, n);
567	uaccess_restore(__ua_flags);
568	return n;
569}
570
571extern unsigned long __must_check
572arm_copy_to_user(void __user *to, const void *from, unsigned long n);
573extern unsigned long __must_check
574__copy_to_user_std(void __user *to, const void *from, unsigned long n);
575
576static inline unsigned long __must_check
577raw_copy_to_user(void __user *to, const void *from, unsigned long n)
578{
579#ifndef CONFIG_UACCESS_WITH_MEMCPY
580	unsigned int __ua_flags;
581	__ua_flags = uaccess_save_and_enable();
582	n = arm_copy_to_user(to, from, n);
583	uaccess_restore(__ua_flags);
584	return n;
585#else
586	return arm_copy_to_user(to, from, n);
587#endif
588}
589
590extern unsigned long __must_check
591arm_clear_user(void __user *addr, unsigned long n);
592extern unsigned long __must_check
593__clear_user_std(void __user *addr, unsigned long n);
594
595static inline unsigned long __must_check
596__clear_user(void __user *addr, unsigned long n)
597{
598	unsigned int __ua_flags = uaccess_save_and_enable();
599	n = arm_clear_user(addr, n);
600	uaccess_restore(__ua_flags);
601	return n;
602}
603
604#else
605static inline unsigned long
606raw_copy_from_user(void *to, const void __user *from, unsigned long n)
607{
608	memcpy(to, (const void __force *)from, n);
609	return 0;
610}
611static inline unsigned long
612raw_copy_to_user(void __user *to, const void *from, unsigned long n)
613{
614	memcpy((void __force *)to, from, n);
615	return 0;
616}
617#define __clear_user(addr, n)		(memset((void __force *)addr, 0, n), 0)
618#endif
619#define INLINE_COPY_TO_USER
620#define INLINE_COPY_FROM_USER
621
622static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
623{
624	if (access_ok(to, n))
625		n = __clear_user(to, n);
626	return n;
627}
628
629/* These are from lib/ code, and use __get_user() and friends */
630extern long strncpy_from_user(char *dest, const char __user *src, long count);
631
632extern __must_check long strnlen_user(const char __user *str, long n);
633
634#endif /* _ASMARM_UACCESS_H */