uaccess.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef __LINUX_UACCESS_H__
  3#define __LINUX_UACCESS_H__
  4
  5#include <linux/cleanup.h>
  6#include <linux/fault-inject-usercopy.h>
  7#include <linux/instrumented.h>
  8#include <linux/minmax.h>
  9#include <linux/nospec.h>
 10#include <linux/sched.h>
 11#include <linux/ucopysize.h>
 12
 13#include <asm/uaccess.h>
 14
 15/*
 16 * Architectures that support memory tagging (assigning tags to memory regions,
 17 * embedding these tags into addresses that point to these memory regions, and
 18 * checking that the memory and the pointer tags match on memory accesses)
 19 * redefine this macro to strip tags from pointers.
 20 *
 21 * Passing down mm_struct allows to define untagging rules on per-process
 22 * basis.
 23 *
 24 * It's defined as noop for architectures that don't support memory tagging.
 25 */
 26#ifndef untagged_addr
 27#define untagged_addr(addr) (addr)
 28#endif
 29
 30#ifndef untagged_addr_remote
 31#define untagged_addr_remote(mm, addr)	({		\
 32	mmap_assert_locked(mm);				\
 33	untagged_addr(addr);				\
 34})
 35#endif
 36
 37#ifdef masked_user_access_begin
 38 #define can_do_masked_user_access() 1
 39# ifndef masked_user_write_access_begin
 40#  define masked_user_write_access_begin masked_user_access_begin
 41# endif
 42# ifndef masked_user_read_access_begin
 43#  define masked_user_read_access_begin masked_user_access_begin
 44#endif
 45#else
 46 #define can_do_masked_user_access() 0
 47 #define masked_user_access_begin(src) NULL
 48 #define masked_user_read_access_begin(src) NULL
 49 #define masked_user_write_access_begin(src) NULL
 50 #define mask_user_address(src) (src)
 51#endif
 52
 53/*
 54 * Architectures should provide two primitives (raw_copy_{to,from}_user())
 55 * and get rid of their private instances of copy_{to,from}_user() and
 56 * __copy_{to,from}_user{,_inatomic}().
 57 *
 58 * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and
 59 * return the amount left to copy.  They should assume that access_ok() has
 60 * already been checked (and succeeded); they should *not* zero-pad anything.
 61 * No KASAN or object size checks either - those belong here.
 62 *
 63 * Both of these functions should attempt to copy size bytes starting at from
 64 * into the area starting at to.  They must not fetch or store anything
 65 * outside of those areas.  Return value must be between 0 (everything
 66 * copied successfully) and size (nothing copied).
 67 *
 68 * If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting
 69 * at to must become equal to the bytes fetched from the corresponding area
 70 * starting at from.  All data past to + size - N must be left unmodified.
 71 *
 72 * If copying succeeds, the return value must be 0.  If some data cannot be
 73 * fetched, it is permitted to copy less than had been fetched; the only
 74 * hard requirement is that not storing anything at all (i.e. returning size)
 75 * should happen only when nothing could be copied.  In other words, you don't
 76 * have to squeeze as much as possible - it is allowed, but not necessary.
 77 *
 78 * For raw_copy_from_user() to always points to kernel memory and no faults
 79 * on store should happen.  Interpretation of from is affected by set_fs().
 80 * For raw_copy_to_user() it's the other way round.
 81 *
 82 * Both can be inlined - it's up to architectures whether it wants to bother
 83 * with that.  They should not be used directly; they are used to implement
 84 * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic())
 85 * that are used instead.  Out of those, __... ones are inlined.  Plain
 86 * copy_{to,from}_user() might or might not be inlined.  If you want them
 87 * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER.
 88 *
 89 * NOTE: only copy_from_user() zero-pads the destination in case of short copy.
 90 * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything
 91 * at all; their callers absolutely must check the return value.
 92 *
 93 * Biarch ones should also provide raw_copy_in_user() - similar to the above,
 94 * but both source and destination are __user pointers (affected by set_fs()
 95 * as usual) and both source and destination can trigger faults.
 96 */
 97
 98static __always_inline __must_check unsigned long
 99__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
100{
101	unsigned long res;
102
103	instrument_copy_from_user_before(to, from, n);
104	check_object_size(to, n, false);
105	res = raw_copy_from_user(to, from, n);
106	instrument_copy_from_user_after(to, from, n, res);
107	return res;
108}
109
110static __always_inline __must_check unsigned long
111__copy_from_user(void *to, const void __user *from, unsigned long n)
112{
113	unsigned long res;
114
115	might_fault();
116	instrument_copy_from_user_before(to, from, n);
117	if (should_fail_usercopy())
118		return n;
119	check_object_size(to, n, false);
120	res = raw_copy_from_user(to, from, n);
121	instrument_copy_from_user_after(to, from, n, res);
122	return res;
123}
124
125/**
126 * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
127 * @to:   Destination address, in user space.
128 * @from: Source address, in kernel space.
129 * @n:    Number of bytes to copy.
130 *
131 * Context: User context only.
132 *
133 * Copy data from kernel space to user space.  Caller must check
134 * the specified block with access_ok() before calling this function.
135 * The caller should also make sure he pins the user space address
136 * so that we don't result in page fault and sleep.
137 */
138static __always_inline __must_check unsigned long
139__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
140{
141	if (should_fail_usercopy())
142		return n;
143	instrument_copy_to_user(to, from, n);
144	check_object_size(from, n, true);
145	return raw_copy_to_user(to, from, n);
146}
147
148static __always_inline __must_check unsigned long
149__copy_to_user(void __user *to, const void *from, unsigned long n)
150{
151	might_fault();
152	if (should_fail_usercopy())
153		return n;
154	instrument_copy_to_user(to, from, n);
155	check_object_size(from, n, true);
156	return raw_copy_to_user(to, from, n);
157}
158
159/*
160 * Architectures that #define INLINE_COPY_TO_USER use this function
161 * directly in the normal copy_to/from_user(), the other ones go
162 * through an extern _copy_to/from_user(), which expands the same code
163 * here.
164 */
165static inline __must_check unsigned long
166_inline_copy_from_user(void *to, const void __user *from, unsigned long n)
167{
168	unsigned long res = n;
169	might_fault();
170	if (should_fail_usercopy())
171		goto fail;
172	if (can_do_masked_user_access())
173		from = mask_user_address(from);
174	else {
175		if (!access_ok(from, n))
176			goto fail;
177		/*
178		 * Ensure that bad access_ok() speculation will not
179		 * lead to nasty side effects *after* the copy is
180		 * finished:
181		 */
182		barrier_nospec();
183	}
184	instrument_copy_from_user_before(to, from, n);
185	res = raw_copy_from_user(to, from, n);
186	instrument_copy_from_user_after(to, from, n, res);
187	if (likely(!res))
188		return 0;
189fail:
190	memset(to + (n - res), 0, res);
191	return res;
192}
193#ifndef INLINE_COPY_FROM_USER
194extern __must_check unsigned long
195_copy_from_user(void *, const void __user *, unsigned long);
196#endif
197
198static inline __must_check unsigned long
199_inline_copy_to_user(void __user *to, const void *from, unsigned long n)
200{
201	might_fault();
202	if (should_fail_usercopy())
203		return n;
204	if (access_ok(to, n)) {
205		instrument_copy_to_user(to, from, n);
206		n = raw_copy_to_user(to, from, n);
207	}
208	return n;
209}
210#ifndef INLINE_COPY_TO_USER
211extern __must_check unsigned long
212_copy_to_user(void __user *, const void *, unsigned long);
213#endif
214
215static __always_inline unsigned long __must_check
216copy_from_user(void *to, const void __user *from, unsigned long n)
217{
218	if (!check_copy_size(to, n, false))
219		return n;
220#ifdef INLINE_COPY_FROM_USER
221	return _inline_copy_from_user(to, from, n);
222#else
223	return _copy_from_user(to, from, n);
224#endif
225}
226
227static __always_inline unsigned long __must_check
228copy_to_user(void __user *to, const void *from, unsigned long n)
229{
230	if (!check_copy_size(from, n, true))
231		return n;
232
233#ifdef INLINE_COPY_TO_USER
234	return _inline_copy_to_user(to, from, n);
235#else
236	return _copy_to_user(to, from, n);
237#endif
238}
239
240#ifndef copy_mc_to_kernel
241/*
242 * Without arch opt-in this generic copy_mc_to_kernel() will not handle
243 * #MC (or arch equivalent) during source read.
244 */
245static inline unsigned long __must_check
246copy_mc_to_kernel(void *dst, const void *src, size_t cnt)
247{
248	memcpy(dst, src, cnt);
249	return 0;
250}
251#endif
252
253static __always_inline void pagefault_disabled_inc(void)
254{
255	current->pagefault_disabled++;
256}
257
258static __always_inline void pagefault_disabled_dec(void)
259{
260	current->pagefault_disabled--;
261}
262
263/*
264 * These routines enable/disable the pagefault handler. If disabled, it will
265 * not take any locks and go straight to the fixup table.
266 *
267 * User access methods will not sleep when called from a pagefault_disabled()
268 * environment.
269 */
270static inline void pagefault_disable(void)
271{
272	pagefault_disabled_inc();
273	/*
274	 * make sure to have issued the store before a pagefault
275	 * can hit.
276	 */
277	barrier();
278}
279
280static inline void pagefault_enable(void)
281{
282	/*
283	 * make sure to issue those last loads/stores before enabling
284	 * the pagefault handler again.
285	 */
286	barrier();
287	pagefault_disabled_dec();
288}
289
290/*
291 * Is the pagefault handler disabled? If so, user access methods will not sleep.
292 */
293static inline bool pagefault_disabled(void)
294{
295	return current->pagefault_disabled != 0;
296}
297
298/*
299 * The pagefault handler is in general disabled by pagefault_disable() or
300 * when in irq context (via in_atomic()).
301 *
302 * This function should only be used by the fault handlers. Other users should
303 * stick to pagefault_disabled().
304 * Please NEVER use preempt_disable() to disable the fault handler. With
305 * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled.
306 * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT.
307 */
308#define faulthandler_disabled() (pagefault_disabled() || in_atomic())
309
310DEFINE_LOCK_GUARD_0(pagefault, pagefault_disable(), pagefault_enable())
311
312#ifndef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
313
314/**
315 * probe_subpage_writeable: probe the user range for write faults at sub-page
316 *			    granularity (e.g. arm64 MTE)
317 * @uaddr: start of address range
318 * @size: size of address range
319 *
320 * Returns 0 on success, the number of bytes not probed on fault.
321 *
322 * It is expected that the caller checked for the write permission of each
323 * page in the range either by put_user() or GUP. The architecture port can
324 * implement a more efficient get_user() probing if the same sub-page faults
325 * are triggered by either a read or a write.
326 */
327static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size)
328{
329	return 0;
330}
331
332#endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
333
334#ifndef ARCH_HAS_NONTEMPORAL_UACCESS
335
336static inline __must_check unsigned long
337copy_from_user_inatomic_nontemporal(void *to, const void __user *from,
338				  unsigned long n)
339{
340	if (can_do_masked_user_access())
341		from = mask_user_address(from);
342	else
343		if (!access_ok(from, n))
344			return n;
345	return __copy_from_user_inatomic(to, from, n);
346}
347
348#endif		/* ARCH_HAS_NONTEMPORAL_UACCESS */
349
350extern __must_check int check_zeroed_user(const void __user *from, size_t size);
351
352/**
353 * copy_struct_from_user: copy a struct from userspace
354 * @dst:   Destination address, in kernel space. This buffer must be @ksize
355 *         bytes long.
356 * @ksize: Size of @dst struct.
357 * @src:   Source address, in userspace.
358 * @usize: (Alleged) size of @src struct.
359 *
360 * Copies a struct from userspace to kernel space, in a way that guarantees
361 * backwards-compatibility for struct syscall arguments (as long as future
362 * struct extensions are made such that all new fields are *appended* to the
363 * old struct, and zeroed-out new fields have the same meaning as the old
364 * struct).
365 *
366 * @ksize is just sizeof(*dst), and @usize should've been passed by userspace.
367 * The recommended usage is something like the following:
368 *
369 *   SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize)
370 *   {
371 *      int err;
372 *      struct foo karg = {};
373 *
374 *      if (usize > PAGE_SIZE)
375 *        return -E2BIG;
376 *      if (usize < FOO_SIZE_VER0)
377 *        return -EINVAL;
378 *
379 *      err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize);
380 *      if (err)
381 *        return err;
382 *
383 *      // ...
384 *   }
385 *
386 * There are three cases to consider:
387 *  * If @usize == @ksize, then it's copied verbatim.
388 *  * If @usize < @ksize, then the userspace has passed an old struct to a
389 *    newer kernel. The rest of the trailing bytes in @dst (@ksize - @usize)
390 *    are to be zero-filled.
391 *  * If @usize > @ksize, then the userspace has passed a new struct to an
392 *    older kernel. The trailing bytes unknown to the kernel (@usize - @ksize)
393 *    are checked to ensure they are zeroed, otherwise -E2BIG is returned.
394 *
395 * Returns (in all cases, some data may have been copied):
396 *  * -E2BIG:  (@usize > @ksize) and there are non-zero trailing bytes in @src.
397 *  * -EFAULT: access to userspace failed.
398 */
399static __always_inline __must_check int
400copy_struct_from_user(void *dst, size_t ksize, const void __user *src,
401		      size_t usize)
402{
403	size_t size = min(ksize, usize);
404	size_t rest = max(ksize, usize) - size;
405
406	/* Double check if ksize is larger than a known object size. */
407	if (WARN_ON_ONCE(ksize > __builtin_object_size(dst, 1)))
408		return -E2BIG;
409
410	/* Deal with trailing bytes. */
411	if (usize < ksize) {
412		memset(dst + size, 0, rest);
413	} else if (usize > ksize) {
414		int ret = check_zeroed_user(src + size, rest);
415		if (ret <= 0)
416			return ret ?: -E2BIG;
417	}
418	/* Copy the interoperable parts of the struct. */
419	if (copy_from_user(dst, src, size))
420		return -EFAULT;
421	return 0;
422}
423
424/**
425 * copy_struct_to_user: copy a struct to userspace
426 * @dst:   Destination address, in userspace. This buffer must be @ksize
427 *         bytes long.
428 * @usize: (Alleged) size of @dst struct.
429 * @src:   Source address, in kernel space.
430 * @ksize: Size of @src struct.
431 * @ignored_trailing: Set to %true if there was a non-zero byte in @src that
432 * userspace cannot see because they are using an smaller struct.
433 *
434 * Copies a struct from kernel space to userspace, in a way that guarantees
435 * backwards-compatibility for struct syscall arguments (as long as future
436 * struct extensions are made such that all new fields are *appended* to the
437 * old struct, and zeroed-out new fields have the same meaning as the old
438 * struct).
439 *
440 * Some syscalls may wish to make sure that userspace knows about everything in
441 * the struct, and if there is a non-zero value that userspce doesn't know
442 * about, they want to return an error (such as -EMSGSIZE) or have some other
443 * fallback (such as adding a "you're missing some information" flag). If
444 * @ignored_trailing is non-%NULL, it will be set to %true if there was a
445 * non-zero byte that could not be copied to userspace (ie. was past @usize).
446 *
447 * While unconditionally returning an error in this case is the simplest
448 * solution, for maximum backward compatibility you should try to only return
449 * -EMSGSIZE if the user explicitly requested the data that couldn't be copied.
450 * Note that structure sizes can change due to header changes and simple
451 * recompilations without code changes(!), so if you care about
452 * @ignored_trailing you probably want to make sure that any new field data is
453 * associated with a flag. Otherwise you might assume that a program knows
454 * about data it does not.
455 *
456 * @ksize is just sizeof(*src), and @usize should've been passed by userspace.
457 * The recommended usage is something like the following:
458 *
459 *   SYSCALL_DEFINE2(foobar, struct foo __user *, uarg, size_t, usize)
460 *   {
461 *      int err;
462 *      bool ignored_trailing;
463 *      struct foo karg = {};
464 *
465 *      if (usize > PAGE_SIZE)
466 *		return -E2BIG;
467 *      if (usize < FOO_SIZE_VER0)
468 *		return -EINVAL;
469 *
470 *      // ... modify karg somehow ...
471 *
472 *      err = copy_struct_to_user(uarg, usize, &karg, sizeof(karg),
473 *				  &ignored_trailing);
474 *      if (err)
475 *		return err;
476 *      if (ignored_trailing)
477 *		return -EMSGSIZE:
478 *
479 *      // ...
480 *   }
481 *
482 * There are three cases to consider:
483 *  * If @usize == @ksize, then it's copied verbatim.
484 *  * If @usize < @ksize, then the kernel is trying to pass userspace a newer
485 *    struct than it supports. Thus we only copy the interoperable portions
486 *    (@usize) and ignore the rest (but @ignored_trailing is set to %true if
487 *    any of the trailing (@ksize - @usize) bytes are non-zero).
488 *  * If @usize > @ksize, then the kernel is trying to pass userspace an older
489 *    struct than userspace supports. In order to make sure the
490 *    unknown-to-the-kernel fields don't contain garbage values, we zero the
491 *    trailing (@usize - @ksize) bytes.
492 *
493 * Returns (in all cases, some data may have been copied):
494 *  * -EFAULT: access to userspace failed.
495 */
496static __always_inline __must_check int
497copy_struct_to_user(void __user *dst, size_t usize, const void *src,
498		    size_t ksize, bool *ignored_trailing)
499{
500	size_t size = min(ksize, usize);
501	size_t rest = max(ksize, usize) - size;
502
503	/* Double check if ksize is larger than a known object size. */
504	if (WARN_ON_ONCE(ksize > __builtin_object_size(src, 1)))
505		return -E2BIG;
506
507	/* Deal with trailing bytes. */
508	if (usize > ksize) {
509		if (clear_user(dst + size, rest))
510			return -EFAULT;
511	}
512	if (ignored_trailing)
513		*ignored_trailing = ksize < usize &&
514			memchr_inv(src + size, 0, rest) != NULL;
515	/* Copy the interoperable parts of the struct. */
516	if (copy_to_user(dst, src, size))
517		return -EFAULT;
518	return 0;
519}
520
521bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size);
522
523long copy_from_kernel_nofault(void *dst, const void *src, size_t size);
524long notrace copy_to_kernel_nofault(void *dst, const void *src, size_t size);
525
526long copy_from_user_nofault(void *dst, const void __user *src, size_t size);
527long notrace copy_to_user_nofault(void __user *dst, const void *src,
528		size_t size);
529
530long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr,
531		long count);
532
533long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
534		long count);
535long strnlen_user_nofault(const void __user *unsafe_addr, long count);
536
537#ifdef arch_get_kernel_nofault
538/*
539 * Wrap the architecture implementation so that @label can be outside of a
540 * cleanup() scope. A regular C goto works correctly, but ASM goto does
541 * not. Clang rejects such an attempt, but GCC silently emits buggy code.
542 */
543#define __get_kernel_nofault(dst, src, type, label)		\
544do {								\
545	__label__ local_label;					\
546	arch_get_kernel_nofault(dst, src, type, local_label);	\
547	if (0) {						\
548	local_label:						\
549		goto label;					\
550	}							\
551} while (0)
552
553#define __put_kernel_nofault(dst, src, type, label)		\
554do {								\
555	__label__ local_label;					\
556	arch_put_kernel_nofault(dst, src, type, local_label);	\
557	if (0) {						\
558	local_label:						\
559		goto label;					\
560	}							\
561} while (0)
562
563#elif !defined(__get_kernel_nofault) /* arch_get_kernel_nofault */
564
565#define __get_kernel_nofault(dst, src, type, label)	\
566do {							\
567	type __user *p = (type __force __user *)(src);	\
568	type data;					\
569	if (__get_user(data, p))			\
570		goto label;				\
571	*(type *)dst = data;				\
572} while (0)
573
574#define __put_kernel_nofault(dst, src, type, label)	\
575do {							\
576	type __user *p = (type __force __user *)(dst);	\
577	type data = *(type *)src;			\
578	if (__put_user(data, p))			\
579		goto label;				\
580} while (0)
581
582#endif  /* !__get_kernel_nofault */
583
584/**
585 * get_kernel_nofault(): safely attempt to read from a location
586 * @val: read into this variable
587 * @ptr: address to read from
588 *
589 * Returns 0 on success, or -EFAULT.
590 */
591#define get_kernel_nofault(val, ptr) ({				\
592	const typeof(val) *__gk_ptr = (ptr);			\
593	copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\
594})
595
596#ifdef user_access_begin
597
598#ifdef arch_unsafe_get_user
599/*
600 * Wrap the architecture implementation so that @label can be outside of a
601 * cleanup() scope. A regular C goto works correctly, but ASM goto does
602 * not. Clang rejects such an attempt, but GCC silently emits buggy code.
603 *
604 * Some architectures use internal local labels already, but this extra
605 * indirection here is harmless because the compiler optimizes it out
606 * completely in any case. This construct just ensures that the ASM GOTO
607 * target is always in the local scope. The C goto 'label' works correctly
608 * when leaving a cleanup() scope.
609 */
610#define unsafe_get_user(x, ptr, label)			\
611do {							\
612	__label__ local_label;				\
613	arch_unsafe_get_user(x, ptr, local_label);	\
614	if (0) {					\
615	local_label:					\
616		goto label;				\
617	}						\
618} while (0)
619
620#define unsafe_put_user(x, ptr, label)			\
621do {							\
622	__label__ local_label;				\
623	arch_unsafe_put_user(x, ptr, local_label);	\
624	if (0) {					\
625	local_label:					\
626		goto label;				\
627	}						\
628} while (0)
629#endif /* arch_unsafe_get_user */
630
631#else /* user_access_begin */
632#define user_access_begin(ptr,len) access_ok(ptr, len)
633#define user_access_end() do { } while (0)
634#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
635#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)
636#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)
637#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)
638#define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)
639static inline unsigned long user_access_save(void) { return 0UL; }
640static inline void user_access_restore(unsigned long flags) { }
641#endif /* !user_access_begin */
642
643#ifndef user_write_access_begin
644#define user_write_access_begin user_access_begin
645#define user_write_access_end user_access_end
646#endif
647#ifndef user_read_access_begin
648#define user_read_access_begin user_access_begin
649#define user_read_access_end user_access_end
650#endif
651
652/* Define RW variant so the below _mode macro expansion works */
653#define masked_user_rw_access_begin(u)	masked_user_access_begin(u)
654#define user_rw_access_begin(u, s)	user_access_begin(u, s)
655
656/* Scoped user access */
657
658/* Cleanup wrapper functions */
659static __always_inline void __scoped_user_read_access_end(const void *p)
660{
661	user_read_access_end();
662};
663static __always_inline void __scoped_user_write_access_end(const void *p)
664{
665	user_write_access_end();
666};
667static __always_inline void __scoped_user_rw_access_end(const void *p)
668{
669	user_access_end();
670};
671
672/**
673 * __scoped_user_access_begin - Start a scoped user access
674 * @mode:	The mode of the access class (read, write, rw)
675 * @uptr:	The pointer to access user space memory
676 * @size:	Size of the access
677 * @elbl:	Error label to goto when the access region is rejected
678 *
679 * Internal helper for __scoped_user_access(). Don't use directly.
680 */
681#define __scoped_user_access_begin(mode, uptr, size, elbl)		\
682({									\
683	typeof(uptr) __retptr;						\
684									\
685	if (can_do_masked_user_access()) {				\
686		__retptr = masked_user_##mode##_access_begin(uptr);	\
687	} else {							\
688		__retptr = uptr;					\
689		if (!user_##mode##_access_begin(uptr, size))		\
690			goto elbl;					\
691	}								\
692	__retptr;							\
693})
694
695/**
696 * __scoped_user_access - Open a scope for user access
697 * @mode:	The mode of the access class (read, write, rw)
698 * @uptr:	The pointer to access user space memory
699 * @size:	Size of the access
700 * @elbl:	Error label to goto when the access region is rejected. It
701 *		must be placed outside the scope
702 *
703 * If the user access function inside the scope requires a fault label, it
704 * can use @elbl or a different label outside the scope, which requires
705 * that user access which is implemented with ASM GOTO has been properly
706 * wrapped. See unsafe_get_user() for reference.
707 *
708 *	scoped_user_rw_access(ptr, efault) {
709 *		unsafe_get_user(rval, &ptr->rval, efault);
710 *		unsafe_put_user(wval, &ptr->wval, efault);
711 *	}
712 *	return 0;
713 *  efault:
714 *	return -EFAULT;
715 *
716 * The scope is internally implemented as a autoterminating nested for()
717 * loop, which can be left with 'return', 'break' and 'goto' at any
718 * point.
719 *
720 * When the scope is left user_##@_mode##_access_end() is automatically
721 * invoked.
722 *
723 * When the architecture supports masked user access and the access region
724 * which is determined by @uptr and @size is not a valid user space
725 * address, i.e. < TASK_SIZE, the scope sets the pointer to a faulting user
726 * space address and does not terminate early. This optimizes for the good
727 * case and lets the performance uncritical bad case go through the fault.
728 *
729 * The eventual modification of the pointer is limited to the scope.
730 * Outside of the scope the original pointer value is unmodified, so that
731 * the original pointer value is available for diagnostic purposes in an
732 * out of scope fault path.
733 *
734 * Nesting scoped user access into a user access scope is invalid and fails
735 * the build. Nesting into other guards, e.g. pagefault is safe.
736 *
737 * The masked variant does not check the size of the access and relies on a
738 * mapping hole (e.g. guard page) to catch an out of range pointer, the
739 * first access to user memory inside the scope has to be within
740 * @uptr ... @uptr + PAGE_SIZE - 1
741 *
742 * Don't use directly. Use scoped_masked_user_$MODE_access() instead.
743 */
744#define __scoped_user_access(mode, uptr, size, elbl)				\
745for (bool done = false; !done; done = true)					\
746	for (auto _tmpptr = __scoped_user_access_begin(mode, uptr, size, elbl);	\
747	     !done; done = true)						\
748		/* Force modified pointer usage within the scope */		\
749		for (const auto uptr  __cleanup(__scoped_user_##mode##_access_end) = \
750		     _tmpptr; !done; done = true)
751
752/**
753 * scoped_user_read_access_size - Start a scoped user read access with given size
754 * @usrc:	Pointer to the user space address to read from
755 * @size:	Size of the access starting from @usrc
756 * @elbl:	Error label to goto when the access region is rejected
757 *
758 * For further information see __scoped_user_access() above.
759 */
760#define scoped_user_read_access_size(usrc, size, elbl)		\
761	__scoped_user_access(read, usrc, size, elbl)
762
763/**
764 * scoped_user_read_access - Start a scoped user read access
765 * @usrc:	Pointer to the user space address to read from
766 * @elbl:	Error label to goto when the access region is rejected
767 *
768 * The size of the access starting from @usrc is determined via sizeof(*@usrc)).
769 *
770 * For further information see __scoped_user_access() above.
771 */
772#define scoped_user_read_access(usrc, elbl)				\
773	scoped_user_read_access_size(usrc, sizeof(*(usrc)), elbl)
774
775/**
776 * scoped_user_write_access_size - Start a scoped user write access with given size
777 * @udst:	Pointer to the user space address to write to
778 * @size:	Size of the access starting from @udst
779 * @elbl:	Error label to goto when the access region is rejected
780 *
781 * For further information see __scoped_user_access() above.
782 */
783#define scoped_user_write_access_size(udst, size, elbl)			\
784	__scoped_user_access(write, udst, size, elbl)
785
786/**
787 * scoped_user_write_access - Start a scoped user write access
788 * @udst:	Pointer to the user space address to write to
789 * @elbl:	Error label to goto when the access region is rejected
790 *
791 * The size of the access starting from @udst is determined via sizeof(*@udst)).
792 *
793 * For further information see __scoped_user_access() above.
794 */
795#define scoped_user_write_access(udst, elbl)				\
796	scoped_user_write_access_size(udst, sizeof(*(udst)), elbl)
797
798/**
799 * scoped_user_rw_access_size - Start a scoped user read/write access with given size
800 * @uptr	Pointer to the user space address to read from and write to
801 * @size:	Size of the access starting from @uptr
802 * @elbl:	Error label to goto when the access region is rejected
803 *
804 * For further information see __scoped_user_access() above.
805 */
806#define scoped_user_rw_access_size(uptr, size, elbl)			\
807	__scoped_user_access(rw, uptr, size, elbl)
808
809/**
810 * scoped_user_rw_access - Start a scoped user read/write access
811 * @uptr	Pointer to the user space address to read from and write to
812 * @elbl:	Error label to goto when the access region is rejected
813 *
814 * The size of the access starting from @uptr is determined via sizeof(*@uptr)).
815 *
816 * For further information see __scoped_user_access() above.
817 */
818#define scoped_user_rw_access(uptr, elbl)				\
819	scoped_user_rw_access_size(uptr, sizeof(*(uptr)), elbl)
820
821/**
822 * get_user_inline - Read user data inlined
823 * @val:	The variable to store the value read from user memory
824 * @usrc:	Pointer to the user space memory to read from
825 *
826 * Return: 0 if successful, -EFAULT when faulted
827 *
828 * Inlined variant of get_user(). Only use when there is a demonstrable
829 * performance reason.
830 */
831#define get_user_inline(val, usrc)				\
832({								\
833	__label__ efault;					\
834	typeof(usrc) _tmpsrc = usrc;				\
835	int _ret = 0;						\
836								\
837	scoped_user_read_access(_tmpsrc, efault)		\
838		unsafe_get_user(val, _tmpsrc, efault);		\
839	if (0) {						\
840	efault:							\
841		_ret = -EFAULT;					\
842	}							\
843	_ret;							\
844})
845
846/**
847 * put_user_inline - Write to user memory inlined
848 * @val:	The value to write
849 * @udst:	Pointer to the user space memory to write to
850 *
851 * Return: 0 if successful, -EFAULT when faulted
852 *
853 * Inlined variant of put_user(). Only use when there is a demonstrable
854 * performance reason.
855 */
856#define put_user_inline(val, udst)				\
857({								\
858	__label__ efault;					\
859	typeof(udst) _tmpdst = udst;				\
860	int _ret = 0;						\
861								\
862	scoped_user_write_access(_tmpdst, efault)		\
863		unsafe_put_user(val, _tmpdst, efault);		\
864	if (0) {						\
865	efault:							\
866		_ret = -EFAULT;					\
867	}							\
868	_ret;							\
869})
870
871#ifdef CONFIG_HARDENED_USERCOPY
872void __noreturn usercopy_abort(const char *name, const char *detail,
873			       bool to_user, unsigned long offset,
874			       unsigned long len);
875#endif
876
877#endif		/* __LINUX_UACCESS_H__ */