maccess.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Access kernel or user memory without faulting.
  4 */
  5#include <linux/export.h>
  6#include <linux/mm.h>
  7#include <linux/uaccess.h>
  8#include <asm/tlb.h>
  9
 10bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
 11		size_t size)
 12{
 13	return true;
 14}
 15
 16/*
 17 * The below only uses kmsan_check_memory() to ensure uninitialized kernel
 18 * memory isn't leaked.
 19 */
 20#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)	\
 21	while (len >= sizeof(type)) {					\
 22		__get_kernel_nofault(dst, src, type, err_label);	\
 23		kmsan_check_memory(src, sizeof(type));			\
 24		dst += sizeof(type);					\
 25		src += sizeof(type);					\
 26		len -= sizeof(type);					\
 27	}
 28
 29long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
 30{
 31	unsigned long align = 0;
 32
 33	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
 34		align = (unsigned long)dst | (unsigned long)src;
 35
 36	if (!copy_from_kernel_nofault_allowed(src, size))
 37		return -ERANGE;
 38
 39	pagefault_disable();
 40	if (!(align & 7))
 41		copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
 42	if (!(align & 3))
 43		copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
 44	if (!(align & 1))
 45		copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
 46	copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
 47	pagefault_enable();
 48	return 0;
 49Efault:
 50	pagefault_enable();
 51	return -EFAULT;
 52}
 53EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
 54
 55#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)	\
 56	while (len >= sizeof(type)) {					\
 57		__put_kernel_nofault(dst, src, type, err_label);	\
 58		instrument_write(dst, sizeof(type));			\
 59		dst += sizeof(type);					\
 60		src += sizeof(type);					\
 61		len -= sizeof(type);					\
 62	}
 63
 64long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
 65{
 66	unsigned long align = 0;
 67
 68	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
 69		align = (unsigned long)dst | (unsigned long)src;
 70
 71	pagefault_disable();
 72	if (!(align & 7))
 73		copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
 74	if (!(align & 3))
 75		copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
 76	if (!(align & 1))
 77		copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
 78	copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
 79	pagefault_enable();
 80	return 0;
 81Efault:
 82	pagefault_enable();
 83	return -EFAULT;
 84}
 85
 86long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
 87{
 88	const void *src = unsafe_addr;
 89
 90	if (unlikely(count <= 0))
 91		return 0;
 92	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
 93		return -ERANGE;
 94
 95	pagefault_disable();
 96	do {
 97		__get_kernel_nofault(dst, src, u8, Efault);
 98		dst++;
 99		src++;
100	} while (dst[-1] && src - unsafe_addr < count);
101	pagefault_enable();
102
103	dst[-1] = '\0';
104	return src - unsafe_addr;
105Efault:
106	pagefault_enable();
107	dst[0] = '\0';
108	return -EFAULT;
109}
110
111/**
112 * copy_from_user_nofault(): safely attempt to read from a user-space location
113 * @dst: pointer to the buffer that shall take the data
114 * @src: address to read from. This must be a user address.
115 * @size: size of the data chunk
116 *
117 * Safely read from user address @src to the buffer at @dst. If a kernel fault
118 * happens, handle that and return -EFAULT.
119 */
120long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
121{
122	long ret = -EFAULT;
123
124	if (!__access_ok(src, size))
125		return ret;
126
127	if (!nmi_uaccess_okay())
128		return ret;
129
130	pagefault_disable();
131	ret = __copy_from_user_inatomic(dst, src, size);
132	pagefault_enable();
133
134	if (ret)
135		return -EFAULT;
136	return 0;
137}
138EXPORT_SYMBOL_GPL(copy_from_user_nofault);
139
140/**
141 * copy_to_user_nofault(): safely attempt to write to a user-space location
142 * @dst: address to write to
143 * @src: pointer to the data that shall be written
144 * @size: size of the data chunk
145 *
146 * Safely write to address @dst from the buffer at @src.  If a kernel fault
147 * happens, handle that and return -EFAULT.
148 */
149long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
150{
151	long ret = -EFAULT;
152
153	if (access_ok(dst, size)) {
154		pagefault_disable();
155		ret = __copy_to_user_inatomic(dst, src, size);
156		pagefault_enable();
157	}
158
159	if (ret)
160		return -EFAULT;
161	return 0;
162}
163EXPORT_SYMBOL_GPL(copy_to_user_nofault);
164
165/**
166 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
167 *				address.
168 * @dst:   Destination address, in kernel space.  This buffer must be at
169 *         least @count bytes long.
170 * @unsafe_addr: Unsafe user address.
171 * @count: Maximum number of bytes to copy, including the trailing NUL.
172 *
173 * Copies a NUL-terminated string from unsafe user address to kernel buffer.
174 *
175 * On success, returns the length of the string INCLUDING the trailing NUL.
176 *
177 * If access fails, returns -EFAULT (some data may have been copied
178 * and the trailing NUL added).
179 *
180 * If @count is smaller than the length of the string, copies @count-1 bytes,
181 * sets the last byte of @dst buffer to NUL and returns @count.
182 */
183long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
184			      long count)
185{
186	long ret;
187
188	if (unlikely(count <= 0))
189		return 0;
190
191	pagefault_disable();
192	ret = strncpy_from_user(dst, unsafe_addr, count);
193	pagefault_enable();
194
195	if (ret >= count) {
196		ret = count;
197		dst[ret - 1] = '\0';
198	} else if (ret >= 0) {
199		ret++;
200	}
201
202	return ret;
203}
204
205/**
206 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
207 * @unsafe_addr: The string to measure.
208 * @count: Maximum count (including NUL)
209 *
210 * Get the size of a NUL-terminated string in user space without pagefault.
211 *
212 * Returns the size of the string INCLUDING the terminating NUL.
213 *
214 * If the string is too long, returns a number larger than @count. User
215 * has to check the return value against "> count".
216 * On exception (or invalid count), returns 0.
217 *
218 * Unlike strnlen_user, this can be used from IRQ handler etc. because
219 * it disables pagefaults.
220 */
221long strnlen_user_nofault(const void __user *unsafe_addr, long count)
222{
223	int ret;
224
225	pagefault_disable();
226	ret = strnlen_user(unsafe_addr, count);
227	pagefault_enable();
228
229	return ret;
230}
231
232void __copy_overflow(int size, unsigned long count)
233{
234	WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
235}
236EXPORT_SYMBOL(__copy_overflow);