uaccess.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *  Standard user space access functions based on mvcp/mvcs and doing
  4 *  interesting things in the secondary space mode.
  5 *
  6 *    Copyright IBM Corp. 2006,2014
  7 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
  8 *		 Gerald Schaefer (gerald.schaefer@de.ibm.com)
  9 */
 10
 11#include <linux/kprobes.h>
 12#include <linux/uaccess.h>
 13#include <linux/export.h>
 14#include <linux/mm.h>
 15#include <asm/asm-extable.h>
 16#include <asm/ctlreg.h>
 17#include <asm/skey.h>
 18
 19#ifdef CONFIG_DEBUG_ENTRY
 20void debug_user_asce(int exit)
 21{
 22	struct lowcore *lc = get_lowcore();
 23	struct ctlreg cr1, cr7;
 24
 25	local_ctl_store(1, &cr1);
 26	local_ctl_store(7, &cr7);
 27	if (cr1.val == lc->user_asce.val && cr7.val == lc->user_asce.val)
 28		return;
 29	panic("incorrect ASCE on kernel %s\n"
 30	      "cr1:    %016lx cr7:  %016lx\n"
 31	      "kernel: %016lx user: %016lx\n",
 32	      exit ? "exit" : "entry", cr1.val, cr7.val,
 33	      lc->kernel_asce.val, lc->user_asce.val);
 34}
 35#endif /*CONFIG_DEBUG_ENTRY */
 36
 37union oac {
 38	unsigned int val;
 39	struct {
 40		struct {
 41			unsigned short key : 4;
 42			unsigned short	   : 4;
 43			unsigned short as  : 2;
 44			unsigned short	   : 4;
 45			unsigned short k   : 1;
 46			unsigned short a   : 1;
 47		} oac1;
 48		struct {
 49			unsigned short key : 4;
 50			unsigned short	   : 4;
 51			unsigned short as  : 2;
 52			unsigned short	   : 4;
 53			unsigned short k   : 1;
 54			unsigned short a   : 1;
 55		} oac2;
 56	};
 57};
 58
 59static uaccess_kmsan_or_inline __must_check unsigned long
 60raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
 61{
 62	unsigned long osize;
 63	union oac spec = {
 64		.oac2.key = key,
 65		.oac2.as = PSW_BITS_AS_SECONDARY,
 66		.oac2.k = 1,
 67		.oac2.a = 1,
 68	};
 69	int cc;
 70
 71	while (1) {
 72		osize = size;
 73		asm_inline volatile(
 74			"	lr	%%r0,%[spec]\n"
 75			"0:	mvcos	%[to],%[from],%[size]\n"
 76			"1:	nopr	%%r7\n"
 77			CC_IPM(cc)
 78			EX_TABLE_UA_MVCOS_FROM(0b, 0b)
 79			EX_TABLE_UA_MVCOS_FROM(1b, 0b)
 80			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
 81			: [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
 82			: CC_CLOBBER_LIST("memory", "0"));
 83		if (CC_TRANSFORM(cc) == 0)
 84			return osize - size;
 85		size -= 4096;
 86		to += 4096;
 87		from += 4096;
 88	}
 89}
 90
 91unsigned long _copy_from_user_key(void *to, const void __user *from,
 92				  unsigned long n, unsigned long key)
 93{
 94	unsigned long res = n;
 95
 96	might_fault();
 97	if (!should_fail_usercopy()) {
 98		instrument_copy_from_user_before(to, from, n);
 99		res = raw_copy_from_user_key(to, from, n, key);
100		instrument_copy_from_user_after(to, from, n, res);
101	}
102	if (unlikely(res))
103		memset(to + (n - res), 0, res);
104	return res;
105}
106EXPORT_SYMBOL(_copy_from_user_key);
107
108static uaccess_kmsan_or_inline __must_check unsigned long
109raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
110{
111	unsigned long osize;
112	union oac spec = {
113		.oac1.key = key,
114		.oac1.as = PSW_BITS_AS_SECONDARY,
115		.oac1.k = 1,
116		.oac1.a = 1,
117	};
118	int cc;
119
120	while (1) {
121		osize = size;
122		asm_inline volatile(
123			"	lr	%%r0,%[spec]\n"
124			"0:	mvcos	%[to],%[from],%[size]\n"
125			"1:	nopr	%%r7\n"
126			CC_IPM(cc)
127			EX_TABLE_UA_MVCOS_TO(0b, 0b)
128			EX_TABLE_UA_MVCOS_TO(1b, 0b)
129			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
130			: [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
131			: CC_CLOBBER_LIST("memory", "0"));
132		if (CC_TRANSFORM(cc) == 0)
133			return osize - size;
134		size -= 4096;
135		to += 4096;
136		from += 4096;
137	}
138}
139
140unsigned long _copy_to_user_key(void __user *to, const void *from,
141				unsigned long n, unsigned long key)
142{
143	might_fault();
144	if (should_fail_usercopy())
145		return n;
146	instrument_copy_to_user(to, from, n);
147	return raw_copy_to_user_key(to, from, n, key);
148}
149EXPORT_SYMBOL(_copy_to_user_key);
150
151#define CMPXCHG_USER_KEY_MAX_LOOPS 128
152
153static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsigned int *uval,
154						    unsigned int old, unsigned int new,
155						    unsigned int mask, unsigned long key)
156{
157	unsigned long count;
158	unsigned int prev;
159	bool sacf_flag;
160	int rc = 0;
161
162	skey_regions_initialize();
163	sacf_flag = enable_sacf_uaccess();
164	asm_inline volatile(
165		"20:	spka	0(%[key])\n"
166		"	sacf	256\n"
167		"	llill	%[count],%[max_loops]\n"
168		"0:	l	%[prev],%[address]\n"
169		"1:	nr	%[prev],%[mask]\n"
170		"	xilf	%[mask],0xffffffff\n"
171		"	or	%[new],%[prev]\n"
172		"	or	%[prev],%[tmp]\n"
173		"2:	lr	%[tmp],%[prev]\n"
174		"3:	cs	%[prev],%[new],%[address]\n"
175		"4:	jnl	5f\n"
176		"	xr	%[tmp],%[prev]\n"
177		"	xr	%[new],%[tmp]\n"
178		"	nr	%[tmp],%[mask]\n"
179		"	jnz	5f\n"
180		"	brct	%[count],2b\n"
181		"5:	sacf	768\n"
182		"	spka	%[default_key]\n"
183		"21:\n"
184		EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
185		EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
186		EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
187		EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
188		SKEY_REGION(20b, 21b)
189		: [rc] "+&d" (rc),
190		[prev] "=&d" (prev),
191		[address] "+Q" (*(int *)address),
192		[tmp] "+&d" (old),
193		[new] "+&d" (new),
194		[mask] "+&d" (mask),
195		[count] "=a" (count)
196		: [key] "%[count]" (key << 4),
197		[default_key] "J" (PAGE_DEFAULT_KEY),
198		[max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
199		: "memory", "cc");
200	disable_sacf_uaccess(sacf_flag);
201	*uval = prev;
202	if (!count)
203		rc = -EAGAIN;
204	return rc;
205}
206
207int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
208				  unsigned char old, unsigned char new, unsigned long key)
209{
210	unsigned int prev, shift, mask, _old, _new;
211	int rc;
212
213	shift = (3 ^ (address & 3)) << 3;
214	address ^= address & 3;
215	_old = (unsigned int)old << shift;
216	_new = (unsigned int)new << shift;
217	mask = ~(0xff << shift);
218	rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key);
219	*uval = prev >> shift;
220	return rc;
221}
222EXPORT_SYMBOL(__cmpxchg_user_key1);
223
224int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
225				  unsigned short old, unsigned short new, unsigned long key)
226{
227	unsigned int prev, shift, mask, _old, _new;
228	int rc;
229
230	shift = (2 ^ (address & 2)) << 3;
231	address ^= address & 2;
232	_old = (unsigned int)old << shift;
233	_new = (unsigned int)new << shift;
234	mask = ~(0xffff << shift);
235	rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key);
236	*uval = prev >> shift;
237	return rc;
238}
239EXPORT_SYMBOL(__cmpxchg_user_key2);
240
241int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval,
242				  unsigned int old, unsigned int new, unsigned long key)
243{
244	unsigned int prev = old;
245	bool sacf_flag;
246	int rc = 0;
247
248	skey_regions_initialize();
249	sacf_flag = enable_sacf_uaccess();
250	asm_inline volatile(
251		"20:	spka	0(%[key])\n"
252		"	sacf	256\n"
253		"0:	cs	%[prev],%[new],%[address]\n"
254		"1:	sacf	768\n"
255		"	spka	%[default_key]\n"
256		"21:\n"
257		EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
258		EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
259		SKEY_REGION(20b, 21b)
260		: [rc] "+&d" (rc),
261		[prev] "+&d" (prev),
262		[address] "+Q" (*(int *)address)
263		: [new] "d" (new),
264		[key] "a" (key << 4),
265		[default_key] "J" (PAGE_DEFAULT_KEY)
266		: "memory", "cc");
267	disable_sacf_uaccess(sacf_flag);
268	*uval = prev;
269	return rc;
270}
271EXPORT_SYMBOL(__cmpxchg_user_key4);
272
273int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval,
274				  unsigned long old, unsigned long new, unsigned long key)
275{
276	unsigned long prev = old;
277	bool sacf_flag;
278	int rc = 0;
279
280	skey_regions_initialize();
281	sacf_flag = enable_sacf_uaccess();
282	asm_inline volatile(
283		"20:	spka	0(%[key])\n"
284		"	sacf	256\n"
285		"0:	csg	%[prev],%[new],%[address]\n"
286		"1:	sacf	768\n"
287		"	spka	%[default_key]\n"
288		"21:\n"
289		EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
290		EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
291		SKEY_REGION(20b, 21b)
292		: [rc] "+&d" (rc),
293		[prev] "+&d" (prev),
294		[address] "+QS" (*(long *)address)
295		: [new] "d" (new),
296		[key] "a" (key << 4),
297		[default_key] "J" (PAGE_DEFAULT_KEY)
298		: "memory", "cc");
299	disable_sacf_uaccess(sacf_flag);
300	*uval = prev;
301	return rc;
302}
303EXPORT_SYMBOL(__cmpxchg_user_key8);
304
305int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval,
306				   __uint128_t old, __uint128_t new, unsigned long key)
307{
308	__uint128_t prev = old;
309	bool sacf_flag;
310	int rc = 0;
311
312	skey_regions_initialize();
313	sacf_flag = enable_sacf_uaccess();
314	asm_inline volatile(
315		"20:	spka	0(%[key])\n"
316		"	sacf	256\n"
317		"0:	cdsg	%[prev],%[new],%[address]\n"
318		"1:	sacf	768\n"
319		"	spka	%[default_key]\n"
320		"21:\n"
321		EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
322		EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
323		SKEY_REGION(20b, 21b)
324		: [rc] "+&d" (rc),
325		[prev] "+&d" (prev),
326		[address] "+QS" (*(__int128_t *)address)
327		: [new] "d" (new),
328		[key] "a" (key << 4),
329		[default_key] "J" (PAGE_DEFAULT_KEY)
330		: "memory", "cc");
331	disable_sacf_uaccess(sacf_flag);
332	*uval = prev;
333	return rc;
334}
335EXPORT_SYMBOL(__cmpxchg_user_key16);