futex.h

 1/* SPDX-License-Identifier: GPL-2.0 */
 2#ifndef _ASM_X86_FUTEX_H
 3#define _ASM_X86_FUTEX_H
 4
 5#ifdef __KERNEL__
 6
 7#include <linux/futex.h>
 8#include <linux/uaccess.h>
 9
10#include <asm/asm.h>
11#include <asm/errno.h>
12#include <asm/processor.h>
13#include <asm/smap.h>
14
15#define unsafe_atomic_op1(insn, oval, uaddr, oparg, label)	\
16do {								\
17	int oldval = 0, ret;					\
18	asm volatile("1:\t" insn "\n"				\
19		     "2:\n"					\
20		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %1) \
21		     : "=r" (oldval), "=r" (ret), "+m" (*uaddr)	\
22		     : "0" (oparg), "1" (0));	\
23	if (ret)						\
24		goto label;					\
25	*oval = oldval;						\
26} while(0)
27
28
29#define unsafe_atomic_op2(insn, oval, uaddr, oparg, label)	\
30do {								\
31	int oldval = 0, ret, tem;				\
32	asm volatile("1:\tmovl	%2, %0\n"			\
33		     "2:\tmovl\t%0, %3\n"			\
34		     "\t" insn "\n"				\
35		     "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n"	\
36		     "\tjnz\t2b\n"				\
37		     "4:\n"					\
38		     _ASM_EXTABLE_TYPE_REG(1b, 4b, EX_TYPE_EFAULT_REG, %1) \
39		     _ASM_EXTABLE_TYPE_REG(3b, 4b, EX_TYPE_EFAULT_REG, %1) \
40		     : "=&a" (oldval), "=&r" (ret),		\
41		       "+m" (*uaddr), "=&r" (tem)		\
42		     : "r" (oparg), "1" (0));			\
43	if (ret)						\
44		goto label;					\
45	*oval = oldval;						\
46} while(0)
47
48static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
49						       u32 __user *uaddr)
50{
51	scoped_user_rw_access(uaddr, Efault) {
52		switch (op) {
53		case FUTEX_OP_SET:
54			unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault);
55			break;
56		case FUTEX_OP_ADD:
57			unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, uaddr, oparg, Efault);
58			break;
59		case FUTEX_OP_OR:
60			unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault);
61			break;
62		case FUTEX_OP_ANDN:
63			unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault);
64			break;
65		case FUTEX_OP_XOR:
66			unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault);
67			break;
68		default:
69			return -ENOSYS;
70		}
71	}
72	return 0;
73Efault:
74	return -EFAULT;
75}
76
77static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
78						u32 oldval, u32 newval)
79{
80	int ret = 0;
81
82	scoped_user_rw_access(uaddr, Efault) {
83		asm_inline volatile("\n"
84				    "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n"
85				    "2:\n"
86				    _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %0)
87				    : "+r" (ret), "=a" (oldval), "+m" (*uaddr)
88				    : "r" (newval), "1" (oldval)
89				    : "memory");
90		*uval = oldval;
91	}
92	return ret;
93Efault:
94	return -EFAULT;
95}
96
97#endif
98#endif /* _ASM_X86_FUTEX_H */