cmpxchg.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef ASM_X86_CMPXCHG_H
  3#define ASM_X86_CMPXCHG_H
  4
  5#include <linux/compiler.h>
  6#include <asm/cpufeatures.h>
  7#include <asm/alternative.h> /* Provides LOCK_PREFIX */
  8
  9/*
 10 * Non-existent functions to indicate usage errors at link time
 11 * (or compile-time if the compiler implements __compiletime_error().
 12 */
 13extern void __xchg_wrong_size(void)
 14	__compiletime_error("Bad argument size for xchg");
 15extern void __cmpxchg_wrong_size(void)
 16	__compiletime_error("Bad argument size for cmpxchg");
 17extern void __xadd_wrong_size(void)
 18	__compiletime_error("Bad argument size for xadd");
 19extern void __add_wrong_size(void)
 20	__compiletime_error("Bad argument size for add");
 21
 22/*
 23 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
 24 * -1 because sizeof will never return -1, thereby making those switch
 25 * case statements guaranteed dead code which the compiler will
 26 * eliminate, and allowing the "missing symbol in the default case" to
 27 * indicate a usage error.
 28 */
 29#define __X86_CASE_B	1
 30#define __X86_CASE_W	2
 31#define __X86_CASE_L	4
 32#ifdef CONFIG_64BIT
 33#define __X86_CASE_Q	8
 34#else
 35#define	__X86_CASE_Q	-1		/* sizeof will never return -1 */
 36#endif
 37
 38/* 
 39 * An exchange-type operation, which takes a value and a pointer, and
 40 * returns the old value.
 41 */
 42#define __xchg_op(ptr, arg, op, lock)					\
 43	({								\
 44	        __typeof__ (*(ptr)) __ret = (arg);			\
 45		switch (sizeof(*(ptr))) {				\
 46		case __X86_CASE_B:					\
 47			asm_inline volatile (lock #op "b %b0, %1"	\
 48				      : "+q" (__ret), "+m" (*(ptr))	\
 49				      : : "memory", "cc");		\
 50			break;						\
 51		case __X86_CASE_W:					\
 52			asm_inline volatile (lock #op "w %w0, %1"	\
 53				      : "+r" (__ret), "+m" (*(ptr))	\
 54				      : : "memory", "cc");		\
 55			break;						\
 56		case __X86_CASE_L:					\
 57			asm_inline volatile (lock #op "l %0, %1"	\
 58				      : "+r" (__ret), "+m" (*(ptr))	\
 59				      : : "memory", "cc");		\
 60			break;						\
 61		case __X86_CASE_Q:					\
 62			asm_inline volatile (lock #op "q %q0, %1"	\
 63				      : "+r" (__ret), "+m" (*(ptr))	\
 64				      : : "memory", "cc");		\
 65			break;						\
 66		default:						\
 67			__ ## op ## _wrong_size();			\
 68		}							\
 69		__ret;							\
 70	})
 71
 72/*
 73 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
 74 * Since this is generally used to protect other memory information, we
 75 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
 76 * information around.
 77 */
 78#define arch_xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")
 79
 80/*
 81 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 82 * store NEW in MEM.  Return the initial value in MEM.  Success is
 83 * indicated by comparing RETURN with OLD.
 84 */
 85#define __raw_cmpxchg(ptr, old, new, size, lock)			\
 86({									\
 87	__typeof__(*(ptr)) __ret;					\
 88	__typeof__(*(ptr)) __old = (old);				\
 89	__typeof__(*(ptr)) __new = (new);				\
 90	switch (size) {							\
 91	case __X86_CASE_B:						\
 92	{								\
 93		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
 94		asm_inline volatile(lock "cmpxchgb %2, %1"		\
 95			     : "=a" (__ret), "+m" (*__ptr)		\
 96			     : "q" (__new), "0" (__old)			\
 97			     : "memory");				\
 98		break;							\
 99	}								\
100	case __X86_CASE_W:						\
101	{								\
102		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
103		asm_inline volatile(lock "cmpxchgw %2, %1"		\
104			     : "=a" (__ret), "+m" (*__ptr)		\
105			     : "r" (__new), "0" (__old)			\
106			     : "memory");				\
107		break;							\
108	}								\
109	case __X86_CASE_L:						\
110	{								\
111		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
112		asm_inline volatile(lock "cmpxchgl %2, %1"		\
113			     : "=a" (__ret), "+m" (*__ptr)		\
114			     : "r" (__new), "0" (__old)			\
115			     : "memory");				\
116		break;							\
117	}								\
118	case __X86_CASE_Q:						\
119	{								\
120		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
121		asm_inline volatile(lock "cmpxchgq %2, %1"		\
122			     : "=a" (__ret), "+m" (*__ptr)		\
123			     : "r" (__new), "0" (__old)			\
124			     : "memory");				\
125		break;							\
126	}								\
127	default:							\
128		__cmpxchg_wrong_size();					\
129	}								\
130	__ret;								\
131})
132
133#define __cmpxchg(ptr, old, new, size)					\
134	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
135
136#define __sync_cmpxchg(ptr, old, new, size)				\
137	__raw_cmpxchg((ptr), (old), (new), (size), "lock ")
138
139#define __cmpxchg_local(ptr, old, new, size)				\
140	__raw_cmpxchg((ptr), (old), (new), (size), "")
141
142#ifdef CONFIG_X86_32
143# include <asm/cmpxchg_32.h>
144#else
145# include <asm/cmpxchg_64.h>
146#endif
147
148#define arch_cmpxchg(ptr, old, new)					\
149	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
150
151#define arch_sync_cmpxchg(ptr, old, new)				\
152	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
153
154#define arch_cmpxchg_local(ptr, old, new)				\
155	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
156
157
158#define __raw_try_cmpxchg(_ptr, _pold, _new, size, lock)		\
159({									\
160	bool success;							\
161	__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold);		\
162	__typeof__(*(_ptr)) __old = *_old;				\
163	__typeof__(*(_ptr)) __new = (_new);				\
164	switch (size) {							\
165	case __X86_CASE_B:						\
166	{								\
167		volatile u8 *__ptr = (volatile u8 *)(_ptr);		\
168		asm_inline volatile(lock "cmpxchgb %[new], %[ptr]"	\
169			     : "=@ccz" (success),			\
170			       [ptr] "+m" (*__ptr),			\
171			       [old] "+a" (__old)			\
172			     : [new] "q" (__new)			\
173			     : "memory");				\
174		break;							\
175	}								\
176	case __X86_CASE_W:						\
177	{								\
178		volatile u16 *__ptr = (volatile u16 *)(_ptr);		\
179		asm_inline volatile(lock "cmpxchgw %[new], %[ptr]"	\
180			     : "=@ccz" (success),			\
181			       [ptr] "+m" (*__ptr),			\
182			       [old] "+a" (__old)			\
183			     : [new] "r" (__new)			\
184			     : "memory");				\
185		break;							\
186	}								\
187	case __X86_CASE_L:						\
188	{								\
189		volatile u32 *__ptr = (volatile u32 *)(_ptr);		\
190		asm_inline volatile(lock "cmpxchgl %[new], %[ptr]"	\
191			     : "=@ccz" (success),			\
192			       [ptr] "+m" (*__ptr),			\
193			       [old] "+a" (__old)			\
194			     : [new] "r" (__new)			\
195			     : "memory");				\
196		break;							\
197	}								\
198	case __X86_CASE_Q:						\
199	{								\
200		volatile u64 *__ptr = (volatile u64 *)(_ptr);		\
201		asm_inline volatile(lock "cmpxchgq %[new], %[ptr]"	\
202			     : "=@ccz" (success),			\
203			       [ptr] "+m" (*__ptr),			\
204			       [old] "+a" (__old)			\
205			     : [new] "r" (__new)			\
206			     : "memory");				\
207		break;							\
208	}								\
209	default:							\
210		__cmpxchg_wrong_size();					\
211	}								\
212	if (unlikely(!success))						\
213		*_old = __old;						\
214	likely(success);						\
215})
216
217#define __try_cmpxchg(ptr, pold, new, size)				\
218	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
219
220#define __sync_try_cmpxchg(ptr, pold, new, size)			\
221	__raw_try_cmpxchg((ptr), (pold), (new), (size), "lock ")
222
223#define __try_cmpxchg_local(ptr, pold, new, size)			\
224	__raw_try_cmpxchg((ptr), (pold), (new), (size), "")
225
226#define arch_try_cmpxchg(ptr, pold, new) 				\
227	__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
228
229#define arch_sync_try_cmpxchg(ptr, pold, new) 				\
230	__sync_try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
231
232#define arch_try_cmpxchg_local(ptr, pold, new)				\
233	__try_cmpxchg_local((ptr), (pold), (new), sizeof(*(ptr)))
234
235/*
236 * xadd() adds "inc" to "*ptr" and atomically returns the previous
237 * value of "*ptr".
238 *
239 * xadd() is locked when multiple CPUs are online
240 */
241#define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)
242#define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)
243
244#endif	/* ASM_X86_CMPXCHG_H */