cmpxchg.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _ALPHA_CMPXCHG_H
  3#define _ALPHA_CMPXCHG_H
  4
  5/*
  6 * Atomic exchange.
  7 * Since it can be used to implement critical sections
  8 * it must clobber "memory" (also for interrupts in UP).
  9 */
 10
 11static inline unsigned long
 12____xchg_u8(volatile char *m, unsigned long val)
 13{
 14	unsigned long ret, tmp, addr64;
 15
 16	__asm__ __volatile__(
 17	"	andnot	%4,7,%3\n"
 18	"	insbl	%1,%4,%1\n"
 19	"1:	ldq_l	%2,0(%3)\n"
 20	"	extbl	%2,%4,%0\n"
 21	"	mskbl	%2,%4,%2\n"
 22	"	or	%1,%2,%2\n"
 23	"	stq_c	%2,0(%3)\n"
 24	"	beq	%2,2f\n"
 25	".subsection 2\n"
 26	"2:	br	1b\n"
 27	".previous"
 28	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 29	: "r" ((long)m), "1" (val) : "memory");
 30
 31	return ret;
 32}
 33
 34static inline unsigned long
 35____xchg_u16(volatile short *m, unsigned long val)
 36{
 37	unsigned long ret, tmp, addr64;
 38
 39	__asm__ __volatile__(
 40	"	andnot	%4,7,%3\n"
 41	"	inswl	%1,%4,%1\n"
 42	"1:	ldq_l	%2,0(%3)\n"
 43	"	extwl	%2,%4,%0\n"
 44	"	mskwl	%2,%4,%2\n"
 45	"	or	%1,%2,%2\n"
 46	"	stq_c	%2,0(%3)\n"
 47	"	beq	%2,2f\n"
 48	".subsection 2\n"
 49	"2:	br	1b\n"
 50	".previous"
 51	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 52	: "r" ((long)m), "1" (val) : "memory");
 53
 54	return ret;
 55}
 56
 57static inline unsigned long
 58____xchg_u32(volatile int *m, unsigned long val)
 59{
 60	unsigned long dummy;
 61
 62	__asm__ __volatile__(
 63	"1:	ldl_l %0,%4\n"
 64	"	bis $31,%3,%1\n"
 65	"	stl_c %1,%2\n"
 66	"	beq %1,2f\n"
 67	".subsection 2\n"
 68	"2:	br 1b\n"
 69	".previous"
 70	: "=&r" (val), "=&r" (dummy), "=m" (*m)
 71	: "rI" (val), "m" (*m) : "memory");
 72
 73	return val;
 74}
 75
 76static inline unsigned long
 77____xchg_u64(volatile long *m, unsigned long val)
 78{
 79	unsigned long dummy;
 80
 81	__asm__ __volatile__(
 82	"1:	ldq_l %0,%4\n"
 83	"	bis $31,%3,%1\n"
 84	"	stq_c %1,%2\n"
 85	"	beq %1,2f\n"
 86	".subsection 2\n"
 87	"2:	br 1b\n"
 88	".previous"
 89	: "=&r" (val), "=&r" (dummy), "=m" (*m)
 90	: "rI" (val), "m" (*m) : "memory");
 91
 92	return val;
 93}
 94
 95/* This function doesn't exist, so you'll get a linker error
 96   if something tries to do an invalid xchg().  */
 97extern void __xchg_called_with_bad_pointer(void);
 98
 99static __always_inline unsigned long
100____xchg(volatile void *ptr, unsigned long x, int size)
101{
102	return
103		size == 1 ? ____xchg_u8(ptr, x) :
104		size == 2 ? ____xchg_u16(ptr, x) :
105		size == 4 ? ____xchg_u32(ptr, x) :
106		size == 8 ? ____xchg_u64(ptr, x) :
107			(__xchg_called_with_bad_pointer(), x);
108}
109
110/*
111 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
112 * store NEW in MEM.  Return the initial value in MEM.  Success is
113 * indicated by comparing RETURN with OLD.
114 */
115
116static inline unsigned long
117____cmpxchg_u8(volatile char *m, unsigned char old, unsigned char new)
118{
119	unsigned long prev, tmp, cmp, addr64;
120
121	__asm__ __volatile__(
122	"	andnot	%5,7,%4\n"
123	"	insbl	%1,%5,%1\n"
124	"1:	ldq_l	%2,0(%4)\n"
125	"	extbl	%2,%5,%0\n"
126	"	cmpeq	%0,%6,%3\n"
127	"	beq	%3,2f\n"
128	"	mskbl	%2,%5,%2\n"
129	"	or	%1,%2,%2\n"
130	"	stq_c	%2,0(%4)\n"
131	"	beq	%2,3f\n"
132	"2:\n"
133	".subsection 2\n"
134	"3:	br	1b\n"
135	".previous"
136	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
137	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
138
139	return prev;
140}
141
142static inline unsigned long
143____cmpxchg_u16(volatile short *m, unsigned short old, unsigned short new)
144{
145	unsigned long prev, tmp, cmp, addr64;
146
147	__asm__ __volatile__(
148	"	andnot	%5,7,%4\n"
149	"	inswl	%1,%5,%1\n"
150	"1:	ldq_l	%2,0(%4)\n"
151	"	extwl	%2,%5,%0\n"
152	"	cmpeq	%0,%6,%3\n"
153	"	beq	%3,2f\n"
154	"	mskwl	%2,%5,%2\n"
155	"	or	%1,%2,%2\n"
156	"	stq_c	%2,0(%4)\n"
157	"	beq	%2,3f\n"
158	"2:\n"
159	".subsection 2\n"
160	"3:	br	1b\n"
161	".previous"
162	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
163	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
164
165	return prev;
166}
167
168static inline unsigned long
169____cmpxchg_u32(volatile int *m, int old, int new)
170{
171	unsigned long prev, cmp;
172
173	__asm__ __volatile__(
174	"1:	ldl_l %0,%5\n"
175	"	cmpeq %0,%3,%1\n"
176	"	beq %1,2f\n"
177	"	mov %4,%1\n"
178	"	stl_c %1,%2\n"
179	"	beq %1,3f\n"
180	"2:\n"
181	".subsection 2\n"
182	"3:	br 1b\n"
183	".previous"
184	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
185	: "r"((long) old), "r"(new), "m"(*m) : "memory");
186
187	return prev;
188}
189
190static inline unsigned long
191____cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
192{
193	unsigned long prev, cmp;
194
195	__asm__ __volatile__(
196	"1:	ldq_l %0,%5\n"
197	"	cmpeq %0,%3,%1\n"
198	"	beq %1,2f\n"
199	"	mov %4,%1\n"
200	"	stq_c %1,%2\n"
201	"	beq %1,3f\n"
202	"2:\n"
203	".subsection 2\n"
204	"3:	br 1b\n"
205	".previous"
206	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
207	: "r"((long) old), "r"(new), "m"(*m) : "memory");
208
209	return prev;
210}
211
212/* This function doesn't exist, so you'll get a linker error
213   if something tries to do an invalid cmpxchg().  */
214extern void __cmpxchg_called_with_bad_pointer(void);
215
216static __always_inline unsigned long
217____cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
218	      int size)
219{
220	return
221		size == 1 ? ____cmpxchg_u8(ptr, old, new) :
222		size == 2 ? ____cmpxchg_u16(ptr, old, new) :
223		size == 4 ? ____cmpxchg_u32(ptr, old, new) :
224		size == 8 ? ____cmpxchg_u64(ptr, old, new) :
225			(__cmpxchg_called_with_bad_pointer(), old);
226}
227
228#define xchg_local(ptr, x)						\
229({									\
230	__typeof__(*(ptr)) _x_ = (x);					\
231	(__typeof__(*(ptr))) ____xchg((ptr), (unsigned long)_x_,	\
232					       sizeof(*(ptr)));		\
233})
234
235#define arch_cmpxchg_local(ptr, o, n)					\
236({									\
237	__typeof__(*(ptr)) _o_ = (o);					\
238	__typeof__(*(ptr)) _n_ = (n);					\
239	(__typeof__(*(ptr))) ____cmpxchg((ptr), (unsigned long)_o_,	\
240					  (unsigned long)_n_,		\
241					  sizeof(*(ptr)));		\
242})
243
244#define arch_cmpxchg64_local(ptr, o, n)					\
245({									\
246	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
247	cmpxchg_local((ptr), (o), (n));					\
248})
249
250/*
251 * The leading and the trailing memory barriers guarantee that these
252 * operations are fully ordered.
253 */
254#define arch_xchg(ptr, x)						\
255({									\
256	__typeof__(*(ptr)) __ret;					\
257	__typeof__(*(ptr)) _x_ = (x);					\
258	smp_mb();							\
259	__ret = (__typeof__(*(ptr)))					\
260		____xchg((ptr), (unsigned long)_x_, sizeof(*(ptr)));	\
261	smp_mb();							\
262	__ret;								\
263})
264
265#define arch_cmpxchg(ptr, o, n)						\
266({									\
267	__typeof__(*(ptr)) __ret;					\
268	__typeof__(*(ptr)) _o_ = (o);					\
269	__typeof__(*(ptr)) _n_ = (n);					\
270	smp_mb();							\
271	__ret = (__typeof__(*(ptr))) ____cmpxchg((ptr),			\
272		(unsigned long)_o_, (unsigned long)_n_, sizeof(*(ptr)));\
273	smp_mb();							\
274	__ret;								\
275})
276
277#define arch_cmpxchg64(ptr, o, n)					\
278({									\
279	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
280	arch_cmpxchg((ptr), (o), (n));					\
281})
282
283#endif /* _ALPHA_CMPXCHG_H */