cmpxchg.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 */
  5#ifndef __ASM_CMPXCHG_H
  6#define __ASM_CMPXCHG_H
  7
  8#include <linux/bits.h>
  9#include <linux/build_bug.h>
 10#include <asm/barrier.h>
 11
 12#define __xchg_amo_asm(amswap_db, m, val)	\
 13({						\
 14	__typeof(val) __ret;			\
 15						\
 16	__asm__ __volatile__ (			\
 17	" "amswap_db" %1, %z2, %0 \n"		\
 18	: "+ZB" (*m), "=&r" (__ret)		\
 19	: "Jr" (val)				\
 20	: "memory");				\
 21						\
 22	__ret;					\
 23})
 24
 25#define __xchg_llsc_asm(ld, st, m, val)			\
 26({							\
 27	__typeof(val) __ret, __tmp;			\
 28							\
 29	asm volatile (					\
 30	"1:	ll.w	%0, %3		\n"		\
 31	"	move	%1, %z4		\n"		\
 32	"	sc.w	%1, %2		\n"		\
 33	"	beqz	%1, 1b		\n"		\
 34	: "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m)	\
 35	: "ZC" (*m), "Jr" (val)				\
 36	: "memory");					\
 37							\
 38	__ret;						\
 39})
 40
 41static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,
 42					unsigned int size)
 43{
 44	unsigned int shift;
 45	u32 old32, mask, temp;
 46	volatile u32 *ptr32;
 47
 48	/* Mask value to the correct size. */
 49	mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
 50	val &= mask;
 51
 52	/*
 53	 * Calculate a shift & mask that correspond to the value we wish to
 54	 * exchange within the naturally aligned 4 byte integerthat includes
 55	 * it.
 56	 */
 57	shift = (unsigned long)ptr & 0x3;
 58	shift *= BITS_PER_BYTE;
 59	mask <<= shift;
 60
 61	/*
 62	 * Calculate a pointer to the naturally aligned 4 byte integer that
 63	 * includes our byte of interest, and load its value.
 64	 */
 65	ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
 66
 67	asm volatile (
 68	"1:	ll.w		%0, %3		\n"
 69	"	andn		%1, %0, %z4	\n"
 70	"	or		%1, %1, %z5	\n"
 71	"	sc.w		%1, %2		\n"
 72	"	beqz		%1, 1b		\n"
 73	: "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
 74	: "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift)
 75	: "memory");
 76
 77	return (old32 & mask) >> shift;
 78}
 79
 80static __always_inline unsigned long
 81__arch_xchg(volatile void *ptr, unsigned long x, int size)
 82{
 83	switch (size) {
 84	case 1:
 85	case 2:
 86		return __xchg_small((volatile void *)ptr, x, size);
 87
 88	case 4:
 89#ifdef CONFIG_CPU_HAS_AMO
 90		return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
 91#else
 92		return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x);
 93#endif /* CONFIG_CPU_HAS_AMO */
 94
 95#ifdef CONFIG_64BIT
 96	case 8:
 97#ifdef CONFIG_CPU_HAS_AMO
 98		return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
 99#else
100		return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x);
101#endif /* CONFIG_CPU_HAS_AMO */
102#endif /* CONFIG_64BIT */
103
104	default:
105		BUILD_BUG();
106	}
107
108	return 0;
109}
110
111#define arch_xchg(ptr, x)						\
112({									\
113	__typeof__(*(ptr)) __res;					\
114									\
115	__res = (__typeof__(*(ptr)))					\
116		__arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr)));	\
117									\
118	__res;								\
119})
120
121#define __cmpxchg_asm(ld, st, m, old, new)				\
122({									\
123	__typeof(old) __ret;						\
124									\
125	__asm__ __volatile__(						\
126	"1:	" ld "	%0, %2		# __cmpxchg_asm \n"		\
127	"	bne	%0, %z3, 2f			\n"		\
128	"	move	$t0, %z4			\n"		\
129	"	" st "	$t0, %1				\n"		\
130	"	beqz	$t0, 1b				\n"		\
131	"2:						\n"		\
132	__WEAK_LLSC_MB							\
133	: "=&r" (__ret), "=ZB"(*m)					\
134	: "ZB"(*m), "Jr" (old), "Jr" (new)				\
135	: "t0", "memory");						\
136									\
137	__ret;								\
138})
139
140static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old,
141					   unsigned int new, unsigned int size)
142{
143	unsigned int shift;
144	u32 old32, mask, temp;
145	volatile u32 *ptr32;
146
147	/* Mask inputs to the correct size. */
148	mask = GENMASK((size * BITS_PER_BYTE) - 1, 0);
149	old &= mask;
150	new &= mask;
151
152	/*
153	 * Calculate a shift & mask that correspond to the value we wish to
154	 * compare & exchange within the naturally aligned 4 byte integer
155	 * that includes it.
156	 */
157	shift = (unsigned long)ptr & 0x3;
158	shift *= BITS_PER_BYTE;
159	old <<= shift;
160	new <<= shift;
161	mask <<= shift;
162
163	/*
164	 * Calculate a pointer to the naturally aligned 4 byte integer that
165	 * includes our byte of interest, and load its value.
166	 */
167	ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3);
168
169	asm volatile (
170	"1:	ll.w		%0, %3		\n"
171	"	and		%1, %0, %z4	\n"
172	"	bne		%1, %z5, 2f	\n"
173	"	andn		%1, %0, %z4	\n"
174	"	or		%1, %1, %z6	\n"
175	"	sc.w		%1, %2		\n"
176	"	beqz		%1, 1b		\n"
177	"	b		3f		\n"
178	"2:					\n"
179	__WEAK_LLSC_MB
180	"3:					\n"
181	: "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32)
182	: "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new)
183	: "memory");
184
185	return (old32 & mask) >> shift;
186}
187
188static __always_inline unsigned long
189__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size)
190{
191	switch (size) {
192	case 1:
193	case 2:
194		return __cmpxchg_small(ptr, old, new, size);
195
196	case 4:
197		return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr,
198				     (u32)old, new);
199
200	case 8:
201		return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr,
202				     (u64)old, new);
203
204	default:
205		BUILD_BUG();
206	}
207
208	return 0;
209}
210
211#define arch_cmpxchg_local(ptr, old, new)				\
212	((__typeof__(*(ptr)))						\
213		__cmpxchg((ptr),					\
214			  (unsigned long)(__typeof__(*(ptr)))(old),	\
215			  (unsigned long)(__typeof__(*(ptr)))(new),	\
216			  sizeof(*(ptr))))
217
218#define arch_cmpxchg(ptr, old, new)					\
219({									\
220	__typeof__(*(ptr)) __res;					\
221									\
222	__res = arch_cmpxchg_local((ptr), (old), (new));		\
223									\
224	__res;								\
225})
226
227#ifdef CONFIG_64BIT
228#define arch_cmpxchg64_local(ptr, o, n)					\
229  ({									\
230	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
231	arch_cmpxchg_local((ptr), (o), (n));				\
232  })
233
234#define arch_cmpxchg64(ptr, o, n)					\
235  ({									\
236	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
237	arch_cmpxchg((ptr), (o), (n));					\
238  })
239#else
240#include <asm-generic/cmpxchg-local.h>
241#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
242#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n))
243#endif
244
245#endif /* __ASM_CMPXCHG_H */