atomic.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/*
  3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
  4 * Copyright (C) 2012 Regents of the University of California
  5 * Copyright (C) 2017 SiFive
  6 */
  7
  8#ifndef _ASM_RISCV_ATOMIC_H
  9#define _ASM_RISCV_ATOMIC_H
 10
 11#ifdef CONFIG_GENERIC_ATOMIC64
 12# include <asm-generic/atomic64.h>
 13#else
 14# if (__riscv_xlen < 64)
 15#  error "64-bit atomics require XLEN to be at least 64"
 16# endif
 17#endif
 18
 19#include <asm/cmpxchg.h>
 20
 21#define __atomic_acquire_fence()					\
 22	__asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
 23
 24#define __atomic_release_fence()					\
 25	__asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
 26
 27static __always_inline int arch_atomic_read(const atomic_t *v)
 28{
 29	return READ_ONCE(v->counter);
 30}
 31static __always_inline void arch_atomic_set(atomic_t *v, int i)
 32{
 33	WRITE_ONCE(v->counter, i);
 34}
 35
 36#ifndef CONFIG_GENERIC_ATOMIC64
 37#define ATOMIC64_INIT(i) { (i) }
 38static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
 39{
 40	return READ_ONCE(v->counter);
 41}
 42static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
 43{
 44	WRITE_ONCE(v->counter, i);
 45}
 46#endif
 47
 48/*
 49 * First, the atomic ops that have no ordering constraints and therefor don't
 50 * have the AQ or RL bits set.  These don't return anything, so there's only
 51 * one version to worry about.
 52 */
 53#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix)		\
 54static __always_inline							\
 55void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v)	\
 56{									\
 57	__asm__ __volatile__ (						\
 58		"	amo" #asm_op "." #asm_type " zero, %1, %0"	\
 59		: "+A" (v->counter)					\
 60		: "r" (I)						\
 61		: "memory");						\
 62}									\
 63
 64#ifdef CONFIG_GENERIC_ATOMIC64
 65#define ATOMIC_OPS(op, asm_op, I)					\
 66        ATOMIC_OP (op, asm_op, I, w, int,   )
 67#else
 68#define ATOMIC_OPS(op, asm_op, I)					\
 69        ATOMIC_OP (op, asm_op, I, w, int,   )				\
 70        ATOMIC_OP (op, asm_op, I, d, s64, 64)
 71#endif
 72
 73ATOMIC_OPS(add, add,  i)
 74ATOMIC_OPS(sub, add, -i)
 75ATOMIC_OPS(and, and,  i)
 76ATOMIC_OPS( or,  or,  i)
 77ATOMIC_OPS(xor, xor,  i)
 78
 79#undef ATOMIC_OP
 80#undef ATOMIC_OPS
 81
 82/*
 83 * Atomic ops that have ordered, relaxed, acquire, and release variants.
 84 * There's two flavors of these: the arithmatic ops have both fetch and return
 85 * versions, while the logical ops only have fetch versions.
 86 */
 87#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix)	\
 88static __always_inline							\
 89c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i,		\
 90					     atomic##prefix##_t *v)	\
 91{									\
 92	register c_type ret;						\
 93	__asm__ __volatile__ (						\
 94		"	amo" #asm_op "." #asm_type " %1, %2, %0"	\
 95		: "+A" (v->counter), "=r" (ret)				\
 96		: "r" (I)						\
 97		: "memory");						\
 98	return ret;							\
 99}									\
100static __always_inline							\
101c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v)	\
102{									\
103	register c_type ret;						\
104	__asm__ __volatile__ (						\
105		"	amo" #asm_op "." #asm_type ".aqrl  %1, %2, %0"	\
106		: "+A" (v->counter), "=r" (ret)				\
107		: "r" (I)						\
108		: "memory");						\
109	return ret;							\
110}
111
112#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix)	\
113static __always_inline							\
114c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i,		\
115					      atomic##prefix##_t *v)	\
116{									\
117        return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I;	\
118}									\
119static __always_inline							\
120c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v)	\
121{									\
122        return arch_atomic##prefix##_fetch_##op(i, v) c_op I;		\
123}
124
125#ifdef CONFIG_GENERIC_ATOMIC64
126#define ATOMIC_OPS(op, asm_op, c_op, I)					\
127        ATOMIC_FETCH_OP( op, asm_op,       I, w, int,   )		\
128        ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int,   )
129#else
130#define ATOMIC_OPS(op, asm_op, c_op, I)					\
131        ATOMIC_FETCH_OP( op, asm_op,       I, w, int,   )		\
132        ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int,   )		\
133        ATOMIC_FETCH_OP( op, asm_op,       I, d, s64, 64)		\
134        ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64)
135#endif
136
137ATOMIC_OPS(add, add, +,  i)
138ATOMIC_OPS(sub, add, +, -i)
139
140#define arch_atomic_add_return_relaxed	arch_atomic_add_return_relaxed
141#define arch_atomic_sub_return_relaxed	arch_atomic_sub_return_relaxed
142#define arch_atomic_add_return		arch_atomic_add_return
143#define arch_atomic_sub_return		arch_atomic_sub_return
144
145#define arch_atomic_fetch_add_relaxed	arch_atomic_fetch_add_relaxed
146#define arch_atomic_fetch_sub_relaxed	arch_atomic_fetch_sub_relaxed
147#define arch_atomic_fetch_add		arch_atomic_fetch_add
148#define arch_atomic_fetch_sub		arch_atomic_fetch_sub
149
150#ifndef CONFIG_GENERIC_ATOMIC64
151#define arch_atomic64_add_return_relaxed	arch_atomic64_add_return_relaxed
152#define arch_atomic64_sub_return_relaxed	arch_atomic64_sub_return_relaxed
153#define arch_atomic64_add_return		arch_atomic64_add_return
154#define arch_atomic64_sub_return		arch_atomic64_sub_return
155
156#define arch_atomic64_fetch_add_relaxed	arch_atomic64_fetch_add_relaxed
157#define arch_atomic64_fetch_sub_relaxed	arch_atomic64_fetch_sub_relaxed
158#define arch_atomic64_fetch_add		arch_atomic64_fetch_add
159#define arch_atomic64_fetch_sub		arch_atomic64_fetch_sub
160#endif
161
162#undef ATOMIC_OPS
163
164#ifdef CONFIG_GENERIC_ATOMIC64
165#define ATOMIC_OPS(op, asm_op, I)					\
166        ATOMIC_FETCH_OP(op, asm_op, I, w, int,   )
167#else
168#define ATOMIC_OPS(op, asm_op, I)					\
169        ATOMIC_FETCH_OP(op, asm_op, I, w, int,   )			\
170        ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64)
171#endif
172
173ATOMIC_OPS(and, and, i)
174ATOMIC_OPS( or,  or, i)
175ATOMIC_OPS(xor, xor, i)
176
177#define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed
178#define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed
179#define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed
180#define arch_atomic_fetch_and		arch_atomic_fetch_and
181#define arch_atomic_fetch_or		arch_atomic_fetch_or
182#define arch_atomic_fetch_xor		arch_atomic_fetch_xor
183
184#ifndef CONFIG_GENERIC_ATOMIC64
185#define arch_atomic64_fetch_and_relaxed	arch_atomic64_fetch_and_relaxed
186#define arch_atomic64_fetch_or_relaxed	arch_atomic64_fetch_or_relaxed
187#define arch_atomic64_fetch_xor_relaxed	arch_atomic64_fetch_xor_relaxed
188#define arch_atomic64_fetch_and		arch_atomic64_fetch_and
189#define arch_atomic64_fetch_or		arch_atomic64_fetch_or
190#define arch_atomic64_fetch_xor		arch_atomic64_fetch_xor
191#endif
192
193#undef ATOMIC_OPS
194
195#undef ATOMIC_FETCH_OP
196#undef ATOMIC_OP_RETURN
197
198#define _arch_atomic_fetch_add_unless(_prev, _rc, counter, _a, _u, sfx)	\
199({									\
200	__asm__ __volatile__ (						\
201		"0:	lr." sfx "     %[p],  %[c]\n"			\
202		"	beq	       %[p],  %[u], 1f\n"		\
203		"	add            %[rc], %[p], %[a]\n"		\
204		"	sc." sfx ".rl  %[rc], %[rc], %[c]\n"		\
205		"	bnez           %[rc], 0b\n"			\
206		RISCV_FULL_BARRIER					\
207		"1:\n"							\
208		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
209		: [a]"r" (_a), [u]"r" (_u)				\
210		: "memory");						\
211})
212
213/* This is required to provide a full barrier on success. */
214static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
215{
216       int prev, rc;
217
218	_arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "w");
219
220	return prev;
221}
222#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
223
224#ifndef CONFIG_GENERIC_ATOMIC64
225static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
226{
227       s64 prev;
228       long rc;
229
230	_arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "d");
231
232	return prev;
233}
234#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
235#endif
236
237#define _arch_atomic_inc_unless_negative(_prev, _rc, counter, sfx)	\
238({									\
239	__asm__ __volatile__ (						\
240		"0:	lr." sfx "      %[p],  %[c]\n"			\
241		"	bltz            %[p],  1f\n"			\
242		"	addi            %[rc], %[p], 1\n"		\
243		"	sc." sfx ".rl   %[rc], %[rc], %[c]\n"		\
244		"	bnez            %[rc], 0b\n"			\
245		RISCV_FULL_BARRIER					\
246		"1:\n"							\
247		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
248		:							\
249		: "memory");						\
250})
251
252static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
253{
254	int prev, rc;
255
256	_arch_atomic_inc_unless_negative(prev, rc, v->counter, "w");
257
258	return !(prev < 0);
259}
260
261#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
262
263#define _arch_atomic_dec_unless_positive(_prev, _rc, counter, sfx)	\
264({									\
265	__asm__ __volatile__ (						\
266		"0:	lr." sfx "      %[p],  %[c]\n"			\
267		"	bgtz            %[p],  1f\n"			\
268		"	addi            %[rc], %[p], -1\n"		\
269		"	sc." sfx ".rl   %[rc], %[rc], %[c]\n"		\
270		"	bnez            %[rc], 0b\n"			\
271		RISCV_FULL_BARRIER					\
272		"1:\n"							\
273		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
274		:							\
275		: "memory");						\
276})
277
278static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v)
279{
280	int prev, rc;
281
282	_arch_atomic_dec_unless_positive(prev, rc, v->counter, "w");
283
284	return !(prev > 0);
285}
286
287#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
288
289#define _arch_atomic_dec_if_positive(_prev, _rc, counter, sfx)		\
290({									\
291	__asm__ __volatile__ (						\
292		"0:	lr." sfx "     %[p],  %[c]\n"			\
293		"	addi           %[rc], %[p], -1\n"		\
294		"	bltz           %[rc], 1f\n"			\
295		"	sc." sfx ".rl  %[rc], %[rc], %[c]\n"		\
296		"	bnez           %[rc], 0b\n"			\
297		RISCV_FULL_BARRIER					\
298		"1:\n"							\
299		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
300		:							\
301		: "memory");						\
302})
303
304static __always_inline int arch_atomic_dec_if_positive(atomic_t *v)
305{
306       int prev, rc;
307
308	_arch_atomic_dec_if_positive(prev, rc, v->counter, "w");
309
310	return prev - 1;
311}
312
313#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
314
315#ifndef CONFIG_GENERIC_ATOMIC64
316static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v)
317{
318	s64 prev;
319	long rc;
320
321	_arch_atomic_inc_unless_negative(prev, rc, v->counter, "d");
322
323	return !(prev < 0);
324}
325
326#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
327
328static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v)
329{
330	s64 prev;
331	long rc;
332
333	_arch_atomic_dec_unless_positive(prev, rc, v->counter, "d");
334
335	return !(prev > 0);
336}
337
338#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
339
340static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
341{
342       s64 prev;
343       long rc;
344
345	_arch_atomic_dec_if_positive(prev, rc, v->counter, "d");
346
347	return prev - 1;
348}
349
350#define arch_atomic64_dec_if_positive	arch_atomic64_dec_if_positive
351#endif
352
353#endif /* _ASM_RISCV_ATOMIC_H */