atomic_ops.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Low level function for atomic operations
  4 *
  5 * Copyright IBM Corp. 1999, 2016
  6 */
  7
  8#ifndef __ARCH_S390_ATOMIC_OPS__
  9#define __ARCH_S390_ATOMIC_OPS__
 10
 11#include <linux/limits.h>
 12#include <asm/march.h>
 13#include <asm/asm.h>
 14
 15static __always_inline int __atomic_read(const int *ptr)
 16{
 17	int val;
 18
 19	asm volatile(
 20		"	l	%[val],%[ptr]"
 21		: [val] "=d" (val) : [ptr] "R" (*ptr));
 22	return val;
 23}
 24
 25static __always_inline void __atomic_set(int *ptr, int val)
 26{
 27	if (__builtin_constant_p(val) && val >= S16_MIN && val <= S16_MAX) {
 28		asm volatile(
 29			"	mvhi	%[ptr],%[val]"
 30			: [ptr] "=Q" (*ptr) : [val] "K" (val));
 31	} else {
 32		asm volatile(
 33			"	st	%[val],%[ptr]"
 34			: [ptr] "=R" (*ptr) : [val] "d" (val));
 35	}
 36}
 37
 38static __always_inline long __atomic64_read(const long *ptr)
 39{
 40	long val;
 41
 42	asm volatile(
 43		"	lg	%[val],%[ptr]"
 44		: [val] "=d" (val) : [ptr] "RT" (*ptr));
 45	return val;
 46}
 47
 48static __always_inline void __atomic64_set(long *ptr, long val)
 49{
 50	if (__builtin_constant_p(val) && val >= S16_MIN && val <= S16_MAX) {
 51		asm volatile(
 52			"	mvghi	%[ptr],%[val]"
 53			: [ptr] "=Q" (*ptr) : [val] "K" (val));
 54	} else {
 55		asm volatile(
 56			"	stg	%[val],%[ptr]"
 57			: [ptr] "=RT" (*ptr) : [val] "d" (val));
 58	}
 59}
 60
 61#ifdef MARCH_HAS_Z196_FEATURES
 62
 63#define __ATOMIC_OP(op_name, op_type, op_string, op_barrier)		\
 64static __always_inline op_type op_name(op_type val, op_type *ptr)	\
 65{									\
 66	op_type old;							\
 67									\
 68	asm volatile(							\
 69		op_string "	%[old],%[val],%[ptr]"			\
 70		op_barrier						\
 71		: [old] "=d" (old), [ptr] "+QS" (*ptr)			\
 72		: [val] "d" (val) : "cc", "memory");			\
 73	return old;							\
 74}									\
 75
 76#define __ATOMIC_OPS(op_name, op_type, op_string)			\
 77	__ATOMIC_OP(op_name, op_type, op_string, "")			\
 78	__ATOMIC_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
 79
 80__ATOMIC_OPS(__atomic_add, int, "laa")
 81__ATOMIC_OPS(__atomic_and, int, "lan")
 82__ATOMIC_OPS(__atomic_or,  int, "lao")
 83__ATOMIC_OPS(__atomic_xor, int, "lax")
 84
 85__ATOMIC_OPS(__atomic64_add, long, "laag")
 86__ATOMIC_OPS(__atomic64_and, long, "lang")
 87__ATOMIC_OPS(__atomic64_or,  long, "laog")
 88__ATOMIC_OPS(__atomic64_xor, long, "laxg")
 89
 90#undef __ATOMIC_OPS
 91#undef __ATOMIC_OP
 92
 93#define __ATOMIC_CONST_OP(op_name, op_type, op_string, op_barrier)	\
 94static __always_inline void op_name(op_type val, op_type *ptr)		\
 95{									\
 96	asm volatile(							\
 97		op_string "	%[ptr],%[val]"				\
 98		op_barrier						\
 99		: [ptr] "+QS" (*ptr) : [val] "i" (val) : "cc", "memory");\
100}
101
102#define __ATOMIC_CONST_OPS(op_name, op_type, op_string)			\
103	__ATOMIC_CONST_OP(op_name, op_type, op_string, "")		\
104	__ATOMIC_CONST_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
105
106__ATOMIC_CONST_OPS(__atomic_add_const, int, "asi")
107__ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")
108
109#undef __ATOMIC_CONST_OPS
110#undef __ATOMIC_CONST_OP
111
112#else /* MARCH_HAS_Z196_FEATURES */
113
114#define __ATOMIC_OP(op_name, op_string)					\
115static __always_inline int op_name(int val, int *ptr)			\
116{									\
117	int old, new;							\
118									\
119	asm volatile(							\
120		"0:	lr	%[new],%[old]\n"			\
121		op_string "	%[new],%[val]\n"			\
122		"	cs	%[old],%[new],%[ptr]\n"			\
123		"	jl	0b"					\
124		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
125		: [val] "d" (val), "0" (*ptr) : "cc", "memory");	\
126	return old;							\
127}
128
129#define __ATOMIC_OPS(op_name, op_string)				\
130	__ATOMIC_OP(op_name, op_string)					\
131	__ATOMIC_OP(op_name##_barrier, op_string)
132
133__ATOMIC_OPS(__atomic_add, "ar")
134__ATOMIC_OPS(__atomic_and, "nr")
135__ATOMIC_OPS(__atomic_or,  "or")
136__ATOMIC_OPS(__atomic_xor, "xr")
137
138#undef __ATOMIC_OPS
139
140#define __ATOMIC64_OP(op_name, op_string)				\
141static __always_inline long op_name(long val, long *ptr)		\
142{									\
143	long old, new;							\
144									\
145	asm volatile(							\
146		"0:	lgr	%[new],%[old]\n"			\
147		op_string "	%[new],%[val]\n"			\
148		"	csg	%[old],%[new],%[ptr]\n"			\
149		"	jl	0b"					\
150		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+QS" (*ptr)\
151		: [val] "d" (val), "0" (*ptr) : "cc", "memory");	\
152	return old;							\
153}
154
155#define __ATOMIC64_OPS(op_name, op_string)				\
156	__ATOMIC64_OP(op_name, op_string)				\
157	__ATOMIC64_OP(op_name##_barrier, op_string)
158
159__ATOMIC64_OPS(__atomic64_add, "agr")
160__ATOMIC64_OPS(__atomic64_and, "ngr")
161__ATOMIC64_OPS(__atomic64_or,  "ogr")
162__ATOMIC64_OPS(__atomic64_xor, "xgr")
163
164#undef __ATOMIC64_OPS
165
166#define __atomic_add_const(val, ptr)		((void)__atomic_add(val, ptr))
167#define __atomic_add_const_barrier(val, ptr)	((void)__atomic_add(val, ptr))
168#define __atomic64_add_const(val, ptr)		((void)__atomic64_add(val, ptr))
169#define __atomic64_add_const_barrier(val, ptr)	((void)__atomic64_add(val, ptr))
170
171#endif /* MARCH_HAS_Z196_FEATURES */
172
173#if defined(MARCH_HAS_Z196_FEATURES) && defined(__HAVE_ASM_FLAG_OUTPUTS__)
174
175#define __ATOMIC_TEST_OP(op_name, op_type, op_string, op_barrier)	\
176static __always_inline bool op_name(op_type val, op_type *ptr)		\
177{									\
178	op_type tmp;							\
179	int cc;								\
180									\
181	asm volatile(							\
182		op_string "	%[tmp],%[val],%[ptr]"			\
183		op_barrier						\
184		: "=@cc" (cc), [tmp] "=d" (tmp), [ptr] "+QS" (*ptr)	\
185		: [val] "d" (val)					\
186		: "memory");						\
187	return (cc == 0) || (cc == 2);					\
188}									\
189
190#define __ATOMIC_TEST_OPS(op_name, op_type, op_string)			\
191	__ATOMIC_TEST_OP(op_name, op_type, op_string, "")		\
192	__ATOMIC_TEST_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
193
194__ATOMIC_TEST_OPS(__atomic_add_and_test, int, "laal")
195__ATOMIC_TEST_OPS(__atomic64_add_and_test, long, "laalg")
196
197#undef __ATOMIC_TEST_OPS
198#undef __ATOMIC_TEST_OP
199
200#define __ATOMIC_CONST_TEST_OP(op_name, op_type, op_string, op_barrier)	\
201static __always_inline bool op_name(op_type val, op_type *ptr)		\
202{									\
203	int cc;								\
204									\
205	asm volatile(							\
206		op_string "	%[ptr],%[val]"				\
207		op_barrier						\
208		: "=@cc" (cc), [ptr] "+QS" (*ptr)			\
209		: [val] "i" (val)					\
210		: "memory");						\
211	return (cc == 0) || (cc == 2);					\
212}
213
214#define __ATOMIC_CONST_TEST_OPS(op_name, op_type, op_string)		\
215	__ATOMIC_CONST_TEST_OP(op_name, op_type, op_string, "")		\
216	__ATOMIC_CONST_TEST_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
217
218__ATOMIC_CONST_TEST_OPS(__atomic_add_const_and_test, int, "alsi")
219__ATOMIC_CONST_TEST_OPS(__atomic64_add_const_and_test, long, "algsi")
220
221#undef __ATOMIC_CONST_TEST_OPS
222#undef __ATOMIC_CONST_TEST_OP
223
224#else /* defined(MARCH_HAS_Z196_FEATURES) && defined(__HAVE_ASM_FLAG_OUTPUTS__) */
225
226#define __ATOMIC_TEST_OP(op_name, op_func, op_type)			\
227static __always_inline bool op_name(op_type val, op_type *ptr)		\
228{									\
229	return op_func(val, ptr) == -val;				\
230}
231
232__ATOMIC_TEST_OP(__atomic_add_and_test,			__atomic_add,		int)
233__ATOMIC_TEST_OP(__atomic_add_and_test_barrier,		__atomic_add_barrier,	int)
234__ATOMIC_TEST_OP(__atomic_add_const_and_test,		__atomic_add,		int)
235__ATOMIC_TEST_OP(__atomic_add_const_and_test_barrier,	__atomic_add_barrier,	int)
236__ATOMIC_TEST_OP(__atomic64_add_and_test,		__atomic64_add,		long)
237__ATOMIC_TEST_OP(__atomic64_add_and_test_barrier,	__atomic64_add_barrier, long)
238__ATOMIC_TEST_OP(__atomic64_add_const_and_test,		__atomic64_add,		long)
239__ATOMIC_TEST_OP(__atomic64_add_const_and_test_barrier,	__atomic64_add_barrier,	long)
240
241#undef __ATOMIC_TEST_OP
242
243#endif /* defined(MARCH_HAS_Z196_FEATURES) && defined(__HAVE_ASM_FLAG_OUTPUTS__) */
244
245#endif /* __ARCH_S390_ATOMIC_OPS__  */