percpu.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef __ARCH_S390_PERCPU__
  3#define __ARCH_S390_PERCPU__
  4
  5#include <linux/preempt.h>
  6#include <asm/cmpxchg.h>
  7#include <asm/march.h>
  8
  9/*
 10 * s390 uses its own implementation for per cpu data, the offset of
 11 * the cpu local data area is cached in the cpu's lowcore memory.
 12 */
 13#define __my_cpu_offset get_lowcore()->percpu_offset
 14
 15/*
 16 * We use a compare-and-swap loop since that uses less cpu cycles than
 17 * disabling and enabling interrupts like the generic variant would do.
 18 */
 19#define arch_this_cpu_to_op_simple(pcp, val, op)			\
 20({									\
 21	typedef typeof(pcp) pcp_op_T__;					\
 22	pcp_op_T__ old__, new__, prev__;				\
 23	pcp_op_T__ *ptr__;						\
 24	preempt_disable_notrace();					\
 25	ptr__ = raw_cpu_ptr(&(pcp));					\
 26	prev__ = READ_ONCE(*ptr__);					\
 27	do {								\
 28		old__ = prev__;						\
 29		new__ = old__ op (val);					\
 30		prev__ = cmpxchg(ptr__, old__, new__);			\
 31	} while (prev__ != old__);					\
 32	preempt_enable_notrace();					\
 33	new__;								\
 34})
 35
 36#define this_cpu_add_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
 37#define this_cpu_add_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
 38#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
 39#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
 40#define this_cpu_and_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
 41#define this_cpu_and_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
 42#define this_cpu_or_1(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
 43#define this_cpu_or_2(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
 44
 45#ifndef MARCH_HAS_Z196_FEATURES
 46
 47#define this_cpu_add_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
 48#define this_cpu_add_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
 49#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
 50#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
 51#define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
 52#define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
 53#define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
 54#define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
 55
 56#else /* MARCH_HAS_Z196_FEATURES */
 57
 58#define arch_this_cpu_add(pcp, val, op1, op2, szcast)			\
 59{									\
 60	typedef typeof(pcp) pcp_op_T__; 				\
 61	pcp_op_T__ val__ = (val);					\
 62	pcp_op_T__ old__, *ptr__;					\
 63	preempt_disable_notrace();					\
 64	ptr__ = raw_cpu_ptr(&(pcp)); 				\
 65	if (__builtin_constant_p(val__) &&				\
 66	    ((szcast)val__ > -129) && ((szcast)val__ < 128)) {		\
 67		asm volatile(						\
 68			op2 "   %[ptr__],%[val__]"			\
 69			: [ptr__] "+Q" (*ptr__) 			\
 70			: [val__] "i" ((szcast)val__)			\
 71			: "cc");					\
 72	} else {							\
 73		asm volatile(						\
 74			op1 "   %[old__],%[val__],%[ptr__]"		\
 75			: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)	\
 76			: [val__] "d" (val__)				\
 77			: "cc");					\
 78	}								\
 79	preempt_enable_notrace();					\
 80}
 81
 82#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
 83#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
 84
 85#define arch_this_cpu_add_return(pcp, val, op)				\
 86({									\
 87	typedef typeof(pcp) pcp_op_T__; 				\
 88	pcp_op_T__ val__ = (val);					\
 89	pcp_op_T__ old__, *ptr__;					\
 90	preempt_disable_notrace();					\
 91	ptr__ = raw_cpu_ptr(&(pcp));	 				\
 92	asm volatile(							\
 93		op "    %[old__],%[val__],%[ptr__]"			\
 94		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
 95		: [val__] "d" (val__)					\
 96		: "cc");						\
 97	preempt_enable_notrace();						\
 98	old__ + val__;							\
 99})
100
101#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
102#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
103
104#define arch_this_cpu_to_op(pcp, val, op)				\
105{									\
106	typedef typeof(pcp) pcp_op_T__; 				\
107	pcp_op_T__ val__ = (val);					\
108	pcp_op_T__ old__, *ptr__;					\
109	preempt_disable_notrace();					\
110	ptr__ = raw_cpu_ptr(&(pcp));	 				\
111	asm volatile(							\
112		op "    %[old__],%[val__],%[ptr__]"			\
113		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
114		: [val__] "d" (val__)					\
115		: "cc");						\
116	preempt_enable_notrace();					\
117}
118
119#define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op(pcp, val, "lan")
120#define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op(pcp, val, "lang")
121#define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op(pcp, val, "lao")
122#define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op(pcp, val, "laog")
123
124#endif /* MARCH_HAS_Z196_FEATURES */
125
126#define arch_this_cpu_cmpxchg(pcp, oval, nval)				\
127({									\
128	typedef typeof(pcp) pcp_op_T__;					\
129	pcp_op_T__ ret__;						\
130	pcp_op_T__ *ptr__;						\
131	preempt_disable_notrace();					\
132	ptr__ = raw_cpu_ptr(&(pcp));					\
133	ret__ = cmpxchg(ptr__, oval, nval);				\
134	preempt_enable_notrace();					\
135	ret__;								\
136})
137
138#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
139#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
140#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
141#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
142
143#define this_cpu_cmpxchg64(pcp, o, n)	this_cpu_cmpxchg_8(pcp, o, n)
144
145#define this_cpu_cmpxchg128(pcp, oval, nval)				\
146({									\
147	typedef typeof(pcp) pcp_op_T__;					\
148	u128 old__, new__, ret__;					\
149	pcp_op_T__ *ptr__;						\
150	old__ = oval;							\
151	new__ = nval;							\
152	preempt_disable_notrace();					\
153	ptr__ = raw_cpu_ptr(&(pcp));					\
154	ret__ = cmpxchg128((void *)ptr__, old__, new__);		\
155	preempt_enable_notrace();					\
156	ret__;								\
157})
158
159#define arch_this_cpu_xchg(pcp, nval)					\
160({									\
161	typeof(pcp) *ptr__;						\
162	typeof(pcp) ret__;						\
163	preempt_disable_notrace();					\
164	ptr__ = raw_cpu_ptr(&(pcp));					\
165	ret__ = xchg(ptr__, nval);					\
166	preempt_enable_notrace();					\
167	ret__;								\
168})
169
170#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
171#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
172#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
173#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
174
175#include <asm-generic/percpu.h>
176
177#endif /* __ARCH_S390_PERCPU__ */