percpu.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _ASM_X86_PERCPU_H
  3#define _ASM_X86_PERCPU_H
  4
  5#ifdef CONFIG_X86_64
  6# define __percpu_seg		gs
  7# define __percpu_rel		(%rip)
  8#else
  9# define __percpu_seg		fs
 10# define __percpu_rel
 11#endif
 12
 13#ifdef __ASSEMBLER__
 14
 15#ifdef CONFIG_SMP
 16# define __percpu		%__percpu_seg:
 17#else
 18# define __percpu
 19#endif
 20
 21#define PER_CPU_VAR(var)	__percpu(var)__percpu_rel
 22
 23#else /* !__ASSEMBLY__: */
 24
 25#include <linux/args.h>
 26#include <linux/bits.h>
 27#include <linux/build_bug.h>
 28#include <linux/stringify.h>
 29#include <asm/asm.h>
 30
 31#ifdef CONFIG_SMP
 32
 33#define __force_percpu_prefix	"%%"__stringify(__percpu_seg)":"
 34
 35#ifdef CONFIG_CC_HAS_NAMED_AS
 36
 37#ifdef __CHECKER__
 38# define __seg_gs		__attribute__((address_space(__seg_gs)))
 39# define __seg_fs		__attribute__((address_space(__seg_fs)))
 40#endif
 41
 42#define __percpu_prefix
 43#define __percpu_seg_override	CONCATENATE(__seg_, __percpu_seg)
 44
 45#else /* !CONFIG_CC_HAS_NAMED_AS: */
 46
 47#define __percpu_prefix		__force_percpu_prefix
 48#define __percpu_seg_override
 49
 50#endif /* CONFIG_CC_HAS_NAMED_AS */
 51
 52/*
 53 * Compared to the generic __my_cpu_offset version, the following
 54 * saves one instruction and avoids clobbering a temp register.
 55 */
 56#define __my_cpu_offset		this_cpu_read(this_cpu_off)
 57
 58/*
 59 * arch_raw_cpu_ptr should not be used in 32-bit VDSO for a 64-bit
 60 * kernel, because games are played with CONFIG_X86_64 there and
 61 * sizeof(this_cpu_off) becames 4.
 62 */
 63#ifndef BUILD_VDSO32_64
 64#define arch_raw_cpu_ptr(_ptr)						\
 65({									\
 66	unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off);	\
 67									\
 68	tcp_ptr__ += (__force unsigned long)(_ptr);			\
 69	(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__;		\
 70})
 71#else
 72#define arch_raw_cpu_ptr(_ptr)						\
 73({									\
 74	BUILD_BUG();							\
 75	(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)0;			\
 76})
 77#endif
 78
 79#define PER_CPU_VAR(var)	%__percpu_seg:(var)__percpu_rel
 80
 81#else /* !CONFIG_SMP: */
 82
 83#define __force_percpu_prefix
 84#define __percpu_prefix
 85#define __percpu_seg_override
 86
 87#define PER_CPU_VAR(var)	(var)__percpu_rel
 88
 89#endif /* CONFIG_SMP */
 90
 91#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL)
 92# define __my_cpu_type(var)	typeof(var)
 93# define __my_cpu_ptr(ptr)	(ptr)
 94# define __my_cpu_var(var)	(var)
 95
 96# define __percpu_qual		__percpu_seg_override
 97#else
 98# define __my_cpu_type(var)	typeof(var) __percpu_seg_override
 99# define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
100# define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
101#endif
102
103#define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
104#define __percpu_arg(x)		__percpu_prefix "%" #x
105
106/*
107 * For arch-specific code, we can use direct single-insn ops (they
108 * don't give an lvalue though).
109 */
110
111#define __pcpu_type_1		u8
112#define __pcpu_type_2		u16
113#define __pcpu_type_4		u32
114#define __pcpu_type_8		u64
115
116#define __pcpu_cast_1(val)	((u8)(((unsigned long) val) & 0xff))
117#define __pcpu_cast_2(val)	((u16)(((unsigned long) val) & 0xffff))
118#define __pcpu_cast_4(val)	((u32)(((unsigned long) val) & 0xffffffff))
119#define __pcpu_cast_8(val)	((u64)(val))
120
121#define __pcpu_op_1(op)		op "b "
122#define __pcpu_op_2(op)		op "w "
123#define __pcpu_op_4(op)		op "l "
124#define __pcpu_op_8(op)		op "q "
125
126#define __pcpu_reg_1(mod, x)	mod "q" (x)
127#define __pcpu_reg_2(mod, x)	mod "r" (x)
128#define __pcpu_reg_4(mod, x)	mod "r" (x)
129#define __pcpu_reg_8(mod, x)	mod "r" (x)
130
131#define __pcpu_reg_imm_1(x)	"qi" (x)
132#define __pcpu_reg_imm_2(x)	"ri" (x)
133#define __pcpu_reg_imm_4(x)	"ri" (x)
134#define __pcpu_reg_imm_8(x)	"re" (x)
135
136#ifdef CONFIG_USE_X86_SEG_SUPPORT
137
138#define __raw_cpu_read(size, qual, pcp)					\
139({									\
140	*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp));		\
141})
142
143#define __raw_cpu_write(size, qual, pcp, val)				\
144do {									\
145	*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val);	\
146} while (0)
147
148#define __raw_cpu_read_const(pcp)	__raw_cpu_read(, , pcp)
149
150#else /* !CONFIG_USE_X86_SEG_SUPPORT: */
151
152#define __raw_cpu_read(size, qual, _var)				\
153({									\
154	__pcpu_type_##size pfo_val__;					\
155									\
156	asm qual (__pcpu_op_##size("mov")				\
157		  __percpu_arg([var]) ", %[val]"			\
158	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
159	    : [var] "m" (__my_cpu_var(_var)));				\
160									\
161	(typeof(_var))(unsigned long) pfo_val__;			\
162})
163
164#define __raw_cpu_write(size, qual, _var, _val)				\
165do {									\
166	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
167									\
168	if (0) {		                                        \
169		TYPEOF_UNQUAL(_var) pto_tmp__;				\
170		pto_tmp__ = (_val);					\
171		(void)pto_tmp__;					\
172	}								\
173	asm qual (__pcpu_op_##size("mov") "%[val], "			\
174		  __percpu_arg([var])					\
175	    : [var] "=m" (__my_cpu_var(_var))				\
176	    : [val] __pcpu_reg_imm_##size(pto_val__));			\
177} while (0)
178
179/*
180 * The generic per-CPU infrastrucutre is not suitable for
181 * reading const-qualified variables.
182 */
183#define __raw_cpu_read_const(pcp)	({ BUILD_BUG(); (typeof(pcp))0; })
184
185#endif /* CONFIG_USE_X86_SEG_SUPPORT */
186
187#define __raw_cpu_read_stable(size, _var)				\
188({									\
189	__pcpu_type_##size pfo_val__;					\
190									\
191	asm(__pcpu_op_##size("mov")					\
192	    __force_percpu_arg(a[var]) ", %[val]"			\
193	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
194	    : [var] "i" (&(_var)));					\
195									\
196	(typeof(_var))(unsigned long) pfo_val__;			\
197})
198
199#define percpu_unary_op(size, qual, op, _var)				\
200({									\
201	asm qual (__pcpu_op_##size(op) __percpu_arg([var])		\
202	    : [var] "+m" (__my_cpu_var(_var)));				\
203})
204
205#define percpu_binary_op(size, qual, op, _var, _val)			\
206do {									\
207	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
208									\
209	if (0) {		                                        \
210		TYPEOF_UNQUAL(_var) pto_tmp__;				\
211		pto_tmp__ = (_val);					\
212		(void)pto_tmp__;					\
213	}								\
214	asm qual (__pcpu_op_##size(op) "%[val], " __percpu_arg([var])	\
215	    : [var] "+m" (__my_cpu_var(_var))				\
216	    : [val] __pcpu_reg_imm_##size(pto_val__));			\
217} while (0)
218
219/*
220 * Generate a per-CPU add to memory instruction and optimize code
221 * if one is added or subtracted.
222 */
223#define percpu_add_op(size, qual, var, val)				\
224do {									\
225	const int pao_ID__ =						\
226		(__builtin_constant_p(val) &&				\
227			((val) == 1 ||					\
228			 (val) == (typeof(val))-1)) ? (int)(val) : 0;	\
229									\
230	if (0) {							\
231		TYPEOF_UNQUAL(var) pao_tmp__;				\
232		pao_tmp__ = (val);					\
233		(void)pao_tmp__;					\
234	}								\
235	if (pao_ID__ == 1)						\
236		percpu_unary_op(size, qual, "inc", var);		\
237	else if (pao_ID__ == -1)					\
238		percpu_unary_op(size, qual, "dec", var);		\
239	else								\
240		percpu_binary_op(size, qual, "add", var, val);		\
241} while (0)
242
243/*
244 * Add return operation
245 */
246#define percpu_add_return_op(size, qual, _var, _val)			\
247({									\
248	__pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val);	\
249									\
250	asm qual (__pcpu_op_##size("xadd") "%[tmp], "			\
251		  __percpu_arg([var])					\
252		  : [tmp] __pcpu_reg_##size("+", paro_tmp__),		\
253		    [var] "+m" (__my_cpu_var(_var))			\
254		  : : "memory");					\
255	(typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
256})
257
258/*
259 * raw_cpu_xchg() can use a load-store since
260 * it is not required to be IRQ-safe.
261 */
262#define raw_percpu_xchg_op(_var, _nval)					\
263({									\
264	TYPEOF_UNQUAL(_var) pxo_old__ = raw_cpu_read(_var);		\
265									\
266	raw_cpu_write(_var, _nval);					\
267									\
268	pxo_old__;							\
269})
270
271/*
272 * this_cpu_xchg() is implemented using CMPXCHG without a LOCK prefix.
273 * XCHG is expensive due to the implied LOCK prefix. The processor
274 * cannot prefetch cachelines if XCHG is used.
275 */
276#define this_percpu_xchg_op(_var, _nval)				\
277({									\
278	TYPEOF_UNQUAL(_var) pxo_old__ = this_cpu_read(_var);		\
279									\
280	do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval));	\
281									\
282	pxo_old__;							\
283})
284
285/*
286 * CMPXCHG has no such implied lock semantics as a result it is much
287 * more efficient for CPU-local operations.
288 */
289#define percpu_cmpxchg_op(size, qual, _var, _oval, _nval)		\
290({									\
291	__pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval);	\
292	__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval);	\
293									\
294	asm qual (__pcpu_op_##size("cmpxchg") "%[nval], "		\
295		  __percpu_arg([var])					\
296		  : [oval] "+a" (pco_old__),				\
297		    [var] "+m" (__my_cpu_var(_var))			\
298		  : [nval] __pcpu_reg_##size(, pco_new__)		\
299		  : "memory");						\
300									\
301	(typeof(_var))(unsigned long) pco_old__;			\
302})
303
304#define percpu_try_cmpxchg_op(size, qual, _var, _ovalp, _nval)		\
305({									\
306	bool success;							\
307	__pcpu_type_##size *pco_oval__ = (__pcpu_type_##size *)(_ovalp); \
308	__pcpu_type_##size pco_old__ = *pco_oval__;			\
309	__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval);	\
310									\
311	asm qual (__pcpu_op_##size("cmpxchg") "%[nval], "		\
312		  __percpu_arg([var])					\
313		  : "=@ccz" (success),					\
314		    [oval] "+a" (pco_old__),				\
315		    [var] "+m" (__my_cpu_var(_var))			\
316		  : [nval] __pcpu_reg_##size(, pco_new__)		\
317		  : "memory");						\
318	if (unlikely(!success))						\
319		*pco_oval__ = pco_old__;				\
320									\
321	likely(success);						\
322})
323
324#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
325
326#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval)		\
327({									\
328	union {								\
329		u64 var;						\
330		struct {						\
331			u32 low, high;					\
332		};							\
333	} old__, new__;							\
334									\
335	old__.var = _oval;						\
336	new__.var = _nval;						\
337									\
338	asm_inline qual (						\
339		ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
340			    "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
341		: ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)),	\
342				"+a" (old__.low), "+d" (old__.high))	\
343		: "b" (new__.low), "c" (new__.high),			\
344		  "S" (&(_var))						\
345		: "memory");						\
346									\
347	old__.var;							\
348})
349
350#define raw_cpu_cmpxchg64(pcp, oval, nval)		percpu_cmpxchg64_op(8,         , pcp, oval, nval)
351#define this_cpu_cmpxchg64(pcp, oval, nval)		percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
352
353#define percpu_try_cmpxchg64_op(size, qual, _var, _ovalp, _nval)	\
354({									\
355	bool success;							\
356	u64 *_oval = (u64 *)(_ovalp);					\
357	union {								\
358		u64 var;						\
359		struct {						\
360			u32 low, high;					\
361		};							\
362	} old__, new__;							\
363									\
364	old__.var = *_oval;						\
365	new__.var = _nval;						\
366									\
367	asm_inline qual (						\
368		ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
369			    "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
370		: ALT_OUTPUT_SP("=@ccz" (success),			\
371				[var] "+m" (__my_cpu_var(_var)),	\
372				"+a" (old__.low), "+d" (old__.high))	\
373		: "b" (new__.low), "c" (new__.high),			\
374		  "S" (&(_var))						\
375		: "memory");						\
376	if (unlikely(!success))						\
377		*_oval = old__.var;					\
378									\
379	likely(success);						\
380})
381
382#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval)		percpu_try_cmpxchg64_op(8,         , pcp, ovalp, nval)
383#define this_cpu_try_cmpxchg64(pcp, ovalp, nval)	percpu_try_cmpxchg64_op(8, volatile, pcp, ovalp, nval)
384
385#endif /* defined(CONFIG_X86_32) && !defined(CONFIG_UML) */
386
387#ifdef CONFIG_X86_64
388#define raw_cpu_cmpxchg64(pcp, oval, nval)		percpu_cmpxchg_op(8,         , pcp, oval, nval);
389#define this_cpu_cmpxchg64(pcp, oval, nval)		percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
390
391#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval)		percpu_try_cmpxchg_op(8,         , pcp, ovalp, nval);
392#define this_cpu_try_cmpxchg64(pcp, ovalp, nval)	percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval);
393
394#define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval)		\
395({									\
396	union {								\
397		u128 var;						\
398		struct {						\
399			u64 low, high;					\
400		};							\
401	} old__, new__;							\
402									\
403	old__.var = _oval;						\
404	new__.var = _nval;						\
405									\
406	asm_inline qual (						\
407		ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
408			    "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
409		: ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)),	\
410				"+a" (old__.low), "+d" (old__.high))	\
411		: "b" (new__.low), "c" (new__.high),			\
412		  "S" (&(_var))						\
413		: "memory");						\
414									\
415	old__.var;							\
416})
417
418#define raw_cpu_cmpxchg128(pcp, oval, nval)		percpu_cmpxchg128_op(16,         , pcp, oval, nval)
419#define this_cpu_cmpxchg128(pcp, oval, nval)		percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
420
421#define percpu_try_cmpxchg128_op(size, qual, _var, _ovalp, _nval)	\
422({									\
423	bool success;							\
424	u128 *_oval = (u128 *)(_ovalp);					\
425	union {								\
426		u128 var;						\
427		struct {						\
428			u64 low, high;					\
429		};							\
430	} old__, new__;							\
431									\
432	old__.var = *_oval;						\
433	new__.var = _nval;						\
434									\
435	asm_inline qual (						\
436		ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
437			    "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
438		: ALT_OUTPUT_SP("=@ccz" (success),			\
439				[var] "+m" (__my_cpu_var(_var)),	\
440				"+a" (old__.low), "+d" (old__.high))	\
441		: "b" (new__.low), "c" (new__.high),			\
442		  "S" (&(_var))						\
443		: "memory");						\
444	if (unlikely(!success))						\
445		*_oval = old__.var;					\
446									\
447	likely(success);						\
448})
449
450#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval)	percpu_try_cmpxchg128_op(16,         , pcp, ovalp, nval)
451#define this_cpu_try_cmpxchg128(pcp, ovalp, nval)	percpu_try_cmpxchg128_op(16, volatile, pcp, ovalp, nval)
452
453#endif /* CONFIG_X86_64 */
454
455#define raw_cpu_read_1(pcp)				__raw_cpu_read(1, , pcp)
456#define raw_cpu_read_2(pcp)				__raw_cpu_read(2, , pcp)
457#define raw_cpu_read_4(pcp)				__raw_cpu_read(4, , pcp)
458#define raw_cpu_write_1(pcp, val)			__raw_cpu_write(1, , pcp, val)
459#define raw_cpu_write_2(pcp, val)			__raw_cpu_write(2, , pcp, val)
460#define raw_cpu_write_4(pcp, val)			__raw_cpu_write(4, , pcp, val)
461
462#define this_cpu_read_1(pcp)				__raw_cpu_read(1, volatile, pcp)
463#define this_cpu_read_2(pcp)				__raw_cpu_read(2, volatile, pcp)
464#define this_cpu_read_4(pcp)				__raw_cpu_read(4, volatile, pcp)
465#define this_cpu_write_1(pcp, val)			__raw_cpu_write(1, volatile, pcp, val)
466#define this_cpu_write_2(pcp, val)			__raw_cpu_write(2, volatile, pcp, val)
467#define this_cpu_write_4(pcp, val)			__raw_cpu_write(4, volatile, pcp, val)
468
469#define this_cpu_read_stable_1(pcp)			__raw_cpu_read_stable(1, pcp)
470#define this_cpu_read_stable_2(pcp)			__raw_cpu_read_stable(2, pcp)
471#define this_cpu_read_stable_4(pcp)			__raw_cpu_read_stable(4, pcp)
472
473#define raw_cpu_add_1(pcp, val)				percpu_add_op(1, , (pcp), val)
474#define raw_cpu_add_2(pcp, val)				percpu_add_op(2, , (pcp), val)
475#define raw_cpu_add_4(pcp, val)				percpu_add_op(4, , (pcp), val)
476#define raw_cpu_and_1(pcp, val)				percpu_binary_op(1, , "and", (pcp), val)
477#define raw_cpu_and_2(pcp, val)				percpu_binary_op(2, , "and", (pcp), val)
478#define raw_cpu_and_4(pcp, val)				percpu_binary_op(4, , "and", (pcp), val)
479#define raw_cpu_or_1(pcp, val)				percpu_binary_op(1, , "or", (pcp), val)
480#define raw_cpu_or_2(pcp, val)				percpu_binary_op(2, , "or", (pcp), val)
481#define raw_cpu_or_4(pcp, val)				percpu_binary_op(4, , "or", (pcp), val)
482#define raw_cpu_xchg_1(pcp, val)			raw_percpu_xchg_op(pcp, val)
483#define raw_cpu_xchg_2(pcp, val)			raw_percpu_xchg_op(pcp, val)
484#define raw_cpu_xchg_4(pcp, val)			raw_percpu_xchg_op(pcp, val)
485
486#define this_cpu_add_1(pcp, val)			percpu_add_op(1, volatile, (pcp), val)
487#define this_cpu_add_2(pcp, val)			percpu_add_op(2, volatile, (pcp), val)
488#define this_cpu_add_4(pcp, val)			percpu_add_op(4, volatile, (pcp), val)
489#define this_cpu_and_1(pcp, val)			percpu_binary_op(1, volatile, "and", (pcp), val)
490#define this_cpu_and_2(pcp, val)			percpu_binary_op(2, volatile, "and", (pcp), val)
491#define this_cpu_and_4(pcp, val)			percpu_binary_op(4, volatile, "and", (pcp), val)
492#define this_cpu_or_1(pcp, val)				percpu_binary_op(1, volatile, "or", (pcp), val)
493#define this_cpu_or_2(pcp, val)				percpu_binary_op(2, volatile, "or", (pcp), val)
494#define this_cpu_or_4(pcp, val)				percpu_binary_op(4, volatile, "or", (pcp), val)
495#define this_cpu_xchg_1(pcp, nval)			this_percpu_xchg_op(pcp, nval)
496#define this_cpu_xchg_2(pcp, nval)			this_percpu_xchg_op(pcp, nval)
497#define this_cpu_xchg_4(pcp, nval)			this_percpu_xchg_op(pcp, nval)
498
499#define raw_cpu_add_return_1(pcp, val)			percpu_add_return_op(1, , pcp, val)
500#define raw_cpu_add_return_2(pcp, val)			percpu_add_return_op(2, , pcp, val)
501#define raw_cpu_add_return_4(pcp, val)			percpu_add_return_op(4, , pcp, val)
502#define raw_cpu_cmpxchg_1(pcp, oval, nval)		percpu_cmpxchg_op(1, , pcp, oval, nval)
503#define raw_cpu_cmpxchg_2(pcp, oval, nval)		percpu_cmpxchg_op(2, , pcp, oval, nval)
504#define raw_cpu_cmpxchg_4(pcp, oval, nval)		percpu_cmpxchg_op(4, , pcp, oval, nval)
505#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval)		percpu_try_cmpxchg_op(1, , pcp, ovalp, nval)
506#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval)		percpu_try_cmpxchg_op(2, , pcp, ovalp, nval)
507#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval)		percpu_try_cmpxchg_op(4, , pcp, ovalp, nval)
508
509#define this_cpu_add_return_1(pcp, val)			percpu_add_return_op(1, volatile, pcp, val)
510#define this_cpu_add_return_2(pcp, val)			percpu_add_return_op(2, volatile, pcp, val)
511#define this_cpu_add_return_4(pcp, val)			percpu_add_return_op(4, volatile, pcp, val)
512#define this_cpu_cmpxchg_1(pcp, oval, nval)		percpu_cmpxchg_op(1, volatile, pcp, oval, nval)
513#define this_cpu_cmpxchg_2(pcp, oval, nval)		percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
514#define this_cpu_cmpxchg_4(pcp, oval, nval)		percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
515#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval)	percpu_try_cmpxchg_op(1, volatile, pcp, ovalp, nval)
516#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval)	percpu_try_cmpxchg_op(2, volatile, pcp, ovalp, nval)
517#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval)	percpu_try_cmpxchg_op(4, volatile, pcp, ovalp, nval)
518
519/*
520 * Per-CPU atomic 64-bit operations are only available under 64-bit kernels.
521 * 32-bit kernels must fall back to generic operations.
522 */
523#ifdef CONFIG_X86_64
524
525#define raw_cpu_read_8(pcp)				__raw_cpu_read(8, , pcp)
526#define raw_cpu_write_8(pcp, val)			__raw_cpu_write(8, , pcp, val)
527
528#define this_cpu_read_8(pcp)				__raw_cpu_read(8, volatile, pcp)
529#define this_cpu_write_8(pcp, val)			__raw_cpu_write(8, volatile, pcp, val)
530
531#define this_cpu_read_stable_8(pcp)			__raw_cpu_read_stable(8, pcp)
532
533#define raw_cpu_add_8(pcp, val)				percpu_add_op(8, , (pcp), val)
534#define raw_cpu_and_8(pcp, val)				percpu_binary_op(8, , "and", (pcp), val)
535#define raw_cpu_or_8(pcp, val)				percpu_binary_op(8, , "or", (pcp), val)
536#define raw_cpu_add_return_8(pcp, val)			percpu_add_return_op(8, , pcp, val)
537#define raw_cpu_xchg_8(pcp, nval)			raw_percpu_xchg_op(pcp, nval)
538#define raw_cpu_cmpxchg_8(pcp, oval, nval)		percpu_cmpxchg_op(8, , pcp, oval, nval)
539#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval)		percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
540
541#define this_cpu_add_8(pcp, val)			percpu_add_op(8, volatile, (pcp), val)
542#define this_cpu_and_8(pcp, val)			percpu_binary_op(8, volatile, "and", (pcp), val)
543#define this_cpu_or_8(pcp, val)				percpu_binary_op(8, volatile, "or", (pcp), val)
544#define this_cpu_add_return_8(pcp, val)			percpu_add_return_op(8, volatile, pcp, val)
545#define this_cpu_xchg_8(pcp, nval)			this_percpu_xchg_op(pcp, nval)
546#define this_cpu_cmpxchg_8(pcp, oval, nval)		percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
547#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval)	percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
548
549#define raw_cpu_read_long(pcp)				raw_cpu_read_8(pcp)
550
551#else /* !CONFIG_X86_64: */
552
553/* There is no generic 64-bit read stable operation for 32-bit targets. */
554#define this_cpu_read_stable_8(pcp)			({ BUILD_BUG(); (typeof(pcp))0; })
555
556#define raw_cpu_read_long(pcp)				raw_cpu_read_4(pcp)
557
558#endif /* CONFIG_X86_64 */
559
560#define this_cpu_read_const(pcp)			__raw_cpu_read_const(pcp)
561
562/*
563 * this_cpu_read() makes the compiler load the per-CPU variable every time
564 * it is accessed while this_cpu_read_stable() allows the value to be cached.
565 * this_cpu_read_stable() is more efficient and can be used if its value
566 * is guaranteed to be valid across CPUs.  The current users include
567 * current_task and cpu_current_top_of_stack, both of which are
568 * actually per-thread variables implemented as per-CPU variables and
569 * thus stable for the duration of the respective task.
570 */
571#define this_cpu_read_stable(pcp)			__pcpu_size_call_return(this_cpu_read_stable_, pcp)
572
573#define x86_this_cpu_constant_test_bit(_nr, _var)			\
574({									\
575	unsigned long __percpu *addr__ =				\
576		(unsigned long __percpu *)&(_var) + BIT_WORD(_nr);	\
577									\
578	!!(BIT_MASK(_nr) & raw_cpu_read(*addr__));			\
579})
580
581#define x86_this_cpu_variable_test_bit(_nr, _var)			\
582({									\
583	bool oldbit;							\
584									\
585	asm volatile("btl %[nr], " __percpu_arg([var])			\
586		     : "=@ccc" (oldbit)					\
587		     : [var] "m" (__my_cpu_var(_var)),			\
588		       [nr] "rI" (_nr));				\
589	oldbit;								\
590})
591
592#define x86_this_cpu_test_bit(_nr, _var)				\
593	(__builtin_constant_p(_nr)					\
594	 ? x86_this_cpu_constant_test_bit(_nr, _var)			\
595	 : x86_this_cpu_variable_test_bit(_nr, _var))
596
597
598#include <asm-generic/percpu.h>
599
600/* We can use this directly for local CPU (faster). */
601DECLARE_PER_CPU_CACHE_HOT(unsigned long, this_cpu_off);
602
603#endif /* !__ASSEMBLER__ */
604
605#ifdef CONFIG_SMP
606
607/*
608 * Define the "EARLY_PER_CPU" macros.  These are used for some per_cpu
609 * variables that are initialized and accessed before there are per_cpu
610 * areas allocated.
611 */
612
613#define	DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)			\
614	DEFINE_PER_CPU(_type, _name) = _initvalue;			\
615	__typeof__(_type) _name##_early_map[NR_CPUS] __initdata =	\
616				{ [0 ... NR_CPUS-1] = _initvalue };	\
617	__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
618
619#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue)	\
620	DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue;		\
621	__typeof__(_type) _name##_early_map[NR_CPUS] __initdata =	\
622				{ [0 ... NR_CPUS-1] = _initvalue };	\
623	__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
624
625#define EXPORT_EARLY_PER_CPU_SYMBOL(_name)				\
626	EXPORT_PER_CPU_SYMBOL(_name)
627
628#define DECLARE_EARLY_PER_CPU(_type, _name)				\
629	DECLARE_PER_CPU(_type, _name);					\
630	extern __typeof__(_type) *_name##_early_ptr;			\
631	extern __typeof__(_type)  _name##_early_map[]
632
633#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name)			\
634	DECLARE_PER_CPU_READ_MOSTLY(_type, _name);			\
635	extern __typeof__(_type) *_name##_early_ptr;			\
636	extern __typeof__(_type)  _name##_early_map[]
637
638#define	early_per_cpu_ptr(_name)			(_name##_early_ptr)
639#define	early_per_cpu_map(_name, _idx)			(_name##_early_map[_idx])
640
641#define	early_per_cpu(_name, _cpu)					\
642	*(early_per_cpu_ptr(_name) ?					\
643		&early_per_cpu_ptr(_name)[_cpu] :			\
644		&per_cpu(_name, _cpu))
645
646#else /* !CONFIG_SMP: */
647#define	DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)			\
648	DEFINE_PER_CPU(_type, _name) = _initvalue
649
650#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue)	\
651	DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue
652
653#define EXPORT_EARLY_PER_CPU_SYMBOL(_name)				\
654	EXPORT_PER_CPU_SYMBOL(_name)
655
656#define DECLARE_EARLY_PER_CPU(_type, _name)				\
657	DECLARE_PER_CPU(_type, _name)
658
659#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name)			\
660	DECLARE_PER_CPU_READ_MOSTLY(_type, _name)
661
662#define	early_per_cpu(_name, _cpu)			per_cpu(_name, _cpu)
663#define	early_per_cpu_ptr(_name)			NULL
664/* no early_per_cpu_map() */
665
666#endif /* !CONFIG_SMP */
667
668#endif /* _ASM_X86_PERCPU_H */