barrier.h

 1/* SPDX-License-Identifier: GPL-2.0 */
 2/*
 3 * Copyright IBM Corp. 1999, 2009
 4 *
 5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
 6 */
 7
 8#ifndef __ASM_BARRIER_H
 9#define __ASM_BARRIER_H
10
11#include <asm/march.h>
12
13/*
14 * Force strict CPU ordering.
15 * And yes, this is required on UP too when we're talking
16 * to devices.
17 */
18
19#ifdef MARCH_HAS_Z196_FEATURES
20/* Fast-BCR without checkpoint synchronization */
21#define __ASM_BCR_SERIALIZE "bcr 14,0"
22#else
23#define __ASM_BCR_SERIALIZE "bcr 15,0"
24#endif
25
26static __always_inline void bcr_serialize(void)
27{
28	asm volatile(__ASM_BCR_SERIALIZE : : : "memory");
29}
30
31#define __mb()		bcr_serialize()
32#define __rmb()		barrier()
33#define __wmb()		barrier()
34#define __dma_rmb()	__mb()
35#define __dma_wmb()	__mb()
36#define __smp_mb()	__mb()
37#define __smp_rmb()	__rmb()
38#define __smp_wmb()	__wmb()
39
40#define __smp_store_release(p, v)					\
41do {									\
42	compiletime_assert_atomic_type(*p);				\
43	barrier();							\
44	WRITE_ONCE(*p, v);						\
45} while (0)
46
47#define __smp_load_acquire(p)						\
48({									\
49	typeof(*p) ___p1 = READ_ONCE(*p);				\
50	compiletime_assert_atomic_type(*p);				\
51	barrier();							\
52	___p1;								\
53})
54
55#define __smp_mb__before_atomic()	barrier()
56#define __smp_mb__after_atomic()	barrier()
57
58/**
59 * array_index_mask_nospec - generate a mask for array_idx() that is
60 * ~0UL when the bounds check succeeds and 0 otherwise
61 * @index: array element index
62 * @size: number of elements in array
63 */
64#define array_index_mask_nospec array_index_mask_nospec
65static __always_inline unsigned long array_index_mask_nospec(unsigned long index,
66							     unsigned long size)
67{
68	unsigned long mask;
69
70	if (__builtin_constant_p(size) && size > 0) {
71		asm("	clgr	%2,%1\n"
72		    "	slbgr	%0,%0"
73		    :"=d" (mask) : "d" (size-1), "d" (index) :"cc");
74		return mask;
75	}
76	asm("	clgr	%1,%2\n"
77	    "	slbgr	%0,%0"
78	    :"=d" (mask) : "d" (size), "d" (index) :"cc");
79	return ~mask;
80}
81
82#include <asm-generic/barrier.h>
83
84#endif /* __ASM_BARRIER_H */