barrier.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Based on arch/arm/include/asm/barrier.h
  4 *
  5 * Copyright (C) 2012 ARM Ltd.
  6 */
  7#ifndef __ASM_BARRIER_H
  8#define __ASM_BARRIER_H
  9
 10#ifndef __ASSEMBLER__
 11
 12#include <linux/kasan-checks.h>
 13
 14#include <asm/alternative-macros.h>
 15
 16#define __nops(n)	".rept	" #n "\nnop\n.endr\n"
 17#define nops(n)		asm volatile(__nops(n))
 18
 19#define sev()		asm volatile("sev" : : : "memory")
 20#define wfe()		asm volatile("wfe" : : : "memory")
 21#define wfet(val)	asm volatile("msr s0_3_c1_c0_0, %0"	\
 22				     : : "r" (val) : "memory")
 23#define wfi()		asm volatile("wfi" : : : "memory")
 24#define wfit(val)	asm volatile("msr s0_3_c1_c0_1, %0"	\
 25				     : : "r" (val) : "memory")
 26
 27#define isb()		asm volatile("isb" : : : "memory")
 28#define dmb(opt)	asm volatile("dmb " #opt : : : "memory")
 29#define dsb(opt)	asm volatile("dsb " #opt : : : "memory")
 30
 31#define psb_csync()	asm volatile("hint #17" : : : "memory")
 32#define __tsb_csync()	asm volatile("hint #18" : : : "memory")
 33#define csdb()		asm volatile("hint #20" : : : "memory")
 34
 35/*
 36 * Data Gathering Hint:
 37 * This instruction prevents merging memory accesses with Normal-NC or
 38 * Device-GRE attributes before the hint instruction with any memory accesses
 39 * appearing after the hint instruction.
 40 */
 41#define dgh()		asm volatile("hint #6" : : : "memory")
 42
 43#define spec_bar()	asm volatile(ALTERNATIVE("dsb nsh\nisb\n",		\
 44						 SB_BARRIER_INSN"nop\n",	\
 45						 ARM64_HAS_SB))
 46
 47#define gsb_ack()	asm volatile(GSB_ACK_BARRIER_INSN : : : "memory")
 48#define gsb_sys()	asm volatile(GSB_SYS_BARRIER_INSN : : : "memory")
 49
 50#ifdef CONFIG_ARM64_PSEUDO_NMI
 51#define pmr_sync()						\
 52	do {							\
 53		asm volatile(					\
 54		ALTERNATIVE_CB("dsb sy",			\
 55			       ARM64_HAS_GIC_PRIO_RELAXED_SYNC,	\
 56			       alt_cb_patch_nops)		\
 57		);						\
 58	} while(0)
 59#else
 60#define pmr_sync()	do {} while (0)
 61#endif
 62
 63#define __mb()		dsb(sy)
 64#define __rmb()		dsb(ld)
 65#define __wmb()		dsb(st)
 66
 67#define __dma_mb()	dmb(osh)
 68#define __dma_rmb()	dmb(oshld)
 69#define __dma_wmb()	dmb(oshst)
 70
 71#define io_stop_wc()	dgh()
 72
 73#define tsb_csync()								\
 74	do {									\
 75		/*								\
 76		 * CPUs affected by Arm Erratum 2054223 or 2067961 needs	\
 77		 * another TSB to ensure the trace is flushed. The barriers	\
 78		 * don't have to be strictly back to back, as long as the	\
 79		 * CPU is in trace prohibited state.				\
 80		 */								\
 81		if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE))	\
 82			__tsb_csync();						\
 83		__tsb_csync();							\
 84	} while (0)
 85
 86/*
 87 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
 88 * and 0 otherwise.
 89 */
 90#define array_index_mask_nospec array_index_mask_nospec
 91static inline unsigned long array_index_mask_nospec(unsigned long idx,
 92						    unsigned long sz)
 93{
 94	unsigned long mask;
 95
 96	asm volatile(
 97	"	cmp	%1, %2\n"
 98	"	sbc	%0, xzr, xzr\n"
 99	: "=r" (mask)
100	: "r" (idx), "Ir" (sz)
101	: "cc");
102
103	csdb();
104	return mask;
105}
106
107/*
108 * Ensure that reads of the counter are treated the same as memory reads
109 * for the purposes of ordering by subsequent memory barriers.
110 *
111 * This insanity brought to you by speculative system register reads,
112 * out-of-order memory accesses, sequence locks and Thomas Gleixner.
113 *
114 * https://lore.kernel.org/r/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/
115 */
116#define arch_counter_enforce_ordering(val) do {				\
117	u64 tmp, _val = (val);						\
118									\
119	asm volatile(							\
120	"	eor	%0, %1, %1\n"					\
121	"	add	%0, sp, %0\n"					\
122	"	ldr	xzr, [%0]"					\
123	: "=r" (tmp) : "r" (_val));					\
124} while (0)
125
126#define __smp_mb()	dmb(ish)
127#define __smp_rmb()	dmb(ishld)
128#define __smp_wmb()	dmb(ishst)
129
130#define __smp_store_release(p, v)					\
131do {									\
132	typeof(p) __p = (p);						\
133	union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u =	\
134		{ .__val = (__force __unqual_scalar_typeof(*p)) (v) };	\
135	compiletime_assert_atomic_type(*p);				\
136	kasan_check_write(__p, sizeof(*p));				\
137	switch (sizeof(*p)) {						\
138	case 1:								\
139		asm volatile ("stlrb %w1, %0"				\
140				: "=Q" (*__p)				\
141				: "rZ" (*(__u8 *)__u.__c)		\
142				: "memory");				\
143		break;							\
144	case 2:								\
145		asm volatile ("stlrh %w1, %0"				\
146				: "=Q" (*__p)				\
147				: "rZ" (*(__u16 *)__u.__c)		\
148				: "memory");				\
149		break;							\
150	case 4:								\
151		asm volatile ("stlr %w1, %0"				\
152				: "=Q" (*__p)				\
153				: "rZ" (*(__u32 *)__u.__c)		\
154				: "memory");				\
155		break;							\
156	case 8:								\
157		asm volatile ("stlr %x1, %0"				\
158				: "=Q" (*__p)				\
159				: "rZ" (*(__u64 *)__u.__c)		\
160				: "memory");				\
161		break;							\
162	}								\
163} while (0)
164
165#define __smp_load_acquire(p)						\
166({									\
167	union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u;	\
168	typeof(p) __p = (p);						\
169	compiletime_assert_atomic_type(*p);				\
170	kasan_check_read(__p, sizeof(*p));				\
171	switch (sizeof(*p)) {						\
172	case 1:								\
173		asm volatile ("ldarb %w0, %1"				\
174			: "=r" (*(__u8 *)__u.__c)			\
175			: "Q" (*__p) : "memory");			\
176		break;							\
177	case 2:								\
178		asm volatile ("ldarh %w0, %1"				\
179			: "=r" (*(__u16 *)__u.__c)			\
180			: "Q" (*__p) : "memory");			\
181		break;							\
182	case 4:								\
183		asm volatile ("ldar %w0, %1"				\
184			: "=r" (*(__u32 *)__u.__c)			\
185			: "Q" (*__p) : "memory");			\
186		break;							\
187	case 8:								\
188		asm volatile ("ldar %0, %1"				\
189			: "=r" (*(__u64 *)__u.__c)			\
190			: "Q" (*__p) : "memory");			\
191		break;							\
192	}								\
193	(typeof(*p))__u.__val;						\
194})
195
196#define smp_cond_load_relaxed(ptr, cond_expr)				\
197({									\
198	typeof(ptr) __PTR = (ptr);					\
199	__unqual_scalar_typeof(*ptr) VAL;				\
200	for (;;) {							\
201		VAL = READ_ONCE(*__PTR);				\
202		if (cond_expr)						\
203			break;						\
204		__cmpwait_relaxed(__PTR, VAL);				\
205	}								\
206	(typeof(*ptr))VAL;						\
207})
208
209#define smp_cond_load_acquire(ptr, cond_expr)				\
210({									\
211	typeof(ptr) __PTR = (ptr);					\
212	__unqual_scalar_typeof(*ptr) VAL;				\
213	for (;;) {							\
214		VAL = smp_load_acquire(__PTR);				\
215		if (cond_expr)						\
216			break;						\
217		__cmpwait_relaxed(__PTR, VAL);				\
218	}								\
219	(typeof(*ptr))VAL;						\
220})
221
222#include <asm-generic/barrier.h>
223
224#endif	/* __ASSEMBLER__ */
225
226#endif	/* __ASM_BARRIER_H */