bitops.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2012 Regents of the University of California
  4 */
  5
  6#ifndef _ASM_RISCV_BITOPS_H
  7#define _ASM_RISCV_BITOPS_H
  8
  9#ifndef _LINUX_BITOPS_H
 10#error "Only <linux/bitops.h> can be included directly"
 11#endif /* _LINUX_BITOPS_H */
 12
 13#include <linux/compiler.h>
 14#include <asm/barrier.h>
 15#include <asm/bitsperlong.h>
 16
 17#if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE)
 18#include <asm-generic/bitops/__ffs.h>
 19#include <asm-generic/bitops/__fls.h>
 20#include <asm-generic/bitops/ffs.h>
 21#include <asm-generic/bitops/fls.h>
 22
 23#else
 24#define __HAVE_ARCH___FFS
 25#define __HAVE_ARCH___FLS
 26#define __HAVE_ARCH_FFS
 27#define __HAVE_ARCH_FLS
 28
 29#include <asm-generic/bitops/__ffs.h>
 30#include <asm-generic/bitops/__fls.h>
 31#include <asm-generic/bitops/ffs.h>
 32#include <asm-generic/bitops/fls.h>
 33
 34#include <asm/alternative-macros.h>
 35#include <asm/hwcap.h>
 36
 37#if (BITS_PER_LONG == 64)
 38#define CTZW	"ctzw "
 39#define CLZW	"clzw "
 40#elif (BITS_PER_LONG == 32)
 41#define CTZW	"ctz "
 42#define CLZW	"clz "
 43#else
 44#error "Unexpected BITS_PER_LONG"
 45#endif
 46
 47static __always_inline __attribute_const__ unsigned long variable__ffs(unsigned long word)
 48{
 49	if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
 50		return generic___ffs(word);
 51
 52	asm volatile (".option push\n"
 53		      ".option arch,+zbb\n"
 54		      "ctz %0, %1\n"
 55		      ".option pop\n"
 56		      : "=r" (word) : "r" (word) :);
 57
 58	return word;
 59}
 60
 61/**
 62 * __ffs - find first set bit in a long word
 63 * @word: The word to search
 64 *
 65 * Undefined if no set bit exists, so code should check against 0 first.
 66 */
 67#define __ffs(word)				\
 68	(__builtin_constant_p(word) ?		\
 69	 (unsigned long)__builtin_ctzl(word) :	\
 70	 variable__ffs(word))
 71
 72static __always_inline __attribute_const__ unsigned long variable__fls(unsigned long word)
 73{
 74	if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
 75		return generic___fls(word);
 76
 77	asm volatile (".option push\n"
 78		      ".option arch,+zbb\n"
 79		      "clz %0, %1\n"
 80		      ".option pop\n"
 81		      : "=r" (word) : "r" (word) :);
 82
 83	return BITS_PER_LONG - 1 - word;
 84}
 85
 86/**
 87 * __fls - find last set bit in a long word
 88 * @word: the word to search
 89 *
 90 * Undefined if no set bit exists, so code should check against 0 first.
 91 */
 92#define __fls(word)							\
 93	(__builtin_constant_p(word) ?					\
 94	 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) :	\
 95	 variable__fls(word))
 96
 97static __always_inline __attribute_const__ int variable_ffs(int x)
 98{
 99	if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
100		return generic_ffs(x);
101
102	if (!x)
103		return 0;
104
105	asm volatile (".option push\n"
106		      ".option arch,+zbb\n"
107		      CTZW "%0, %1\n"
108		      ".option pop\n"
109		      : "=r" (x) : "r" (x) :);
110
111	return x + 1;
112}
113
114/**
115 * ffs - find first set bit in a word
116 * @x: the word to search
117 *
118 * This is defined the same way as the libc and compiler builtin ffs routines.
119 *
120 * ffs(value) returns 0 if value is 0 or the position of the first set bit if
121 * value is nonzero. The first (least significant) bit is at position 1.
122 */
123#define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x))
124
125static __always_inline int variable_fls(unsigned int x)
126{
127	if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
128		return generic_fls(x);
129
130	if (!x)
131		return 0;
132
133	asm volatile (".option push\n"
134		      ".option arch,+zbb\n"
135		      CLZW "%0, %1\n"
136		      ".option pop\n"
137		      : "=r" (x) : "r" (x) :);
138
139	return 32 - x;
140}
141
142/**
143 * fls - find last set bit in a word
144 * @x: the word to search
145 *
146 * This is defined in a similar way as ffs, but returns the position of the most
147 * significant set bit.
148 *
149 * fls(value) returns 0 if value is 0 or the position of the last set bit if
150 * value is nonzero. The last (most significant) bit is at position 32.
151 */
152#define fls(x)							\
153({								\
154	typeof(x) x_ = (x);					\
155	__builtin_constant_p(x_) ?				\
156	 ((x_ != 0) ? (32 - __builtin_clz(x_)) : 0)		\
157	 :							\
158	 variable_fls(x_);					\
159})
160
161#endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */
162
163#include <asm-generic/bitops/ffz.h>
164#include <asm-generic/bitops/fls64.h>
165#include <asm-generic/bitops/sched.h>
166
167#include <asm/arch_hweight.h>
168
169#include <asm-generic/bitops/const_hweight.h>
170
171#if (BITS_PER_LONG == 64)
172#define __AMO(op)	"amo" #op ".d"
173#elif (BITS_PER_LONG == 32)
174#define __AMO(op)	"amo" #op ".w"
175#else
176#error "Unexpected BITS_PER_LONG"
177#endif
178
179#define __test_and_op_bit_ord(op, mod, nr, addr, ord)		\
180({								\
181	unsigned long __res, __mask;				\
182	__mask = BIT_MASK(nr);					\
183	__asm__ __volatile__ (					\
184		__AMO(op) #ord " %0, %2, %1"			\
185		: "=r" (__res), "+A" (addr[BIT_WORD(nr)])	\
186		: "r" (mod(__mask))				\
187		: "memory");					\
188	((__res & __mask) != 0);				\
189})
190
191#define __op_bit_ord(op, mod, nr, addr, ord)			\
192	__asm__ __volatile__ (					\
193		__AMO(op) #ord " zero, %1, %0"			\
194		: "+A" (addr[BIT_WORD(nr)])			\
195		: "r" (mod(BIT_MASK(nr)))			\
196		: "memory");
197
198#define __test_and_op_bit(op, mod, nr, addr) 			\
199	__test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
200#define __op_bit(op, mod, nr, addr)				\
201	__op_bit_ord(op, mod, nr, addr, )
202
203/* Bitmask modifiers */
204#define __NOP(x)	(x)
205#define __NOT(x)	(~(x))
206
207/**
208 * arch_test_and_set_bit - Set a bit and return its old value
209 * @nr: Bit to set
210 * @addr: Address to count from
211 *
212 * This is an atomic fully-ordered operation (implied full memory barrier).
213 */
214static __always_inline int arch_test_and_set_bit(int nr, volatile unsigned long *addr)
215{
216	return __test_and_op_bit(or, __NOP, nr, addr);
217}
218
219/**
220 * arch_test_and_clear_bit - Clear a bit and return its old value
221 * @nr: Bit to clear
222 * @addr: Address to count from
223 *
224 * This is an atomic fully-ordered operation (implied full memory barrier).
225 */
226static __always_inline int arch_test_and_clear_bit(int nr, volatile unsigned long *addr)
227{
228	return __test_and_op_bit(and, __NOT, nr, addr);
229}
230
231/**
232 * arch_test_and_change_bit - Change a bit and return its old value
233 * @nr: Bit to change
234 * @addr: Address to count from
235 *
236 * This operation is atomic and cannot be reordered.
237 * It also implies a memory barrier.
238 */
239static __always_inline int arch_test_and_change_bit(int nr, volatile unsigned long *addr)
240{
241	return __test_and_op_bit(xor, __NOP, nr, addr);
242}
243
244/**
245 * arch_set_bit - Atomically set a bit in memory
246 * @nr: the bit to set
247 * @addr: the address to start counting from
248 *
249 * Note: there are no guarantees that this function will not be reordered
250 * on non x86 architectures, so if you are writing portable code,
251 * make sure not to rely on its reordering guarantees.
252 *
253 * Note that @nr may be almost arbitrarily large; this function is not
254 * restricted to acting on a single-word quantity.
255 */
256static __always_inline void arch_set_bit(int nr, volatile unsigned long *addr)
257{
258	__op_bit(or, __NOP, nr, addr);
259}
260
261/**
262 * arch_clear_bit - Clears a bit in memory
263 * @nr: Bit to clear
264 * @addr: Address to start counting from
265 *
266 * Note: there are no guarantees that this function will not be reordered
267 * on non x86 architectures, so if you are writing portable code,
268 * make sure not to rely on its reordering guarantees.
269 */
270static __always_inline void arch_clear_bit(int nr, volatile unsigned long *addr)
271{
272	__op_bit(and, __NOT, nr, addr);
273}
274
275/**
276 * arch_change_bit - Toggle a bit in memory
277 * @nr: Bit to change
278 * @addr: Address to start counting from
279 *
280 * change_bit()  may be reordered on other architectures than x86.
281 * Note that @nr may be almost arbitrarily large; this function is not
282 * restricted to acting on a single-word quantity.
283 */
284static __always_inline void arch_change_bit(int nr, volatile unsigned long *addr)
285{
286	__op_bit(xor, __NOP, nr, addr);
287}
288
289/**
290 * arch_test_and_set_bit_lock - Set a bit and return its old value, for lock
291 * @nr: Bit to set
292 * @addr: Address to count from
293 *
294 * This operation is atomic and provides acquire barrier semantics.
295 * It can be used to implement bit locks.
296 */
297static __always_inline int arch_test_and_set_bit_lock(
298	unsigned long nr, volatile unsigned long *addr)
299{
300	return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq);
301}
302
303/**
304 * arch_clear_bit_unlock - Clear a bit in memory, for unlock
305 * @nr: the bit to set
306 * @addr: the address to start counting from
307 *
308 * This operation is atomic and provides release barrier semantics.
309 */
310static __always_inline void arch_clear_bit_unlock(
311	unsigned long nr, volatile unsigned long *addr)
312{
313	__op_bit_ord(and, __NOT, nr, addr, .rl);
314}
315
316/**
317 * arch___clear_bit_unlock - Clear a bit in memory, for unlock
318 * @nr: the bit to set
319 * @addr: the address to start counting from
320 *
321 * This operation is like clear_bit_unlock, however it is not atomic.
322 * It does provide release barrier semantics so it can be used to unlock
323 * a bit lock, however it would only be used if no other CPU can modify
324 * any bits in the memory until the lock is released (a good example is
325 * if the bit lock itself protects access to the other bits in the word).
326 *
327 * On RISC-V systems there seems to be no benefit to taking advantage of the
328 * non-atomic property here: it's a lot more instructions and we still have to
329 * provide release semantics anyway.
330 */
331static __always_inline void arch___clear_bit_unlock(
332	unsigned long nr, volatile unsigned long *addr)
333{
334	arch_clear_bit_unlock(nr, addr);
335}
336
337static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
338		volatile unsigned long *addr)
339{
340	unsigned long res;
341	__asm__ __volatile__ (
342		__AMO(xor) ".rl %0, %2, %1"
343		: "=r" (res), "+A" (*addr)
344		: "r" (__NOP(mask))
345		: "memory");
346	return (res & BIT(7)) != 0;
347}
348
349#undef __test_and_op_bit
350#undef __op_bit
351#undef __NOP
352#undef __NOT
353#undef __AMO
354
355#include <asm-generic/bitops/instrumented-atomic.h>
356#include <asm-generic/bitops/instrumented-lock.h>
357
358#include <asm-generic/bitops/non-atomic.h>
359#include <asm-generic/bitops/le.h>
360#include <asm-generic/bitops/ext2-atomic.h>
361
362#endif /* _ASM_RISCV_BITOPS_H */