atomic-arch-fallback.h

   1// SPDX-License-Identifier: GPL-2.0
   2
   3// Generated by scripts/atomic/gen-atomic-fallback.sh
   4// DO NOT MODIFY THIS FILE DIRECTLY
   5
   6#ifndef _LINUX_ATOMIC_FALLBACK_H
   7#define _LINUX_ATOMIC_FALLBACK_H
   8
   9#include <linux/compiler.h>
  10
  11#if defined(arch_xchg)
  12#define raw_xchg arch_xchg
  13#elif defined(arch_xchg_relaxed)
  14#define raw_xchg(...) \
  15	__atomic_op_fence(arch_xchg, __VA_ARGS__)
  16#else
  17extern void raw_xchg_not_implemented(void);
  18#define raw_xchg(...) raw_xchg_not_implemented()
  19#endif
  20
  21#if defined(arch_xchg_acquire)
  22#define raw_xchg_acquire arch_xchg_acquire
  23#elif defined(arch_xchg_relaxed)
  24#define raw_xchg_acquire(...) \
  25	__atomic_op_acquire(arch_xchg, __VA_ARGS__)
  26#elif defined(arch_xchg)
  27#define raw_xchg_acquire arch_xchg
  28#else
  29extern void raw_xchg_acquire_not_implemented(void);
  30#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
  31#endif
  32
  33#if defined(arch_xchg_release)
  34#define raw_xchg_release arch_xchg_release
  35#elif defined(arch_xchg_relaxed)
  36#define raw_xchg_release(...) \
  37	__atomic_op_release(arch_xchg, __VA_ARGS__)
  38#elif defined(arch_xchg)
  39#define raw_xchg_release arch_xchg
  40#else
  41extern void raw_xchg_release_not_implemented(void);
  42#define raw_xchg_release(...) raw_xchg_release_not_implemented()
  43#endif
  44
  45#if defined(arch_xchg_relaxed)
  46#define raw_xchg_relaxed arch_xchg_relaxed
  47#elif defined(arch_xchg)
  48#define raw_xchg_relaxed arch_xchg
  49#else
  50extern void raw_xchg_relaxed_not_implemented(void);
  51#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
  52#endif
  53
  54#if defined(arch_cmpxchg)
  55#define raw_cmpxchg arch_cmpxchg
  56#elif defined(arch_cmpxchg_relaxed)
  57#define raw_cmpxchg(...) \
  58	__atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
  59#else
  60extern void raw_cmpxchg_not_implemented(void);
  61#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
  62#endif
  63
  64#if defined(arch_cmpxchg_acquire)
  65#define raw_cmpxchg_acquire arch_cmpxchg_acquire
  66#elif defined(arch_cmpxchg_relaxed)
  67#define raw_cmpxchg_acquire(...) \
  68	__atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
  69#elif defined(arch_cmpxchg)
  70#define raw_cmpxchg_acquire arch_cmpxchg
  71#else
  72extern void raw_cmpxchg_acquire_not_implemented(void);
  73#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
  74#endif
  75
  76#if defined(arch_cmpxchg_release)
  77#define raw_cmpxchg_release arch_cmpxchg_release
  78#elif defined(arch_cmpxchg_relaxed)
  79#define raw_cmpxchg_release(...) \
  80	__atomic_op_release(arch_cmpxchg, __VA_ARGS__)
  81#elif defined(arch_cmpxchg)
  82#define raw_cmpxchg_release arch_cmpxchg
  83#else
  84extern void raw_cmpxchg_release_not_implemented(void);
  85#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
  86#endif
  87
  88#if defined(arch_cmpxchg_relaxed)
  89#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
  90#elif defined(arch_cmpxchg)
  91#define raw_cmpxchg_relaxed arch_cmpxchg
  92#else
  93extern void raw_cmpxchg_relaxed_not_implemented(void);
  94#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
  95#endif
  96
  97#if defined(arch_cmpxchg64)
  98#define raw_cmpxchg64 arch_cmpxchg64
  99#elif defined(arch_cmpxchg64_relaxed)
 100#define raw_cmpxchg64(...) \
 101	__atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
 102#else
 103extern void raw_cmpxchg64_not_implemented(void);
 104#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
 105#endif
 106
 107#if defined(arch_cmpxchg64_acquire)
 108#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
 109#elif defined(arch_cmpxchg64_relaxed)
 110#define raw_cmpxchg64_acquire(...) \
 111	__atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
 112#elif defined(arch_cmpxchg64)
 113#define raw_cmpxchg64_acquire arch_cmpxchg64
 114#else
 115extern void raw_cmpxchg64_acquire_not_implemented(void);
 116#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
 117#endif
 118
 119#if defined(arch_cmpxchg64_release)
 120#define raw_cmpxchg64_release arch_cmpxchg64_release
 121#elif defined(arch_cmpxchg64_relaxed)
 122#define raw_cmpxchg64_release(...) \
 123	__atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
 124#elif defined(arch_cmpxchg64)
 125#define raw_cmpxchg64_release arch_cmpxchg64
 126#else
 127extern void raw_cmpxchg64_release_not_implemented(void);
 128#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
 129#endif
 130
 131#if defined(arch_cmpxchg64_relaxed)
 132#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
 133#elif defined(arch_cmpxchg64)
 134#define raw_cmpxchg64_relaxed arch_cmpxchg64
 135#else
 136extern void raw_cmpxchg64_relaxed_not_implemented(void);
 137#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
 138#endif
 139
 140#if defined(arch_cmpxchg128)
 141#define raw_cmpxchg128 arch_cmpxchg128
 142#elif defined(arch_cmpxchg128_relaxed)
 143#define raw_cmpxchg128(...) \
 144	__atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
 145#else
 146extern void raw_cmpxchg128_not_implemented(void);
 147#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
 148#endif
 149
 150#if defined(arch_cmpxchg128_acquire)
 151#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
 152#elif defined(arch_cmpxchg128_relaxed)
 153#define raw_cmpxchg128_acquire(...) \
 154	__atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
 155#elif defined(arch_cmpxchg128)
 156#define raw_cmpxchg128_acquire arch_cmpxchg128
 157#else
 158extern void raw_cmpxchg128_acquire_not_implemented(void);
 159#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
 160#endif
 161
 162#if defined(arch_cmpxchg128_release)
 163#define raw_cmpxchg128_release arch_cmpxchg128_release
 164#elif defined(arch_cmpxchg128_relaxed)
 165#define raw_cmpxchg128_release(...) \
 166	__atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
 167#elif defined(arch_cmpxchg128)
 168#define raw_cmpxchg128_release arch_cmpxchg128
 169#else
 170extern void raw_cmpxchg128_release_not_implemented(void);
 171#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
 172#endif
 173
 174#if defined(arch_cmpxchg128_relaxed)
 175#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
 176#elif defined(arch_cmpxchg128)
 177#define raw_cmpxchg128_relaxed arch_cmpxchg128
 178#else
 179extern void raw_cmpxchg128_relaxed_not_implemented(void);
 180#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
 181#endif
 182
 183#if defined(arch_try_cmpxchg)
 184#define raw_try_cmpxchg arch_try_cmpxchg
 185#elif defined(arch_try_cmpxchg_relaxed)
 186#define raw_try_cmpxchg(...) \
 187	__atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
 188#else
 189#define raw_try_cmpxchg(_ptr, _oldp, _new) \
 190({ \
 191	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 192	___r = raw_cmpxchg((_ptr), ___o, (_new)); \
 193	if (unlikely(___r != ___o)) \
 194		*___op = ___r; \
 195	likely(___r == ___o); \
 196})
 197#endif
 198
 199#if defined(arch_try_cmpxchg_acquire)
 200#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
 201#elif defined(arch_try_cmpxchg_relaxed)
 202#define raw_try_cmpxchg_acquire(...) \
 203	__atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
 204#elif defined(arch_try_cmpxchg)
 205#define raw_try_cmpxchg_acquire arch_try_cmpxchg
 206#else
 207#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
 208({ \
 209	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 210	___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
 211	if (unlikely(___r != ___o)) \
 212		*___op = ___r; \
 213	likely(___r == ___o); \
 214})
 215#endif
 216
 217#if defined(arch_try_cmpxchg_release)
 218#define raw_try_cmpxchg_release arch_try_cmpxchg_release
 219#elif defined(arch_try_cmpxchg_relaxed)
 220#define raw_try_cmpxchg_release(...) \
 221	__atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
 222#elif defined(arch_try_cmpxchg)
 223#define raw_try_cmpxchg_release arch_try_cmpxchg
 224#else
 225#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
 226({ \
 227	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 228	___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
 229	if (unlikely(___r != ___o)) \
 230		*___op = ___r; \
 231	likely(___r == ___o); \
 232})
 233#endif
 234
 235#if defined(arch_try_cmpxchg_relaxed)
 236#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
 237#elif defined(arch_try_cmpxchg)
 238#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
 239#else
 240#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
 241({ \
 242	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 243	___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
 244	if (unlikely(___r != ___o)) \
 245		*___op = ___r; \
 246	likely(___r == ___o); \
 247})
 248#endif
 249
 250#if defined(arch_try_cmpxchg64)
 251#define raw_try_cmpxchg64 arch_try_cmpxchg64
 252#elif defined(arch_try_cmpxchg64_relaxed)
 253#define raw_try_cmpxchg64(...) \
 254	__atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
 255#else
 256#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
 257({ \
 258	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 259	___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
 260	if (unlikely(___r != ___o)) \
 261		*___op = ___r; \
 262	likely(___r == ___o); \
 263})
 264#endif
 265
 266#if defined(arch_try_cmpxchg64_acquire)
 267#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
 268#elif defined(arch_try_cmpxchg64_relaxed)
 269#define raw_try_cmpxchg64_acquire(...) \
 270	__atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
 271#elif defined(arch_try_cmpxchg64)
 272#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
 273#else
 274#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
 275({ \
 276	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 277	___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
 278	if (unlikely(___r != ___o)) \
 279		*___op = ___r; \
 280	likely(___r == ___o); \
 281})
 282#endif
 283
 284#if defined(arch_try_cmpxchg64_release)
 285#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
 286#elif defined(arch_try_cmpxchg64_relaxed)
 287#define raw_try_cmpxchg64_release(...) \
 288	__atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
 289#elif defined(arch_try_cmpxchg64)
 290#define raw_try_cmpxchg64_release arch_try_cmpxchg64
 291#else
 292#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
 293({ \
 294	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 295	___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
 296	if (unlikely(___r != ___o)) \
 297		*___op = ___r; \
 298	likely(___r == ___o); \
 299})
 300#endif
 301
 302#if defined(arch_try_cmpxchg64_relaxed)
 303#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
 304#elif defined(arch_try_cmpxchg64)
 305#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
 306#else
 307#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
 308({ \
 309	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 310	___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
 311	if (unlikely(___r != ___o)) \
 312		*___op = ___r; \
 313	likely(___r == ___o); \
 314})
 315#endif
 316
 317#if defined(arch_try_cmpxchg128)
 318#define raw_try_cmpxchg128 arch_try_cmpxchg128
 319#elif defined(arch_try_cmpxchg128_relaxed)
 320#define raw_try_cmpxchg128(...) \
 321	__atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
 322#else
 323#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
 324({ \
 325	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 326	___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
 327	if (unlikely(___r != ___o)) \
 328		*___op = ___r; \
 329	likely(___r == ___o); \
 330})
 331#endif
 332
 333#if defined(arch_try_cmpxchg128_acquire)
 334#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
 335#elif defined(arch_try_cmpxchg128_relaxed)
 336#define raw_try_cmpxchg128_acquire(...) \
 337	__atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
 338#elif defined(arch_try_cmpxchg128)
 339#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
 340#else
 341#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
 342({ \
 343	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 344	___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
 345	if (unlikely(___r != ___o)) \
 346		*___op = ___r; \
 347	likely(___r == ___o); \
 348})
 349#endif
 350
 351#if defined(arch_try_cmpxchg128_release)
 352#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
 353#elif defined(arch_try_cmpxchg128_relaxed)
 354#define raw_try_cmpxchg128_release(...) \
 355	__atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
 356#elif defined(arch_try_cmpxchg128)
 357#define raw_try_cmpxchg128_release arch_try_cmpxchg128
 358#else
 359#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
 360({ \
 361	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 362	___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
 363	if (unlikely(___r != ___o)) \
 364		*___op = ___r; \
 365	likely(___r == ___o); \
 366})
 367#endif
 368
 369#if defined(arch_try_cmpxchg128_relaxed)
 370#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
 371#elif defined(arch_try_cmpxchg128)
 372#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
 373#else
 374#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
 375({ \
 376	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 377	___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
 378	if (unlikely(___r != ___o)) \
 379		*___op = ___r; \
 380	likely(___r == ___o); \
 381})
 382#endif
 383
 384#define raw_cmpxchg_local arch_cmpxchg_local
 385
 386#ifdef arch_try_cmpxchg_local
 387#define raw_try_cmpxchg_local arch_try_cmpxchg_local
 388#else
 389#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
 390({ \
 391	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 392	___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
 393	if (unlikely(___r != ___o)) \
 394		*___op = ___r; \
 395	likely(___r == ___o); \
 396})
 397#endif
 398
 399#define raw_cmpxchg64_local arch_cmpxchg64_local
 400
 401#ifdef arch_try_cmpxchg64_local
 402#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
 403#else
 404#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
 405({ \
 406	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 407	___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
 408	if (unlikely(___r != ___o)) \
 409		*___op = ___r; \
 410	likely(___r == ___o); \
 411})
 412#endif
 413
 414#define raw_cmpxchg128_local arch_cmpxchg128_local
 415
 416#ifdef arch_try_cmpxchg128_local
 417#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
 418#else
 419#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
 420({ \
 421	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 422	___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
 423	if (unlikely(___r != ___o)) \
 424		*___op = ___r; \
 425	likely(___r == ___o); \
 426})
 427#endif
 428
 429#define raw_sync_cmpxchg arch_sync_cmpxchg
 430
 431#ifdef arch_sync_try_cmpxchg
 432#define raw_sync_try_cmpxchg arch_sync_try_cmpxchg
 433#else
 434#define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \
 435({ \
 436	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 437	___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \
 438	if (unlikely(___r != ___o)) \
 439		*___op = ___r; \
 440	likely(___r == ___o); \
 441})
 442#endif
 443
 444/**
 445 * raw_atomic_read() - atomic load with relaxed ordering
 446 * @v: pointer to atomic_t
 447 *
 448 * Atomically loads the value of @v with relaxed ordering.
 449 *
 450 * Safe to use in noinstr code; prefer atomic_read() elsewhere.
 451 *
 452 * Return: The value loaded from @v.
 453 */
 454static __always_inline int
 455raw_atomic_read(const atomic_t *v)
 456{
 457	return arch_atomic_read(v);
 458}
 459
 460/**
 461 * raw_atomic_read_acquire() - atomic load with acquire ordering
 462 * @v: pointer to atomic_t
 463 *
 464 * Atomically loads the value of @v with acquire ordering.
 465 *
 466 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
 467 *
 468 * Return: The value loaded from @v.
 469 */
 470static __always_inline int
 471raw_atomic_read_acquire(const atomic_t *v)
 472{
 473#if defined(arch_atomic_read_acquire)
 474	return arch_atomic_read_acquire(v);
 475#else
 476	int ret;
 477
 478	if (__native_word(atomic_t)) {
 479		ret = smp_load_acquire(&(v)->counter);
 480	} else {
 481		ret = raw_atomic_read(v);
 482		__atomic_acquire_fence();
 483	}
 484
 485	return ret;
 486#endif
 487}
 488
 489/**
 490 * raw_atomic_set() - atomic set with relaxed ordering
 491 * @v: pointer to atomic_t
 492 * @i: int value to assign
 493 *
 494 * Atomically sets @v to @i with relaxed ordering.
 495 *
 496 * Safe to use in noinstr code; prefer atomic_set() elsewhere.
 497 *
 498 * Return: Nothing.
 499 */
 500static __always_inline void
 501raw_atomic_set(atomic_t *v, int i)
 502{
 503	arch_atomic_set(v, i);
 504}
 505
 506/**
 507 * raw_atomic_set_release() - atomic set with release ordering
 508 * @v: pointer to atomic_t
 509 * @i: int value to assign
 510 *
 511 * Atomically sets @v to @i with release ordering.
 512 *
 513 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
 514 *
 515 * Return: Nothing.
 516 */
 517static __always_inline void
 518raw_atomic_set_release(atomic_t *v, int i)
 519{
 520#if defined(arch_atomic_set_release)
 521	arch_atomic_set_release(v, i);
 522#else
 523	if (__native_word(atomic_t)) {
 524		smp_store_release(&(v)->counter, i);
 525	} else {
 526		__atomic_release_fence();
 527		raw_atomic_set(v, i);
 528	}
 529#endif
 530}
 531
 532/**
 533 * raw_atomic_add() - atomic add with relaxed ordering
 534 * @i: int value to add
 535 * @v: pointer to atomic_t
 536 *
 537 * Atomically updates @v to (@v + @i) with relaxed ordering.
 538 *
 539 * Safe to use in noinstr code; prefer atomic_add() elsewhere.
 540 *
 541 * Return: Nothing.
 542 */
 543static __always_inline void
 544raw_atomic_add(int i, atomic_t *v)
 545{
 546	arch_atomic_add(i, v);
 547}
 548
 549/**
 550 * raw_atomic_add_return() - atomic add with full ordering
 551 * @i: int value to add
 552 * @v: pointer to atomic_t
 553 *
 554 * Atomically updates @v to (@v + @i) with full ordering.
 555 *
 556 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
 557 *
 558 * Return: The updated value of @v.
 559 */
 560static __always_inline int
 561raw_atomic_add_return(int i, atomic_t *v)
 562{
 563#if defined(arch_atomic_add_return)
 564	return arch_atomic_add_return(i, v);
 565#elif defined(arch_atomic_add_return_relaxed)
 566	int ret;
 567	__atomic_pre_full_fence();
 568	ret = arch_atomic_add_return_relaxed(i, v);
 569	__atomic_post_full_fence();
 570	return ret;
 571#else
 572#error "Unable to define raw_atomic_add_return"
 573#endif
 574}
 575
 576/**
 577 * raw_atomic_add_return_acquire() - atomic add with acquire ordering
 578 * @i: int value to add
 579 * @v: pointer to atomic_t
 580 *
 581 * Atomically updates @v to (@v + @i) with acquire ordering.
 582 *
 583 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
 584 *
 585 * Return: The updated value of @v.
 586 */
 587static __always_inline int
 588raw_atomic_add_return_acquire(int i, atomic_t *v)
 589{
 590#if defined(arch_atomic_add_return_acquire)
 591	return arch_atomic_add_return_acquire(i, v);
 592#elif defined(arch_atomic_add_return_relaxed)
 593	int ret = arch_atomic_add_return_relaxed(i, v);
 594	__atomic_acquire_fence();
 595	return ret;
 596#elif defined(arch_atomic_add_return)
 597	return arch_atomic_add_return(i, v);
 598#else
 599#error "Unable to define raw_atomic_add_return_acquire"
 600#endif
 601}
 602
 603/**
 604 * raw_atomic_add_return_release() - atomic add with release ordering
 605 * @i: int value to add
 606 * @v: pointer to atomic_t
 607 *
 608 * Atomically updates @v to (@v + @i) with release ordering.
 609 *
 610 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
 611 *
 612 * Return: The updated value of @v.
 613 */
 614static __always_inline int
 615raw_atomic_add_return_release(int i, atomic_t *v)
 616{
 617#if defined(arch_atomic_add_return_release)
 618	return arch_atomic_add_return_release(i, v);
 619#elif defined(arch_atomic_add_return_relaxed)
 620	__atomic_release_fence();
 621	return arch_atomic_add_return_relaxed(i, v);
 622#elif defined(arch_atomic_add_return)
 623	return arch_atomic_add_return(i, v);
 624#else
 625#error "Unable to define raw_atomic_add_return_release"
 626#endif
 627}
 628
 629/**
 630 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
 631 * @i: int value to add
 632 * @v: pointer to atomic_t
 633 *
 634 * Atomically updates @v to (@v + @i) with relaxed ordering.
 635 *
 636 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
 637 *
 638 * Return: The updated value of @v.
 639 */
 640static __always_inline int
 641raw_atomic_add_return_relaxed(int i, atomic_t *v)
 642{
 643#if defined(arch_atomic_add_return_relaxed)
 644	return arch_atomic_add_return_relaxed(i, v);
 645#elif defined(arch_atomic_add_return)
 646	return arch_atomic_add_return(i, v);
 647#else
 648#error "Unable to define raw_atomic_add_return_relaxed"
 649#endif
 650}
 651
 652/**
 653 * raw_atomic_fetch_add() - atomic add with full ordering
 654 * @i: int value to add
 655 * @v: pointer to atomic_t
 656 *
 657 * Atomically updates @v to (@v + @i) with full ordering.
 658 *
 659 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
 660 *
 661 * Return: The original value of @v.
 662 */
 663static __always_inline int
 664raw_atomic_fetch_add(int i, atomic_t *v)
 665{
 666#if defined(arch_atomic_fetch_add)
 667	return arch_atomic_fetch_add(i, v);
 668#elif defined(arch_atomic_fetch_add_relaxed)
 669	int ret;
 670	__atomic_pre_full_fence();
 671	ret = arch_atomic_fetch_add_relaxed(i, v);
 672	__atomic_post_full_fence();
 673	return ret;
 674#else
 675#error "Unable to define raw_atomic_fetch_add"
 676#endif
 677}
 678
 679/**
 680 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
 681 * @i: int value to add
 682 * @v: pointer to atomic_t
 683 *
 684 * Atomically updates @v to (@v + @i) with acquire ordering.
 685 *
 686 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
 687 *
 688 * Return: The original value of @v.
 689 */
 690static __always_inline int
 691raw_atomic_fetch_add_acquire(int i, atomic_t *v)
 692{
 693#if defined(arch_atomic_fetch_add_acquire)
 694	return arch_atomic_fetch_add_acquire(i, v);
 695#elif defined(arch_atomic_fetch_add_relaxed)
 696	int ret = arch_atomic_fetch_add_relaxed(i, v);
 697	__atomic_acquire_fence();
 698	return ret;
 699#elif defined(arch_atomic_fetch_add)
 700	return arch_atomic_fetch_add(i, v);
 701#else
 702#error "Unable to define raw_atomic_fetch_add_acquire"
 703#endif
 704}
 705
 706/**
 707 * raw_atomic_fetch_add_release() - atomic add with release ordering
 708 * @i: int value to add
 709 * @v: pointer to atomic_t
 710 *
 711 * Atomically updates @v to (@v + @i) with release ordering.
 712 *
 713 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
 714 *
 715 * Return: The original value of @v.
 716 */
 717static __always_inline int
 718raw_atomic_fetch_add_release(int i, atomic_t *v)
 719{
 720#if defined(arch_atomic_fetch_add_release)
 721	return arch_atomic_fetch_add_release(i, v);
 722#elif defined(arch_atomic_fetch_add_relaxed)
 723	__atomic_release_fence();
 724	return arch_atomic_fetch_add_relaxed(i, v);
 725#elif defined(arch_atomic_fetch_add)
 726	return arch_atomic_fetch_add(i, v);
 727#else
 728#error "Unable to define raw_atomic_fetch_add_release"
 729#endif
 730}
 731
 732/**
 733 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
 734 * @i: int value to add
 735 * @v: pointer to atomic_t
 736 *
 737 * Atomically updates @v to (@v + @i) with relaxed ordering.
 738 *
 739 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
 740 *
 741 * Return: The original value of @v.
 742 */
 743static __always_inline int
 744raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
 745{
 746#if defined(arch_atomic_fetch_add_relaxed)
 747	return arch_atomic_fetch_add_relaxed(i, v);
 748#elif defined(arch_atomic_fetch_add)
 749	return arch_atomic_fetch_add(i, v);
 750#else
 751#error "Unable to define raw_atomic_fetch_add_relaxed"
 752#endif
 753}
 754
 755/**
 756 * raw_atomic_sub() - atomic subtract with relaxed ordering
 757 * @i: int value to subtract
 758 * @v: pointer to atomic_t
 759 *
 760 * Atomically updates @v to (@v - @i) with relaxed ordering.
 761 *
 762 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
 763 *
 764 * Return: Nothing.
 765 */
 766static __always_inline void
 767raw_atomic_sub(int i, atomic_t *v)
 768{
 769	arch_atomic_sub(i, v);
 770}
 771
 772/**
 773 * raw_atomic_sub_return() - atomic subtract with full ordering
 774 * @i: int value to subtract
 775 * @v: pointer to atomic_t
 776 *
 777 * Atomically updates @v to (@v - @i) with full ordering.
 778 *
 779 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
 780 *
 781 * Return: The updated value of @v.
 782 */
 783static __always_inline int
 784raw_atomic_sub_return(int i, atomic_t *v)
 785{
 786#if defined(arch_atomic_sub_return)
 787	return arch_atomic_sub_return(i, v);
 788#elif defined(arch_atomic_sub_return_relaxed)
 789	int ret;
 790	__atomic_pre_full_fence();
 791	ret = arch_atomic_sub_return_relaxed(i, v);
 792	__atomic_post_full_fence();
 793	return ret;
 794#else
 795#error "Unable to define raw_atomic_sub_return"
 796#endif
 797}
 798
 799/**
 800 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
 801 * @i: int value to subtract
 802 * @v: pointer to atomic_t
 803 *
 804 * Atomically updates @v to (@v - @i) with acquire ordering.
 805 *
 806 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
 807 *
 808 * Return: The updated value of @v.
 809 */
 810static __always_inline int
 811raw_atomic_sub_return_acquire(int i, atomic_t *v)
 812{
 813#if defined(arch_atomic_sub_return_acquire)
 814	return arch_atomic_sub_return_acquire(i, v);
 815#elif defined(arch_atomic_sub_return_relaxed)
 816	int ret = arch_atomic_sub_return_relaxed(i, v);
 817	__atomic_acquire_fence();
 818	return ret;
 819#elif defined(arch_atomic_sub_return)
 820	return arch_atomic_sub_return(i, v);
 821#else
 822#error "Unable to define raw_atomic_sub_return_acquire"
 823#endif
 824}
 825
 826/**
 827 * raw_atomic_sub_return_release() - atomic subtract with release ordering
 828 * @i: int value to subtract
 829 * @v: pointer to atomic_t
 830 *
 831 * Atomically updates @v to (@v - @i) with release ordering.
 832 *
 833 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
 834 *
 835 * Return: The updated value of @v.
 836 */
 837static __always_inline int
 838raw_atomic_sub_return_release(int i, atomic_t *v)
 839{
 840#if defined(arch_atomic_sub_return_release)
 841	return arch_atomic_sub_return_release(i, v);
 842#elif defined(arch_atomic_sub_return_relaxed)
 843	__atomic_release_fence();
 844	return arch_atomic_sub_return_relaxed(i, v);
 845#elif defined(arch_atomic_sub_return)
 846	return arch_atomic_sub_return(i, v);
 847#else
 848#error "Unable to define raw_atomic_sub_return_release"
 849#endif
 850}
 851
 852/**
 853 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
 854 * @i: int value to subtract
 855 * @v: pointer to atomic_t
 856 *
 857 * Atomically updates @v to (@v - @i) with relaxed ordering.
 858 *
 859 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
 860 *
 861 * Return: The updated value of @v.
 862 */
 863static __always_inline int
 864raw_atomic_sub_return_relaxed(int i, atomic_t *v)
 865{
 866#if defined(arch_atomic_sub_return_relaxed)
 867	return arch_atomic_sub_return_relaxed(i, v);
 868#elif defined(arch_atomic_sub_return)
 869	return arch_atomic_sub_return(i, v);
 870#else
 871#error "Unable to define raw_atomic_sub_return_relaxed"
 872#endif
 873}
 874
 875/**
 876 * raw_atomic_fetch_sub() - atomic subtract with full ordering
 877 * @i: int value to subtract
 878 * @v: pointer to atomic_t
 879 *
 880 * Atomically updates @v to (@v - @i) with full ordering.
 881 *
 882 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
 883 *
 884 * Return: The original value of @v.
 885 */
 886static __always_inline int
 887raw_atomic_fetch_sub(int i, atomic_t *v)
 888{
 889#if defined(arch_atomic_fetch_sub)
 890	return arch_atomic_fetch_sub(i, v);
 891#elif defined(arch_atomic_fetch_sub_relaxed)
 892	int ret;
 893	__atomic_pre_full_fence();
 894	ret = arch_atomic_fetch_sub_relaxed(i, v);
 895	__atomic_post_full_fence();
 896	return ret;
 897#else
 898#error "Unable to define raw_atomic_fetch_sub"
 899#endif
 900}
 901
 902/**
 903 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
 904 * @i: int value to subtract
 905 * @v: pointer to atomic_t
 906 *
 907 * Atomically updates @v to (@v - @i) with acquire ordering.
 908 *
 909 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
 910 *
 911 * Return: The original value of @v.
 912 */
 913static __always_inline int
 914raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
 915{
 916#if defined(arch_atomic_fetch_sub_acquire)
 917	return arch_atomic_fetch_sub_acquire(i, v);
 918#elif defined(arch_atomic_fetch_sub_relaxed)
 919	int ret = arch_atomic_fetch_sub_relaxed(i, v);
 920	__atomic_acquire_fence();
 921	return ret;
 922#elif defined(arch_atomic_fetch_sub)
 923	return arch_atomic_fetch_sub(i, v);
 924#else
 925#error "Unable to define raw_atomic_fetch_sub_acquire"
 926#endif
 927}
 928
 929/**
 930 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
 931 * @i: int value to subtract
 932 * @v: pointer to atomic_t
 933 *
 934 * Atomically updates @v to (@v - @i) with release ordering.
 935 *
 936 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
 937 *
 938 * Return: The original value of @v.
 939 */
 940static __always_inline int
 941raw_atomic_fetch_sub_release(int i, atomic_t *v)
 942{
 943#if defined(arch_atomic_fetch_sub_release)
 944	return arch_atomic_fetch_sub_release(i, v);
 945#elif defined(arch_atomic_fetch_sub_relaxed)
 946	__atomic_release_fence();
 947	return arch_atomic_fetch_sub_relaxed(i, v);
 948#elif defined(arch_atomic_fetch_sub)
 949	return arch_atomic_fetch_sub(i, v);
 950#else
 951#error "Unable to define raw_atomic_fetch_sub_release"
 952#endif
 953}
 954
 955/**
 956 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 957 * @i: int value to subtract
 958 * @v: pointer to atomic_t
 959 *
 960 * Atomically updates @v to (@v - @i) with relaxed ordering.
 961 *
 962 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
 963 *
 964 * Return: The original value of @v.
 965 */
 966static __always_inline int
 967raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
 968{
 969#if defined(arch_atomic_fetch_sub_relaxed)
 970	return arch_atomic_fetch_sub_relaxed(i, v);
 971#elif defined(arch_atomic_fetch_sub)
 972	return arch_atomic_fetch_sub(i, v);
 973#else
 974#error "Unable to define raw_atomic_fetch_sub_relaxed"
 975#endif
 976}
 977
 978/**
 979 * raw_atomic_inc() - atomic increment with relaxed ordering
 980 * @v: pointer to atomic_t
 981 *
 982 * Atomically updates @v to (@v + 1) with relaxed ordering.
 983 *
 984 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
 985 *
 986 * Return: Nothing.
 987 */
 988static __always_inline void
 989raw_atomic_inc(atomic_t *v)
 990{
 991#if defined(arch_atomic_inc)
 992	arch_atomic_inc(v);
 993#else
 994	raw_atomic_add(1, v);
 995#endif
 996}
 997
 998/**
 999 * raw_atomic_inc_return() - atomic increment with full ordering
1000 * @v: pointer to atomic_t
1001 *
1002 * Atomically updates @v to (@v + 1) with full ordering.
1003 *
1004 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
1005 *
1006 * Return: The updated value of @v.
1007 */
1008static __always_inline int
1009raw_atomic_inc_return(atomic_t *v)
1010{
1011#if defined(arch_atomic_inc_return)
1012	return arch_atomic_inc_return(v);
1013#elif defined(arch_atomic_inc_return_relaxed)
1014	int ret;
1015	__atomic_pre_full_fence();
1016	ret = arch_atomic_inc_return_relaxed(v);
1017	__atomic_post_full_fence();
1018	return ret;
1019#else
1020	return raw_atomic_add_return(1, v);
1021#endif
1022}
1023
1024/**
1025 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
1026 * @v: pointer to atomic_t
1027 *
1028 * Atomically updates @v to (@v + 1) with acquire ordering.
1029 *
1030 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1031 *
1032 * Return: The updated value of @v.
1033 */
1034static __always_inline int
1035raw_atomic_inc_return_acquire(atomic_t *v)
1036{
1037#if defined(arch_atomic_inc_return_acquire)
1038	return arch_atomic_inc_return_acquire(v);
1039#elif defined(arch_atomic_inc_return_relaxed)
1040	int ret = arch_atomic_inc_return_relaxed(v);
1041	__atomic_acquire_fence();
1042	return ret;
1043#elif defined(arch_atomic_inc_return)
1044	return arch_atomic_inc_return(v);
1045#else
1046	return raw_atomic_add_return_acquire(1, v);
1047#endif
1048}
1049
1050/**
1051 * raw_atomic_inc_return_release() - atomic increment with release ordering
1052 * @v: pointer to atomic_t
1053 *
1054 * Atomically updates @v to (@v + 1) with release ordering.
1055 *
1056 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1057 *
1058 * Return: The updated value of @v.
1059 */
1060static __always_inline int
1061raw_atomic_inc_return_release(atomic_t *v)
1062{
1063#if defined(arch_atomic_inc_return_release)
1064	return arch_atomic_inc_return_release(v);
1065#elif defined(arch_atomic_inc_return_relaxed)
1066	__atomic_release_fence();
1067	return arch_atomic_inc_return_relaxed(v);
1068#elif defined(arch_atomic_inc_return)
1069	return arch_atomic_inc_return(v);
1070#else
1071	return raw_atomic_add_return_release(1, v);
1072#endif
1073}
1074
1075/**
1076 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
1077 * @v: pointer to atomic_t
1078 *
1079 * Atomically updates @v to (@v + 1) with relaxed ordering.
1080 *
1081 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
1082 *
1083 * Return: The updated value of @v.
1084 */
1085static __always_inline int
1086raw_atomic_inc_return_relaxed(atomic_t *v)
1087{
1088#if defined(arch_atomic_inc_return_relaxed)
1089	return arch_atomic_inc_return_relaxed(v);
1090#elif defined(arch_atomic_inc_return)
1091	return arch_atomic_inc_return(v);
1092#else
1093	return raw_atomic_add_return_relaxed(1, v);
1094#endif
1095}
1096
1097/**
1098 * raw_atomic_fetch_inc() - atomic increment with full ordering
1099 * @v: pointer to atomic_t
1100 *
1101 * Atomically updates @v to (@v + 1) with full ordering.
1102 *
1103 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1104 *
1105 * Return: The original value of @v.
1106 */
1107static __always_inline int
1108raw_atomic_fetch_inc(atomic_t *v)
1109{
1110#if defined(arch_atomic_fetch_inc)
1111	return arch_atomic_fetch_inc(v);
1112#elif defined(arch_atomic_fetch_inc_relaxed)
1113	int ret;
1114	__atomic_pre_full_fence();
1115	ret = arch_atomic_fetch_inc_relaxed(v);
1116	__atomic_post_full_fence();
1117	return ret;
1118#else
1119	return raw_atomic_fetch_add(1, v);
1120#endif
1121}
1122
1123/**
1124 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
1125 * @v: pointer to atomic_t
1126 *
1127 * Atomically updates @v to (@v + 1) with acquire ordering.
1128 *
1129 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1130 *
1131 * Return: The original value of @v.
1132 */
1133static __always_inline int
1134raw_atomic_fetch_inc_acquire(atomic_t *v)
1135{
1136#if defined(arch_atomic_fetch_inc_acquire)
1137	return arch_atomic_fetch_inc_acquire(v);
1138#elif defined(arch_atomic_fetch_inc_relaxed)
1139	int ret = arch_atomic_fetch_inc_relaxed(v);
1140	__atomic_acquire_fence();
1141	return ret;
1142#elif defined(arch_atomic_fetch_inc)
1143	return arch_atomic_fetch_inc(v);
1144#else
1145	return raw_atomic_fetch_add_acquire(1, v);
1146#endif
1147}
1148
1149/**
1150 * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1151 * @v: pointer to atomic_t
1152 *
1153 * Atomically updates @v to (@v + 1) with release ordering.
1154 *
1155 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1156 *
1157 * Return: The original value of @v.
1158 */
1159static __always_inline int
1160raw_atomic_fetch_inc_release(atomic_t *v)
1161{
1162#if defined(arch_atomic_fetch_inc_release)
1163	return arch_atomic_fetch_inc_release(v);
1164#elif defined(arch_atomic_fetch_inc_relaxed)
1165	__atomic_release_fence();
1166	return arch_atomic_fetch_inc_relaxed(v);
1167#elif defined(arch_atomic_fetch_inc)
1168	return arch_atomic_fetch_inc(v);
1169#else
1170	return raw_atomic_fetch_add_release(1, v);
1171#endif
1172}
1173
1174/**
1175 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1176 * @v: pointer to atomic_t
1177 *
1178 * Atomically updates @v to (@v + 1) with relaxed ordering.
1179 *
1180 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1181 *
1182 * Return: The original value of @v.
1183 */
1184static __always_inline int
1185raw_atomic_fetch_inc_relaxed(atomic_t *v)
1186{
1187#if defined(arch_atomic_fetch_inc_relaxed)
1188	return arch_atomic_fetch_inc_relaxed(v);
1189#elif defined(arch_atomic_fetch_inc)
1190	return arch_atomic_fetch_inc(v);
1191#else
1192	return raw_atomic_fetch_add_relaxed(1, v);
1193#endif
1194}
1195
1196/**
1197 * raw_atomic_dec() - atomic decrement with relaxed ordering
1198 * @v: pointer to atomic_t
1199 *
1200 * Atomically updates @v to (@v - 1) with relaxed ordering.
1201 *
1202 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1203 *
1204 * Return: Nothing.
1205 */
1206static __always_inline void
1207raw_atomic_dec(atomic_t *v)
1208{
1209#if defined(arch_atomic_dec)
1210	arch_atomic_dec(v);
1211#else
1212	raw_atomic_sub(1, v);
1213#endif
1214}
1215
1216/**
1217 * raw_atomic_dec_return() - atomic decrement with full ordering
1218 * @v: pointer to atomic_t
1219 *
1220 * Atomically updates @v to (@v - 1) with full ordering.
1221 *
1222 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1223 *
1224 * Return: The updated value of @v.
1225 */
1226static __always_inline int
1227raw_atomic_dec_return(atomic_t *v)
1228{
1229#if defined(arch_atomic_dec_return)
1230	return arch_atomic_dec_return(v);
1231#elif defined(arch_atomic_dec_return_relaxed)
1232	int ret;
1233	__atomic_pre_full_fence();
1234	ret = arch_atomic_dec_return_relaxed(v);
1235	__atomic_post_full_fence();
1236	return ret;
1237#else
1238	return raw_atomic_sub_return(1, v);
1239#endif
1240}
1241
1242/**
1243 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
1244 * @v: pointer to atomic_t
1245 *
1246 * Atomically updates @v to (@v - 1) with acquire ordering.
1247 *
1248 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1249 *
1250 * Return: The updated value of @v.
1251 */
1252static __always_inline int
1253raw_atomic_dec_return_acquire(atomic_t *v)
1254{
1255#if defined(arch_atomic_dec_return_acquire)
1256	return arch_atomic_dec_return_acquire(v);
1257#elif defined(arch_atomic_dec_return_relaxed)
1258	int ret = arch_atomic_dec_return_relaxed(v);
1259	__atomic_acquire_fence();
1260	return ret;
1261#elif defined(arch_atomic_dec_return)
1262	return arch_atomic_dec_return(v);
1263#else
1264	return raw_atomic_sub_return_acquire(1, v);
1265#endif
1266}
1267
1268/**
1269 * raw_atomic_dec_return_release() - atomic decrement with release ordering
1270 * @v: pointer to atomic_t
1271 *
1272 * Atomically updates @v to (@v - 1) with release ordering.
1273 *
1274 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1275 *
1276 * Return: The updated value of @v.
1277 */
1278static __always_inline int
1279raw_atomic_dec_return_release(atomic_t *v)
1280{
1281#if defined(arch_atomic_dec_return_release)
1282	return arch_atomic_dec_return_release(v);
1283#elif defined(arch_atomic_dec_return_relaxed)
1284	__atomic_release_fence();
1285	return arch_atomic_dec_return_relaxed(v);
1286#elif defined(arch_atomic_dec_return)
1287	return arch_atomic_dec_return(v);
1288#else
1289	return raw_atomic_sub_return_release(1, v);
1290#endif
1291}
1292
1293/**
1294 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1295 * @v: pointer to atomic_t
1296 *
1297 * Atomically updates @v to (@v - 1) with relaxed ordering.
1298 *
1299 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1300 *
1301 * Return: The updated value of @v.
1302 */
1303static __always_inline int
1304raw_atomic_dec_return_relaxed(atomic_t *v)
1305{
1306#if defined(arch_atomic_dec_return_relaxed)
1307	return arch_atomic_dec_return_relaxed(v);
1308#elif defined(arch_atomic_dec_return)
1309	return arch_atomic_dec_return(v);
1310#else
1311	return raw_atomic_sub_return_relaxed(1, v);
1312#endif
1313}
1314
1315/**
1316 * raw_atomic_fetch_dec() - atomic decrement with full ordering
1317 * @v: pointer to atomic_t
1318 *
1319 * Atomically updates @v to (@v - 1) with full ordering.
1320 *
1321 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1322 *
1323 * Return: The original value of @v.
1324 */
1325static __always_inline int
1326raw_atomic_fetch_dec(atomic_t *v)
1327{
1328#if defined(arch_atomic_fetch_dec)
1329	return arch_atomic_fetch_dec(v);
1330#elif defined(arch_atomic_fetch_dec_relaxed)
1331	int ret;
1332	__atomic_pre_full_fence();
1333	ret = arch_atomic_fetch_dec_relaxed(v);
1334	__atomic_post_full_fence();
1335	return ret;
1336#else
1337	return raw_atomic_fetch_sub(1, v);
1338#endif
1339}
1340
1341/**
1342 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1343 * @v: pointer to atomic_t
1344 *
1345 * Atomically updates @v to (@v - 1) with acquire ordering.
1346 *
1347 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1348 *
1349 * Return: The original value of @v.
1350 */
1351static __always_inline int
1352raw_atomic_fetch_dec_acquire(atomic_t *v)
1353{
1354#if defined(arch_atomic_fetch_dec_acquire)
1355	return arch_atomic_fetch_dec_acquire(v);
1356#elif defined(arch_atomic_fetch_dec_relaxed)
1357	int ret = arch_atomic_fetch_dec_relaxed(v);
1358	__atomic_acquire_fence();
1359	return ret;
1360#elif defined(arch_atomic_fetch_dec)
1361	return arch_atomic_fetch_dec(v);
1362#else
1363	return raw_atomic_fetch_sub_acquire(1, v);
1364#endif
1365}
1366
1367/**
1368 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1369 * @v: pointer to atomic_t
1370 *
1371 * Atomically updates @v to (@v - 1) with release ordering.
1372 *
1373 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1374 *
1375 * Return: The original value of @v.
1376 */
1377static __always_inline int
1378raw_atomic_fetch_dec_release(atomic_t *v)
1379{
1380#if defined(arch_atomic_fetch_dec_release)
1381	return arch_atomic_fetch_dec_release(v);
1382#elif defined(arch_atomic_fetch_dec_relaxed)
1383	__atomic_release_fence();
1384	return arch_atomic_fetch_dec_relaxed(v);
1385#elif defined(arch_atomic_fetch_dec)
1386	return arch_atomic_fetch_dec(v);
1387#else
1388	return raw_atomic_fetch_sub_release(1, v);
1389#endif
1390}
1391
1392/**
1393 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1394 * @v: pointer to atomic_t
1395 *
1396 * Atomically updates @v to (@v - 1) with relaxed ordering.
1397 *
1398 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1399 *
1400 * Return: The original value of @v.
1401 */
1402static __always_inline int
1403raw_atomic_fetch_dec_relaxed(atomic_t *v)
1404{
1405#if defined(arch_atomic_fetch_dec_relaxed)
1406	return arch_atomic_fetch_dec_relaxed(v);
1407#elif defined(arch_atomic_fetch_dec)
1408	return arch_atomic_fetch_dec(v);
1409#else
1410	return raw_atomic_fetch_sub_relaxed(1, v);
1411#endif
1412}
1413
1414/**
1415 * raw_atomic_and() - atomic bitwise AND with relaxed ordering
1416 * @i: int value
1417 * @v: pointer to atomic_t
1418 *
1419 * Atomically updates @v to (@v & @i) with relaxed ordering.
1420 *
1421 * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1422 *
1423 * Return: Nothing.
1424 */
1425static __always_inline void
1426raw_atomic_and(int i, atomic_t *v)
1427{
1428	arch_atomic_and(i, v);
1429}
1430
1431/**
1432 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1433 * @i: int value
1434 * @v: pointer to atomic_t
1435 *
1436 * Atomically updates @v to (@v & @i) with full ordering.
1437 *
1438 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1439 *
1440 * Return: The original value of @v.
1441 */
1442static __always_inline int
1443raw_atomic_fetch_and(int i, atomic_t *v)
1444{
1445#if defined(arch_atomic_fetch_and)
1446	return arch_atomic_fetch_and(i, v);
1447#elif defined(arch_atomic_fetch_and_relaxed)
1448	int ret;
1449	__atomic_pre_full_fence();
1450	ret = arch_atomic_fetch_and_relaxed(i, v);
1451	__atomic_post_full_fence();
1452	return ret;
1453#else
1454#error "Unable to define raw_atomic_fetch_and"
1455#endif
1456}
1457
1458/**
1459 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
1460 * @i: int value
1461 * @v: pointer to atomic_t
1462 *
1463 * Atomically updates @v to (@v & @i) with acquire ordering.
1464 *
1465 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
1466 *
1467 * Return: The original value of @v.
1468 */
1469static __always_inline int
1470raw_atomic_fetch_and_acquire(int i, atomic_t *v)
1471{
1472#if defined(arch_atomic_fetch_and_acquire)
1473	return arch_atomic_fetch_and_acquire(i, v);
1474#elif defined(arch_atomic_fetch_and_relaxed)
1475	int ret = arch_atomic_fetch_and_relaxed(i, v);
1476	__atomic_acquire_fence();
1477	return ret;
1478#elif defined(arch_atomic_fetch_and)
1479	return arch_atomic_fetch_and(i, v);
1480#else
1481#error "Unable to define raw_atomic_fetch_and_acquire"
1482#endif
1483}
1484
1485/**
1486 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1487 * @i: int value
1488 * @v: pointer to atomic_t
1489 *
1490 * Atomically updates @v to (@v & @i) with release ordering.
1491 *
1492 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1493 *
1494 * Return: The original value of @v.
1495 */
1496static __always_inline int
1497raw_atomic_fetch_and_release(int i, atomic_t *v)
1498{
1499#if defined(arch_atomic_fetch_and_release)
1500	return arch_atomic_fetch_and_release(i, v);
1501#elif defined(arch_atomic_fetch_and_relaxed)
1502	__atomic_release_fence();
1503	return arch_atomic_fetch_and_relaxed(i, v);
1504#elif defined(arch_atomic_fetch_and)
1505	return arch_atomic_fetch_and(i, v);
1506#else
1507#error "Unable to define raw_atomic_fetch_and_release"
1508#endif
1509}
1510
1511/**
1512 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1513 * @i: int value
1514 * @v: pointer to atomic_t
1515 *
1516 * Atomically updates @v to (@v & @i) with relaxed ordering.
1517 *
1518 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1519 *
1520 * Return: The original value of @v.
1521 */
1522static __always_inline int
1523raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
1524{
1525#if defined(arch_atomic_fetch_and_relaxed)
1526	return arch_atomic_fetch_and_relaxed(i, v);
1527#elif defined(arch_atomic_fetch_and)
1528	return arch_atomic_fetch_and(i, v);
1529#else
1530#error "Unable to define raw_atomic_fetch_and_relaxed"
1531#endif
1532}
1533
1534/**
1535 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
1536 * @i: int value
1537 * @v: pointer to atomic_t
1538 *
1539 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1540 *
1541 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1542 *
1543 * Return: Nothing.
1544 */
1545static __always_inline void
1546raw_atomic_andnot(int i, atomic_t *v)
1547{
1548#if defined(arch_atomic_andnot)
1549	arch_atomic_andnot(i, v);
1550#else
1551	raw_atomic_and(~i, v);
1552#endif
1553}
1554
1555/**
1556 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
1557 * @i: int value
1558 * @v: pointer to atomic_t
1559 *
1560 * Atomically updates @v to (@v & ~@i) with full ordering.
1561 *
1562 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
1563 *
1564 * Return: The original value of @v.
1565 */
1566static __always_inline int
1567raw_atomic_fetch_andnot(int i, atomic_t *v)
1568{
1569#if defined(arch_atomic_fetch_andnot)
1570	return arch_atomic_fetch_andnot(i, v);
1571#elif defined(arch_atomic_fetch_andnot_relaxed)
1572	int ret;
1573	__atomic_pre_full_fence();
1574	ret = arch_atomic_fetch_andnot_relaxed(i, v);
1575	__atomic_post_full_fence();
1576	return ret;
1577#else
1578	return raw_atomic_fetch_and(~i, v);
1579#endif
1580}
1581
1582/**
1583 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1584 * @i: int value
1585 * @v: pointer to atomic_t
1586 *
1587 * Atomically updates @v to (@v & ~@i) with acquire ordering.
1588 *
1589 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1590 *
1591 * Return: The original value of @v.
1592 */
1593static __always_inline int
1594raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
1595{
1596#if defined(arch_atomic_fetch_andnot_acquire)
1597	return arch_atomic_fetch_andnot_acquire(i, v);
1598#elif defined(arch_atomic_fetch_andnot_relaxed)
1599	int ret = arch_atomic_fetch_andnot_relaxed(i, v);
1600	__atomic_acquire_fence();
1601	return ret;
1602#elif defined(arch_atomic_fetch_andnot)
1603	return arch_atomic_fetch_andnot(i, v);
1604#else
1605	return raw_atomic_fetch_and_acquire(~i, v);
1606#endif
1607}
1608
1609/**
1610 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1611 * @i: int value
1612 * @v: pointer to atomic_t
1613 *
1614 * Atomically updates @v to (@v & ~@i) with release ordering.
1615 *
1616 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1617 *
1618 * Return: The original value of @v.
1619 */
1620static __always_inline int
1621raw_atomic_fetch_andnot_release(int i, atomic_t *v)
1622{
1623#if defined(arch_atomic_fetch_andnot_release)
1624	return arch_atomic_fetch_andnot_release(i, v);
1625#elif defined(arch_atomic_fetch_andnot_relaxed)
1626	__atomic_release_fence();
1627	return arch_atomic_fetch_andnot_relaxed(i, v);
1628#elif defined(arch_atomic_fetch_andnot)
1629	return arch_atomic_fetch_andnot(i, v);
1630#else
1631	return raw_atomic_fetch_and_release(~i, v);
1632#endif
1633}
1634
1635/**
1636 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1637 * @i: int value
1638 * @v: pointer to atomic_t
1639 *
1640 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1641 *
1642 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1643 *
1644 * Return: The original value of @v.
1645 */
1646static __always_inline int
1647raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1648{
1649#if defined(arch_atomic_fetch_andnot_relaxed)
1650	return arch_atomic_fetch_andnot_relaxed(i, v);
1651#elif defined(arch_atomic_fetch_andnot)
1652	return arch_atomic_fetch_andnot(i, v);
1653#else
1654	return raw_atomic_fetch_and_relaxed(~i, v);
1655#endif
1656}
1657
1658/**
1659 * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1660 * @i: int value
1661 * @v: pointer to atomic_t
1662 *
1663 * Atomically updates @v to (@v | @i) with relaxed ordering.
1664 *
1665 * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1666 *
1667 * Return: Nothing.
1668 */
1669static __always_inline void
1670raw_atomic_or(int i, atomic_t *v)
1671{
1672	arch_atomic_or(i, v);
1673}
1674
1675/**
1676 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1677 * @i: int value
1678 * @v: pointer to atomic_t
1679 *
1680 * Atomically updates @v to (@v | @i) with full ordering.
1681 *
1682 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1683 *
1684 * Return: The original value of @v.
1685 */
1686static __always_inline int
1687raw_atomic_fetch_or(int i, atomic_t *v)
1688{
1689#if defined(arch_atomic_fetch_or)
1690	return arch_atomic_fetch_or(i, v);
1691#elif defined(arch_atomic_fetch_or_relaxed)
1692	int ret;
1693	__atomic_pre_full_fence();
1694	ret = arch_atomic_fetch_or_relaxed(i, v);
1695	__atomic_post_full_fence();
1696	return ret;
1697#else
1698#error "Unable to define raw_atomic_fetch_or"
1699#endif
1700}
1701
1702/**
1703 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1704 * @i: int value
1705 * @v: pointer to atomic_t
1706 *
1707 * Atomically updates @v to (@v | @i) with acquire ordering.
1708 *
1709 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
1710 *
1711 * Return: The original value of @v.
1712 */
1713static __always_inline int
1714raw_atomic_fetch_or_acquire(int i, atomic_t *v)
1715{
1716#if defined(arch_atomic_fetch_or_acquire)
1717	return arch_atomic_fetch_or_acquire(i, v);
1718#elif defined(arch_atomic_fetch_or_relaxed)
1719	int ret = arch_atomic_fetch_or_relaxed(i, v);
1720	__atomic_acquire_fence();
1721	return ret;
1722#elif defined(arch_atomic_fetch_or)
1723	return arch_atomic_fetch_or(i, v);
1724#else
1725#error "Unable to define raw_atomic_fetch_or_acquire"
1726#endif
1727}
1728
1729/**
1730 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1731 * @i: int value
1732 * @v: pointer to atomic_t
1733 *
1734 * Atomically updates @v to (@v | @i) with release ordering.
1735 *
1736 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1737 *
1738 * Return: The original value of @v.
1739 */
1740static __always_inline int
1741raw_atomic_fetch_or_release(int i, atomic_t *v)
1742{
1743#if defined(arch_atomic_fetch_or_release)
1744	return arch_atomic_fetch_or_release(i, v);
1745#elif defined(arch_atomic_fetch_or_relaxed)
1746	__atomic_release_fence();
1747	return arch_atomic_fetch_or_relaxed(i, v);
1748#elif defined(arch_atomic_fetch_or)
1749	return arch_atomic_fetch_or(i, v);
1750#else
1751#error "Unable to define raw_atomic_fetch_or_release"
1752#endif
1753}
1754
1755/**
1756 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1757 * @i: int value
1758 * @v: pointer to atomic_t
1759 *
1760 * Atomically updates @v to (@v | @i) with relaxed ordering.
1761 *
1762 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1763 *
1764 * Return: The original value of @v.
1765 */
1766static __always_inline int
1767raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1768{
1769#if defined(arch_atomic_fetch_or_relaxed)
1770	return arch_atomic_fetch_or_relaxed(i, v);
1771#elif defined(arch_atomic_fetch_or)
1772	return arch_atomic_fetch_or(i, v);
1773#else
1774#error "Unable to define raw_atomic_fetch_or_relaxed"
1775#endif
1776}
1777
1778/**
1779 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1780 * @i: int value
1781 * @v: pointer to atomic_t
1782 *
1783 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1784 *
1785 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1786 *
1787 * Return: Nothing.
1788 */
1789static __always_inline void
1790raw_atomic_xor(int i, atomic_t *v)
1791{
1792	arch_atomic_xor(i, v);
1793}
1794
1795/**
1796 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1797 * @i: int value
1798 * @v: pointer to atomic_t
1799 *
1800 * Atomically updates @v to (@v ^ @i) with full ordering.
1801 *
1802 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1803 *
1804 * Return: The original value of @v.
1805 */
1806static __always_inline int
1807raw_atomic_fetch_xor(int i, atomic_t *v)
1808{
1809#if defined(arch_atomic_fetch_xor)
1810	return arch_atomic_fetch_xor(i, v);
1811#elif defined(arch_atomic_fetch_xor_relaxed)
1812	int ret;
1813	__atomic_pre_full_fence();
1814	ret = arch_atomic_fetch_xor_relaxed(i, v);
1815	__atomic_post_full_fence();
1816	return ret;
1817#else
1818#error "Unable to define raw_atomic_fetch_xor"
1819#endif
1820}
1821
1822/**
1823 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1824 * @i: int value
1825 * @v: pointer to atomic_t
1826 *
1827 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1828 *
1829 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
1830 *
1831 * Return: The original value of @v.
1832 */
1833static __always_inline int
1834raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
1835{
1836#if defined(arch_atomic_fetch_xor_acquire)
1837	return arch_atomic_fetch_xor_acquire(i, v);
1838#elif defined(arch_atomic_fetch_xor_relaxed)
1839	int ret = arch_atomic_fetch_xor_relaxed(i, v);
1840	__atomic_acquire_fence();
1841	return ret;
1842#elif defined(arch_atomic_fetch_xor)
1843	return arch_atomic_fetch_xor(i, v);
1844#else
1845#error "Unable to define raw_atomic_fetch_xor_acquire"
1846#endif
1847}
1848
1849/**
1850 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1851 * @i: int value
1852 * @v: pointer to atomic_t
1853 *
1854 * Atomically updates @v to (@v ^ @i) with release ordering.
1855 *
1856 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1857 *
1858 * Return: The original value of @v.
1859 */
1860static __always_inline int
1861raw_atomic_fetch_xor_release(int i, atomic_t *v)
1862{
1863#if defined(arch_atomic_fetch_xor_release)
1864	return arch_atomic_fetch_xor_release(i, v);
1865#elif defined(arch_atomic_fetch_xor_relaxed)
1866	__atomic_release_fence();
1867	return arch_atomic_fetch_xor_relaxed(i, v);
1868#elif defined(arch_atomic_fetch_xor)
1869	return arch_atomic_fetch_xor(i, v);
1870#else
1871#error "Unable to define raw_atomic_fetch_xor_release"
1872#endif
1873}
1874
1875/**
1876 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1877 * @i: int value
1878 * @v: pointer to atomic_t
1879 *
1880 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1881 *
1882 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1883 *
1884 * Return: The original value of @v.
1885 */
1886static __always_inline int
1887raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1888{
1889#if defined(arch_atomic_fetch_xor_relaxed)
1890	return arch_atomic_fetch_xor_relaxed(i, v);
1891#elif defined(arch_atomic_fetch_xor)
1892	return arch_atomic_fetch_xor(i, v);
1893#else
1894#error "Unable to define raw_atomic_fetch_xor_relaxed"
1895#endif
1896}
1897
1898/**
1899 * raw_atomic_xchg() - atomic exchange with full ordering
1900 * @v: pointer to atomic_t
1901 * @new: int value to assign
1902 *
1903 * Atomically updates @v to @new with full ordering.
1904 *
1905 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1906 *
1907 * Return: The original value of @v.
1908 */
1909static __always_inline int
1910raw_atomic_xchg(atomic_t *v, int new)
1911{
1912#if defined(arch_atomic_xchg)
1913	return arch_atomic_xchg(v, new);
1914#elif defined(arch_atomic_xchg_relaxed)
1915	int ret;
1916	__atomic_pre_full_fence();
1917	ret = arch_atomic_xchg_relaxed(v, new);
1918	__atomic_post_full_fence();
1919	return ret;
1920#else
1921	return raw_xchg(&v->counter, new);
1922#endif
1923}
1924
1925/**
1926 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1927 * @v: pointer to atomic_t
1928 * @new: int value to assign
1929 *
1930 * Atomically updates @v to @new with acquire ordering.
1931 *
1932 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1933 *
1934 * Return: The original value of @v.
1935 */
1936static __always_inline int
1937raw_atomic_xchg_acquire(atomic_t *v, int new)
1938{
1939#if defined(arch_atomic_xchg_acquire)
1940	return arch_atomic_xchg_acquire(v, new);
1941#elif defined(arch_atomic_xchg_relaxed)
1942	int ret = arch_atomic_xchg_relaxed(v, new);
1943	__atomic_acquire_fence();
1944	return ret;
1945#elif defined(arch_atomic_xchg)
1946	return arch_atomic_xchg(v, new);
1947#else
1948	return raw_xchg_acquire(&v->counter, new);
1949#endif
1950}
1951
1952/**
1953 * raw_atomic_xchg_release() - atomic exchange with release ordering
1954 * @v: pointer to atomic_t
1955 * @new: int value to assign
1956 *
1957 * Atomically updates @v to @new with release ordering.
1958 *
1959 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1960 *
1961 * Return: The original value of @v.
1962 */
1963static __always_inline int
1964raw_atomic_xchg_release(atomic_t *v, int new)
1965{
1966#if defined(arch_atomic_xchg_release)
1967	return arch_atomic_xchg_release(v, new);
1968#elif defined(arch_atomic_xchg_relaxed)
1969	__atomic_release_fence();
1970	return arch_atomic_xchg_relaxed(v, new);
1971#elif defined(arch_atomic_xchg)
1972	return arch_atomic_xchg(v, new);
1973#else
1974	return raw_xchg_release(&v->counter, new);
1975#endif
1976}
1977
1978/**
1979 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1980 * @v: pointer to atomic_t
1981 * @new: int value to assign
1982 *
1983 * Atomically updates @v to @new with relaxed ordering.
1984 *
1985 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1986 *
1987 * Return: The original value of @v.
1988 */
1989static __always_inline int
1990raw_atomic_xchg_relaxed(atomic_t *v, int new)
1991{
1992#if defined(arch_atomic_xchg_relaxed)
1993	return arch_atomic_xchg_relaxed(v, new);
1994#elif defined(arch_atomic_xchg)
1995	return arch_atomic_xchg(v, new);
1996#else
1997	return raw_xchg_relaxed(&v->counter, new);
1998#endif
1999}
2000
2001/**
2002 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
2003 * @v: pointer to atomic_t
2004 * @old: int value to compare with
2005 * @new: int value to assign
2006 *
2007 * If (@v == @old), atomically updates @v to @new with full ordering.
2008 * Otherwise, @v is not modified and relaxed ordering is provided.
2009 *
2010 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
2011 *
2012 * Return: The original value of @v.
2013 */
2014static __always_inline int
2015raw_atomic_cmpxchg(atomic_t *v, int old, int new)
2016{
2017#if defined(arch_atomic_cmpxchg)
2018	return arch_atomic_cmpxchg(v, old, new);
2019#elif defined(arch_atomic_cmpxchg_relaxed)
2020	int ret;
2021	__atomic_pre_full_fence();
2022	ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2023	__atomic_post_full_fence();
2024	return ret;
2025#else
2026	return raw_cmpxchg(&v->counter, old, new);
2027#endif
2028}
2029
2030/**
2031 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2032 * @v: pointer to atomic_t
2033 * @old: int value to compare with
2034 * @new: int value to assign
2035 *
2036 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2037 * Otherwise, @v is not modified and relaxed ordering is provided.
2038 *
2039 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2040 *
2041 * Return: The original value of @v.
2042 */
2043static __always_inline int
2044raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
2045{
2046#if defined(arch_atomic_cmpxchg_acquire)
2047	return arch_atomic_cmpxchg_acquire(v, old, new);
2048#elif defined(arch_atomic_cmpxchg_relaxed)
2049	int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2050	__atomic_acquire_fence();
2051	return ret;
2052#elif defined(arch_atomic_cmpxchg)
2053	return arch_atomic_cmpxchg(v, old, new);
2054#else
2055	return raw_cmpxchg_acquire(&v->counter, old, new);
2056#endif
2057}
2058
2059/**
2060 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2061 * @v: pointer to atomic_t
2062 * @old: int value to compare with
2063 * @new: int value to assign
2064 *
2065 * If (@v == @old), atomically updates @v to @new with release ordering.
2066 * Otherwise, @v is not modified and relaxed ordering is provided.
2067 *
2068 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2069 *
2070 * Return: The original value of @v.
2071 */
2072static __always_inline int
2073raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
2074{
2075#if defined(arch_atomic_cmpxchg_release)
2076	return arch_atomic_cmpxchg_release(v, old, new);
2077#elif defined(arch_atomic_cmpxchg_relaxed)
2078	__atomic_release_fence();
2079	return arch_atomic_cmpxchg_relaxed(v, old, new);
2080#elif defined(arch_atomic_cmpxchg)
2081	return arch_atomic_cmpxchg(v, old, new);
2082#else
2083	return raw_cmpxchg_release(&v->counter, old, new);
2084#endif
2085}
2086
2087/**
2088 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2089 * @v: pointer to atomic_t
2090 * @old: int value to compare with
2091 * @new: int value to assign
2092 *
2093 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2094 * Otherwise, @v is not modified and relaxed ordering is provided.
2095 *
2096 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2097 *
2098 * Return: The original value of @v.
2099 */
2100static __always_inline int
2101raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
2102{
2103#if defined(arch_atomic_cmpxchg_relaxed)
2104	return arch_atomic_cmpxchg_relaxed(v, old, new);
2105#elif defined(arch_atomic_cmpxchg)
2106	return arch_atomic_cmpxchg(v, old, new);
2107#else
2108	return raw_cmpxchg_relaxed(&v->counter, old, new);
2109#endif
2110}
2111
2112/**
2113 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
2114 * @v: pointer to atomic_t
2115 * @old: pointer to int value to compare with
2116 * @new: int value to assign
2117 *
2118 * If (@v == @old), atomically updates @v to @new with full ordering.
2119 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2120 * and relaxed ordering is provided.
2121 *
2122 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2123 *
2124 * Return: @true if the exchange occured, @false otherwise.
2125 */
2126static __always_inline bool
2127raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
2128{
2129#if defined(arch_atomic_try_cmpxchg)
2130	return arch_atomic_try_cmpxchg(v, old, new);
2131#elif defined(arch_atomic_try_cmpxchg_relaxed)
2132	bool ret;
2133	__atomic_pre_full_fence();
2134	ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2135	__atomic_post_full_fence();
2136	return ret;
2137#else
2138	int r, o = *old;
2139	r = raw_atomic_cmpxchg(v, o, new);
2140	if (unlikely(r != o))
2141		*old = r;
2142	return likely(r == o);
2143#endif
2144}
2145
2146/**
2147 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2148 * @v: pointer to atomic_t
2149 * @old: pointer to int value to compare with
2150 * @new: int value to assign
2151 *
2152 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2153 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2154 * and relaxed ordering is provided.
2155 *
2156 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2157 *
2158 * Return: @true if the exchange occured, @false otherwise.
2159 */
2160static __always_inline bool
2161raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
2162{
2163#if defined(arch_atomic_try_cmpxchg_acquire)
2164	return arch_atomic_try_cmpxchg_acquire(v, old, new);
2165#elif defined(arch_atomic_try_cmpxchg_relaxed)
2166	bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2167	__atomic_acquire_fence();
2168	return ret;
2169#elif defined(arch_atomic_try_cmpxchg)
2170	return arch_atomic_try_cmpxchg(v, old, new);
2171#else
2172	int r, o = *old;
2173	r = raw_atomic_cmpxchg_acquire(v, o, new);
2174	if (unlikely(r != o))
2175		*old = r;
2176	return likely(r == o);
2177#endif
2178}
2179
2180/**
2181 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2182 * @v: pointer to atomic_t
2183 * @old: pointer to int value to compare with
2184 * @new: int value to assign
2185 *
2186 * If (@v == @old), atomically updates @v to @new with release ordering.
2187 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2188 * and relaxed ordering is provided.
2189 *
2190 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2191 *
2192 * Return: @true if the exchange occured, @false otherwise.
2193 */
2194static __always_inline bool
2195raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
2196{
2197#if defined(arch_atomic_try_cmpxchg_release)
2198	return arch_atomic_try_cmpxchg_release(v, old, new);
2199#elif defined(arch_atomic_try_cmpxchg_relaxed)
2200	__atomic_release_fence();
2201	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2202#elif defined(arch_atomic_try_cmpxchg)
2203	return arch_atomic_try_cmpxchg(v, old, new);
2204#else
2205	int r, o = *old;
2206	r = raw_atomic_cmpxchg_release(v, o, new);
2207	if (unlikely(r != o))
2208		*old = r;
2209	return likely(r == o);
2210#endif
2211}
2212
2213/**
2214 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2215 * @v: pointer to atomic_t
2216 * @old: pointer to int value to compare with
2217 * @new: int value to assign
2218 *
2219 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2220 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2221 * and relaxed ordering is provided.
2222 *
2223 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2224 *
2225 * Return: @true if the exchange occured, @false otherwise.
2226 */
2227static __always_inline bool
2228raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
2229{
2230#if defined(arch_atomic_try_cmpxchg_relaxed)
2231	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2232#elif defined(arch_atomic_try_cmpxchg)
2233	return arch_atomic_try_cmpxchg(v, old, new);
2234#else
2235	int r, o = *old;
2236	r = raw_atomic_cmpxchg_relaxed(v, o, new);
2237	if (unlikely(r != o))
2238		*old = r;
2239	return likely(r == o);
2240#endif
2241}
2242
2243/**
2244 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
2245 * @i: int value to subtract
2246 * @v: pointer to atomic_t
2247 *
2248 * Atomically updates @v to (@v - @i) with full ordering.
2249 *
2250 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
2251 *
2252 * Return: @true if the resulting value of @v is zero, @false otherwise.
2253 */
2254static __always_inline bool
2255raw_atomic_sub_and_test(int i, atomic_t *v)
2256{
2257#if defined(arch_atomic_sub_and_test)
2258	return arch_atomic_sub_and_test(i, v);
2259#else
2260	return raw_atomic_sub_return(i, v) == 0;
2261#endif
2262}
2263
2264/**
2265 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
2266 * @v: pointer to atomic_t
2267 *
2268 * Atomically updates @v to (@v - 1) with full ordering.
2269 *
2270 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
2271 *
2272 * Return: @true if the resulting value of @v is zero, @false otherwise.
2273 */
2274static __always_inline bool
2275raw_atomic_dec_and_test(atomic_t *v)
2276{
2277#if defined(arch_atomic_dec_and_test)
2278	return arch_atomic_dec_and_test(v);
2279#else
2280	return raw_atomic_dec_return(v) == 0;
2281#endif
2282}
2283
2284/**
2285 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
2286 * @v: pointer to atomic_t
2287 *
2288 * Atomically updates @v to (@v + 1) with full ordering.
2289 *
2290 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
2291 *
2292 * Return: @true if the resulting value of @v is zero, @false otherwise.
2293 */
2294static __always_inline bool
2295raw_atomic_inc_and_test(atomic_t *v)
2296{
2297#if defined(arch_atomic_inc_and_test)
2298	return arch_atomic_inc_and_test(v);
2299#else
2300	return raw_atomic_inc_return(v) == 0;
2301#endif
2302}
2303
2304/**
2305 * raw_atomic_add_negative() - atomic add and test if negative with full ordering
2306 * @i: int value to add
2307 * @v: pointer to atomic_t
2308 *
2309 * Atomically updates @v to (@v + @i) with full ordering.
2310 *
2311 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2312 *
2313 * Return: @true if the resulting value of @v is negative, @false otherwise.
2314 */
2315static __always_inline bool
2316raw_atomic_add_negative(int i, atomic_t *v)
2317{
2318#if defined(arch_atomic_add_negative)
2319	return arch_atomic_add_negative(i, v);
2320#elif defined(arch_atomic_add_negative_relaxed)
2321	bool ret;
2322	__atomic_pre_full_fence();
2323	ret = arch_atomic_add_negative_relaxed(i, v);
2324	__atomic_post_full_fence();
2325	return ret;
2326#else
2327	return raw_atomic_add_return(i, v) < 0;
2328#endif
2329}
2330
2331/**
2332 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
2333 * @i: int value to add
2334 * @v: pointer to atomic_t
2335 *
2336 * Atomically updates @v to (@v + @i) with acquire ordering.
2337 *
2338 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
2339 *
2340 * Return: @true if the resulting value of @v is negative, @false otherwise.
2341 */
2342static __always_inline bool
2343raw_atomic_add_negative_acquire(int i, atomic_t *v)
2344{
2345#if defined(arch_atomic_add_negative_acquire)
2346	return arch_atomic_add_negative_acquire(i, v);
2347#elif defined(arch_atomic_add_negative_relaxed)
2348	bool ret = arch_atomic_add_negative_relaxed(i, v);
2349	__atomic_acquire_fence();
2350	return ret;
2351#elif defined(arch_atomic_add_negative)
2352	return arch_atomic_add_negative(i, v);
2353#else
2354	return raw_atomic_add_return_acquire(i, v) < 0;
2355#endif
2356}
2357
2358/**
2359 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
2360 * @i: int value to add
2361 * @v: pointer to atomic_t
2362 *
2363 * Atomically updates @v to (@v + @i) with release ordering.
2364 *
2365 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2366 *
2367 * Return: @true if the resulting value of @v is negative, @false otherwise.
2368 */
2369static __always_inline bool
2370raw_atomic_add_negative_release(int i, atomic_t *v)
2371{
2372#if defined(arch_atomic_add_negative_release)
2373	return arch_atomic_add_negative_release(i, v);
2374#elif defined(arch_atomic_add_negative_relaxed)
2375	__atomic_release_fence();
2376	return arch_atomic_add_negative_relaxed(i, v);
2377#elif defined(arch_atomic_add_negative)
2378	return arch_atomic_add_negative(i, v);
2379#else
2380	return raw_atomic_add_return_release(i, v) < 0;
2381#endif
2382}
2383
2384/**
2385 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
2386 * @i: int value to add
2387 * @v: pointer to atomic_t
2388 *
2389 * Atomically updates @v to (@v + @i) with relaxed ordering.
2390 *
2391 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
2392 *
2393 * Return: @true if the resulting value of @v is negative, @false otherwise.
2394 */
2395static __always_inline bool
2396raw_atomic_add_negative_relaxed(int i, atomic_t *v)
2397{
2398#if defined(arch_atomic_add_negative_relaxed)
2399	return arch_atomic_add_negative_relaxed(i, v);
2400#elif defined(arch_atomic_add_negative)
2401	return arch_atomic_add_negative(i, v);
2402#else
2403	return raw_atomic_add_return_relaxed(i, v) < 0;
2404#endif
2405}
2406
2407/**
2408 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
2409 * @v: pointer to atomic_t
2410 * @a: int value to add
2411 * @u: int value to compare with
2412 *
2413 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2414 * Otherwise, @v is not modified and relaxed ordering is provided.
2415 *
2416 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2417 *
2418 * Return: The original value of @v.
2419 */
2420static __always_inline int
2421raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
2422{
2423#if defined(arch_atomic_fetch_add_unless)
2424	return arch_atomic_fetch_add_unless(v, a, u);
2425#else
2426	int c = raw_atomic_read(v);
2427
2428	do {
2429		if (unlikely(c == u))
2430			break;
2431	} while (!raw_atomic_try_cmpxchg(v, &c, c + a));
2432
2433	return c;
2434#endif
2435}
2436
2437/**
2438 * raw_atomic_add_unless() - atomic add unless value with full ordering
2439 * @v: pointer to atomic_t
2440 * @a: int value to add
2441 * @u: int value to compare with
2442 *
2443 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2444 * Otherwise, @v is not modified and relaxed ordering is provided.
2445 *
2446 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2447 *
2448 * Return: @true if @v was updated, @false otherwise.
2449 */
2450static __always_inline bool
2451raw_atomic_add_unless(atomic_t *v, int a, int u)
2452{
2453#if defined(arch_atomic_add_unless)
2454	return arch_atomic_add_unless(v, a, u);
2455#else
2456	return raw_atomic_fetch_add_unless(v, a, u) != u;
2457#endif
2458}
2459
2460/**
2461 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
2462 * @v: pointer to atomic_t
2463 *
2464 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2465 * Otherwise, @v is not modified and relaxed ordering is provided.
2466 *
2467 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2468 *
2469 * Return: @true if @v was updated, @false otherwise.
2470 */
2471static __always_inline bool
2472raw_atomic_inc_not_zero(atomic_t *v)
2473{
2474#if defined(arch_atomic_inc_not_zero)
2475	return arch_atomic_inc_not_zero(v);
2476#else
2477	return raw_atomic_add_unless(v, 1, 0);
2478#endif
2479}
2480
2481/**
2482 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2483 * @v: pointer to atomic_t
2484 *
2485 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2486 * Otherwise, @v is not modified and relaxed ordering is provided.
2487 *
2488 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2489 *
2490 * Return: @true if @v was updated, @false otherwise.
2491 */
2492static __always_inline bool
2493raw_atomic_inc_unless_negative(atomic_t *v)
2494{
2495#if defined(arch_atomic_inc_unless_negative)
2496	return arch_atomic_inc_unless_negative(v);
2497#else
2498	int c = raw_atomic_read(v);
2499
2500	do {
2501		if (unlikely(c < 0))
2502			return false;
2503	} while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
2504
2505	return true;
2506#endif
2507}
2508
2509/**
2510 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2511 * @v: pointer to atomic_t
2512 *
2513 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2514 * Otherwise, @v is not modified and relaxed ordering is provided.
2515 *
2516 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2517 *
2518 * Return: @true if @v was updated, @false otherwise.
2519 */
2520static __always_inline bool
2521raw_atomic_dec_unless_positive(atomic_t *v)
2522{
2523#if defined(arch_atomic_dec_unless_positive)
2524	return arch_atomic_dec_unless_positive(v);
2525#else
2526	int c = raw_atomic_read(v);
2527
2528	do {
2529		if (unlikely(c > 0))
2530			return false;
2531	} while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
2532
2533	return true;
2534#endif
2535}
2536
2537/**
2538 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2539 * @v: pointer to atomic_t
2540 *
2541 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2542 * Otherwise, @v is not modified and relaxed ordering is provided.
2543 *
2544 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2545 *
2546 * Return: The old value of (@v - 1), regardless of whether @v was updated.
2547 */
2548static __always_inline int
2549raw_atomic_dec_if_positive(atomic_t *v)
2550{
2551#if defined(arch_atomic_dec_if_positive)
2552	return arch_atomic_dec_if_positive(v);
2553#else
2554	int dec, c = raw_atomic_read(v);
2555
2556	do {
2557		dec = c - 1;
2558		if (unlikely(dec < 0))
2559			break;
2560	} while (!raw_atomic_try_cmpxchg(v, &c, dec));
2561
2562	return dec;
2563#endif
2564}
2565
2566#ifdef CONFIG_GENERIC_ATOMIC64
2567#include <asm-generic/atomic64.h>
2568#endif
2569
2570/**
2571 * raw_atomic64_read() - atomic load with relaxed ordering
2572 * @v: pointer to atomic64_t
2573 *
2574 * Atomically loads the value of @v with relaxed ordering.
2575 *
2576 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2577 *
2578 * Return: The value loaded from @v.
2579 */
2580static __always_inline s64
2581raw_atomic64_read(const atomic64_t *v)
2582{
2583	return arch_atomic64_read(v);
2584}
2585
2586/**
2587 * raw_atomic64_read_acquire() - atomic load with acquire ordering
2588 * @v: pointer to atomic64_t
2589 *
2590 * Atomically loads the value of @v with acquire ordering.
2591 *
2592 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2593 *
2594 * Return: The value loaded from @v.
2595 */
2596static __always_inline s64
2597raw_atomic64_read_acquire(const atomic64_t *v)
2598{
2599#if defined(arch_atomic64_read_acquire)
2600	return arch_atomic64_read_acquire(v);
2601#else
2602	s64 ret;
2603
2604	if (__native_word(atomic64_t)) {
2605		ret = smp_load_acquire(&(v)->counter);
2606	} else {
2607		ret = raw_atomic64_read(v);
2608		__atomic_acquire_fence();
2609	}
2610
2611	return ret;
2612#endif
2613}
2614
2615/**
2616 * raw_atomic64_set() - atomic set with relaxed ordering
2617 * @v: pointer to atomic64_t
2618 * @i: s64 value to assign
2619 *
2620 * Atomically sets @v to @i with relaxed ordering.
2621 *
2622 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2623 *
2624 * Return: Nothing.
2625 */
2626static __always_inline void
2627raw_atomic64_set(atomic64_t *v, s64 i)
2628{
2629	arch_atomic64_set(v, i);
2630}
2631
2632/**
2633 * raw_atomic64_set_release() - atomic set with release ordering
2634 * @v: pointer to atomic64_t
2635 * @i: s64 value to assign
2636 *
2637 * Atomically sets @v to @i with release ordering.
2638 *
2639 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2640 *
2641 * Return: Nothing.
2642 */
2643static __always_inline void
2644raw_atomic64_set_release(atomic64_t *v, s64 i)
2645{
2646#if defined(arch_atomic64_set_release)
2647	arch_atomic64_set_release(v, i);
2648#else
2649	if (__native_word(atomic64_t)) {
2650		smp_store_release(&(v)->counter, i);
2651	} else {
2652		__atomic_release_fence();
2653		raw_atomic64_set(v, i);
2654	}
2655#endif
2656}
2657
2658/**
2659 * raw_atomic64_add() - atomic add with relaxed ordering
2660 * @i: s64 value to add
2661 * @v: pointer to atomic64_t
2662 *
2663 * Atomically updates @v to (@v + @i) with relaxed ordering.
2664 *
2665 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2666 *
2667 * Return: Nothing.
2668 */
2669static __always_inline void
2670raw_atomic64_add(s64 i, atomic64_t *v)
2671{
2672	arch_atomic64_add(i, v);
2673}
2674
2675/**
2676 * raw_atomic64_add_return() - atomic add with full ordering
2677 * @i: s64 value to add
2678 * @v: pointer to atomic64_t
2679 *
2680 * Atomically updates @v to (@v + @i) with full ordering.
2681 *
2682 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2683 *
2684 * Return: The updated value of @v.
2685 */
2686static __always_inline s64
2687raw_atomic64_add_return(s64 i, atomic64_t *v)
2688{
2689#if defined(arch_atomic64_add_return)
2690	return arch_atomic64_add_return(i, v);
2691#elif defined(arch_atomic64_add_return_relaxed)
2692	s64 ret;
2693	__atomic_pre_full_fence();
2694	ret = arch_atomic64_add_return_relaxed(i, v);
2695	__atomic_post_full_fence();
2696	return ret;
2697#else
2698#error "Unable to define raw_atomic64_add_return"
2699#endif
2700}
2701
2702/**
2703 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2704 * @i: s64 value to add
2705 * @v: pointer to atomic64_t
2706 *
2707 * Atomically updates @v to (@v + @i) with acquire ordering.
2708 *
2709 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2710 *
2711 * Return: The updated value of @v.
2712 */
2713static __always_inline s64
2714raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
2715{
2716#if defined(arch_atomic64_add_return_acquire)
2717	return arch_atomic64_add_return_acquire(i, v);
2718#elif defined(arch_atomic64_add_return_relaxed)
2719	s64 ret = arch_atomic64_add_return_relaxed(i, v);
2720	__atomic_acquire_fence();
2721	return ret;
2722#elif defined(arch_atomic64_add_return)
2723	return arch_atomic64_add_return(i, v);
2724#else
2725#error "Unable to define raw_atomic64_add_return_acquire"
2726#endif
2727}
2728
2729/**
2730 * raw_atomic64_add_return_release() - atomic add with release ordering
2731 * @i: s64 value to add
2732 * @v: pointer to atomic64_t
2733 *
2734 * Atomically updates @v to (@v + @i) with release ordering.
2735 *
2736 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2737 *
2738 * Return: The updated value of @v.
2739 */
2740static __always_inline s64
2741raw_atomic64_add_return_release(s64 i, atomic64_t *v)
2742{
2743#if defined(arch_atomic64_add_return_release)
2744	return arch_atomic64_add_return_release(i, v);
2745#elif defined(arch_atomic64_add_return_relaxed)
2746	__atomic_release_fence();
2747	return arch_atomic64_add_return_relaxed(i, v);
2748#elif defined(arch_atomic64_add_return)
2749	return arch_atomic64_add_return(i, v);
2750#else
2751#error "Unable to define raw_atomic64_add_return_release"
2752#endif
2753}
2754
2755/**
2756 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2757 * @i: s64 value to add
2758 * @v: pointer to atomic64_t
2759 *
2760 * Atomically updates @v to (@v + @i) with relaxed ordering.
2761 *
2762 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2763 *
2764 * Return: The updated value of @v.
2765 */
2766static __always_inline s64
2767raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2768{
2769#if defined(arch_atomic64_add_return_relaxed)
2770	return arch_atomic64_add_return_relaxed(i, v);
2771#elif defined(arch_atomic64_add_return)
2772	return arch_atomic64_add_return(i, v);
2773#else
2774#error "Unable to define raw_atomic64_add_return_relaxed"
2775#endif
2776}
2777
2778/**
2779 * raw_atomic64_fetch_add() - atomic add with full ordering
2780 * @i: s64 value to add
2781 * @v: pointer to atomic64_t
2782 *
2783 * Atomically updates @v to (@v + @i) with full ordering.
2784 *
2785 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2786 *
2787 * Return: The original value of @v.
2788 */
2789static __always_inline s64
2790raw_atomic64_fetch_add(s64 i, atomic64_t *v)
2791{
2792#if defined(arch_atomic64_fetch_add)
2793	return arch_atomic64_fetch_add(i, v);
2794#elif defined(arch_atomic64_fetch_add_relaxed)
2795	s64 ret;
2796	__atomic_pre_full_fence();
2797	ret = arch_atomic64_fetch_add_relaxed(i, v);
2798	__atomic_post_full_fence();
2799	return ret;
2800#else
2801#error "Unable to define raw_atomic64_fetch_add"
2802#endif
2803}
2804
2805/**
2806 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
2807 * @i: s64 value to add
2808 * @v: pointer to atomic64_t
2809 *
2810 * Atomically updates @v to (@v + @i) with acquire ordering.
2811 *
2812 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
2813 *
2814 * Return: The original value of @v.
2815 */
2816static __always_inline s64
2817raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
2818{
2819#if defined(arch_atomic64_fetch_add_acquire)
2820	return arch_atomic64_fetch_add_acquire(i, v);
2821#elif defined(arch_atomic64_fetch_add_relaxed)
2822	s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
2823	__atomic_acquire_fence();
2824	return ret;
2825#elif defined(arch_atomic64_fetch_add)
2826	return arch_atomic64_fetch_add(i, v);
2827#else
2828#error "Unable to define raw_atomic64_fetch_add_acquire"
2829#endif
2830}
2831
2832/**
2833 * raw_atomic64_fetch_add_release() - atomic add with release ordering
2834 * @i: s64 value to add
2835 * @v: pointer to atomic64_t
2836 *
2837 * Atomically updates @v to (@v + @i) with release ordering.
2838 *
2839 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2840 *
2841 * Return: The original value of @v.
2842 */
2843static __always_inline s64
2844raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
2845{
2846#if defined(arch_atomic64_fetch_add_release)
2847	return arch_atomic64_fetch_add_release(i, v);
2848#elif defined(arch_atomic64_fetch_add_relaxed)
2849	__atomic_release_fence();
2850	return arch_atomic64_fetch_add_relaxed(i, v);
2851#elif defined(arch_atomic64_fetch_add)
2852	return arch_atomic64_fetch_add(i, v);
2853#else
2854#error "Unable to define raw_atomic64_fetch_add_release"
2855#endif
2856}
2857
2858/**
2859 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2860 * @i: s64 value to add
2861 * @v: pointer to atomic64_t
2862 *
2863 * Atomically updates @v to (@v + @i) with relaxed ordering.
2864 *
2865 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2866 *
2867 * Return: The original value of @v.
2868 */
2869static __always_inline s64
2870raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2871{
2872#if defined(arch_atomic64_fetch_add_relaxed)
2873	return arch_atomic64_fetch_add_relaxed(i, v);
2874#elif defined(arch_atomic64_fetch_add)
2875	return arch_atomic64_fetch_add(i, v);
2876#else
2877#error "Unable to define raw_atomic64_fetch_add_relaxed"
2878#endif
2879}
2880
2881/**
2882 * raw_atomic64_sub() - atomic subtract with relaxed ordering
2883 * @i: s64 value to subtract
2884 * @v: pointer to atomic64_t
2885 *
2886 * Atomically updates @v to (@v - @i) with relaxed ordering.
2887 *
2888 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2889 *
2890 * Return: Nothing.
2891 */
2892static __always_inline void
2893raw_atomic64_sub(s64 i, atomic64_t *v)
2894{
2895	arch_atomic64_sub(i, v);
2896}
2897
2898/**
2899 * raw_atomic64_sub_return() - atomic subtract with full ordering
2900 * @i: s64 value to subtract
2901 * @v: pointer to atomic64_t
2902 *
2903 * Atomically updates @v to (@v - @i) with full ordering.
2904 *
2905 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2906 *
2907 * Return: The updated value of @v.
2908 */
2909static __always_inline s64
2910raw_atomic64_sub_return(s64 i, atomic64_t *v)
2911{
2912#if defined(arch_atomic64_sub_return)
2913	return arch_atomic64_sub_return(i, v);
2914#elif defined(arch_atomic64_sub_return_relaxed)
2915	s64 ret;
2916	__atomic_pre_full_fence();
2917	ret = arch_atomic64_sub_return_relaxed(i, v);
2918	__atomic_post_full_fence();
2919	return ret;
2920#else
2921#error "Unable to define raw_atomic64_sub_return"
2922#endif
2923}
2924
2925/**
2926 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
2927 * @i: s64 value to subtract
2928 * @v: pointer to atomic64_t
2929 *
2930 * Atomically updates @v to (@v - @i) with acquire ordering.
2931 *
2932 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
2933 *
2934 * Return: The updated value of @v.
2935 */
2936static __always_inline s64
2937raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
2938{
2939#if defined(arch_atomic64_sub_return_acquire)
2940	return arch_atomic64_sub_return_acquire(i, v);
2941#elif defined(arch_atomic64_sub_return_relaxed)
2942	s64 ret = arch_atomic64_sub_return_relaxed(i, v);
2943	__atomic_acquire_fence();
2944	return ret;
2945#elif defined(arch_atomic64_sub_return)
2946	return arch_atomic64_sub_return(i, v);
2947#else
2948#error "Unable to define raw_atomic64_sub_return_acquire"
2949#endif
2950}
2951
2952/**
2953 * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2954 * @i: s64 value to subtract
2955 * @v: pointer to atomic64_t
2956 *
2957 * Atomically updates @v to (@v - @i) with release ordering.
2958 *
2959 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2960 *
2961 * Return: The updated value of @v.
2962 */
2963static __always_inline s64
2964raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
2965{
2966#if defined(arch_atomic64_sub_return_release)
2967	return arch_atomic64_sub_return_release(i, v);
2968#elif defined(arch_atomic64_sub_return_relaxed)
2969	__atomic_release_fence();
2970	return arch_atomic64_sub_return_relaxed(i, v);
2971#elif defined(arch_atomic64_sub_return)
2972	return arch_atomic64_sub_return(i, v);
2973#else
2974#error "Unable to define raw_atomic64_sub_return_release"
2975#endif
2976}
2977
2978/**
2979 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2980 * @i: s64 value to subtract
2981 * @v: pointer to atomic64_t
2982 *
2983 * Atomically updates @v to (@v - @i) with relaxed ordering.
2984 *
2985 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2986 *
2987 * Return: The updated value of @v.
2988 */
2989static __always_inline s64
2990raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2991{
2992#if defined(arch_atomic64_sub_return_relaxed)
2993	return arch_atomic64_sub_return_relaxed(i, v);
2994#elif defined(arch_atomic64_sub_return)
2995	return arch_atomic64_sub_return(i, v);
2996#else
2997#error "Unable to define raw_atomic64_sub_return_relaxed"
2998#endif
2999}
3000
3001/**
3002 * raw_atomic64_fetch_sub() - atomic subtract with full ordering
3003 * @i: s64 value to subtract
3004 * @v: pointer to atomic64_t
3005 *
3006 * Atomically updates @v to (@v - @i) with full ordering.
3007 *
3008 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
3009 *
3010 * Return: The original value of @v.
3011 */
3012static __always_inline s64
3013raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
3014{
3015#if defined(arch_atomic64_fetch_sub)
3016	return arch_atomic64_fetch_sub(i, v);
3017#elif defined(arch_atomic64_fetch_sub_relaxed)
3018	s64 ret;
3019	__atomic_pre_full_fence();
3020	ret = arch_atomic64_fetch_sub_relaxed(i, v);
3021	__atomic_post_full_fence();
3022	return ret;
3023#else
3024#error "Unable to define raw_atomic64_fetch_sub"
3025#endif
3026}
3027
3028/**
3029 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
3030 * @i: s64 value to subtract
3031 * @v: pointer to atomic64_t
3032 *
3033 * Atomically updates @v to (@v - @i) with acquire ordering.
3034 *
3035 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
3036 *
3037 * Return: The original value of @v.
3038 */
3039static __always_inline s64
3040raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
3041{
3042#if defined(arch_atomic64_fetch_sub_acquire)
3043	return arch_atomic64_fetch_sub_acquire(i, v);
3044#elif defined(arch_atomic64_fetch_sub_relaxed)
3045	s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3046	__atomic_acquire_fence();
3047	return ret;
3048#elif defined(arch_atomic64_fetch_sub)
3049	return arch_atomic64_fetch_sub(i, v);
3050#else
3051#error "Unable to define raw_atomic64_fetch_sub_acquire"
3052#endif
3053}
3054
3055/**
3056 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3057 * @i: s64 value to subtract
3058 * @v: pointer to atomic64_t
3059 *
3060 * Atomically updates @v to (@v - @i) with release ordering.
3061 *
3062 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3063 *
3064 * Return: The original value of @v.
3065 */
3066static __always_inline s64
3067raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
3068{
3069#if defined(arch_atomic64_fetch_sub_release)
3070	return arch_atomic64_fetch_sub_release(i, v);
3071#elif defined(arch_atomic64_fetch_sub_relaxed)
3072	__atomic_release_fence();
3073	return arch_atomic64_fetch_sub_relaxed(i, v);
3074#elif defined(arch_atomic64_fetch_sub)
3075	return arch_atomic64_fetch_sub(i, v);
3076#else
3077#error "Unable to define raw_atomic64_fetch_sub_release"
3078#endif
3079}
3080
3081/**
3082 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3083 * @i: s64 value to subtract
3084 * @v: pointer to atomic64_t
3085 *
3086 * Atomically updates @v to (@v - @i) with relaxed ordering.
3087 *
3088 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3089 *
3090 * Return: The original value of @v.
3091 */
3092static __always_inline s64
3093raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
3094{
3095#if defined(arch_atomic64_fetch_sub_relaxed)
3096	return arch_atomic64_fetch_sub_relaxed(i, v);
3097#elif defined(arch_atomic64_fetch_sub)
3098	return arch_atomic64_fetch_sub(i, v);
3099#else
3100#error "Unable to define raw_atomic64_fetch_sub_relaxed"
3101#endif
3102}
3103
3104/**
3105 * raw_atomic64_inc() - atomic increment with relaxed ordering
3106 * @v: pointer to atomic64_t
3107 *
3108 * Atomically updates @v to (@v + 1) with relaxed ordering.
3109 *
3110 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
3111 *
3112 * Return: Nothing.
3113 */
3114static __always_inline void
3115raw_atomic64_inc(atomic64_t *v)
3116{
3117#if defined(arch_atomic64_inc)
3118	arch_atomic64_inc(v);
3119#else
3120	raw_atomic64_add(1, v);
3121#endif
3122}
3123
3124/**
3125 * raw_atomic64_inc_return() - atomic increment with full ordering
3126 * @v: pointer to atomic64_t
3127 *
3128 * Atomically updates @v to (@v + 1) with full ordering.
3129 *
3130 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3131 *
3132 * Return: The updated value of @v.
3133 */
3134static __always_inline s64
3135raw_atomic64_inc_return(atomic64_t *v)
3136{
3137#if defined(arch_atomic64_inc_return)
3138	return arch_atomic64_inc_return(v);
3139#elif defined(arch_atomic64_inc_return_relaxed)
3140	s64 ret;
3141	__atomic_pre_full_fence();
3142	ret = arch_atomic64_inc_return_relaxed(v);
3143	__atomic_post_full_fence();
3144	return ret;
3145#else
3146	return raw_atomic64_add_return(1, v);
3147#endif
3148}
3149
3150/**
3151 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
3152 * @v: pointer to atomic64_t
3153 *
3154 * Atomically updates @v to (@v + 1) with acquire ordering.
3155 *
3156 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3157 *
3158 * Return: The updated value of @v.
3159 */
3160static __always_inline s64
3161raw_atomic64_inc_return_acquire(atomic64_t *v)
3162{
3163#if defined(arch_atomic64_inc_return_acquire)
3164	return arch_atomic64_inc_return_acquire(v);
3165#elif defined(arch_atomic64_inc_return_relaxed)
3166	s64 ret = arch_atomic64_inc_return_relaxed(v);
3167	__atomic_acquire_fence();
3168	return ret;
3169#elif defined(arch_atomic64_inc_return)
3170	return arch_atomic64_inc_return(v);
3171#else
3172	return raw_atomic64_add_return_acquire(1, v);
3173#endif
3174}
3175
3176/**
3177 * raw_atomic64_inc_return_release() - atomic increment with release ordering
3178 * @v: pointer to atomic64_t
3179 *
3180 * Atomically updates @v to (@v + 1) with release ordering.
3181 *
3182 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3183 *
3184 * Return: The updated value of @v.
3185 */
3186static __always_inline s64
3187raw_atomic64_inc_return_release(atomic64_t *v)
3188{
3189#if defined(arch_atomic64_inc_return_release)
3190	return arch_atomic64_inc_return_release(v);
3191#elif defined(arch_atomic64_inc_return_relaxed)
3192	__atomic_release_fence();
3193	return arch_atomic64_inc_return_relaxed(v);
3194#elif defined(arch_atomic64_inc_return)
3195	return arch_atomic64_inc_return(v);
3196#else
3197	return raw_atomic64_add_return_release(1, v);
3198#endif
3199}
3200
3201/**
3202 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
3203 * @v: pointer to atomic64_t
3204 *
3205 * Atomically updates @v to (@v + 1) with relaxed ordering.
3206 *
3207 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
3208 *
3209 * Return: The updated value of @v.
3210 */
3211static __always_inline s64
3212raw_atomic64_inc_return_relaxed(atomic64_t *v)
3213{
3214#if defined(arch_atomic64_inc_return_relaxed)
3215	return arch_atomic64_inc_return_relaxed(v);
3216#elif defined(arch_atomic64_inc_return)
3217	return arch_atomic64_inc_return(v);
3218#else
3219	return raw_atomic64_add_return_relaxed(1, v);
3220#endif
3221}
3222
3223/**
3224 * raw_atomic64_fetch_inc() - atomic increment with full ordering
3225 * @v: pointer to atomic64_t
3226 *
3227 * Atomically updates @v to (@v + 1) with full ordering.
3228 *
3229 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3230 *
3231 * Return: The original value of @v.
3232 */
3233static __always_inline s64
3234raw_atomic64_fetch_inc(atomic64_t *v)
3235{
3236#if defined(arch_atomic64_fetch_inc)
3237	return arch_atomic64_fetch_inc(v);
3238#elif defined(arch_atomic64_fetch_inc_relaxed)
3239	s64 ret;
3240	__atomic_pre_full_fence();
3241	ret = arch_atomic64_fetch_inc_relaxed(v);
3242	__atomic_post_full_fence();
3243	return ret;
3244#else
3245	return raw_atomic64_fetch_add(1, v);
3246#endif
3247}
3248
3249/**
3250 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3251 * @v: pointer to atomic64_t
3252 *
3253 * Atomically updates @v to (@v + 1) with acquire ordering.
3254 *
3255 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3256 *
3257 * Return: The original value of @v.
3258 */
3259static __always_inline s64
3260raw_atomic64_fetch_inc_acquire(atomic64_t *v)
3261{
3262#if defined(arch_atomic64_fetch_inc_acquire)
3263	return arch_atomic64_fetch_inc_acquire(v);
3264#elif defined(arch_atomic64_fetch_inc_relaxed)
3265	s64 ret = arch_atomic64_fetch_inc_relaxed(v);
3266	__atomic_acquire_fence();
3267	return ret;
3268#elif defined(arch_atomic64_fetch_inc)
3269	return arch_atomic64_fetch_inc(v);
3270#else
3271	return raw_atomic64_fetch_add_acquire(1, v);
3272#endif
3273}
3274
3275/**
3276 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3277 * @v: pointer to atomic64_t
3278 *
3279 * Atomically updates @v to (@v + 1) with release ordering.
3280 *
3281 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3282 *
3283 * Return: The original value of @v.
3284 */
3285static __always_inline s64
3286raw_atomic64_fetch_inc_release(atomic64_t *v)
3287{
3288#if defined(arch_atomic64_fetch_inc_release)
3289	return arch_atomic64_fetch_inc_release(v);
3290#elif defined(arch_atomic64_fetch_inc_relaxed)
3291	__atomic_release_fence();
3292	return arch_atomic64_fetch_inc_relaxed(v);
3293#elif defined(arch_atomic64_fetch_inc)
3294	return arch_atomic64_fetch_inc(v);
3295#else
3296	return raw_atomic64_fetch_add_release(1, v);
3297#endif
3298}
3299
3300/**
3301 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
3302 * @v: pointer to atomic64_t
3303 *
3304 * Atomically updates @v to (@v + 1) with relaxed ordering.
3305 *
3306 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
3307 *
3308 * Return: The original value of @v.
3309 */
3310static __always_inline s64
3311raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
3312{
3313#if defined(arch_atomic64_fetch_inc_relaxed)
3314	return arch_atomic64_fetch_inc_relaxed(v);
3315#elif defined(arch_atomic64_fetch_inc)
3316	return arch_atomic64_fetch_inc(v);
3317#else
3318	return raw_atomic64_fetch_add_relaxed(1, v);
3319#endif
3320}
3321
3322/**
3323 * raw_atomic64_dec() - atomic decrement with relaxed ordering
3324 * @v: pointer to atomic64_t
3325 *
3326 * Atomically updates @v to (@v - 1) with relaxed ordering.
3327 *
3328 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3329 *
3330 * Return: Nothing.
3331 */
3332static __always_inline void
3333raw_atomic64_dec(atomic64_t *v)
3334{
3335#if defined(arch_atomic64_dec)
3336	arch_atomic64_dec(v);
3337#else
3338	raw_atomic64_sub(1, v);
3339#endif
3340}
3341
3342/**
3343 * raw_atomic64_dec_return() - atomic decrement with full ordering
3344 * @v: pointer to atomic64_t
3345 *
3346 * Atomically updates @v to (@v - 1) with full ordering.
3347 *
3348 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3349 *
3350 * Return: The updated value of @v.
3351 */
3352static __always_inline s64
3353raw_atomic64_dec_return(atomic64_t *v)
3354{
3355#if defined(arch_atomic64_dec_return)
3356	return arch_atomic64_dec_return(v);
3357#elif defined(arch_atomic64_dec_return_relaxed)
3358	s64 ret;
3359	__atomic_pre_full_fence();
3360	ret = arch_atomic64_dec_return_relaxed(v);
3361	__atomic_post_full_fence();
3362	return ret;
3363#else
3364	return raw_atomic64_sub_return(1, v);
3365#endif
3366}
3367
3368/**
3369 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
3370 * @v: pointer to atomic64_t
3371 *
3372 * Atomically updates @v to (@v - 1) with acquire ordering.
3373 *
3374 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3375 *
3376 * Return: The updated value of @v.
3377 */
3378static __always_inline s64
3379raw_atomic64_dec_return_acquire(atomic64_t *v)
3380{
3381#if defined(arch_atomic64_dec_return_acquire)
3382	return arch_atomic64_dec_return_acquire(v);
3383#elif defined(arch_atomic64_dec_return_relaxed)
3384	s64 ret = arch_atomic64_dec_return_relaxed(v);
3385	__atomic_acquire_fence();
3386	return ret;
3387#elif defined(arch_atomic64_dec_return)
3388	return arch_atomic64_dec_return(v);
3389#else
3390	return raw_atomic64_sub_return_acquire(1, v);
3391#endif
3392}
3393
3394/**
3395 * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3396 * @v: pointer to atomic64_t
3397 *
3398 * Atomically updates @v to (@v - 1) with release ordering.
3399 *
3400 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3401 *
3402 * Return: The updated value of @v.
3403 */
3404static __always_inline s64
3405raw_atomic64_dec_return_release(atomic64_t *v)
3406{
3407#if defined(arch_atomic64_dec_return_release)
3408	return arch_atomic64_dec_return_release(v);
3409#elif defined(arch_atomic64_dec_return_relaxed)
3410	__atomic_release_fence();
3411	return arch_atomic64_dec_return_relaxed(v);
3412#elif defined(arch_atomic64_dec_return)
3413	return arch_atomic64_dec_return(v);
3414#else
3415	return raw_atomic64_sub_return_release(1, v);
3416#endif
3417}
3418
3419/**
3420 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3421 * @v: pointer to atomic64_t
3422 *
3423 * Atomically updates @v to (@v - 1) with relaxed ordering.
3424 *
3425 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3426 *
3427 * Return: The updated value of @v.
3428 */
3429static __always_inline s64
3430raw_atomic64_dec_return_relaxed(atomic64_t *v)
3431{
3432#if defined(arch_atomic64_dec_return_relaxed)
3433	return arch_atomic64_dec_return_relaxed(v);
3434#elif defined(arch_atomic64_dec_return)
3435	return arch_atomic64_dec_return(v);
3436#else
3437	return raw_atomic64_sub_return_relaxed(1, v);
3438#endif
3439}
3440
3441/**
3442 * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3443 * @v: pointer to atomic64_t
3444 *
3445 * Atomically updates @v to (@v - 1) with full ordering.
3446 *
3447 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3448 *
3449 * Return: The original value of @v.
3450 */
3451static __always_inline s64
3452raw_atomic64_fetch_dec(atomic64_t *v)
3453{
3454#if defined(arch_atomic64_fetch_dec)
3455	return arch_atomic64_fetch_dec(v);
3456#elif defined(arch_atomic64_fetch_dec_relaxed)
3457	s64 ret;
3458	__atomic_pre_full_fence();
3459	ret = arch_atomic64_fetch_dec_relaxed(v);
3460	__atomic_post_full_fence();
3461	return ret;
3462#else
3463	return raw_atomic64_fetch_sub(1, v);
3464#endif
3465}
3466
3467/**
3468 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3469 * @v: pointer to atomic64_t
3470 *
3471 * Atomically updates @v to (@v - 1) with acquire ordering.
3472 *
3473 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3474 *
3475 * Return: The original value of @v.
3476 */
3477static __always_inline s64
3478raw_atomic64_fetch_dec_acquire(atomic64_t *v)
3479{
3480#if defined(arch_atomic64_fetch_dec_acquire)
3481	return arch_atomic64_fetch_dec_acquire(v);
3482#elif defined(arch_atomic64_fetch_dec_relaxed)
3483	s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3484	__atomic_acquire_fence();
3485	return ret;
3486#elif defined(arch_atomic64_fetch_dec)
3487	return arch_atomic64_fetch_dec(v);
3488#else
3489	return raw_atomic64_fetch_sub_acquire(1, v);
3490#endif
3491}
3492
3493/**
3494 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3495 * @v: pointer to atomic64_t
3496 *
3497 * Atomically updates @v to (@v - 1) with release ordering.
3498 *
3499 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3500 *
3501 * Return: The original value of @v.
3502 */
3503static __always_inline s64
3504raw_atomic64_fetch_dec_release(atomic64_t *v)
3505{
3506#if defined(arch_atomic64_fetch_dec_release)
3507	return arch_atomic64_fetch_dec_release(v);
3508#elif defined(arch_atomic64_fetch_dec_relaxed)
3509	__atomic_release_fence();
3510	return arch_atomic64_fetch_dec_relaxed(v);
3511#elif defined(arch_atomic64_fetch_dec)
3512	return arch_atomic64_fetch_dec(v);
3513#else
3514	return raw_atomic64_fetch_sub_release(1, v);
3515#endif
3516}
3517
3518/**
3519 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3520 * @v: pointer to atomic64_t
3521 *
3522 * Atomically updates @v to (@v - 1) with relaxed ordering.
3523 *
3524 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3525 *
3526 * Return: The original value of @v.
3527 */
3528static __always_inline s64
3529raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
3530{
3531#if defined(arch_atomic64_fetch_dec_relaxed)
3532	return arch_atomic64_fetch_dec_relaxed(v);
3533#elif defined(arch_atomic64_fetch_dec)
3534	return arch_atomic64_fetch_dec(v);
3535#else
3536	return raw_atomic64_fetch_sub_relaxed(1, v);
3537#endif
3538}
3539
3540/**
3541 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
3542 * @i: s64 value
3543 * @v: pointer to atomic64_t
3544 *
3545 * Atomically updates @v to (@v & @i) with relaxed ordering.
3546 *
3547 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3548 *
3549 * Return: Nothing.
3550 */
3551static __always_inline void
3552raw_atomic64_and(s64 i, atomic64_t *v)
3553{
3554	arch_atomic64_and(i, v);
3555}
3556
3557/**
3558 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3559 * @i: s64 value
3560 * @v: pointer to atomic64_t
3561 *
3562 * Atomically updates @v to (@v & @i) with full ordering.
3563 *
3564 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3565 *
3566 * Return: The original value of @v.
3567 */
3568static __always_inline s64
3569raw_atomic64_fetch_and(s64 i, atomic64_t *v)
3570{
3571#if defined(arch_atomic64_fetch_and)
3572	return arch_atomic64_fetch_and(i, v);
3573#elif defined(arch_atomic64_fetch_and_relaxed)
3574	s64 ret;
3575	__atomic_pre_full_fence();
3576	ret = arch_atomic64_fetch_and_relaxed(i, v);
3577	__atomic_post_full_fence();
3578	return ret;
3579#else
3580#error "Unable to define raw_atomic64_fetch_and"
3581#endif
3582}
3583
3584/**
3585 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3586 * @i: s64 value
3587 * @v: pointer to atomic64_t
3588 *
3589 * Atomically updates @v to (@v & @i) with acquire ordering.
3590 *
3591 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
3592 *
3593 * Return: The original value of @v.
3594 */
3595static __always_inline s64
3596raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
3597{
3598#if defined(arch_atomic64_fetch_and_acquire)
3599	return arch_atomic64_fetch_and_acquire(i, v);
3600#elif defined(arch_atomic64_fetch_and_relaxed)
3601	s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
3602	__atomic_acquire_fence();
3603	return ret;
3604#elif defined(arch_atomic64_fetch_and)
3605	return arch_atomic64_fetch_and(i, v);
3606#else
3607#error "Unable to define raw_atomic64_fetch_and_acquire"
3608#endif
3609}
3610
3611/**
3612 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3613 * @i: s64 value
3614 * @v: pointer to atomic64_t
3615 *
3616 * Atomically updates @v to (@v & @i) with release ordering.
3617 *
3618 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3619 *
3620 * Return: The original value of @v.
3621 */
3622static __always_inline s64
3623raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
3624{
3625#if defined(arch_atomic64_fetch_and_release)
3626	return arch_atomic64_fetch_and_release(i, v);
3627#elif defined(arch_atomic64_fetch_and_relaxed)
3628	__atomic_release_fence();
3629	return arch_atomic64_fetch_and_relaxed(i, v);
3630#elif defined(arch_atomic64_fetch_and)
3631	return arch_atomic64_fetch_and(i, v);
3632#else
3633#error "Unable to define raw_atomic64_fetch_and_release"
3634#endif
3635}
3636
3637/**
3638 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3639 * @i: s64 value
3640 * @v: pointer to atomic64_t
3641 *
3642 * Atomically updates @v to (@v & @i) with relaxed ordering.
3643 *
3644 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3645 *
3646 * Return: The original value of @v.
3647 */
3648static __always_inline s64
3649raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
3650{
3651#if defined(arch_atomic64_fetch_and_relaxed)
3652	return arch_atomic64_fetch_and_relaxed(i, v);
3653#elif defined(arch_atomic64_fetch_and)
3654	return arch_atomic64_fetch_and(i, v);
3655#else
3656#error "Unable to define raw_atomic64_fetch_and_relaxed"
3657#endif
3658}
3659
3660/**
3661 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
3662 * @i: s64 value
3663 * @v: pointer to atomic64_t
3664 *
3665 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3666 *
3667 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3668 *
3669 * Return: Nothing.
3670 */
3671static __always_inline void
3672raw_atomic64_andnot(s64 i, atomic64_t *v)
3673{
3674#if defined(arch_atomic64_andnot)
3675	arch_atomic64_andnot(i, v);
3676#else
3677	raw_atomic64_and(~i, v);
3678#endif
3679}
3680
3681/**
3682 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3683 * @i: s64 value
3684 * @v: pointer to atomic64_t
3685 *
3686 * Atomically updates @v to (@v & ~@i) with full ordering.
3687 *
3688 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3689 *
3690 * Return: The original value of @v.
3691 */
3692static __always_inline s64
3693raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
3694{
3695#if defined(arch_atomic64_fetch_andnot)
3696	return arch_atomic64_fetch_andnot(i, v);
3697#elif defined(arch_atomic64_fetch_andnot_relaxed)
3698	s64 ret;
3699	__atomic_pre_full_fence();
3700	ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3701	__atomic_post_full_fence();
3702	return ret;
3703#else
3704	return raw_atomic64_fetch_and(~i, v);
3705#endif
3706}
3707
3708/**
3709 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3710 * @i: s64 value
3711 * @v: pointer to atomic64_t
3712 *
3713 * Atomically updates @v to (@v & ~@i) with acquire ordering.
3714 *
3715 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3716 *
3717 * Return: The original value of @v.
3718 */
3719static __always_inline s64
3720raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
3721{
3722#if defined(arch_atomic64_fetch_andnot_acquire)
3723	return arch_atomic64_fetch_andnot_acquire(i, v);
3724#elif defined(arch_atomic64_fetch_andnot_relaxed)
3725	s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3726	__atomic_acquire_fence();
3727	return ret;
3728#elif defined(arch_atomic64_fetch_andnot)
3729	return arch_atomic64_fetch_andnot(i, v);
3730#else
3731	return raw_atomic64_fetch_and_acquire(~i, v);
3732#endif
3733}
3734
3735/**
3736 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3737 * @i: s64 value
3738 * @v: pointer to atomic64_t
3739 *
3740 * Atomically updates @v to (@v & ~@i) with release ordering.
3741 *
3742 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3743 *
3744 * Return: The original value of @v.
3745 */
3746static __always_inline s64
3747raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
3748{
3749#if defined(arch_atomic64_fetch_andnot_release)
3750	return arch_atomic64_fetch_andnot_release(i, v);
3751#elif defined(arch_atomic64_fetch_andnot_relaxed)
3752	__atomic_release_fence();
3753	return arch_atomic64_fetch_andnot_relaxed(i, v);
3754#elif defined(arch_atomic64_fetch_andnot)
3755	return arch_atomic64_fetch_andnot(i, v);
3756#else
3757	return raw_atomic64_fetch_and_release(~i, v);
3758#endif
3759}
3760
3761/**
3762 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
3763 * @i: s64 value
3764 * @v: pointer to atomic64_t
3765 *
3766 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3767 *
3768 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
3769 *
3770 * Return: The original value of @v.
3771 */
3772static __always_inline s64
3773raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
3774{
3775#if defined(arch_atomic64_fetch_andnot_relaxed)
3776	return arch_atomic64_fetch_andnot_relaxed(i, v);
3777#elif defined(arch_atomic64_fetch_andnot)
3778	return arch_atomic64_fetch_andnot(i, v);
3779#else
3780	return raw_atomic64_fetch_and_relaxed(~i, v);
3781#endif
3782}
3783
3784/**
3785 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
3786 * @i: s64 value
3787 * @v: pointer to atomic64_t
3788 *
3789 * Atomically updates @v to (@v | @i) with relaxed ordering.
3790 *
3791 * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
3792 *
3793 * Return: Nothing.
3794 */
3795static __always_inline void
3796raw_atomic64_or(s64 i, atomic64_t *v)
3797{
3798	arch_atomic64_or(i, v);
3799}
3800
3801/**
3802 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
3803 * @i: s64 value
3804 * @v: pointer to atomic64_t
3805 *
3806 * Atomically updates @v to (@v | @i) with full ordering.
3807 *
3808 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
3809 *
3810 * Return: The original value of @v.
3811 */
3812static __always_inline s64
3813raw_atomic64_fetch_or(s64 i, atomic64_t *v)
3814{
3815#if defined(arch_atomic64_fetch_or)
3816	return arch_atomic64_fetch_or(i, v);
3817#elif defined(arch_atomic64_fetch_or_relaxed)
3818	s64 ret;
3819	__atomic_pre_full_fence();
3820	ret = arch_atomic64_fetch_or_relaxed(i, v);
3821	__atomic_post_full_fence();
3822	return ret;
3823#else
3824#error "Unable to define raw_atomic64_fetch_or"
3825#endif
3826}
3827
3828/**
3829 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
3830 * @i: s64 value
3831 * @v: pointer to atomic64_t
3832 *
3833 * Atomically updates @v to (@v | @i) with acquire ordering.
3834 *
3835 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
3836 *
3837 * Return: The original value of @v.
3838 */
3839static __always_inline s64
3840raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
3841{
3842#if defined(arch_atomic64_fetch_or_acquire)
3843	return arch_atomic64_fetch_or_acquire(i, v);
3844#elif defined(arch_atomic64_fetch_or_relaxed)
3845	s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
3846	__atomic_acquire_fence();
3847	return ret;
3848#elif defined(arch_atomic64_fetch_or)
3849	return arch_atomic64_fetch_or(i, v);
3850#else
3851#error "Unable to define raw_atomic64_fetch_or_acquire"
3852#endif
3853}
3854
3855/**
3856 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
3857 * @i: s64 value
3858 * @v: pointer to atomic64_t
3859 *
3860 * Atomically updates @v to (@v | @i) with release ordering.
3861 *
3862 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
3863 *
3864 * Return: The original value of @v.
3865 */
3866static __always_inline s64
3867raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
3868{
3869#if defined(arch_atomic64_fetch_or_release)
3870	return arch_atomic64_fetch_or_release(i, v);
3871#elif defined(arch_atomic64_fetch_or_relaxed)
3872	__atomic_release_fence();
3873	return arch_atomic64_fetch_or_relaxed(i, v);
3874#elif defined(arch_atomic64_fetch_or)
3875	return arch_atomic64_fetch_or(i, v);
3876#else
3877#error "Unable to define raw_atomic64_fetch_or_release"
3878#endif
3879}
3880
3881/**
3882 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
3883 * @i: s64 value
3884 * @v: pointer to atomic64_t
3885 *
3886 * Atomically updates @v to (@v | @i) with relaxed ordering.
3887 *
3888 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
3889 *
3890 * Return: The original value of @v.
3891 */
3892static __always_inline s64
3893raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
3894{
3895#if defined(arch_atomic64_fetch_or_relaxed)
3896	return arch_atomic64_fetch_or_relaxed(i, v);
3897#elif defined(arch_atomic64_fetch_or)
3898	return arch_atomic64_fetch_or(i, v);
3899#else
3900#error "Unable to define raw_atomic64_fetch_or_relaxed"
3901#endif
3902}
3903
3904/**
3905 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
3906 * @i: s64 value
3907 * @v: pointer to atomic64_t
3908 *
3909 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3910 *
3911 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
3912 *
3913 * Return: Nothing.
3914 */
3915static __always_inline void
3916raw_atomic64_xor(s64 i, atomic64_t *v)
3917{
3918	arch_atomic64_xor(i, v);
3919}
3920
3921/**
3922 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
3923 * @i: s64 value
3924 * @v: pointer to atomic64_t
3925 *
3926 * Atomically updates @v to (@v ^ @i) with full ordering.
3927 *
3928 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
3929 *
3930 * Return: The original value of @v.
3931 */
3932static __always_inline s64
3933raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
3934{
3935#if defined(arch_atomic64_fetch_xor)
3936	return arch_atomic64_fetch_xor(i, v);
3937#elif defined(arch_atomic64_fetch_xor_relaxed)
3938	s64 ret;
3939	__atomic_pre_full_fence();
3940	ret = arch_atomic64_fetch_xor_relaxed(i, v);
3941	__atomic_post_full_fence();
3942	return ret;
3943#else
3944#error "Unable to define raw_atomic64_fetch_xor"
3945#endif
3946}
3947
3948/**
3949 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
3950 * @i: s64 value
3951 * @v: pointer to atomic64_t
3952 *
3953 * Atomically updates @v to (@v ^ @i) with acquire ordering.
3954 *
3955 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
3956 *
3957 * Return: The original value of @v.
3958 */
3959static __always_inline s64
3960raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
3961{
3962#if defined(arch_atomic64_fetch_xor_acquire)
3963	return arch_atomic64_fetch_xor_acquire(i, v);
3964#elif defined(arch_atomic64_fetch_xor_relaxed)
3965	s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3966	__atomic_acquire_fence();
3967	return ret;
3968#elif defined(arch_atomic64_fetch_xor)
3969	return arch_atomic64_fetch_xor(i, v);
3970#else
3971#error "Unable to define raw_atomic64_fetch_xor_acquire"
3972#endif
3973}
3974
3975/**
3976 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
3977 * @i: s64 value
3978 * @v: pointer to atomic64_t
3979 *
3980 * Atomically updates @v to (@v ^ @i) with release ordering.
3981 *
3982 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
3983 *
3984 * Return: The original value of @v.
3985 */
3986static __always_inline s64
3987raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
3988{
3989#if defined(arch_atomic64_fetch_xor_release)
3990	return arch_atomic64_fetch_xor_release(i, v);
3991#elif defined(arch_atomic64_fetch_xor_relaxed)
3992	__atomic_release_fence();
3993	return arch_atomic64_fetch_xor_relaxed(i, v);
3994#elif defined(arch_atomic64_fetch_xor)
3995	return arch_atomic64_fetch_xor(i, v);
3996#else
3997#error "Unable to define raw_atomic64_fetch_xor_release"
3998#endif
3999}
4000
4001/**
4002 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4003 * @i: s64 value
4004 * @v: pointer to atomic64_t
4005 *
4006 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4007 *
4008 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
4009 *
4010 * Return: The original value of @v.
4011 */
4012static __always_inline s64
4013raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
4014{
4015#if defined(arch_atomic64_fetch_xor_relaxed)
4016	return arch_atomic64_fetch_xor_relaxed(i, v);
4017#elif defined(arch_atomic64_fetch_xor)
4018	return arch_atomic64_fetch_xor(i, v);
4019#else
4020#error "Unable to define raw_atomic64_fetch_xor_relaxed"
4021#endif
4022}
4023
4024/**
4025 * raw_atomic64_xchg() - atomic exchange with full ordering
4026 * @v: pointer to atomic64_t
4027 * @new: s64 value to assign
4028 *
4029 * Atomically updates @v to @new with full ordering.
4030 *
4031 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
4032 *
4033 * Return: The original value of @v.
4034 */
4035static __always_inline s64
4036raw_atomic64_xchg(atomic64_t *v, s64 new)
4037{
4038#if defined(arch_atomic64_xchg)
4039	return arch_atomic64_xchg(v, new);
4040#elif defined(arch_atomic64_xchg_relaxed)
4041	s64 ret;
4042	__atomic_pre_full_fence();
4043	ret = arch_atomic64_xchg_relaxed(v, new);
4044	__atomic_post_full_fence();
4045	return ret;
4046#else
4047	return raw_xchg(&v->counter, new);
4048#endif
4049}
4050
4051/**
4052 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
4053 * @v: pointer to atomic64_t
4054 * @new: s64 value to assign
4055 *
4056 * Atomically updates @v to @new with acquire ordering.
4057 *
4058 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
4059 *
4060 * Return: The original value of @v.
4061 */
4062static __always_inline s64
4063raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
4064{
4065#if defined(arch_atomic64_xchg_acquire)
4066	return arch_atomic64_xchg_acquire(v, new);
4067#elif defined(arch_atomic64_xchg_relaxed)
4068	s64 ret = arch_atomic64_xchg_relaxed(v, new);
4069	__atomic_acquire_fence();
4070	return ret;
4071#elif defined(arch_atomic64_xchg)
4072	return arch_atomic64_xchg(v, new);
4073#else
4074	return raw_xchg_acquire(&v->counter, new);
4075#endif
4076}
4077
4078/**
4079 * raw_atomic64_xchg_release() - atomic exchange with release ordering
4080 * @v: pointer to atomic64_t
4081 * @new: s64 value to assign
4082 *
4083 * Atomically updates @v to @new with release ordering.
4084 *
4085 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
4086 *
4087 * Return: The original value of @v.
4088 */
4089static __always_inline s64
4090raw_atomic64_xchg_release(atomic64_t *v, s64 new)
4091{
4092#if defined(arch_atomic64_xchg_release)
4093	return arch_atomic64_xchg_release(v, new);
4094#elif defined(arch_atomic64_xchg_relaxed)
4095	__atomic_release_fence();
4096	return arch_atomic64_xchg_relaxed(v, new);
4097#elif defined(arch_atomic64_xchg)
4098	return arch_atomic64_xchg(v, new);
4099#else
4100	return raw_xchg_release(&v->counter, new);
4101#endif
4102}
4103
4104/**
4105 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
4106 * @v: pointer to atomic64_t
4107 * @new: s64 value to assign
4108 *
4109 * Atomically updates @v to @new with relaxed ordering.
4110 *
4111 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
4112 *
4113 * Return: The original value of @v.
4114 */
4115static __always_inline s64
4116raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
4117{
4118#if defined(arch_atomic64_xchg_relaxed)
4119	return arch_atomic64_xchg_relaxed(v, new);
4120#elif defined(arch_atomic64_xchg)
4121	return arch_atomic64_xchg(v, new);
4122#else
4123	return raw_xchg_relaxed(&v->counter, new);
4124#endif
4125}
4126
4127/**
4128 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
4129 * @v: pointer to atomic64_t
4130 * @old: s64 value to compare with
4131 * @new: s64 value to assign
4132 *
4133 * If (@v == @old), atomically updates @v to @new with full ordering.
4134 * Otherwise, @v is not modified and relaxed ordering is provided.
4135 *
4136 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4137 *
4138 * Return: The original value of @v.
4139 */
4140static __always_inline s64
4141raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
4142{
4143#if defined(arch_atomic64_cmpxchg)
4144	return arch_atomic64_cmpxchg(v, old, new);
4145#elif defined(arch_atomic64_cmpxchg_relaxed)
4146	s64 ret;
4147	__atomic_pre_full_fence();
4148	ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4149	__atomic_post_full_fence();
4150	return ret;
4151#else
4152	return raw_cmpxchg(&v->counter, old, new);
4153#endif
4154}
4155
4156/**
4157 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4158 * @v: pointer to atomic64_t
4159 * @old: s64 value to compare with
4160 * @new: s64 value to assign
4161 *
4162 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4163 * Otherwise, @v is not modified and relaxed ordering is provided.
4164 *
4165 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4166 *
4167 * Return: The original value of @v.
4168 */
4169static __always_inline s64
4170raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
4171{
4172#if defined(arch_atomic64_cmpxchg_acquire)
4173	return arch_atomic64_cmpxchg_acquire(v, old, new);
4174#elif defined(arch_atomic64_cmpxchg_relaxed)
4175	s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4176	__atomic_acquire_fence();
4177	return ret;
4178#elif defined(arch_atomic64_cmpxchg)
4179	return arch_atomic64_cmpxchg(v, old, new);
4180#else
4181	return raw_cmpxchg_acquire(&v->counter, old, new);
4182#endif
4183}
4184
4185/**
4186 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
4187 * @v: pointer to atomic64_t
4188 * @old: s64 value to compare with
4189 * @new: s64 value to assign
4190 *
4191 * If (@v == @old), atomically updates @v to @new with release ordering.
4192 * Otherwise, @v is not modified and relaxed ordering is provided.
4193 *
4194 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4195 *
4196 * Return: The original value of @v.
4197 */
4198static __always_inline s64
4199raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
4200{
4201#if defined(arch_atomic64_cmpxchg_release)
4202	return arch_atomic64_cmpxchg_release(v, old, new);
4203#elif defined(arch_atomic64_cmpxchg_relaxed)
4204	__atomic_release_fence();
4205	return arch_atomic64_cmpxchg_relaxed(v, old, new);
4206#elif defined(arch_atomic64_cmpxchg)
4207	return arch_atomic64_cmpxchg(v, old, new);
4208#else
4209	return raw_cmpxchg_release(&v->counter, old, new);
4210#endif
4211}
4212
4213/**
4214 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4215 * @v: pointer to atomic64_t
4216 * @old: s64 value to compare with
4217 * @new: s64 value to assign
4218 *
4219 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4220 * Otherwise, @v is not modified and relaxed ordering is provided.
4221 *
4222 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4223 *
4224 * Return: The original value of @v.
4225 */
4226static __always_inline s64
4227raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
4228{
4229#if defined(arch_atomic64_cmpxchg_relaxed)
4230	return arch_atomic64_cmpxchg_relaxed(v, old, new);
4231#elif defined(arch_atomic64_cmpxchg)
4232	return arch_atomic64_cmpxchg(v, old, new);
4233#else
4234	return raw_cmpxchg_relaxed(&v->counter, old, new);
4235#endif
4236}
4237
4238/**
4239 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
4240 * @v: pointer to atomic64_t
4241 * @old: pointer to s64 value to compare with
4242 * @new: s64 value to assign
4243 *
4244 * If (@v == @old), atomically updates @v to @new with full ordering.
4245 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4246 * and relaxed ordering is provided.
4247 *
4248 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4249 *
4250 * Return: @true if the exchange occured, @false otherwise.
4251 */
4252static __always_inline bool
4253raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
4254{
4255#if defined(arch_atomic64_try_cmpxchg)
4256	return arch_atomic64_try_cmpxchg(v, old, new);
4257#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4258	bool ret;
4259	__atomic_pre_full_fence();
4260	ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4261	__atomic_post_full_fence();
4262	return ret;
4263#else
4264	s64 r, o = *old;
4265	r = raw_atomic64_cmpxchg(v, o, new);
4266	if (unlikely(r != o))
4267		*old = r;
4268	return likely(r == o);
4269#endif
4270}
4271
4272/**
4273 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4274 * @v: pointer to atomic64_t
4275 * @old: pointer to s64 value to compare with
4276 * @new: s64 value to assign
4277 *
4278 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4279 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4280 * and relaxed ordering is provided.
4281 *
4282 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4283 *
4284 * Return: @true if the exchange occured, @false otherwise.
4285 */
4286static __always_inline bool
4287raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
4288{
4289#if defined(arch_atomic64_try_cmpxchg_acquire)
4290	return arch_atomic64_try_cmpxchg_acquire(v, old, new);
4291#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4292	bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4293	__atomic_acquire_fence();
4294	return ret;
4295#elif defined(arch_atomic64_try_cmpxchg)
4296	return arch_atomic64_try_cmpxchg(v, old, new);
4297#else
4298	s64 r, o = *old;
4299	r = raw_atomic64_cmpxchg_acquire(v, o, new);
4300	if (unlikely(r != o))
4301		*old = r;
4302	return likely(r == o);
4303#endif
4304}
4305
4306/**
4307 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
4308 * @v: pointer to atomic64_t
4309 * @old: pointer to s64 value to compare with
4310 * @new: s64 value to assign
4311 *
4312 * If (@v == @old), atomically updates @v to @new with release ordering.
4313 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4314 * and relaxed ordering is provided.
4315 *
4316 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4317 *
4318 * Return: @true if the exchange occured, @false otherwise.
4319 */
4320static __always_inline bool
4321raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
4322{
4323#if defined(arch_atomic64_try_cmpxchg_release)
4324	return arch_atomic64_try_cmpxchg_release(v, old, new);
4325#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4326	__atomic_release_fence();
4327	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4328#elif defined(arch_atomic64_try_cmpxchg)
4329	return arch_atomic64_try_cmpxchg(v, old, new);
4330#else
4331	s64 r, o = *old;
4332	r = raw_atomic64_cmpxchg_release(v, o, new);
4333	if (unlikely(r != o))
4334		*old = r;
4335	return likely(r == o);
4336#endif
4337}
4338
4339/**
4340 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4341 * @v: pointer to atomic64_t
4342 * @old: pointer to s64 value to compare with
4343 * @new: s64 value to assign
4344 *
4345 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4346 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4347 * and relaxed ordering is provided.
4348 *
4349 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4350 *
4351 * Return: @true if the exchange occured, @false otherwise.
4352 */
4353static __always_inline bool
4354raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
4355{
4356#if defined(arch_atomic64_try_cmpxchg_relaxed)
4357	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4358#elif defined(arch_atomic64_try_cmpxchg)
4359	return arch_atomic64_try_cmpxchg(v, old, new);
4360#else
4361	s64 r, o = *old;
4362	r = raw_atomic64_cmpxchg_relaxed(v, o, new);
4363	if (unlikely(r != o))
4364		*old = r;
4365	return likely(r == o);
4366#endif
4367}
4368
4369/**
4370 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
4371 * @i: s64 value to subtract
4372 * @v: pointer to atomic64_t
4373 *
4374 * Atomically updates @v to (@v - @i) with full ordering.
4375 *
4376 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
4377 *
4378 * Return: @true if the resulting value of @v is zero, @false otherwise.
4379 */
4380static __always_inline bool
4381raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
4382{
4383#if defined(arch_atomic64_sub_and_test)
4384	return arch_atomic64_sub_and_test(i, v);
4385#else
4386	return raw_atomic64_sub_return(i, v) == 0;
4387#endif
4388}
4389
4390/**
4391 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
4392 * @v: pointer to atomic64_t
4393 *
4394 * Atomically updates @v to (@v - 1) with full ordering.
4395 *
4396 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
4397 *
4398 * Return: @true if the resulting value of @v is zero, @false otherwise.
4399 */
4400static __always_inline bool
4401raw_atomic64_dec_and_test(atomic64_t *v)
4402{
4403#if defined(arch_atomic64_dec_and_test)
4404	return arch_atomic64_dec_and_test(v);
4405#else
4406	return raw_atomic64_dec_return(v) == 0;
4407#endif
4408}
4409
4410/**
4411 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
4412 * @v: pointer to atomic64_t
4413 *
4414 * Atomically updates @v to (@v + 1) with full ordering.
4415 *
4416 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
4417 *
4418 * Return: @true if the resulting value of @v is zero, @false otherwise.
4419 */
4420static __always_inline bool
4421raw_atomic64_inc_and_test(atomic64_t *v)
4422{
4423#if defined(arch_atomic64_inc_and_test)
4424	return arch_atomic64_inc_and_test(v);
4425#else
4426	return raw_atomic64_inc_return(v) == 0;
4427#endif
4428}
4429
4430/**
4431 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
4432 * @i: s64 value to add
4433 * @v: pointer to atomic64_t
4434 *
4435 * Atomically updates @v to (@v + @i) with full ordering.
4436 *
4437 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
4438 *
4439 * Return: @true if the resulting value of @v is negative, @false otherwise.
4440 */
4441static __always_inline bool
4442raw_atomic64_add_negative(s64 i, atomic64_t *v)
4443{
4444#if defined(arch_atomic64_add_negative)
4445	return arch_atomic64_add_negative(i, v);
4446#elif defined(arch_atomic64_add_negative_relaxed)
4447	bool ret;
4448	__atomic_pre_full_fence();
4449	ret = arch_atomic64_add_negative_relaxed(i, v);
4450	__atomic_post_full_fence();
4451	return ret;
4452#else
4453	return raw_atomic64_add_return(i, v) < 0;
4454#endif
4455}
4456
4457/**
4458 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
4459 * @i: s64 value to add
4460 * @v: pointer to atomic64_t
4461 *
4462 * Atomically updates @v to (@v + @i) with acquire ordering.
4463 *
4464 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
4465 *
4466 * Return: @true if the resulting value of @v is negative, @false otherwise.
4467 */
4468static __always_inline bool
4469raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
4470{
4471#if defined(arch_atomic64_add_negative_acquire)
4472	return arch_atomic64_add_negative_acquire(i, v);
4473#elif defined(arch_atomic64_add_negative_relaxed)
4474	bool ret = arch_atomic64_add_negative_relaxed(i, v);
4475	__atomic_acquire_fence();
4476	return ret;
4477#elif defined(arch_atomic64_add_negative)
4478	return arch_atomic64_add_negative(i, v);
4479#else
4480	return raw_atomic64_add_return_acquire(i, v) < 0;
4481#endif
4482}
4483
4484/**
4485 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
4486 * @i: s64 value to add
4487 * @v: pointer to atomic64_t
4488 *
4489 * Atomically updates @v to (@v + @i) with release ordering.
4490 *
4491 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
4492 *
4493 * Return: @true if the resulting value of @v is negative, @false otherwise.
4494 */
4495static __always_inline bool
4496raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
4497{
4498#if defined(arch_atomic64_add_negative_release)
4499	return arch_atomic64_add_negative_release(i, v);
4500#elif defined(arch_atomic64_add_negative_relaxed)
4501	__atomic_release_fence();
4502	return arch_atomic64_add_negative_relaxed(i, v);
4503#elif defined(arch_atomic64_add_negative)
4504	return arch_atomic64_add_negative(i, v);
4505#else
4506	return raw_atomic64_add_return_release(i, v) < 0;
4507#endif
4508}
4509
4510/**
4511 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4512 * @i: s64 value to add
4513 * @v: pointer to atomic64_t
4514 *
4515 * Atomically updates @v to (@v + @i) with relaxed ordering.
4516 *
4517 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
4518 *
4519 * Return: @true if the resulting value of @v is negative, @false otherwise.
4520 */
4521static __always_inline bool
4522raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
4523{
4524#if defined(arch_atomic64_add_negative_relaxed)
4525	return arch_atomic64_add_negative_relaxed(i, v);
4526#elif defined(arch_atomic64_add_negative)
4527	return arch_atomic64_add_negative(i, v);
4528#else
4529	return raw_atomic64_add_return_relaxed(i, v) < 0;
4530#endif
4531}
4532
4533/**
4534 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
4535 * @v: pointer to atomic64_t
4536 * @a: s64 value to add
4537 * @u: s64 value to compare with
4538 *
4539 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4540 * Otherwise, @v is not modified and relaxed ordering is provided.
4541 *
4542 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4543 *
4544 * Return: The original value of @v.
4545 */
4546static __always_inline s64
4547raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
4548{
4549#if defined(arch_atomic64_fetch_add_unless)
4550	return arch_atomic64_fetch_add_unless(v, a, u);
4551#else
4552	s64 c = raw_atomic64_read(v);
4553
4554	do {
4555		if (unlikely(c == u))
4556			break;
4557	} while (!raw_atomic64_try_cmpxchg(v, &c, c + a));
4558
4559	return c;
4560#endif
4561}
4562
4563/**
4564 * raw_atomic64_add_unless() - atomic add unless value with full ordering
4565 * @v: pointer to atomic64_t
4566 * @a: s64 value to add
4567 * @u: s64 value to compare with
4568 *
4569 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4570 * Otherwise, @v is not modified and relaxed ordering is provided.
4571 *
4572 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4573 *
4574 * Return: @true if @v was updated, @false otherwise.
4575 */
4576static __always_inline bool
4577raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
4578{
4579#if defined(arch_atomic64_add_unless)
4580	return arch_atomic64_add_unless(v, a, u);
4581#else
4582	return raw_atomic64_fetch_add_unless(v, a, u) != u;
4583#endif
4584}
4585
4586/**
4587 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
4588 * @v: pointer to atomic64_t
4589 *
4590 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4591 * Otherwise, @v is not modified and relaxed ordering is provided.
4592 *
4593 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4594 *
4595 * Return: @true if @v was updated, @false otherwise.
4596 */
4597static __always_inline bool
4598raw_atomic64_inc_not_zero(atomic64_t *v)
4599{
4600#if defined(arch_atomic64_inc_not_zero)
4601	return arch_atomic64_inc_not_zero(v);
4602#else
4603	return raw_atomic64_add_unless(v, 1, 0);
4604#endif
4605}
4606
4607/**
4608 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
4609 * @v: pointer to atomic64_t
4610 *
4611 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4612 * Otherwise, @v is not modified and relaxed ordering is provided.
4613 *
4614 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4615 *
4616 * Return: @true if @v was updated, @false otherwise.
4617 */
4618static __always_inline bool
4619raw_atomic64_inc_unless_negative(atomic64_t *v)
4620{
4621#if defined(arch_atomic64_inc_unless_negative)
4622	return arch_atomic64_inc_unless_negative(v);
4623#else
4624	s64 c = raw_atomic64_read(v);
4625
4626	do {
4627		if (unlikely(c < 0))
4628			return false;
4629	} while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));
4630
4631	return true;
4632#endif
4633}
4634
4635/**
4636 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
4637 * @v: pointer to atomic64_t
4638 *
4639 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4640 * Otherwise, @v is not modified and relaxed ordering is provided.
4641 *
4642 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4643 *
4644 * Return: @true if @v was updated, @false otherwise.
4645 */
4646static __always_inline bool
4647raw_atomic64_dec_unless_positive(atomic64_t *v)
4648{
4649#if defined(arch_atomic64_dec_unless_positive)
4650	return arch_atomic64_dec_unless_positive(v);
4651#else
4652	s64 c = raw_atomic64_read(v);
4653
4654	do {
4655		if (unlikely(c > 0))
4656			return false;
4657	} while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));
4658
4659	return true;
4660#endif
4661}
4662
4663/**
4664 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
4665 * @v: pointer to atomic64_t
4666 *
4667 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4668 * Otherwise, @v is not modified and relaxed ordering is provided.
4669 *
4670 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4671 *
4672 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4673 */
4674static __always_inline s64
4675raw_atomic64_dec_if_positive(atomic64_t *v)
4676{
4677#if defined(arch_atomic64_dec_if_positive)
4678	return arch_atomic64_dec_if_positive(v);
4679#else
4680	s64 dec, c = raw_atomic64_read(v);
4681
4682	do {
4683		dec = c - 1;
4684		if (unlikely(dec < 0))
4685			break;
4686	} while (!raw_atomic64_try_cmpxchg(v, &c, dec));
4687
4688	return dec;
4689#endif
4690}
4691
4692#endif /* _LINUX_ATOMIC_FALLBACK_H */
4693// b565db590afeeff0d7c9485ccbca5bb6e155749f