qspinlock.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/bug.h>
  3#include <linux/compiler.h>
  4#include <linux/export.h>
  5#include <linux/percpu.h>
  6#include <linux/processor.h>
  7#include <linux/smp.h>
  8#include <linux/topology.h>
  9#include <linux/sched/clock.h>
 10#include <asm/qspinlock.h>
 11#include <asm/paravirt.h>
 12#include <trace/events/lock.h>
 13
 14#define MAX_NODES	4
 15
 16struct qnode {
 17	struct qnode	*next;
 18	struct qspinlock *lock;
 19	int		cpu;
 20	u8		sleepy; /* 1 if the previous vCPU was preempted or
 21				 * if the previous node was sleepy */
 22	u8		locked; /* 1 if lock acquired */
 23};
 24
 25struct qnodes {
 26	int		count;
 27	struct qnode nodes[MAX_NODES];
 28};
 29
 30/* Tuning parameters */
 31static int steal_spins __read_mostly = (1 << 5);
 32static int remote_steal_spins __read_mostly = (1 << 2);
 33#if _Q_SPIN_TRY_LOCK_STEAL == 1
 34static const bool maybe_stealers = true;
 35#else
 36static bool maybe_stealers __read_mostly = true;
 37#endif
 38static int head_spins __read_mostly = (1 << 8);
 39
 40static bool pv_yield_owner __read_mostly = true;
 41static bool pv_yield_allow_steal __read_mostly = false;
 42static bool pv_spin_on_preempted_owner __read_mostly = false;
 43static bool pv_sleepy_lock __read_mostly = true;
 44static bool pv_sleepy_lock_sticky __read_mostly = false;
 45static u64 pv_sleepy_lock_interval_ns __read_mostly = 0;
 46static int pv_sleepy_lock_factor __read_mostly = 256;
 47static bool pv_yield_prev __read_mostly = true;
 48static bool pv_yield_sleepy_owner __read_mostly = true;
 49static bool pv_prod_head __read_mostly = false;
 50
 51static DEFINE_PER_CPU_ALIGNED(struct qnodes, qnodes);
 52static DEFINE_PER_CPU_ALIGNED(u64, sleepy_lock_seen_clock);
 53
 54#if _Q_SPIN_SPEC_BARRIER == 1
 55#define spec_barrier() do { asm volatile("ori 31,31,0" ::: "memory"); } while (0)
 56#else
 57#define spec_barrier() do { } while (0)
 58#endif
 59
 60static __always_inline bool recently_sleepy(void)
 61{
 62	/* pv_sleepy_lock is true when this is called */
 63	if (pv_sleepy_lock_interval_ns) {
 64		u64 seen = this_cpu_read(sleepy_lock_seen_clock);
 65
 66		if (seen) {
 67			u64 delta = sched_clock() - seen;
 68			if (delta < pv_sleepy_lock_interval_ns)
 69				return true;
 70			this_cpu_write(sleepy_lock_seen_clock, 0);
 71		}
 72	}
 73
 74	return false;
 75}
 76
 77static __always_inline int get_steal_spins(bool paravirt, bool sleepy)
 78{
 79	if (paravirt && sleepy)
 80		return steal_spins * pv_sleepy_lock_factor;
 81	else
 82		return steal_spins;
 83}
 84
 85static __always_inline int get_remote_steal_spins(bool paravirt, bool sleepy)
 86{
 87	if (paravirt && sleepy)
 88		return remote_steal_spins * pv_sleepy_lock_factor;
 89	else
 90		return remote_steal_spins;
 91}
 92
 93static __always_inline int get_head_spins(bool paravirt, bool sleepy)
 94{
 95	if (paravirt && sleepy)
 96		return head_spins * pv_sleepy_lock_factor;
 97	else
 98		return head_spins;
 99}
100
101static inline u32 encode_tail_cpu(int cpu)
102{
103	return (cpu + 1) << _Q_TAIL_CPU_OFFSET;
104}
105
106static inline int decode_tail_cpu(u32 val)
107{
108	return (val >> _Q_TAIL_CPU_OFFSET) - 1;
109}
110
111static inline int get_owner_cpu(u32 val)
112{
113	return (val & _Q_OWNER_CPU_MASK) >> _Q_OWNER_CPU_OFFSET;
114}
115
116/*
117 * Try to acquire the lock if it was not already locked. If the tail matches
118 * mytail then clear it, otherwise leave it unchnaged. Return previous value.
119 *
120 * This is used by the head of the queue to acquire the lock and clean up
121 * its tail if it was the last one queued.
122 */
123static __always_inline u32 trylock_clean_tail(struct qspinlock *lock, u32 tail)
124{
125	u32 newval = queued_spin_encode_locked_val();
126	u32 prev, tmp;
127
128	asm volatile(
129"1:	lwarx	%0,0,%2,%7	# trylock_clean_tail			\n"
130	/* This test is necessary if there could be stealers */
131"	andi.	%1,%0,%5						\n"
132"	bne	3f							\n"
133	/* Test whether the lock tail == mytail */
134"	and	%1,%0,%6						\n"
135"	cmpw	0,%1,%3							\n"
136	/* Merge the new locked value */
137"	or	%1,%1,%4						\n"
138"	bne	2f							\n"
139	/* If the lock tail matched, then clear it, otherwise leave it. */
140"	andc	%1,%1,%6						\n"
141"2:	stwcx.	%1,0,%2							\n"
142"	bne-	1b							\n"
143"\t"	PPC_ACQUIRE_BARRIER "						\n"
144"3:									\n"
145	: "=&r" (prev), "=&r" (tmp)
146	: "r" (&lock->val), "r"(tail), "r" (newval),
147	  "i" (_Q_LOCKED_VAL),
148	  "r" (_Q_TAIL_CPU_MASK),
149	  "i" (_Q_SPIN_EH_HINT)
150	: "cr0", "memory");
151
152	return prev;
153}
154
155/*
156 * Publish our tail, replacing previous tail. Return previous value.
157 *
158 * This provides a release barrier for publishing node, this pairs with the
159 * acquire barrier in get_tail_qnode() when the next CPU finds this tail
160 * value.
161 */
162static __always_inline u32 publish_tail_cpu(struct qspinlock *lock, u32 tail)
163{
164	u32 prev, tmp;
165
166	kcsan_release();
167
168	asm volatile(
169"\t"	PPC_RELEASE_BARRIER "						\n"
170"1:	lwarx	%0,0,%2		# publish_tail_cpu			\n"
171"	andc	%1,%0,%4						\n"
172"	or	%1,%1,%3						\n"
173"	stwcx.	%1,0,%2							\n"
174"	bne-	1b							\n"
175	: "=&r" (prev), "=&r"(tmp)
176	: "r" (&lock->val), "r" (tail), "r"(_Q_TAIL_CPU_MASK)
177	: "cr0", "memory");
178
179	return prev;
180}
181
182static __always_inline u32 set_mustq(struct qspinlock *lock)
183{
184	u32 prev;
185
186	asm volatile(
187"1:	lwarx	%0,0,%1		# set_mustq				\n"
188"	or	%0,%0,%2						\n"
189"	stwcx.	%0,0,%1							\n"
190"	bne-	1b							\n"
191	: "=&r" (prev)
192	: "r" (&lock->val), "r" (_Q_MUST_Q_VAL)
193	: "cr0", "memory");
194
195	return prev;
196}
197
198static __always_inline u32 clear_mustq(struct qspinlock *lock)
199{
200	u32 prev;
201
202	asm volatile(
203"1:	lwarx	%0,0,%1		# clear_mustq				\n"
204"	andc	%0,%0,%2						\n"
205"	stwcx.	%0,0,%1							\n"
206"	bne-	1b							\n"
207	: "=&r" (prev)
208	: "r" (&lock->val), "r" (_Q_MUST_Q_VAL)
209	: "cr0", "memory");
210
211	return prev;
212}
213
214static __always_inline bool try_set_sleepy(struct qspinlock *lock, u32 old)
215{
216	u32 prev;
217	u32 new = old | _Q_SLEEPY_VAL;
218
219	BUG_ON(!(old & _Q_LOCKED_VAL));
220	BUG_ON(old & _Q_SLEEPY_VAL);
221
222	asm volatile(
223"1:	lwarx	%0,0,%1		# try_set_sleepy			\n"
224"	cmpw	0,%0,%2							\n"
225"	bne-	2f							\n"
226"	stwcx.	%3,0,%1							\n"
227"	bne-	1b							\n"
228"2:									\n"
229	: "=&r" (prev)
230	: "r" (&lock->val), "r"(old), "r" (new)
231	: "cr0", "memory");
232
233	return likely(prev == old);
234}
235
236static __always_inline void seen_sleepy_owner(struct qspinlock *lock, u32 val)
237{
238	if (pv_sleepy_lock) {
239		if (pv_sleepy_lock_interval_ns)
240			this_cpu_write(sleepy_lock_seen_clock, sched_clock());
241		if (!(val & _Q_SLEEPY_VAL))
242			try_set_sleepy(lock, val);
243	}
244}
245
246static __always_inline void seen_sleepy_lock(void)
247{
248	if (pv_sleepy_lock && pv_sleepy_lock_interval_ns)
249		this_cpu_write(sleepy_lock_seen_clock, sched_clock());
250}
251
252static __always_inline void seen_sleepy_node(void)
253{
254	if (pv_sleepy_lock) {
255		if (pv_sleepy_lock_interval_ns)
256			this_cpu_write(sleepy_lock_seen_clock, sched_clock());
257		/* Don't set sleepy because we likely have a stale val */
258	}
259}
260
261static struct qnode *get_tail_qnode(struct qspinlock *lock, int prev_cpu)
262{
263	struct qnodes *qnodesp = per_cpu_ptr(&qnodes, prev_cpu);
264	int idx;
265
266	/*
267	 * After publishing the new tail and finding a previous tail in the
268	 * previous val (which is the control dependency), this barrier
269	 * orders the release barrier in publish_tail_cpu performed by the
270	 * last CPU, with subsequently looking at its qnode structures
271	 * after the barrier.
272	 */
273	smp_acquire__after_ctrl_dep();
274
275	for (idx = 0; idx < MAX_NODES; idx++) {
276		struct qnode *qnode = &qnodesp->nodes[idx];
277		if (qnode->lock == lock)
278			return qnode;
279	}
280
281	BUG();
282}
283
284/* Called inside spin_begin(). Returns whether or not the vCPU was preempted. */
285static __always_inline bool __yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt, bool mustq)
286{
287	int owner;
288	u32 yield_count;
289	bool preempted = false;
290
291	BUG_ON(!(val & _Q_LOCKED_VAL));
292
293	if (!paravirt)
294		goto relax;
295
296	if (!pv_yield_owner)
297		goto relax;
298
299	owner = get_owner_cpu(val);
300	yield_count = yield_count_of(owner);
301
302	if ((yield_count & 1) == 0)
303		goto relax; /* owner vcpu is running */
304
305	spin_end();
306
307	seen_sleepy_owner(lock, val);
308	preempted = true;
309
310	/*
311	 * Read the lock word after sampling the yield count. On the other side
312	 * there may a wmb because the yield count update is done by the
313	 * hypervisor preemption and the value update by the OS, however this
314	 * ordering might reduce the chance of out of order accesses and
315	 * improve the heuristic.
316	 */
317	smp_rmb();
318
319	if (READ_ONCE(lock->val) == val) {
320		if (mustq)
321			clear_mustq(lock);
322		yield_to_preempted(owner, yield_count);
323		if (mustq)
324			set_mustq(lock);
325		spin_begin();
326
327		/* Don't relax if we yielded. Maybe we should? */
328		return preempted;
329	}
330	spin_begin();
331relax:
332	spin_cpu_relax();
333
334	return preempted;
335}
336
337/* Called inside spin_begin(). Returns whether or not the vCPU was preempted. */
338static __always_inline bool yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt)
339{
340	return __yield_to_locked_owner(lock, val, paravirt, false);
341}
342
343/* Called inside spin_begin(). Returns whether or not the vCPU was preempted. */
344static __always_inline bool yield_head_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt)
345{
346	bool mustq = false;
347
348	if ((val & _Q_MUST_Q_VAL) && pv_yield_allow_steal)
349		mustq = true;
350
351	return __yield_to_locked_owner(lock, val, paravirt, mustq);
352}
353
354static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool paravirt)
355{
356	struct qnode *next;
357	int owner;
358
359	if (!paravirt)
360		return;
361	if (!pv_yield_sleepy_owner)
362		return;
363
364	next = READ_ONCE(node->next);
365	if (!next)
366		return;
367
368	if (next->sleepy)
369		return;
370
371	owner = get_owner_cpu(val);
372	if (vcpu_is_preempted(owner))
373		next->sleepy = 1;
374}
375
376/* Called inside spin_begin() */
377static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, int prev_cpu, bool paravirt)
378{
379	u32 yield_count;
380	bool preempted = false;
381
382	if (!paravirt)
383		goto relax;
384
385	if (!pv_yield_sleepy_owner)
386		goto yield_prev;
387
388	/*
389	 * If the previous waiter was preempted it might not be able to
390	 * propagate sleepy to us, so check the lock in that case too.
391	 */
392	if (node->sleepy || vcpu_is_preempted(prev_cpu)) {
393		u32 val = READ_ONCE(lock->val);
394
395		if (val & _Q_LOCKED_VAL) {
396			if (node->next && !node->next->sleepy) {
397				/*
398				 * Propagate sleepy to next waiter. Only if
399				 * owner is preempted, which allows the queue
400				 * to become "non-sleepy" if vCPU preemption
401				 * ceases to occur, even if the lock remains
402				 * highly contended.
403				 */
404				if (vcpu_is_preempted(get_owner_cpu(val)))
405					node->next->sleepy = 1;
406			}
407
408			preempted = yield_to_locked_owner(lock, val, paravirt);
409			if (preempted)
410				return preempted;
411		}
412		node->sleepy = false;
413	}
414
415yield_prev:
416	if (!pv_yield_prev)
417		goto relax;
418
419	yield_count = yield_count_of(prev_cpu);
420	if ((yield_count & 1) == 0)
421		goto relax; /* owner vcpu is running */
422
423	spin_end();
424
425	preempted = true;
426	seen_sleepy_node();
427
428	smp_rmb(); /* See __yield_to_locked_owner comment */
429
430	if (!READ_ONCE(node->locked)) {
431		yield_to_preempted(prev_cpu, yield_count);
432		spin_begin();
433		return preempted;
434	}
435	spin_begin();
436
437relax:
438	spin_cpu_relax();
439
440	return preempted;
441}
442
443static __always_inline bool steal_break(u32 val, int iters, bool paravirt, bool sleepy)
444{
445	if (iters >= get_steal_spins(paravirt, sleepy))
446		return true;
447
448	if (IS_ENABLED(CONFIG_NUMA) &&
449	    (iters >= get_remote_steal_spins(paravirt, sleepy))) {
450		int cpu = get_owner_cpu(val);
451		if (numa_node_id() != cpu_to_node(cpu))
452			return true;
453	}
454	return false;
455}
456
457static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool paravirt)
458{
459	bool seen_preempted = false;
460	bool sleepy = false;
461	int iters = 0;
462	u32 val;
463
464	if (!steal_spins) {
465		/* XXX: should spin_on_preempted_owner do anything here? */
466		return false;
467	}
468
469	/* Attempt to steal the lock */
470	spin_begin();
471	do {
472		bool preempted = false;
473
474		val = READ_ONCE(lock->val);
475		if (val & _Q_MUST_Q_VAL)
476			break;
477		spec_barrier();
478
479		if (unlikely(!(val & _Q_LOCKED_VAL))) {
480			spin_end();
481			if (__queued_spin_trylock_steal(lock))
482				return true;
483			spin_begin();
484		} else {
485			preempted = yield_to_locked_owner(lock, val, paravirt);
486		}
487
488		if (paravirt && pv_sleepy_lock) {
489			if (!sleepy) {
490				if (val & _Q_SLEEPY_VAL) {
491					seen_sleepy_lock();
492					sleepy = true;
493				} else if (recently_sleepy()) {
494					sleepy = true;
495				}
496			}
497			if (pv_sleepy_lock_sticky && seen_preempted &&
498			    !(val & _Q_SLEEPY_VAL)) {
499				if (try_set_sleepy(lock, val))
500					val |= _Q_SLEEPY_VAL;
501			}
502		}
503
504		if (preempted) {
505			seen_preempted = true;
506			sleepy = true;
507			if (!pv_spin_on_preempted_owner)
508				iters++;
509			/*
510			 * pv_spin_on_preempted_owner don't increase iters
511			 * while the owner is preempted -- we won't interfere
512			 * with it by definition. This could introduce some
513			 * latency issue if we continually observe preempted
514			 * owners, but hopefully that's a rare corner case of
515			 * a badly oversubscribed system.
516			 */
517		} else {
518			iters++;
519		}
520	} while (!steal_break(val, iters, paravirt, sleepy));
521
522	spin_end();
523
524	return false;
525}
526
527static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, bool paravirt)
528{
529	struct qnodes *qnodesp;
530	struct qnode *next, *node;
531	u32 val, old, tail;
532	bool seen_preempted = false;
533	bool sleepy = false;
534	bool mustq = false;
535	int idx;
536	int iters = 0;
537
538	BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
539
540	qnodesp = this_cpu_ptr(&qnodes);
541	if (unlikely(qnodesp->count >= MAX_NODES)) {
542		spec_barrier();
543		while (!queued_spin_trylock(lock))
544			cpu_relax();
545		return;
546	}
547
548	idx = qnodesp->count++;
549	/*
550	 * Ensure that we increment the head node->count before initialising
551	 * the actual node. If the compiler is kind enough to reorder these
552	 * stores, then an IRQ could overwrite our assignments.
553	 */
554	barrier();
555	node = &qnodesp->nodes[idx];
556	node->next = NULL;
557	node->lock = lock;
558	node->cpu = smp_processor_id();
559	node->sleepy = 0;
560	node->locked = 0;
561
562	tail = encode_tail_cpu(node->cpu);
563
564	/*
565	 * Assign all attributes of a node before it can be published.
566	 * Issues an lwsync, serving as a release barrier, as well as a
567	 * compiler barrier.
568	 */
569	old = publish_tail_cpu(lock, tail);
570
571	/*
572	 * If there was a previous node; link it and wait until reaching the
573	 * head of the waitqueue.
574	 */
575	if (old & _Q_TAIL_CPU_MASK) {
576		int prev_cpu = decode_tail_cpu(old);
577		struct qnode *prev = get_tail_qnode(lock, prev_cpu);
578
579		/* Link @node into the waitqueue. */
580		WRITE_ONCE(prev->next, node);
581
582		/* Wait for mcs node lock to be released */
583		spin_begin();
584		while (!READ_ONCE(node->locked)) {
585			spec_barrier();
586
587			if (yield_to_prev(lock, node, prev_cpu, paravirt))
588				seen_preempted = true;
589		}
590		spec_barrier();
591		spin_end();
592
593		smp_rmb(); /* acquire barrier for the mcs lock */
594
595		/*
596		 * Generic qspinlocks have this prefetch here, but it seems
597		 * like it could cause additional line transitions because
598		 * the waiter will keep loading from it.
599		 */
600		if (_Q_SPIN_PREFETCH_NEXT) {
601			next = READ_ONCE(node->next);
602			if (next)
603				prefetchw(next);
604		}
605	}
606
607	/* We're at the head of the waitqueue, wait for the lock. */
608again:
609	spin_begin();
610	for (;;) {
611		bool preempted;
612
613		val = READ_ONCE(lock->val);
614		if (!(val & _Q_LOCKED_VAL))
615			break;
616		spec_barrier();
617
618		if (paravirt && pv_sleepy_lock && maybe_stealers) {
619			if (!sleepy) {
620				if (val & _Q_SLEEPY_VAL) {
621					seen_sleepy_lock();
622					sleepy = true;
623				} else if (recently_sleepy()) {
624					sleepy = true;
625				}
626			}
627			if (pv_sleepy_lock_sticky && seen_preempted &&
628			    !(val & _Q_SLEEPY_VAL)) {
629				if (try_set_sleepy(lock, val))
630					val |= _Q_SLEEPY_VAL;
631			}
632		}
633
634		propagate_sleepy(node, val, paravirt);
635		preempted = yield_head_to_locked_owner(lock, val, paravirt);
636		if (!maybe_stealers)
637			continue;
638
639		if (preempted)
640			seen_preempted = true;
641
642		if (paravirt && preempted) {
643			sleepy = true;
644
645			if (!pv_spin_on_preempted_owner)
646				iters++;
647		} else {
648			iters++;
649		}
650
651		if (!mustq && iters >= get_head_spins(paravirt, sleepy)) {
652			mustq = true;
653			set_mustq(lock);
654			val |= _Q_MUST_Q_VAL;
655		}
656	}
657	spec_barrier();
658	spin_end();
659
660	/* If we're the last queued, must clean up the tail. */
661	old = trylock_clean_tail(lock, tail);
662	if (unlikely(old & _Q_LOCKED_VAL)) {
663		BUG_ON(!maybe_stealers);
664		goto again; /* Can only be true if maybe_stealers. */
665	}
666
667	if ((old & _Q_TAIL_CPU_MASK) == tail)
668		goto release; /* We were the tail, no next. */
669
670	/* There is a next, must wait for node->next != NULL (MCS protocol) */
671	next = READ_ONCE(node->next);
672	if (!next) {
673		spin_begin();
674		while (!(next = READ_ONCE(node->next)))
675			cpu_relax();
676		spin_end();
677	}
678	spec_barrier();
679
680	/*
681	 * Unlock the next mcs waiter node. Release barrier is not required
682	 * here because the acquirer is only accessing the lock word, and
683	 * the acquire barrier we took the lock with orders that update vs
684	 * this store to locked. The corresponding barrier is the smp_rmb()
685	 * acquire barrier for mcs lock, above.
686	 */
687	if (paravirt && pv_prod_head) {
688		int next_cpu = next->cpu;
689		WRITE_ONCE(next->locked, 1);
690		if (_Q_SPIN_MISO)
691			asm volatile("miso" ::: "memory");
692		if (vcpu_is_preempted(next_cpu))
693			prod_cpu(next_cpu);
694	} else {
695		WRITE_ONCE(next->locked, 1);
696		if (_Q_SPIN_MISO)
697			asm volatile("miso" ::: "memory");
698	}
699
700release:
701	/*
702	 * Clear the lock before releasing the node, as another CPU might see stale
703	 * values if an interrupt occurs after we increment qnodesp->count
704	 * but before node->lock is initialized. The barrier ensures that
705	 * there are no further stores to the node after it has been released.
706	 */
707	node->lock = NULL;
708	barrier();
709	qnodesp->count--;
710}
711
712void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock)
713{
714	trace_contention_begin(lock, LCB_F_SPIN);
715	/*
716	 * This looks funny, but it induces the compiler to inline both
717	 * sides of the branch rather than share code as when the condition
718	 * is passed as the paravirt argument to the functions.
719	 */
720	if (IS_ENABLED(CONFIG_PARAVIRT_SPINLOCKS) && is_shared_processor()) {
721		if (try_to_steal_lock(lock, true))
722			spec_barrier();
723		else
724			queued_spin_lock_mcs_queue(lock, true);
725	} else {
726		if (try_to_steal_lock(lock, false))
727			spec_barrier();
728		else
729			queued_spin_lock_mcs_queue(lock, false);
730	}
731	trace_contention_end(lock, 0);
732}
733EXPORT_SYMBOL(queued_spin_lock_slowpath);
734
735#ifdef CONFIG_PARAVIRT_SPINLOCKS
736void pv_spinlocks_init(void)
737{
738}
739#endif
740
741#include <linux/debugfs.h>
742static int steal_spins_set(void *data, u64 val)
743{
744#if _Q_SPIN_TRY_LOCK_STEAL == 1
745	/* MAYBE_STEAL remains true */
746	steal_spins = val;
747#else
748	static DEFINE_MUTEX(lock);
749
750	/*
751	 * The lock slow path has a !maybe_stealers case that can assume
752	 * the head of queue will not see concurrent waiters. That waiter
753	 * is unsafe in the presence of stealers, so must keep them away
754	 * from one another.
755	 */
756
757	mutex_lock(&lock);
758	if (val && !steal_spins) {
759		maybe_stealers = true;
760		/* wait for queue head waiter to go away */
761		synchronize_rcu();
762		steal_spins = val;
763	} else if (!val && steal_spins) {
764		steal_spins = val;
765		/* wait for all possible stealers to go away */
766		synchronize_rcu();
767		maybe_stealers = false;
768	} else {
769		steal_spins = val;
770	}
771	mutex_unlock(&lock);
772#endif
773
774	return 0;
775}
776
777static int steal_spins_get(void *data, u64 *val)
778{
779	*val = steal_spins;
780
781	return 0;
782}
783
784DEFINE_SIMPLE_ATTRIBUTE(fops_steal_spins, steal_spins_get, steal_spins_set, "%llu\n");
785
786static int remote_steal_spins_set(void *data, u64 val)
787{
788	remote_steal_spins = val;
789
790	return 0;
791}
792
793static int remote_steal_spins_get(void *data, u64 *val)
794{
795	*val = remote_steal_spins;
796
797	return 0;
798}
799
800DEFINE_SIMPLE_ATTRIBUTE(fops_remote_steal_spins, remote_steal_spins_get, remote_steal_spins_set, "%llu\n");
801
802static int head_spins_set(void *data, u64 val)
803{
804	head_spins = val;
805
806	return 0;
807}
808
809static int head_spins_get(void *data, u64 *val)
810{
811	*val = head_spins;
812
813	return 0;
814}
815
816DEFINE_SIMPLE_ATTRIBUTE(fops_head_spins, head_spins_get, head_spins_set, "%llu\n");
817
818static int pv_yield_owner_set(void *data, u64 val)
819{
820	pv_yield_owner = !!val;
821
822	return 0;
823}
824
825static int pv_yield_owner_get(void *data, u64 *val)
826{
827	*val = pv_yield_owner;
828
829	return 0;
830}
831
832DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_owner, pv_yield_owner_get, pv_yield_owner_set, "%llu\n");
833
834static int pv_yield_allow_steal_set(void *data, u64 val)
835{
836	pv_yield_allow_steal = !!val;
837
838	return 0;
839}
840
841static int pv_yield_allow_steal_get(void *data, u64 *val)
842{
843	*val = pv_yield_allow_steal;
844
845	return 0;
846}
847
848DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_allow_steal, pv_yield_allow_steal_get, pv_yield_allow_steal_set, "%llu\n");
849
850static int pv_spin_on_preempted_owner_set(void *data, u64 val)
851{
852	pv_spin_on_preempted_owner = !!val;
853
854	return 0;
855}
856
857static int pv_spin_on_preempted_owner_get(void *data, u64 *val)
858{
859	*val = pv_spin_on_preempted_owner;
860
861	return 0;
862}
863
864DEFINE_SIMPLE_ATTRIBUTE(fops_pv_spin_on_preempted_owner, pv_spin_on_preempted_owner_get, pv_spin_on_preempted_owner_set, "%llu\n");
865
866static int pv_sleepy_lock_set(void *data, u64 val)
867{
868	pv_sleepy_lock = !!val;
869
870	return 0;
871}
872
873static int pv_sleepy_lock_get(void *data, u64 *val)
874{
875	*val = pv_sleepy_lock;
876
877	return 0;
878}
879
880DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock, pv_sleepy_lock_get, pv_sleepy_lock_set, "%llu\n");
881
882static int pv_sleepy_lock_sticky_set(void *data, u64 val)
883{
884	pv_sleepy_lock_sticky = !!val;
885
886	return 0;
887}
888
889static int pv_sleepy_lock_sticky_get(void *data, u64 *val)
890{
891	*val = pv_sleepy_lock_sticky;
892
893	return 0;
894}
895
896DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock_sticky, pv_sleepy_lock_sticky_get, pv_sleepy_lock_sticky_set, "%llu\n");
897
898static int pv_sleepy_lock_interval_ns_set(void *data, u64 val)
899{
900	pv_sleepy_lock_interval_ns = val;
901
902	return 0;
903}
904
905static int pv_sleepy_lock_interval_ns_get(void *data, u64 *val)
906{
907	*val = pv_sleepy_lock_interval_ns;
908
909	return 0;
910}
911
912DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock_interval_ns, pv_sleepy_lock_interval_ns_get, pv_sleepy_lock_interval_ns_set, "%llu\n");
913
914static int pv_sleepy_lock_factor_set(void *data, u64 val)
915{
916	pv_sleepy_lock_factor = val;
917
918	return 0;
919}
920
921static int pv_sleepy_lock_factor_get(void *data, u64 *val)
922{
923	*val = pv_sleepy_lock_factor;
924
925	return 0;
926}
927
928DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock_factor, pv_sleepy_lock_factor_get, pv_sleepy_lock_factor_set, "%llu\n");
929
930static int pv_yield_prev_set(void *data, u64 val)
931{
932	pv_yield_prev = !!val;
933
934	return 0;
935}
936
937static int pv_yield_prev_get(void *data, u64 *val)
938{
939	*val = pv_yield_prev;
940
941	return 0;
942}
943
944DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_prev, pv_yield_prev_get, pv_yield_prev_set, "%llu\n");
945
946static int pv_yield_sleepy_owner_set(void *data, u64 val)
947{
948	pv_yield_sleepy_owner = !!val;
949
950	return 0;
951}
952
953static int pv_yield_sleepy_owner_get(void *data, u64 *val)
954{
955	*val = pv_yield_sleepy_owner;
956
957	return 0;
958}
959
960DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_sleepy_owner, pv_yield_sleepy_owner_get, pv_yield_sleepy_owner_set, "%llu\n");
961
962static int pv_prod_head_set(void *data, u64 val)
963{
964	pv_prod_head = !!val;
965
966	return 0;
967}
968
969static int pv_prod_head_get(void *data, u64 *val)
970{
971	*val = pv_prod_head;
972
973	return 0;
974}
975
976DEFINE_SIMPLE_ATTRIBUTE(fops_pv_prod_head, pv_prod_head_get, pv_prod_head_set, "%llu\n");
977
978static __init int spinlock_debugfs_init(void)
979{
980	debugfs_create_file("qspl_steal_spins", 0600, arch_debugfs_dir, NULL, &fops_steal_spins);
981	debugfs_create_file("qspl_remote_steal_spins", 0600, arch_debugfs_dir, NULL, &fops_remote_steal_spins);
982	debugfs_create_file("qspl_head_spins", 0600, arch_debugfs_dir, NULL, &fops_head_spins);
983	if (is_shared_processor()) {
984		debugfs_create_file("qspl_pv_yield_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_owner);
985		debugfs_create_file("qspl_pv_yield_allow_steal", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_allow_steal);
986		debugfs_create_file("qspl_pv_spin_on_preempted_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_spin_on_preempted_owner);
987		debugfs_create_file("qspl_pv_sleepy_lock", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock);
988		debugfs_create_file("qspl_pv_sleepy_lock_sticky", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_sticky);
989		debugfs_create_file("qspl_pv_sleepy_lock_interval_ns", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_interval_ns);
990		debugfs_create_file("qspl_pv_sleepy_lock_factor", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_factor);
991		debugfs_create_file("qspl_pv_yield_prev", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_prev);
992		debugfs_create_file("qspl_pv_yield_sleepy_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_sleepy_owner);
993		debugfs_create_file("qspl_pv_prod_head", 0600, arch_debugfs_dir, NULL, &fops_pv_prod_head);
994	}
995
996	return 0;
997}
998device_initcall(spinlock_debugfs_init);