spinlock.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *    Out of line spinlock code.
  4 *
  5 *    Copyright IBM Corp. 2004, 2006
  6 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
  7 */
  8
  9#include <linux/types.h>
 10#include <linux/export.h>
 11#include <linux/spinlock.h>
 12#include <linux/jiffies.h>
 13#include <linux/sysctl.h>
 14#include <linux/init.h>
 15#include <linux/smp.h>
 16#include <linux/percpu.h>
 17#include <linux/io.h>
 18#include <asm/alternative.h>
 19#include <asm/machine.h>
 20#include <asm/asm.h>
 21
 22int spin_retry = -1;
 23
 24static int __init spin_retry_init(void)
 25{
 26	if (spin_retry < 0)
 27		spin_retry = 1000;
 28	return 0;
 29}
 30early_initcall(spin_retry_init);
 31
 32/*
 33 * spin_retry= parameter
 34 */
 35static int __init spin_retry_setup(char *str)
 36{
 37	spin_retry = simple_strtoul(str, &str, 0);
 38	return 1;
 39}
 40__setup("spin_retry=", spin_retry_setup);
 41
 42static const struct ctl_table s390_spin_sysctl_table[] = {
 43	{
 44		.procname	= "spin_retry",
 45		.data		= &spin_retry,
 46		.maxlen		= sizeof(int),
 47		.mode		= 0644,
 48		.proc_handler	= proc_dointvec,
 49	},
 50};
 51
 52static int __init init_s390_spin_sysctls(void)
 53{
 54	register_sysctl_init("kernel", s390_spin_sysctl_table);
 55	return 0;
 56}
 57arch_initcall(init_s390_spin_sysctls);
 58
 59struct spin_wait {
 60	struct spin_wait *next, *prev;
 61	int node_id;
 62} __aligned(32);
 63
 64static DEFINE_PER_CPU_ALIGNED(struct spin_wait, spin_wait[4]);
 65
 66#define _Q_LOCK_CPU_OFFSET	0
 67#define _Q_LOCK_STEAL_OFFSET	16
 68#define _Q_TAIL_IDX_OFFSET	18
 69#define _Q_TAIL_CPU_OFFSET	20
 70
 71#define _Q_LOCK_CPU_MASK	0x0000ffff
 72#define _Q_LOCK_STEAL_ADD	0x00010000
 73#define _Q_LOCK_STEAL_MASK	0x00030000
 74#define _Q_TAIL_IDX_MASK	0x000c0000
 75#define _Q_TAIL_CPU_MASK	0xfff00000
 76
 77#define _Q_LOCK_MASK		(_Q_LOCK_CPU_MASK | _Q_LOCK_STEAL_MASK)
 78#define _Q_TAIL_MASK		(_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
 79
 80void arch_spin_lock_setup(int cpu)
 81{
 82	struct spin_wait *node;
 83	int ix;
 84
 85	node = per_cpu_ptr(&spin_wait[0], cpu);
 86	for (ix = 0; ix < 4; ix++, node++) {
 87		memset(node, 0, sizeof(*node));
 88		node->node_id = ((cpu + 1) << _Q_TAIL_CPU_OFFSET) +
 89			(ix << _Q_TAIL_IDX_OFFSET);
 90	}
 91}
 92
 93static inline int arch_load_niai4(int *lock)
 94{
 95	int owner;
 96
 97	asm_inline volatile(
 98		ALTERNATIVE("nop", ".insn rre,0xb2fa0000,4,0", ALT_FACILITY(49)) /* NIAI 4 */
 99		"	l	%[owner],%[lock]"
100		: [owner] "=d" (owner) : [lock] "R" (*lock) : "memory");
101	return owner;
102}
103
104#ifdef __HAVE_ASM_FLAG_OUTPUTS__
105
106static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)
107{
108	int cc;
109
110	asm_inline volatile(
111		ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */
112		"	cs	%[old],%[new],%[lock]"
113		: [old] "+d" (old), [lock] "+Q" (*lock), "=@cc" (cc)
114		: [new] "d" (new)
115		: "memory");
116	return cc == 0;
117}
118
119#else /* __HAVE_ASM_FLAG_OUTPUTS__ */
120
121static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)
122{
123	int expected = old;
124
125	asm_inline volatile(
126		ALTERNATIVE("nop", ".insn rre,0xb2fa0000,8,0", ALT_FACILITY(49)) /* NIAI 8 */
127		"	cs	%[old],%[new],%[lock]"
128		: [old] "+d" (old), [lock] "+Q" (*lock)
129		: [new] "d" (new)
130		: "cc", "memory");
131	return expected == old;
132}
133
134#endif /* __HAVE_ASM_FLAG_OUTPUTS__ */
135
136static inline struct spin_wait *arch_spin_decode_tail(int lock)
137{
138	int ix, cpu;
139
140	ix = (lock & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
141	cpu = (lock & _Q_TAIL_CPU_MASK) >> _Q_TAIL_CPU_OFFSET;
142	return per_cpu_ptr(&spin_wait[ix], cpu - 1);
143}
144
145static inline int arch_spin_yield_target(int lock, struct spin_wait *node)
146{
147	if (lock & _Q_LOCK_CPU_MASK)
148		return lock & _Q_LOCK_CPU_MASK;
149	if (node == NULL || node->prev == NULL)
150		return 0;	/* 0 -> no target cpu */
151	while (node->prev)
152		node = node->prev;
153	return node->node_id >> _Q_TAIL_CPU_OFFSET;
154}
155
156static inline void arch_spin_lock_queued(arch_spinlock_t *lp)
157{
158	struct spin_wait *node, *next;
159	int lockval, ix, node_id, tail_id, old, new, owner, count;
160
161	ix = get_lowcore()->spinlock_index++;
162	barrier();
163	lockval = spinlock_lockval();	/* cpu + 1 */
164	node = this_cpu_ptr(&spin_wait[ix]);
165	node->prev = node->next = NULL;
166	node_id = node->node_id;
167
168	/* Enqueue the node for this CPU in the spinlock wait queue */
169	old = READ_ONCE(lp->lock);
170	while (1) {
171		if ((old & _Q_LOCK_CPU_MASK) == 0 &&
172		    (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {
173			/*
174			 * The lock is free but there may be waiters.
175			 * With no waiters simply take the lock, if there
176			 * are waiters try to steal the lock. The lock may
177			 * be stolen three times before the next queued
178			 * waiter will get the lock.
179			 */
180			new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;
181			if (arch_try_cmpxchg(&lp->lock, &old, new))
182				/* Got the lock */
183				goto out;
184			/* lock passing in progress */
185			continue;
186		}
187		/* Make the node of this CPU the new tail. */
188		new = node_id | (old & _Q_LOCK_MASK);
189		if (arch_try_cmpxchg(&lp->lock, &old, new))
190			break;
191	}
192	/* Set the 'next' pointer of the tail node in the queue */
193	tail_id = old & _Q_TAIL_MASK;
194	if (tail_id != 0) {
195		node->prev = arch_spin_decode_tail(tail_id);
196		WRITE_ONCE(node->prev->next, node);
197	}
198
199	/* Pass the virtual CPU to the lock holder if it is not running */
200	owner = arch_spin_yield_target(old, node);
201	if (owner && arch_vcpu_is_preempted(owner - 1))
202		smp_yield_cpu(owner - 1);
203
204	/* Spin on the CPU local node->prev pointer */
205	if (tail_id != 0) {
206		count = spin_retry;
207		while (READ_ONCE(node->prev) != NULL) {
208			if (count-- >= 0)
209				continue;
210			count = spin_retry;
211			/* Query running state of lock holder again. */
212			owner = arch_spin_yield_target(old, node);
213			if (owner && arch_vcpu_is_preempted(owner - 1))
214				smp_yield_cpu(owner - 1);
215		}
216	}
217
218	/* Spin on the lock value in the spinlock_t */
219	count = spin_retry;
220	while (1) {
221		old = READ_ONCE(lp->lock);
222		owner = old & _Q_LOCK_CPU_MASK;
223		if (!owner) {
224			tail_id = old & _Q_TAIL_MASK;
225			new = ((tail_id != node_id) ? tail_id : 0) | lockval;
226			if (arch_try_cmpxchg(&lp->lock, &old, new))
227				/* Got the lock */
228				break;
229			continue;
230		}
231		if (count-- >= 0)
232			continue;
233		count = spin_retry;
234		if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1))
235			smp_yield_cpu(owner - 1);
236	}
237
238	/* Pass lock_spin job to next CPU in the queue */
239	if (node_id && tail_id != node_id) {
240		/* Wait until the next CPU has set up the 'next' pointer */
241		while ((next = READ_ONCE(node->next)) == NULL)
242			;
243		next->prev = NULL;
244	}
245
246 out:
247	get_lowcore()->spinlock_index--;
248}
249
250static inline void arch_spin_lock_classic(arch_spinlock_t *lp)
251{
252	int lockval, old, new, owner, count;
253
254	lockval = spinlock_lockval();	/* cpu + 1 */
255
256	/* Pass the virtual CPU to the lock holder if it is not running */
257	owner = arch_spin_yield_target(READ_ONCE(lp->lock), NULL);
258	if (owner && arch_vcpu_is_preempted(owner - 1))
259		smp_yield_cpu(owner - 1);
260
261	count = spin_retry;
262	while (1) {
263		old = arch_load_niai4(&lp->lock);
264		owner = old & _Q_LOCK_CPU_MASK;
265		/* Try to get the lock if it is free. */
266		if (!owner) {
267			new = (old & _Q_TAIL_MASK) | lockval;
268			if (arch_try_cmpxchg_niai8(&lp->lock, old, new)) {
269				/* Got the lock */
270				return;
271			}
272			continue;
273		}
274		if (count-- >= 0)
275			continue;
276		count = spin_retry;
277		if (!machine_is_lpar() || arch_vcpu_is_preempted(owner - 1))
278			smp_yield_cpu(owner - 1);
279	}
280}
281
282void arch_spin_lock_wait(arch_spinlock_t *lp)
283{
284	if (test_cpu_flag(CIF_DEDICATED_CPU))
285		arch_spin_lock_queued(lp);
286	else
287		arch_spin_lock_classic(lp);
288}
289EXPORT_SYMBOL(arch_spin_lock_wait);
290
291int arch_spin_trylock_retry(arch_spinlock_t *lp)
292{
293	int cpu = spinlock_lockval();
294	int owner, count;
295
296	for (count = spin_retry; count > 0; count--) {
297		owner = READ_ONCE(lp->lock);
298		/* Try to get the lock if it is free. */
299		if (!owner) {
300			if (arch_try_cmpxchg(&lp->lock, &owner, cpu))
301				return 1;
302		}
303	}
304	return 0;
305}
306EXPORT_SYMBOL(arch_spin_trylock_retry);
307
308void arch_read_lock_wait(arch_rwlock_t *rw)
309{
310	if (unlikely(in_interrupt())) {
311		while (READ_ONCE(rw->cnts) & 0x10000)
312			barrier();
313		return;
314	}
315
316	/* Remove this reader again to allow recursive read locking */
317	__atomic_add_const(-1, &rw->cnts);
318	/* Put the reader into the wait queue */
319	arch_spin_lock(&rw->wait);
320	/* Now add this reader to the count value again */
321	__atomic_add_const(1, &rw->cnts);
322	/* Loop until the writer is done */
323	while (READ_ONCE(rw->cnts) & 0x10000)
324		barrier();
325	arch_spin_unlock(&rw->wait);
326}
327EXPORT_SYMBOL(arch_read_lock_wait);
328
329void arch_write_lock_wait(arch_rwlock_t *rw)
330{
331	int old;
332
333	/* Add this CPU to the write waiters */
334	__atomic_add(0x20000, &rw->cnts);
335
336	/* Put the writer into the wait queue */
337	arch_spin_lock(&rw->wait);
338
339	while (1) {
340		old = READ_ONCE(rw->cnts);
341		if ((old & 0x1ffff) == 0 &&
342		    arch_try_cmpxchg(&rw->cnts, &old, old | 0x10000))
343			/* Got the lock */
344			break;
345		barrier();
346	}
347
348	arch_spin_unlock(&rw->wait);
349}
350EXPORT_SYMBOL(arch_write_lock_wait);
351
352void arch_spin_relax(arch_spinlock_t *lp)
353{
354	int cpu;
355
356	cpu = READ_ONCE(lp->lock) & _Q_LOCK_CPU_MASK;
357	if (!cpu)
358		return;
359	if (machine_is_lpar() && !arch_vcpu_is_preempted(cpu - 1))
360		return;
361	smp_yield_cpu(cpu - 1);
362}
363EXPORT_SYMBOL(arch_spin_relax);