local_lock_internal.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _LINUX_LOCAL_LOCK_H
  3# error "Do not include directly, include linux/local_lock.h"
  4#endif
  5
  6#include <linux/percpu-defs.h>
  7#include <linux/lockdep.h>
  8
  9#ifndef CONFIG_PREEMPT_RT
 10
 11typedef struct {
 12#ifdef CONFIG_DEBUG_LOCK_ALLOC
 13	struct lockdep_map	dep_map;
 14	struct task_struct	*owner;
 15#endif
 16} local_lock_t;
 17
 18/* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */
 19typedef struct {
 20#ifdef CONFIG_DEBUG_LOCK_ALLOC
 21	struct lockdep_map	dep_map;
 22	struct task_struct	*owner;
 23#endif
 24	u8		acquired;
 25} local_trylock_t;
 26
 27#ifdef CONFIG_DEBUG_LOCK_ALLOC
 28# define LOCAL_LOCK_DEBUG_INIT(lockname)		\
 29	.dep_map = {					\
 30		.name = #lockname,			\
 31		.wait_type_inner = LD_WAIT_CONFIG,	\
 32		.lock_type = LD_LOCK_PERCPU,		\
 33	},						\
 34	.owner = NULL,
 35
 36# define LOCAL_TRYLOCK_DEBUG_INIT(lockname)		\
 37	LOCAL_LOCK_DEBUG_INIT(lockname)
 38
 39static inline void local_lock_acquire(local_lock_t *l)
 40{
 41	lock_map_acquire(&l->dep_map);
 42	DEBUG_LOCKS_WARN_ON(l->owner);
 43	l->owner = current;
 44}
 45
 46static inline void local_trylock_acquire(local_lock_t *l)
 47{
 48	lock_map_acquire_try(&l->dep_map);
 49	DEBUG_LOCKS_WARN_ON(l->owner);
 50	l->owner = current;
 51}
 52
 53static inline void local_lock_release(local_lock_t *l)
 54{
 55	DEBUG_LOCKS_WARN_ON(l->owner != current);
 56	l->owner = NULL;
 57	lock_map_release(&l->dep_map);
 58}
 59
 60static inline void local_lock_debug_init(local_lock_t *l)
 61{
 62	l->owner = NULL;
 63}
 64#else /* CONFIG_DEBUG_LOCK_ALLOC */
 65# define LOCAL_LOCK_DEBUG_INIT(lockname)
 66# define LOCAL_TRYLOCK_DEBUG_INIT(lockname)
 67static inline void local_lock_acquire(local_lock_t *l) { }
 68static inline void local_trylock_acquire(local_lock_t *l) { }
 69static inline void local_lock_release(local_lock_t *l) { }
 70static inline void local_lock_debug_init(local_lock_t *l) { }
 71#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
 72
 73#define INIT_LOCAL_LOCK(lockname)	{ LOCAL_LOCK_DEBUG_INIT(lockname) }
 74#define INIT_LOCAL_TRYLOCK(lockname)	{ LOCAL_TRYLOCK_DEBUG_INIT(lockname) }
 75
 76#define __local_lock_init(lock)					\
 77do {								\
 78	static struct lock_class_key __key;			\
 79								\
 80	debug_check_no_locks_freed((void *)lock, sizeof(*lock));\
 81	lockdep_init_map_type(&(lock)->dep_map, #lock, &__key,  \
 82			      0, LD_WAIT_CONFIG, LD_WAIT_INV,	\
 83			      LD_LOCK_PERCPU);			\
 84	local_lock_debug_init(lock);				\
 85} while (0)
 86
 87#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock)
 88
 89#define __spinlock_nested_bh_init(lock)				\
 90do {								\
 91	static struct lock_class_key __key;			\
 92								\
 93	debug_check_no_locks_freed((void *)lock, sizeof(*lock));\
 94	lockdep_init_map_type(&(lock)->dep_map, #lock, &__key,  \
 95			      0, LD_WAIT_CONFIG, LD_WAIT_INV,	\
 96			      LD_LOCK_NORMAL);			\
 97	local_lock_debug_init(lock);				\
 98} while (0)
 99
100#define __local_lock_acquire(lock)					\
101	do {								\
102		local_trylock_t *__tl;					\
103		local_lock_t *__l;					\
104									\
105		__l = (local_lock_t *)(lock);				\
106		__tl = (local_trylock_t *)__l;				\
107		_Generic((lock),					\
108			local_trylock_t *: ({				\
109				lockdep_assert(__tl->acquired == 0);	\
110				WRITE_ONCE(__tl->acquired, 1);		\
111			}),						\
112			local_lock_t *: (void)0);			\
113		local_lock_acquire(__l);				\
114	} while (0)
115
116#define __local_lock(lock)					\
117	do {							\
118		preempt_disable();				\
119		__local_lock_acquire(lock);			\
120	} while (0)
121
122#define __local_lock_irq(lock)					\
123	do {							\
124		local_irq_disable();				\
125		__local_lock_acquire(lock);			\
126	} while (0)
127
128#define __local_lock_irqsave(lock, flags)			\
129	do {							\
130		local_irq_save(flags);				\
131		__local_lock_acquire(lock);			\
132	} while (0)
133
134#define __local_trylock(lock)					\
135	({							\
136		local_trylock_t *__tl;				\
137								\
138		preempt_disable();				\
139		__tl = (lock);					\
140		if (READ_ONCE(__tl->acquired)) {		\
141			preempt_enable();			\
142			__tl = NULL;				\
143		} else {					\
144			WRITE_ONCE(__tl->acquired, 1);		\
145			local_trylock_acquire(			\
146				(local_lock_t *)__tl);		\
147		}						\
148		!!__tl;						\
149	})
150
151#define __local_trylock_irqsave(lock, flags)			\
152	({							\
153		local_trylock_t *__tl;				\
154								\
155		local_irq_save(flags);				\
156		__tl = (lock);					\
157		if (READ_ONCE(__tl->acquired)) {		\
158			local_irq_restore(flags);		\
159			__tl = NULL;				\
160		} else {					\
161			WRITE_ONCE(__tl->acquired, 1);		\
162			local_trylock_acquire(			\
163				(local_lock_t *)__tl);		\
164		}						\
165		!!__tl;						\
166	})
167
168/* preemption or migration must be disabled before calling __local_lock_is_locked */
169#define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired)
170
171#define __local_lock_release(lock)					\
172	do {								\
173		local_trylock_t *__tl;					\
174		local_lock_t *__l;					\
175									\
176		__l = (local_lock_t *)(lock);				\
177		__tl = (local_trylock_t *)__l;				\
178		local_lock_release(__l);				\
179		_Generic((lock),					\
180			local_trylock_t *: ({				\
181				lockdep_assert(__tl->acquired == 1);	\
182				WRITE_ONCE(__tl->acquired, 0);		\
183			}),						\
184			local_lock_t *: (void)0);			\
185	} while (0)
186
187#define __local_unlock(lock)					\
188	do {							\
189		__local_lock_release(lock);			\
190		preempt_enable();				\
191	} while (0)
192
193#define __local_unlock_irq(lock)				\
194	do {							\
195		__local_lock_release(lock);			\
196		local_irq_enable();				\
197	} while (0)
198
199#define __local_unlock_irqrestore(lock, flags)			\
200	do {							\
201		__local_lock_release(lock);			\
202		local_irq_restore(flags);			\
203	} while (0)
204
205#define __local_lock_nested_bh(lock)				\
206	do {							\
207		lockdep_assert_in_softirq();			\
208		local_lock_acquire((lock));			\
209	} while (0)
210
211#define __local_unlock_nested_bh(lock)				\
212	local_lock_release((lock))
213
214#else /* !CONFIG_PREEMPT_RT */
215
216/*
217 * On PREEMPT_RT local_lock maps to a per CPU spinlock, which protects the
218 * critical section while staying preemptible.
219 */
220typedef spinlock_t local_lock_t;
221typedef spinlock_t local_trylock_t;
222
223#define INIT_LOCAL_LOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname))
224#define INIT_LOCAL_TRYLOCK(lockname) __LOCAL_SPIN_LOCK_UNLOCKED((lockname))
225
226#define __local_lock_init(__l)					\
227	do {							\
228		local_spin_lock_init((__l));			\
229	} while (0)
230
231#define __local_trylock_init(__l)			__local_lock_init(__l)
232
233#define __local_lock(__lock)					\
234	do {							\
235		migrate_disable();				\
236		spin_lock((__lock));				\
237	} while (0)
238
239#define __local_lock_irq(lock)			__local_lock(lock)
240
241#define __local_lock_irqsave(lock, flags)			\
242	do {							\
243		typecheck(unsigned long, flags);		\
244		flags = 0;					\
245		__local_lock(lock);				\
246	} while (0)
247
248#define __local_unlock(__lock)					\
249	do {							\
250		spin_unlock((__lock));				\
251		migrate_enable();				\
252	} while (0)
253
254#define __local_unlock_irq(lock)		__local_unlock(lock)
255
256#define __local_unlock_irqrestore(lock, flags)	__local_unlock(lock)
257
258#define __local_lock_nested_bh(lock)				\
259do {								\
260	lockdep_assert_in_softirq_func();			\
261	spin_lock((lock));					\
262} while (0)
263
264#define __local_unlock_nested_bh(lock)				\
265do {								\
266	spin_unlock((lock));					\
267} while (0)
268
269#define __local_trylock(lock)					\
270	({							\
271		int __locked;					\
272								\
273		if (in_nmi() | in_hardirq()) {			\
274			__locked = 0;				\
275		} else {					\
276			migrate_disable();			\
277			__locked = spin_trylock((lock));	\
278			if (!__locked)				\
279				migrate_enable();		\
280		}						\
281		__locked;					\
282	})
283
284#define __local_trylock_irqsave(lock, flags)			\
285	({							\
286		typecheck(unsigned long, flags);		\
287		flags = 0;					\
288		__local_trylock(lock);				\
289	})
290
291/* migration must be disabled before calling __local_lock_is_locked */
292#define __local_lock_is_locked(__lock)					\
293	(rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current)
294
295#endif /* CONFIG_PREEMPT_RT */