futex.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _FUTEX_H
  3#define _FUTEX_H
  4
  5#include <linux/futex.h>
  6#include <linux/rtmutex.h>
  7#include <linux/sched/wake_q.h>
  8#include <linux/compat.h>
  9#include <linux/uaccess.h>
 10#include <linux/cleanup.h>
 11
 12#ifdef CONFIG_PREEMPT_RT
 13#include <linux/rcuwait.h>
 14#endif
 15
 16#include <asm/futex.h>
 17
 18/*
 19 * Futex flags used to encode options to functions and preserve them across
 20 * restarts.
 21 */
 22#define FLAGS_SIZE_8		0x0000
 23#define FLAGS_SIZE_16		0x0001
 24#define FLAGS_SIZE_32		0x0002
 25#define FLAGS_SIZE_64		0x0003
 26
 27#define FLAGS_SIZE_MASK		0x0003
 28
 29#ifdef CONFIG_MMU
 30# define FLAGS_SHARED		0x0010
 31#else
 32/*
 33 * NOMMU does not have per process address space. Let the compiler optimize
 34 * code away.
 35 */
 36# define FLAGS_SHARED		0x0000
 37#endif
 38#define FLAGS_CLOCKRT		0x0020
 39#define FLAGS_HAS_TIMEOUT	0x0040
 40#define FLAGS_NUMA		0x0080
 41#define FLAGS_STRICT		0x0100
 42#define FLAGS_MPOL		0x0200
 43
 44/* FUTEX_ to FLAGS_ */
 45static inline unsigned int futex_to_flags(unsigned int op)
 46{
 47	unsigned int flags = FLAGS_SIZE_32;
 48
 49	if (!(op & FUTEX_PRIVATE_FLAG))
 50		flags |= FLAGS_SHARED;
 51
 52	if (op & FUTEX_CLOCK_REALTIME)
 53		flags |= FLAGS_CLOCKRT;
 54
 55	return flags;
 56}
 57
 58#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_NUMA | FUTEX2_MPOL | FUTEX2_PRIVATE)
 59
 60/* FUTEX2_ to FLAGS_ */
 61static inline unsigned int futex2_to_flags(unsigned int flags2)
 62{
 63	unsigned int flags = flags2 & FUTEX2_SIZE_MASK;
 64
 65	if (!(flags2 & FUTEX2_PRIVATE))
 66		flags |= FLAGS_SHARED;
 67
 68	if (flags2 & FUTEX2_NUMA)
 69		flags |= FLAGS_NUMA;
 70
 71	if (flags2 & FUTEX2_MPOL)
 72		flags |= FLAGS_MPOL;
 73
 74	return flags;
 75}
 76
 77static inline unsigned int futex_size(unsigned int flags)
 78{
 79	return 1 << (flags & FLAGS_SIZE_MASK);
 80}
 81
 82static inline bool futex_flags_valid(unsigned int flags)
 83{
 84	/* Only 64bit futexes for 64bit code */
 85	if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) {
 86		if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
 87			return false;
 88	}
 89
 90	/* Only 32bit futexes are implemented -- for now */
 91	if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
 92		return false;
 93
 94	/*
 95	 * Must be able to represent both FUTEX_NO_NODE and every valid nodeid
 96	 * in a futex word.
 97	 */
 98	if (flags & FLAGS_NUMA) {
 99		int bits = 8 * futex_size(flags);
100		u64 max = ~0ULL;
101
102		max >>= 64 - bits;
103		if (nr_node_ids >= max)
104			return false;
105	}
106
107	return true;
108}
109
110static inline bool futex_validate_input(unsigned int flags, u64 val)
111{
112	int bits = 8 * futex_size(flags);
113
114	if (bits < 64 && (val >> bits))
115		return false;
116
117	return true;
118}
119
120#ifdef CONFIG_FAIL_FUTEX
121extern bool should_fail_futex(bool fshared);
122#else
123static inline bool should_fail_futex(bool fshared)
124{
125	return false;
126}
127#endif
128
129/*
130 * Hash buckets are shared by all the futex_keys that hash to the same
131 * location.  Each key may have multiple futex_q structures, one for each task
132 * waiting on a futex.
133 */
134struct futex_hash_bucket {
135	atomic_t waiters;
136	spinlock_t lock;
137	struct plist_head chain;
138	struct futex_private_hash *priv;
139} ____cacheline_aligned_in_smp;
140
141/*
142 * Priority Inheritance state:
143 */
144struct futex_pi_state {
145	/*
146	 * list of 'owned' pi_state instances - these have to be
147	 * cleaned up in do_exit() if the task exits prematurely:
148	 */
149	struct list_head list;
150
151	/*
152	 * The PI object:
153	 */
154	struct rt_mutex_base pi_mutex;
155
156	struct task_struct *owner;
157	refcount_t refcount;
158
159	union futex_key key;
160} __randomize_layout;
161
162struct futex_q;
163typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q);
164
165/**
166 * struct futex_q - The hashed futex queue entry, one per waiting task
167 * @list:		priority-sorted list of tasks waiting on this futex
168 * @task:		the task waiting on the futex
169 * @lock_ptr:		the hash bucket lock
170 * @wake:		the wake handler for this queue
171 * @wake_data:		data associated with the wake handler
172 * @key:		the key the futex is hashed on
173 * @pi_state:		optional priority inheritance state
174 * @rt_waiter:		rt_waiter storage for use with requeue_pi
175 * @requeue_pi_key:	the requeue_pi target futex key
176 * @bitset:		bitset for the optional bitmasked wakeup
177 * @requeue_state:	State field for futex_requeue_pi()
178 * @drop_hb_ref:	Waiter should drop the extra hash bucket reference if true
179 * @requeue_wait:	RCU wait for futex_requeue_pi() (RT only)
180 *
181 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
182 * we can wake only the relevant ones (hashed queues may be shared).
183 *
184 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
185 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
186 * The order of wakeup is always to make the first condition true, then
187 * the second.
188 *
189 * PI futexes are typically woken before they are removed from the hash list via
190 * the rt_mutex code. See futex_unqueue_pi().
191 */
192struct futex_q {
193	struct plist_node list;
194
195	struct task_struct *task;
196	spinlock_t *lock_ptr;
197	futex_wake_fn *wake;
198	void *wake_data;
199	union futex_key key;
200	struct futex_pi_state *pi_state;
201	struct rt_mutex_waiter *rt_waiter;
202	union futex_key *requeue_pi_key;
203	u32 bitset;
204	atomic_t requeue_state;
205	bool drop_hb_ref;
206#ifdef CONFIG_PREEMPT_RT
207	struct rcuwait requeue_wait;
208#endif
209} __randomize_layout;
210
211extern const struct futex_q futex_q_init;
212
213enum futex_access {
214	FUTEX_READ,
215	FUTEX_WRITE
216};
217
218extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
219			 enum futex_access rw);
220extern void futex_q_lockptr_lock(struct futex_q *q);
221extern struct hrtimer_sleeper *
222futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
223		  int flags, u64 range_ns);
224
225extern struct futex_hash_bucket *futex_hash(union futex_key *key);
226#ifdef CONFIG_FUTEX_PRIVATE_HASH
227extern void futex_hash_get(struct futex_hash_bucket *hb);
228extern void futex_hash_put(struct futex_hash_bucket *hb);
229
230extern struct futex_private_hash *futex_private_hash(void);
231extern void futex_private_hash_put(struct futex_private_hash *fph);
232
233#else /* !CONFIG_FUTEX_PRIVATE_HASH */
234static inline void futex_hash_get(struct futex_hash_bucket *hb) { }
235static inline void futex_hash_put(struct futex_hash_bucket *hb) { }
236static inline struct futex_private_hash *futex_private_hash(void) { return NULL; }
237static inline void futex_private_hash_put(struct futex_private_hash *fph) { }
238#endif
239
240DEFINE_CLASS(hb, struct futex_hash_bucket *,
241	     if (_T) futex_hash_put(_T),
242	     futex_hash(key), union futex_key *key);
243
244DEFINE_CLASS(private_hash, struct futex_private_hash *,
245	     if (_T) futex_private_hash_put(_T),
246	     futex_private_hash(), void);
247
248/**
249 * futex_match - Check whether two futex keys are equal
250 * @key1:	Pointer to key1
251 * @key2:	Pointer to key2
252 *
253 * Return 1 if two futex_keys are equal, 0 otherwise.
254 */
255static inline int futex_match(union futex_key *key1, union futex_key *key2)
256{
257	return (key1 && key2
258		&& key1->both.word == key2->both.word
259		&& key1->both.ptr == key2->both.ptr
260		&& key1->both.offset == key2->both.offset);
261}
262
263extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
264			    struct futex_q *q, union futex_key *key2,
265			    struct task_struct *task);
266extern void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout);
267extern bool __futex_wake_mark(struct futex_q *q);
268extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
269
270extern int fault_in_user_writeable(u32 __user *uaddr);
271extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
272
273static inline int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval)
274{
275	int ret;
276
277	pagefault_disable();
278	ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);
279	pagefault_enable();
280
281	return ret;
282}
283
284/* Read from user memory with pagefaults disabled */
285static inline int futex_get_value_locked(u32 *dest, u32 __user *from)
286{
287	guard(pagefault)();
288	return get_user_inline(*dest, from);
289}
290
291extern void __futex_unqueue(struct futex_q *q);
292extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
293				struct task_struct *task);
294extern int futex_unqueue(struct futex_q *q);
295
296/**
297 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
298 * @q:	The futex_q to enqueue
299 * @hb:	The destination hash bucket
300 * @task: Task queueing this futex
301 *
302 * The hb->lock must be held by the caller, and is released here. A call to
303 * futex_queue() is typically paired with exactly one call to futex_unqueue().  The
304 * exceptions involve the PI related operations, which may use futex_unqueue_pi()
305 * or nothing if the unqueue is done as part of the wake process and the unqueue
306 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
307 * an example).
308 *
309 * Note that @task may be NULL, for async usage of futexes.
310 */
311static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
312			       struct task_struct *task)
313	__releases(&hb->lock)
314{
315	__futex_queue(q, hb, task);
316	spin_unlock(&hb->lock);
317}
318
319extern void futex_unqueue_pi(struct futex_q *q);
320
321extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
322
323/*
324 * Reflects a new waiter being added to the waitqueue.
325 */
326static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
327{
328#ifdef CONFIG_SMP
329	atomic_inc(&hb->waiters);
330	/*
331	 * Full barrier (A), see the ordering comment above.
332	 */
333	smp_mb__after_atomic();
334#endif
335}
336
337/*
338 * Reflects a waiter being removed from the waitqueue by wakeup
339 * paths.
340 */
341static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
342{
343#ifdef CONFIG_SMP
344	atomic_dec(&hb->waiters);
345#endif
346}
347
348static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
349{
350#ifdef CONFIG_SMP
351	/*
352	 * Full barrier (B), see the ordering comment above.
353	 */
354	smp_mb();
355	return atomic_read(&hb->waiters);
356#else
357	return 1;
358#endif
359}
360
361extern void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb);
362extern void futex_q_unlock(struct futex_hash_bucket *hb);
363
364
365extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
366				union futex_key *key,
367				struct futex_pi_state **ps,
368				struct task_struct *task,
369				struct task_struct **exiting,
370				int set_waiters);
371
372extern int refill_pi_state_cache(void);
373extern void get_pi_state(struct futex_pi_state *pi_state);
374extern void put_pi_state(struct futex_pi_state *pi_state);
375extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
376
377/*
378 * Express the locking dependencies for lockdep:
379 */
380static inline void
381double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
382{
383	if (hb1 > hb2)
384		swap(hb1, hb2);
385
386	spin_lock(&hb1->lock);
387	if (hb1 != hb2)
388		spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
389}
390
391static inline void
392double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
393{
394	spin_unlock(&hb1->lock);
395	if (hb1 != hb2)
396		spin_unlock(&hb2->lock);
397}
398
399/* syscalls */
400
401extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
402				 val, ktime_t *abs_time, u32 bitset, u32 __user
403				 *uaddr2);
404
405extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
406			 u32 __user *uaddr2, unsigned int flags2,
407			 int nr_wake, int nr_requeue,
408			 u32 *cmpval, int requeue_pi);
409
410extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
411			struct hrtimer_sleeper *to, u32 bitset);
412
413extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
414		      ktime_t *abs_time, u32 bitset);
415
416/**
417 * struct futex_vector - Auxiliary struct for futex_waitv()
418 * @w: Userspace provided data
419 * @q: Kernel side data
420 *
421 * Struct used to build an array with all data need for futex_waitv()
422 */
423struct futex_vector {
424	struct futex_waitv w;
425	struct futex_q q;
426};
427
428extern int futex_parse_waitv(struct futex_vector *futexv,
429			     struct futex_waitv __user *uwaitv,
430			     unsigned int nr_futexes, futex_wake_fn *wake,
431			     void *wake_data);
432
433extern int futex_wait_multiple_setup(struct futex_vector *vs, int count,
434				     int *woken);
435
436extern int futex_unqueue_multiple(struct futex_vector *v, int count);
437
438extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
439			       struct hrtimer_sleeper *to);
440
441extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
442
443extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
444			 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
445
446extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
447
448extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
449
450#endif /* _FUTEX_H */