poll.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/errno.h>
  4#include <linux/fs.h>
  5#include <linux/file.h>
  6#include <linux/mm.h>
  7#include <linux/slab.h>
  8#include <linux/poll.h>
  9#include <linux/hashtable.h>
 10#include <linux/io_uring.h>
 11
 12#include <trace/events/io_uring.h>
 13
 14#include <uapi/linux/io_uring.h>
 15
 16#include "io_uring.h"
 17#include "alloc_cache.h"
 18#include "refs.h"
 19#include "napi.h"
 20#include "opdef.h"
 21#include "kbuf.h"
 22#include "poll.h"
 23#include "cancel.h"
 24
 25struct io_poll_update {
 26	struct file			*file;
 27	u64				old_user_data;
 28	u64				new_user_data;
 29	__poll_t			events;
 30	bool				update_events;
 31	bool				update_user_data;
 32};
 33
 34struct io_poll_table {
 35	struct poll_table_struct pt;
 36	struct io_kiocb *req;
 37	int nr_entries;
 38	int error;
 39	bool owning;
 40	/* output value, set only if arm poll returns >0 */
 41	__poll_t result_mask;
 42};
 43
 44#define IO_POLL_CANCEL_FLAG	BIT(31)
 45#define IO_POLL_RETRY_FLAG	BIT(30)
 46#define IO_POLL_REF_MASK	GENMASK(29, 0)
 47
 48/*
 49 * We usually have 1-2 refs taken, 128 is more than enough and we want to
 50 * maximise the margin between this amount and the moment when it overflows.
 51 */
 52#define IO_POLL_REF_BIAS	128
 53
 54#define IO_WQE_F_DOUBLE		1
 55
 56static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
 57			void *key);
 58
 59static inline struct io_kiocb *wqe_to_req(struct wait_queue_entry *wqe)
 60{
 61	unsigned long priv = (unsigned long)wqe->private;
 62
 63	return (struct io_kiocb *)(priv & ~IO_WQE_F_DOUBLE);
 64}
 65
 66static inline bool wqe_is_double(struct wait_queue_entry *wqe)
 67{
 68	unsigned long priv = (unsigned long)wqe->private;
 69
 70	return priv & IO_WQE_F_DOUBLE;
 71}
 72
 73static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)
 74{
 75	int v;
 76
 77	/*
 78	 * poll_refs are already elevated and we don't have much hope for
 79	 * grabbing the ownership. Instead of incrementing set a retry flag
 80	 * to notify the loop that there might have been some change.
 81	 */
 82	v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs);
 83	if (v & IO_POLL_REF_MASK)
 84		return false;
 85	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
 86}
 87
 88/*
 89 * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can
 90 * bump it and acquire ownership. It's disallowed to modify requests while not
 91 * owning it, that prevents from races for enqueueing task_work's and b/w
 92 * arming poll and wakeups.
 93 */
 94static inline bool io_poll_get_ownership(struct io_kiocb *req)
 95{
 96	if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))
 97		return io_poll_get_ownership_slowpath(req);
 98	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
 99}
100
101static void io_poll_mark_cancelled(struct io_kiocb *req)
102{
103	atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);
104}
105
106static struct io_poll *io_poll_get_double(struct io_kiocb *req)
107{
108	/* pure poll stashes this in ->async_data, poll driven retry elsewhere */
109	if (req->opcode == IORING_OP_POLL_ADD)
110		return req->async_data;
111	return req->apoll->double_poll;
112}
113
114static struct io_poll *io_poll_get_single(struct io_kiocb *req)
115{
116	if (req->opcode == IORING_OP_POLL_ADD)
117		return io_kiocb_to_cmd(req, struct io_poll);
118	return &req->apoll->poll;
119}
120
121static void io_poll_req_insert(struct io_kiocb *req)
122{
123	struct io_hash_table *table = &req->ctx->cancel_table;
124	u32 index = hash_long(req->cqe.user_data, table->hash_bits);
125
126	lockdep_assert_held(&req->ctx->uring_lock);
127
128	hlist_add_head(&req->hash_node, &table->hbs[index].list);
129}
130
131static void io_init_poll_iocb(struct io_poll *poll, __poll_t events)
132{
133	poll->head = NULL;
134#define IO_POLL_UNMASK	(EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
135	/* mask in events that we always want/need */
136	poll->events = events | IO_POLL_UNMASK;
137	INIT_LIST_HEAD(&poll->wait.entry);
138	init_waitqueue_func_entry(&poll->wait, io_poll_wake);
139}
140
141static void io_poll_remove_waitq(struct io_poll *poll)
142{
143	/*
144	 * If the waitqueue is being freed early but someone is already holds
145	 * ownership over it, we have to tear down the request as best we can.
146	 * That means immediately removing the request from its waitqueue and
147	 * preventing all further accesses to the waitqueue via the request.
148	 */
149	list_del_init(&poll->wait.entry);
150
151	/*
152	 * Careful: this *must* be the last step, since as soon as req->head is
153	 * NULL'ed out, the request can be completed and freed, since
154	 * io_poll_remove_entry() will no longer need to take the waitqueue
155	 * lock.
156	 */
157	smp_store_release(&poll->head, NULL);
158}
159
160static inline void io_poll_remove_entry(struct io_poll *poll)
161{
162	struct wait_queue_head *head = smp_load_acquire(&poll->head);
163
164	if (head) {
165		spin_lock_irq(&head->lock);
166		io_poll_remove_waitq(poll);
167		spin_unlock_irq(&head->lock);
168	}
169}
170
171static void io_poll_remove_entries(struct io_kiocb *req)
172{
173	/*
174	 * Nothing to do if neither of those flags are set. Avoid dipping
175	 * into the poll/apoll/double cachelines if we can.
176	 */
177	if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL)))
178		return;
179
180	/*
181	 * While we hold the waitqueue lock and the waitqueue is nonempty,
182	 * wake_up_pollfree() will wait for us.  However, taking the waitqueue
183	 * lock in the first place can race with the waitqueue being freed.
184	 *
185	 * We solve this as eventpoll does: by taking advantage of the fact that
186	 * all users of wake_up_pollfree() will RCU-delay the actual free.  If
187	 * we enter rcu_read_lock() and see that the pointer to the queue is
188	 * non-NULL, we can then lock it without the memory being freed out from
189	 * under us.
190	 *
191	 * Keep holding rcu_read_lock() as long as we hold the queue lock, in
192	 * case the caller deletes the entry from the queue, leaving it empty.
193	 * In that case, only RCU prevents the queue memory from being freed.
194	 */
195	rcu_read_lock();
196	if (req->flags & REQ_F_SINGLE_POLL)
197		io_poll_remove_entry(io_poll_get_single(req));
198	if (req->flags & REQ_F_DOUBLE_POLL)
199		io_poll_remove_entry(io_poll_get_double(req));
200	rcu_read_unlock();
201}
202
203enum {
204	IOU_POLL_DONE = 0,
205	IOU_POLL_NO_ACTION = 1,
206	IOU_POLL_REMOVE_POLL_USE_RES = 2,
207	IOU_POLL_REISSUE = 3,
208	IOU_POLL_REQUEUE = 4,
209};
210
211static void __io_poll_execute(struct io_kiocb *req, int mask)
212{
213	unsigned flags = 0;
214
215	io_req_set_res(req, mask, 0);
216	req->io_task_work.func = io_poll_task_func;
217
218	trace_io_uring_task_add(req, mask);
219
220	if (!(req->flags & REQ_F_POLL_NO_LAZY))
221		flags = IOU_F_TWQ_LAZY_WAKE;
222	__io_req_task_work_add(req, flags);
223}
224
225static inline void io_poll_execute(struct io_kiocb *req, int res)
226{
227	if (io_poll_get_ownership(req))
228		__io_poll_execute(req, res);
229}
230
231/*
232 * All poll tw should go through this. Checks for poll events, manages
233 * references, does rewait, etc.
234 *
235 * Returns a negative error on failure. IOU_POLL_NO_ACTION when no action
236 * require, which is either spurious wakeup or multishot CQE is served.
237 * IOU_POLL_DONE when it's done with the request, then the mask is stored in
238 * req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
239 * poll and that the result is stored in req->cqe.
240 */
241static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
242{
243	int v;
244
245	if (unlikely(tw.cancel))
246		return -ECANCELED;
247
248	do {
249		v = atomic_read(&req->poll_refs);
250
251		if (unlikely(v != 1)) {
252			/* tw should be the owner and so have some refs */
253			if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK)))
254				return IOU_POLL_NO_ACTION;
255			if (v & IO_POLL_CANCEL_FLAG)
256				return -ECANCELED;
257			/*
258			 * cqe.res contains only events of the first wake up
259			 * and all others are to be lost. Redo vfs_poll() to get
260			 * up to date state.
261			 */
262			if ((v & IO_POLL_REF_MASK) != 1)
263				req->cqe.res = 0;
264
265			if (v & IO_POLL_RETRY_FLAG) {
266				req->cqe.res = 0;
267				/*
268				 * We won't find new events that came in between
269				 * vfs_poll and the ref put unless we clear the
270				 * flag in advance.
271				 */
272				atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);
273				v &= ~IO_POLL_RETRY_FLAG;
274			}
275			v &= IO_POLL_REF_MASK;
276		}
277
278		/* the mask was stashed in __io_poll_execute */
279		if (!req->cqe.res) {
280			struct poll_table_struct pt = { ._key = req->apoll_events };
281			req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;
282			/*
283			 * We got woken with a mask, but someone else got to
284			 * it first. The above vfs_poll() doesn't add us back
285			 * to the waitqueue, so if we get nothing back, we
286			 * should be safe and attempt a reissue.
287			 */
288			if (unlikely(!req->cqe.res)) {
289				/* Multishot armed need not reissue */
290				if (!(req->apoll_events & EPOLLONESHOT))
291					continue;
292				return IOU_POLL_REISSUE;
293			}
294		}
295		if (req->apoll_events & EPOLLONESHOT)
296			return IOU_POLL_DONE;
297
298		/* multishot, just fill a CQE and proceed */
299		if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
300			__poll_t mask = mangle_poll(req->cqe.res &
301						    req->apoll_events);
302
303			if (!io_req_post_cqe(req, mask, IORING_CQE_F_MORE)) {
304				io_req_set_res(req, mask, 0);
305				return IOU_POLL_REMOVE_POLL_USE_RES;
306			}
307		} else {
308			int ret;
309
310			/* multiple refs and HUP, ensure we loop once more */
311			if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1)
312				v--;
313
314			ret = io_poll_issue(req, tw);
315			if (ret == IOU_COMPLETE)
316				return IOU_POLL_REMOVE_POLL_USE_RES;
317			else if (ret == IOU_REQUEUE)
318				return IOU_POLL_REQUEUE;
319			if (ret != IOU_RETRY && ret < 0)
320				return ret;
321		}
322
323		/* force the next iteration to vfs_poll() */
324		req->cqe.res = 0;
325
326		/*
327		 * Release all references, retry if someone tried to restart
328		 * task_work while we were executing it.
329		 */
330	} while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK);
331
332	io_napi_add(req);
333	return IOU_POLL_NO_ACTION;
334}
335
336void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw)
337{
338	struct io_kiocb *req = tw_req.req;
339	int ret;
340
341	ret = io_poll_check_events(req, tw);
342	if (ret == IOU_POLL_NO_ACTION) {
343		return;
344	} else if (ret == IOU_POLL_REQUEUE) {
345		__io_poll_execute(req, 0);
346		return;
347	}
348	io_poll_remove_entries(req);
349	/* task_work always has ->uring_lock held */
350	hash_del(&req->hash_node);
351
352	if (req->opcode == IORING_OP_POLL_ADD) {
353		if (ret == IOU_POLL_DONE) {
354			struct io_poll *poll;
355
356			poll = io_kiocb_to_cmd(req, struct io_poll);
357			req->cqe.res = mangle_poll(req->cqe.res & poll->events);
358		} else if (ret == IOU_POLL_REISSUE) {
359			io_req_task_submit(tw_req, tw);
360			return;
361		} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
362			req->cqe.res = ret;
363			req_set_fail(req);
364		}
365
366		io_req_set_res(req, req->cqe.res, 0);
367		io_req_task_complete(tw_req, tw);
368	} else {
369		io_tw_lock(req->ctx, tw);
370
371		if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
372			io_req_task_complete(tw_req, tw);
373		else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
374			io_req_task_submit(tw_req, tw);
375		else
376			io_req_defer_failed(req, ret);
377	}
378}
379
380static void io_poll_cancel_req(struct io_kiocb *req)
381{
382	io_poll_mark_cancelled(req);
383	/* kick tw, which should complete the request */
384	io_poll_execute(req, 0);
385}
386
387#define IO_ASYNC_POLL_COMMON	(EPOLLONESHOT | EPOLLPRI)
388
389static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)
390{
391	io_poll_mark_cancelled(req);
392	/* we have to kick tw in case it's not already */
393	io_poll_execute(req, 0);
394	io_poll_remove_waitq(poll);
395	return 1;
396}
397
398static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
399			void *key)
400{
401	struct io_kiocb *req = wqe_to_req(wait);
402	struct io_poll *poll = container_of(wait, struct io_poll, wait);
403	__poll_t mask = key_to_poll(key);
404
405	if (unlikely(mask & POLLFREE))
406		return io_pollfree_wake(req, poll);
407
408	/* for instances that support it check for an event match first */
409	if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON)))
410		return 0;
411
412	if (io_poll_get_ownership(req)) {
413		/*
414		 * If we trigger a multishot poll off our own wakeup path,
415		 * disable multishot as there is a circular dependency between
416		 * CQ posting and triggering the event.
417		 */
418		if (mask & EPOLL_URING_WAKE)
419			poll->events |= EPOLLONESHOT;
420
421		/* optional, saves extra locking for removal in tw handler */
422		if (mask && poll->events & EPOLLONESHOT) {
423			io_poll_remove_waitq(poll);
424			if (wqe_is_double(wait))
425				req->flags &= ~REQ_F_DOUBLE_POLL;
426			else
427				req->flags &= ~REQ_F_SINGLE_POLL;
428		}
429		__io_poll_execute(req, mask);
430	}
431	return 1;
432}
433
434/* fails only when polling is already completing by the first entry */
435static bool io_poll_double_prepare(struct io_kiocb *req)
436{
437	struct wait_queue_head *head;
438	struct io_poll *poll = io_poll_get_single(req);
439
440	/* head is RCU protected, see io_poll_remove_entries() comments */
441	rcu_read_lock();
442	head = smp_load_acquire(&poll->head);
443	/*
444	 * poll arm might not hold ownership and so race for req->flags with
445	 * io_poll_wake(). There is only one poll entry queued, serialise with
446	 * it by taking its head lock. As we're still arming the tw hanlder
447	 * is not going to be run, so there are no races with it.
448	 */
449	if (head) {
450		spin_lock_irq(&head->lock);
451		req->flags |= REQ_F_DOUBLE_POLL;
452		if (req->opcode == IORING_OP_POLL_ADD)
453			req->flags |= REQ_F_ASYNC_DATA;
454		spin_unlock_irq(&head->lock);
455	}
456	rcu_read_unlock();
457	return !!head;
458}
459
460static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,
461			    struct wait_queue_head *head,
462			    struct io_poll **poll_ptr)
463{
464	struct io_kiocb *req = pt->req;
465	unsigned long wqe_private = (unsigned long) req;
466
467	/*
468	 * The file being polled uses multiple waitqueues for poll handling
469	 * (e.g. one for read, one for write). Setup a separate io_poll
470	 * if this happens.
471	 */
472	if (unlikely(pt->nr_entries)) {
473		struct io_poll *first = poll;
474
475		/* double add on the same waitqueue head, ignore */
476		if (first->head == head)
477			return;
478		/* already have a 2nd entry, fail a third attempt */
479		if (*poll_ptr) {
480			if ((*poll_ptr)->head == head)
481				return;
482			pt->error = -EINVAL;
483			return;
484		}
485
486		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
487		if (!poll) {
488			pt->error = -ENOMEM;
489			return;
490		}
491
492		/* mark as double wq entry */
493		wqe_private |= IO_WQE_F_DOUBLE;
494		io_init_poll_iocb(poll, first->events);
495		if (!io_poll_double_prepare(req)) {
496			/* the request is completing, just back off */
497			kfree(poll);
498			return;
499		}
500		*poll_ptr = poll;
501	} else {
502		/* fine to modify, there is no poll queued to race with us */
503		req->flags |= REQ_F_SINGLE_POLL;
504	}
505
506	pt->nr_entries++;
507	poll->head = head;
508	poll->wait.private = (void *) wqe_private;
509
510	if (poll->events & EPOLLEXCLUSIVE) {
511		add_wait_queue_exclusive(head, &poll->wait);
512	} else {
513		add_wait_queue(head, &poll->wait);
514	}
515}
516
517static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
518			       struct poll_table_struct *p)
519{
520	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
521	struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);
522
523	__io_queue_proc(poll, pt, head,
524			(struct io_poll **) &pt->req->async_data);
525}
526
527static bool io_poll_can_finish_inline(struct io_kiocb *req,
528				      struct io_poll_table *pt)
529{
530	return pt->owning || io_poll_get_ownership(req);
531}
532
533static void io_poll_add_hash(struct io_kiocb *req, unsigned int issue_flags)
534{
535	struct io_ring_ctx *ctx = req->ctx;
536
537	io_ring_submit_lock(ctx, issue_flags);
538	io_poll_req_insert(req);
539	io_ring_submit_unlock(ctx, issue_flags);
540}
541
542/*
543 * Returns 0 when it's handed over for polling. The caller owns the requests if
544 * it returns non-zero, but otherwise should not touch it. Negative values
545 * contain an error code. When the result is >0, the polling has completed
546 * inline and ipt.result_mask is set to the mask.
547 */
548static int __io_arm_poll_handler(struct io_kiocb *req,
549				 struct io_poll *poll,
550				 struct io_poll_table *ipt, __poll_t mask,
551				 unsigned issue_flags)
552{
553	INIT_HLIST_NODE(&req->hash_node);
554	io_init_poll_iocb(poll, mask);
555	poll->file = req->file;
556	req->apoll_events = poll->events;
557
558	ipt->pt._key = mask;
559	ipt->req = req;
560	ipt->error = 0;
561	ipt->nr_entries = 0;
562	/*
563	 * Polling is either completed here or via task_work, so if we're in the
564	 * task context we're naturally serialised with tw by merit of running
565	 * the same task. When it's io-wq, take the ownership to prevent tw
566	 * from running. However, when we're in the task context, skip taking
567	 * it as an optimisation.
568	 *
569	 * Note: even though the request won't be completed/freed, without
570	 * ownership we still can race with io_poll_wake().
571	 * io_poll_can_finish_inline() tries to deal with that.
572	 */
573	ipt->owning = issue_flags & IO_URING_F_UNLOCKED;
574	atomic_set(&req->poll_refs, (int)ipt->owning);
575
576	/*
577	 * Exclusive waits may only wake a limited amount of entries
578	 * rather than all of them, this may interfere with lazy
579	 * wake if someone does wait(events > 1). Ensure we don't do
580	 * lazy wake for those, as we need to process each one as they
581	 * come in.
582	 */
583	if (poll->events & EPOLLEXCLUSIVE)
584		req->flags |= REQ_F_POLL_NO_LAZY;
585
586	mask = vfs_poll(req->file, &ipt->pt) & poll->events;
587
588	if (unlikely(ipt->error || !ipt->nr_entries)) {
589		io_poll_remove_entries(req);
590
591		if (!io_poll_can_finish_inline(req, ipt)) {
592			io_poll_mark_cancelled(req);
593			return 0;
594		} else if (mask && (poll->events & EPOLLET)) {
595			ipt->result_mask = mask;
596			return 1;
597		}
598		return ipt->error ?: -EINVAL;
599	}
600
601	if (mask &&
602	   ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) {
603		if (!io_poll_can_finish_inline(req, ipt)) {
604			io_poll_add_hash(req, issue_flags);
605			return 0;
606		}
607		io_poll_remove_entries(req);
608		ipt->result_mask = mask;
609		/* no one else has access to the req, forget about the ref */
610		return 1;
611	}
612
613	io_poll_add_hash(req, issue_flags);
614
615	if (mask && (poll->events & EPOLLET) &&
616	    io_poll_can_finish_inline(req, ipt)) {
617		__io_poll_execute(req, mask);
618		return 0;
619	}
620	io_napi_add(req);
621
622	if (ipt->owning) {
623		/*
624		 * Try to release ownership. If we see a change of state, e.g.
625		 * poll was waken up, queue up a tw, it'll deal with it.
626		 */
627		if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
628			__io_poll_execute(req, 0);
629	}
630	return 0;
631}
632
633static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
634			       struct poll_table_struct *p)
635{
636	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
637	struct async_poll *apoll = pt->req->apoll;
638
639	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
640}
641
642/*
643 * We can't reliably detect loops in repeated poll triggers and issue
644 * subsequently failing. But rather than fail these immediately, allow a
645 * certain amount of retries before we give up. Given that this condition
646 * should _rarely_ trigger even once, we should be fine with a larger value.
647 */
648#define APOLL_MAX_RETRY		128
649
650static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req,
651					     unsigned issue_flags)
652{
653	struct io_ring_ctx *ctx = req->ctx;
654	struct async_poll *apoll;
655
656	if (req->flags & REQ_F_POLLED) {
657		apoll = req->apoll;
658		kfree(apoll->double_poll);
659	} else {
660		if (!(issue_flags & IO_URING_F_UNLOCKED))
661			apoll = io_cache_alloc(&ctx->apoll_cache, GFP_ATOMIC);
662		else
663			apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
664		if (!apoll)
665			return NULL;
666		apoll->poll.retries = APOLL_MAX_RETRY;
667	}
668	apoll->double_poll = NULL;
669	req->apoll = apoll;
670	if (unlikely(!--apoll->poll.retries))
671		return NULL;
672	return apoll;
673}
674
675int io_arm_apoll(struct io_kiocb *req, unsigned issue_flags, __poll_t mask)
676{
677	struct async_poll *apoll;
678	struct io_poll_table ipt;
679	int ret;
680
681	mask |= EPOLLET;
682	if (!io_file_can_poll(req))
683		return IO_APOLL_ABORTED;
684	if (!(req->flags & REQ_F_APOLL_MULTISHOT))
685		mask |= EPOLLONESHOT;
686
687	apoll = io_req_alloc_apoll(req, issue_flags);
688	if (!apoll)
689		return IO_APOLL_ABORTED;
690	req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL);
691	req->flags |= REQ_F_POLLED;
692	ipt.pt._qproc = io_async_queue_proc;
693
694	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);
695	if (ret)
696		return ret > 0 ? IO_APOLL_READY : IO_APOLL_ABORTED;
697	trace_io_uring_poll_arm(req, mask, apoll->poll.events);
698	return IO_APOLL_OK;
699}
700
701int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
702{
703	const struct io_issue_def *def = &io_issue_defs[req->opcode];
704	__poll_t mask = POLLPRI | POLLERR;
705
706	if (!def->pollin && !def->pollout)
707		return IO_APOLL_ABORTED;
708	if (!io_file_can_poll(req))
709		return IO_APOLL_ABORTED;
710
711	if (def->pollin) {
712		mask |= EPOLLIN | EPOLLRDNORM;
713
714		/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
715		if (req->flags & REQ_F_CLEAR_POLLIN)
716			mask &= ~EPOLLIN;
717	} else {
718		mask |= EPOLLOUT | EPOLLWRNORM;
719	}
720	if (def->poll_exclusive)
721		mask |= EPOLLEXCLUSIVE;
722
723	return io_arm_apoll(req, issue_flags, mask);
724}
725
726/*
727 * Returns true if we found and killed one or more poll requests
728 */
729__cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
730			       bool cancel_all)
731{
732	unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits;
733	struct hlist_node *tmp;
734	struct io_kiocb *req;
735	bool found = false;
736	int i;
737
738	lockdep_assert_held(&ctx->uring_lock);
739
740	for (i = 0; i < nr_buckets; i++) {
741		struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i];
742
743		hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {
744			if (io_match_task_safe(req, tctx, cancel_all)) {
745				hlist_del_init(&req->hash_node);
746				io_poll_cancel_req(req);
747				found = true;
748			}
749		}
750	}
751	return found;
752}
753
754static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,
755				     struct io_cancel_data *cd)
756{
757	struct io_kiocb *req;
758	u32 index = hash_long(cd->data, ctx->cancel_table.hash_bits);
759	struct io_hash_bucket *hb = &ctx->cancel_table.hbs[index];
760
761	hlist_for_each_entry(req, &hb->list, hash_node) {
762		if (cd->data != req->cqe.user_data)
763			continue;
764		if (poll_only && req->opcode != IORING_OP_POLL_ADD)
765			continue;
766		if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
767			if (io_cancel_match_sequence(req, cd->seq))
768				continue;
769		}
770		return req;
771	}
772	return NULL;
773}
774
775static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,
776					  struct io_cancel_data *cd)
777{
778	unsigned nr_buckets = 1U << ctx->cancel_table.hash_bits;
779	struct io_kiocb *req;
780	int i;
781
782	for (i = 0; i < nr_buckets; i++) {
783		struct io_hash_bucket *hb = &ctx->cancel_table.hbs[i];
784
785		hlist_for_each_entry(req, &hb->list, hash_node) {
786			if (io_cancel_req_match(req, cd))
787				return req;
788		}
789	}
790	return NULL;
791}
792
793static int io_poll_disarm(struct io_kiocb *req)
794{
795	if (!req)
796		return -ENOENT;
797	if (!io_poll_get_ownership(req))
798		return -EALREADY;
799	io_poll_remove_entries(req);
800	hash_del(&req->hash_node);
801	return 0;
802}
803
804static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
805{
806	struct io_kiocb *req;
807
808	if (cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP |
809			 IORING_ASYNC_CANCEL_ANY))
810		req = io_poll_file_find(ctx, cd);
811	else
812		req = io_poll_find(ctx, false, cd);
813
814	if (req) {
815		io_poll_cancel_req(req);
816		return 0;
817	}
818	return -ENOENT;
819}
820
821int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
822		   unsigned issue_flags)
823{
824	int ret;
825
826	io_ring_submit_lock(ctx, issue_flags);
827	ret = __io_poll_cancel(ctx, cd);
828	io_ring_submit_unlock(ctx, issue_flags);
829	return ret;
830}
831
832static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
833				     unsigned int flags)
834{
835	u32 events;
836
837	events = READ_ONCE(sqe->poll32_events);
838#ifdef __BIG_ENDIAN
839	events = swahw32(events);
840#endif
841	if (!(flags & IORING_POLL_ADD_MULTI))
842		events |= EPOLLONESHOT;
843	if (!(flags & IORING_POLL_ADD_LEVEL))
844		events |= EPOLLET;
845	return demangle_poll(events) |
846		(events & (EPOLLEXCLUSIVE|EPOLLONESHOT|EPOLLET));
847}
848
849int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
850{
851	struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update);
852	u32 flags;
853
854	if (sqe->buf_index || sqe->splice_fd_in)
855		return -EINVAL;
856	flags = READ_ONCE(sqe->len);
857	if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
858		      IORING_POLL_ADD_MULTI))
859		return -EINVAL;
860	/* meaningless without update */
861	if (flags == IORING_POLL_ADD_MULTI)
862		return -EINVAL;
863
864	upd->old_user_data = READ_ONCE(sqe->addr);
865	upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
866	upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
867
868	upd->new_user_data = READ_ONCE(sqe->off);
869	if (!upd->update_user_data && upd->new_user_data)
870		return -EINVAL;
871	if (upd->update_events)
872		upd->events = io_poll_parse_events(sqe, flags);
873	else if (sqe->poll32_events)
874		return -EINVAL;
875
876	return 0;
877}
878
879int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
880{
881	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
882	u32 flags;
883
884	if (sqe->buf_index || sqe->off || sqe->addr)
885		return -EINVAL;
886	flags = READ_ONCE(sqe->len);
887	if (flags & ~IORING_POLL_ADD_MULTI)
888		return -EINVAL;
889	if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))
890		return -EINVAL;
891
892	poll->events = io_poll_parse_events(sqe, flags);
893	return 0;
894}
895
896int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
897{
898	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
899	struct io_poll_table ipt;
900	int ret;
901
902	ipt.pt._qproc = io_poll_queue_proc;
903
904	ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);
905	if (ret > 0) {
906		io_req_set_res(req, ipt.result_mask, 0);
907		return IOU_COMPLETE;
908	}
909	return ret ?: IOU_ISSUE_SKIP_COMPLETE;
910}
911
912int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
913{
914	struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update);
915	struct io_ring_ctx *ctx = req->ctx;
916	struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, };
917	struct io_kiocb *preq;
918	int ret2, ret = 0;
919
920	io_ring_submit_lock(ctx, issue_flags);
921	preq = io_poll_find(ctx, true, &cd);
922	ret2 = io_poll_disarm(preq);
923	if (ret2) {
924		ret = ret2;
925		goto out;
926	}
927	if (WARN_ON_ONCE(preq->opcode != IORING_OP_POLL_ADD)) {
928		ret = -EFAULT;
929		goto out;
930	}
931
932	if (poll_update->update_events || poll_update->update_user_data) {
933		/* only mask one event flags, keep behavior flags */
934		if (poll_update->update_events) {
935			struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll);
936
937			poll->events &= ~0xffff;
938			poll->events |= poll_update->events & 0xffff;
939			poll->events |= IO_POLL_UNMASK;
940		}
941		if (poll_update->update_user_data)
942			preq->cqe.user_data = poll_update->new_user_data;
943
944		ret2 = io_poll_add(preq, issue_flags & ~IO_URING_F_UNLOCKED);
945		/* successfully updated, don't complete poll request */
946		if (ret2 == IOU_ISSUE_SKIP_COMPLETE)
947			goto out;
948		/* request completed as part of the update, complete it */
949		else if (ret2 == IOU_COMPLETE)
950			goto complete;
951	}
952
953	io_req_set_res(preq, -ECANCELED, 0);
954complete:
955	if (preq->cqe.res < 0)
956		req_set_fail(preq);
957	preq->io_task_work.func = io_req_task_complete;
958	io_req_task_work_add(preq);
959out:
960	io_ring_submit_unlock(ctx, issue_flags);
961	if (ret < 0) {
962		req_set_fail(req);
963		return ret;
964	}
965	/* complete update request, we're done with it */
966	io_req_set_res(req, ret, 0);
967	return IOU_COMPLETE;
968}