register.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Code related to the io_uring_register() syscall
  4 *
  5 * Copyright (C) 2023 Jens Axboe
  6 */
  7#include <linux/kernel.h>
  8#include <linux/errno.h>
  9#include <linux/syscalls.h>
 10#include <linux/refcount.h>
 11#include <linux/bits.h>
 12#include <linux/fs.h>
 13#include <linux/file.h>
 14#include <linux/slab.h>
 15#include <linux/uaccess.h>
 16#include <linux/nospec.h>
 17#include <linux/compat.h>
 18#include <linux/io_uring.h>
 19#include <linux/io_uring_types.h>
 20
 21#include "filetable.h"
 22#include "io_uring.h"
 23#include "opdef.h"
 24#include "tctx.h"
 25#include "rsrc.h"
 26#include "sqpoll.h"
 27#include "register.h"
 28#include "cancel.h"
 29#include "kbuf.h"
 30#include "napi.h"
 31#include "eventfd.h"
 32#include "msg_ring.h"
 33#include "memmap.h"
 34#include "zcrx.h"
 35#include "query.h"
 36
 37#define IORING_MAX_RESTRICTIONS	(IORING_RESTRICTION_LAST + \
 38				 IORING_REGISTER_LAST + IORING_OP_LAST)
 39
 40static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,
 41			   unsigned nr_args)
 42{
 43	struct io_uring_probe *p;
 44	size_t size;
 45	int i, ret;
 46
 47	if (nr_args > IORING_OP_LAST)
 48		nr_args = IORING_OP_LAST;
 49
 50	size = struct_size(p, ops, nr_args);
 51	p = memdup_user(arg, size);
 52	if (IS_ERR(p))
 53		return PTR_ERR(p);
 54	ret = -EINVAL;
 55	if (memchr_inv(p, 0, size))
 56		goto out;
 57
 58	p->last_op = IORING_OP_LAST - 1;
 59
 60	for (i = 0; i < nr_args; i++) {
 61		p->ops[i].op = i;
 62		if (io_uring_op_supported(i))
 63			p->ops[i].flags = IO_URING_OP_SUPPORTED;
 64	}
 65	p->ops_len = i;
 66
 67	ret = 0;
 68	if (copy_to_user(arg, p, size))
 69		ret = -EFAULT;
 70out:
 71	kfree(p);
 72	return ret;
 73}
 74
 75int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
 76{
 77	const struct cred *creds;
 78
 79	creds = xa_erase(&ctx->personalities, id);
 80	if (creds) {
 81		put_cred(creds);
 82		return 0;
 83	}
 84
 85	return -EINVAL;
 86}
 87
 88
 89static int io_register_personality(struct io_ring_ctx *ctx)
 90{
 91	const struct cred *creds;
 92	u32 id;
 93	int ret;
 94
 95	creds = get_current_cred();
 96
 97	ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
 98			XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
 99	if (ret < 0) {
100		put_cred(creds);
101		return ret;
102	}
103	return id;
104}
105
106static __cold int io_parse_restrictions(void __user *arg, unsigned int nr_args,
107					struct io_restriction *restrictions)
108{
109	struct io_uring_restriction *res;
110	size_t size;
111	int i, ret;
112
113	if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
114		return -EINVAL;
115
116	size = array_size(nr_args, sizeof(*res));
117	if (size == SIZE_MAX)
118		return -EOVERFLOW;
119
120	res = memdup_user(arg, size);
121	if (IS_ERR(res))
122		return PTR_ERR(res);
123
124	ret = -EINVAL;
125
126	for (i = 0; i < nr_args; i++) {
127		switch (res[i].opcode) {
128		case IORING_RESTRICTION_REGISTER_OP:
129			if (res[i].register_op >= IORING_REGISTER_LAST)
130				goto err;
131			__set_bit(res[i].register_op, restrictions->register_op);
132			break;
133		case IORING_RESTRICTION_SQE_OP:
134			if (res[i].sqe_op >= IORING_OP_LAST)
135				goto err;
136			__set_bit(res[i].sqe_op, restrictions->sqe_op);
137			break;
138		case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
139			restrictions->sqe_flags_allowed = res[i].sqe_flags;
140			break;
141		case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
142			restrictions->sqe_flags_required = res[i].sqe_flags;
143			break;
144		default:
145			goto err;
146		}
147	}
148
149	ret = 0;
150
151err:
152	kfree(res);
153	return ret;
154}
155
156static __cold int io_register_restrictions(struct io_ring_ctx *ctx,
157					   void __user *arg, unsigned int nr_args)
158{
159	int ret;
160
161	/* Restrictions allowed only if rings started disabled */
162	if (!(ctx->flags & IORING_SETUP_R_DISABLED))
163		return -EBADFD;
164
165	/* We allow only a single restrictions registration */
166	if (ctx->restrictions.registered)
167		return -EBUSY;
168
169	ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions);
170	/* Reset all restrictions if an error happened */
171	if (ret != 0)
172		memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
173	else
174		ctx->restrictions.registered = true;
175	return ret;
176}
177
178static int io_register_enable_rings(struct io_ring_ctx *ctx)
179{
180	if (!(ctx->flags & IORING_SETUP_R_DISABLED))
181		return -EBADFD;
182
183	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) {
184		WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
185		/*
186		 * Lazy activation attempts would fail if it was polled before
187		 * submitter_task is set.
188		 */
189		if (wq_has_sleeper(&ctx->poll_wq))
190			io_activate_pollwq(ctx);
191	}
192
193	if (ctx->restrictions.registered)
194		ctx->restricted = 1;
195
196	/* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */
197	smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED);
198	if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
199		wake_up(&ctx->sq_data->wait);
200	return 0;
201}
202
203static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,
204					 cpumask_var_t new_mask)
205{
206	int ret;
207
208	if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
209		ret = io_wq_cpu_affinity(current->io_uring, new_mask);
210	} else {
211		mutex_unlock(&ctx->uring_lock);
212		ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);
213		mutex_lock(&ctx->uring_lock);
214	}
215
216	return ret;
217}
218
219static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
220				       void __user *arg, unsigned len)
221{
222	cpumask_var_t new_mask;
223	int ret;
224
225	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
226		return -ENOMEM;
227
228	cpumask_clear(new_mask);
229	if (len > cpumask_size())
230		len = cpumask_size();
231
232#ifdef CONFIG_COMPAT
233	if (in_compat_syscall())
234		ret = compat_get_bitmap(cpumask_bits(new_mask),
235					(const compat_ulong_t __user *)arg,
236					len * 8 /* CHAR_BIT */);
237	else
238#endif
239		ret = copy_from_user(new_mask, arg, len);
240
241	if (ret) {
242		free_cpumask_var(new_mask);
243		return -EFAULT;
244	}
245
246	ret = __io_register_iowq_aff(ctx, new_mask);
247	free_cpumask_var(new_mask);
248	return ret;
249}
250
251static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
252{
253	return __io_register_iowq_aff(ctx, NULL);
254}
255
256static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
257					       void __user *arg)
258	__must_hold(&ctx->uring_lock)
259{
260	struct io_tctx_node *node;
261	struct io_uring_task *tctx = NULL;
262	struct io_sq_data *sqd = NULL;
263	__u32 new_count[2];
264	int i, ret;
265
266	if (copy_from_user(new_count, arg, sizeof(new_count)))
267		return -EFAULT;
268	for (i = 0; i < ARRAY_SIZE(new_count); i++)
269		if (new_count[i] > INT_MAX)
270			return -EINVAL;
271
272	if (ctx->flags & IORING_SETUP_SQPOLL) {
273		sqd = ctx->sq_data;
274		if (sqd) {
275			struct task_struct *tsk;
276
277			/*
278			 * Observe the correct sqd->lock -> ctx->uring_lock
279			 * ordering. Fine to drop uring_lock here, we hold
280			 * a ref to the ctx.
281			 */
282			refcount_inc(&sqd->refs);
283			mutex_unlock(&ctx->uring_lock);
284			mutex_lock(&sqd->lock);
285			mutex_lock(&ctx->uring_lock);
286			tsk = sqpoll_task_locked(sqd);
287			if (tsk)
288				tctx = tsk->io_uring;
289		}
290	} else {
291		tctx = current->io_uring;
292	}
293
294	BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
295
296	for (i = 0; i < ARRAY_SIZE(new_count); i++)
297		if (new_count[i])
298			ctx->iowq_limits[i] = new_count[i];
299	ctx->iowq_limits_set = true;
300
301	if (tctx && tctx->io_wq) {
302		ret = io_wq_max_workers(tctx->io_wq, new_count);
303		if (ret)
304			goto err;
305	} else {
306		memset(new_count, 0, sizeof(new_count));
307	}
308
309	if (sqd) {
310		mutex_unlock(&ctx->uring_lock);
311		mutex_unlock(&sqd->lock);
312		io_put_sq_data(sqd);
313		mutex_lock(&ctx->uring_lock);
314	}
315
316	if (copy_to_user(arg, new_count, sizeof(new_count)))
317		return -EFAULT;
318
319	/* that's it for SQPOLL, only the SQPOLL task creates requests */
320	if (sqd)
321		return 0;
322
323	/* now propagate the restriction to all registered users */
324	mutex_lock(&ctx->tctx_lock);
325	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
326		tctx = node->task->io_uring;
327		if (WARN_ON_ONCE(!tctx->io_wq))
328			continue;
329
330		for (i = 0; i < ARRAY_SIZE(new_count); i++)
331			new_count[i] = ctx->iowq_limits[i];
332		/* ignore errors, it always returns zero anyway */
333		(void)io_wq_max_workers(tctx->io_wq, new_count);
334	}
335	mutex_unlock(&ctx->tctx_lock);
336	return 0;
337err:
338	if (sqd) {
339		mutex_unlock(&ctx->uring_lock);
340		mutex_unlock(&sqd->lock);
341		io_put_sq_data(sqd);
342		mutex_lock(&ctx->uring_lock);
343	}
344	return ret;
345}
346
347static int io_register_clock(struct io_ring_ctx *ctx,
348			     struct io_uring_clock_register __user *arg)
349{
350	struct io_uring_clock_register reg;
351
352	if (copy_from_user(&reg, arg, sizeof(reg)))
353		return -EFAULT;
354	if (memchr_inv(&reg.__resv, 0, sizeof(reg.__resv)))
355		return -EINVAL;
356
357	switch (reg.clockid) {
358	case CLOCK_MONOTONIC:
359		ctx->clock_offset = 0;
360		break;
361	case CLOCK_BOOTTIME:
362		ctx->clock_offset = TK_OFFS_BOOT;
363		break;
364	default:
365		return -EINVAL;
366	}
367
368	ctx->clockid = reg.clockid;
369	return 0;
370}
371
372/*
373 * State to maintain until we can swap. Both new and old state, used for
374 * either mapping or freeing.
375 */
376struct io_ring_ctx_rings {
377	struct io_rings *rings;
378	struct io_uring_sqe *sq_sqes;
379
380	struct io_mapped_region sq_region;
381	struct io_mapped_region ring_region;
382};
383
384static void io_register_free_rings(struct io_ring_ctx *ctx,
385				   struct io_ring_ctx_rings *r)
386{
387	io_free_region(ctx->user, &r->sq_region);
388	io_free_region(ctx->user, &r->ring_region);
389}
390
391#define swap_old(ctx, o, n, field)		\
392	do {					\
393		(o).field = (ctx)->field;	\
394		(ctx)->field = (n).field;	\
395	} while (0)
396
397#define RESIZE_FLAGS	(IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP)
398#define COPY_FLAGS	(IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \
399			 IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \
400			 IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED)
401
402static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
403{
404	struct io_ctx_config config;
405	struct io_uring_region_desc rd;
406	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
407	unsigned i, tail, old_head;
408	struct io_uring_params *p = &config.p;
409	struct io_rings_layout *rl = &config.layout;
410	int ret;
411
412	memset(&config, 0, sizeof(config));
413
414	/* limited to DEFER_TASKRUN for now */
415	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
416		return -EINVAL;
417	if (copy_from_user(p, arg, sizeof(*p)))
418		return -EFAULT;
419	if (p->flags & ~RESIZE_FLAGS)
420		return -EINVAL;
421
422	/* properties that are always inherited */
423	p->flags |= (ctx->flags & COPY_FLAGS);
424
425	ret = io_prepare_config(&config);
426	if (unlikely(ret))
427		return ret;
428
429	memset(&rd, 0, sizeof(rd));
430	rd.size = PAGE_ALIGN(rl->rings_size);
431	if (p->flags & IORING_SETUP_NO_MMAP) {
432		rd.user_addr = p->cq_off.user_addr;
433		rd.flags |= IORING_MEM_REGION_TYPE_USER;
434	}
435	ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
436	if (ret)
437		return ret;
438
439	n.rings = io_region_get_ptr(&n.ring_region);
440
441	/*
442	 * At this point n.rings is shared with userspace, just like o.rings
443	 * is as well. While we don't expect userspace to modify it while
444	 * a resize is in progress, and it's most likely that userspace will
445	 * shoot itself in the foot if it does, we can't always assume good
446	 * intent... Use read/write once helpers from here on to indicate the
447	 * shared nature of it.
448	 */
449	WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1);
450	WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1);
451	WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries);
452	WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries);
453
454	if (copy_to_user(arg, p, sizeof(*p))) {
455		io_register_free_rings(ctx, &n);
456		return -EFAULT;
457	}
458
459	memset(&rd, 0, sizeof(rd));
460	rd.size = PAGE_ALIGN(rl->sq_size);
461	if (p->flags & IORING_SETUP_NO_MMAP) {
462		rd.user_addr = p->sq_off.user_addr;
463		rd.flags |= IORING_MEM_REGION_TYPE_USER;
464	}
465	ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
466	if (ret) {
467		io_register_free_rings(ctx, &n);
468		return ret;
469	}
470	n.sq_sqes = io_region_get_ptr(&n.sq_region);
471
472	/*
473	 * If using SQPOLL, park the thread
474	 */
475	if (ctx->sq_data) {
476		mutex_unlock(&ctx->uring_lock);
477		io_sq_thread_park(ctx->sq_data);
478		mutex_lock(&ctx->uring_lock);
479	}
480
481	/*
482	 * We'll do the swap. Grab the ctx->mmap_lock, which will exclude
483	 * any new mmap's on the ring fd. Clear out existing mappings to prevent
484	 * mmap from seeing them, as we'll unmap them. Any attempt to mmap
485	 * existing rings beyond this point will fail. Not that it could proceed
486	 * at this point anyway, as the io_uring mmap side needs go grab the
487	 * ctx->mmap_lock as well. Likewise, hold the completion lock over the
488	 * duration of the actual swap.
489	 */
490	mutex_lock(&ctx->mmap_lock);
491	spin_lock(&ctx->completion_lock);
492	o.rings = ctx->rings;
493	ctx->rings = NULL;
494	o.sq_sqes = ctx->sq_sqes;
495	ctx->sq_sqes = NULL;
496
497	/*
498	 * Now copy SQ and CQ entries, if any. If either of the destination
499	 * rings can't hold what is already there, then fail the operation.
500	 */
501	tail = READ_ONCE(o.rings->sq.tail);
502	old_head = READ_ONCE(o.rings->sq.head);
503	if (tail - old_head > p->sq_entries)
504		goto overflow;
505	for (i = old_head; i < tail; i++) {
506		unsigned src_head = i & (ctx->sq_entries - 1);
507		unsigned dst_head = i & (p->sq_entries - 1);
508
509		n.sq_sqes[dst_head] = o.sq_sqes[src_head];
510	}
511	WRITE_ONCE(n.rings->sq.head, old_head);
512	WRITE_ONCE(n.rings->sq.tail, tail);
513
514	tail = READ_ONCE(o.rings->cq.tail);
515	old_head = READ_ONCE(o.rings->cq.head);
516	if (tail - old_head > p->cq_entries) {
517overflow:
518		/* restore old rings, and return -EOVERFLOW via cleanup path */
519		ctx->rings = o.rings;
520		ctx->sq_sqes = o.sq_sqes;
521		to_free = &n;
522		ret = -EOVERFLOW;
523		goto out;
524	}
525	for (i = old_head; i < tail; i++) {
526		unsigned src_head = i & (ctx->cq_entries - 1);
527		unsigned dst_head = i & (p->cq_entries - 1);
528
529		n.rings->cqes[dst_head] = o.rings->cqes[src_head];
530	}
531	WRITE_ONCE(n.rings->cq.head, old_head);
532	WRITE_ONCE(n.rings->cq.tail, tail);
533	/* invalidate cached cqe refill */
534	ctx->cqe_cached = ctx->cqe_sentinel = NULL;
535
536	WRITE_ONCE(n.rings->sq_dropped, READ_ONCE(o.rings->sq_dropped));
537	atomic_set(&n.rings->sq_flags, atomic_read(&o.rings->sq_flags));
538	WRITE_ONCE(n.rings->cq_flags, READ_ONCE(o.rings->cq_flags));
539	WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow));
540
541	/* all done, store old pointers and assign new ones */
542	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
543		ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
544
545	ctx->sq_entries = p->sq_entries;
546	ctx->cq_entries = p->cq_entries;
547
548	/*
549	 * Just mark any flag we may have missed and that the application
550	 * should act on unconditionally. Worst case it'll be an extra
551	 * syscall.
552	 */
553	atomic_or(IORING_SQ_TASKRUN | IORING_SQ_NEED_WAKEUP, &n.rings->sq_flags);
554	ctx->rings = n.rings;
555	rcu_assign_pointer(ctx->rings_rcu, n.rings);
556
557	ctx->sq_sqes = n.sq_sqes;
558	swap_old(ctx, o, n, ring_region);
559	swap_old(ctx, o, n, sq_region);
560	to_free = &o;
561	ret = 0;
562out:
563	spin_unlock(&ctx->completion_lock);
564	mutex_unlock(&ctx->mmap_lock);
565
566	/* Wait for concurrent io_ctx_mark_taskrun() */
567	if (to_free == &o)
568		synchronize_rcu_expedited();
569	io_register_free_rings(ctx, to_free);
570
571	if (ctx->sq_data)
572		io_sq_thread_unpark(ctx->sq_data);
573
574	return ret;
575}
576
577static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
578{
579	struct io_uring_mem_region_reg __user *reg_uptr = uarg;
580	struct io_uring_mem_region_reg reg;
581	struct io_uring_region_desc __user *rd_uptr;
582	struct io_uring_region_desc rd;
583	struct io_mapped_region region = {};
584	int ret;
585
586	if (io_region_is_set(&ctx->param_region))
587		return -EBUSY;
588	if (copy_from_user(&reg, reg_uptr, sizeof(reg)))
589		return -EFAULT;
590	rd_uptr = u64_to_user_ptr(reg.region_uptr);
591	if (copy_from_user(&rd, rd_uptr, sizeof(rd)))
592		return -EFAULT;
593	if (memchr_inv(&reg.__resv, 0, sizeof(reg.__resv)))
594		return -EINVAL;
595	if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG)
596		return -EINVAL;
597
598	/*
599	 * This ensures there are no waiters. Waiters are unlocked and it's
600	 * hard to synchronise with them, especially if we need to initialise
601	 * the region.
602	 */
603	if ((reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) &&
604	    !(ctx->flags & IORING_SETUP_R_DISABLED))
605		return -EINVAL;
606
607	ret = io_create_region(ctx, &region, &rd, IORING_MAP_OFF_PARAM_REGION);
608	if (ret)
609		return ret;
610	if (copy_to_user(rd_uptr, &rd, sizeof(rd))) {
611		io_free_region(ctx->user, &region);
612		return -EFAULT;
613	}
614
615	if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) {
616		ctx->cq_wait_arg = io_region_get_ptr(&region);
617		ctx->cq_wait_size = rd.size;
618	}
619
620	io_region_publish(ctx, &region, &ctx->param_region);
621	return 0;
622}
623
624static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
625			       void __user *arg, unsigned nr_args)
626	__releases(ctx->uring_lock)
627	__acquires(ctx->uring_lock)
628{
629	int ret;
630
631	/*
632	 * We don't quiesce the refs for register anymore and so it can't be
633	 * dying as we're holding a file ref here.
634	 */
635	if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))
636		return -ENXIO;
637
638	if (ctx->submitter_task && ctx->submitter_task != current)
639		return -EEXIST;
640
641	if (ctx->restricted) {
642		opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
643		if (!test_bit(opcode, ctx->restrictions.register_op))
644			return -EACCES;
645	}
646
647	switch (opcode) {
648	case IORING_REGISTER_BUFFERS:
649		ret = -EFAULT;
650		if (!arg)
651			break;
652		ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
653		break;
654	case IORING_UNREGISTER_BUFFERS:
655		ret = -EINVAL;
656		if (arg || nr_args)
657			break;
658		ret = io_sqe_buffers_unregister(ctx);
659		break;
660	case IORING_REGISTER_FILES:
661		ret = -EFAULT;
662		if (!arg)
663			break;
664		ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
665		break;
666	case IORING_UNREGISTER_FILES:
667		ret = -EINVAL;
668		if (arg || nr_args)
669			break;
670		ret = io_sqe_files_unregister(ctx);
671		break;
672	case IORING_REGISTER_FILES_UPDATE:
673		ret = io_register_files_update(ctx, arg, nr_args);
674		break;
675	case IORING_REGISTER_EVENTFD:
676		ret = -EINVAL;
677		if (nr_args != 1)
678			break;
679		ret = io_eventfd_register(ctx, arg, 0);
680		break;
681	case IORING_REGISTER_EVENTFD_ASYNC:
682		ret = -EINVAL;
683		if (nr_args != 1)
684			break;
685		ret = io_eventfd_register(ctx, arg, 1);
686		break;
687	case IORING_UNREGISTER_EVENTFD:
688		ret = -EINVAL;
689		if (arg || nr_args)
690			break;
691		ret = io_eventfd_unregister(ctx);
692		break;
693	case IORING_REGISTER_PROBE:
694		ret = -EINVAL;
695		if (!arg || nr_args > 256)
696			break;
697		ret = io_probe(ctx, arg, nr_args);
698		break;
699	case IORING_REGISTER_PERSONALITY:
700		ret = -EINVAL;
701		if (arg || nr_args)
702			break;
703		ret = io_register_personality(ctx);
704		break;
705	case IORING_UNREGISTER_PERSONALITY:
706		ret = -EINVAL;
707		if (arg)
708			break;
709		ret = io_unregister_personality(ctx, nr_args);
710		break;
711	case IORING_REGISTER_ENABLE_RINGS:
712		ret = -EINVAL;
713		if (arg || nr_args)
714			break;
715		ret = io_register_enable_rings(ctx);
716		break;
717	case IORING_REGISTER_RESTRICTIONS:
718		ret = io_register_restrictions(ctx, arg, nr_args);
719		break;
720	case IORING_REGISTER_FILES2:
721		ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
722		break;
723	case IORING_REGISTER_FILES_UPDATE2:
724		ret = io_register_rsrc_update(ctx, arg, nr_args,
725					      IORING_RSRC_FILE);
726		break;
727	case IORING_REGISTER_BUFFERS2:
728		ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
729		break;
730	case IORING_REGISTER_BUFFERS_UPDATE:
731		ret = io_register_rsrc_update(ctx, arg, nr_args,
732					      IORING_RSRC_BUFFER);
733		break;
734	case IORING_REGISTER_IOWQ_AFF:
735		ret = -EINVAL;
736		if (!arg || !nr_args)
737			break;
738		ret = io_register_iowq_aff(ctx, arg, nr_args);
739		break;
740	case IORING_UNREGISTER_IOWQ_AFF:
741		ret = -EINVAL;
742		if (arg || nr_args)
743			break;
744		ret = io_unregister_iowq_aff(ctx);
745		break;
746	case IORING_REGISTER_IOWQ_MAX_WORKERS:
747		ret = -EINVAL;
748		if (!arg || nr_args != 2)
749			break;
750		ret = io_register_iowq_max_workers(ctx, arg);
751		break;
752	case IORING_REGISTER_RING_FDS:
753		ret = io_ringfd_register(ctx, arg, nr_args);
754		break;
755	case IORING_UNREGISTER_RING_FDS:
756		ret = io_ringfd_unregister(ctx, arg, nr_args);
757		break;
758	case IORING_REGISTER_PBUF_RING:
759		ret = -EINVAL;
760		if (!arg || nr_args != 1)
761			break;
762		ret = io_register_pbuf_ring(ctx, arg);
763		break;
764	case IORING_UNREGISTER_PBUF_RING:
765		ret = -EINVAL;
766		if (!arg || nr_args != 1)
767			break;
768		ret = io_unregister_pbuf_ring(ctx, arg);
769		break;
770	case IORING_REGISTER_SYNC_CANCEL:
771		ret = -EINVAL;
772		if (!arg || nr_args != 1)
773			break;
774		ret = io_sync_cancel(ctx, arg);
775		break;
776	case IORING_REGISTER_FILE_ALLOC_RANGE:
777		ret = -EINVAL;
778		if (!arg || nr_args)
779			break;
780		ret = io_register_file_alloc_range(ctx, arg);
781		break;
782	case IORING_REGISTER_PBUF_STATUS:
783		ret = -EINVAL;
784		if (!arg || nr_args != 1)
785			break;
786		ret = io_register_pbuf_status(ctx, arg);
787		break;
788	case IORING_REGISTER_NAPI:
789		ret = -EINVAL;
790		if (!arg || nr_args != 1)
791			break;
792		ret = io_register_napi(ctx, arg);
793		break;
794	case IORING_UNREGISTER_NAPI:
795		ret = -EINVAL;
796		if (nr_args != 1)
797			break;
798		ret = io_unregister_napi(ctx, arg);
799		break;
800	case IORING_REGISTER_CLOCK:
801		ret = -EINVAL;
802		if (!arg || nr_args)
803			break;
804		ret = io_register_clock(ctx, arg);
805		break;
806	case IORING_REGISTER_CLONE_BUFFERS:
807		ret = -EINVAL;
808		if (!arg || nr_args != 1)
809			break;
810		ret = io_register_clone_buffers(ctx, arg);
811		break;
812	case IORING_REGISTER_ZCRX_IFQ:
813		ret = -EINVAL;
814		if (!arg || nr_args != 1)
815			break;
816		ret = io_register_zcrx_ifq(ctx, arg);
817		break;
818	case IORING_REGISTER_RESIZE_RINGS:
819		ret = -EINVAL;
820		if (!arg || nr_args != 1)
821			break;
822		ret = io_register_resize_rings(ctx, arg);
823		break;
824	case IORING_REGISTER_MEM_REGION:
825		ret = -EINVAL;
826		if (!arg || nr_args != 1)
827			break;
828		ret = io_register_mem_region(ctx, arg);
829		break;
830	case IORING_REGISTER_QUERY:
831		ret = io_query(arg, nr_args);
832		break;
833	case IORING_REGISTER_ZCRX_CTRL:
834		ret = io_zcrx_ctrl(ctx, arg, nr_args);
835		break;
836	default:
837		ret = -EINVAL;
838		break;
839	}
840
841	return ret;
842}
843
844/*
845 * Given an 'fd' value, return the ctx associated with if. If 'registered' is
846 * true, then the registered index is used. Otherwise, the normal fd table.
847 * Caller must call fput() on the returned file, unless it's an ERR_PTR.
848 */
849struct file *io_uring_register_get_file(unsigned int fd, bool registered)
850{
851	struct file *file;
852
853	if (registered) {
854		/*
855		 * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
856		 * need only dereference our task private array to find it.
857		 */
858		struct io_uring_task *tctx = current->io_uring;
859
860		if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
861			return ERR_PTR(-EINVAL);
862		fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
863		file = tctx->registered_rings[fd];
864		if (file)
865			get_file(file);
866	} else {
867		file = fget(fd);
868	}
869
870	if (unlikely(!file))
871		return ERR_PTR(-EBADF);
872	if (io_is_uring_fops(file))
873		return file;
874	fput(file);
875	return ERR_PTR(-EOPNOTSUPP);
876}
877
878static int io_uring_register_send_msg_ring(void __user *arg, unsigned int nr_args)
879{
880	struct io_uring_sqe sqe;
881
882	if (!arg || nr_args != 1)
883		return -EINVAL;
884	if (copy_from_user(&sqe, arg, sizeof(sqe)))
885		return -EFAULT;
886	/* no flags supported */
887	if (sqe.flags)
888		return -EINVAL;
889	if (sqe.opcode != IORING_OP_MSG_RING)
890		return -EINVAL;
891
892	return io_uring_sync_msg_ring(&sqe);
893}
894
895/*
896 * "blind" registration opcodes are ones where there's no ring given, and
897 * hence the source fd must be -1.
898 */
899static int io_uring_register_blind(unsigned int opcode, void __user *arg,
900				   unsigned int nr_args)
901{
902	switch (opcode) {
903	case IORING_REGISTER_SEND_MSG_RING:
904		return io_uring_register_send_msg_ring(arg, nr_args);
905	case IORING_REGISTER_QUERY:
906		return io_query(arg, nr_args);
907	}
908	return -EINVAL;
909}
910
911SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
912		void __user *, arg, unsigned int, nr_args)
913{
914	struct io_ring_ctx *ctx;
915	long ret = -EBADF;
916	struct file *file;
917	bool use_registered_ring;
918
919	use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING);
920	opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;
921
922	if (opcode >= IORING_REGISTER_LAST)
923		return -EINVAL;
924
925	if (fd == -1)
926		return io_uring_register_blind(opcode, arg, nr_args);
927
928	file = io_uring_register_get_file(fd, use_registered_ring);
929	if (IS_ERR(file))
930		return PTR_ERR(file);
931	ctx = file->private_data;
932
933	mutex_lock(&ctx->uring_lock);
934	ret = __io_uring_register(ctx, opcode, arg, nr_args);
935
936	trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr,
937				ctx->buf_table.nr, ret);
938	mutex_unlock(&ctx->uring_lock);
939
940	fput(file);
941	return ret;
942}