opdef.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * io_uring opcode handling table
  4 */
  5#include <linux/kernel.h>
  6#include <linux/errno.h>
  7#include <linux/fs.h>
  8#include <linux/file.h>
  9#include <linux/io_uring.h>
 10#include <linux/io_uring/cmd.h>
 11
 12#include "io_uring.h"
 13#include "opdef.h"
 14#include "refs.h"
 15#include "tctx.h"
 16#include "sqpoll.h"
 17#include "fdinfo.h"
 18#include "kbuf.h"
 19#include "rsrc.h"
 20
 21#include "xattr.h"
 22#include "nop.h"
 23#include "fs.h"
 24#include "splice.h"
 25#include "sync.h"
 26#include "advise.h"
 27#include "openclose.h"
 28#include "uring_cmd.h"
 29#include "epoll.h"
 30#include "statx.h"
 31#include "net.h"
 32#include "msg_ring.h"
 33#include "timeout.h"
 34#include "poll.h"
 35#include "cancel.h"
 36#include "rw.h"
 37#include "waitid.h"
 38#include "futex.h"
 39#include "truncate.h"
 40#include "zcrx.h"
 41
 42static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
 43{
 44	WARN_ON_ONCE(1);
 45	return -ECANCELED;
 46}
 47
 48static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
 49					     const struct io_uring_sqe *sqe)
 50{
 51	return -EOPNOTSUPP;
 52}
 53
 54const struct io_issue_def io_issue_defs[] = {
 55	[IORING_OP_NOP] = {
 56		.audit_skip		= 1,
 57		.iopoll			= 1,
 58		.prep			= io_nop_prep,
 59		.issue			= io_nop,
 60	},
 61	[IORING_OP_READV] = {
 62		.needs_file		= 1,
 63		.unbound_nonreg_file	= 1,
 64		.pollin			= 1,
 65		.buffer_select		= 1,
 66		.plug			= 1,
 67		.audit_skip		= 1,
 68		.ioprio			= 1,
 69		.iopoll			= 1,
 70		.iopoll_queue		= 1,
 71		.vectored		= 1,
 72		.async_size		= sizeof(struct io_async_rw),
 73		.prep			= io_prep_readv,
 74		.issue			= io_read,
 75	},
 76	[IORING_OP_WRITEV] = {
 77		.needs_file		= 1,
 78		.hash_reg_file		= 1,
 79		.unbound_nonreg_file	= 1,
 80		.pollout		= 1,
 81		.plug			= 1,
 82		.audit_skip		= 1,
 83		.ioprio			= 1,
 84		.iopoll			= 1,
 85		.iopoll_queue		= 1,
 86		.vectored		= 1,
 87		.async_size		= sizeof(struct io_async_rw),
 88		.prep			= io_prep_writev,
 89		.issue			= io_write,
 90	},
 91	[IORING_OP_FSYNC] = {
 92		.needs_file		= 1,
 93		.audit_skip		= 1,
 94		.prep			= io_fsync_prep,
 95		.issue			= io_fsync,
 96	},
 97	[IORING_OP_READ_FIXED] = {
 98		.needs_file		= 1,
 99		.unbound_nonreg_file	= 1,
100		.pollin			= 1,
101		.plug			= 1,
102		.audit_skip		= 1,
103		.ioprio			= 1,
104		.iopoll			= 1,
105		.iopoll_queue		= 1,
106		.async_size		= sizeof(struct io_async_rw),
107		.prep			= io_prep_read_fixed,
108		.issue			= io_read_fixed,
109	},
110	[IORING_OP_WRITE_FIXED] = {
111		.needs_file		= 1,
112		.hash_reg_file		= 1,
113		.unbound_nonreg_file	= 1,
114		.pollout		= 1,
115		.plug			= 1,
116		.audit_skip		= 1,
117		.ioprio			= 1,
118		.iopoll			= 1,
119		.iopoll_queue		= 1,
120		.async_size		= sizeof(struct io_async_rw),
121		.prep			= io_prep_write_fixed,
122		.issue			= io_write_fixed,
123	},
124	[IORING_OP_POLL_ADD] = {
125		.needs_file		= 1,
126		.unbound_nonreg_file	= 1,
127		.audit_skip		= 1,
128		.prep			= io_poll_add_prep,
129		.issue			= io_poll_add,
130	},
131	[IORING_OP_POLL_REMOVE] = {
132		.audit_skip		= 1,
133		.prep			= io_poll_remove_prep,
134		.issue			= io_poll_remove,
135	},
136	[IORING_OP_SYNC_FILE_RANGE] = {
137		.needs_file		= 1,
138		.audit_skip		= 1,
139		.prep			= io_sfr_prep,
140		.issue			= io_sync_file_range,
141	},
142	[IORING_OP_SENDMSG] = {
143		.needs_file		= 1,
144		.unbound_nonreg_file	= 1,
145		.pollout		= 1,
146		.ioprio			= 1,
147#if defined(CONFIG_NET)
148		.async_size		= sizeof(struct io_async_msghdr),
149		.prep			= io_sendmsg_prep,
150		.issue			= io_sendmsg,
151#else
152		.prep			= io_eopnotsupp_prep,
153#endif
154	},
155	[IORING_OP_RECVMSG] = {
156		.needs_file		= 1,
157		.unbound_nonreg_file	= 1,
158		.pollin			= 1,
159		.buffer_select		= 1,
160		.ioprio			= 1,
161#if defined(CONFIG_NET)
162		.async_size		= sizeof(struct io_async_msghdr),
163		.prep			= io_recvmsg_prep,
164		.issue			= io_recvmsg,
165#else
166		.prep			= io_eopnotsupp_prep,
167#endif
168	},
169	[IORING_OP_TIMEOUT] = {
170		.audit_skip		= 1,
171		.async_size		= sizeof(struct io_timeout_data),
172		.prep			= io_timeout_prep,
173		.issue			= io_timeout,
174	},
175	[IORING_OP_TIMEOUT_REMOVE] = {
176		/* used by timeout updates' prep() */
177		.audit_skip		= 1,
178		.prep			= io_timeout_remove_prep,
179		.issue			= io_timeout_remove,
180	},
181	[IORING_OP_ACCEPT] = {
182		.needs_file		= 1,
183		.unbound_nonreg_file	= 1,
184		.pollin			= 1,
185		.poll_exclusive		= 1,
186		.ioprio			= 1,	/* used for flags */
187#if defined(CONFIG_NET)
188		.prep			= io_accept_prep,
189		.issue			= io_accept,
190#else
191		.prep			= io_eopnotsupp_prep,
192#endif
193	},
194	[IORING_OP_ASYNC_CANCEL] = {
195		.audit_skip		= 1,
196		.prep			= io_async_cancel_prep,
197		.issue			= io_async_cancel,
198	},
199	[IORING_OP_LINK_TIMEOUT] = {
200		.audit_skip		= 1,
201		.async_size		= sizeof(struct io_timeout_data),
202		.prep			= io_link_timeout_prep,
203		.issue			= io_no_issue,
204	},
205	[IORING_OP_CONNECT] = {
206		.needs_file		= 1,
207		.unbound_nonreg_file	= 1,
208		.pollout		= 1,
209#if defined(CONFIG_NET)
210		.async_size		= sizeof(struct io_async_msghdr),
211		.prep			= io_connect_prep,
212		.issue			= io_connect,
213#else
214		.prep			= io_eopnotsupp_prep,
215#endif
216	},
217	[IORING_OP_FALLOCATE] = {
218		.needs_file		= 1,
219		.hash_reg_file          = 1,
220		.prep			= io_fallocate_prep,
221		.issue			= io_fallocate,
222	},
223	[IORING_OP_OPENAT] = {
224		.prep			= io_openat_prep,
225		.issue			= io_openat,
226	},
227	[IORING_OP_CLOSE] = {
228		.prep			= io_close_prep,
229		.issue			= io_close,
230	},
231	[IORING_OP_FILES_UPDATE] = {
232		.audit_skip		= 1,
233		.iopoll			= 1,
234		.prep			= io_files_update_prep,
235		.issue			= io_files_update,
236	},
237	[IORING_OP_STATX] = {
238		.audit_skip		= 1,
239		.prep			= io_statx_prep,
240		.issue			= io_statx,
241	},
242	[IORING_OP_READ] = {
243		.needs_file		= 1,
244		.unbound_nonreg_file	= 1,
245		.pollin			= 1,
246		.buffer_select		= 1,
247		.plug			= 1,
248		.audit_skip		= 1,
249		.ioprio			= 1,
250		.iopoll			= 1,
251		.iopoll_queue		= 1,
252		.async_size		= sizeof(struct io_async_rw),
253		.prep			= io_prep_read,
254		.issue			= io_read,
255	},
256	[IORING_OP_WRITE] = {
257		.needs_file		= 1,
258		.hash_reg_file		= 1,
259		.unbound_nonreg_file	= 1,
260		.pollout		= 1,
261		.plug			= 1,
262		.audit_skip		= 1,
263		.ioprio			= 1,
264		.iopoll			= 1,
265		.iopoll_queue		= 1,
266		.async_size		= sizeof(struct io_async_rw),
267		.prep			= io_prep_write,
268		.issue			= io_write,
269	},
270	[IORING_OP_FADVISE] = {
271		.needs_file		= 1,
272		.audit_skip		= 1,
273		.prep			= io_fadvise_prep,
274		.issue			= io_fadvise,
275	},
276	[IORING_OP_MADVISE] = {
277		.audit_skip		= 1,
278		.prep			= io_madvise_prep,
279		.issue			= io_madvise,
280	},
281	[IORING_OP_SEND] = {
282		.needs_file		= 1,
283		.unbound_nonreg_file	= 1,
284		.pollout		= 1,
285		.audit_skip		= 1,
286		.ioprio			= 1,
287		.buffer_select		= 1,
288#if defined(CONFIG_NET)
289		.async_size		= sizeof(struct io_async_msghdr),
290		.prep			= io_sendmsg_prep,
291		.issue			= io_send,
292#else
293		.prep			= io_eopnotsupp_prep,
294#endif
295	},
296	[IORING_OP_RECV] = {
297		.needs_file		= 1,
298		.unbound_nonreg_file	= 1,
299		.pollin			= 1,
300		.buffer_select		= 1,
301		.audit_skip		= 1,
302		.ioprio			= 1,
303#if defined(CONFIG_NET)
304		.async_size		= sizeof(struct io_async_msghdr),
305		.prep			= io_recvmsg_prep,
306		.issue			= io_recv,
307#else
308		.prep			= io_eopnotsupp_prep,
309#endif
310	},
311	[IORING_OP_OPENAT2] = {
312		.prep			= io_openat2_prep,
313		.issue			= io_openat2,
314	},
315	[IORING_OP_EPOLL_CTL] = {
316		.unbound_nonreg_file	= 1,
317		.audit_skip		= 1,
318#if defined(CONFIG_EPOLL)
319		.prep			= io_epoll_ctl_prep,
320		.issue			= io_epoll_ctl,
321#else
322		.prep			= io_eopnotsupp_prep,
323#endif
324	},
325	[IORING_OP_SPLICE] = {
326		.needs_file		= 1,
327		.hash_reg_file		= 1,
328		.unbound_nonreg_file	= 1,
329		.audit_skip		= 1,
330		.prep			= io_splice_prep,
331		.issue			= io_splice,
332	},
333	[IORING_OP_PROVIDE_BUFFERS] = {
334		.audit_skip		= 1,
335		.iopoll			= 1,
336		.prep			= io_provide_buffers_prep,
337		.issue			= io_manage_buffers_legacy,
338	},
339	[IORING_OP_REMOVE_BUFFERS] = {
340		.audit_skip		= 1,
341		.iopoll			= 1,
342		.prep			= io_remove_buffers_prep,
343		.issue			= io_manage_buffers_legacy,
344	},
345	[IORING_OP_TEE] = {
346		.needs_file		= 1,
347		.hash_reg_file		= 1,
348		.unbound_nonreg_file	= 1,
349		.audit_skip		= 1,
350		.prep			= io_tee_prep,
351		.issue			= io_tee,
352	},
353	[IORING_OP_SHUTDOWN] = {
354		.needs_file		= 1,
355#if defined(CONFIG_NET)
356		.prep			= io_shutdown_prep,
357		.issue			= io_shutdown,
358#else
359		.prep			= io_eopnotsupp_prep,
360#endif
361	},
362	[IORING_OP_RENAMEAT] = {
363		.prep			= io_renameat_prep,
364		.issue			= io_renameat,
365	},
366	[IORING_OP_UNLINKAT] = {
367		.prep			= io_unlinkat_prep,
368		.issue			= io_unlinkat,
369	},
370	[IORING_OP_MKDIRAT] = {
371		.prep			= io_mkdirat_prep,
372		.issue			= io_mkdirat,
373	},
374	[IORING_OP_SYMLINKAT] = {
375		.prep			= io_symlinkat_prep,
376		.issue			= io_symlinkat,
377	},
378	[IORING_OP_LINKAT] = {
379		.prep			= io_linkat_prep,
380		.issue			= io_linkat,
381	},
382	[IORING_OP_MSG_RING] = {
383		.needs_file		= 1,
384		.iopoll			= 1,
385		.prep			= io_msg_ring_prep,
386		.issue			= io_msg_ring,
387	},
388	[IORING_OP_FSETXATTR] = {
389		.needs_file = 1,
390		.prep			= io_fsetxattr_prep,
391		.issue			= io_fsetxattr,
392	},
393	[IORING_OP_SETXATTR] = {
394		.prep			= io_setxattr_prep,
395		.issue			= io_setxattr,
396	},
397	[IORING_OP_FGETXATTR] = {
398		.needs_file = 1,
399		.prep			= io_fgetxattr_prep,
400		.issue			= io_fgetxattr,
401	},
402	[IORING_OP_GETXATTR] = {
403		.prep			= io_getxattr_prep,
404		.issue			= io_getxattr,
405	},
406	[IORING_OP_SOCKET] = {
407		.audit_skip		= 1,
408#if defined(CONFIG_NET)
409		.prep			= io_socket_prep,
410		.issue			= io_socket,
411#else
412		.prep			= io_eopnotsupp_prep,
413#endif
414	},
415	[IORING_OP_URING_CMD] = {
416		.buffer_select		= 1,
417		.needs_file		= 1,
418		.plug			= 1,
419		.iopoll			= 1,
420		.iopoll_queue		= 1,
421		.async_size		= sizeof(struct io_async_cmd),
422		.prep			= io_uring_cmd_prep,
423		.issue			= io_uring_cmd,
424	},
425	[IORING_OP_SEND_ZC] = {
426		.needs_file		= 1,
427		.unbound_nonreg_file	= 1,
428		.pollout		= 1,
429		.audit_skip		= 1,
430		.ioprio			= 1,
431#if defined(CONFIG_NET)
432		.async_size		= sizeof(struct io_async_msghdr),
433		.prep			= io_send_zc_prep,
434		.issue			= io_send_zc,
435#else
436		.prep			= io_eopnotsupp_prep,
437#endif
438	},
439	[IORING_OP_SENDMSG_ZC] = {
440		.needs_file		= 1,
441		.unbound_nonreg_file	= 1,
442		.pollout		= 1,
443		.ioprio			= 1,
444#if defined(CONFIG_NET)
445		.async_size		= sizeof(struct io_async_msghdr),
446		.prep			= io_send_zc_prep,
447		.issue			= io_sendmsg_zc,
448#else
449		.prep			= io_eopnotsupp_prep,
450#endif
451	},
452	[IORING_OP_READ_MULTISHOT] = {
453		.needs_file		= 1,
454		.unbound_nonreg_file	= 1,
455		.pollin			= 1,
456		.buffer_select		= 1,
457		.audit_skip		= 1,
458		.async_size		= sizeof(struct io_async_rw),
459		.prep			= io_read_mshot_prep,
460		.issue			= io_read_mshot,
461	},
462	[IORING_OP_WAITID] = {
463		.async_size		= sizeof(struct io_waitid_async),
464		.prep			= io_waitid_prep,
465		.issue			= io_waitid,
466	},
467	[IORING_OP_FUTEX_WAIT] = {
468#if defined(CONFIG_FUTEX)
469		.prep			= io_futex_prep,
470		.issue			= io_futex_wait,
471#else
472		.prep			= io_eopnotsupp_prep,
473#endif
474	},
475	[IORING_OP_FUTEX_WAKE] = {
476#if defined(CONFIG_FUTEX)
477		.prep			= io_futex_prep,
478		.issue			= io_futex_wake,
479#else
480		.prep			= io_eopnotsupp_prep,
481#endif
482	},
483	[IORING_OP_FUTEX_WAITV] = {
484#if defined(CONFIG_FUTEX)
485		.prep			= io_futexv_prep,
486		.issue			= io_futexv_wait,
487#else
488		.prep			= io_eopnotsupp_prep,
489#endif
490	},
491	[IORING_OP_FIXED_FD_INSTALL] = {
492		.needs_file		= 1,
493		.prep			= io_install_fixed_fd_prep,
494		.issue			= io_install_fixed_fd,
495	},
496	[IORING_OP_FTRUNCATE] = {
497		.needs_file		= 1,
498		.hash_reg_file		= 1,
499		.prep			= io_ftruncate_prep,
500		.issue			= io_ftruncate,
501	},
502	[IORING_OP_BIND] = {
503#if defined(CONFIG_NET)
504		.needs_file		= 1,
505		.prep			= io_bind_prep,
506		.issue			= io_bind,
507		.async_size		= sizeof(struct io_async_msghdr),
508#else
509		.prep			= io_eopnotsupp_prep,
510#endif
511	},
512	[IORING_OP_LISTEN] = {
513#if defined(CONFIG_NET)
514		.needs_file		= 1,
515		.prep			= io_listen_prep,
516		.issue			= io_listen,
517		.async_size		= sizeof(struct io_async_msghdr),
518#else
519		.prep			= io_eopnotsupp_prep,
520#endif
521	},
522	[IORING_OP_RECV_ZC] = {
523		.needs_file		= 1,
524		.unbound_nonreg_file	= 1,
525		.pollin			= 1,
526		.ioprio			= 1,
527#if defined(CONFIG_NET)
528		.prep			= io_recvzc_prep,
529		.issue			= io_recvzc,
530#else
531		.prep			= io_eopnotsupp_prep,
532#endif
533	},
534	[IORING_OP_EPOLL_WAIT] = {
535		.needs_file		= 1,
536		.audit_skip		= 1,
537		.pollin			= 1,
538#if defined(CONFIG_EPOLL)
539		.prep			= io_epoll_wait_prep,
540		.issue			= io_epoll_wait,
541#else
542		.prep			= io_eopnotsupp_prep,
543#endif
544	},
545	[IORING_OP_READV_FIXED] = {
546		.needs_file		= 1,
547		.unbound_nonreg_file	= 1,
548		.pollin			= 1,
549		.plug			= 1,
550		.audit_skip		= 1,
551		.ioprio			= 1,
552		.iopoll			= 1,
553		.iopoll_queue		= 1,
554		.vectored		= 1,
555		.async_size		= sizeof(struct io_async_rw),
556		.prep			= io_prep_readv_fixed,
557		.issue			= io_read,
558	},
559	[IORING_OP_WRITEV_FIXED] = {
560		.needs_file		= 1,
561		.hash_reg_file		= 1,
562		.unbound_nonreg_file	= 1,
563		.pollout		= 1,
564		.plug			= 1,
565		.audit_skip		= 1,
566		.ioprio			= 1,
567		.iopoll			= 1,
568		.iopoll_queue		= 1,
569		.vectored		= 1,
570		.async_size		= sizeof(struct io_async_rw),
571		.prep			= io_prep_writev_fixed,
572		.issue			= io_write,
573	},
574	[IORING_OP_PIPE] = {
575		.prep			= io_pipe_prep,
576		.issue			= io_pipe,
577	},
578	[IORING_OP_NOP128] = {
579		.audit_skip		= 1,
580		.iopoll			= 1,
581		.is_128			= 1,
582		.prep			= io_nop_prep,
583		.issue			= io_nop,
584	},
585	[IORING_OP_URING_CMD128] = {
586		.buffer_select		= 1,
587		.needs_file		= 1,
588		.plug			= 1,
589		.iopoll			= 1,
590		.iopoll_queue		= 1,
591		.is_128			= 1,
592		.async_size		= sizeof(struct io_async_cmd),
593		.prep			= io_uring_cmd_prep,
594		.issue			= io_uring_cmd,
595	},
596};
597
598const struct io_cold_def io_cold_defs[] = {
599	[IORING_OP_NOP] = {
600		.name			= "NOP",
601	},
602	[IORING_OP_READV] = {
603		.name			= "READV",
604		.cleanup		= io_readv_writev_cleanup,
605		.fail			= io_rw_fail,
606	},
607	[IORING_OP_WRITEV] = {
608		.name			= "WRITEV",
609		.cleanup		= io_readv_writev_cleanup,
610		.fail			= io_rw_fail,
611	},
612	[IORING_OP_FSYNC] = {
613		.name			= "FSYNC",
614	},
615	[IORING_OP_READ_FIXED] = {
616		.name			= "READ_FIXED",
617		.cleanup		= io_readv_writev_cleanup,
618		.fail			= io_rw_fail,
619	},
620	[IORING_OP_WRITE_FIXED] = {
621		.name			= "WRITE_FIXED",
622		.cleanup		= io_readv_writev_cleanup,
623		.fail			= io_rw_fail,
624	},
625	[IORING_OP_POLL_ADD] = {
626		.name			= "POLL_ADD",
627	},
628	[IORING_OP_POLL_REMOVE] = {
629		.name			= "POLL_REMOVE",
630	},
631	[IORING_OP_SYNC_FILE_RANGE] = {
632		.name			= "SYNC_FILE_RANGE",
633	},
634	[IORING_OP_SENDMSG] = {
635		.name			= "SENDMSG",
636#if defined(CONFIG_NET)
637		.cleanup		= io_sendmsg_recvmsg_cleanup,
638		.fail			= io_sendrecv_fail,
639#endif
640	},
641	[IORING_OP_RECVMSG] = {
642		.name			= "RECVMSG",
643#if defined(CONFIG_NET)
644		.cleanup		= io_sendmsg_recvmsg_cleanup,
645		.fail			= io_sendrecv_fail,
646#endif
647	},
648	[IORING_OP_TIMEOUT] = {
649		.name			= "TIMEOUT",
650	},
651	[IORING_OP_TIMEOUT_REMOVE] = {
652		.name			= "TIMEOUT_REMOVE",
653	},
654	[IORING_OP_ACCEPT] = {
655		.name			= "ACCEPT",
656	},
657	[IORING_OP_ASYNC_CANCEL] = {
658		.name			= "ASYNC_CANCEL",
659	},
660	[IORING_OP_LINK_TIMEOUT] = {
661		.name			= "LINK_TIMEOUT",
662	},
663	[IORING_OP_CONNECT] = {
664		.name			= "CONNECT",
665	},
666	[IORING_OP_FALLOCATE] = {
667		.name			= "FALLOCATE",
668	},
669	[IORING_OP_OPENAT] = {
670		.name			= "OPENAT",
671		.cleanup		= io_open_cleanup,
672	},
673	[IORING_OP_CLOSE] = {
674		.name			= "CLOSE",
675	},
676	[IORING_OP_FILES_UPDATE] = {
677		.name			= "FILES_UPDATE",
678	},
679	[IORING_OP_STATX] = {
680		.name			= "STATX",
681		.cleanup		= io_statx_cleanup,
682	},
683	[IORING_OP_READ] = {
684		.name			= "READ",
685		.cleanup		= io_readv_writev_cleanup,
686		.fail			= io_rw_fail,
687	},
688	[IORING_OP_WRITE] = {
689		.name			= "WRITE",
690		.cleanup		= io_readv_writev_cleanup,
691		.fail			= io_rw_fail,
692	},
693	[IORING_OP_FADVISE] = {
694		.name			= "FADVISE",
695	},
696	[IORING_OP_MADVISE] = {
697		.name			= "MADVISE",
698	},
699	[IORING_OP_SEND] = {
700		.name			= "SEND",
701#if defined(CONFIG_NET)
702		.cleanup		= io_sendmsg_recvmsg_cleanup,
703		.fail			= io_sendrecv_fail,
704#endif
705	},
706	[IORING_OP_RECV] = {
707		.name			= "RECV",
708#if defined(CONFIG_NET)
709		.cleanup		= io_sendmsg_recvmsg_cleanup,
710		.fail			= io_sendrecv_fail,
711#endif
712	},
713	[IORING_OP_OPENAT2] = {
714		.name			= "OPENAT2",
715		.cleanup		= io_open_cleanup,
716	},
717	[IORING_OP_EPOLL_CTL] = {
718		.name			= "EPOLL",
719	},
720	[IORING_OP_SPLICE] = {
721		.name			= "SPLICE",
722		.cleanup		= io_splice_cleanup,
723	},
724	[IORING_OP_PROVIDE_BUFFERS] = {
725		.name			= "PROVIDE_BUFFERS",
726	},
727	[IORING_OP_REMOVE_BUFFERS] = {
728		.name			= "REMOVE_BUFFERS",
729	},
730	[IORING_OP_TEE] = {
731		.name			= "TEE",
732		.cleanup		= io_splice_cleanup,
733	},
734	[IORING_OP_SHUTDOWN] = {
735		.name			= "SHUTDOWN",
736	},
737	[IORING_OP_RENAMEAT] = {
738		.name			= "RENAMEAT",
739		.cleanup		= io_renameat_cleanup,
740	},
741	[IORING_OP_UNLINKAT] = {
742		.name			= "UNLINKAT",
743		.cleanup		= io_unlinkat_cleanup,
744	},
745	[IORING_OP_MKDIRAT] = {
746		.name			= "MKDIRAT",
747		.cleanup		= io_mkdirat_cleanup,
748	},
749	[IORING_OP_SYMLINKAT] = {
750		.name			= "SYMLINKAT",
751		.cleanup		= io_link_cleanup,
752	},
753	[IORING_OP_LINKAT] = {
754		.name			= "LINKAT",
755		.cleanup		= io_link_cleanup,
756	},
757	[IORING_OP_MSG_RING] = {
758		.name			= "MSG_RING",
759		.cleanup		= io_msg_ring_cleanup,
760	},
761	[IORING_OP_FSETXATTR] = {
762		.name			= "FSETXATTR",
763		.cleanup		= io_xattr_cleanup,
764	},
765	[IORING_OP_SETXATTR] = {
766		.name			= "SETXATTR",
767		.cleanup		= io_xattr_cleanup,
768	},
769	[IORING_OP_FGETXATTR] = {
770		.name			= "FGETXATTR",
771		.cleanup		= io_xattr_cleanup,
772	},
773	[IORING_OP_GETXATTR] = {
774		.name			= "GETXATTR",
775		.cleanup		= io_xattr_cleanup,
776	},
777	[IORING_OP_SOCKET] = {
778		.name			= "SOCKET",
779	},
780	[IORING_OP_URING_CMD] = {
781		.name			= "URING_CMD",
782		.sqe_copy		= io_uring_cmd_sqe_copy,
783		.cleanup		= io_uring_cmd_cleanup,
784	},
785	[IORING_OP_SEND_ZC] = {
786		.name			= "SEND_ZC",
787#if defined(CONFIG_NET)
788		.cleanup		= io_send_zc_cleanup,
789		.fail			= io_sendrecv_fail,
790#endif
791	},
792	[IORING_OP_SENDMSG_ZC] = {
793		.name			= "SENDMSG_ZC",
794#if defined(CONFIG_NET)
795		.cleanup		= io_send_zc_cleanup,
796		.fail			= io_sendrecv_fail,
797#endif
798	},
799	[IORING_OP_READ_MULTISHOT] = {
800		.name			= "READ_MULTISHOT",
801		.cleanup		= io_readv_writev_cleanup,
802	},
803	[IORING_OP_WAITID] = {
804		.name			= "WAITID",
805	},
806	[IORING_OP_FUTEX_WAIT] = {
807		.name			= "FUTEX_WAIT",
808	},
809	[IORING_OP_FUTEX_WAKE] = {
810		.name			= "FUTEX_WAKE",
811	},
812	[IORING_OP_FUTEX_WAITV] = {
813		.name			= "FUTEX_WAITV",
814	},
815	[IORING_OP_FIXED_FD_INSTALL] = {
816		.name			= "FIXED_FD_INSTALL",
817	},
818	[IORING_OP_FTRUNCATE] = {
819		.name			= "FTRUNCATE",
820	},
821	[IORING_OP_BIND] = {
822		.name			= "BIND",
823	},
824	[IORING_OP_LISTEN] = {
825		.name			= "LISTEN",
826	},
827	[IORING_OP_RECV_ZC] = {
828		.name			= "RECV_ZC",
829	},
830	[IORING_OP_EPOLL_WAIT] = {
831		.name			= "EPOLL_WAIT",
832	},
833	[IORING_OP_READV_FIXED] = {
834		.name			= "READV_FIXED",
835		.cleanup		= io_readv_writev_cleanup,
836		.fail			= io_rw_fail,
837	},
838	[IORING_OP_WRITEV_FIXED] = {
839		.name			= "WRITEV_FIXED",
840		.cleanup		= io_readv_writev_cleanup,
841		.fail			= io_rw_fail,
842	},
843	[IORING_OP_PIPE] = {
844		.name			= "PIPE",
845	},
846	[IORING_OP_NOP128] = {
847		.name			= "NOP128",
848	},
849	[IORING_OP_URING_CMD128] = {
850		.name			= "URING_CMD128",
851		.sqe_copy		= io_uring_cmd_sqe_copy,
852		.cleanup		= io_uring_cmd_cleanup,
853	},
854};
855
856const char *io_uring_get_opcode(u8 opcode)
857{
858	if (opcode < IORING_OP_LAST)
859		return io_cold_defs[opcode].name;
860	return "INVALID";
861}
862
863bool io_uring_op_supported(u8 opcode)
864{
865	if (opcode < IORING_OP_LAST &&
866	    io_issue_defs[opcode].prep != io_eopnotsupp_prep)
867		return true;
868	return false;
869}
870
871void __init io_uring_optable_init(void)
872{
873	int i;
874
875	BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
876	BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
877
878	for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
879		BUG_ON(!io_issue_defs[i].prep);
880		if (io_issue_defs[i].prep != io_eopnotsupp_prep)
881			BUG_ON(!io_issue_defs[i].issue);
882		WARN_ON_ONCE(!io_cold_defs[i].name);
883	}
884}