io_uring_types.h

  1#ifndef IO_URING_TYPES_H
  2#define IO_URING_TYPES_H
  3
  4#include <linux/blkdev.h>
  5#include <linux/hashtable.h>
  6#include <linux/task_work.h>
  7#include <linux/bitmap.h>
  8#include <linux/llist.h>
  9#include <uapi/linux/io_uring.h>
 10
 11enum {
 12	/*
 13	 * A hint to not wake right away but delay until there are enough of
 14	 * tw's queued to match the number of CQEs the task is waiting for.
 15	 *
 16	 * Must not be used with requests generating more than one CQE.
 17	 * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set.
 18	 */
 19	IOU_F_TWQ_LAZY_WAKE			= 1,
 20};
 21
 22enum io_uring_cmd_flags {
 23	IO_URING_F_COMPLETE_DEFER	= 1,
 24	IO_URING_F_UNLOCKED		= 2,
 25	/* the request is executed from poll, it should not be freed */
 26	IO_URING_F_MULTISHOT		= 4,
 27	/* executed by io-wq */
 28	IO_URING_F_IOWQ			= 8,
 29	/* executed inline from syscall */
 30	IO_URING_F_INLINE		= 16,
 31	/* int's last bit, sign checks are usually faster than a bit test */
 32	IO_URING_F_NONBLOCK		= INT_MIN,
 33
 34	/* ctx state flags, for URING_CMD */
 35	IO_URING_F_SQE128		= (1 << 8),
 36	IO_URING_F_CQE32		= (1 << 9),
 37	IO_URING_F_IOPOLL		= (1 << 10),
 38
 39	/* set when uring wants to cancel a previously issued command */
 40	IO_URING_F_CANCEL		= (1 << 11),
 41	IO_URING_F_COMPAT		= (1 << 12),
 42};
 43
 44struct io_wq_work_node {
 45	struct io_wq_work_node *next;
 46};
 47
 48struct io_wq_work_list {
 49	struct io_wq_work_node *first;
 50	struct io_wq_work_node *last;
 51};
 52
 53struct io_wq_work {
 54	struct io_wq_work_node list;
 55	atomic_t flags;
 56	/* place it here instead of io_kiocb as it fills padding and saves 4B */
 57	int cancel_seq;
 58};
 59
 60struct io_rsrc_data {
 61	unsigned int			nr;
 62	struct io_rsrc_node		**nodes;
 63};
 64
 65struct io_file_table {
 66	struct io_rsrc_data data;
 67	unsigned long *bitmap;
 68	unsigned int alloc_hint;
 69};
 70
 71struct io_hash_bucket {
 72	struct hlist_head	list;
 73} ____cacheline_aligned_in_smp;
 74
 75struct io_hash_table {
 76	struct io_hash_bucket	*hbs;
 77	unsigned		hash_bits;
 78};
 79
 80struct io_mapped_region {
 81	struct page		**pages;
 82	void			*ptr;
 83	unsigned		nr_pages;
 84	unsigned		flags;
 85};
 86
 87/*
 88 * Return value from io_buffer_list selection, to avoid stashing it in
 89 * struct io_kiocb. For legacy/classic provided buffers, keeping a reference
 90 * across execution contexts are fine. But for ring provided buffers, the
 91 * list may go away as soon as ->uring_lock is dropped. As the io_kiocb
 92 * persists, it's better to just keep the buffer local for those cases.
 93 */
 94struct io_br_sel {
 95	struct io_buffer_list *buf_list;
 96	/*
 97	 * Some selection parts return the user address, others return an error.
 98	 */
 99	union {
100		void __user *addr;
101		ssize_t val;
102	};
103};
104
105
106/*
107 * Arbitrary limit, can be raised if need be
108 */
109#define IO_RINGFD_REG_MAX 16
110
111struct io_uring_task {
112	/* submission side */
113	int				cached_refs;
114	const struct io_ring_ctx 	*last;
115	struct task_struct		*task;
116	struct io_wq			*io_wq;
117	struct file			*registered_rings[IO_RINGFD_REG_MAX];
118
119	struct xarray			xa;
120	struct wait_queue_head		wait;
121	atomic_t			in_cancel;
122	atomic_t			inflight_tracked;
123	struct percpu_counter		inflight;
124
125	struct { /* task_work */
126		struct llist_head	task_list;
127		struct callback_head	task_work;
128	} ____cacheline_aligned_in_smp;
129};
130
131struct iou_vec {
132	union {
133		struct iovec	*iovec;
134		struct bio_vec	*bvec;
135	};
136	unsigned		nr; /* number of struct iovec it can hold */
137};
138
139struct io_uring {
140	u32 head;
141	u32 tail;
142};
143
144/*
145 * This data is shared with the application through the mmap at offsets
146 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
147 *
148 * The offsets to the member fields are published through struct
149 * io_sqring_offsets when calling io_uring_setup.
150 */
151struct io_rings {
152	/*
153	 * Head and tail offsets into the ring; the offsets need to be
154	 * masked to get valid indices.
155	 *
156	 * The kernel controls head of the sq ring and the tail of the cq ring,
157	 * and the application controls tail of the sq ring and the head of the
158	 * cq ring.
159	 */
160	struct io_uring		sq, cq;
161	/*
162	 * Bitmasks to apply to head and tail offsets (constant, equals
163	 * ring_entries - 1)
164	 */
165	u32			sq_ring_mask, cq_ring_mask;
166	/* Ring sizes (constant, power of 2) */
167	u32			sq_ring_entries, cq_ring_entries;
168	/*
169	 * Number of invalid entries dropped by the kernel due to
170	 * invalid index stored in array
171	 *
172	 * Written by the kernel, shouldn't be modified by the
173	 * application (i.e. get number of "new events" by comparing to
174	 * cached value).
175	 *
176	 * After a new SQ head value was read by the application this
177	 * counter includes all submissions that were dropped reaching
178	 * the new SQ head (and possibly more).
179	 */
180	u32			sq_dropped;
181	/*
182	 * Runtime SQ flags
183	 *
184	 * Written by the kernel, shouldn't be modified by the
185	 * application.
186	 *
187	 * The application needs a full memory barrier before checking
188	 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
189	 */
190	atomic_t		sq_flags;
191	/*
192	 * Runtime CQ flags
193	 *
194	 * Written by the application, shouldn't be modified by the
195	 * kernel.
196	 */
197	u32			cq_flags;
198	/*
199	 * Number of completion events lost because the queue was full;
200	 * this should be avoided by the application by making sure
201	 * there are not more requests pending than there is space in
202	 * the completion queue.
203	 *
204	 * Written by the kernel, shouldn't be modified by the
205	 * application (i.e. get number of "new events" by comparing to
206	 * cached value).
207	 *
208	 * As completion events come in out of order this counter is not
209	 * ordered with any other data.
210	 */
211	u32			cq_overflow;
212	/*
213	 * Ring buffer of completion events.
214	 *
215	 * The kernel writes completion events fresh every time they are
216	 * produced, so the application is allowed to modify pending
217	 * entries.
218	 */
219	struct io_uring_cqe	cqes[] ____cacheline_aligned_in_smp;
220};
221
222struct io_restriction {
223	DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
224	DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
225	u8 sqe_flags_allowed;
226	u8 sqe_flags_required;
227	bool registered;
228};
229
230struct io_submit_link {
231	struct io_kiocb		*head;
232	struct io_kiocb		*last;
233};
234
235struct io_submit_state {
236	/* inline/task_work completion list, under ->uring_lock */
237	struct io_wq_work_node	free_list;
238	/* batch completion logic */
239	struct io_wq_work_list	compl_reqs;
240	struct io_submit_link	link;
241
242	bool			plug_started;
243	bool			need_plug;
244	bool			cq_flush;
245	unsigned short		submit_nr;
246	struct blk_plug		plug;
247};
248
249struct io_alloc_cache {
250	void			**entries;
251	unsigned int		nr_cached;
252	unsigned int		max_cached;
253	unsigned int		elem_size;
254	unsigned int		init_clear;
255};
256
257struct io_ring_ctx {
258	/* const or read-mostly hot data */
259	struct {
260		unsigned int		flags;
261		unsigned int		drain_next: 1;
262		unsigned int		restricted: 1;
263		unsigned int		off_timeout_used: 1;
264		unsigned int		drain_active: 1;
265		unsigned int		has_evfd: 1;
266		/* all CQEs should be posted only by the submitter task */
267		unsigned int		task_complete: 1;
268		unsigned int		lockless_cq: 1;
269		unsigned int		syscall_iopoll: 1;
270		unsigned int		poll_activated: 1;
271		unsigned int		drain_disabled: 1;
272		unsigned int		compat: 1;
273		unsigned int		iowq_limits_set : 1;
274
275		struct task_struct	*submitter_task;
276		struct io_rings		*rings;
277		struct percpu_ref	refs;
278
279		clockid_t		clockid;
280		enum tk_offsets		clock_offset;
281
282		enum task_work_notify_mode	notify_method;
283		unsigned			sq_thread_idle;
284	} ____cacheline_aligned_in_smp;
285
286	/* submission data */
287	struct {
288		struct mutex		uring_lock;
289
290		/*
291		 * Ring buffer of indices into array of io_uring_sqe, which is
292		 * mmapped by the application using the IORING_OFF_SQES offset.
293		 *
294		 * This indirection could e.g. be used to assign fixed
295		 * io_uring_sqe entries to operations and only submit them to
296		 * the queue when needed.
297		 *
298		 * The kernel modifies neither the indices array nor the entries
299		 * array.
300		 */
301		u32			*sq_array;
302		struct io_uring_sqe	*sq_sqes;
303		unsigned		cached_sq_head;
304		unsigned		sq_entries;
305
306		/*
307		 * Fixed resources fast path, should be accessed only under
308		 * uring_lock, and updated through io_uring_register(2)
309		 */
310		atomic_t		cancel_seq;
311
312		/*
313		 * ->iopoll_list is protected by the ctx->uring_lock for
314		 * io_uring instances that don't use IORING_SETUP_SQPOLL.
315		 * For SQPOLL, only the single threaded io_sq_thread() will
316		 * manipulate the list, hence no extra locking is needed there.
317		 */
318		bool			poll_multi_queue;
319		struct io_wq_work_list	iopoll_list;
320
321		struct io_file_table	file_table;
322		struct io_rsrc_data	buf_table;
323		struct io_alloc_cache	node_cache;
324		struct io_alloc_cache	imu_cache;
325
326		struct io_submit_state	submit_state;
327
328		/*
329		 * Modifications are protected by ->uring_lock and ->mmap_lock.
330		 * The buffer list's io mapped region should be stable once
331		 * published.
332		 */
333		struct xarray		io_bl_xa;
334
335		struct io_hash_table	cancel_table;
336		struct io_alloc_cache	apoll_cache;
337		struct io_alloc_cache	netmsg_cache;
338		struct io_alloc_cache	rw_cache;
339		struct io_alloc_cache	cmd_cache;
340
341		/*
342		 * Any cancelable uring_cmd is added to this list in
343		 * ->uring_cmd() by io_uring_cmd_insert_cancelable()
344		 */
345		struct hlist_head	cancelable_uring_cmd;
346		/*
347		 * For Hybrid IOPOLL, runtime in hybrid polling, without
348		 * scheduling time
349		 */
350		u64					hybrid_poll_time;
351	} ____cacheline_aligned_in_smp;
352
353	struct {
354		/*
355		 * We cache a range of free CQEs we can use, once exhausted it
356		 * should go through a slower range setup, see __io_get_cqe()
357		 */
358		struct io_uring_cqe	*cqe_cached;
359		struct io_uring_cqe	*cqe_sentinel;
360
361		unsigned		cached_cq_tail;
362		unsigned		cq_entries;
363		struct io_ev_fd	__rcu	*io_ev_fd;
364
365		void			*cq_wait_arg;
366		size_t			cq_wait_size;
367	} ____cacheline_aligned_in_smp;
368
369	/*
370	 * task_work and async notification delivery cacheline. Expected to
371	 * regularly bounce b/w CPUs.
372	 */
373	struct {
374		struct io_rings	__rcu	*rings_rcu;
375		struct llist_head	work_llist;
376		struct llist_head	retry_llist;
377		unsigned long		check_cq;
378		atomic_t		cq_wait_nr;
379		atomic_t		cq_timeouts;
380		struct wait_queue_head	cq_wait;
381	} ____cacheline_aligned_in_smp;
382
383	/* timeouts */
384	struct {
385		raw_spinlock_t		timeout_lock;
386		struct list_head	timeout_list;
387		struct list_head	ltimeout_list;
388		unsigned		cq_last_tm_flush;
389	} ____cacheline_aligned_in_smp;
390
391	spinlock_t		completion_lock;
392
393	struct list_head	cq_overflow_list;
394
395	struct hlist_head	waitid_list;
396
397#ifdef CONFIG_FUTEX
398	struct hlist_head	futex_list;
399	struct io_alloc_cache	futex_cache;
400#endif
401
402	const struct cred	*sq_creds;	/* cred used for __io_sq_thread() */
403	struct io_sq_data	*sq_data;	/* if using sq thread polling */
404
405	struct wait_queue_head	sqo_sq_wait;
406	struct list_head	sqd_list;
407
408	unsigned int		file_alloc_start;
409	unsigned int		file_alloc_end;
410
411	/* Keep this last, we don't need it for the fast path */
412	struct wait_queue_head		poll_wq;
413	struct io_restriction		restrictions;
414
415	/* Stores zcrx object pointers of type struct io_zcrx_ifq */
416	struct xarray			zcrx_ctxs;
417
418	u32			pers_next;
419	struct xarray		personalities;
420
421	/* hashed buffered write serialization */
422	struct io_wq_hash		*hash_map;
423
424	/* Only used for accounting purposes */
425	struct user_struct		*user;
426	struct mm_struct		*mm_account;
427
428	/*
429	 * List of tctx nodes for this ctx, protected by tctx_lock. For
430	 * cancelation purposes, nests under uring_lock.
431	 */
432	struct list_head		tctx_list;
433	struct mutex			tctx_lock;
434
435	/* ctx exit and cancelation */
436	struct llist_head		fallback_llist;
437	struct delayed_work		fallback_work;
438	struct work_struct		exit_work;
439	struct completion		ref_comp;
440
441	/* io-wq management, e.g. thread count */
442	u32				iowq_limits[2];
443
444	struct callback_head		poll_wq_task_work;
445	struct list_head		defer_list;
446	unsigned			nr_drained;
447
448	/* protected by ->completion_lock */
449	unsigned			nr_req_allocated;
450
451#ifdef CONFIG_NET_RX_BUSY_POLL
452	struct list_head	napi_list;	/* track busy poll napi_id */
453	spinlock_t		napi_lock;	/* napi_list lock */
454
455	/* napi busy poll default timeout */
456	ktime_t			napi_busy_poll_dt;
457	bool			napi_prefer_busy_poll;
458	u8			napi_track_mode;
459
460	DECLARE_HASHTABLE(napi_ht, 4);
461#endif
462
463	/*
464	 * Protection for resize vs mmap races - both the mmap and resize
465	 * side will need to grab this lock, to prevent either side from
466	 * being run concurrently with the other.
467	 */
468	struct mutex			mmap_lock;
469
470	struct io_mapped_region		sq_region;
471	struct io_mapped_region		ring_region;
472	/* used for optimised request parameter and wait argument passing  */
473	struct io_mapped_region		param_region;
474};
475
476/*
477 * Token indicating function is called in task work context:
478 * ctx->uring_lock is held and any completions generated will be flushed.
479 * ONLY core io_uring.c should instantiate this struct.
480 */
481struct io_tw_state {
482	bool cancel;
483};
484/* Alias to use in code that doesn't instantiate struct io_tw_state */
485typedef struct io_tw_state io_tw_token_t;
486
487enum {
488	REQ_F_FIXED_FILE_BIT	= IOSQE_FIXED_FILE_BIT,
489	REQ_F_IO_DRAIN_BIT	= IOSQE_IO_DRAIN_BIT,
490	REQ_F_LINK_BIT		= IOSQE_IO_LINK_BIT,
491	REQ_F_HARDLINK_BIT	= IOSQE_IO_HARDLINK_BIT,
492	REQ_F_FORCE_ASYNC_BIT	= IOSQE_ASYNC_BIT,
493	REQ_F_BUFFER_SELECT_BIT	= IOSQE_BUFFER_SELECT_BIT,
494	REQ_F_CQE_SKIP_BIT	= IOSQE_CQE_SKIP_SUCCESS_BIT,
495
496	/* first byte is taken by user flags, shift it to not overlap */
497	REQ_F_FAIL_BIT		= 8,
498	REQ_F_INFLIGHT_BIT,
499	REQ_F_CUR_POS_BIT,
500	REQ_F_NOWAIT_BIT,
501	REQ_F_LINK_TIMEOUT_BIT,
502	REQ_F_NEED_CLEANUP_BIT,
503	REQ_F_POLLED_BIT,
504	REQ_F_HYBRID_IOPOLL_STATE_BIT,
505	REQ_F_BUFFER_SELECTED_BIT,
506	REQ_F_BUFFER_RING_BIT,
507	REQ_F_REISSUE_BIT,
508	REQ_F_CREDS_BIT,
509	REQ_F_REFCOUNT_BIT,
510	REQ_F_ARM_LTIMEOUT_BIT,
511	REQ_F_ASYNC_DATA_BIT,
512	REQ_F_SKIP_LINK_CQES_BIT,
513	REQ_F_SINGLE_POLL_BIT,
514	REQ_F_DOUBLE_POLL_BIT,
515	REQ_F_MULTISHOT_BIT,
516	REQ_F_APOLL_MULTISHOT_BIT,
517	REQ_F_CLEAR_POLLIN_BIT,
518	/* keep async read/write and isreg together and in order */
519	REQ_F_SUPPORT_NOWAIT_BIT,
520	REQ_F_ISREG_BIT,
521	REQ_F_POLL_NO_LAZY_BIT,
522	REQ_F_CAN_POLL_BIT,
523	REQ_F_BL_EMPTY_BIT,
524	REQ_F_BL_NO_RECYCLE_BIT,
525	REQ_F_BUFFERS_COMMIT_BIT,
526	REQ_F_BUF_NODE_BIT,
527	REQ_F_BUF_MORE_BIT,
528	REQ_F_HAS_METADATA_BIT,
529	REQ_F_IMPORT_BUFFER_BIT,
530	REQ_F_SQE_COPIED_BIT,
531
532	/* not a real bit, just to check we're not overflowing the space */
533	__REQ_F_LAST_BIT,
534};
535
536typedef u64 __bitwise io_req_flags_t;
537#define IO_REQ_FLAG(bitno)	((__force io_req_flags_t) BIT_ULL((bitno)))
538
539enum {
540	/* ctx owns file */
541	REQ_F_FIXED_FILE	= IO_REQ_FLAG(REQ_F_FIXED_FILE_BIT),
542	/* drain existing IO first */
543	REQ_F_IO_DRAIN		= IO_REQ_FLAG(REQ_F_IO_DRAIN_BIT),
544	/* linked sqes */
545	REQ_F_LINK		= IO_REQ_FLAG(REQ_F_LINK_BIT),
546	/* doesn't sever on completion < 0 */
547	REQ_F_HARDLINK		= IO_REQ_FLAG(REQ_F_HARDLINK_BIT),
548	/* IOSQE_ASYNC */
549	REQ_F_FORCE_ASYNC	= IO_REQ_FLAG(REQ_F_FORCE_ASYNC_BIT),
550	/* IOSQE_BUFFER_SELECT */
551	REQ_F_BUFFER_SELECT	= IO_REQ_FLAG(REQ_F_BUFFER_SELECT_BIT),
552	/* IOSQE_CQE_SKIP_SUCCESS */
553	REQ_F_CQE_SKIP		= IO_REQ_FLAG(REQ_F_CQE_SKIP_BIT),
554
555	/* fail rest of links */
556	REQ_F_FAIL		= IO_REQ_FLAG(REQ_F_FAIL_BIT),
557	/* on inflight list, should be cancelled and waited on exit reliably */
558	REQ_F_INFLIGHT		= IO_REQ_FLAG(REQ_F_INFLIGHT_BIT),
559	/* read/write uses file position */
560	REQ_F_CUR_POS		= IO_REQ_FLAG(REQ_F_CUR_POS_BIT),
561	/* must not punt to workers */
562	REQ_F_NOWAIT		= IO_REQ_FLAG(REQ_F_NOWAIT_BIT),
563	/* has or had linked timeout */
564	REQ_F_LINK_TIMEOUT	= IO_REQ_FLAG(REQ_F_LINK_TIMEOUT_BIT),
565	/* needs cleanup */
566	REQ_F_NEED_CLEANUP	= IO_REQ_FLAG(REQ_F_NEED_CLEANUP_BIT),
567	/* already went through poll handler */
568	REQ_F_POLLED		= IO_REQ_FLAG(REQ_F_POLLED_BIT),
569	/* every req only blocks once in hybrid poll */
570	REQ_F_IOPOLL_STATE        = IO_REQ_FLAG(REQ_F_HYBRID_IOPOLL_STATE_BIT),
571	/* buffer already selected */
572	REQ_F_BUFFER_SELECTED	= IO_REQ_FLAG(REQ_F_BUFFER_SELECTED_BIT),
573	/* buffer selected from ring, needs commit */
574	REQ_F_BUFFER_RING	= IO_REQ_FLAG(REQ_F_BUFFER_RING_BIT),
575	/* caller should reissue async */
576	REQ_F_REISSUE		= IO_REQ_FLAG(REQ_F_REISSUE_BIT),
577	/* supports async reads/writes */
578	REQ_F_SUPPORT_NOWAIT	= IO_REQ_FLAG(REQ_F_SUPPORT_NOWAIT_BIT),
579	/* regular file */
580	REQ_F_ISREG		= IO_REQ_FLAG(REQ_F_ISREG_BIT),
581	/* has creds assigned */
582	REQ_F_CREDS		= IO_REQ_FLAG(REQ_F_CREDS_BIT),
583	/* skip refcounting if not set */
584	REQ_F_REFCOUNT		= IO_REQ_FLAG(REQ_F_REFCOUNT_BIT),
585	/* there is a linked timeout that has to be armed */
586	REQ_F_ARM_LTIMEOUT	= IO_REQ_FLAG(REQ_F_ARM_LTIMEOUT_BIT),
587	/* ->async_data allocated */
588	REQ_F_ASYNC_DATA	= IO_REQ_FLAG(REQ_F_ASYNC_DATA_BIT),
589	/* don't post CQEs while failing linked requests */
590	REQ_F_SKIP_LINK_CQES	= IO_REQ_FLAG(REQ_F_SKIP_LINK_CQES_BIT),
591	/* single poll may be active */
592	REQ_F_SINGLE_POLL	= IO_REQ_FLAG(REQ_F_SINGLE_POLL_BIT),
593	/* double poll may active */
594	REQ_F_DOUBLE_POLL	= IO_REQ_FLAG(REQ_F_DOUBLE_POLL_BIT),
595	/* request posts multiple completions, should be set at prep time */
596	REQ_F_MULTISHOT		= IO_REQ_FLAG(REQ_F_MULTISHOT_BIT),
597	/* fast poll multishot mode */
598	REQ_F_APOLL_MULTISHOT	= IO_REQ_FLAG(REQ_F_APOLL_MULTISHOT_BIT),
599	/* recvmsg special flag, clear EPOLLIN */
600	REQ_F_CLEAR_POLLIN	= IO_REQ_FLAG(REQ_F_CLEAR_POLLIN_BIT),
601	/* don't use lazy poll wake for this request */
602	REQ_F_POLL_NO_LAZY	= IO_REQ_FLAG(REQ_F_POLL_NO_LAZY_BIT),
603	/* file is pollable */
604	REQ_F_CAN_POLL		= IO_REQ_FLAG(REQ_F_CAN_POLL_BIT),
605	/* buffer list was empty after selection of buffer */
606	REQ_F_BL_EMPTY		= IO_REQ_FLAG(REQ_F_BL_EMPTY_BIT),
607	/* don't recycle provided buffers for this request */
608	REQ_F_BL_NO_RECYCLE	= IO_REQ_FLAG(REQ_F_BL_NO_RECYCLE_BIT),
609	/* buffer ring head needs incrementing on put */
610	REQ_F_BUFFERS_COMMIT	= IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
611	/* buf node is valid */
612	REQ_F_BUF_NODE		= IO_REQ_FLAG(REQ_F_BUF_NODE_BIT),
613	/* incremental buffer consumption, more space available */
614	REQ_F_BUF_MORE		= IO_REQ_FLAG(REQ_F_BUF_MORE_BIT),
615	/* request has read/write metadata assigned */
616	REQ_F_HAS_METADATA	= IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
617	/*
618	 * For vectored fixed buffers, resolve iovec to registered buffers.
619	 * For SEND_ZC, whether to import buffers (i.e. the first issue).
620	 */
621	REQ_F_IMPORT_BUFFER	= IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT),
622	/* ->sqe_copy() has been called, if necessary */
623	REQ_F_SQE_COPIED	= IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT),
624};
625
626struct io_tw_req {
627	struct io_kiocb *req;
628};
629
630typedef void (*io_req_tw_func_t)(struct io_tw_req tw_req, io_tw_token_t tw);
631
632struct io_task_work {
633	struct llist_node		node;
634	io_req_tw_func_t		func;
635};
636
637struct io_cqe {
638	__u64	user_data;
639	__s32	res;
640	/* fd initially, then cflags for completion */
641	union {
642		__u32	flags;
643		int	fd;
644	};
645};
646
647/*
648 * Each request type overlays its private data structure on top of this one.
649 * They must not exceed this one in size.
650 */
651struct io_cmd_data {
652	struct file		*file;
653	/* each command gets 56 bytes of data */
654	__u8			data[56];
655};
656
657static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)
658{
659	BUILD_BUG_ON(cmd_sz > sizeof(struct io_cmd_data));
660}
661#define io_kiocb_to_cmd(req, cmd_type) ( \
662	io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \
663	((cmd_type *)&(req)->cmd) \
664)
665
666static inline struct io_kiocb *cmd_to_io_kiocb(void *ptr)
667{
668	return ptr;
669}
670
671struct io_kiocb {
672	union {
673		/*
674		 * NOTE! Each of the io_kiocb union members has the file pointer
675		 * as the first entry in their struct definition. So you can
676		 * access the file pointer through any of the sub-structs,
677		 * or directly as just 'file' in this struct.
678		 */
679		struct file		*file;
680		struct io_cmd_data	cmd;
681	};
682
683	u8				opcode;
684	/* polled IO has completed */
685	u8				iopoll_completed;
686	/*
687	 * Can be either a fixed buffer index, or used with provided buffers.
688	 * For the latter, it points to the selected buffer ID.
689	 */
690	u16				buf_index;
691
692	unsigned			nr_tw;
693
694	/* REQ_F_* flags */
695	io_req_flags_t			flags;
696
697	struct io_cqe			cqe;
698
699	struct io_ring_ctx		*ctx;
700	struct io_uring_task		*tctx;
701
702	union {
703		/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
704		struct io_buffer	*kbuf;
705
706		struct io_rsrc_node	*buf_node;
707	};
708
709	union {
710		/* used by request caches, completion batching and iopoll */
711		struct io_wq_work_node	comp_list;
712		/* cache ->apoll->events */
713		__poll_t apoll_events;
714	};
715
716	struct io_rsrc_node		*file_node;
717
718	atomic_t			refs;
719	bool				cancel_seq_set;
720	struct io_task_work		io_task_work;
721	union {
722		/*
723		 * for polled requests, i.e. IORING_OP_POLL_ADD and async armed
724		 * poll
725		 */
726		struct hlist_node	hash_node;
727		/* For IOPOLL setup queues, with hybrid polling */
728		u64                     iopoll_start;
729		/* for private io_kiocb freeing */
730		struct rcu_head		rcu_head;
731	};
732	/* internal polling, see IORING_FEAT_FAST_POLL */
733	struct async_poll		*apoll;
734	/* opcode allocated if it needs to store data for async defer */
735	void				*async_data;
736	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
737	atomic_t			poll_refs;
738	struct io_kiocb			*link;
739	/* custom credentials, valid IFF REQ_F_CREDS is set */
740	const struct cred		*creds;
741	struct io_wq_work		work;
742
743	struct io_big_cqe {
744		u64			extra1;
745		u64			extra2;
746	} big_cqe;
747};
748
749struct io_overflow_cqe {
750	struct list_head list;
751	struct io_uring_cqe cqe;
752};
753#endif