uring_cmd.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/errno.h>
  4#include <linux/file.h>
  5#include <linux/io_uring/cmd.h>
  6#include <linux/security.h>
  7#include <linux/nospec.h>
  8
  9#include <uapi/linux/io_uring.h>
 10
 11#include "io_uring.h"
 12#include "alloc_cache.h"
 13#include "rsrc.h"
 14#include "kbuf.h"
 15#include "uring_cmd.h"
 16#include "poll.h"
 17
 18void io_cmd_cache_free(const void *entry)
 19{
 20	struct io_async_cmd *ac = (struct io_async_cmd *)entry;
 21
 22	io_vec_free(&ac->vec);
 23	kfree(ac);
 24}
 25
 26static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
 27{
 28	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
 29	struct io_async_cmd *ac = req->async_data;
 30
 31	if (issue_flags & IO_URING_F_UNLOCKED)
 32		return;
 33
 34	io_alloc_cache_vec_kasan(&ac->vec);
 35	if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
 36		io_vec_free(&ac->vec);
 37
 38	if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) {
 39		ioucmd->sqe = NULL;
 40		io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
 41	}
 42}
 43
 44void io_uring_cmd_cleanup(struct io_kiocb *req)
 45{
 46	io_req_uring_cleanup(req, 0);
 47}
 48
 49bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
 50				   struct io_uring_task *tctx, bool cancel_all)
 51{
 52	struct hlist_node *tmp;
 53	struct io_kiocb *req;
 54	bool ret = false;
 55
 56	lockdep_assert_held(&ctx->uring_lock);
 57
 58	hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
 59			hash_node) {
 60		struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
 61				struct io_uring_cmd);
 62		struct file *file = req->file;
 63
 64		if (!cancel_all && req->tctx != tctx)
 65			continue;
 66
 67		if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
 68			file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL |
 69						   IO_URING_F_COMPLETE_DEFER);
 70			ret = true;
 71		}
 72	}
 73	io_submit_flush_completions(ctx);
 74	return ret;
 75}
 76
 77static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
 78		unsigned int issue_flags)
 79{
 80	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
 81	struct io_ring_ctx *ctx = req->ctx;
 82
 83	if (!(cmd->flags & IORING_URING_CMD_CANCELABLE))
 84		return;
 85
 86	cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
 87	io_ring_submit_lock(ctx, issue_flags);
 88	hlist_del(&req->hash_node);
 89	io_ring_submit_unlock(ctx, issue_flags);
 90}
 91
 92/*
 93 * Mark this command as concelable, then io_uring_try_cancel_uring_cmd()
 94 * will try to cancel this issued command by sending ->uring_cmd() with
 95 * issue_flags of IO_URING_F_CANCEL.
 96 *
 97 * The command is guaranteed to not be done when calling ->uring_cmd()
 98 * with IO_URING_F_CANCEL, but it is driver's responsibility to deal
 99 * with race between io_uring canceling and normal completion.
100 */
101void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
102		unsigned int issue_flags)
103{
104	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
105	struct io_ring_ctx *ctx = req->ctx;
106
107	if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
108		cmd->flags |= IORING_URING_CMD_CANCELABLE;
109		io_ring_submit_lock(ctx, issue_flags);
110		hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);
111		io_ring_submit_unlock(ctx, issue_flags);
112	}
113}
114EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
115
116void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
117			io_req_tw_func_t task_work_cb,
118			unsigned flags)
119{
120	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
121
122	if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
123		return;
124
125	req->io_task_work.func = task_work_cb;
126	__io_req_task_work_add(req, flags);
127}
128EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
129
130static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
131					  u64 extra1, u64 extra2)
132{
133	req->big_cqe.extra1 = extra1;
134	req->big_cqe.extra2 = extra2;
135}
136
137/*
138 * Called by consumers of io_uring_cmd, if they originally returned
139 * -EIOCBQUEUED upon receiving the command.
140 */
141void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
142		       unsigned issue_flags, bool is_cqe32)
143{
144	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
145
146	if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
147		return;
148
149	io_uring_cmd_del_cancelable(ioucmd, issue_flags);
150
151	if (ret < 0)
152		req_set_fail(req);
153
154	io_req_set_res(req, ret, 0);
155	if (is_cqe32) {
156		if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
157			req->cqe.flags |= IORING_CQE_F_32;
158		io_req_set_cqe32_extra(req, res2, 0);
159	}
160	io_req_uring_cleanup(req, issue_flags);
161	if (req->ctx->flags & IORING_SETUP_IOPOLL) {
162		/* order with io_iopoll_req_issued() checking ->iopoll_complete */
163		smp_store_release(&req->iopoll_completed, 1);
164	} else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
165		if (WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED))
166			return;
167		io_req_complete_defer(req);
168	} else {
169		req->io_task_work.func = io_req_task_complete;
170		io_req_task_work_add(req);
171	}
172}
173EXPORT_SYMBOL_GPL(__io_uring_cmd_done);
174
175int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
176{
177	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
178	struct io_async_cmd *ac;
179
180	if (sqe->__pad1)
181		return -EINVAL;
182
183	ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
184	if (ioucmd->flags & ~IORING_URING_CMD_MASK)
185		return -EINVAL;
186
187	if (ioucmd->flags & IORING_URING_CMD_FIXED) {
188		if (ioucmd->flags & IORING_URING_CMD_MULTISHOT)
189			return -EINVAL;
190		req->buf_index = READ_ONCE(sqe->buf_index);
191	}
192
193	if (!!(ioucmd->flags & IORING_URING_CMD_MULTISHOT) !=
194	    !!(req->flags & REQ_F_BUFFER_SELECT))
195		return -EINVAL;
196
197	ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
198
199	ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req);
200	if (!ac)
201		return -ENOMEM;
202	ioucmd->sqe = sqe;
203	return 0;
204}
205
206/*
207 * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
208 * slot.
209 */
210static inline size_t uring_sqe_size(struct io_kiocb *req)
211{
212	if (req->ctx->flags & IORING_SETUP_SQE128 ||
213	    req->opcode == IORING_OP_URING_CMD128)
214		return 2 * sizeof(struct io_uring_sqe);
215	return sizeof(struct io_uring_sqe);
216}
217
218void io_uring_cmd_sqe_copy(struct io_kiocb *req)
219{
220	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
221	struct io_async_cmd *ac = req->async_data;
222
223	/* Should not happen, as REQ_F_SQE_COPIED covers this */
224	if (WARN_ON_ONCE(ioucmd->sqe == ac->sqes))
225		return;
226	memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req));
227	ioucmd->sqe = ac->sqes;
228}
229
230int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
231{
232	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
233	struct io_ring_ctx *ctx = req->ctx;
234	struct file *file = req->file;
235	int ret;
236
237	if (!file->f_op->uring_cmd)
238		return -EOPNOTSUPP;
239
240	ret = security_uring_cmd(ioucmd);
241	if (ret)
242		return ret;
243
244	if (ctx->flags & IORING_SETUP_SQE128 ||
245	    req->opcode == IORING_OP_URING_CMD128)
246		issue_flags |= IO_URING_F_SQE128;
247	if (ctx->flags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED))
248		issue_flags |= IO_URING_F_CQE32;
249	if (io_is_compat(ctx))
250		issue_flags |= IO_URING_F_COMPAT;
251	if (ctx->flags & IORING_SETUP_IOPOLL) {
252		if (!file->f_op->uring_cmd_iopoll)
253			return -EOPNOTSUPP;
254		issue_flags |= IO_URING_F_IOPOLL;
255		req->iopoll_completed = 0;
256		if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) {
257			/* make sure every req only blocks once */
258			req->flags &= ~REQ_F_IOPOLL_STATE;
259			req->iopoll_start = ktime_get_ns();
260		}
261	}
262
263	ret = file->f_op->uring_cmd(ioucmd, issue_flags);
264	if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) {
265		if (ret >= 0)
266			return IOU_ISSUE_SKIP_COMPLETE;
267	}
268	if (ret == -EAGAIN) {
269		ioucmd->flags |= IORING_URING_CMD_REISSUE;
270		return ret;
271	}
272	if (ret == -EIOCBQUEUED)
273		return ret;
274	if (ret < 0)
275		req_set_fail(req);
276	io_req_uring_cleanup(req, issue_flags);
277	io_req_set_res(req, ret, 0);
278	return IOU_COMPLETE;
279}
280
281int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
282			      struct iov_iter *iter,
283			      struct io_uring_cmd *ioucmd,
284			      unsigned int issue_flags)
285{
286	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
287
288	if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
289		return -EINVAL;
290
291	return io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags);
292}
293EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);
294
295int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
296				  const struct iovec __user *uvec,
297				  size_t uvec_segs,
298				  int ddir, struct iov_iter *iter,
299				  unsigned issue_flags)
300{
301	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
302	struct io_async_cmd *ac = req->async_data;
303	int ret;
304
305	if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
306		return -EINVAL;
307
308	ret = io_prep_reg_iovec(req, &ac->vec, uvec, uvec_segs);
309	if (ret)
310		return ret;
311
312	return io_import_reg_vec(ddir, iter, req, &ac->vec, uvec_segs,
313				 issue_flags);
314}
315EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);
316
317void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
318{
319	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
320
321	io_req_queue_iowq(req);
322}
323
324int io_cmd_poll_multishot(struct io_uring_cmd *cmd,
325			  unsigned int issue_flags, __poll_t mask)
326{
327	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
328	int ret;
329
330	if (likely(req->flags & REQ_F_APOLL_MULTISHOT))
331		return 0;
332
333	req->flags |= REQ_F_APOLL_MULTISHOT;
334	mask &= ~EPOLLONESHOT;
335
336	ret = io_arm_apoll(req, issue_flags, mask);
337	return ret == IO_APOLL_OK ? -EIOCBQUEUED : -ECANCELED;
338}
339
340bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd,
341				   unsigned int issue_flags,
342				   struct io_uring_cqe cqe[2])
343{
344	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
345
346	if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_MULTISHOT)))
347		return false;
348	return io_req_post_cqe32(req, cqe);
349}
350
351/*
352 * Work with io_uring_mshot_cmd_post_cqe() together for committing the
353 * provided buffer upfront
354 */
355struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,
356					    unsigned buf_group, size_t *len,
357					    unsigned int issue_flags)
358{
359	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
360
361	if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
362		return (struct io_br_sel) { .val = -EINVAL };
363
364	if (WARN_ON_ONCE(!io_do_buffer_select(req)))
365		return (struct io_br_sel) { .val = -EINVAL };
366
367	return io_buffer_select(req, len, buf_group, issue_flags);
368}
369EXPORT_SYMBOL_GPL(io_uring_cmd_buffer_select);
370
371/*
372 * Return true if this multishot uring_cmd needs to be completed, otherwise
373 * the event CQE is posted successfully.
374 *
375 * This function must use `struct io_br_sel` returned from
376 * io_uring_cmd_buffer_select() for committing the buffer in the same
377 * uring_cmd submission context.
378 */
379bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
380				 struct io_br_sel *sel, unsigned int issue_flags)
381{
382	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
383	unsigned int cflags = 0;
384
385	if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
386		return true;
387
388	if (sel->val > 0) {
389		cflags = io_put_kbuf(req, sel->val, sel->buf_list);
390		if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE))
391			return false;
392	}
393
394	io_kbuf_recycle(req, sel->buf_list, issue_flags);
395	if (sel->val < 0)
396		req_set_fail(req);
397	io_req_set_res(req, sel->val, cflags);
398	return true;
399}
400EXPORT_SYMBOL_GPL(io_uring_mshot_cmd_post_cqe);