splice.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/kernel.h>
  3#include <linux/errno.h>
  4#include <linux/fs.h>
  5#include <linux/file.h>
  6#include <linux/mm.h>
  7#include <linux/slab.h>
  8#include <linux/namei.h>
  9#include <linux/io_uring.h>
 10#include <linux/splice.h>
 11
 12#include <uapi/linux/io_uring.h>
 13
 14#include "filetable.h"
 15#include "io_uring.h"
 16#include "splice.h"
 17
 18struct io_splice {
 19	struct file			*file_out;
 20	loff_t				off_out;
 21	loff_t				off_in;
 22	u64				len;
 23	int				splice_fd_in;
 24	unsigned int			flags;
 25	struct io_rsrc_node		*rsrc_node;
 26};
 27
 28static int __io_splice_prep(struct io_kiocb *req,
 29			    const struct io_uring_sqe *sqe)
 30{
 31	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 32	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
 33
 34	sp->len = READ_ONCE(sqe->len);
 35	sp->flags = READ_ONCE(sqe->splice_flags);
 36	if (unlikely(sp->flags & ~valid_flags))
 37		return -EINVAL;
 38	sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
 39	sp->rsrc_node = NULL;
 40	req->flags |= REQ_F_FORCE_ASYNC;
 41	return 0;
 42}
 43
 44int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 45{
 46	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
 47		return -EINVAL;
 48	return __io_splice_prep(req, sqe);
 49}
 50
 51void io_splice_cleanup(struct io_kiocb *req)
 52{
 53	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 54
 55	if (sp->rsrc_node)
 56		io_put_rsrc_node(req->ctx, sp->rsrc_node);
 57}
 58
 59static struct file *io_splice_get_file(struct io_kiocb *req,
 60				       unsigned int issue_flags)
 61{
 62	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 63	struct io_ring_ctx *ctx = req->ctx;
 64	struct io_rsrc_node *node;
 65	struct file *file = NULL;
 66
 67	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
 68		return io_file_get_normal(req, sp->splice_fd_in);
 69
 70	io_ring_submit_lock(ctx, issue_flags);
 71	node = io_rsrc_node_lookup(&ctx->file_table.data, sp->splice_fd_in);
 72	if (node) {
 73		node->refs++;
 74		sp->rsrc_node = node;
 75		file = io_slot_file(node);
 76		req->flags |= REQ_F_NEED_CLEANUP;
 77	}
 78	io_ring_submit_unlock(ctx, issue_flags);
 79	return file;
 80}
 81
 82int io_tee(struct io_kiocb *req, unsigned int issue_flags)
 83{
 84	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
 85	struct file *out = sp->file_out;
 86	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
 87	struct file *in;
 88	ssize_t ret = 0;
 89
 90	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
 91
 92	in = io_splice_get_file(req, issue_flags);
 93	if (!in) {
 94		ret = -EBADF;
 95		goto done;
 96	}
 97
 98	if (sp->len)
 99		ret = do_tee(in, out, sp->len, flags);
100
101	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
102		fput(in);
103done:
104	if (ret != sp->len)
105		req_set_fail(req);
106	io_req_set_res(req, ret, 0);
107	return IOU_COMPLETE;
108}
109
110int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
111{
112	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
113
114	sp->off_in = READ_ONCE(sqe->splice_off_in);
115	sp->off_out = READ_ONCE(sqe->off);
116	return __io_splice_prep(req, sqe);
117}
118
119int io_splice(struct io_kiocb *req, unsigned int issue_flags)
120{
121	struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
122	struct file *out = sp->file_out;
123	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
124	loff_t *poff_in, *poff_out;
125	struct file *in;
126	ssize_t ret = 0;
127
128	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
129
130	in = io_splice_get_file(req, issue_flags);
131	if (!in) {
132		ret = -EBADF;
133		goto done;
134	}
135
136	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
137	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
138
139	if (sp->len)
140		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
141
142	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
143		fput(in);
144done:
145	if (ret != sp->len)
146		req_set_fail(req);
147	io_req_set_res(req, ret, 0);
148	return IOU_COMPLETE;
149}