splice.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * "splice": joining two ropes together by interweaving their strands.
   4 *
   5 * This is the "extended pipe" functionality, where a pipe is used as
   6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
   7 * buffer that you can use to transfer data from one end to the other.
   8 *
   9 * The traditional unix read/write is extended with a "splice()" operation
  10 * that transfers data buffers to or from a pipe buffer.
  11 *
  12 * Named by Larry McVoy, original implementation from Linus, extended by
  13 * Jens to support splicing to files, network, direct splicing, etc and
  14 * fixing lots of bugs.
  15 *
  16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
  17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
  18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  19 *
  20 */
  21#include <linux/bvec.h>
  22#include <linux/fs.h>
  23#include <linux/file.h>
  24#include <linux/pagemap.h>
  25#include <linux/splice.h>
  26#include <linux/memcontrol.h>
  27#include <linux/mm_inline.h>
  28#include <linux/swap.h>
  29#include <linux/writeback.h>
  30#include <linux/export.h>
  31#include <linux/syscalls.h>
  32#include <linux/uio.h>
  33#include <linux/fsnotify.h>
  34#include <linux/security.h>
  35#include <linux/gfp.h>
  36#include <linux/net.h>
  37#include <linux/socket.h>
  38#include <linux/sched/signal.h>
  39
  40#include "internal.h"
  41
  42/*
  43 * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to
  44 * indicate they support non-blocking reads or writes, we must clear it
  45 * here if set to avoid blocking other users of this pipe if splice is
  46 * being done on it.
  47 */
  48static noinline void pipe_clear_nowait(struct file *file)
  49{
  50	fmode_t fmode = READ_ONCE(file->f_mode);
  51
  52	do {
  53		if (!(fmode & FMODE_NOWAIT))
  54			break;
  55	} while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));
  56}
  57
  58/*
  59 * Attempt to steal a page from a pipe buffer. This should perhaps go into
  60 * a vm helper function, it's already simplified quite a bit by the
  61 * addition of remove_mapping(). If success is returned, the caller may
  62 * attempt to reuse this page for another destination.
  63 */
  64static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
  65		struct pipe_buffer *buf)
  66{
  67	struct folio *folio = page_folio(buf->page);
  68	struct address_space *mapping;
  69
  70	folio_lock(folio);
  71
  72	mapping = folio_mapping(folio);
  73	if (mapping) {
  74		WARN_ON(!folio_test_uptodate(folio));
  75
  76		/*
  77		 * At least for ext2 with nobh option, we need to wait on
  78		 * writeback completing on this folio, since we'll remove it
  79		 * from the pagecache.  Otherwise truncate wont wait on the
  80		 * folio, allowing the disk blocks to be reused by someone else
  81		 * before we actually wrote our data to them. fs corruption
  82		 * ensues.
  83		 */
  84		folio_wait_writeback(folio);
  85
  86		if (!filemap_release_folio(folio, GFP_KERNEL))
  87			goto out_unlock;
  88
  89		/*
  90		 * If we succeeded in removing the mapping, set LRU flag
  91		 * and return good.
  92		 */
  93		if (remove_mapping(mapping, folio)) {
  94			buf->flags |= PIPE_BUF_FLAG_LRU;
  95			return true;
  96		}
  97	}
  98
  99	/*
 100	 * Raced with truncate or failed to remove folio from current
 101	 * address space, unlock and return failure.
 102	 */
 103out_unlock:
 104	folio_unlock(folio);
 105	return false;
 106}
 107
 108static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
 109					struct pipe_buffer *buf)
 110{
 111	put_page(buf->page);
 112	buf->flags &= ~PIPE_BUF_FLAG_LRU;
 113}
 114
 115/*
 116 * Check whether the contents of buf is OK to access. Since the content
 117 * is a page cache page, IO may be in flight.
 118 */
 119static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
 120				       struct pipe_buffer *buf)
 121{
 122	struct folio *folio = page_folio(buf->page);
 123	int err;
 124
 125	if (!folio_test_uptodate(folio)) {
 126		folio_lock(folio);
 127
 128		/*
 129		 * Folio got truncated/unhashed. This will cause a 0-byte
 130		 * splice, if this is the first page.
 131		 */
 132		if (!folio->mapping) {
 133			err = -ENODATA;
 134			goto error;
 135		}
 136
 137		/*
 138		 * Uh oh, read-error from disk.
 139		 */
 140		if (!folio_test_uptodate(folio)) {
 141			err = -EIO;
 142			goto error;
 143		}
 144
 145		/* Folio is ok after all, we are done */
 146		folio_unlock(folio);
 147	}
 148
 149	return 0;
 150error:
 151	folio_unlock(folio);
 152	return err;
 153}
 154
 155const struct pipe_buf_operations page_cache_pipe_buf_ops = {
 156	.confirm	= page_cache_pipe_buf_confirm,
 157	.release	= page_cache_pipe_buf_release,
 158	.try_steal	= page_cache_pipe_buf_try_steal,
 159	.get		= generic_pipe_buf_get,
 160};
 161
 162static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
 163		struct pipe_buffer *buf)
 164{
 165	if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
 166		return false;
 167
 168	buf->flags |= PIPE_BUF_FLAG_LRU;
 169	return generic_pipe_buf_try_steal(pipe, buf);
 170}
 171
 172static const struct pipe_buf_operations user_page_pipe_buf_ops = {
 173	.release	= page_cache_pipe_buf_release,
 174	.try_steal	= user_page_pipe_buf_try_steal,
 175	.get		= generic_pipe_buf_get,
 176};
 177
 178static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
 179{
 180	smp_mb();
 181	if (waitqueue_active(&pipe->rd_wait))
 182		wake_up_interruptible(&pipe->rd_wait);
 183	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 184}
 185
 186/**
 187 * splice_to_pipe - fill passed data into a pipe
 188 * @pipe:	pipe to fill
 189 * @spd:	data to fill
 190 *
 191 * Description:
 192 *    @spd contains a map of pages and len/offset tuples, along with
 193 *    the struct pipe_buf_operations associated with these pages. This
 194 *    function will link that data to the pipe.
 195 *
 196 */
 197ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
 198		       struct splice_pipe_desc *spd)
 199{
 200	unsigned int spd_pages = spd->nr_pages;
 201	unsigned int tail = pipe->tail;
 202	unsigned int head = pipe->head;
 203	ssize_t ret = 0;
 204	int page_nr = 0;
 205
 206	if (!spd_pages)
 207		return 0;
 208
 209	if (unlikely(!pipe->readers)) {
 210		send_sig(SIGPIPE, current, 0);
 211		ret = -EPIPE;
 212		goto out;
 213	}
 214
 215	while (!pipe_full(head, tail, pipe->max_usage)) {
 216		struct pipe_buffer *buf = pipe_buf(pipe, head);
 217
 218		buf->page = spd->pages[page_nr];
 219		buf->offset = spd->partial[page_nr].offset;
 220		buf->len = spd->partial[page_nr].len;
 221		buf->private = spd->partial[page_nr].private;
 222		buf->ops = spd->ops;
 223		buf->flags = 0;
 224
 225		head++;
 226		pipe->head = head;
 227		page_nr++;
 228		ret += buf->len;
 229
 230		if (!--spd->nr_pages)
 231			break;
 232	}
 233
 234	if (!ret)
 235		ret = -EAGAIN;
 236
 237out:
 238	while (page_nr < spd_pages)
 239		spd->spd_release(spd, page_nr++);
 240
 241	return ret;
 242}
 243EXPORT_SYMBOL_GPL(splice_to_pipe);
 244
 245ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
 246{
 247	unsigned int head = pipe->head;
 248	unsigned int tail = pipe->tail;
 249	int ret;
 250
 251	if (unlikely(!pipe->readers)) {
 252		send_sig(SIGPIPE, current, 0);
 253		ret = -EPIPE;
 254	} else if (pipe_full(head, tail, pipe->max_usage)) {
 255		ret = -EAGAIN;
 256	} else {
 257		*pipe_buf(pipe, head) = *buf;
 258		pipe->head = head + 1;
 259		return buf->len;
 260	}
 261	pipe_buf_release(pipe, buf);
 262	return ret;
 263}
 264EXPORT_SYMBOL(add_to_pipe);
 265
 266/*
 267 * Check if we need to grow the arrays holding pages and partial page
 268 * descriptions.
 269 */
 270int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
 271{
 272	unsigned int max_usage = READ_ONCE(pipe->max_usage);
 273
 274	spd->nr_pages_max = max_usage;
 275	if (max_usage <= PIPE_DEF_BUFFERS)
 276		return 0;
 277
 278	spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
 279	spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
 280				     GFP_KERNEL);
 281
 282	if (spd->pages && spd->partial)
 283		return 0;
 284
 285	kfree(spd->pages);
 286	kfree(spd->partial);
 287	return -ENOMEM;
 288}
 289
 290void splice_shrink_spd(struct splice_pipe_desc *spd)
 291{
 292	if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
 293		return;
 294
 295	kfree(spd->pages);
 296	kfree(spd->partial);
 297}
 298
 299/**
 300 * copy_splice_read -  Copy data from a file and splice the copy into a pipe
 301 * @in: The file to read from
 302 * @ppos: Pointer to the file position to read from
 303 * @pipe: The pipe to splice into
 304 * @len: The amount to splice
 305 * @flags: The SPLICE_F_* flags
 306 *
 307 * This function allocates a bunch of pages sufficient to hold the requested
 308 * amount of data (but limited by the remaining pipe capacity), passes it to
 309 * the file's ->read_iter() to read into and then splices the used pages into
 310 * the pipe.
 311 *
 312 * Return: On success, the number of bytes read will be returned and *@ppos
 313 * will be updated if appropriate; 0 will be returned if there is no more data
 314 * to be read; -EAGAIN will be returned if the pipe had no space, and some
 315 * other negative error code will be returned on error.  A short read may occur
 316 * if the pipe has insufficient space, we reach the end of the data or we hit a
 317 * hole.
 318 */
 319ssize_t copy_splice_read(struct file *in, loff_t *ppos,
 320			 struct pipe_inode_info *pipe,
 321			 size_t len, unsigned int flags)
 322{
 323	struct iov_iter to;
 324	struct bio_vec *bv;
 325	struct kiocb kiocb;
 326	struct page **pages;
 327	ssize_t ret;
 328	size_t used, npages, chunk, remain, keep = 0;
 329	int i;
 330
 331	/* Work out how much data we can actually add into the pipe */
 332	used = pipe_buf_usage(pipe);
 333	npages = max_t(ssize_t, pipe->max_usage - used, 0);
 334	len = min_t(size_t, len, npages * PAGE_SIZE);
 335	npages = DIV_ROUND_UP(len, PAGE_SIZE);
 336
 337	bv = kzalloc(array_size(npages, sizeof(bv[0])) +
 338		     array_size(npages, sizeof(struct page *)), GFP_KERNEL);
 339	if (!bv)
 340		return -ENOMEM;
 341
 342	pages = (struct page **)(bv + npages);
 343	npages = alloc_pages_bulk(GFP_USER, npages, pages);
 344	if (!npages) {
 345		kfree(bv);
 346		return -ENOMEM;
 347	}
 348
 349	remain = len = min_t(size_t, len, npages * PAGE_SIZE);
 350
 351	for (i = 0; i < npages; i++) {
 352		chunk = min_t(size_t, PAGE_SIZE, remain);
 353		bv[i].bv_page = pages[i];
 354		bv[i].bv_offset = 0;
 355		bv[i].bv_len = chunk;
 356		remain -= chunk;
 357	}
 358
 359	/* Do the I/O */
 360	iov_iter_bvec(&to, ITER_DEST, bv, npages, len);
 361	init_sync_kiocb(&kiocb, in);
 362	kiocb.ki_pos = *ppos;
 363	ret = in->f_op->read_iter(&kiocb, &to);
 364
 365	if (ret > 0) {
 366		keep = DIV_ROUND_UP(ret, PAGE_SIZE);
 367		*ppos = kiocb.ki_pos;
 368	}
 369
 370	/*
 371	 * Callers of ->splice_read() expect -EAGAIN on "can't put anything in
 372	 * there", rather than -EFAULT.
 373	 */
 374	if (ret == -EFAULT)
 375		ret = -EAGAIN;
 376
 377	/* Free any pages that didn't get touched at all. */
 378	if (keep < npages)
 379		release_pages(pages + keep, npages - keep);
 380
 381	/* Push the remaining pages into the pipe. */
 382	remain = ret;
 383	for (i = 0; i < keep; i++) {
 384		struct pipe_buffer *buf = pipe_head_buf(pipe);
 385
 386		chunk = min_t(size_t, remain, PAGE_SIZE);
 387		*buf = (struct pipe_buffer) {
 388			.ops	= &default_pipe_buf_ops,
 389			.page	= bv[i].bv_page,
 390			.offset	= 0,
 391			.len	= chunk,
 392		};
 393		pipe->head++;
 394		remain -= chunk;
 395	}
 396
 397	kfree(bv);
 398	return ret;
 399}
 400EXPORT_SYMBOL(copy_splice_read);
 401
 402const struct pipe_buf_operations default_pipe_buf_ops = {
 403	.release	= generic_pipe_buf_release,
 404	.try_steal	= generic_pipe_buf_try_steal,
 405	.get		= generic_pipe_buf_get,
 406};
 407
 408/* Pipe buffer operations for a socket and similar. */
 409const struct pipe_buf_operations nosteal_pipe_buf_ops = {
 410	.release	= generic_pipe_buf_release,
 411	.get		= generic_pipe_buf_get,
 412};
 413EXPORT_SYMBOL(nosteal_pipe_buf_ops);
 414
 415static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
 416{
 417	smp_mb();
 418	if (waitqueue_active(&pipe->wr_wait))
 419		wake_up_interruptible(&pipe->wr_wait);
 420	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 421}
 422
 423/**
 424 * splice_from_pipe_feed - feed available data from a pipe to a file
 425 * @pipe:	pipe to splice from
 426 * @sd:		information to @actor
 427 * @actor:	handler that splices the data
 428 *
 429 * Description:
 430 *    This function loops over the pipe and calls @actor to do the
 431 *    actual moving of a single struct pipe_buffer to the desired
 432 *    destination.  It returns when there's no more buffers left in
 433 *    the pipe or if the requested number of bytes (@sd->total_len)
 434 *    have been copied.  It returns a positive number (one) if the
 435 *    pipe needs to be filled with more data, zero if the required
 436 *    number of bytes have been copied and -errno on error.
 437 *
 438 *    This, together with splice_from_pipe_{begin,end,next}, may be
 439 *    used to implement the functionality of __splice_from_pipe() when
 440 *    locking is required around copying the pipe buffers to the
 441 *    destination.
 442 */
 443static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
 444			  splice_actor *actor)
 445{
 446	unsigned int head = pipe->head;
 447	unsigned int tail = pipe->tail;
 448	int ret;
 449
 450	while (!pipe_empty(head, tail)) {
 451		struct pipe_buffer *buf = pipe_buf(pipe, tail);
 452
 453		sd->len = buf->len;
 454		if (sd->len > sd->total_len)
 455			sd->len = sd->total_len;
 456
 457		ret = pipe_buf_confirm(pipe, buf);
 458		if (unlikely(ret)) {
 459			if (ret == -ENODATA)
 460				ret = 0;
 461			return ret;
 462		}
 463
 464		ret = actor(pipe, buf, sd);
 465		if (ret <= 0)
 466			return ret;
 467
 468		buf->offset += ret;
 469		buf->len -= ret;
 470
 471		sd->num_spliced += ret;
 472		sd->len -= ret;
 473		sd->pos += ret;
 474		sd->total_len -= ret;
 475
 476		if (!buf->len) {
 477			pipe_buf_release(pipe, buf);
 478			tail++;
 479			pipe->tail = tail;
 480			if (pipe->files)
 481				sd->need_wakeup = true;
 482		}
 483
 484		if (!sd->total_len)
 485			return 0;
 486	}
 487
 488	return 1;
 489}
 490
 491/* We know we have a pipe buffer, but maybe it's empty? */
 492static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
 493{
 494	unsigned int tail = pipe->tail;
 495	struct pipe_buffer *buf = pipe_buf(pipe, tail);
 496
 497	if (unlikely(!buf->len)) {
 498		pipe_buf_release(pipe, buf);
 499		pipe->tail = tail+1;
 500		return true;
 501	}
 502
 503	return false;
 504}
 505
 506/**
 507 * splice_from_pipe_next - wait for some data to splice from
 508 * @pipe:	pipe to splice from
 509 * @sd:		information about the splice operation
 510 *
 511 * Description:
 512 *    This function will wait for some data and return a positive
 513 *    value (one) if pipe buffers are available.  It will return zero
 514 *    or -errno if no more data needs to be spliced.
 515 */
 516static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
 517{
 518	/*
 519	 * Check for signal early to make process killable when there are
 520	 * always buffers available
 521	 */
 522	if (signal_pending(current))
 523		return -ERESTARTSYS;
 524
 525repeat:
 526	while (pipe_is_empty(pipe)) {
 527		if (!pipe->writers)
 528			return 0;
 529
 530		if (sd->num_spliced)
 531			return 0;
 532
 533		if (sd->flags & SPLICE_F_NONBLOCK)
 534			return -EAGAIN;
 535
 536		if (signal_pending(current))
 537			return -ERESTARTSYS;
 538
 539		if (sd->need_wakeup) {
 540			wakeup_pipe_writers(pipe);
 541			sd->need_wakeup = false;
 542		}
 543
 544		pipe_wait_readable(pipe);
 545	}
 546
 547	if (eat_empty_buffer(pipe))
 548		goto repeat;
 549
 550	return 1;
 551}
 552
 553/**
 554 * splice_from_pipe_begin - start splicing from pipe
 555 * @sd:		information about the splice operation
 556 *
 557 * Description:
 558 *    This function should be called before a loop containing
 559 *    splice_from_pipe_next() and splice_from_pipe_feed() to
 560 *    initialize the necessary fields of @sd.
 561 */
 562static void splice_from_pipe_begin(struct splice_desc *sd)
 563{
 564	sd->num_spliced = 0;
 565	sd->need_wakeup = false;
 566}
 567
 568/**
 569 * splice_from_pipe_end - finish splicing from pipe
 570 * @pipe:	pipe to splice from
 571 * @sd:		information about the splice operation
 572 *
 573 * Description:
 574 *    This function will wake up pipe writers if necessary.  It should
 575 *    be called after a loop containing splice_from_pipe_next() and
 576 *    splice_from_pipe_feed().
 577 */
 578static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
 579{
 580	if (sd->need_wakeup)
 581		wakeup_pipe_writers(pipe);
 582}
 583
 584/**
 585 * __splice_from_pipe - splice data from a pipe to given actor
 586 * @pipe:	pipe to splice from
 587 * @sd:		information to @actor
 588 * @actor:	handler that splices the data
 589 *
 590 * Description:
 591 *    This function does little more than loop over the pipe and call
 592 *    @actor to do the actual moving of a single struct pipe_buffer to
 593 *    the desired destination. See pipe_to_file, pipe_to_sendmsg, or
 594 *    pipe_to_user.
 595 *
 596 */
 597ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
 598			   splice_actor *actor)
 599{
 600	int ret;
 601
 602	splice_from_pipe_begin(sd);
 603	do {
 604		cond_resched();
 605		ret = splice_from_pipe_next(pipe, sd);
 606		if (ret > 0)
 607			ret = splice_from_pipe_feed(pipe, sd, actor);
 608	} while (ret > 0);
 609	splice_from_pipe_end(pipe, sd);
 610
 611	return sd->num_spliced ? sd->num_spliced : ret;
 612}
 613EXPORT_SYMBOL(__splice_from_pipe);
 614
 615/**
 616 * splice_from_pipe - splice data from a pipe to a file
 617 * @pipe:	pipe to splice from
 618 * @out:	file to splice to
 619 * @ppos:	position in @out
 620 * @len:	how many bytes to splice
 621 * @flags:	splice modifier flags
 622 * @actor:	handler that splices the data
 623 *
 624 * Description:
 625 *    See __splice_from_pipe. This function locks the pipe inode,
 626 *    otherwise it's identical to __splice_from_pipe().
 627 *
 628 */
 629ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
 630			 loff_t *ppos, size_t len, unsigned int flags,
 631			 splice_actor *actor)
 632{
 633	ssize_t ret;
 634	struct splice_desc sd = {
 635		.total_len = len,
 636		.flags = flags,
 637		.pos = *ppos,
 638		.u.file = out,
 639	};
 640
 641	pipe_lock(pipe);
 642	ret = __splice_from_pipe(pipe, &sd, actor);
 643	pipe_unlock(pipe);
 644
 645	return ret;
 646}
 647
 648/**
 649 * iter_file_splice_write - splice data from a pipe to a file
 650 * @pipe:	pipe info
 651 * @out:	file to write to
 652 * @ppos:	position in @out
 653 * @len:	number of bytes to splice
 654 * @flags:	splice modifier flags
 655 *
 656 * Description:
 657 *    Will either move or copy pages (determined by @flags options) from
 658 *    the given pipe inode to the given file.
 659 *    This one is ->write_iter-based.
 660 *
 661 */
 662ssize_t
 663iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
 664			  loff_t *ppos, size_t len, unsigned int flags)
 665{
 666	struct splice_desc sd = {
 667		.total_len = len,
 668		.flags = flags,
 669		.pos = *ppos,
 670		.u.file = out,
 671	};
 672	int nbufs = pipe->max_usage;
 673	struct bio_vec *array;
 674	ssize_t ret;
 675
 676	if (!out->f_op->write_iter)
 677		return -EINVAL;
 678
 679	array = kcalloc(nbufs, sizeof(struct bio_vec), GFP_KERNEL);
 680	if (unlikely(!array))
 681		return -ENOMEM;
 682
 683	pipe_lock(pipe);
 684
 685	splice_from_pipe_begin(&sd);
 686	while (sd.total_len) {
 687		struct kiocb kiocb;
 688		struct iov_iter from;
 689		unsigned int head, tail;
 690		size_t left;
 691		int n;
 692
 693		ret = splice_from_pipe_next(pipe, &sd);
 694		if (ret <= 0)
 695			break;
 696
 697		if (unlikely(nbufs < pipe->max_usage)) {
 698			kfree(array);
 699			nbufs = pipe->max_usage;
 700			array = kcalloc(nbufs, sizeof(struct bio_vec),
 701					GFP_KERNEL);
 702			if (!array) {
 703				ret = -ENOMEM;
 704				break;
 705			}
 706		}
 707
 708		head = pipe->head;
 709		tail = pipe->tail;
 710
 711		/* build the vector */
 712		left = sd.total_len;
 713		for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
 714			struct pipe_buffer *buf = pipe_buf(pipe, tail);
 715			size_t this_len = buf->len;
 716
 717			/* zero-length bvecs are not supported, skip them */
 718			if (!this_len)
 719				continue;
 720			this_len = min(this_len, left);
 721
 722			ret = pipe_buf_confirm(pipe, buf);
 723			if (unlikely(ret)) {
 724				if (ret == -ENODATA)
 725					ret = 0;
 726				goto done;
 727			}
 728
 729			bvec_set_page(&array[n], buf->page, this_len,
 730				      buf->offset);
 731			left -= this_len;
 732			n++;
 733		}
 734
 735		iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);
 736		init_sync_kiocb(&kiocb, out);
 737		kiocb.ki_pos = sd.pos;
 738		ret = out->f_op->write_iter(&kiocb, &from);
 739		sd.pos = kiocb.ki_pos;
 740		if (ret <= 0)
 741			break;
 742		WARN_ONCE(ret > sd.total_len - left,
 743			  "Splice Exceeded! ret=%zd tot=%zu left=%zu\n",
 744			  ret, sd.total_len, left);
 745
 746		sd.num_spliced += ret;
 747		sd.total_len -= ret;
 748		*ppos = sd.pos;
 749
 750		/* dismiss the fully eaten buffers, adjust the partial one */
 751		tail = pipe->tail;
 752		while (ret) {
 753			struct pipe_buffer *buf = pipe_buf(pipe, tail);
 754			if (ret >= buf->len) {
 755				ret -= buf->len;
 756				buf->len = 0;
 757				pipe_buf_release(pipe, buf);
 758				tail++;
 759				pipe->tail = tail;
 760				if (pipe->files)
 761					sd.need_wakeup = true;
 762			} else {
 763				buf->offset += ret;
 764				buf->len -= ret;
 765				ret = 0;
 766			}
 767		}
 768	}
 769done:
 770	kfree(array);
 771	splice_from_pipe_end(pipe, &sd);
 772
 773	pipe_unlock(pipe);
 774
 775	if (sd.num_spliced)
 776		ret = sd.num_spliced;
 777
 778	return ret;
 779}
 780
 781EXPORT_SYMBOL(iter_file_splice_write);
 782
 783#ifdef CONFIG_NET
 784/**
 785 * splice_to_socket - splice data from a pipe to a socket
 786 * @pipe:	pipe to splice from
 787 * @out:	socket to write to
 788 * @ppos:	position in @out
 789 * @len:	number of bytes to splice
 790 * @flags:	splice modifier flags
 791 *
 792 * Description:
 793 *    Will send @len bytes from the pipe to a network socket. No data copying
 794 *    is involved.
 795 *
 796 */
 797ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out,
 798			 loff_t *ppos, size_t len, unsigned int flags)
 799{
 800	struct socket *sock = sock_from_file(out);
 801	struct bio_vec bvec[16];
 802	struct msghdr msg = {};
 803	ssize_t ret = 0;
 804	size_t spliced = 0;
 805	bool need_wakeup = false;
 806
 807	pipe_lock(pipe);
 808
 809	while (len > 0) {
 810		unsigned int head, tail, bc = 0;
 811		size_t remain = len;
 812
 813		/*
 814		 * Check for signal early to make process killable when there
 815		 * are always buffers available
 816		 */
 817		ret = -ERESTARTSYS;
 818		if (signal_pending(current))
 819			break;
 820
 821		while (pipe_is_empty(pipe)) {
 822			ret = 0;
 823			if (!pipe->writers)
 824				goto out;
 825
 826			if (spliced)
 827				goto out;
 828
 829			ret = -EAGAIN;
 830			if (flags & SPLICE_F_NONBLOCK)
 831				goto out;
 832
 833			ret = -ERESTARTSYS;
 834			if (signal_pending(current))
 835				goto out;
 836
 837			if (need_wakeup) {
 838				wakeup_pipe_writers(pipe);
 839				need_wakeup = false;
 840			}
 841
 842			pipe_wait_readable(pipe);
 843		}
 844
 845		head = pipe->head;
 846		tail = pipe->tail;
 847
 848		while (!pipe_empty(head, tail)) {
 849			struct pipe_buffer *buf = pipe_buf(pipe, tail);
 850			size_t seg;
 851
 852			if (!buf->len) {
 853				tail++;
 854				continue;
 855			}
 856
 857			seg = min_t(size_t, remain, buf->len);
 858
 859			ret = pipe_buf_confirm(pipe, buf);
 860			if (unlikely(ret)) {
 861				if (ret == -ENODATA)
 862					ret = 0;
 863				break;
 864			}
 865
 866			bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset);
 867			remain -= seg;
 868			if (remain == 0 || bc >= ARRAY_SIZE(bvec))
 869				break;
 870			tail++;
 871		}
 872
 873		if (!bc)
 874			break;
 875
 876		msg.msg_flags = MSG_SPLICE_PAGES;
 877		if (flags & SPLICE_F_MORE)
 878			msg.msg_flags |= MSG_MORE;
 879		if (remain && pipe_occupancy(pipe->head, tail) > 0)
 880			msg.msg_flags |= MSG_MORE;
 881		if (out->f_flags & O_NONBLOCK)
 882			msg.msg_flags |= MSG_DONTWAIT;
 883
 884		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc,
 885			      len - remain);
 886		ret = sock_sendmsg(sock, &msg);
 887		if (ret <= 0)
 888			break;
 889
 890		spliced += ret;
 891		len -= ret;
 892		tail = pipe->tail;
 893		while (ret > 0) {
 894			struct pipe_buffer *buf = pipe_buf(pipe, tail);
 895			size_t seg = min_t(size_t, ret, buf->len);
 896
 897			buf->offset += seg;
 898			buf->len -= seg;
 899			ret -= seg;
 900
 901			if (!buf->len) {
 902				pipe_buf_release(pipe, buf);
 903				tail++;
 904			}
 905		}
 906
 907		if (tail != pipe->tail) {
 908			pipe->tail = tail;
 909			if (pipe->files)
 910				need_wakeup = true;
 911		}
 912	}
 913
 914out:
 915	pipe_unlock(pipe);
 916	if (need_wakeup)
 917		wakeup_pipe_writers(pipe);
 918	return spliced ?: ret;
 919}
 920#endif
 921
 922static int warn_unsupported(struct file *file, const char *op)
 923{
 924	pr_debug_ratelimited(
 925		"splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
 926		op, file, current->pid, current->comm);
 927	return -EINVAL;
 928}
 929
 930/*
 931 * Attempt to initiate a splice from pipe to file.
 932 */
 933static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out,
 934			      loff_t *ppos, size_t len, unsigned int flags)
 935{
 936	if (unlikely(!out->f_op->splice_write))
 937		return warn_unsupported(out, "write");
 938	return out->f_op->splice_write(pipe, out, ppos, len, flags);
 939}
 940
 941/*
 942 * Indicate to the caller that there was a premature EOF when reading from the
 943 * source and the caller didn't indicate they would be sending more data after
 944 * this.
 945 */
 946static void do_splice_eof(struct splice_desc *sd)
 947{
 948	if (sd->splice_eof)
 949		sd->splice_eof(sd);
 950}
 951
 952/*
 953 * Callers already called rw_verify_area() on the entire range.
 954 * No need to call it for sub ranges.
 955 */
 956static ssize_t do_splice_read(struct file *in, loff_t *ppos,
 957			      struct pipe_inode_info *pipe, size_t len,
 958			      unsigned int flags)
 959{
 960	unsigned int p_space;
 961
 962	if (unlikely(!(in->f_mode & FMODE_READ)))
 963		return -EBADF;
 964	if (!len)
 965		return 0;
 966
 967	/* Don't try to read more the pipe has space for. */
 968	p_space = pipe->max_usage - pipe_buf_usage(pipe);
 969	len = min_t(size_t, len, p_space << PAGE_SHIFT);
 970
 971	if (unlikely(len > MAX_RW_COUNT))
 972		len = MAX_RW_COUNT;
 973
 974	if (unlikely(!in->f_op->splice_read))
 975		return warn_unsupported(in, "read");
 976	/*
 977	 * O_DIRECT and DAX don't deal with the pagecache, so we allocate a
 978	 * buffer, copy into it and splice that into the pipe.
 979	 */
 980	if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host))
 981		return copy_splice_read(in, ppos, pipe, len, flags);
 982	return in->f_op->splice_read(in, ppos, pipe, len, flags);
 983}
 984
 985/**
 986 * vfs_splice_read - Read data from a file and splice it into a pipe
 987 * @in:		File to splice from
 988 * @ppos:	Input file offset
 989 * @pipe:	Pipe to splice to
 990 * @len:	Number of bytes to splice
 991 * @flags:	Splice modifier flags (SPLICE_F_*)
 992 *
 993 * Splice the requested amount of data from the input file to the pipe.  This
 994 * is synchronous as the caller must hold the pipe lock across the entire
 995 * operation.
 996 *
 997 * If successful, it returns the amount of data spliced, 0 if it hit the EOF or
 998 * a hole and a negative error code otherwise.
 999 */
1000ssize_t vfs_splice_read(struct file *in, loff_t *ppos,
1001			struct pipe_inode_info *pipe, size_t len,
1002			unsigned int flags)
1003{
1004	ssize_t ret;
1005
1006	ret = rw_verify_area(READ, in, ppos, len);
1007	if (unlikely(ret < 0))
1008		return ret;
1009
1010	return do_splice_read(in, ppos, pipe, len, flags);
1011}
1012EXPORT_SYMBOL_GPL(vfs_splice_read);
1013
1014/**
1015 * splice_direct_to_actor - splices data directly between two non-pipes
1016 * @in:		file to splice from
1017 * @sd:		actor information on where to splice to
1018 * @actor:	handles the data splicing
1019 *
1020 * Description:
1021 *    This is a special case helper to splice directly between two
1022 *    points, without requiring an explicit pipe. Internally an allocated
1023 *    pipe is cached in the process, and reused during the lifetime of
1024 *    that process.
1025 *
1026 */
1027ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1028			       splice_direct_actor *actor)
1029{
1030	struct pipe_inode_info *pipe;
1031	ssize_t ret, bytes;
1032	size_t len;
1033	int i, flags, more;
1034
1035	/*
1036	 * We require the input to be seekable, as we don't want to randomly
1037	 * drop data for eg socket -> socket splicing. Use the piped splicing
1038	 * for that!
1039	 */
1040	if (unlikely(!(in->f_mode & FMODE_LSEEK)))
1041		return -EINVAL;
1042
1043	/*
1044	 * neither in nor out is a pipe, setup an internal pipe attached to
1045	 * 'out' and transfer the wanted data from 'in' to 'out' through that
1046	 */
1047	pipe = current->splice_pipe;
1048	if (unlikely(!pipe)) {
1049		pipe = alloc_pipe_info();
1050		if (!pipe)
1051			return -ENOMEM;
1052
1053		/*
1054		 * We don't have an immediate reader, but we'll read the stuff
1055		 * out of the pipe right after the splice_to_pipe(). So set
1056		 * PIPE_READERS appropriately.
1057		 */
1058		pipe->readers = 1;
1059
1060		current->splice_pipe = pipe;
1061	}
1062
1063	/*
1064	 * Do the splice.
1065	 */
1066	bytes = 0;
1067	len = sd->total_len;
1068
1069	/* Don't block on output, we have to drain the direct pipe. */
1070	flags = sd->flags;
1071	sd->flags &= ~SPLICE_F_NONBLOCK;
1072
1073	/*
1074	 * We signal MORE until we've read sufficient data to fulfill the
1075	 * request and we keep signalling it if the caller set it.
1076	 */
1077	more = sd->flags & SPLICE_F_MORE;
1078	sd->flags |= SPLICE_F_MORE;
1079
1080	WARN_ON_ONCE(!pipe_is_empty(pipe));
1081
1082	while (len) {
1083		size_t read_len;
1084		loff_t pos = sd->pos, prev_pos = pos;
1085
1086		ret = do_splice_read(in, &pos, pipe, len, flags);
1087		if (unlikely(ret <= 0))
1088			goto read_failure;
1089
1090		read_len = ret;
1091		sd->total_len = read_len;
1092
1093		/*
1094		 * If we now have sufficient data to fulfill the request then
1095		 * we clear SPLICE_F_MORE if it was not set initially.
1096		 */
1097		if (read_len >= len && !more)
1098			sd->flags &= ~SPLICE_F_MORE;
1099
1100		/*
1101		 * NOTE: nonblocking mode only applies to the input. We
1102		 * must not do the output in nonblocking mode as then we
1103		 * could get stuck data in the internal pipe:
1104		 */
1105		ret = actor(pipe, sd);
1106		if (unlikely(ret <= 0)) {
1107			sd->pos = prev_pos;
1108			goto out_release;
1109		}
1110
1111		bytes += ret;
1112		len -= ret;
1113		sd->pos = pos;
1114
1115		if (ret < read_len) {
1116			sd->pos = prev_pos + ret;
1117			goto out_release;
1118		}
1119	}
1120
1121done:
1122	pipe->tail = pipe->head = 0;
1123	file_accessed(in);
1124	return bytes;
1125
1126read_failure:
1127	/*
1128	 * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that
1129	 * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a
1130	 * "->splice_in()" that returned EOF (ie zero) *and* we have sent at
1131	 * least 1 byte *then* we will also do the ->splice_eof() call.
1132	 */
1133	if (ret == 0 && !more && len > 0 && bytes)
1134		do_splice_eof(sd);
1135out_release:
1136	/*
1137	 * If we did an incomplete transfer we must release
1138	 * the pipe buffers in question:
1139	 */
1140	for (i = 0; i < pipe->ring_size; i++) {
1141		struct pipe_buffer *buf = &pipe->bufs[i];
1142
1143		if (buf->ops)
1144			pipe_buf_release(pipe, buf);
1145	}
1146
1147	if (!bytes)
1148		bytes = ret;
1149
1150	goto done;
1151}
1152EXPORT_SYMBOL(splice_direct_to_actor);
1153
1154static int direct_splice_actor(struct pipe_inode_info *pipe,
1155			       struct splice_desc *sd)
1156{
1157	struct file *file = sd->u.file;
1158	long ret;
1159
1160	file_start_write(file);
1161	ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
1162	file_end_write(file);
1163	return ret;
1164}
1165
1166static int splice_file_range_actor(struct pipe_inode_info *pipe,
1167					struct splice_desc *sd)
1168{
1169	struct file *file = sd->u.file;
1170
1171	return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
1172}
1173
1174static void direct_file_splice_eof(struct splice_desc *sd)
1175{
1176	struct file *file = sd->u.file;
1177
1178	if (file->f_op->splice_eof)
1179		file->f_op->splice_eof(file);
1180}
1181
1182static ssize_t do_splice_direct_actor(struct file *in, loff_t *ppos,
1183				      struct file *out, loff_t *opos,
1184				      size_t len, unsigned int flags,
1185				      splice_direct_actor *actor)
1186{
1187	struct splice_desc sd = {
1188		.len		= len,
1189		.total_len	= len,
1190		.flags		= flags,
1191		.pos		= *ppos,
1192		.u.file		= out,
1193		.splice_eof	= direct_file_splice_eof,
1194		.opos		= opos,
1195	};
1196	ssize_t ret;
1197
1198	if (unlikely(!(out->f_mode & FMODE_WRITE)))
1199		return -EBADF;
1200
1201	if (unlikely(out->f_flags & O_APPEND))
1202		return -EINVAL;
1203
1204	ret = splice_direct_to_actor(in, &sd, actor);
1205	if (ret > 0)
1206		*ppos = sd.pos;
1207
1208	return ret;
1209}
1210/**
1211 * do_splice_direct - splices data directly between two files
1212 * @in:		file to splice from
1213 * @ppos:	input file offset
1214 * @out:	file to splice to
1215 * @opos:	output file offset
1216 * @len:	number of bytes to splice
1217 * @flags:	splice modifier flags
1218 *
1219 * Description:
1220 *    For use by do_sendfile(). splice can easily emulate sendfile, but
1221 *    doing it in the application would incur an extra system call
1222 *    (splice in + splice out, as compared to just sendfile()). So this helper
1223 *    can splice directly through a process-private pipe.
1224 *
1225 * Callers already called rw_verify_area() on the entire range.
1226 */
1227ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1228			 loff_t *opos, size_t len, unsigned int flags)
1229{
1230	return do_splice_direct_actor(in, ppos, out, opos, len, flags,
1231				      direct_splice_actor);
1232}
1233EXPORT_SYMBOL(do_splice_direct);
1234
1235/**
1236 * splice_file_range - splices data between two files for copy_file_range()
1237 * @in:		file to splice from
1238 * @ppos:	input file offset
1239 * @out:	file to splice to
1240 * @opos:	output file offset
1241 * @len:	number of bytes to splice
1242 *
1243 * Description:
1244 *    For use by ->copy_file_range() methods.
1245 *    Like do_splice_direct(), but vfs_copy_file_range() already holds
1246 *    start_file_write() on @out file.
1247 *
1248 * Callers already called rw_verify_area() on the entire range.
1249 */
1250ssize_t splice_file_range(struct file *in, loff_t *ppos, struct file *out,
1251			  loff_t *opos, size_t len)
1252{
1253	lockdep_assert(file_write_started(out));
1254
1255	return do_splice_direct_actor(in, ppos, out, opos,
1256				      min_t(size_t, len, MAX_RW_COUNT),
1257				      0, splice_file_range_actor);
1258}
1259EXPORT_SYMBOL(splice_file_range);
1260
1261static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
1262{
1263	for (;;) {
1264		if (unlikely(!pipe->readers)) {
1265			send_sig(SIGPIPE, current, 0);
1266			return -EPIPE;
1267		}
1268		if (!pipe_is_full(pipe))
1269			return 0;
1270		if (flags & SPLICE_F_NONBLOCK)
1271			return -EAGAIN;
1272		if (signal_pending(current))
1273			return -ERESTARTSYS;
1274		pipe_wait_writable(pipe);
1275	}
1276}
1277
1278static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1279			       struct pipe_inode_info *opipe,
1280			       size_t len, unsigned int flags);
1281
1282ssize_t splice_file_to_pipe(struct file *in,
1283			    struct pipe_inode_info *opipe,
1284			    loff_t *offset,
1285			    size_t len, unsigned int flags)
1286{
1287	ssize_t ret;
1288
1289	pipe_lock(opipe);
1290	ret = wait_for_space(opipe, flags);
1291	if (!ret)
1292		ret = do_splice_read(in, offset, opipe, len, flags);
1293	pipe_unlock(opipe);
1294	if (ret > 0)
1295		wakeup_pipe_readers(opipe);
1296	return ret;
1297}
1298
1299/*
1300 * Determine where to splice to/from.
1301 */
1302ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out,
1303		  loff_t *off_out, size_t len, unsigned int flags)
1304{
1305	struct pipe_inode_info *ipipe;
1306	struct pipe_inode_info *opipe;
1307	loff_t offset;
1308	ssize_t ret;
1309
1310	if (unlikely(!(in->f_mode & FMODE_READ) ||
1311		     !(out->f_mode & FMODE_WRITE)))
1312		return -EBADF;
1313
1314	ipipe = get_pipe_info(in, true);
1315	opipe = get_pipe_info(out, true);
1316
1317	if (ipipe && opipe) {
1318		if (off_in || off_out)
1319			return -ESPIPE;
1320
1321		/* Splicing to self would be fun, but... */
1322		if (ipipe == opipe)
1323			return -EINVAL;
1324
1325		if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1326			flags |= SPLICE_F_NONBLOCK;
1327
1328		ret = splice_pipe_to_pipe(ipipe, opipe, len, flags);
1329	} else if (ipipe) {
1330		if (off_in)
1331			return -ESPIPE;
1332		if (off_out) {
1333			if (!(out->f_mode & FMODE_PWRITE))
1334				return -EINVAL;
1335			offset = *off_out;
1336		} else {
1337			offset = out->f_pos;
1338		}
1339
1340		if (unlikely(out->f_flags & O_APPEND))
1341			return -EINVAL;
1342
1343		ret = rw_verify_area(WRITE, out, &offset, len);
1344		if (unlikely(ret < 0))
1345			return ret;
1346
1347		if (in->f_flags & O_NONBLOCK)
1348			flags |= SPLICE_F_NONBLOCK;
1349
1350		file_start_write(out);
1351		ret = do_splice_from(ipipe, out, &offset, len, flags);
1352		file_end_write(out);
1353
1354		if (!off_out)
1355			out->f_pos = offset;
1356		else
1357			*off_out = offset;
1358	} else if (opipe) {
1359		if (off_out)
1360			return -ESPIPE;
1361		if (off_in) {
1362			if (!(in->f_mode & FMODE_PREAD))
1363				return -EINVAL;
1364			offset = *off_in;
1365		} else {
1366			offset = in->f_pos;
1367		}
1368
1369		ret = rw_verify_area(READ, in, &offset, len);
1370		if (unlikely(ret < 0))
1371			return ret;
1372
1373		if (out->f_flags & O_NONBLOCK)
1374			flags |= SPLICE_F_NONBLOCK;
1375
1376		ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
1377
1378		if (!off_in)
1379			in->f_pos = offset;
1380		else
1381			*off_in = offset;
1382	} else {
1383		ret = -EINVAL;
1384	}
1385
1386	if (ret > 0) {
1387		/*
1388		 * Generate modify out before access in:
1389		 * do_splice_from() may've already sent modify out,
1390		 * and this ensures the events get merged.
1391		 */
1392		fsnotify_modify(out);
1393		fsnotify_access(in);
1394	}
1395
1396	return ret;
1397}
1398
1399static ssize_t __do_splice(struct file *in, loff_t __user *off_in,
1400			   struct file *out, loff_t __user *off_out,
1401			   size_t len, unsigned int flags)
1402{
1403	struct pipe_inode_info *ipipe;
1404	struct pipe_inode_info *opipe;
1405	loff_t offset, *__off_in = NULL, *__off_out = NULL;
1406	ssize_t ret;
1407
1408	ipipe = get_pipe_info(in, true);
1409	opipe = get_pipe_info(out, true);
1410
1411	if (ipipe) {
1412		if (off_in)
1413			return -ESPIPE;
1414		pipe_clear_nowait(in);
1415	}
1416	if (opipe) {
1417		if (off_out)
1418			return -ESPIPE;
1419		pipe_clear_nowait(out);
1420	}
1421
1422	if (off_out) {
1423		if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1424			return -EFAULT;
1425		__off_out = &offset;
1426	}
1427	if (off_in) {
1428		if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1429			return -EFAULT;
1430		__off_in = &offset;
1431	}
1432
1433	ret = do_splice(in, __off_in, out, __off_out, len, flags);
1434	if (ret < 0)
1435		return ret;
1436
1437	if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
1438		return -EFAULT;
1439	if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
1440		return -EFAULT;
1441
1442	return ret;
1443}
1444
1445static ssize_t iter_to_pipe(struct iov_iter *from,
1446			    struct pipe_inode_info *pipe,
1447			    unsigned int flags)
1448{
1449	struct pipe_buffer buf = {
1450		.ops = &user_page_pipe_buf_ops,
1451		.flags = flags
1452	};
1453	size_t total = 0;
1454	ssize_t ret = 0;
1455
1456	while (iov_iter_count(from)) {
1457		struct page *pages[16];
1458		ssize_t left;
1459		size_t start;
1460		int i, n;
1461
1462		left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);
1463		if (left <= 0) {
1464			ret = left;
1465			break;
1466		}
1467
1468		n = DIV_ROUND_UP(left + start, PAGE_SIZE);
1469		for (i = 0; i < n; i++) {
1470			int size = min_t(int, left, PAGE_SIZE - start);
1471
1472			buf.page = pages[i];
1473			buf.offset = start;
1474			buf.len = size;
1475			ret = add_to_pipe(pipe, &buf);
1476			if (unlikely(ret < 0)) {
1477				iov_iter_revert(from, left);
1478				// this one got dropped by add_to_pipe()
1479				while (++i < n)
1480					put_page(pages[i]);
1481				goto out;
1482			}
1483			total += ret;
1484			left -= size;
1485			start = 0;
1486		}
1487	}
1488out:
1489	return total ? total : ret;
1490}
1491
1492static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1493			struct splice_desc *sd)
1494{
1495	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1496	return n == sd->len ? n : -EFAULT;
1497}
1498
1499/*
1500 * For lack of a better implementation, implement vmsplice() to userspace
1501 * as a simple copy of the pipe's pages to the user iov.
1502 */
1503static ssize_t vmsplice_to_user(struct file *file, struct iov_iter *iter,
1504				unsigned int flags)
1505{
1506	struct pipe_inode_info *pipe = get_pipe_info(file, true);
1507	struct splice_desc sd = {
1508		.total_len = iov_iter_count(iter),
1509		.flags = flags,
1510		.u.data = iter
1511	};
1512	ssize_t ret = 0;
1513
1514	if (!pipe)
1515		return -EBADF;
1516
1517	pipe_clear_nowait(file);
1518
1519	if (sd.total_len) {
1520		pipe_lock(pipe);
1521		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1522		pipe_unlock(pipe);
1523	}
1524
1525	if (ret > 0)
1526		fsnotify_access(file);
1527
1528	return ret;
1529}
1530
1531/*
1532 * vmsplice splices a user address range into a pipe. It can be thought of
1533 * as splice-from-memory, where the regular splice is splice-from-file (or
1534 * to file). In both cases the output is a pipe, naturally.
1535 */
1536static ssize_t vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1537				unsigned int flags)
1538{
1539	struct pipe_inode_info *pipe;
1540	ssize_t ret = 0;
1541	unsigned buf_flag = 0;
1542
1543	if (flags & SPLICE_F_GIFT)
1544		buf_flag = PIPE_BUF_FLAG_GIFT;
1545
1546	pipe = get_pipe_info(file, true);
1547	if (!pipe)
1548		return -EBADF;
1549
1550	pipe_clear_nowait(file);
1551
1552	pipe_lock(pipe);
1553	ret = wait_for_space(pipe, flags);
1554	if (!ret)
1555		ret = iter_to_pipe(iter, pipe, buf_flag);
1556	pipe_unlock(pipe);
1557	if (ret > 0) {
1558		wakeup_pipe_readers(pipe);
1559		fsnotify_modify(file);
1560	}
1561	return ret;
1562}
1563
1564/*
1565 * Note that vmsplice only really supports true splicing _from_ user memory
1566 * to a pipe, not the other way around. Splicing from user memory is a simple
1567 * operation that can be supported without any funky alignment restrictions
1568 * or nasty vm tricks. We simply map in the user memory and fill them into
1569 * a pipe. The reverse isn't quite as easy, though. There are two possible
1570 * solutions for that:
1571 *
1572 *	- memcpy() the data internally, at which point we might as well just
1573 *	  do a regular read() on the buffer anyway.
1574 *	- Lots of nasty vm tricks, that are neither fast nor flexible (it
1575 *	  has restriction limitations on both ends of the pipe).
1576 *
1577 * Currently we punt and implement it as a normal copy, see pipe_to_user().
1578 *
1579 */
1580SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
1581		unsigned long, nr_segs, unsigned int, flags)
1582{
1583	struct iovec iovstack[UIO_FASTIOV];
1584	struct iovec *iov = iovstack;
1585	struct iov_iter iter;
1586	ssize_t error;
1587	int type;
1588
1589	if (unlikely(flags & ~SPLICE_F_ALL))
1590		return -EINVAL;
1591
1592	CLASS(fd, f)(fd);
1593	if (fd_empty(f))
1594		return -EBADF;
1595	if (fd_file(f)->f_mode & FMODE_WRITE)
1596		type = ITER_SOURCE;
1597	else if (fd_file(f)->f_mode & FMODE_READ)
1598		type = ITER_DEST;
1599	else
1600		return -EBADF;
1601
1602	error = import_iovec(type, uiov, nr_segs,
1603			     ARRAY_SIZE(iovstack), &iov, &iter);
1604	if (error < 0)
1605		return error;
1606
1607	if (!iov_iter_count(&iter))
1608		error = 0;
1609	else if (type == ITER_SOURCE)
1610		error = vmsplice_to_pipe(fd_file(f), &iter, flags);
1611	else
1612		error = vmsplice_to_user(fd_file(f), &iter, flags);
1613
1614	kfree(iov);
1615	return error;
1616}
1617
1618SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1619		int, fd_out, loff_t __user *, off_out,
1620		size_t, len, unsigned int, flags)
1621{
1622	if (unlikely(!len))
1623		return 0;
1624
1625	if (unlikely(flags & ~SPLICE_F_ALL))
1626		return -EINVAL;
1627
1628	CLASS(fd, in)(fd_in);
1629	if (fd_empty(in))
1630		return -EBADF;
1631
1632	CLASS(fd, out)(fd_out);
1633	if (fd_empty(out))
1634		return -EBADF;
1635
1636	return __do_splice(fd_file(in), off_in, fd_file(out), off_out,
1637					    len, flags);
1638}
1639
1640/*
1641 * Make sure there's data to read. Wait for input if we can, otherwise
1642 * return an appropriate error.
1643 */
1644static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1645{
1646	int ret;
1647
1648	/*
1649	 * Check the pipe occupancy without the inode lock first. This function
1650	 * is speculative anyways, so missing one is ok.
1651	 */
1652	if (!pipe_is_empty(pipe))
1653		return 0;
1654
1655	ret = 0;
1656	pipe_lock(pipe);
1657
1658	while (pipe_is_empty(pipe)) {
1659		if (signal_pending(current)) {
1660			ret = -ERESTARTSYS;
1661			break;
1662		}
1663		if (!pipe->writers)
1664			break;
1665		if (flags & SPLICE_F_NONBLOCK) {
1666			ret = -EAGAIN;
1667			break;
1668		}
1669		pipe_wait_readable(pipe);
1670	}
1671
1672	pipe_unlock(pipe);
1673	return ret;
1674}
1675
1676/*
1677 * Make sure there's writeable room. Wait for room if we can, otherwise
1678 * return an appropriate error.
1679 */
1680static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1681{
1682	int ret;
1683
1684	/*
1685	 * Check pipe occupancy without the inode lock first. This function
1686	 * is speculative anyways, so missing one is ok.
1687	 */
1688	if (!pipe_is_full(pipe))
1689		return 0;
1690
1691	ret = 0;
1692	pipe_lock(pipe);
1693
1694	while (pipe_is_full(pipe)) {
1695		if (!pipe->readers) {
1696			send_sig(SIGPIPE, current, 0);
1697			ret = -EPIPE;
1698			break;
1699		}
1700		if (flags & SPLICE_F_NONBLOCK) {
1701			ret = -EAGAIN;
1702			break;
1703		}
1704		if (signal_pending(current)) {
1705			ret = -ERESTARTSYS;
1706			break;
1707		}
1708		pipe_wait_writable(pipe);
1709	}
1710
1711	pipe_unlock(pipe);
1712	return ret;
1713}
1714
1715/*
1716 * Splice contents of ipipe to opipe.
1717 */
1718static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1719			       struct pipe_inode_info *opipe,
1720			       size_t len, unsigned int flags)
1721{
1722	struct pipe_buffer *ibuf, *obuf;
1723	unsigned int i_head, o_head;
1724	unsigned int i_tail, o_tail;
1725	int ret = 0;
1726	bool input_wakeup = false;
1727
1728
1729retry:
1730	ret = ipipe_prep(ipipe, flags);
1731	if (ret)
1732		return ret;
1733
1734	ret = opipe_prep(opipe, flags);
1735	if (ret)
1736		return ret;
1737
1738	/*
1739	 * Potential ABBA deadlock, work around it by ordering lock
1740	 * grabbing by pipe info address. Otherwise two different processes
1741	 * could deadlock (one doing tee from A -> B, the other from B -> A).
1742	 */
1743	pipe_double_lock(ipipe, opipe);
1744
1745	i_tail = ipipe->tail;
1746	o_head = opipe->head;
1747
1748	do {
1749		size_t o_len;
1750
1751		if (!opipe->readers) {
1752			send_sig(SIGPIPE, current, 0);
1753			if (!ret)
1754				ret = -EPIPE;
1755			break;
1756		}
1757
1758		i_head = ipipe->head;
1759		o_tail = opipe->tail;
1760
1761		if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1762			break;
1763
1764		/*
1765		 * Cannot make any progress, because either the input
1766		 * pipe is empty or the output pipe is full.
1767		 */
1768		if (pipe_empty(i_head, i_tail) ||
1769		    pipe_full(o_head, o_tail, opipe->max_usage)) {
1770			/* Already processed some buffers, break */
1771			if (ret)
1772				break;
1773
1774			if (flags & SPLICE_F_NONBLOCK) {
1775				ret = -EAGAIN;
1776				break;
1777			}
1778
1779			/*
1780			 * We raced with another reader/writer and haven't
1781			 * managed to process any buffers.  A zero return
1782			 * value means EOF, so retry instead.
1783			 */
1784			pipe_unlock(ipipe);
1785			pipe_unlock(opipe);
1786			goto retry;
1787		}
1788
1789		ibuf = pipe_buf(ipipe, i_tail);
1790		obuf = pipe_buf(opipe, o_head);
1791
1792		if (len >= ibuf->len) {
1793			/*
1794			 * Simply move the whole buffer from ipipe to opipe
1795			 */
1796			*obuf = *ibuf;
1797			ibuf->ops = NULL;
1798			i_tail++;
1799			ipipe->tail = i_tail;
1800			input_wakeup = true;
1801			o_len = obuf->len;
1802			o_head++;
1803			opipe->head = o_head;
1804		} else {
1805			/*
1806			 * Get a reference to this pipe buffer,
1807			 * so we can copy the contents over.
1808			 */
1809			if (!pipe_buf_get(ipipe, ibuf)) {
1810				if (ret == 0)
1811					ret = -EFAULT;
1812				break;
1813			}
1814			*obuf = *ibuf;
1815
1816			/*
1817			 * Don't inherit the gift and merge flags, we need to
1818			 * prevent multiple steals of this page.
1819			 */
1820			obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1821			obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1822
1823			obuf->len = len;
1824			ibuf->offset += len;
1825			ibuf->len -= len;
1826			o_len = len;
1827			o_head++;
1828			opipe->head = o_head;
1829		}
1830		ret += o_len;
1831		len -= o_len;
1832	} while (len);
1833
1834	pipe_unlock(ipipe);
1835	pipe_unlock(opipe);
1836
1837	/*
1838	 * If we put data in the output pipe, wakeup any potential readers.
1839	 */
1840	if (ret > 0)
1841		wakeup_pipe_readers(opipe);
1842
1843	if (input_wakeup)
1844		wakeup_pipe_writers(ipipe);
1845
1846	return ret;
1847}
1848
1849/*
1850 * Link contents of ipipe to opipe.
1851 */
1852static ssize_t link_pipe(struct pipe_inode_info *ipipe,
1853			 struct pipe_inode_info *opipe,
1854			 size_t len, unsigned int flags)
1855{
1856	struct pipe_buffer *ibuf, *obuf;
1857	unsigned int i_head, o_head;
1858	unsigned int i_tail, o_tail;
1859	ssize_t ret = 0;
1860
1861	/*
1862	 * Potential ABBA deadlock, work around it by ordering lock
1863	 * grabbing by pipe info address. Otherwise two different processes
1864	 * could deadlock (one doing tee from A -> B, the other from B -> A).
1865	 */
1866	pipe_double_lock(ipipe, opipe);
1867
1868	i_tail = ipipe->tail;
1869	o_head = opipe->head;
1870
1871	do {
1872		if (!opipe->readers) {
1873			send_sig(SIGPIPE, current, 0);
1874			if (!ret)
1875				ret = -EPIPE;
1876			break;
1877		}
1878
1879		i_head = ipipe->head;
1880		o_tail = opipe->tail;
1881
1882		/*
1883		 * If we have iterated all input buffers or run out of
1884		 * output room, break.
1885		 */
1886		if (pipe_empty(i_head, i_tail) ||
1887		    pipe_full(o_head, o_tail, opipe->max_usage))
1888			break;
1889
1890		ibuf = pipe_buf(ipipe, i_tail);
1891		obuf = pipe_buf(opipe, o_head);
1892
1893		/*
1894		 * Get a reference to this pipe buffer,
1895		 * so we can copy the contents over.
1896		 */
1897		if (!pipe_buf_get(ipipe, ibuf)) {
1898			if (ret == 0)
1899				ret = -EFAULT;
1900			break;
1901		}
1902
1903		*obuf = *ibuf;
1904
1905		/*
1906		 * Don't inherit the gift and merge flag, we need to prevent
1907		 * multiple steals of this page.
1908		 */
1909		obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1910		obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1911
1912		if (obuf->len > len)
1913			obuf->len = len;
1914		ret += obuf->len;
1915		len -= obuf->len;
1916
1917		o_head++;
1918		opipe->head = o_head;
1919		i_tail++;
1920	} while (len);
1921
1922	pipe_unlock(ipipe);
1923	pipe_unlock(opipe);
1924
1925	/*
1926	 * If we put data in the output pipe, wakeup any potential readers.
1927	 */
1928	if (ret > 0)
1929		wakeup_pipe_readers(opipe);
1930
1931	return ret;
1932}
1933
1934/*
1935 * This is a tee(1) implementation that works on pipes. It doesn't copy
1936 * any data, it simply references the 'in' pages on the 'out' pipe.
1937 * The 'flags' used are the SPLICE_F_* variants, currently the only
1938 * applicable one is SPLICE_F_NONBLOCK.
1939 */
1940ssize_t do_tee(struct file *in, struct file *out, size_t len,
1941	       unsigned int flags)
1942{
1943	struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1944	struct pipe_inode_info *opipe = get_pipe_info(out, true);
1945	ssize_t ret = -EINVAL;
1946
1947	if (unlikely(!(in->f_mode & FMODE_READ) ||
1948		     !(out->f_mode & FMODE_WRITE)))
1949		return -EBADF;
1950
1951	/*
1952	 * Duplicate the contents of ipipe to opipe without actually
1953	 * copying the data.
1954	 */
1955	if (ipipe && opipe && ipipe != opipe) {
1956		if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1957			flags |= SPLICE_F_NONBLOCK;
1958
1959		/*
1960		 * Keep going, unless we encounter an error. The ipipe/opipe
1961		 * ordering doesn't really matter.
1962		 */
1963		ret = ipipe_prep(ipipe, flags);
1964		if (!ret) {
1965			ret = opipe_prep(opipe, flags);
1966			if (!ret)
1967				ret = link_pipe(ipipe, opipe, len, flags);
1968		}
1969	}
1970
1971	if (ret > 0) {
1972		fsnotify_access(in);
1973		fsnotify_modify(out);
1974	}
1975
1976	return ret;
1977}
1978
1979SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1980{
1981	if (unlikely(flags & ~SPLICE_F_ALL))
1982		return -EINVAL;
1983
1984	if (unlikely(!len))
1985		return 0;
1986
1987	CLASS(fd, in)(fdin);
1988	if (fd_empty(in))
1989		return -EBADF;
1990
1991	CLASS(fd, out)(fdout);
1992	if (fd_empty(out))
1993		return -EBADF;
1994
1995	return do_tee(fd_file(in), fd_file(out), len, flags);
1996}