read_write.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  linux/fs/read_write.c
   4 *
   5 *  Copyright (C) 1991, 1992  Linus Torvalds
   6 */
   7
   8#include <linux/slab.h>
   9#include <linux/stat.h>
  10#include <linux/sched/xacct.h>
  11#include <linux/fcntl.h>
  12#include <linux/file.h>
  13#include <linux/uio.h>
  14#include <linux/fsnotify.h>
  15#include <linux/security.h>
  16#include <linux/export.h>
  17#include <linux/syscalls.h>
  18#include <linux/pagemap.h>
  19#include <linux/splice.h>
  20#include <linux/compat.h>
  21#include <linux/mount.h>
  22#include <linux/fs.h>
  23#include "internal.h"
  24
  25#include <linux/uaccess.h>
  26#include <asm/unistd.h>
  27
  28const struct file_operations generic_ro_fops = {
  29	.llseek		= generic_file_llseek,
  30	.read_iter	= generic_file_read_iter,
  31	.mmap_prepare	= generic_file_readonly_mmap_prepare,
  32	.splice_read	= filemap_splice_read,
  33};
  34
  35EXPORT_SYMBOL(generic_ro_fops);
  36
  37static inline bool unsigned_offsets(struct file *file)
  38{
  39	return file->f_op->fop_flags & FOP_UNSIGNED_OFFSET;
  40}
  41
  42/**
  43 * vfs_setpos_cookie - update the file offset for lseek and reset cookie
  44 * @file:	file structure in question
  45 * @offset:	file offset to seek to
  46 * @maxsize:	maximum file size
  47 * @cookie:	cookie to reset
  48 *
  49 * Update the file offset to the value specified by @offset if the given
  50 * offset is valid and it is not equal to the current file offset and
  51 * reset the specified cookie to indicate that a seek happened.
  52 *
  53 * Return the specified offset on success and -EINVAL on invalid offset.
  54 */
  55static loff_t vfs_setpos_cookie(struct file *file, loff_t offset,
  56				loff_t maxsize, u64 *cookie)
  57{
  58	if (offset < 0 && !unsigned_offsets(file))
  59		return -EINVAL;
  60	if (offset > maxsize)
  61		return -EINVAL;
  62
  63	if (offset != file->f_pos) {
  64		file->f_pos = offset;
  65		if (cookie)
  66			*cookie = 0;
  67	}
  68	return offset;
  69}
  70
  71/**
  72 * vfs_setpos - update the file offset for lseek
  73 * @file:	file structure in question
  74 * @offset:	file offset to seek to
  75 * @maxsize:	maximum file size
  76 *
  77 * This is a low-level filesystem helper for updating the file offset to
  78 * the value specified by @offset if the given offset is valid and it is
  79 * not equal to the current file offset.
  80 *
  81 * Return the specified offset on success and -EINVAL on invalid offset.
  82 */
  83loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
  84{
  85	return vfs_setpos_cookie(file, offset, maxsize, NULL);
  86}
  87EXPORT_SYMBOL(vfs_setpos);
  88
  89/**
  90 * must_set_pos - check whether f_pos has to be updated
  91 * @file: file to seek on
  92 * @offset: offset to use
  93 * @whence: type of seek operation
  94 * @eof: end of file
  95 *
  96 * Check whether f_pos needs to be updated and update @offset according
  97 * to @whence.
  98 *
  99 * Return: 0 if f_pos doesn't need to be updated, 1 if f_pos has to be
 100 * updated, and negative error code on failure.
 101 */
 102static int must_set_pos(struct file *file, loff_t *offset, int whence, loff_t eof)
 103{
 104	switch (whence) {
 105	case SEEK_END:
 106		*offset += eof;
 107		break;
 108	case SEEK_CUR:
 109		/*
 110		 * Here we special-case the lseek(fd, 0, SEEK_CUR)
 111		 * position-querying operation.  Avoid rewriting the "same"
 112		 * f_pos value back to the file because a concurrent read(),
 113		 * write() or lseek() might have altered it
 114		 */
 115		if (*offset == 0) {
 116			*offset = file->f_pos;
 117			return 0;
 118		}
 119		break;
 120	case SEEK_DATA:
 121		/*
 122		 * In the generic case the entire file is data, so as long as
 123		 * offset isn't at the end of the file then the offset is data.
 124		 */
 125		if ((unsigned long long)*offset >= eof)
 126			return -ENXIO;
 127		break;
 128	case SEEK_HOLE:
 129		/*
 130		 * There is a virtual hole at the end of the file, so as long as
 131		 * offset isn't i_size or larger, return i_size.
 132		 */
 133		if ((unsigned long long)*offset >= eof)
 134			return -ENXIO;
 135		*offset = eof;
 136		break;
 137	}
 138
 139	return 1;
 140}
 141
 142/**
 143 * generic_file_llseek_size - generic llseek implementation for regular files
 144 * @file:	file structure to seek on
 145 * @offset:	file offset to seek to
 146 * @whence:	type of seek
 147 * @maxsize:	max size of this file in file system
 148 * @eof:	offset used for SEEK_END position
 149 *
 150 * This is a variant of generic_file_llseek that allows passing in a custom
 151 * maximum file size and a custom EOF position, for e.g. hashed directories
 152 *
 153 * Synchronization:
 154 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
 155 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
 156 * read/writes behave like SEEK_SET against seeks.
 157 */
 158loff_t
 159generic_file_llseek_size(struct file *file, loff_t offset, int whence,
 160		loff_t maxsize, loff_t eof)
 161{
 162	int ret;
 163
 164	ret = must_set_pos(file, &offset, whence, eof);
 165	if (ret < 0)
 166		return ret;
 167	if (ret == 0)
 168		return offset;
 169
 170	if (whence == SEEK_CUR) {
 171		/*
 172		 * If the file requires locking via f_pos_lock we know
 173		 * that mutual exclusion for SEEK_CUR on the same file
 174		 * is guaranteed. If the file isn't locked, we take
 175		 * f_lock to protect against f_pos races with other
 176		 * SEEK_CURs.
 177		 */
 178		if (file_seek_cur_needs_f_lock(file)) {
 179			guard(spinlock)(&file->f_lock);
 180			return vfs_setpos(file, file->f_pos + offset, maxsize);
 181		}
 182		return vfs_setpos(file, file->f_pos + offset, maxsize);
 183	}
 184
 185	return vfs_setpos(file, offset, maxsize);
 186}
 187EXPORT_SYMBOL(generic_file_llseek_size);
 188
 189/**
 190 * generic_llseek_cookie - versioned llseek implementation
 191 * @file:	file structure to seek on
 192 * @offset:	file offset to seek to
 193 * @whence:	type of seek
 194 * @cookie:	cookie to update
 195 *
 196 * See generic_file_llseek for a general description and locking assumptions.
 197 *
 198 * In contrast to generic_file_llseek, this function also resets a
 199 * specified cookie to indicate a seek took place.
 200 */
 201loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence,
 202			     u64 *cookie)
 203{
 204	struct inode *inode = file->f_mapping->host;
 205	loff_t maxsize = inode->i_sb->s_maxbytes;
 206	loff_t eof = i_size_read(inode);
 207	int ret;
 208
 209	if (WARN_ON_ONCE(!cookie))
 210		return -EINVAL;
 211
 212	/*
 213	 * Require that this is only used for directories that guarantee
 214	 * synchronization between readdir and seek so that an update to
 215	 * @cookie is correctly synchronized with concurrent readdir.
 216	 */
 217	if (WARN_ON_ONCE(!(file->f_mode & FMODE_ATOMIC_POS)))
 218		return -EINVAL;
 219
 220	ret = must_set_pos(file, &offset, whence, eof);
 221	if (ret < 0)
 222		return ret;
 223	if (ret == 0)
 224		return offset;
 225
 226	/* No need to hold f_lock because we know that f_pos_lock is held. */
 227	if (whence == SEEK_CUR)
 228		return vfs_setpos_cookie(file, file->f_pos + offset, maxsize, cookie);
 229
 230	return vfs_setpos_cookie(file, offset, maxsize, cookie);
 231}
 232EXPORT_SYMBOL(generic_llseek_cookie);
 233
 234/**
 235 * generic_file_llseek - generic llseek implementation for regular files
 236 * @file:	file structure to seek on
 237 * @offset:	file offset to seek to
 238 * @whence:	type of seek
 239 *
 240 * This is a generic implementation of ->llseek useable for all normal local
 241 * filesystems.  It just updates the file offset to the value specified by
 242 * @offset and @whence.
 243 */
 244loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
 245{
 246	struct inode *inode = file->f_mapping->host;
 247
 248	return generic_file_llseek_size(file, offset, whence,
 249					inode->i_sb->s_maxbytes,
 250					i_size_read(inode));
 251}
 252EXPORT_SYMBOL(generic_file_llseek);
 253
 254/**
 255 * fixed_size_llseek - llseek implementation for fixed-sized devices
 256 * @file:	file structure to seek on
 257 * @offset:	file offset to seek to
 258 * @whence:	type of seek
 259 * @size:	size of the file
 260 *
 261 */
 262loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
 263{
 264	switch (whence) {
 265	case SEEK_SET: case SEEK_CUR: case SEEK_END:
 266		return generic_file_llseek_size(file, offset, whence,
 267						size, size);
 268	default:
 269		return -EINVAL;
 270	}
 271}
 272EXPORT_SYMBOL(fixed_size_llseek);
 273
 274/**
 275 * no_seek_end_llseek - llseek implementation for fixed-sized devices
 276 * @file:	file structure to seek on
 277 * @offset:	file offset to seek to
 278 * @whence:	type of seek
 279 *
 280 */
 281loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
 282{
 283	switch (whence) {
 284	case SEEK_SET: case SEEK_CUR:
 285		return generic_file_llseek_size(file, offset, whence,
 286						OFFSET_MAX, 0);
 287	default:
 288		return -EINVAL;
 289	}
 290}
 291EXPORT_SYMBOL(no_seek_end_llseek);
 292
 293/**
 294 * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
 295 * @file:	file structure to seek on
 296 * @offset:	file offset to seek to
 297 * @whence:	type of seek
 298 * @size:	maximal offset allowed
 299 *
 300 */
 301loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
 302{
 303	switch (whence) {
 304	case SEEK_SET: case SEEK_CUR:
 305		return generic_file_llseek_size(file, offset, whence,
 306						size, 0);
 307	default:
 308		return -EINVAL;
 309	}
 310}
 311EXPORT_SYMBOL(no_seek_end_llseek_size);
 312
 313/**
 314 * noop_llseek - No Operation Performed llseek implementation
 315 * @file:	file structure to seek on
 316 * @offset:	file offset to seek to
 317 * @whence:	type of seek
 318 *
 319 * This is an implementation of ->llseek useable for the rare special case when
 320 * userspace expects the seek to succeed but the (device) file is actually not
 321 * able to perform the seek. In this case you use noop_llseek() instead of
 322 * falling back to the default implementation of ->llseek.
 323 */
 324loff_t noop_llseek(struct file *file, loff_t offset, int whence)
 325{
 326	return file->f_pos;
 327}
 328EXPORT_SYMBOL(noop_llseek);
 329
 330loff_t default_llseek(struct file *file, loff_t offset, int whence)
 331{
 332	struct inode *inode = file_inode(file);
 333	loff_t retval;
 334
 335	retval = inode_lock_killable(inode);
 336	if (retval)
 337		return retval;
 338	switch (whence) {
 339		case SEEK_END:
 340			offset += i_size_read(inode);
 341			break;
 342		case SEEK_CUR:
 343			if (offset == 0) {
 344				retval = file->f_pos;
 345				goto out;
 346			}
 347			offset += file->f_pos;
 348			break;
 349		case SEEK_DATA:
 350			/*
 351			 * In the generic case the entire file is data, so as
 352			 * long as offset isn't at the end of the file then the
 353			 * offset is data.
 354			 */
 355			if (offset >= inode->i_size) {
 356				retval = -ENXIO;
 357				goto out;
 358			}
 359			break;
 360		case SEEK_HOLE:
 361			/*
 362			 * There is a virtual hole at the end of the file, so
 363			 * as long as offset isn't i_size or larger, return
 364			 * i_size.
 365			 */
 366			if (offset >= inode->i_size) {
 367				retval = -ENXIO;
 368				goto out;
 369			}
 370			offset = inode->i_size;
 371			break;
 372	}
 373	retval = -EINVAL;
 374	if (offset >= 0 || unsigned_offsets(file)) {
 375		if (offset != file->f_pos)
 376			file->f_pos = offset;
 377		retval = offset;
 378	}
 379out:
 380	inode_unlock(inode);
 381	return retval;
 382}
 383EXPORT_SYMBOL(default_llseek);
 384
 385loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
 386{
 387	if (!(file->f_mode & FMODE_LSEEK))
 388		return -ESPIPE;
 389	return file->f_op->llseek(file, offset, whence);
 390}
 391EXPORT_SYMBOL(vfs_llseek);
 392
 393static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
 394{
 395	off_t retval;
 396	CLASS(fd_pos, f)(fd);
 397	if (fd_empty(f))
 398		return -EBADF;
 399
 400	retval = -EINVAL;
 401	if (whence <= SEEK_MAX) {
 402		loff_t res = vfs_llseek(fd_file(f), offset, whence);
 403		retval = res;
 404		if (res != (loff_t)retval)
 405			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
 406	}
 407	return retval;
 408}
 409
 410SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
 411{
 412	return ksys_lseek(fd, offset, whence);
 413}
 414
 415#ifdef CONFIG_COMPAT
 416COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
 417{
 418	return ksys_lseek(fd, offset, whence);
 419}
 420#endif
 421
 422#if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
 423	defined(__ARCH_WANT_SYS_LLSEEK)
 424SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
 425		unsigned long, offset_low, loff_t __user *, result,
 426		unsigned int, whence)
 427{
 428	int retval;
 429	CLASS(fd_pos, f)(fd);
 430	loff_t offset;
 431
 432	if (fd_empty(f))
 433		return -EBADF;
 434
 435	if (whence > SEEK_MAX)
 436		return -EINVAL;
 437
 438	offset = vfs_llseek(fd_file(f), ((loff_t) offset_high << 32) | offset_low,
 439			whence);
 440
 441	retval = (int)offset;
 442	if (offset >= 0) {
 443		retval = -EFAULT;
 444		if (!copy_to_user(result, &offset, sizeof(offset)))
 445			retval = 0;
 446	}
 447	return retval;
 448}
 449#endif
 450
 451int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
 452{
 453	int mask = read_write == READ ? MAY_READ : MAY_WRITE;
 454	int ret;
 455
 456	if (unlikely((ssize_t) count < 0))
 457		return -EINVAL;
 458
 459	if (ppos) {
 460		loff_t pos = *ppos;
 461
 462		if (unlikely(pos < 0)) {
 463			if (!unsigned_offsets(file))
 464				return -EINVAL;
 465			if (count >= -pos) /* both values are in 0..LLONG_MAX */
 466				return -EOVERFLOW;
 467		} else if (unlikely((loff_t) (pos + count) < 0)) {
 468			if (!unsigned_offsets(file))
 469				return -EINVAL;
 470		}
 471	}
 472
 473	ret = security_file_permission(file, mask);
 474	if (ret)
 475		return ret;
 476
 477	return fsnotify_file_area_perm(file, mask, ppos, count);
 478}
 479EXPORT_SYMBOL(rw_verify_area);
 480
 481static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
 482{
 483	struct kiocb kiocb;
 484	struct iov_iter iter;
 485	ssize_t ret;
 486
 487	init_sync_kiocb(&kiocb, filp);
 488	kiocb.ki_pos = (ppos ? *ppos : 0);
 489	iov_iter_ubuf(&iter, ITER_DEST, buf, len);
 490
 491	ret = filp->f_op->read_iter(&kiocb, &iter);
 492	BUG_ON(ret == -EIOCBQUEUED);
 493	if (ppos)
 494		*ppos = kiocb.ki_pos;
 495	return ret;
 496}
 497
 498static int warn_unsupported(struct file *file, const char *op)
 499{
 500	pr_warn_ratelimited(
 501		"kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
 502		op, file, current->pid, current->comm);
 503	return -EINVAL;
 504}
 505
 506ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
 507{
 508	struct kvec iov = {
 509		.iov_base	= buf,
 510		.iov_len	= min_t(size_t, count, MAX_RW_COUNT),
 511	};
 512	struct kiocb kiocb;
 513	struct iov_iter iter;
 514	ssize_t ret;
 515
 516	if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
 517		return -EINVAL;
 518	if (!(file->f_mode & FMODE_CAN_READ))
 519		return -EINVAL;
 520	/*
 521	 * Also fail if ->read_iter and ->read are both wired up as that
 522	 * implies very convoluted semantics.
 523	 */
 524	if (unlikely(!file->f_op->read_iter || file->f_op->read))
 525		return warn_unsupported(file, "read");
 526
 527	init_sync_kiocb(&kiocb, file);
 528	kiocb.ki_pos = pos ? *pos : 0;
 529	iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len);
 530	ret = file->f_op->read_iter(&kiocb, &iter);
 531	if (ret > 0) {
 532		if (pos)
 533			*pos = kiocb.ki_pos;
 534		fsnotify_access(file);
 535		add_rchar(current, ret);
 536	}
 537	inc_syscr(current);
 538	return ret;
 539}
 540
 541ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
 542{
 543	ssize_t ret;
 544
 545	ret = rw_verify_area(READ, file, pos, count);
 546	if (ret)
 547		return ret;
 548	return __kernel_read(file, buf, count, pos);
 549}
 550EXPORT_SYMBOL(kernel_read);
 551
 552ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
 553{
 554	ssize_t ret;
 555
 556	if (!(file->f_mode & FMODE_READ))
 557		return -EBADF;
 558	if (!(file->f_mode & FMODE_CAN_READ))
 559		return -EINVAL;
 560	if (unlikely(!access_ok(buf, count)))
 561		return -EFAULT;
 562
 563	ret = rw_verify_area(READ, file, pos, count);
 564	if (ret)
 565		return ret;
 566	if (count > MAX_RW_COUNT)
 567		count =  MAX_RW_COUNT;
 568
 569	if (file->f_op->read)
 570		ret = file->f_op->read(file, buf, count, pos);
 571	else if (file->f_op->read_iter)
 572		ret = new_sync_read(file, buf, count, pos);
 573	else
 574		ret = -EINVAL;
 575	if (ret > 0) {
 576		fsnotify_access(file);
 577		add_rchar(current, ret);
 578	}
 579	inc_syscr(current);
 580	return ret;
 581}
 582
 583static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
 584{
 585	struct kiocb kiocb;
 586	struct iov_iter iter;
 587	ssize_t ret;
 588
 589	init_sync_kiocb(&kiocb, filp);
 590	kiocb.ki_pos = (ppos ? *ppos : 0);
 591	iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len);
 592
 593	ret = filp->f_op->write_iter(&kiocb, &iter);
 594	BUG_ON(ret == -EIOCBQUEUED);
 595	if (ret > 0 && ppos)
 596		*ppos = kiocb.ki_pos;
 597	return ret;
 598}
 599
 600/* caller is responsible for file_start_write/file_end_write */
 601ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos)
 602{
 603	struct kiocb kiocb;
 604	ssize_t ret;
 605
 606	if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
 607		return -EBADF;
 608	if (!(file->f_mode & FMODE_CAN_WRITE))
 609		return -EINVAL;
 610	/*
 611	 * Also fail if ->write_iter and ->write are both wired up as that
 612	 * implies very convoluted semantics.
 613	 */
 614	if (unlikely(!file->f_op->write_iter || file->f_op->write))
 615		return warn_unsupported(file, "write");
 616
 617	init_sync_kiocb(&kiocb, file);
 618	kiocb.ki_pos = pos ? *pos : 0;
 619	ret = file->f_op->write_iter(&kiocb, from);
 620	if (ret > 0) {
 621		if (pos)
 622			*pos = kiocb.ki_pos;
 623		fsnotify_modify(file);
 624		add_wchar(current, ret);
 625	}
 626	inc_syscw(current);
 627	return ret;
 628}
 629
 630/* caller is responsible for file_start_write/file_end_write */
 631ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
 632{
 633	struct kvec iov = {
 634		.iov_base	= (void *)buf,
 635		.iov_len	= min_t(size_t, count, MAX_RW_COUNT),
 636	};
 637	struct iov_iter iter;
 638	iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len);
 639	return __kernel_write_iter(file, &iter, pos);
 640}
 641/*
 642 * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
 643 * but autofs is one of the few internal kernel users that actually
 644 * wants this _and_ can be built as a module. So we need to export
 645 * this symbol for autofs, even though it really isn't appropriate
 646 * for any other kernel modules.
 647 */
 648EXPORT_SYMBOL_GPL(__kernel_write);
 649
 650ssize_t kernel_write(struct file *file, const void *buf, size_t count,
 651			    loff_t *pos)
 652{
 653	ssize_t ret;
 654
 655	ret = rw_verify_area(WRITE, file, pos, count);
 656	if (ret)
 657		return ret;
 658
 659	file_start_write(file);
 660	ret =  __kernel_write(file, buf, count, pos);
 661	file_end_write(file);
 662	return ret;
 663}
 664EXPORT_SYMBOL(kernel_write);
 665
 666ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
 667{
 668	ssize_t ret;
 669
 670	if (!(file->f_mode & FMODE_WRITE))
 671		return -EBADF;
 672	if (!(file->f_mode & FMODE_CAN_WRITE))
 673		return -EINVAL;
 674	if (unlikely(!access_ok(buf, count)))
 675		return -EFAULT;
 676
 677	ret = rw_verify_area(WRITE, file, pos, count);
 678	if (ret)
 679		return ret;
 680	if (count > MAX_RW_COUNT)
 681		count =  MAX_RW_COUNT;
 682	file_start_write(file);
 683	if (file->f_op->write)
 684		ret = file->f_op->write(file, buf, count, pos);
 685	else if (file->f_op->write_iter)
 686		ret = new_sync_write(file, buf, count, pos);
 687	else
 688		ret = -EINVAL;
 689	if (ret > 0) {
 690		fsnotify_modify(file);
 691		add_wchar(current, ret);
 692	}
 693	inc_syscw(current);
 694	file_end_write(file);
 695	return ret;
 696}
 697
 698/* file_ppos returns &file->f_pos or NULL if file is stream */
 699static inline loff_t *file_ppos(struct file *file)
 700{
 701	return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
 702}
 703
 704ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
 705{
 706	CLASS(fd_pos, f)(fd);
 707	ssize_t ret = -EBADF;
 708
 709	if (!fd_empty(f)) {
 710		loff_t pos, *ppos = file_ppos(fd_file(f));
 711		if (ppos) {
 712			pos = *ppos;
 713			ppos = &pos;
 714		}
 715		ret = vfs_read(fd_file(f), buf, count, ppos);
 716		if (ret >= 0 && ppos)
 717			fd_file(f)->f_pos = pos;
 718	}
 719	return ret;
 720}
 721
 722SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
 723{
 724	return ksys_read(fd, buf, count);
 725}
 726
 727ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
 728{
 729	CLASS(fd_pos, f)(fd);
 730	ssize_t ret = -EBADF;
 731
 732	if (!fd_empty(f)) {
 733		loff_t pos, *ppos = file_ppos(fd_file(f));
 734		if (ppos) {
 735			pos = *ppos;
 736			ppos = &pos;
 737		}
 738		ret = vfs_write(fd_file(f), buf, count, ppos);
 739		if (ret >= 0 && ppos)
 740			fd_file(f)->f_pos = pos;
 741	}
 742
 743	return ret;
 744}
 745
 746SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
 747		size_t, count)
 748{
 749	return ksys_write(fd, buf, count);
 750}
 751
 752ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
 753		     loff_t pos)
 754{
 755	if (pos < 0)
 756		return -EINVAL;
 757
 758	CLASS(fd, f)(fd);
 759	if (fd_empty(f))
 760		return -EBADF;
 761
 762	if (fd_file(f)->f_mode & FMODE_PREAD)
 763		return vfs_read(fd_file(f), buf, count, &pos);
 764
 765	return -ESPIPE;
 766}
 767
 768SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
 769			size_t, count, loff_t, pos)
 770{
 771	return ksys_pread64(fd, buf, count, pos);
 772}
 773
 774#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64)
 775COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf,
 776		       size_t, count, compat_arg_u64_dual(pos))
 777{
 778	return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos));
 779}
 780#endif
 781
 782ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
 783		      size_t count, loff_t pos)
 784{
 785	if (pos < 0)
 786		return -EINVAL;
 787
 788	CLASS(fd, f)(fd);
 789	if (fd_empty(f))
 790		return -EBADF;
 791
 792	if (fd_file(f)->f_mode & FMODE_PWRITE)
 793		return vfs_write(fd_file(f), buf, count, &pos);
 794
 795	return -ESPIPE;
 796}
 797
 798SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
 799			 size_t, count, loff_t, pos)
 800{
 801	return ksys_pwrite64(fd, buf, count, pos);
 802}
 803
 804#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64)
 805COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf,
 806		       size_t, count, compat_arg_u64_dual(pos))
 807{
 808	return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos));
 809}
 810#endif
 811
 812static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
 813		loff_t *ppos, int type, rwf_t flags)
 814{
 815	struct kiocb kiocb;
 816	ssize_t ret;
 817
 818	init_sync_kiocb(&kiocb, filp);
 819	ret = kiocb_set_rw_flags(&kiocb, flags, type);
 820	if (ret)
 821		return ret;
 822	kiocb.ki_pos = (ppos ? *ppos : 0);
 823
 824	if (type == READ)
 825		ret = filp->f_op->read_iter(&kiocb, iter);
 826	else
 827		ret = filp->f_op->write_iter(&kiocb, iter);
 828	BUG_ON(ret == -EIOCBQUEUED);
 829	if (ppos)
 830		*ppos = kiocb.ki_pos;
 831	return ret;
 832}
 833
 834/* Do it by hand, with file-ops */
 835static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
 836		loff_t *ppos, int type, rwf_t flags)
 837{
 838	ssize_t ret = 0;
 839
 840	if (flags & ~RWF_HIPRI)
 841		return -EOPNOTSUPP;
 842
 843	while (iov_iter_count(iter)) {
 844		ssize_t nr;
 845
 846		if (type == READ) {
 847			nr = filp->f_op->read(filp, iter_iov_addr(iter),
 848						iter_iov_len(iter), ppos);
 849		} else {
 850			nr = filp->f_op->write(filp, iter_iov_addr(iter),
 851						iter_iov_len(iter), ppos);
 852		}
 853
 854		if (nr < 0) {
 855			if (!ret)
 856				ret = nr;
 857			break;
 858		}
 859		ret += nr;
 860		if (nr != iter_iov_len(iter))
 861			break;
 862		iov_iter_advance(iter, nr);
 863	}
 864
 865	return ret;
 866}
 867
 868ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
 869			   struct iov_iter *iter)
 870{
 871	size_t tot_len;
 872	ssize_t ret = 0;
 873
 874	if (!file->f_op->read_iter)
 875		return -EINVAL;
 876	if (!(file->f_mode & FMODE_READ))
 877		return -EBADF;
 878	if (!(file->f_mode & FMODE_CAN_READ))
 879		return -EINVAL;
 880
 881	tot_len = iov_iter_count(iter);
 882	if (!tot_len)
 883		goto out;
 884	ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
 885	if (ret < 0)
 886		return ret;
 887
 888	ret = file->f_op->read_iter(iocb, iter);
 889out:
 890	if (ret >= 0)
 891		fsnotify_access(file);
 892	return ret;
 893}
 894EXPORT_SYMBOL(vfs_iocb_iter_read);
 895
 896ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
 897		      rwf_t flags)
 898{
 899	size_t tot_len;
 900	ssize_t ret = 0;
 901
 902	if (!file->f_op->read_iter)
 903		return -EINVAL;
 904	if (!(file->f_mode & FMODE_READ))
 905		return -EBADF;
 906	if (!(file->f_mode & FMODE_CAN_READ))
 907		return -EINVAL;
 908
 909	tot_len = iov_iter_count(iter);
 910	if (!tot_len)
 911		goto out;
 912	ret = rw_verify_area(READ, file, ppos, tot_len);
 913	if (ret < 0)
 914		return ret;
 915
 916	ret = do_iter_readv_writev(file, iter, ppos, READ, flags);
 917out:
 918	if (ret >= 0)
 919		fsnotify_access(file);
 920	return ret;
 921}
 922EXPORT_SYMBOL(vfs_iter_read);
 923
 924/*
 925 * Caller is responsible for calling kiocb_end_write() on completion
 926 * if async iocb was queued.
 927 */
 928ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
 929			    struct iov_iter *iter)
 930{
 931	size_t tot_len;
 932	ssize_t ret = 0;
 933
 934	if (!file->f_op->write_iter)
 935		return -EINVAL;
 936	if (!(file->f_mode & FMODE_WRITE))
 937		return -EBADF;
 938	if (!(file->f_mode & FMODE_CAN_WRITE))
 939		return -EINVAL;
 940
 941	tot_len = iov_iter_count(iter);
 942	if (!tot_len)
 943		return 0;
 944	ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
 945	if (ret < 0)
 946		return ret;
 947
 948	kiocb_start_write(iocb);
 949	ret = file->f_op->write_iter(iocb, iter);
 950	if (ret != -EIOCBQUEUED)
 951		kiocb_end_write(iocb);
 952	if (ret > 0)
 953		fsnotify_modify(file);
 954
 955	return ret;
 956}
 957EXPORT_SYMBOL(vfs_iocb_iter_write);
 958
 959ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
 960		       rwf_t flags)
 961{
 962	size_t tot_len;
 963	ssize_t ret;
 964
 965	if (!(file->f_mode & FMODE_WRITE))
 966		return -EBADF;
 967	if (!(file->f_mode & FMODE_CAN_WRITE))
 968		return -EINVAL;
 969	if (!file->f_op->write_iter)
 970		return -EINVAL;
 971
 972	tot_len = iov_iter_count(iter);
 973	if (!tot_len)
 974		return 0;
 975
 976	ret = rw_verify_area(WRITE, file, ppos, tot_len);
 977	if (ret < 0)
 978		return ret;
 979
 980	file_start_write(file);
 981	ret = do_iter_readv_writev(file, iter, ppos, WRITE, flags);
 982	if (ret > 0)
 983		fsnotify_modify(file);
 984	file_end_write(file);
 985
 986	return ret;
 987}
 988EXPORT_SYMBOL(vfs_iter_write);
 989
 990static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
 991			 unsigned long vlen, loff_t *pos, rwf_t flags)
 992{
 993	struct iovec iovstack[UIO_FASTIOV];
 994	struct iovec *iov = iovstack;
 995	struct iov_iter iter;
 996	size_t tot_len;
 997	ssize_t ret = 0;
 998
 999	if (!(file->f_mode & FMODE_READ))
1000		return -EBADF;
1001	if (!(file->f_mode & FMODE_CAN_READ))
1002		return -EINVAL;
1003
1004	ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov,
1005			   &iter);
1006	if (ret < 0)
1007		return ret;
1008
1009	tot_len = iov_iter_count(&iter);
1010	if (!tot_len)
1011		goto out;
1012
1013	ret = rw_verify_area(READ, file, pos, tot_len);
1014	if (ret < 0)
1015		goto out;
1016
1017	if (file->f_op->read_iter)
1018		ret = do_iter_readv_writev(file, &iter, pos, READ, flags);
1019	else
1020		ret = do_loop_readv_writev(file, &iter, pos, READ, flags);
1021out:
1022	if (ret >= 0)
1023		fsnotify_access(file);
1024	kfree(iov);
1025	return ret;
1026}
1027
1028static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
1029			  unsigned long vlen, loff_t *pos, rwf_t flags)
1030{
1031	struct iovec iovstack[UIO_FASTIOV];
1032	struct iovec *iov = iovstack;
1033	struct iov_iter iter;
1034	size_t tot_len;
1035	ssize_t ret = 0;
1036
1037	if (!(file->f_mode & FMODE_WRITE))
1038		return -EBADF;
1039	if (!(file->f_mode & FMODE_CAN_WRITE))
1040		return -EINVAL;
1041
1042	ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov,
1043			   &iter);
1044	if (ret < 0)
1045		return ret;
1046
1047	tot_len = iov_iter_count(&iter);
1048	if (!tot_len)
1049		goto out;
1050
1051	ret = rw_verify_area(WRITE, file, pos, tot_len);
1052	if (ret < 0)
1053		goto out;
1054
1055	file_start_write(file);
1056	if (file->f_op->write_iter)
1057		ret = do_iter_readv_writev(file, &iter, pos, WRITE, flags);
1058	else
1059		ret = do_loop_readv_writev(file, &iter, pos, WRITE, flags);
1060	if (ret > 0)
1061		fsnotify_modify(file);
1062	file_end_write(file);
1063out:
1064	kfree(iov);
1065	return ret;
1066}
1067
1068static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
1069			unsigned long vlen, rwf_t flags)
1070{
1071	CLASS(fd_pos, f)(fd);
1072	ssize_t ret = -EBADF;
1073
1074	if (!fd_empty(f)) {
1075		loff_t pos, *ppos = file_ppos(fd_file(f));
1076		if (ppos) {
1077			pos = *ppos;
1078			ppos = &pos;
1079		}
1080		ret = vfs_readv(fd_file(f), vec, vlen, ppos, flags);
1081		if (ret >= 0 && ppos)
1082			fd_file(f)->f_pos = pos;
1083	}
1084
1085	if (ret > 0)
1086		add_rchar(current, ret);
1087	inc_syscr(current);
1088	return ret;
1089}
1090
1091static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
1092			 unsigned long vlen, rwf_t flags)
1093{
1094	CLASS(fd_pos, f)(fd);
1095	ssize_t ret = -EBADF;
1096
1097	if (!fd_empty(f)) {
1098		loff_t pos, *ppos = file_ppos(fd_file(f));
1099		if (ppos) {
1100			pos = *ppos;
1101			ppos = &pos;
1102		}
1103		ret = vfs_writev(fd_file(f), vec, vlen, ppos, flags);
1104		if (ret >= 0 && ppos)
1105			fd_file(f)->f_pos = pos;
1106	}
1107
1108	if (ret > 0)
1109		add_wchar(current, ret);
1110	inc_syscw(current);
1111	return ret;
1112}
1113
1114static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
1115{
1116#define HALF_LONG_BITS (BITS_PER_LONG / 2)
1117	return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
1118}
1119
1120static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
1121			 unsigned long vlen, loff_t pos, rwf_t flags)
1122{
1123	ssize_t ret = -EBADF;
1124
1125	if (pos < 0)
1126		return -EINVAL;
1127
1128	CLASS(fd, f)(fd);
1129	if (!fd_empty(f)) {
1130		ret = -ESPIPE;
1131		if (fd_file(f)->f_mode & FMODE_PREAD)
1132			ret = vfs_readv(fd_file(f), vec, vlen, &pos, flags);
1133	}
1134
1135	if (ret > 0)
1136		add_rchar(current, ret);
1137	inc_syscr(current);
1138	return ret;
1139}
1140
1141static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1142			  unsigned long vlen, loff_t pos, rwf_t flags)
1143{
1144	ssize_t ret = -EBADF;
1145
1146	if (pos < 0)
1147		return -EINVAL;
1148
1149	CLASS(fd, f)(fd);
1150	if (!fd_empty(f)) {
1151		ret = -ESPIPE;
1152		if (fd_file(f)->f_mode & FMODE_PWRITE)
1153			ret = vfs_writev(fd_file(f), vec, vlen, &pos, flags);
1154	}
1155
1156	if (ret > 0)
1157		add_wchar(current, ret);
1158	inc_syscw(current);
1159	return ret;
1160}
1161
1162SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1163		unsigned long, vlen)
1164{
1165	return do_readv(fd, vec, vlen, 0);
1166}
1167
1168SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1169		unsigned long, vlen)
1170{
1171	return do_writev(fd, vec, vlen, 0);
1172}
1173
1174SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1175		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1176{
1177	loff_t pos = pos_from_hilo(pos_h, pos_l);
1178
1179	return do_preadv(fd, vec, vlen, pos, 0);
1180}
1181
1182SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1183		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1184		rwf_t, flags)
1185{
1186	loff_t pos = pos_from_hilo(pos_h, pos_l);
1187
1188	if (pos == -1)
1189		return do_readv(fd, vec, vlen, flags);
1190
1191	return do_preadv(fd, vec, vlen, pos, flags);
1192}
1193
1194SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1195		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1196{
1197	loff_t pos = pos_from_hilo(pos_h, pos_l);
1198
1199	return do_pwritev(fd, vec, vlen, pos, 0);
1200}
1201
1202SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1203		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1204		rwf_t, flags)
1205{
1206	loff_t pos = pos_from_hilo(pos_h, pos_l);
1207
1208	if (pos == -1)
1209		return do_writev(fd, vec, vlen, flags);
1210
1211	return do_pwritev(fd, vec, vlen, pos, flags);
1212}
1213
1214/*
1215 * Various compat syscalls.  Note that they all pretend to take a native
1216 * iovec - import_iovec will properly treat those as compat_iovecs based on
1217 * in_compat_syscall().
1218 */
1219#ifdef CONFIG_COMPAT
1220#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1221COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1222		const struct iovec __user *, vec,
1223		unsigned long, vlen, loff_t, pos)
1224{
1225	return do_preadv(fd, vec, vlen, pos, 0);
1226}
1227#endif
1228
1229COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1230		const struct iovec __user *, vec,
1231		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1232{
1233	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1234
1235	return do_preadv(fd, vec, vlen, pos, 0);
1236}
1237
1238#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1239COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1240		const struct iovec __user *, vec,
1241		unsigned long, vlen, loff_t, pos, rwf_t, flags)
1242{
1243	if (pos == -1)
1244		return do_readv(fd, vec, vlen, flags);
1245	return do_preadv(fd, vec, vlen, pos, flags);
1246}
1247#endif
1248
1249COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1250		const struct iovec __user *, vec,
1251		compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1252		rwf_t, flags)
1253{
1254	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1255
1256	if (pos == -1)
1257		return do_readv(fd, vec, vlen, flags);
1258	return do_preadv(fd, vec, vlen, pos, flags);
1259}
1260
1261#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1262COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1263		const struct iovec __user *, vec,
1264		unsigned long, vlen, loff_t, pos)
1265{
1266	return do_pwritev(fd, vec, vlen, pos, 0);
1267}
1268#endif
1269
1270COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1271		const struct iovec __user *,vec,
1272		compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1273{
1274	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1275
1276	return do_pwritev(fd, vec, vlen, pos, 0);
1277}
1278
1279#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1280COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1281		const struct iovec __user *, vec,
1282		unsigned long, vlen, loff_t, pos, rwf_t, flags)
1283{
1284	if (pos == -1)
1285		return do_writev(fd, vec, vlen, flags);
1286	return do_pwritev(fd, vec, vlen, pos, flags);
1287}
1288#endif
1289
1290COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1291		const struct iovec __user *,vec,
1292		compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1293{
1294	loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1295
1296	if (pos == -1)
1297		return do_writev(fd, vec, vlen, flags);
1298	return do_pwritev(fd, vec, vlen, pos, flags);
1299}
1300#endif /* CONFIG_COMPAT */
1301
1302static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1303			   size_t count, loff_t max)
1304{
1305	struct inode *in_inode, *out_inode;
1306	struct pipe_inode_info *opipe;
1307	loff_t pos;
1308	loff_t out_pos;
1309	ssize_t retval;
1310	int fl;
1311
1312	/*
1313	 * Get input file, and verify that it is ok..
1314	 */
1315	CLASS(fd, in)(in_fd);
1316	if (fd_empty(in))
1317		return -EBADF;
1318	if (!(fd_file(in)->f_mode & FMODE_READ))
1319		return -EBADF;
1320	if (!ppos) {
1321		pos = fd_file(in)->f_pos;
1322	} else {
1323		pos = *ppos;
1324		if (!(fd_file(in)->f_mode & FMODE_PREAD))
1325			return -ESPIPE;
1326	}
1327	retval = rw_verify_area(READ, fd_file(in), &pos, count);
1328	if (retval < 0)
1329		return retval;
1330	if (count > MAX_RW_COUNT)
1331		count =  MAX_RW_COUNT;
1332
1333	/*
1334	 * Get output file, and verify that it is ok..
1335	 */
1336	CLASS(fd, out)(out_fd);
1337	if (fd_empty(out))
1338		return -EBADF;
1339	if (!(fd_file(out)->f_mode & FMODE_WRITE))
1340		return -EBADF;
1341	in_inode = file_inode(fd_file(in));
1342	out_inode = file_inode(fd_file(out));
1343	out_pos = fd_file(out)->f_pos;
1344
1345	if (!max)
1346		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1347
1348	if (unlikely(pos + count > max)) {
1349		if (pos >= max)
1350			return -EOVERFLOW;
1351		count = max - pos;
1352	}
1353
1354	fl = 0;
1355#if 0
1356	/*
1357	 * We need to debate whether we can enable this or not. The
1358	 * man page documents EAGAIN return for the output at least,
1359	 * and the application is arguably buggy if it doesn't expect
1360	 * EAGAIN on a non-blocking file descriptor.
1361	 */
1362	if (fd_file(in)->f_flags & O_NONBLOCK)
1363		fl = SPLICE_F_NONBLOCK;
1364#endif
1365	opipe = get_pipe_info(fd_file(out), true);
1366	if (!opipe) {
1367		retval = rw_verify_area(WRITE, fd_file(out), &out_pos, count);
1368		if (retval < 0)
1369			return retval;
1370		retval = do_splice_direct(fd_file(in), &pos, fd_file(out), &out_pos,
1371					  count, fl);
1372	} else {
1373		if (fd_file(out)->f_flags & O_NONBLOCK)
1374			fl |= SPLICE_F_NONBLOCK;
1375
1376		retval = splice_file_to_pipe(fd_file(in), opipe, &pos, count, fl);
1377	}
1378
1379	if (retval > 0) {
1380		add_rchar(current, retval);
1381		add_wchar(current, retval);
1382		fsnotify_access(fd_file(in));
1383		fsnotify_modify(fd_file(out));
1384		fd_file(out)->f_pos = out_pos;
1385		if (ppos)
1386			*ppos = pos;
1387		else
1388			fd_file(in)->f_pos = pos;
1389	}
1390
1391	inc_syscr(current);
1392	inc_syscw(current);
1393	if (pos > max)
1394		retval = -EOVERFLOW;
1395	return retval;
1396}
1397
1398SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1399{
1400	loff_t pos;
1401	off_t off;
1402	ssize_t ret;
1403
1404	if (offset) {
1405		if (unlikely(get_user(off, offset)))
1406			return -EFAULT;
1407		pos = off;
1408		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1409		if (unlikely(put_user(pos, offset)))
1410			return -EFAULT;
1411		return ret;
1412	}
1413
1414	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1415}
1416
1417SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1418{
1419	loff_t pos;
1420	ssize_t ret;
1421
1422	if (offset) {
1423		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1424			return -EFAULT;
1425		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1426		if (unlikely(put_user(pos, offset)))
1427			return -EFAULT;
1428		return ret;
1429	}
1430
1431	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1432}
1433
1434#ifdef CONFIG_COMPAT
1435COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1436		compat_off_t __user *, offset, compat_size_t, count)
1437{
1438	loff_t pos;
1439	off_t off;
1440	ssize_t ret;
1441
1442	if (offset) {
1443		if (unlikely(get_user(off, offset)))
1444			return -EFAULT;
1445		pos = off;
1446		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1447		if (unlikely(put_user(pos, offset)))
1448			return -EFAULT;
1449		return ret;
1450	}
1451
1452	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1453}
1454
1455COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1456		compat_loff_t __user *, offset, compat_size_t, count)
1457{
1458	loff_t pos;
1459	ssize_t ret;
1460
1461	if (offset) {
1462		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1463			return -EFAULT;
1464		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1465		if (unlikely(put_user(pos, offset)))
1466			return -EFAULT;
1467		return ret;
1468	}
1469
1470	return do_sendfile(out_fd, in_fd, NULL, count, 0);
1471}
1472#endif
1473
1474/*
1475 * Performs necessary checks before doing a file copy
1476 *
1477 * Can adjust amount of bytes to copy via @req_count argument.
1478 * Returns appropriate error code that caller should return or
1479 * zero in case the copy should be allowed.
1480 */
1481static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
1482				    struct file *file_out, loff_t pos_out,
1483				    size_t *req_count, unsigned int flags)
1484{
1485	struct inode *inode_in = file_inode(file_in);
1486	struct inode *inode_out = file_inode(file_out);
1487	uint64_t count = *req_count;
1488	loff_t size_in;
1489	int ret;
1490
1491	ret = generic_file_rw_checks(file_in, file_out);
1492	if (ret)
1493		return ret;
1494
1495	/*
1496	 * We allow some filesystems to handle cross sb copy, but passing
1497	 * a file of the wrong filesystem type to filesystem driver can result
1498	 * in an attempt to dereference the wrong type of ->private_data, so
1499	 * avoid doing that until we really have a good reason.
1500	 *
1501	 * nfs and cifs define several different file_system_type structures
1502	 * and several different sets of file_operations, but they all end up
1503	 * using the same ->copy_file_range() function pointer.
1504	 */
1505	if (flags & COPY_FILE_SPLICE) {
1506		/* cross sb splice is allowed */
1507	} else if (file_out->f_op->copy_file_range) {
1508		if (file_in->f_op->copy_file_range !=
1509		    file_out->f_op->copy_file_range)
1510			return -EXDEV;
1511	} else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) {
1512		return -EXDEV;
1513	}
1514
1515	/* Don't touch certain kinds of inodes */
1516	if (IS_IMMUTABLE(inode_out))
1517		return -EPERM;
1518
1519	if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1520		return -ETXTBSY;
1521
1522	/* Ensure offsets don't wrap. */
1523	if (pos_in + count < pos_in || pos_out + count < pos_out)
1524		return -EOVERFLOW;
1525
1526	/* Shorten the copy to EOF */
1527	size_in = i_size_read(inode_in);
1528	if (pos_in >= size_in)
1529		count = 0;
1530	else
1531		count = min(count, size_in - (uint64_t)pos_in);
1532
1533	ret = generic_write_check_limits(file_out, pos_out, &count);
1534	if (ret)
1535		return ret;
1536
1537	/* Don't allow overlapped copying within the same file. */
1538	if (inode_in == inode_out &&
1539	    pos_out + count > pos_in &&
1540	    pos_out < pos_in + count)
1541		return -EINVAL;
1542
1543	*req_count = count;
1544	return 0;
1545}
1546
1547/*
1548 * copy_file_range() differs from regular file read and write in that it
1549 * specifically allows return partial success.  When it does so is up to
1550 * the copy_file_range method.
1551 */
1552ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1553			    struct file *file_out, loff_t pos_out,
1554			    size_t len, unsigned int flags)
1555{
1556	ssize_t ret;
1557	bool splice = flags & COPY_FILE_SPLICE;
1558	bool samesb = file_inode(file_in)->i_sb == file_inode(file_out)->i_sb;
1559
1560	if (flags & ~COPY_FILE_SPLICE)
1561		return -EINVAL;
1562
1563	ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
1564				       flags);
1565	if (unlikely(ret))
1566		return ret;
1567
1568	ret = rw_verify_area(READ, file_in, &pos_in, len);
1569	if (unlikely(ret))
1570		return ret;
1571
1572	ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1573	if (unlikely(ret))
1574		return ret;
1575
1576	if (len == 0)
1577		return 0;
1578
1579	/*
1580	 * Make sure return value doesn't overflow in 32bit compat mode.  Also
1581	 * limit the size for all cases except when calling ->copy_file_range().
1582	 */
1583	if (splice || !file_out->f_op->copy_file_range || in_compat_syscall())
1584		len = min_t(size_t, MAX_RW_COUNT, len);
1585
1586	file_start_write(file_out);
1587
1588	/*
1589	 * Cloning is supported by more file systems, so we implement copy on
1590	 * same sb using clone, but for filesystems where both clone and copy
1591	 * are supported (e.g. nfs,cifs), we only call the copy method.
1592	 */
1593	if (!splice && file_out->f_op->copy_file_range) {
1594		ret = file_out->f_op->copy_file_range(file_in, pos_in,
1595						      file_out, pos_out,
1596						      len, flags);
1597	} else if (!splice && file_in->f_op->remap_file_range && samesb) {
1598		ret = file_in->f_op->remap_file_range(file_in, pos_in,
1599				file_out, pos_out, len, REMAP_FILE_CAN_SHORTEN);
1600		/* fallback to splice */
1601		if (ret <= 0)
1602			splice = true;
1603	} else if (samesb) {
1604		/* Fallback to splice for same sb copy for backward compat */
1605		splice = true;
1606	}
1607
1608	file_end_write(file_out);
1609
1610	if (!splice)
1611		goto done;
1612
1613	/*
1614	 * We can get here for same sb copy of filesystems that do not implement
1615	 * ->copy_file_range() in case filesystem does not support clone or in
1616	 * case filesystem supports clone but rejected the clone request (e.g.
1617	 * because it was not block aligned).
1618	 *
1619	 * In both cases, fall back to kernel copy so we are able to maintain a
1620	 * consistent story about which filesystems support copy_file_range()
1621	 * and which filesystems do not, that will allow userspace tools to
1622	 * make consistent desicions w.r.t using copy_file_range().
1623	 *
1624	 * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE
1625	 * for server-side-copy between any two sb.
1626	 *
1627	 * In any case, we call do_splice_direct() and not splice_file_range(),
1628	 * without file_start_write() held, to avoid possible deadlocks related
1629	 * to splicing from input file, while file_start_write() is held on
1630	 * the output file on a different sb.
1631	 */
1632	ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out, len, 0);
1633done:
1634	if (ret > 0) {
1635		fsnotify_access(file_in);
1636		add_rchar(current, ret);
1637		fsnotify_modify(file_out);
1638		add_wchar(current, ret);
1639	}
1640
1641	inc_syscr(current);
1642	inc_syscw(current);
1643
1644	return ret;
1645}
1646EXPORT_SYMBOL(vfs_copy_file_range);
1647
1648SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1649		int, fd_out, loff_t __user *, off_out,
1650		size_t, len, unsigned int, flags)
1651{
1652	loff_t pos_in;
1653	loff_t pos_out;
1654	ssize_t ret = -EBADF;
1655
1656	CLASS(fd, f_in)(fd_in);
1657	if (fd_empty(f_in))
1658		return -EBADF;
1659
1660	CLASS(fd, f_out)(fd_out);
1661	if (fd_empty(f_out))
1662		return -EBADF;
1663
1664	if (off_in) {
1665		if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1666			return -EFAULT;
1667	} else {
1668		pos_in = fd_file(f_in)->f_pos;
1669	}
1670
1671	if (off_out) {
1672		if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1673			return -EFAULT;
1674	} else {
1675		pos_out = fd_file(f_out)->f_pos;
1676	}
1677
1678	if (flags != 0)
1679		return -EINVAL;
1680
1681	ret = vfs_copy_file_range(fd_file(f_in), pos_in, fd_file(f_out), pos_out, len,
1682				  flags);
1683	if (ret > 0) {
1684		pos_in += ret;
1685		pos_out += ret;
1686
1687		if (off_in) {
1688			if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1689				ret = -EFAULT;
1690		} else {
1691			fd_file(f_in)->f_pos = pos_in;
1692		}
1693
1694		if (off_out) {
1695			if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1696				ret = -EFAULT;
1697		} else {
1698			fd_file(f_out)->f_pos = pos_out;
1699		}
1700	}
1701	return ret;
1702}
1703
1704/*
1705 * Don't operate on ranges the page cache doesn't support, and don't exceed the
1706 * LFS limits.  If pos is under the limit it becomes a short access.  If it
1707 * exceeds the limit we return -EFBIG.
1708 */
1709int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
1710{
1711	struct inode *inode = file->f_mapping->host;
1712	loff_t max_size = inode->i_sb->s_maxbytes;
1713	loff_t limit = rlimit(RLIMIT_FSIZE);
1714
1715	if (limit != RLIM_INFINITY) {
1716		if (pos >= limit) {
1717			send_sig(SIGXFSZ, current, 0);
1718			return -EFBIG;
1719		}
1720		*count = min(*count, limit - pos);
1721	}
1722
1723	if (!(file->f_flags & O_LARGEFILE))
1724		max_size = MAX_NON_LFS;
1725
1726	if (unlikely(pos >= max_size))
1727		return -EFBIG;
1728
1729	*count = min(*count, max_size - pos);
1730
1731	return 0;
1732}
1733EXPORT_SYMBOL_GPL(generic_write_check_limits);
1734
1735/* Like generic_write_checks(), but takes size of write instead of iter. */
1736int generic_write_checks_count(struct kiocb *iocb, loff_t *count)
1737{
1738	struct file *file = iocb->ki_filp;
1739	struct inode *inode = file->f_mapping->host;
1740
1741	if (IS_SWAPFILE(inode))
1742		return -ETXTBSY;
1743
1744	if (!*count)
1745		return 0;
1746
1747	if (iocb->ki_flags & IOCB_APPEND)
1748		iocb->ki_pos = i_size_read(inode);
1749
1750	if ((iocb->ki_flags & IOCB_NOWAIT) &&
1751	    !((iocb->ki_flags & IOCB_DIRECT) ||
1752	      (file->f_op->fop_flags & FOP_BUFFER_WASYNC)))
1753		return -EINVAL;
1754
1755	return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count);
1756}
1757EXPORT_SYMBOL(generic_write_checks_count);
1758
1759/*
1760 * Performs necessary checks before doing a write
1761 *
1762 * Can adjust writing position or amount of bytes to write.
1763 * Returns appropriate error code that caller should return or
1764 * zero in case that write should be allowed.
1765 */
1766ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1767{
1768	loff_t count = iov_iter_count(from);
1769	int ret;
1770
1771	ret = generic_write_checks_count(iocb, &count);
1772	if (ret)
1773		return ret;
1774
1775	iov_iter_truncate(from, count);
1776	return iov_iter_count(from);
1777}
1778EXPORT_SYMBOL(generic_write_checks);
1779
1780/*
1781 * Performs common checks before doing a file copy/clone
1782 * from @file_in to @file_out.
1783 */
1784int generic_file_rw_checks(struct file *file_in, struct file *file_out)
1785{
1786	struct inode *inode_in = file_inode(file_in);
1787	struct inode *inode_out = file_inode(file_out);
1788
1789	/* Don't copy dirs, pipes, sockets... */
1790	if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1791		return -EISDIR;
1792	if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1793		return -EINVAL;
1794
1795	if (!(file_in->f_mode & FMODE_READ) ||
1796	    !(file_out->f_mode & FMODE_WRITE) ||
1797	    (file_out->f_flags & O_APPEND))
1798		return -EBADF;
1799
1800	return 0;
1801}
1802
1803int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
1804{
1805	size_t len = iov_iter_count(iter);
1806
1807	if (!iter_is_ubuf(iter))
1808		return -EINVAL;
1809
1810	if (!is_power_of_2(len))
1811		return -EINVAL;
1812
1813	if (!IS_ALIGNED(iocb->ki_pos, len))
1814		return -EINVAL;
1815
1816	if (!(iocb->ki_flags & IOCB_DIRECT))
1817		return -EOPNOTSUPP;
1818
1819	return 0;
1820}
1821EXPORT_SYMBOL_GPL(generic_atomic_write_valid);