file.c

   1/*
   2  FUSE: Filesystem in Userspace
   3  Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
   4
   5  This program can be distributed under the terms of the GNU GPL.
   6  See the file COPYING.
   7*/
   8
   9#include "fuse_i.h"
  10
  11#include <linux/pagemap.h>
  12#include <linux/slab.h>
  13#include <linux/kernel.h>
  14#include <linux/sched.h>
  15#include <linux/sched/signal.h>
  16#include <linux/module.h>
  17#include <linux/swap.h>
  18#include <linux/falloc.h>
  19#include <linux/uio.h>
  20#include <linux/fs.h>
  21#include <linux/filelock.h>
  22#include <linux/splice.h>
  23#include <linux/task_io_accounting_ops.h>
  24#include <linux/iomap.h>
  25
  26static int fuse_send_open(struct fuse_mount *fm, u64 nodeid,
  27			  unsigned int open_flags, int opcode,
  28			  struct fuse_open_out *outargp)
  29{
  30	struct fuse_open_in inarg;
  31	FUSE_ARGS(args);
  32
  33	memset(&inarg, 0, sizeof(inarg));
  34	inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
  35	if (!fm->fc->atomic_o_trunc)
  36		inarg.flags &= ~O_TRUNC;
  37
  38	if (fm->fc->handle_killpriv_v2 &&
  39	    (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) {
  40		inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID;
  41	}
  42
  43	args.opcode = opcode;
  44	args.nodeid = nodeid;
  45	args.in_numargs = 1;
  46	args.in_args[0].size = sizeof(inarg);
  47	args.in_args[0].value = &inarg;
  48	args.out_numargs = 1;
  49	args.out_args[0].size = sizeof(*outargp);
  50	args.out_args[0].value = outargp;
  51
  52	return fuse_simple_request(fm, &args);
  53}
  54
  55struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release)
  56{
  57	struct fuse_file *ff;
  58
  59	ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
  60	if (unlikely(!ff))
  61		return NULL;
  62
  63	ff->fm = fm;
  64	if (release) {
  65		ff->args = kzalloc(sizeof(*ff->args), GFP_KERNEL_ACCOUNT);
  66		if (!ff->args) {
  67			kfree(ff);
  68			return NULL;
  69		}
  70	}
  71
  72	INIT_LIST_HEAD(&ff->write_entry);
  73	refcount_set(&ff->count, 1);
  74	RB_CLEAR_NODE(&ff->polled_node);
  75	init_waitqueue_head(&ff->poll_wait);
  76
  77	ff->kh = atomic64_inc_return(&fm->fc->khctr);
  78
  79	return ff;
  80}
  81
  82void fuse_file_free(struct fuse_file *ff)
  83{
  84	kfree(ff->args);
  85	kfree(ff);
  86}
  87
  88static struct fuse_file *fuse_file_get(struct fuse_file *ff)
  89{
  90	refcount_inc(&ff->count);
  91	return ff;
  92}
  93
  94static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
  95			     int error)
  96{
  97	struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
  98
  99	iput(ra->inode);
 100	kfree(ra);
 101}
 102
 103static void fuse_file_put(struct fuse_file *ff, bool sync)
 104{
 105	if (refcount_dec_and_test(&ff->count)) {
 106		struct fuse_release_args *ra = &ff->args->release_args;
 107		struct fuse_args *args = (ra ? &ra->args : NULL);
 108
 109		if (ra && ra->inode)
 110			fuse_file_io_release(ff, ra->inode);
 111
 112		if (!args) {
 113			/* Do nothing when server does not implement 'opendir' */
 114		} else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) {
 115			fuse_release_end(ff->fm, args, 0);
 116		} else if (sync) {
 117			fuse_simple_request(ff->fm, args);
 118			fuse_release_end(ff->fm, args, 0);
 119		} else {
 120			args->end = fuse_release_end;
 121			if (fuse_simple_background(ff->fm, args,
 122						   GFP_KERNEL | __GFP_NOFAIL))
 123				fuse_release_end(ff->fm, args, -ENOTCONN);
 124		}
 125		kfree(ff);
 126	}
 127}
 128
 129struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
 130				 unsigned int open_flags, bool isdir)
 131{
 132	struct fuse_conn *fc = fm->fc;
 133	struct fuse_file *ff;
 134	int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
 135	bool open = isdir ? !fc->no_opendir : !fc->no_open;
 136	bool release = !isdir || open;
 137
 138	/*
 139	 * ff->args->release_args still needs to be allocated (so we can hold an
 140	 * inode reference while there are pending inflight file operations when
 141	 * ->release() is called, see fuse_prepare_release()) even if
 142	 * fc->no_open is set else it becomes possible for reclaim to deadlock
 143	 * if while servicing the readahead request the server triggers reclaim
 144	 * and reclaim evicts the inode of the file being read ahead.
 145	 */
 146	ff = fuse_file_alloc(fm, release);
 147	if (!ff)
 148		return ERR_PTR(-ENOMEM);
 149
 150	ff->fh = 0;
 151	/* Default for no-open */
 152	ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
 153	if (open) {
 154		/* Store outarg for fuse_finish_open() */
 155		struct fuse_open_out *outargp = &ff->args->open_outarg;
 156		int err;
 157
 158		err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp);
 159		if (!err) {
 160			ff->fh = outargp->fh;
 161			ff->open_flags = outargp->open_flags;
 162		} else if (err != -ENOSYS) {
 163			fuse_file_free(ff);
 164			return ERR_PTR(err);
 165		} else {
 166			if (isdir) {
 167				/* No release needed */
 168				kfree(ff->args);
 169				ff->args = NULL;
 170				fc->no_opendir = 1;
 171			} else {
 172				fc->no_open = 1;
 173			}
 174		}
 175	}
 176
 177	if (isdir)
 178		ff->open_flags &= ~FOPEN_DIRECT_IO;
 179
 180	ff->nodeid = nodeid;
 181
 182	return ff;
 183}
 184
 185int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
 186		 bool isdir)
 187{
 188	struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir);
 189
 190	if (!IS_ERR(ff))
 191		file->private_data = ff;
 192
 193	return PTR_ERR_OR_ZERO(ff);
 194}
 195EXPORT_SYMBOL_GPL(fuse_do_open);
 196
 197static void fuse_link_write_file(struct file *file)
 198{
 199	struct inode *inode = file_inode(file);
 200	struct fuse_inode *fi = get_fuse_inode(inode);
 201	struct fuse_file *ff = file->private_data;
 202	/*
 203	 * file may be written through mmap, so chain it onto the
 204	 * inodes's write_file list
 205	 */
 206	spin_lock(&fi->lock);
 207	if (list_empty(&ff->write_entry))
 208		list_add(&ff->write_entry, &fi->write_files);
 209	spin_unlock(&fi->lock);
 210}
 211
 212int fuse_finish_open(struct inode *inode, struct file *file)
 213{
 214	struct fuse_file *ff = file->private_data;
 215	struct fuse_conn *fc = get_fuse_conn(inode);
 216	int err;
 217
 218	err = fuse_file_io_open(file, inode);
 219	if (err)
 220		return err;
 221
 222	if (ff->open_flags & FOPEN_STREAM)
 223		stream_open(inode, file);
 224	else if (ff->open_flags & FOPEN_NONSEEKABLE)
 225		nonseekable_open(inode, file);
 226
 227	if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
 228		fuse_link_write_file(file);
 229
 230	return 0;
 231}
 232
 233static void fuse_truncate_update_attr(struct inode *inode, struct file *file)
 234{
 235	struct fuse_conn *fc = get_fuse_conn(inode);
 236	struct fuse_inode *fi = get_fuse_inode(inode);
 237
 238	spin_lock(&fi->lock);
 239	fi->attr_version = atomic64_inc_return(&fc->attr_version);
 240	i_size_write(inode, 0);
 241	spin_unlock(&fi->lock);
 242	file_update_time(file);
 243	fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
 244}
 245
 246static int fuse_open(struct inode *inode, struct file *file)
 247{
 248	struct fuse_mount *fm = get_fuse_mount(inode);
 249	struct fuse_inode *fi = get_fuse_inode(inode);
 250	struct fuse_conn *fc = fm->fc;
 251	struct fuse_file *ff;
 252	int err;
 253	bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc;
 254	bool is_wb_truncate = is_truncate && fc->writeback_cache;
 255	bool dax_truncate = is_truncate && FUSE_IS_DAX(inode);
 256
 257	if (fuse_is_bad(inode))
 258		return -EIO;
 259
 260	err = generic_file_open(inode, file);
 261	if (err)
 262		return err;
 263
 264	if (is_wb_truncate || dax_truncate)
 265		inode_lock(inode);
 266
 267	if (dax_truncate) {
 268		filemap_invalidate_lock(inode->i_mapping);
 269		err = fuse_dax_break_layouts(inode, 0, -1);
 270		if (err)
 271			goto out_inode_unlock;
 272	}
 273
 274	if (is_wb_truncate || dax_truncate)
 275		fuse_set_nowrite(inode);
 276
 277	err = fuse_do_open(fm, get_node_id(inode), file, false);
 278	if (!err) {
 279		ff = file->private_data;
 280		err = fuse_finish_open(inode, file);
 281		if (err)
 282			fuse_sync_release(fi, ff, file->f_flags);
 283		else if (is_truncate)
 284			fuse_truncate_update_attr(inode, file);
 285	}
 286
 287	if (is_wb_truncate || dax_truncate)
 288		fuse_release_nowrite(inode);
 289	if (!err) {
 290		if (is_truncate)
 291			truncate_pagecache(inode, 0);
 292		else if (!(ff->open_flags & FOPEN_KEEP_CACHE))
 293			invalidate_inode_pages2(inode->i_mapping);
 294	}
 295	if (dax_truncate)
 296		filemap_invalidate_unlock(inode->i_mapping);
 297out_inode_unlock:
 298	if (is_wb_truncate || dax_truncate)
 299		inode_unlock(inode);
 300
 301	return err;
 302}
 303
 304static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
 305				 unsigned int flags, int opcode, bool sync)
 306{
 307	struct fuse_conn *fc = ff->fm->fc;
 308	struct fuse_release_args *ra = &ff->args->release_args;
 309
 310	if (fuse_file_passthrough(ff))
 311		fuse_passthrough_release(ff, fuse_inode_backing(fi));
 312
 313	/* Inode is NULL on error path of fuse_create_open() */
 314	if (likely(fi)) {
 315		spin_lock(&fi->lock);
 316		list_del(&ff->write_entry);
 317		spin_unlock(&fi->lock);
 318	}
 319	spin_lock(&fc->lock);
 320	if (!RB_EMPTY_NODE(&ff->polled_node))
 321		rb_erase(&ff->polled_node, &fc->polled_files);
 322	spin_unlock(&fc->lock);
 323
 324	wake_up_interruptible_all(&ff->poll_wait);
 325
 326	if (!ra)
 327		return;
 328
 329	/* ff->args was used for open outarg */
 330	memset(ff->args, 0, sizeof(*ff->args));
 331	ra->inarg.fh = ff->fh;
 332	ra->inarg.flags = flags;
 333	ra->args.in_numargs = 1;
 334	ra->args.in_args[0].size = sizeof(struct fuse_release_in);
 335	ra->args.in_args[0].value = &ra->inarg;
 336	ra->args.opcode = opcode;
 337	ra->args.nodeid = ff->nodeid;
 338	ra->args.force = true;
 339	ra->args.nocreds = true;
 340
 341	/*
 342	 * Hold inode until release is finished.
 343	 * From fuse_sync_release() the refcount is 1 and everything's
 344	 * synchronous, so we are fine with not doing igrab() here.
 345	 */
 346	ra->inode = sync ? NULL : igrab(&fi->inode);
 347}
 348
 349void fuse_file_release(struct inode *inode, struct fuse_file *ff,
 350		       unsigned int open_flags, fl_owner_t id, bool isdir)
 351{
 352	struct fuse_inode *fi = get_fuse_inode(inode);
 353	struct fuse_release_args *ra = &ff->args->release_args;
 354	int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
 355
 356	fuse_prepare_release(fi, ff, open_flags, opcode, false);
 357
 358	if (ra && ff->flock) {
 359		ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
 360		ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id);
 361	}
 362
 363	/*
 364	 * Normally this will send the RELEASE request, however if
 365	 * some asynchronous READ or WRITE requests are outstanding,
 366	 * the sending will be delayed.
 367	 *
 368	 * Make the release synchronous if this is a fuseblk mount,
 369	 * synchronous RELEASE is allowed (and desirable) in this case
 370	 * because the server can be trusted not to screw up.
 371	 *
 372	 * Always use the asynchronous file put because the current thread
 373	 * might be the fuse server.  This can happen if a process starts some
 374	 * aio and closes the fd before the aio completes.  Since aio takes its
 375	 * own ref to the file, the IO completion has to drop the ref, which is
 376	 * how the fuse server can end up closing its clients' files.
 377	 */
 378	fuse_file_put(ff, false);
 379}
 380
 381void fuse_release_common(struct file *file, bool isdir)
 382{
 383	fuse_file_release(file_inode(file), file->private_data, file->f_flags,
 384			  (fl_owner_t) file, isdir);
 385}
 386
 387static int fuse_release(struct inode *inode, struct file *file)
 388{
 389	struct fuse_conn *fc = get_fuse_conn(inode);
 390
 391	/*
 392	 * Dirty pages might remain despite write_inode_now() call from
 393	 * fuse_flush() due to writes racing with the close.
 394	 */
 395	if (fc->writeback_cache)
 396		write_inode_now(inode, 1);
 397
 398	fuse_release_common(file, false);
 399
 400	/* return value is ignored by VFS */
 401	return 0;
 402}
 403
 404void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff,
 405		       unsigned int flags)
 406{
 407	WARN_ON(refcount_read(&ff->count) > 1);
 408	fuse_prepare_release(fi, ff, flags, FUSE_RELEASE, true);
 409	fuse_file_put(ff, true);
 410}
 411EXPORT_SYMBOL_GPL(fuse_sync_release);
 412
 413/*
 414 * Scramble the ID space with XTEA, so that the value of the files_struct
 415 * pointer is not exposed to userspace.
 416 */
 417u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
 418{
 419	u32 *k = fc->scramble_key;
 420	u64 v = (unsigned long) id;
 421	u32 v0 = v;
 422	u32 v1 = v >> 32;
 423	u32 sum = 0;
 424	int i;
 425
 426	for (i = 0; i < 32; i++) {
 427		v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
 428		sum += 0x9E3779B9;
 429		v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
 430	}
 431
 432	return (u64) v0 + ((u64) v1 << 32);
 433}
 434
 435struct fuse_writepage_args {
 436	struct fuse_io_args ia;
 437	struct list_head queue_entry;
 438	struct inode *inode;
 439	struct fuse_sync_bucket *bucket;
 440};
 441
 442/*
 443 * Wait for all pending writepages on the inode to finish.
 444 *
 445 * This is currently done by blocking further writes with FUSE_NOWRITE
 446 * and waiting for all sent writes to complete.
 447 *
 448 * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
 449 * could conflict with truncation.
 450 */
 451static void fuse_sync_writes(struct inode *inode)
 452{
 453	fuse_set_nowrite(inode);
 454	fuse_release_nowrite(inode);
 455}
 456
 457static int fuse_flush(struct file *file, fl_owner_t id)
 458{
 459	struct inode *inode = file_inode(file);
 460	struct fuse_mount *fm = get_fuse_mount(inode);
 461	struct fuse_file *ff = file->private_data;
 462	struct fuse_flush_in inarg;
 463	FUSE_ARGS(args);
 464	int err;
 465
 466	if (fuse_is_bad(inode))
 467		return -EIO;
 468
 469	if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache)
 470		return 0;
 471
 472	err = write_inode_now(inode, 1);
 473	if (err)
 474		return err;
 475
 476	err = filemap_check_errors(file->f_mapping);
 477	if (err)
 478		return err;
 479
 480	err = 0;
 481	if (fm->fc->no_flush)
 482		goto inval_attr_out;
 483
 484	memset(&inarg, 0, sizeof(inarg));
 485	inarg.fh = ff->fh;
 486	inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
 487	args.opcode = FUSE_FLUSH;
 488	args.nodeid = get_node_id(inode);
 489	args.in_numargs = 1;
 490	args.in_args[0].size = sizeof(inarg);
 491	args.in_args[0].value = &inarg;
 492	args.force = true;
 493
 494	err = fuse_simple_request(fm, &args);
 495	if (err == -ENOSYS) {
 496		fm->fc->no_flush = 1;
 497		err = 0;
 498	}
 499
 500inval_attr_out:
 501	/*
 502	 * In memory i_blocks is not maintained by fuse, if writeback cache is
 503	 * enabled, i_blocks from cached attr may not be accurate.
 504	 */
 505	if (!err && fm->fc->writeback_cache)
 506		fuse_invalidate_attr_mask(inode, STATX_BLOCKS);
 507	return err;
 508}
 509
 510int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
 511		      int datasync, int opcode)
 512{
 513	struct inode *inode = file->f_mapping->host;
 514	struct fuse_mount *fm = get_fuse_mount(inode);
 515	struct fuse_file *ff = file->private_data;
 516	FUSE_ARGS(args);
 517	struct fuse_fsync_in inarg;
 518
 519	memset(&inarg, 0, sizeof(inarg));
 520	inarg.fh = ff->fh;
 521	inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
 522	args.opcode = opcode;
 523	args.nodeid = get_node_id(inode);
 524	args.in_numargs = 1;
 525	args.in_args[0].size = sizeof(inarg);
 526	args.in_args[0].value = &inarg;
 527	return fuse_simple_request(fm, &args);
 528}
 529
 530static int fuse_fsync(struct file *file, loff_t start, loff_t end,
 531		      int datasync)
 532{
 533	struct inode *inode = file->f_mapping->host;
 534	struct fuse_conn *fc = get_fuse_conn(inode);
 535	int err;
 536
 537	if (fuse_is_bad(inode))
 538		return -EIO;
 539
 540	inode_lock(inode);
 541
 542	/*
 543	 * Start writeback against all dirty pages of the inode, then
 544	 * wait for all outstanding writes, before sending the FSYNC
 545	 * request.
 546	 */
 547	err = file_write_and_wait_range(file, start, end);
 548	if (err)
 549		goto out;
 550
 551	fuse_sync_writes(inode);
 552
 553	/*
 554	 * Due to implementation of fuse writeback
 555	 * file_write_and_wait_range() does not catch errors.
 556	 * We have to do this directly after fuse_sync_writes()
 557	 */
 558	err = file_check_and_advance_wb_err(file);
 559	if (err)
 560		goto out;
 561
 562	err = sync_inode_metadata(inode, 1);
 563	if (err)
 564		goto out;
 565
 566	if (fc->no_fsync)
 567		goto out;
 568
 569	err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
 570	if (err == -ENOSYS) {
 571		fc->no_fsync = 1;
 572		err = 0;
 573	}
 574out:
 575	inode_unlock(inode);
 576
 577	return err;
 578}
 579
 580void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 581			 size_t count, int opcode)
 582{
 583	struct fuse_file *ff = file->private_data;
 584	struct fuse_args *args = &ia->ap.args;
 585
 586	ia->read.in.fh = ff->fh;
 587	ia->read.in.offset = pos;
 588	ia->read.in.size = count;
 589	ia->read.in.flags = file->f_flags;
 590	args->opcode = opcode;
 591	args->nodeid = ff->nodeid;
 592	args->in_numargs = 1;
 593	args->in_args[0].size = sizeof(ia->read.in);
 594	args->in_args[0].value = &ia->read.in;
 595	args->out_argvar = true;
 596	args->out_numargs = 1;
 597	args->out_args[0].size = count;
 598}
 599
 600static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres,
 601				    bool should_dirty)
 602{
 603	unsigned int i;
 604
 605	for (i = 0; i < ap->num_folios; i++) {
 606		if (should_dirty)
 607			folio_mark_dirty_lock(ap->folios[i]);
 608		if (ap->args.is_pinned)
 609			unpin_folio(ap->folios[i]);
 610	}
 611
 612	if (nres > 0 && ap->args.invalidate_vmap)
 613		invalidate_kernel_vmap_range(ap->args.vmap_base, nres);
 614}
 615
 616static void fuse_io_release(struct kref *kref)
 617{
 618	kfree(container_of(kref, struct fuse_io_priv, refcnt));
 619}
 620
 621static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
 622{
 623	if (io->err)
 624		return io->err;
 625
 626	if (io->bytes >= 0 && io->write)
 627		return -EIO;
 628
 629	return io->bytes < 0 ? io->size : io->bytes;
 630}
 631
 632/*
 633 * In case of short read, the caller sets 'pos' to the position of
 634 * actual end of fuse request in IO request. Otherwise, if bytes_requested
 635 * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
 636 *
 637 * An example:
 638 * User requested DIO read of 64K. It was split into two 32K fuse requests,
 639 * both submitted asynchronously. The first of them was ACKed by userspace as
 640 * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
 641 * second request was ACKed as short, e.g. only 1K was read, resulting in
 642 * pos == 33K.
 643 *
 644 * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
 645 * will be equal to the length of the longest contiguous fragment of
 646 * transferred data starting from the beginning of IO request.
 647 */
 648static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
 649{
 650	int left;
 651
 652	spin_lock(&io->lock);
 653	if (err)
 654		io->err = io->err ? : err;
 655	else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
 656		io->bytes = pos;
 657
 658	left = --io->reqs;
 659	if (!left && io->blocking)
 660		complete(io->done);
 661	spin_unlock(&io->lock);
 662
 663	if (!left && !io->blocking) {
 664		ssize_t res = fuse_get_res_by_io(io);
 665
 666		if (res >= 0) {
 667			struct inode *inode = file_inode(io->iocb->ki_filp);
 668			struct fuse_conn *fc = get_fuse_conn(inode);
 669			struct fuse_inode *fi = get_fuse_inode(inode);
 670
 671			spin_lock(&fi->lock);
 672			fi->attr_version = atomic64_inc_return(&fc->attr_version);
 673			spin_unlock(&fi->lock);
 674		}
 675
 676		io->iocb->ki_complete(io->iocb, res);
 677	}
 678
 679	kref_put(&io->refcnt, fuse_io_release);
 680}
 681
 682static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
 683						 unsigned int nfolios)
 684{
 685	struct fuse_io_args *ia;
 686
 687	ia = kzalloc(sizeof(*ia), GFP_KERNEL);
 688	if (ia) {
 689		ia->io = io;
 690		ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL,
 691						  &ia->ap.descs);
 692		if (!ia->ap.folios) {
 693			kfree(ia);
 694			ia = NULL;
 695		}
 696	}
 697	return ia;
 698}
 699
 700static void fuse_io_free(struct fuse_io_args *ia)
 701{
 702	kfree(ia->ap.folios);
 703	kfree(ia);
 704}
 705
 706static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
 707				  int err)
 708{
 709	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 710	struct fuse_io_priv *io = ia->io;
 711	ssize_t pos = -1;
 712	size_t nres;
 713
 714	if (err) {
 715		/* Nothing */
 716	} else if (io->write) {
 717		if (ia->write.out.size > ia->write.in.size) {
 718			err = -EIO;
 719		} else {
 720			nres = ia->write.out.size;
 721			if (ia->write.in.size != ia->write.out.size)
 722				pos = ia->write.in.offset - io->offset +
 723				      ia->write.out.size;
 724		}
 725	} else {
 726		u32 outsize = args->out_args[0].size;
 727
 728		nres = outsize;
 729		if (ia->read.in.size != outsize)
 730			pos = ia->read.in.offset - io->offset + outsize;
 731	}
 732
 733	fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty);
 734
 735	fuse_aio_complete(io, err, pos);
 736	fuse_io_free(ia);
 737}
 738
 739static ssize_t fuse_async_req_send(struct fuse_mount *fm,
 740				   struct fuse_io_args *ia, size_t num_bytes)
 741{
 742	ssize_t err;
 743	struct fuse_io_priv *io = ia->io;
 744
 745	spin_lock(&io->lock);
 746	kref_get(&io->refcnt);
 747	io->size += num_bytes;
 748	io->reqs++;
 749	spin_unlock(&io->lock);
 750
 751	ia->ap.args.end = fuse_aio_complete_req;
 752	ia->ap.args.may_block = io->should_dirty;
 753	err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
 754	if (err)
 755		fuse_aio_complete_req(fm, &ia->ap.args, err);
 756
 757	return num_bytes;
 758}
 759
 760static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
 761			      fl_owner_t owner)
 762{
 763	struct file *file = ia->io->iocb->ki_filp;
 764	struct fuse_file *ff = file->private_data;
 765	struct fuse_mount *fm = ff->fm;
 766
 767	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 768	if (owner != NULL) {
 769		ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
 770		ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
 771	}
 772
 773	if (ia->io->async)
 774		return fuse_async_req_send(fm, ia, count);
 775
 776	return fuse_simple_request(fm, &ia->ap.args);
 777}
 778
 779static void fuse_read_update_size(struct inode *inode, loff_t size,
 780				  u64 attr_ver)
 781{
 782	struct fuse_conn *fc = get_fuse_conn(inode);
 783	struct fuse_inode *fi = get_fuse_inode(inode);
 784
 785	spin_lock(&fi->lock);
 786	if (attr_ver >= fi->attr_version && size < inode->i_size &&
 787	    !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
 788		fi->attr_version = atomic64_inc_return(&fc->attr_version);
 789		i_size_write(inode, size);
 790	}
 791	spin_unlock(&fi->lock);
 792}
 793
 794static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
 795			    struct fuse_args_pages *ap)
 796{
 797	struct fuse_conn *fc = get_fuse_conn(inode);
 798
 799	/*
 800	 * If writeback_cache is enabled, a short read means there's a hole in
 801	 * the file.  Some data after the hole is in page cache, but has not
 802	 * reached the client fs yet.  So the hole is not present there.
 803	 */
 804	if (!fc->writeback_cache) {
 805		loff_t pos = folio_pos(ap->folios[0]) + num_read;
 806		fuse_read_update_size(inode, pos, attr_ver);
 807	}
 808}
 809
 810static int fuse_do_readfolio(struct file *file, struct folio *folio,
 811			     size_t off, size_t len)
 812{
 813	struct inode *inode = folio->mapping->host;
 814	struct fuse_mount *fm = get_fuse_mount(inode);
 815	loff_t pos = folio_pos(folio) + off;
 816	struct fuse_folio_desc desc = {
 817		.offset = off,
 818		.length = len,
 819	};
 820	struct fuse_io_args ia = {
 821		.ap.args.page_zeroing = true,
 822		.ap.args.out_pages = true,
 823		.ap.num_folios = 1,
 824		.ap.folios = &folio,
 825		.ap.descs = &desc,
 826	};
 827	ssize_t res;
 828	u64 attr_ver;
 829
 830	attr_ver = fuse_get_attr_version(fm->fc);
 831
 832	/* Don't overflow end offset */
 833	if (pos + (desc.length - 1) == LLONG_MAX)
 834		desc.length--;
 835
 836	fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
 837	res = fuse_simple_request(fm, &ia.ap.args);
 838	if (res < 0)
 839		return res;
 840	/*
 841	 * Short read means EOF.  If file size is larger, truncate it
 842	 */
 843	if (res < desc.length)
 844		fuse_short_read(inode, attr_ver, res, &ia.ap);
 845
 846	return 0;
 847}
 848
 849static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 850			    unsigned int flags, struct iomap *iomap,
 851			    struct iomap *srcmap)
 852{
 853	iomap->type = IOMAP_MAPPED;
 854	iomap->length = length;
 855	iomap->offset = offset;
 856	return 0;
 857}
 858
 859static const struct iomap_ops fuse_iomap_ops = {
 860	.iomap_begin	= fuse_iomap_begin,
 861};
 862
 863struct fuse_fill_read_data {
 864	struct file *file;
 865
 866	/* Fields below are used if sending the read request asynchronously */
 867	struct fuse_conn *fc;
 868	struct fuse_io_args *ia;
 869	unsigned int nr_bytes;
 870};
 871
 872/* forward declarations */
 873static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
 874				  unsigned len, struct fuse_args_pages *ap,
 875				  unsigned cur_bytes, bool write);
 876static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
 877				unsigned int count, bool async);
 878
 879static int fuse_handle_readahead(struct folio *folio,
 880				 struct readahead_control *rac,
 881				 struct fuse_fill_read_data *data, loff_t pos,
 882				 size_t len)
 883{
 884	struct fuse_io_args *ia = data->ia;
 885	size_t off = offset_in_folio(folio, pos);
 886	struct fuse_conn *fc = data->fc;
 887	struct fuse_args_pages *ap;
 888	unsigned int nr_pages;
 889
 890	if (ia && fuse_folios_need_send(fc, pos, len, &ia->ap, data->nr_bytes,
 891					false)) {
 892		fuse_send_readpages(ia, data->file, data->nr_bytes,
 893				    fc->async_read);
 894		data->nr_bytes = 0;
 895		data->ia = NULL;
 896		ia = NULL;
 897	}
 898	if (!ia) {
 899		if (fc->num_background >= fc->congestion_threshold &&
 900		    rac->ra->async_size >= readahead_count(rac))
 901			/*
 902			 * Congested and only async pages left, so skip the
 903			 * rest.
 904			 */
 905			return -EAGAIN;
 906
 907		nr_pages = min(fc->max_pages, readahead_count(rac));
 908		data->ia = fuse_io_alloc(NULL, nr_pages);
 909		if (!data->ia)
 910			return -ENOMEM;
 911		ia = data->ia;
 912	}
 913	folio_get(folio);
 914	ap = &ia->ap;
 915	ap->folios[ap->num_folios] = folio;
 916	ap->descs[ap->num_folios].offset = off;
 917	ap->descs[ap->num_folios].length = len;
 918	data->nr_bytes += len;
 919	ap->num_folios++;
 920
 921	return 0;
 922}
 923
 924static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter,
 925					     struct iomap_read_folio_ctx *ctx,
 926					     size_t len)
 927{
 928	struct fuse_fill_read_data *data = ctx->read_ctx;
 929	struct folio *folio = ctx->cur_folio;
 930	loff_t pos =  iter->pos;
 931	size_t off = offset_in_folio(folio, pos);
 932	struct file *file = data->file;
 933	int ret;
 934
 935	if (ctx->rac) {
 936		ret = fuse_handle_readahead(folio, ctx->rac, data, pos, len);
 937	} else {
 938		/*
 939		 *  for non-readahead read requests, do reads synchronously
 940		 *  since it's not guaranteed that the server can handle
 941		 *  out-of-order reads
 942		 */
 943		ret = fuse_do_readfolio(file, folio, off, len);
 944		if (!ret)
 945			iomap_finish_folio_read(folio, off, len, ret);
 946	}
 947	return ret;
 948}
 949
 950static void fuse_iomap_read_submit(struct iomap_read_folio_ctx *ctx)
 951{
 952	struct fuse_fill_read_data *data = ctx->read_ctx;
 953
 954	if (data->ia)
 955		fuse_send_readpages(data->ia, data->file, data->nr_bytes,
 956				    data->fc->async_read);
 957}
 958
 959static const struct iomap_read_ops fuse_iomap_read_ops = {
 960	.read_folio_range = fuse_iomap_read_folio_range_async,
 961	.submit_read = fuse_iomap_read_submit,
 962};
 963
 964static int fuse_read_folio(struct file *file, struct folio *folio)
 965{
 966	struct inode *inode = folio->mapping->host;
 967	struct fuse_fill_read_data data = {
 968		.file = file,
 969	};
 970	struct iomap_read_folio_ctx ctx = {
 971		.cur_folio = folio,
 972		.ops = &fuse_iomap_read_ops,
 973		.read_ctx = &data,
 974
 975	};
 976
 977	if (fuse_is_bad(inode)) {
 978		folio_unlock(folio);
 979		return -EIO;
 980	}
 981
 982	iomap_read_folio(&fuse_iomap_ops, &ctx);
 983	fuse_invalidate_atime(inode);
 984	return 0;
 985}
 986
 987static int fuse_iomap_read_folio_range(const struct iomap_iter *iter,
 988				       struct folio *folio, loff_t pos,
 989				       size_t len)
 990{
 991	struct file *file = iter->private;
 992	size_t off = offset_in_folio(folio, pos);
 993
 994	return fuse_do_readfolio(file, folio, off, len);
 995}
 996
 997static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
 998			       int err)
 999{
1000	int i;
1001	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
1002	struct fuse_args_pages *ap = &ia->ap;
1003	size_t count = ia->read.in.size;
1004	size_t num_read = args->out_args[0].size;
1005	struct address_space *mapping;
1006	struct inode *inode;
1007
1008	WARN_ON_ONCE(!ap->num_folios);
1009	mapping = ap->folios[0]->mapping;
1010	inode = mapping->host;
1011
1012	/*
1013	 * Short read means EOF. If file size is larger, truncate it
1014	 */
1015	if (!err && num_read < count)
1016		fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
1017
1018	fuse_invalidate_atime(inode);
1019
1020	for (i = 0; i < ap->num_folios; i++) {
1021		iomap_finish_folio_read(ap->folios[i], ap->descs[i].offset,
1022					ap->descs[i].length, err);
1023		folio_put(ap->folios[i]);
1024	}
1025	if (ia->ff)
1026		fuse_file_put(ia->ff, false);
1027
1028	fuse_io_free(ia);
1029}
1030
1031static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
1032				unsigned int count, bool async)
1033{
1034	struct fuse_file *ff = file->private_data;
1035	struct fuse_mount *fm = ff->fm;
1036	struct fuse_args_pages *ap = &ia->ap;
1037	loff_t pos = folio_pos(ap->folios[0]);
1038	ssize_t res;
1039	int err;
1040
1041	ap->args.out_pages = true;
1042	ap->args.page_zeroing = true;
1043	ap->args.page_replace = true;
1044
1045	/* Don't overflow end offset */
1046	if (pos + (count - 1) == LLONG_MAX) {
1047		count--;
1048		ap->descs[ap->num_folios - 1].length--;
1049	}
1050	WARN_ON((loff_t) (pos + count) < 0);
1051
1052	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
1053	ia->read.attr_ver = fuse_get_attr_version(fm->fc);
1054	if (async) {
1055		ia->ff = fuse_file_get(ff);
1056		ap->args.end = fuse_readpages_end;
1057		err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
1058		if (!err)
1059			return;
1060	} else {
1061		res = fuse_simple_request(fm, &ap->args);
1062		err = res < 0 ? res : 0;
1063	}
1064	fuse_readpages_end(fm, &ap->args, err);
1065}
1066
1067static void fuse_readahead(struct readahead_control *rac)
1068{
1069	struct inode *inode = rac->mapping->host;
1070	struct fuse_conn *fc = get_fuse_conn(inode);
1071	struct fuse_fill_read_data data = {
1072		.file = rac->file,
1073		.fc = fc,
1074	};
1075	struct iomap_read_folio_ctx ctx = {
1076		.ops = &fuse_iomap_read_ops,
1077		.rac = rac,
1078		.read_ctx = &data
1079	};
1080
1081	if (fuse_is_bad(inode))
1082		return;
1083
1084	iomap_readahead(&fuse_iomap_ops, &ctx);
1085}
1086
1087static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
1088{
1089	struct inode *inode = iocb->ki_filp->f_mapping->host;
1090	struct fuse_conn *fc = get_fuse_conn(inode);
1091
1092	/*
1093	 * In auto invalidate mode, always update attributes on read.
1094	 * Otherwise, only update if we attempt to read past EOF (to ensure
1095	 * i_size is up to date).
1096	 */
1097	if (fc->auto_inval_data ||
1098	    (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
1099		int err;
1100		err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE);
1101		if (err)
1102			return err;
1103	}
1104
1105	return generic_file_read_iter(iocb, to);
1106}
1107
1108static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
1109				 loff_t pos, size_t count)
1110{
1111	struct fuse_args *args = &ia->ap.args;
1112
1113	ia->write.in.fh = ff->fh;
1114	ia->write.in.offset = pos;
1115	ia->write.in.size = count;
1116	args->opcode = FUSE_WRITE;
1117	args->nodeid = ff->nodeid;
1118	args->in_numargs = 2;
1119	if (ff->fm->fc->minor < 9)
1120		args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
1121	else
1122		args->in_args[0].size = sizeof(ia->write.in);
1123	args->in_args[0].value = &ia->write.in;
1124	args->in_args[1].size = count;
1125	args->out_numargs = 1;
1126	args->out_args[0].size = sizeof(ia->write.out);
1127	args->out_args[0].value = &ia->write.out;
1128}
1129
1130static unsigned int fuse_write_flags(struct kiocb *iocb)
1131{
1132	unsigned int flags = iocb->ki_filp->f_flags;
1133
1134	if (iocb_is_dsync(iocb))
1135		flags |= O_DSYNC;
1136	if (iocb->ki_flags & IOCB_SYNC)
1137		flags |= O_SYNC;
1138
1139	return flags;
1140}
1141
1142static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
1143			       size_t count, fl_owner_t owner)
1144{
1145	struct kiocb *iocb = ia->io->iocb;
1146	struct file *file = iocb->ki_filp;
1147	struct fuse_file *ff = file->private_data;
1148	struct fuse_mount *fm = ff->fm;
1149	struct fuse_write_in *inarg = &ia->write.in;
1150	ssize_t err;
1151
1152	fuse_write_args_fill(ia, ff, pos, count);
1153	inarg->flags = fuse_write_flags(iocb);
1154	if (owner != NULL) {
1155		inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
1156		inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
1157	}
1158
1159	if (ia->io->async)
1160		return fuse_async_req_send(fm, ia, count);
1161
1162	err = fuse_simple_request(fm, &ia->ap.args);
1163	if (!err && ia->write.out.size > count)
1164		err = -EIO;
1165
1166	return err ?: ia->write.out.size;
1167}
1168
1169bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written)
1170{
1171	struct fuse_conn *fc = get_fuse_conn(inode);
1172	struct fuse_inode *fi = get_fuse_inode(inode);
1173	bool ret = false;
1174
1175	spin_lock(&fi->lock);
1176	fi->attr_version = atomic64_inc_return(&fc->attr_version);
1177	if (written > 0 && pos > inode->i_size) {
1178		i_size_write(inode, pos);
1179		ret = true;
1180	}
1181	spin_unlock(&fi->lock);
1182
1183	fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
1184
1185	return ret;
1186}
1187
1188static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
1189				     struct kiocb *iocb, struct inode *inode,
1190				     loff_t pos, size_t count)
1191{
1192	struct fuse_args_pages *ap = &ia->ap;
1193	struct file *file = iocb->ki_filp;
1194	struct fuse_file *ff = file->private_data;
1195	struct fuse_mount *fm = ff->fm;
1196	unsigned int offset, i;
1197	bool short_write;
1198	int err;
1199
1200	for (i = 0; i < ap->num_folios; i++)
1201		folio_wait_writeback(ap->folios[i]);
1202
1203	fuse_write_args_fill(ia, ff, pos, count);
1204	ia->write.in.flags = fuse_write_flags(iocb);
1205	if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID))
1206		ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
1207
1208	err = fuse_simple_request(fm, &ap->args);
1209	if (!err && ia->write.out.size > count)
1210		err = -EIO;
1211
1212	short_write = ia->write.out.size < count;
1213	offset = ap->descs[0].offset;
1214	count = ia->write.out.size;
1215	for (i = 0; i < ap->num_folios; i++) {
1216		struct folio *folio = ap->folios[i];
1217
1218		if (err) {
1219			folio_clear_uptodate(folio);
1220		} else {
1221			if (count >= folio_size(folio) - offset)
1222				count -= folio_size(folio) - offset;
1223			else {
1224				if (short_write)
1225					folio_clear_uptodate(folio);
1226				count = 0;
1227			}
1228			offset = 0;
1229		}
1230		if (ia->write.folio_locked && (i == ap->num_folios - 1))
1231			folio_unlock(folio);
1232		folio_put(folio);
1233	}
1234
1235	return err;
1236}
1237
1238static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
1239				     struct address_space *mapping,
1240				     struct iov_iter *ii, loff_t pos,
1241				     unsigned int max_folios)
1242{
1243	struct fuse_args_pages *ap = &ia->ap;
1244	struct fuse_conn *fc = get_fuse_conn(mapping->host);
1245	unsigned offset = pos & (PAGE_SIZE - 1);
1246	size_t count = 0;
1247	unsigned int num;
1248	int err = 0;
1249
1250	num = min(iov_iter_count(ii), fc->max_write);
1251
1252	ap->args.in_pages = true;
1253
1254	while (num && ap->num_folios < max_folios) {
1255		size_t tmp;
1256		struct folio *folio;
1257		pgoff_t index = pos >> PAGE_SHIFT;
1258		unsigned int bytes;
1259		unsigned int folio_offset;
1260
1261 again:
1262		folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
1263					    mapping_gfp_mask(mapping));
1264		if (IS_ERR(folio)) {
1265			err = PTR_ERR(folio);
1266			break;
1267		}
1268
1269		if (mapping_writably_mapped(mapping))
1270			flush_dcache_folio(folio);
1271
1272		folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
1273		bytes = min(folio_size(folio) - folio_offset, num);
1274
1275		tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii);
1276		flush_dcache_folio(folio);
1277
1278		if (!tmp) {
1279			folio_unlock(folio);
1280			folio_put(folio);
1281
1282			/*
1283			 * Ensure forward progress by faulting in
1284			 * while not holding the folio lock:
1285			 */
1286			if (fault_in_iov_iter_readable(ii, bytes)) {
1287				err = -EFAULT;
1288				break;
1289			}
1290
1291			goto again;
1292		}
1293
1294		ap->folios[ap->num_folios] = folio;
1295		ap->descs[ap->num_folios].offset = folio_offset;
1296		ap->descs[ap->num_folios].length = tmp;
1297		ap->num_folios++;
1298
1299		count += tmp;
1300		pos += tmp;
1301		num -= tmp;
1302		offset += tmp;
1303		if (offset == folio_size(folio))
1304			offset = 0;
1305
1306		/* If we copied full folio, mark it uptodate */
1307		if (tmp == folio_size(folio))
1308			folio_mark_uptodate(folio);
1309
1310		if (folio_test_uptodate(folio)) {
1311			folio_unlock(folio);
1312		} else {
1313			ia->write.folio_locked = true;
1314			break;
1315		}
1316		if (!fc->big_writes || offset != 0)
1317			break;
1318	}
1319
1320	return count > 0 ? count : err;
1321}
1322
1323static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1324				     unsigned int max_pages)
1325{
1326	return min_t(unsigned int,
1327		     ((pos + len - 1) >> PAGE_SHIFT) -
1328		     (pos >> PAGE_SHIFT) + 1,
1329		     max_pages);
1330}
1331
1332static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii)
1333{
1334	struct address_space *mapping = iocb->ki_filp->f_mapping;
1335	struct inode *inode = mapping->host;
1336	struct fuse_conn *fc = get_fuse_conn(inode);
1337	struct fuse_inode *fi = get_fuse_inode(inode);
1338	loff_t pos = iocb->ki_pos;
1339	int err = 0;
1340	ssize_t res = 0;
1341
1342	if (inode->i_size < pos + iov_iter_count(ii))
1343		set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1344
1345	do {
1346		ssize_t count;
1347		struct fuse_io_args ia = {};
1348		struct fuse_args_pages *ap = &ia.ap;
1349		unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1350						      fc->max_pages);
1351
1352		ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs);
1353		if (!ap->folios) {
1354			err = -ENOMEM;
1355			break;
1356		}
1357
1358		count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);
1359		if (count <= 0) {
1360			err = count;
1361		} else {
1362			err = fuse_send_write_pages(&ia, iocb, inode,
1363						    pos, count);
1364			if (!err) {
1365				size_t num_written = ia.write.out.size;
1366
1367				res += num_written;
1368				pos += num_written;
1369
1370				/* break out of the loop on short write */
1371				if (num_written != count)
1372					err = -EIO;
1373			}
1374		}
1375		kfree(ap->folios);
1376	} while (!err && iov_iter_count(ii));
1377
1378	fuse_write_update_attr(inode, pos, res);
1379	clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1380
1381	if (!res)
1382		return err;
1383	iocb->ki_pos += res;
1384	return res;
1385}
1386
1387static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter)
1388{
1389	struct inode *inode = file_inode(iocb->ki_filp);
1390
1391	return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode);
1392}
1393
1394/*
1395 * @return true if an exclusive lock for direct IO writes is needed
1396 */
1397static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from)
1398{
1399	struct file *file = iocb->ki_filp;
1400	struct fuse_file *ff = file->private_data;
1401	struct inode *inode = file_inode(iocb->ki_filp);
1402	struct fuse_inode *fi = get_fuse_inode(inode);
1403
1404	/* Server side has to advise that it supports parallel dio writes. */
1405	if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES))
1406		return true;
1407
1408	/*
1409	 * Append will need to know the eventual EOF - always needs an
1410	 * exclusive lock.
1411	 */
1412	if (iocb->ki_flags & IOCB_APPEND)
1413		return true;
1414
1415	/* shared locks are not allowed with parallel page cache IO */
1416	if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
1417		return true;
1418
1419	/* Parallel dio beyond EOF is not supported, at least for now. */
1420	if (fuse_io_past_eof(iocb, from))
1421		return true;
1422
1423	return false;
1424}
1425
1426static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from,
1427			  bool *exclusive)
1428{
1429	struct inode *inode = file_inode(iocb->ki_filp);
1430	struct fuse_inode *fi = get_fuse_inode(inode);
1431
1432	*exclusive = fuse_dio_wr_exclusive_lock(iocb, from);
1433	if (*exclusive) {
1434		inode_lock(inode);
1435	} else {
1436		inode_lock_shared(inode);
1437		/*
1438		 * New parallal dio allowed only if inode is not in caching
1439		 * mode and denies new opens in caching mode. This check
1440		 * should be performed only after taking shared inode lock.
1441		 * Previous past eof check was without inode lock and might
1442		 * have raced, so check it again.
1443		 */
1444		if (fuse_io_past_eof(iocb, from) ||
1445		    fuse_inode_uncached_io_start(fi, NULL) != 0) {
1446			inode_unlock_shared(inode);
1447			inode_lock(inode);
1448			*exclusive = true;
1449		}
1450	}
1451}
1452
1453static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive)
1454{
1455	struct inode *inode = file_inode(iocb->ki_filp);
1456	struct fuse_inode *fi = get_fuse_inode(inode);
1457
1458	if (exclusive) {
1459		inode_unlock(inode);
1460	} else {
1461		/* Allow opens in caching mode after last parallel dio end */
1462		fuse_inode_uncached_io_end(fi);
1463		inode_unlock_shared(inode);
1464	}
1465}
1466
1467static const struct iomap_write_ops fuse_iomap_write_ops = {
1468	.read_folio_range = fuse_iomap_read_folio_range,
1469};
1470
1471static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
1472{
1473	struct file *file = iocb->ki_filp;
1474	struct mnt_idmap *idmap = file_mnt_idmap(file);
1475	struct address_space *mapping = file->f_mapping;
1476	ssize_t written = 0;
1477	struct inode *inode = mapping->host;
1478	ssize_t err, count;
1479	struct fuse_conn *fc = get_fuse_conn(inode);
1480	bool writeback = false;
1481
1482	if (fc->writeback_cache) {
1483		/* Update size (EOF optimization) and mode (SUID clearing) */
1484		err = fuse_update_attributes(mapping->host, file,
1485					     STATX_SIZE | STATX_MODE);
1486		if (err)
1487			return err;
1488
1489		if (!fc->handle_killpriv_v2 ||
1490		    !setattr_should_drop_suidgid(idmap, file_inode(file)))
1491			writeback = true;
1492	}
1493
1494	inode_lock(inode);
1495
1496	err = count = generic_write_checks(iocb, from);
1497	if (err <= 0)
1498		goto out;
1499
1500	task_io_account_write(count);
1501
1502	err = kiocb_modified(iocb);
1503	if (err)
1504		goto out;
1505
1506	if (iocb->ki_flags & IOCB_DIRECT) {
1507		written = generic_file_direct_write(iocb, from);
1508		if (written < 0 || !iov_iter_count(from))
1509			goto out;
1510		written = direct_write_fallback(iocb, from, written,
1511				fuse_perform_write(iocb, from));
1512	} else if (writeback) {
1513		/*
1514		 * Use iomap so that we can do granular uptodate reads
1515		 * and granular dirty tracking for large folios.
1516		 */
1517		written = iomap_file_buffered_write(iocb, from,
1518						    &fuse_iomap_ops,
1519						    &fuse_iomap_write_ops,
1520						    file);
1521	} else {
1522		written = fuse_perform_write(iocb, from);
1523	}
1524out:
1525	inode_unlock(inode);
1526	if (written > 0)
1527		written = generic_write_sync(iocb, written);
1528
1529	return written ? written : err;
1530}
1531
1532static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1533{
1534	return (unsigned long)iter_iov(ii)->iov_base + ii->iov_offset;
1535}
1536
1537static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1538					size_t max_size)
1539{
1540	return min(iov_iter_single_seg_count(ii), max_size);
1541}
1542
1543static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1544			       size_t *nbytesp, int write,
1545			       unsigned int max_pages,
1546			       bool use_pages_for_kvec_io)
1547{
1548	bool flush_or_invalidate = false;
1549	unsigned int nr_pages = 0;
1550	size_t nbytes = 0;  /* # bytes already packed in req */
1551	ssize_t ret = 0;
1552
1553	/* Special case for kernel I/O: can copy directly into the buffer.
1554	 * However if the implementation of fuse_conn requires pages instead of
1555	 * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead.
1556	 */
1557	if (iov_iter_is_kvec(ii)) {
1558		void *user_addr = (void *)fuse_get_user_addr(ii);
1559
1560		if (!use_pages_for_kvec_io) {
1561			size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1562
1563			if (write)
1564				ap->args.in_args[1].value = user_addr;
1565			else
1566				ap->args.out_args[0].value = user_addr;
1567
1568			iov_iter_advance(ii, frag_size);
1569			*nbytesp = frag_size;
1570			return 0;
1571		}
1572
1573		if (is_vmalloc_addr(user_addr)) {
1574			ap->args.vmap_base = user_addr;
1575			flush_or_invalidate = true;
1576		}
1577	}
1578
1579	/*
1580	 * Until there is support for iov_iter_extract_folios(), we have to
1581	 * manually extract pages using iov_iter_extract_pages() and then
1582	 * copy that to a folios array.
1583	 */
1584	struct page **pages = kzalloc(max_pages * sizeof(struct page *),
1585				      GFP_KERNEL);
1586	if (!pages) {
1587		ret = -ENOMEM;
1588		goto out;
1589	}
1590
1591	while (nbytes < *nbytesp && nr_pages < max_pages) {
1592		unsigned nfolios, i;
1593		size_t start;
1594
1595		ret = iov_iter_extract_pages(ii, &pages,
1596					     *nbytesp - nbytes,
1597					     max_pages - nr_pages,
1598					     0, &start);
1599		if (ret < 0)
1600			break;
1601
1602		nbytes += ret;
1603
1604		nfolios = DIV_ROUND_UP(ret + start, PAGE_SIZE);
1605
1606		for (i = 0; i < nfolios; i++) {
1607			struct folio *folio = page_folio(pages[i]);
1608			unsigned int offset = start +
1609				(folio_page_idx(folio, pages[i]) << PAGE_SHIFT);
1610			unsigned int len = min_t(unsigned int, ret, PAGE_SIZE - start);
1611
1612			ap->descs[ap->num_folios].offset = offset;
1613			ap->descs[ap->num_folios].length = len;
1614			ap->folios[ap->num_folios] = folio;
1615			start = 0;
1616			ret -= len;
1617			ap->num_folios++;
1618		}
1619
1620		nr_pages += nfolios;
1621	}
1622	kfree(pages);
1623
1624	if (write && flush_or_invalidate)
1625		flush_kernel_vmap_range(ap->args.vmap_base, nbytes);
1626
1627	ap->args.invalidate_vmap = !write && flush_or_invalidate;
1628	ap->args.is_pinned = iov_iter_extract_will_pin(ii);
1629	ap->args.user_pages = true;
1630	if (write)
1631		ap->args.in_pages = true;
1632	else
1633		ap->args.out_pages = true;
1634
1635out:
1636	*nbytesp = nbytes;
1637
1638	return ret < 0 ? ret : 0;
1639}
1640
1641ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1642		       loff_t *ppos, int flags)
1643{
1644	int write = flags & FUSE_DIO_WRITE;
1645	int cuse = flags & FUSE_DIO_CUSE;
1646	struct file *file = io->iocb->ki_filp;
1647	struct address_space *mapping = file->f_mapping;
1648	struct inode *inode = mapping->host;
1649	struct fuse_file *ff = file->private_data;
1650	struct fuse_conn *fc = ff->fm->fc;
1651	size_t nmax = write ? fc->max_write : fc->max_read;
1652	loff_t pos = *ppos;
1653	size_t count = iov_iter_count(iter);
1654	pgoff_t idx_from = pos >> PAGE_SHIFT;
1655	pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1656	ssize_t res = 0;
1657	int err = 0;
1658	struct fuse_io_args *ia;
1659	unsigned int max_pages;
1660	bool fopen_direct_io = ff->open_flags & FOPEN_DIRECT_IO;
1661
1662	max_pages = iov_iter_npages(iter, fc->max_pages);
1663	ia = fuse_io_alloc(io, max_pages);
1664	if (!ia)
1665		return -ENOMEM;
1666
1667	if (fopen_direct_io) {
1668		res = filemap_write_and_wait_range(mapping, pos, pos + count - 1);
1669		if (res) {
1670			fuse_io_free(ia);
1671			return res;
1672		}
1673	}
1674	if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) {
1675		if (!write)
1676			inode_lock(inode);
1677		fuse_sync_writes(inode);
1678		if (!write)
1679			inode_unlock(inode);
1680	}
1681
1682	if (fopen_direct_io && write) {
1683		res = invalidate_inode_pages2_range(mapping, idx_from, idx_to);
1684		if (res) {
1685			fuse_io_free(ia);
1686			return res;
1687		}
1688	}
1689
1690	io->should_dirty = !write && user_backed_iter(iter);
1691	while (count) {
1692		ssize_t nres;
1693		fl_owner_t owner = current->files;
1694		size_t nbytes = min(count, nmax);
1695
1696		err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1697					  max_pages, fc->use_pages_for_kvec_io);
1698		if (err && !nbytes)
1699			break;
1700
1701		if (write) {
1702			if (!capable(CAP_FSETID))
1703				ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
1704
1705			nres = fuse_send_write(ia, pos, nbytes, owner);
1706		} else {
1707			nres = fuse_send_read(ia, pos, nbytes, owner);
1708		}
1709
1710		if (!io->async || nres < 0) {
1711			fuse_release_user_pages(&ia->ap, nres, io->should_dirty);
1712			fuse_io_free(ia);
1713		}
1714		ia = NULL;
1715		if (nres < 0) {
1716			iov_iter_revert(iter, nbytes);
1717			err = nres;
1718			break;
1719		}
1720		WARN_ON(nres > nbytes);
1721
1722		count -= nres;
1723		res += nres;
1724		pos += nres;
1725		if (nres != nbytes) {
1726			iov_iter_revert(iter, nbytes - nres);
1727			break;
1728		}
1729		if (count) {
1730			max_pages = iov_iter_npages(iter, fc->max_pages);
1731			ia = fuse_io_alloc(io, max_pages);
1732			if (!ia)
1733				break;
1734		}
1735	}
1736	if (ia)
1737		fuse_io_free(ia);
1738	if (res > 0)
1739		*ppos = pos;
1740
1741	if (res > 0 && write && fopen_direct_io) {
1742		/*
1743		 * As in generic_file_direct_write(), invalidate after the
1744		 * write, to invalidate read-ahead cache that may have competed
1745		 * with the write.
1746		 */
1747		invalidate_inode_pages2_range(mapping, idx_from, idx_to);
1748	}
1749
1750	return res > 0 ? res : err;
1751}
1752EXPORT_SYMBOL_GPL(fuse_direct_io);
1753
1754static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1755				  struct iov_iter *iter,
1756				  loff_t *ppos)
1757{
1758	ssize_t res;
1759	struct inode *inode = file_inode(io->iocb->ki_filp);
1760
1761	res = fuse_direct_io(io, iter, ppos, 0);
1762
1763	fuse_invalidate_atime(inode);
1764
1765	return res;
1766}
1767
1768static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1769
1770static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1771{
1772	ssize_t res;
1773
1774	if (!is_sync_kiocb(iocb)) {
1775		res = fuse_direct_IO(iocb, to);
1776	} else {
1777		struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1778
1779		res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1780	}
1781
1782	return res;
1783}
1784
1785static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1786{
1787	struct inode *inode = file_inode(iocb->ki_filp);
1788	ssize_t res;
1789	bool exclusive;
1790
1791	fuse_dio_lock(iocb, from, &exclusive);
1792	res = generic_write_checks(iocb, from);
1793	if (res > 0) {
1794		task_io_account_write(res);
1795		if (!is_sync_kiocb(iocb)) {
1796			res = fuse_direct_IO(iocb, from);
1797		} else {
1798			struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1799
1800			res = fuse_direct_io(&io, from, &iocb->ki_pos,
1801					     FUSE_DIO_WRITE);
1802			fuse_write_update_attr(inode, iocb->ki_pos, res);
1803		}
1804	}
1805	fuse_dio_unlock(iocb, exclusive);
1806
1807	return res;
1808}
1809
1810static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1811{
1812	struct file *file = iocb->ki_filp;
1813	struct fuse_file *ff = file->private_data;
1814	struct inode *inode = file_inode(file);
1815
1816	if (fuse_is_bad(inode))
1817		return -EIO;
1818
1819	if (FUSE_IS_DAX(inode))
1820		return fuse_dax_read_iter(iocb, to);
1821
1822	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1823	if (ff->open_flags & FOPEN_DIRECT_IO)
1824		return fuse_direct_read_iter(iocb, to);
1825	else if (fuse_file_passthrough(ff))
1826		return fuse_passthrough_read_iter(iocb, to);
1827	else
1828		return fuse_cache_read_iter(iocb, to);
1829}
1830
1831static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1832{
1833	struct file *file = iocb->ki_filp;
1834	struct fuse_file *ff = file->private_data;
1835	struct inode *inode = file_inode(file);
1836
1837	if (fuse_is_bad(inode))
1838		return -EIO;
1839
1840	if (FUSE_IS_DAX(inode))
1841		return fuse_dax_write_iter(iocb, from);
1842
1843	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1844	if (ff->open_flags & FOPEN_DIRECT_IO)
1845		return fuse_direct_write_iter(iocb, from);
1846	else if (fuse_file_passthrough(ff))
1847		return fuse_passthrough_write_iter(iocb, from);
1848	else
1849		return fuse_cache_write_iter(iocb, from);
1850}
1851
1852static ssize_t fuse_splice_read(struct file *in, loff_t *ppos,
1853				struct pipe_inode_info *pipe, size_t len,
1854				unsigned int flags)
1855{
1856	struct fuse_file *ff = in->private_data;
1857
1858	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1859	if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
1860		return fuse_passthrough_splice_read(in, ppos, pipe, len, flags);
1861	else
1862		return filemap_splice_read(in, ppos, pipe, len, flags);
1863}
1864
1865static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out,
1866				 loff_t *ppos, size_t len, unsigned int flags)
1867{
1868	struct fuse_file *ff = out->private_data;
1869
1870	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
1871	if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
1872		return fuse_passthrough_splice_write(pipe, out, ppos, len, flags);
1873	else
1874		return iter_file_splice_write(pipe, out, ppos, len, flags);
1875}
1876
1877static void fuse_writepage_free(struct fuse_writepage_args *wpa)
1878{
1879	struct fuse_args_pages *ap = &wpa->ia.ap;
1880
1881	if (wpa->bucket)
1882		fuse_sync_bucket_dec(wpa->bucket);
1883
1884	fuse_file_put(wpa->ia.ff, false);
1885
1886	kfree(ap->folios);
1887	kfree(wpa);
1888}
1889
1890static void fuse_writepage_finish(struct fuse_writepage_args *wpa)
1891{
1892	struct fuse_args_pages *ap = &wpa->ia.ap;
1893	struct inode *inode = wpa->inode;
1894	struct fuse_inode *fi = get_fuse_inode(inode);
1895	int i;
1896
1897	for (i = 0; i < ap->num_folios; i++)
1898		/*
1899		 * Benchmarks showed that ending writeback within the
1900		 * scope of the fi->lock alleviates xarray lock
1901		 * contention and noticeably improves performance.
1902		 */
1903		iomap_finish_folio_write(inode, ap->folios[i],
1904					 ap->descs[i].length);
1905
1906	wake_up(&fi->page_waitq);
1907}
1908
1909/* Called under fi->lock, may release and reacquire it */
1910static void fuse_send_writepage(struct fuse_mount *fm,
1911				struct fuse_writepage_args *wpa, loff_t size)
1912__releases(fi->lock)
1913__acquires(fi->lock)
1914{
1915	struct fuse_inode *fi = get_fuse_inode(wpa->inode);
1916	struct fuse_args_pages *ap = &wpa->ia.ap;
1917	struct fuse_write_in *inarg = &wpa->ia.write.in;
1918	struct fuse_args *args = &ap->args;
1919	__u64 data_size = 0;
1920	int err, i;
1921
1922	for (i = 0; i < ap->num_folios; i++)
1923		data_size += ap->descs[i].length;
1924
1925	fi->writectr++;
1926	if (inarg->offset + data_size <= size) {
1927		inarg->size = data_size;
1928	} else if (inarg->offset < size) {
1929		inarg->size = size - inarg->offset;
1930	} else {
1931		/* Got truncated off completely */
1932		goto out_free;
1933	}
1934
1935	args->in_args[1].size = inarg->size;
1936	args->force = true;
1937	args->nocreds = true;
1938
1939	err = fuse_simple_background(fm, args, GFP_ATOMIC);
1940	if (err == -ENOMEM) {
1941		spin_unlock(&fi->lock);
1942		err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
1943		spin_lock(&fi->lock);
1944	}
1945
1946	/* Fails on broken connection only */
1947	if (unlikely(err))
1948		goto out_free;
1949
1950	return;
1951
1952 out_free:
1953	fi->writectr--;
1954	fuse_writepage_finish(wpa);
1955	spin_unlock(&fi->lock);
1956	fuse_writepage_free(wpa);
1957	spin_lock(&fi->lock);
1958}
1959
1960/*
1961 * If fi->writectr is positive (no truncate or fsync going on) send
1962 * all queued writepage requests.
1963 *
1964 * Called with fi->lock
1965 */
1966void fuse_flush_writepages(struct inode *inode)
1967__releases(fi->lock)
1968__acquires(fi->lock)
1969{
1970	struct fuse_mount *fm = get_fuse_mount(inode);
1971	struct fuse_inode *fi = get_fuse_inode(inode);
1972	loff_t crop = i_size_read(inode);
1973	struct fuse_writepage_args *wpa;
1974
1975	while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1976		wpa = list_entry(fi->queued_writes.next,
1977				 struct fuse_writepage_args, queue_entry);
1978		list_del_init(&wpa->queue_entry);
1979		fuse_send_writepage(fm, wpa, crop);
1980	}
1981}
1982
1983static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
1984			       int error)
1985{
1986	struct fuse_writepage_args *wpa =
1987		container_of(args, typeof(*wpa), ia.ap.args);
1988	struct inode *inode = wpa->inode;
1989	struct fuse_inode *fi = get_fuse_inode(inode);
1990	struct fuse_conn *fc = get_fuse_conn(inode);
1991
1992	mapping_set_error(inode->i_mapping, error);
1993	/*
1994	 * A writeback finished and this might have updated mtime/ctime on
1995	 * server making local mtime/ctime stale.  Hence invalidate attrs.
1996	 * Do this only if writeback_cache is not enabled.  If writeback_cache
1997	 * is enabled, we trust local ctime/mtime.
1998	 */
1999	if (!fc->writeback_cache)
2000		fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY);
2001	spin_lock(&fi->lock);
2002	fi->writectr--;
2003	fuse_writepage_finish(wpa);
2004	spin_unlock(&fi->lock);
2005	fuse_writepage_free(wpa);
2006}
2007
2008static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi)
2009{
2010	struct fuse_file *ff;
2011
2012	spin_lock(&fi->lock);
2013	ff = list_first_entry_or_null(&fi->write_files, struct fuse_file,
2014				      write_entry);
2015	if (ff)
2016		fuse_file_get(ff);
2017	spin_unlock(&fi->lock);
2018
2019	return ff;
2020}
2021
2022static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi)
2023{
2024	struct fuse_file *ff = __fuse_write_file_get(fi);
2025	WARN_ON(!ff);
2026	return ff;
2027}
2028
2029int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
2030{
2031	struct fuse_inode *fi = get_fuse_inode(inode);
2032	struct fuse_file *ff;
2033	int err;
2034
2035	ff = __fuse_write_file_get(fi);
2036	err = fuse_flush_times(inode, ff);
2037	if (ff)
2038		fuse_file_put(ff, false);
2039
2040	return err;
2041}
2042
2043static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
2044{
2045	struct fuse_writepage_args *wpa;
2046	struct fuse_args_pages *ap;
2047
2048	wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
2049	if (wpa) {
2050		ap = &wpa->ia.ap;
2051		ap->num_folios = 0;
2052		ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs);
2053		if (!ap->folios) {
2054			kfree(wpa);
2055			wpa = NULL;
2056		}
2057	}
2058	return wpa;
2059
2060}
2061
2062static void fuse_writepage_add_to_bucket(struct fuse_conn *fc,
2063					 struct fuse_writepage_args *wpa)
2064{
2065	if (!fc->sync_fs)
2066		return;
2067
2068	rcu_read_lock();
2069	/* Prevent resurrection of dead bucket in unlikely race with syncfs */
2070	do {
2071		wpa->bucket = rcu_dereference(fc->curr_bucket);
2072	} while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count)));
2073	rcu_read_unlock();
2074}
2075
2076static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio,
2077					  uint32_t folio_index, loff_t offset, unsigned len)
2078{
2079	struct fuse_args_pages *ap = &wpa->ia.ap;
2080
2081	ap->folios[folio_index] = folio;
2082	ap->descs[folio_index].offset = offset;
2083	ap->descs[folio_index].length = len;
2084}
2085
2086static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio,
2087							     size_t offset,
2088							     struct fuse_file *ff)
2089{
2090	struct inode *inode = folio->mapping->host;
2091	struct fuse_conn *fc = get_fuse_conn(inode);
2092	struct fuse_writepage_args *wpa;
2093	struct fuse_args_pages *ap;
2094
2095	wpa = fuse_writepage_args_alloc();
2096	if (!wpa)
2097		return NULL;
2098
2099	fuse_writepage_add_to_bucket(fc, wpa);
2100	fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio) + offset, 0);
2101	wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2102	wpa->inode = inode;
2103	wpa->ia.ff = ff;
2104
2105	ap = &wpa->ia.ap;
2106	ap->args.in_pages = true;
2107	ap->args.end = fuse_writepage_end;
2108
2109	return wpa;
2110}
2111
2112struct fuse_fill_wb_data {
2113	struct fuse_writepage_args *wpa;
2114	struct fuse_file *ff;
2115	unsigned int max_folios;
2116	/*
2117	 * nr_bytes won't overflow since fuse_folios_need_send() caps
2118	 * wb requests to never exceed fc->max_pages (which has an upper bound
2119	 * of U16_MAX).
2120	 */
2121	unsigned int nr_bytes;
2122};
2123
2124static bool fuse_pages_realloc(struct fuse_fill_wb_data *data,
2125			       unsigned int max_pages)
2126{
2127	struct fuse_args_pages *ap = &data->wpa->ia.ap;
2128	struct folio **folios;
2129	struct fuse_folio_desc *descs;
2130	unsigned int nfolios = min_t(unsigned int,
2131				     max_t(unsigned int, data->max_folios * 2,
2132					   FUSE_DEFAULT_MAX_PAGES_PER_REQ),
2133				    max_pages);
2134	WARN_ON(nfolios <= data->max_folios);
2135
2136	folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs);
2137	if (!folios)
2138		return false;
2139
2140	memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios);
2141	memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios);
2142	kfree(ap->folios);
2143	ap->folios = folios;
2144	ap->descs = descs;
2145	data->max_folios = nfolios;
2146
2147	return true;
2148}
2149
2150static void fuse_writepages_send(struct inode *inode,
2151				 struct fuse_fill_wb_data *data)
2152{
2153	struct fuse_writepage_args *wpa = data->wpa;
2154	struct fuse_inode *fi = get_fuse_inode(inode);
2155
2156	spin_lock(&fi->lock);
2157	list_add_tail(&wpa->queue_entry, &fi->queued_writes);
2158	fuse_flush_writepages(inode);
2159	spin_unlock(&fi->lock);
2160}
2161
2162static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
2163				  unsigned len, struct fuse_args_pages *ap,
2164				  unsigned cur_bytes, bool write)
2165{
2166	struct folio *prev_folio;
2167	struct fuse_folio_desc prev_desc;
2168	unsigned bytes = cur_bytes + len;
2169	loff_t prev_pos;
2170	size_t max_bytes = write ? fc->max_write : fc->max_read;
2171
2172	WARN_ON(!ap->num_folios);
2173
2174	/* Reached max pages */
2175	if ((bytes + PAGE_SIZE - 1) >> PAGE_SHIFT > fc->max_pages)
2176		return true;
2177
2178	if (bytes > max_bytes)
2179		return true;
2180
2181	/* Discontinuity */
2182	prev_folio = ap->folios[ap->num_folios - 1];
2183	prev_desc = ap->descs[ap->num_folios - 1];
2184	prev_pos = folio_pos(prev_folio) + prev_desc.offset + prev_desc.length;
2185	if (prev_pos != pos)
2186		return true;
2187
2188	return false;
2189}
2190
2191static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc,
2192					  struct folio *folio, u64 pos,
2193					  unsigned len, u64 end_pos)
2194{
2195	struct fuse_fill_wb_data *data = wpc->wb_ctx;
2196	struct fuse_writepage_args *wpa = data->wpa;
2197	struct fuse_args_pages *ap = &wpa->ia.ap;
2198	struct inode *inode = wpc->inode;
2199	struct fuse_inode *fi = get_fuse_inode(inode);
2200	struct fuse_conn *fc = get_fuse_conn(inode);
2201	loff_t offset = offset_in_folio(folio, pos);
2202
2203	WARN_ON_ONCE(!data);
2204
2205	if (!data->ff) {
2206		data->ff = fuse_write_file_get(fi);
2207		if (!data->ff)
2208			return -EIO;
2209	}
2210
2211	if (wpa) {
2212		bool send = fuse_folios_need_send(fc, pos, len, ap,
2213						  data->nr_bytes, true);
2214
2215		if (!send) {
2216			/*
2217			 * Need to grow the pages array?  If so, did the
2218			 * expansion fail?
2219			 */
2220			send = (ap->num_folios == data->max_folios) &&
2221				!fuse_pages_realloc(data, fc->max_pages);
2222		}
2223
2224		if (send) {
2225			fuse_writepages_send(inode, data);
2226			data->wpa = NULL;
2227			data->nr_bytes = 0;
2228		}
2229	}
2230
2231	if (data->wpa == NULL) {
2232		wpa = fuse_writepage_args_setup(folio, offset, data->ff);
2233		if (!wpa)
2234			return -ENOMEM;
2235		fuse_file_get(wpa->ia.ff);
2236		data->max_folios = 1;
2237		ap = &wpa->ia.ap;
2238	}
2239
2240	fuse_writepage_args_page_fill(wpa, folio, ap->num_folios,
2241				      offset, len);
2242	data->nr_bytes += len;
2243
2244	ap->num_folios++;
2245	if (!data->wpa)
2246		data->wpa = wpa;
2247
2248	return len;
2249}
2250
2251static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc,
2252				       int error)
2253{
2254	struct fuse_fill_wb_data *data = wpc->wb_ctx;
2255
2256	WARN_ON_ONCE(!data);
2257
2258	if (data->wpa) {
2259		WARN_ON(!data->wpa->ia.ap.num_folios);
2260		fuse_writepages_send(wpc->inode, data);
2261	}
2262
2263	if (data->ff)
2264		fuse_file_put(data->ff, false);
2265
2266	return error;
2267}
2268
2269static const struct iomap_writeback_ops fuse_writeback_ops = {
2270	.writeback_range	= fuse_iomap_writeback_range,
2271	.writeback_submit	= fuse_iomap_writeback_submit,
2272};
2273
2274static int fuse_writepages(struct address_space *mapping,
2275			   struct writeback_control *wbc)
2276{
2277	struct inode *inode = mapping->host;
2278	struct fuse_conn *fc = get_fuse_conn(inode);
2279	struct fuse_fill_wb_data data = {};
2280	struct iomap_writepage_ctx wpc = {
2281		.inode = inode,
2282		.iomap.type = IOMAP_MAPPED,
2283		.wbc = wbc,
2284		.ops = &fuse_writeback_ops,
2285		.wb_ctx	= &data,
2286	};
2287
2288	if (fuse_is_bad(inode))
2289		return -EIO;
2290
2291	if (wbc->sync_mode == WB_SYNC_NONE &&
2292	    fc->num_background >= fc->congestion_threshold)
2293		return 0;
2294
2295	return iomap_writepages(&wpc);
2296}
2297
2298static int fuse_launder_folio(struct folio *folio)
2299{
2300	int err = 0;
2301	struct fuse_fill_wb_data data = {};
2302	struct iomap_writepage_ctx wpc = {
2303		.inode = folio->mapping->host,
2304		.iomap.type = IOMAP_MAPPED,
2305		.ops = &fuse_writeback_ops,
2306		.wb_ctx	= &data,
2307	};
2308
2309	if (folio_clear_dirty_for_io(folio)) {
2310		err = iomap_writeback_folio(&wpc, folio);
2311		err = fuse_iomap_writeback_submit(&wpc, err);
2312		if (!err)
2313			folio_wait_writeback(folio);
2314	}
2315	return err;
2316}
2317
2318/*
2319 * Write back dirty data/metadata now (there may not be any suitable
2320 * open files later for data)
2321 */
2322static void fuse_vma_close(struct vm_area_struct *vma)
2323{
2324	int err;
2325
2326	err = write_inode_now(vma->vm_file->f_mapping->host, 1);
2327	mapping_set_error(vma->vm_file->f_mapping, err);
2328}
2329
2330/*
2331 * Wait for writeback against this page to complete before allowing it
2332 * to be marked dirty again, and hence written back again, possibly
2333 * before the previous writepage completed.
2334 *
2335 * Block here, instead of in ->writepage(), so that the userspace fs
2336 * can only block processes actually operating on the filesystem.
2337 *
2338 * Otherwise unprivileged userspace fs would be able to block
2339 * unrelated:
2340 *
2341 * - page migration
2342 * - sync(2)
2343 * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
2344 */
2345static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
2346{
2347	struct folio *folio = page_folio(vmf->page);
2348	struct inode *inode = file_inode(vmf->vma->vm_file);
2349
2350	file_update_time(vmf->vma->vm_file);
2351	folio_lock(folio);
2352	if (folio->mapping != inode->i_mapping) {
2353		folio_unlock(folio);
2354		return VM_FAULT_NOPAGE;
2355	}
2356
2357	folio_wait_writeback(folio);
2358	return VM_FAULT_LOCKED;
2359}
2360
2361static const struct vm_operations_struct fuse_file_vm_ops = {
2362	.close		= fuse_vma_close,
2363	.fault		= filemap_fault,
2364	.map_pages	= filemap_map_pages,
2365	.page_mkwrite	= fuse_page_mkwrite,
2366};
2367
2368static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2369{
2370	struct fuse_file *ff = file->private_data;
2371	struct fuse_conn *fc = ff->fm->fc;
2372	struct inode *inode = file_inode(file);
2373	int rc;
2374
2375	/* DAX mmap is superior to direct_io mmap */
2376	if (FUSE_IS_DAX(inode))
2377		return fuse_dax_mmap(file, vma);
2378
2379	/*
2380	 * If inode is in passthrough io mode, because it has some file open
2381	 * in passthrough mode, either mmap to backing file or fail mmap,
2382	 * because mixing cached mmap and passthrough io mode is not allowed.
2383	 */
2384	if (fuse_file_passthrough(ff))
2385		return fuse_passthrough_mmap(file, vma);
2386	else if (fuse_inode_backing(get_fuse_inode(inode)))
2387		return -ENODEV;
2388
2389	/*
2390	 * FOPEN_DIRECT_IO handling is special compared to O_DIRECT,
2391	 * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP.
2392	 */
2393	if (ff->open_flags & FOPEN_DIRECT_IO) {
2394		/*
2395		 * Can't provide the coherency needed for MAP_SHARED
2396		 * if FUSE_DIRECT_IO_ALLOW_MMAP isn't set.
2397		 */
2398		if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_allow_mmap)
2399			return -ENODEV;
2400
2401		invalidate_inode_pages2(file->f_mapping);
2402
2403		if (!(vma->vm_flags & VM_MAYSHARE)) {
2404			/* MAP_PRIVATE */
2405			return generic_file_mmap(file, vma);
2406		}
2407
2408		/*
2409		 * First mmap of direct_io file enters caching inode io mode.
2410		 * Also waits for parallel dio writers to go into serial mode
2411		 * (exclusive instead of shared lock).
2412		 * After first mmap, the inode stays in caching io mode until
2413		 * the direct_io file release.
2414		 */
2415		rc = fuse_file_cached_io_open(inode, ff);
2416		if (rc)
2417			return rc;
2418	}
2419
2420	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2421		fuse_link_write_file(file);
2422
2423	file_accessed(file);
2424	vma->vm_ops = &fuse_file_vm_ops;
2425	return 0;
2426}
2427
2428static int convert_fuse_file_lock(struct fuse_conn *fc,
2429				  const struct fuse_file_lock *ffl,
2430				  struct file_lock *fl)
2431{
2432	switch (ffl->type) {
2433	case F_UNLCK:
2434		break;
2435
2436	case F_RDLCK:
2437	case F_WRLCK:
2438		if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2439		    ffl->end < ffl->start)
2440			return -EIO;
2441
2442		fl->fl_start = ffl->start;
2443		fl->fl_end = ffl->end;
2444
2445		/*
2446		 * Convert pid into init's pid namespace.  The locks API will
2447		 * translate it into the caller's pid namespace.
2448		 */
2449		rcu_read_lock();
2450		fl->c.flc_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
2451		rcu_read_unlock();
2452		break;
2453
2454	default:
2455		return -EIO;
2456	}
2457	fl->c.flc_type = ffl->type;
2458	return 0;
2459}
2460
2461static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2462			 const struct file_lock *fl, int opcode, pid_t pid,
2463			 int flock, struct fuse_lk_in *inarg)
2464{
2465	struct inode *inode = file_inode(file);
2466	struct fuse_conn *fc = get_fuse_conn(inode);
2467	struct fuse_file *ff = file->private_data;
2468
2469	memset(inarg, 0, sizeof(*inarg));
2470	inarg->fh = ff->fh;
2471	inarg->owner = fuse_lock_owner_id(fc, fl->c.flc_owner);
2472	inarg->lk.start = fl->fl_start;
2473	inarg->lk.end = fl->fl_end;
2474	inarg->lk.type = fl->c.flc_type;
2475	inarg->lk.pid = pid;
2476	if (flock)
2477		inarg->lk_flags |= FUSE_LK_FLOCK;
2478	args->opcode = opcode;
2479	args->nodeid = get_node_id(inode);
2480	args->in_numargs = 1;
2481	args->in_args[0].size = sizeof(*inarg);
2482	args->in_args[0].value = inarg;
2483}
2484
2485static int fuse_getlk(struct file *file, struct file_lock *fl)
2486{
2487	struct inode *inode = file_inode(file);
2488	struct fuse_mount *fm = get_fuse_mount(inode);
2489	FUSE_ARGS(args);
2490	struct fuse_lk_in inarg;
2491	struct fuse_lk_out outarg;
2492	int err;
2493
2494	fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2495	args.out_numargs = 1;
2496	args.out_args[0].size = sizeof(outarg);
2497	args.out_args[0].value = &outarg;
2498	err = fuse_simple_request(fm, &args);
2499	if (!err)
2500		err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
2501
2502	return err;
2503}
2504
2505static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2506{
2507	struct inode *inode = file_inode(file);
2508	struct fuse_mount *fm = get_fuse_mount(inode);
2509	FUSE_ARGS(args);
2510	struct fuse_lk_in inarg;
2511	int opcode = (fl->c.flc_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2512	struct pid *pid = fl->c.flc_type != F_UNLCK ? task_tgid(current) : NULL;
2513	pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
2514	int err;
2515
2516	if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2517		/* NLM needs asynchronous locks, which we don't support yet */
2518		return -ENOLCK;
2519	}
2520
2521	fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
2522	err = fuse_simple_request(fm, &args);
2523
2524	/* locking is restartable */
2525	if (err == -EINTR)
2526		err = -ERESTARTSYS;
2527
2528	return err;
2529}
2530
2531static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2532{
2533	struct inode *inode = file_inode(file);
2534	struct fuse_conn *fc = get_fuse_conn(inode);
2535	int err;
2536
2537	if (cmd == F_CANCELLK) {
2538		err = 0;
2539	} else if (cmd == F_GETLK) {
2540		if (fc->no_lock) {
2541			posix_test_lock(file, fl);
2542			err = 0;
2543		} else
2544			err = fuse_getlk(file, fl);
2545	} else {
2546		if (fc->no_lock)
2547			err = posix_lock_file(file, fl, NULL);
2548		else
2549			err = fuse_setlk(file, fl, 0);
2550	}
2551	return err;
2552}
2553
2554static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2555{
2556	struct inode *inode = file_inode(file);
2557	struct fuse_conn *fc = get_fuse_conn(inode);
2558	int err;
2559
2560	if (fc->no_flock) {
2561		err = locks_lock_file_wait(file, fl);
2562	} else {
2563		struct fuse_file *ff = file->private_data;
2564
2565		/* emulate flock with POSIX locks */
2566		ff->flock = true;
2567		err = fuse_setlk(file, fl, 1);
2568	}
2569
2570	return err;
2571}
2572
2573static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2574{
2575	struct inode *inode = mapping->host;
2576	struct fuse_mount *fm = get_fuse_mount(inode);
2577	FUSE_ARGS(args);
2578	struct fuse_bmap_in inarg;
2579	struct fuse_bmap_out outarg;
2580	int err;
2581
2582	if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
2583		return 0;
2584
2585	memset(&inarg, 0, sizeof(inarg));
2586	inarg.block = block;
2587	inarg.blocksize = inode->i_sb->s_blocksize;
2588	args.opcode = FUSE_BMAP;
2589	args.nodeid = get_node_id(inode);
2590	args.in_numargs = 1;
2591	args.in_args[0].size = sizeof(inarg);
2592	args.in_args[0].value = &inarg;
2593	args.out_numargs = 1;
2594	args.out_args[0].size = sizeof(outarg);
2595	args.out_args[0].value = &outarg;
2596	err = fuse_simple_request(fm, &args);
2597	if (err == -ENOSYS)
2598		fm->fc->no_bmap = 1;
2599
2600	return err ? 0 : outarg.block;
2601}
2602
2603static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2604{
2605	struct inode *inode = file->f_mapping->host;
2606	struct fuse_mount *fm = get_fuse_mount(inode);
2607	struct fuse_file *ff = file->private_data;
2608	FUSE_ARGS(args);
2609	struct fuse_lseek_in inarg = {
2610		.fh = ff->fh,
2611		.offset = offset,
2612		.whence = whence
2613	};
2614	struct fuse_lseek_out outarg;
2615	int err;
2616
2617	if (fm->fc->no_lseek)
2618		goto fallback;
2619
2620	args.opcode = FUSE_LSEEK;
2621	args.nodeid = ff->nodeid;
2622	args.in_numargs = 1;
2623	args.in_args[0].size = sizeof(inarg);
2624	args.in_args[0].value = &inarg;
2625	args.out_numargs = 1;
2626	args.out_args[0].size = sizeof(outarg);
2627	args.out_args[0].value = &outarg;
2628	err = fuse_simple_request(fm, &args);
2629	if (err) {
2630		if (err == -ENOSYS) {
2631			fm->fc->no_lseek = 1;
2632			goto fallback;
2633		}
2634		return err;
2635	}
2636
2637	return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2638
2639fallback:
2640	err = fuse_update_attributes(inode, file, STATX_SIZE);
2641	if (!err)
2642		return generic_file_llseek(file, offset, whence);
2643	else
2644		return err;
2645}
2646
2647static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2648{
2649	loff_t retval;
2650	struct inode *inode = file_inode(file);
2651
2652	switch (whence) {
2653	case SEEK_SET:
2654	case SEEK_CUR:
2655		 /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
2656		retval = generic_file_llseek(file, offset, whence);
2657		break;
2658	case SEEK_END:
2659		inode_lock(inode);
2660		retval = fuse_update_attributes(inode, file, STATX_SIZE);
2661		if (!retval)
2662			retval = generic_file_llseek(file, offset, whence);
2663		inode_unlock(inode);
2664		break;
2665	case SEEK_HOLE:
2666	case SEEK_DATA:
2667		inode_lock(inode);
2668		retval = fuse_lseek(file, offset, whence);
2669		inode_unlock(inode);
2670		break;
2671	default:
2672		retval = -EINVAL;
2673	}
2674
2675	return retval;
2676}
2677
2678/*
2679 * All files which have been polled are linked to RB tree
2680 * fuse_conn->polled_files which is indexed by kh.  Walk the tree and
2681 * find the matching one.
2682 */
2683static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2684					      struct rb_node **parent_out)
2685{
2686	struct rb_node **link = &fc->polled_files.rb_node;
2687	struct rb_node *last = NULL;
2688
2689	while (*link) {
2690		struct fuse_file *ff;
2691
2692		last = *link;
2693		ff = rb_entry(last, struct fuse_file, polled_node);
2694
2695		if (kh < ff->kh)
2696			link = &last->rb_left;
2697		else if (kh > ff->kh)
2698			link = &last->rb_right;
2699		else
2700			return link;
2701	}
2702
2703	if (parent_out)
2704		*parent_out = last;
2705	return link;
2706}
2707
2708/*
2709 * The file is about to be polled.  Make sure it's on the polled_files
2710 * RB tree.  Note that files once added to the polled_files tree are
2711 * not removed before the file is released.  This is because a file
2712 * polled once is likely to be polled again.
2713 */
2714static void fuse_register_polled_file(struct fuse_conn *fc,
2715				      struct fuse_file *ff)
2716{
2717	spin_lock(&fc->lock);
2718	if (RB_EMPTY_NODE(&ff->polled_node)) {
2719		struct rb_node **link, *parent;
2720
2721		link = fuse_find_polled_node(fc, ff->kh, &parent);
2722		BUG_ON(*link);
2723		rb_link_node(&ff->polled_node, parent, link);
2724		rb_insert_color(&ff->polled_node, &fc->polled_files);
2725	}
2726	spin_unlock(&fc->lock);
2727}
2728
2729__poll_t fuse_file_poll(struct file *file, poll_table *wait)
2730{
2731	struct fuse_file *ff = file->private_data;
2732	struct fuse_mount *fm = ff->fm;
2733	struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2734	struct fuse_poll_out outarg;
2735	FUSE_ARGS(args);
2736	int err;
2737
2738	if (fm->fc->no_poll)
2739		return DEFAULT_POLLMASK;
2740
2741	poll_wait(file, &ff->poll_wait, wait);
2742	inarg.events = mangle_poll(poll_requested_events(wait));
2743
2744	/*
2745	 * Ask for notification iff there's someone waiting for it.
2746	 * The client may ignore the flag and always notify.
2747	 */
2748	if (waitqueue_active(&ff->poll_wait)) {
2749		inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
2750		fuse_register_polled_file(fm->fc, ff);
2751	}
2752
2753	args.opcode = FUSE_POLL;
2754	args.nodeid = ff->nodeid;
2755	args.in_numargs = 1;
2756	args.in_args[0].size = sizeof(inarg);
2757	args.in_args[0].value = &inarg;
2758	args.out_numargs = 1;
2759	args.out_args[0].size = sizeof(outarg);
2760	args.out_args[0].value = &outarg;
2761	err = fuse_simple_request(fm, &args);
2762
2763	if (!err)
2764		return demangle_poll(outarg.revents);
2765	if (err == -ENOSYS) {
2766		fm->fc->no_poll = 1;
2767		return DEFAULT_POLLMASK;
2768	}
2769	return EPOLLERR;
2770}
2771EXPORT_SYMBOL_GPL(fuse_file_poll);
2772
2773/*
2774 * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
2775 * wakes up the poll waiters.
2776 */
2777int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2778			    struct fuse_notify_poll_wakeup_out *outarg)
2779{
2780	u64 kh = outarg->kh;
2781	struct rb_node **link;
2782
2783	spin_lock(&fc->lock);
2784
2785	link = fuse_find_polled_node(fc, kh, NULL);
2786	if (*link) {
2787		struct fuse_file *ff;
2788
2789		ff = rb_entry(*link, struct fuse_file, polled_node);
2790		wake_up_interruptible_sync(&ff->poll_wait);
2791	}
2792
2793	spin_unlock(&fc->lock);
2794	return 0;
2795}
2796
2797static void fuse_do_truncate(struct file *file)
2798{
2799	struct inode *inode = file->f_mapping->host;
2800	struct iattr attr;
2801
2802	attr.ia_valid = ATTR_SIZE;
2803	attr.ia_size = i_size_read(inode);
2804
2805	attr.ia_file = file;
2806	attr.ia_valid |= ATTR_FILE;
2807
2808	fuse_do_setattr(file_mnt_idmap(file), file_dentry(file), &attr, file);
2809}
2810
2811static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
2812{
2813	return round_up(off, fc->max_pages << PAGE_SHIFT);
2814}
2815
2816static ssize_t
2817fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2818{
2819	DECLARE_COMPLETION_ONSTACK(wait);
2820	ssize_t ret = 0;
2821	struct file *file = iocb->ki_filp;
2822	struct fuse_file *ff = file->private_data;
2823	loff_t pos = 0;
2824	struct inode *inode;
2825	loff_t i_size;
2826	size_t count = iov_iter_count(iter), shortened = 0;
2827	loff_t offset = iocb->ki_pos;
2828	struct fuse_io_priv *io;
2829
2830	pos = offset;
2831	inode = file->f_mapping->host;
2832	i_size = i_size_read(inode);
2833
2834	if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
2835		return 0;
2836
2837	io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
2838	if (!io)
2839		return -ENOMEM;
2840	spin_lock_init(&io->lock);
2841	kref_init(&io->refcnt);
2842	io->reqs = 1;
2843	io->bytes = -1;
2844	io->size = 0;
2845	io->offset = offset;
2846	io->write = (iov_iter_rw(iter) == WRITE);
2847	io->err = 0;
2848	/*
2849	 * By default, we want to optimize all I/Os with async request
2850	 * submission to the client filesystem if supported.
2851	 */
2852	io->async = ff->fm->fc->async_dio;
2853	io->iocb = iocb;
2854	io->blocking = is_sync_kiocb(iocb);
2855
2856	/* optimization for short read */
2857	if (io->async && !io->write && offset + count > i_size) {
2858		iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
2859		shortened = count - iov_iter_count(iter);
2860		count -= shortened;
2861	}
2862
2863	/*
2864	 * We cannot asynchronously extend the size of a file.
2865	 * In such case the aio will behave exactly like sync io.
2866	 */
2867	if ((offset + count > i_size) && io->write)
2868		io->blocking = true;
2869
2870	if (io->async && io->blocking) {
2871		/*
2872		 * Additional reference to keep io around after
2873		 * calling fuse_aio_complete()
2874		 */
2875		kref_get(&io->refcnt);
2876		io->done = &wait;
2877	}
2878
2879	if (iov_iter_rw(iter) == WRITE) {
2880		ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
2881		fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
2882	} else {
2883		ret = __fuse_direct_read(io, iter, &pos);
2884	}
2885	iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
2886
2887	if (io->async) {
2888		bool blocking = io->blocking;
2889
2890		fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2891
2892		/* we have a non-extending, async request, so return */
2893		if (!blocking)
2894			return -EIOCBQUEUED;
2895
2896		wait_for_completion(&wait);
2897		ret = fuse_get_res_by_io(io);
2898	}
2899
2900	kref_put(&io->refcnt, fuse_io_release);
2901
2902	if (iov_iter_rw(iter) == WRITE) {
2903		fuse_write_update_attr(inode, pos, ret);
2904		/* For extending writes we already hold exclusive lock */
2905		if (ret < 0 && offset + count > i_size)
2906			fuse_do_truncate(file);
2907	}
2908
2909	return ret;
2910}
2911
2912static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
2913{
2914	int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
2915
2916	if (!err)
2917		fuse_sync_writes(inode);
2918
2919	return err;
2920}
2921
2922static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
2923				loff_t length)
2924{
2925	struct fuse_file *ff = file->private_data;
2926	struct inode *inode = file_inode(file);
2927	struct fuse_inode *fi = get_fuse_inode(inode);
2928	struct fuse_mount *fm = ff->fm;
2929	FUSE_ARGS(args);
2930	struct fuse_fallocate_in inarg = {
2931		.fh = ff->fh,
2932		.offset = offset,
2933		.length = length,
2934		.mode = mode
2935	};
2936	int err;
2937	bool block_faults = FUSE_IS_DAX(inode) &&
2938		(!(mode & FALLOC_FL_KEEP_SIZE) ||
2939		 (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)));
2940
2941	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
2942		     FALLOC_FL_ZERO_RANGE))
2943		return -EOPNOTSUPP;
2944
2945	if (fm->fc->no_fallocate)
2946		return -EOPNOTSUPP;
2947
2948	inode_lock(inode);
2949	if (block_faults) {
2950		filemap_invalidate_lock(inode->i_mapping);
2951		err = fuse_dax_break_layouts(inode, 0, -1);
2952		if (err)
2953			goto out;
2954	}
2955
2956	if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) {
2957		loff_t endbyte = offset + length - 1;
2958
2959		err = fuse_writeback_range(inode, offset, endbyte);
2960		if (err)
2961			goto out;
2962	}
2963
2964	if (!(mode & FALLOC_FL_KEEP_SIZE) &&
2965	    offset + length > i_size_read(inode)) {
2966		err = inode_newsize_ok(inode, offset + length);
2967		if (err)
2968			goto out;
2969	}
2970
2971	err = file_modified(file);
2972	if (err)
2973		goto out;
2974
2975	if (!(mode & FALLOC_FL_KEEP_SIZE))
2976		set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2977
2978	args.opcode = FUSE_FALLOCATE;
2979	args.nodeid = ff->nodeid;
2980	args.in_numargs = 1;
2981	args.in_args[0].size = sizeof(inarg);
2982	args.in_args[0].value = &inarg;
2983	err = fuse_simple_request(fm, &args);
2984	if (err == -ENOSYS) {
2985		fm->fc->no_fallocate = 1;
2986		err = -EOPNOTSUPP;
2987	}
2988	if (err)
2989		goto out;
2990
2991	/* we could have extended the file */
2992	if (!(mode & FALLOC_FL_KEEP_SIZE)) {
2993		if (fuse_write_update_attr(inode, offset + length, length))
2994			file_update_time(file);
2995	}
2996
2997	if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))
2998		truncate_pagecache_range(inode, offset, offset + length - 1);
2999
3000	fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
3001
3002out:
3003	if (!(mode & FALLOC_FL_KEEP_SIZE))
3004		clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3005
3006	if (block_faults)
3007		filemap_invalidate_unlock(inode->i_mapping);
3008
3009	inode_unlock(inode);
3010
3011	fuse_flush_time_update(inode);
3012
3013	return err;
3014}
3015
3016static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
3017				      struct file *file_out, loff_t pos_out,
3018				      size_t len, unsigned int flags)
3019{
3020	struct fuse_file *ff_in = file_in->private_data;
3021	struct fuse_file *ff_out = file_out->private_data;
3022	struct inode *inode_in = file_inode(file_in);
3023	struct inode *inode_out = file_inode(file_out);
3024	struct fuse_inode *fi_out = get_fuse_inode(inode_out);
3025	struct fuse_mount *fm = ff_in->fm;
3026	struct fuse_conn *fc = fm->fc;
3027	FUSE_ARGS(args);
3028	struct fuse_copy_file_range_in inarg = {
3029		.fh_in = ff_in->fh,
3030		.off_in = pos_in,
3031		.nodeid_out = ff_out->nodeid,
3032		.fh_out = ff_out->fh,
3033		.off_out = pos_out,
3034		.len = len,
3035		.flags = flags
3036	};
3037	struct fuse_write_out outarg;
3038	struct fuse_copy_file_range_out outarg_64;
3039	u64 bytes_copied;
3040	ssize_t err;
3041	/* mark unstable when write-back is not used, and file_out gets
3042	 * extended */
3043	bool is_unstable = (!fc->writeback_cache) &&
3044			   ((pos_out + len) > inode_out->i_size);
3045
3046	if (fc->no_copy_file_range)
3047		return -EOPNOTSUPP;
3048
3049	if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3050		return -EXDEV;
3051
3052	inode_lock(inode_in);
3053	err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
3054	inode_unlock(inode_in);
3055	if (err)
3056		return err;
3057
3058	inode_lock(inode_out);
3059
3060	err = file_modified(file_out);
3061	if (err)
3062		goto out;
3063
3064	/*
3065	 * Write out dirty pages in the destination file before sending the COPY
3066	 * request to userspace.  After the request is completed, truncate off
3067	 * pages (including partial ones) from the cache that have been copied,
3068	 * since these contain stale data at that point.
3069	 *
3070	 * This should be mostly correct, but if the COPY writes to partial
3071	 * pages (at the start or end) and the parts not covered by the COPY are
3072	 * written through a memory map after calling fuse_writeback_range(),
3073	 * then these partial page modifications will be lost on truncation.
3074	 *
3075	 * It is unlikely that someone would rely on such mixed style
3076	 * modifications.  Yet this does give less guarantees than if the
3077	 * copying was performed with write(2).
3078	 *
3079	 * To fix this a mapping->invalidate_lock could be used to prevent new
3080	 * faults while the copy is ongoing.
3081	 */
3082	err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
3083	if (err)
3084		goto out;
3085
3086	if (is_unstable)
3087		set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3088
3089	args.opcode = FUSE_COPY_FILE_RANGE_64;
3090	args.nodeid = ff_in->nodeid;
3091	args.in_numargs = 1;
3092	args.in_args[0].size = sizeof(inarg);
3093	args.in_args[0].value = &inarg;
3094	args.out_numargs = 1;
3095	args.out_args[0].size = sizeof(outarg_64);
3096	args.out_args[0].value = &outarg_64;
3097	if (fc->no_copy_file_range_64) {
3098fallback:
3099		/* Fall back to old op that can't handle large copy length */
3100		args.opcode = FUSE_COPY_FILE_RANGE;
3101		args.out_args[0].size = sizeof(outarg);
3102		args.out_args[0].value = &outarg;
3103		inarg.len = len = min_t(size_t, len, UINT_MAX & PAGE_MASK);
3104	}
3105	err = fuse_simple_request(fm, &args);
3106	if (err == -ENOSYS) {
3107		if (fc->no_copy_file_range_64) {
3108			fc->no_copy_file_range = 1;
3109			err = -EOPNOTSUPP;
3110		} else {
3111			fc->no_copy_file_range_64 = 1;
3112			goto fallback;
3113		}
3114	}
3115	if (err)
3116		goto out;
3117
3118	bytes_copied = fc->no_copy_file_range_64 ?
3119		outarg.size : outarg_64.bytes_copied;
3120
3121	if (bytes_copied > len) {
3122		err = -EIO;
3123		goto out;
3124	}
3125
3126	truncate_inode_pages_range(inode_out->i_mapping,
3127				   ALIGN_DOWN(pos_out, PAGE_SIZE),
3128				   ALIGN(pos_out + bytes_copied, PAGE_SIZE) - 1);
3129
3130	file_update_time(file_out);
3131	fuse_write_update_attr(inode_out, pos_out + bytes_copied, bytes_copied);
3132
3133	err = bytes_copied;
3134out:
3135	if (is_unstable)
3136		clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3137
3138	inode_unlock(inode_out);
3139	file_accessed(file_in);
3140
3141	fuse_flush_time_update(inode_out);
3142
3143	return err;
3144}
3145
3146static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3147				    struct file *dst_file, loff_t dst_off,
3148				    size_t len, unsigned int flags)
3149{
3150	ssize_t ret;
3151
3152	ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3153				     len, flags);
3154
3155	if (ret == -EOPNOTSUPP || ret == -EXDEV)
3156		ret = splice_copy_file_range(src_file, src_off, dst_file,
3157					     dst_off, len);
3158	return ret;
3159}
3160
3161static const struct file_operations fuse_file_operations = {
3162	.llseek		= fuse_file_llseek,
3163	.read_iter	= fuse_file_read_iter,
3164	.write_iter	= fuse_file_write_iter,
3165	.mmap		= fuse_file_mmap,
3166	.open		= fuse_open,
3167	.flush		= fuse_flush,
3168	.release	= fuse_release,
3169	.fsync		= fuse_fsync,
3170	.lock		= fuse_file_lock,
3171	.get_unmapped_area = thp_get_unmapped_area,
3172	.flock		= fuse_file_flock,
3173	.splice_read	= fuse_splice_read,
3174	.splice_write	= fuse_splice_write,
3175	.unlocked_ioctl	= fuse_file_ioctl,
3176	.compat_ioctl	= fuse_file_compat_ioctl,
3177	.poll		= fuse_file_poll,
3178	.fallocate	= fuse_file_fallocate,
3179	.copy_file_range = fuse_copy_file_range,
3180};
3181
3182static const struct address_space_operations fuse_file_aops  = {
3183	.read_folio	= fuse_read_folio,
3184	.readahead	= fuse_readahead,
3185	.writepages	= fuse_writepages,
3186	.launder_folio	= fuse_launder_folio,
3187	.dirty_folio	= iomap_dirty_folio,
3188	.release_folio	= iomap_release_folio,
3189	.invalidate_folio = iomap_invalidate_folio,
3190	.is_partially_uptodate = iomap_is_partially_uptodate,
3191	.migrate_folio	= filemap_migrate_folio,
3192	.bmap		= fuse_bmap,
3193	.direct_IO	= fuse_direct_IO,
3194};
3195
3196void fuse_init_file_inode(struct inode *inode, unsigned int flags)
3197{
3198	struct fuse_inode *fi = get_fuse_inode(inode);
3199	struct fuse_conn *fc = get_fuse_conn(inode);
3200
3201	inode->i_fop = &fuse_file_operations;
3202	inode->i_data.a_ops = &fuse_file_aops;
3203	if (fc->writeback_cache)
3204		mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data);
3205
3206	INIT_LIST_HEAD(&fi->write_files);
3207	INIT_LIST_HEAD(&fi->queued_writes);
3208	fi->writectr = 0;
3209	fi->iocachectr = 0;
3210	init_waitqueue_head(&fi->page_waitq);
3211	init_waitqueue_head(&fi->direct_io_waitq);
3212
3213	if (IS_ENABLED(CONFIG_FUSE_DAX))
3214		fuse_dax_inode_init(inode, flags);
3215}