dax.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * dax: direct host memory access
   4 * Copyright (C) 2020 Red Hat, Inc.
   5 */
   6
   7#include "fuse_i.h"
   8
   9#include <linux/delay.h>
  10#include <linux/dax.h>
  11#include <linux/uio.h>
  12#include <linux/pagemap.h>
  13#include <linux/iomap.h>
  14#include <linux/interval_tree.h>
  15
  16/*
  17 * Default memory range size.  A power of 2 so it agrees with common FUSE_INIT
  18 * map_alignment values 4KB and 64KB.
  19 */
  20#define FUSE_DAX_SHIFT	21
  21#define FUSE_DAX_SZ	(1 << FUSE_DAX_SHIFT)
  22#define FUSE_DAX_PAGES	(FUSE_DAX_SZ / PAGE_SIZE)
  23
  24/* Number of ranges reclaimer will try to free in one invocation */
  25#define FUSE_DAX_RECLAIM_CHUNK		(10)
  26
  27/*
  28 * Dax memory reclaim threshold in percetage of total ranges. When free
  29 * number of free ranges drops below this threshold, reclaim can trigger
  30 * Default is 20%
  31 */
  32#define FUSE_DAX_RECLAIM_THRESHOLD	(20)
  33
  34/** Translation information for file offsets to DAX window offsets */
  35struct fuse_dax_mapping {
  36	/* Pointer to inode where this memory range is mapped */
  37	struct inode *inode;
  38
  39	/* Will connect in fcd->free_ranges to keep track of free memory */
  40	struct list_head list;
  41
  42	/* For interval tree in file/inode */
  43	struct interval_tree_node itn;
  44
  45	/* Will connect in fc->busy_ranges to keep track busy memory */
  46	struct list_head busy_list;
  47
  48	/** Position in DAX window */
  49	u64 window_offset;
  50
  51	/** Length of mapping, in bytes */
  52	loff_t length;
  53
  54	/* Is this mapping read-only or read-write */
  55	bool writable;
  56
  57	/* reference count when the mapping is used by dax iomap. */
  58	refcount_t refcnt;
  59};
  60
  61/* Per-inode dax map */
  62struct fuse_inode_dax {
  63	/* Semaphore to protect modifications to the dmap tree */
  64	struct rw_semaphore sem;
  65
  66	/* Sorted rb tree of struct fuse_dax_mapping elements */
  67	struct rb_root_cached tree;
  68	unsigned long nr;
  69};
  70
  71struct fuse_conn_dax {
  72	/* DAX device */
  73	struct dax_device *dev;
  74
  75	/* Lock protecting accessess to  members of this structure */
  76	spinlock_t lock;
  77
  78	/* List of memory ranges which are busy */
  79	unsigned long nr_busy_ranges;
  80	struct list_head busy_ranges;
  81
  82	/* Worker to free up memory ranges */
  83	struct delayed_work free_work;
  84
  85	/* Wait queue for a dax range to become free */
  86	wait_queue_head_t range_waitq;
  87
  88	/* DAX Window Free Ranges */
  89	long nr_free_ranges;
  90	struct list_head free_ranges;
  91
  92	unsigned long nr_ranges;
  93};
  94
  95static inline struct fuse_dax_mapping *
  96node_to_dmap(struct interval_tree_node *node)
  97{
  98	if (!node)
  99		return NULL;
 100
 101	return container_of(node, struct fuse_dax_mapping, itn);
 102}
 103
 104static struct fuse_dax_mapping *
 105alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);
 106
 107static void
 108__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
 109{
 110	unsigned long free_threshold;
 111
 112	/* If number of free ranges are below threshold, start reclaim */
 113	free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,
 114			     1);
 115	if (fcd->nr_free_ranges < free_threshold)
 116		queue_delayed_work(system_long_wq, &fcd->free_work,
 117				   msecs_to_jiffies(delay_ms));
 118}
 119
 120static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
 121				  unsigned long delay_ms)
 122{
 123	spin_lock(&fcd->lock);
 124	__kick_dmap_free_worker(fcd, delay_ms);
 125	spin_unlock(&fcd->lock);
 126}
 127
 128static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
 129{
 130	struct fuse_dax_mapping *dmap;
 131
 132	spin_lock(&fcd->lock);
 133	dmap = list_first_entry_or_null(&fcd->free_ranges,
 134					struct fuse_dax_mapping, list);
 135	if (dmap) {
 136		list_del_init(&dmap->list);
 137		WARN_ON(fcd->nr_free_ranges <= 0);
 138		fcd->nr_free_ranges--;
 139	}
 140	__kick_dmap_free_worker(fcd, 0);
 141	spin_unlock(&fcd->lock);
 142
 143	return dmap;
 144}
 145
 146/* This assumes fcd->lock is held */
 147static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
 148				    struct fuse_dax_mapping *dmap)
 149{
 150	list_del_init(&dmap->busy_list);
 151	WARN_ON(fcd->nr_busy_ranges == 0);
 152	fcd->nr_busy_ranges--;
 153}
 154
 155static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
 156				  struct fuse_dax_mapping *dmap)
 157{
 158	spin_lock(&fcd->lock);
 159	__dmap_remove_busy_list(fcd, dmap);
 160	spin_unlock(&fcd->lock);
 161}
 162
 163/* This assumes fcd->lock is held */
 164static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
 165				struct fuse_dax_mapping *dmap)
 166{
 167	list_add_tail(&dmap->list, &fcd->free_ranges);
 168	fcd->nr_free_ranges++;
 169	wake_up(&fcd->range_waitq);
 170}
 171
 172static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
 173				struct fuse_dax_mapping *dmap)
 174{
 175	/* Return fuse_dax_mapping to free list */
 176	spin_lock(&fcd->lock);
 177	__dmap_add_to_free_pool(fcd, dmap);
 178	spin_unlock(&fcd->lock);
 179}
 180
 181static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
 182				  struct fuse_dax_mapping *dmap, bool writable,
 183				  bool upgrade)
 184{
 185	struct fuse_mount *fm = get_fuse_mount(inode);
 186	struct fuse_conn_dax *fcd = fm->fc->dax;
 187	struct fuse_inode *fi = get_fuse_inode(inode);
 188	struct fuse_setupmapping_in inarg;
 189	loff_t offset = start_idx << FUSE_DAX_SHIFT;
 190	FUSE_ARGS(args);
 191	ssize_t err;
 192
 193	WARN_ON(fcd->nr_free_ranges < 0);
 194
 195	/* Ask fuse daemon to setup mapping */
 196	memset(&inarg, 0, sizeof(inarg));
 197	inarg.foffset = offset;
 198	inarg.fh = -1;
 199	inarg.moffset = dmap->window_offset;
 200	inarg.len = FUSE_DAX_SZ;
 201	inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ;
 202	if (writable)
 203		inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE;
 204	args.opcode = FUSE_SETUPMAPPING;
 205	args.nodeid = fi->nodeid;
 206	args.in_numargs = 1;
 207	args.in_args[0].size = sizeof(inarg);
 208	args.in_args[0].value = &inarg;
 209	err = fuse_simple_request(fm, &args);
 210	if (err < 0)
 211		return err;
 212	dmap->writable = writable;
 213	if (!upgrade) {
 214		/*
 215		 * We don't take a reference on inode. inode is valid right now
 216		 * and when inode is going away, cleanup logic should first
 217		 * cleanup dmap entries.
 218		 */
 219		dmap->inode = inode;
 220		dmap->itn.start = dmap->itn.last = start_idx;
 221		/* Protected by fi->dax->sem */
 222		interval_tree_insert(&dmap->itn, &fi->dax->tree);
 223		fi->dax->nr++;
 224		spin_lock(&fcd->lock);
 225		list_add_tail(&dmap->busy_list, &fcd->busy_ranges);
 226		fcd->nr_busy_ranges++;
 227		spin_unlock(&fcd->lock);
 228	}
 229	return 0;
 230}
 231
 232static int fuse_send_removemapping(struct inode *inode,
 233				   struct fuse_removemapping_in *inargp,
 234				   struct fuse_removemapping_one *remove_one)
 235{
 236	struct fuse_inode *fi = get_fuse_inode(inode);
 237	struct fuse_mount *fm = get_fuse_mount(inode);
 238	FUSE_ARGS(args);
 239
 240	args.opcode = FUSE_REMOVEMAPPING;
 241	args.nodeid = fi->nodeid;
 242	args.in_numargs = 3;
 243	fuse_set_zero_arg0(&args);
 244	args.in_args[1].size = sizeof(*inargp);
 245	args.in_args[1].value = inargp;
 246	args.in_args[2].size = inargp->count * sizeof(*remove_one);
 247	args.in_args[2].value = remove_one;
 248	return fuse_simple_request(fm, &args);
 249}
 250
 251static int dmap_removemapping_list(struct inode *inode, unsigned int num,
 252				   struct list_head *to_remove)
 253{
 254	struct fuse_removemapping_one *remove_one, *ptr;
 255	struct fuse_removemapping_in inarg;
 256	struct fuse_dax_mapping *dmap;
 257	int ret, i = 0, nr_alloc;
 258
 259	nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY);
 260	remove_one = kmalloc_array(nr_alloc, sizeof(*remove_one), GFP_NOFS);
 261	if (!remove_one)
 262		return -ENOMEM;
 263
 264	ptr = remove_one;
 265	list_for_each_entry(dmap, to_remove, list) {
 266		ptr->moffset = dmap->window_offset;
 267		ptr->len = dmap->length;
 268		ptr++;
 269		i++;
 270		num--;
 271		if (i >= nr_alloc || num == 0) {
 272			memset(&inarg, 0, sizeof(inarg));
 273			inarg.count = i;
 274			ret = fuse_send_removemapping(inode, &inarg,
 275						      remove_one);
 276			if (ret)
 277				goto out;
 278			ptr = remove_one;
 279			i = 0;
 280		}
 281	}
 282out:
 283	kfree(remove_one);
 284	return ret;
 285}
 286
 287/*
 288 * Cleanup dmap entry and add back to free list. This should be called with
 289 * fcd->lock held.
 290 */
 291static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
 292					    struct fuse_dax_mapping *dmap)
 293{
 294	pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n",
 295		 dmap->itn.start, dmap->itn.last, dmap->window_offset,
 296		 dmap->length);
 297	__dmap_remove_busy_list(fcd, dmap);
 298	dmap->inode = NULL;
 299	dmap->itn.start = dmap->itn.last = 0;
 300	__dmap_add_to_free_pool(fcd, dmap);
 301}
 302
 303/*
 304 * Free inode dmap entries whose range falls inside [start, end].
 305 * Does not take any locks. At this point of time it should only be
 306 * called from evict_inode() path where we know all dmap entries can be
 307 * reclaimed.
 308 */
 309static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
 310				     struct inode *inode,
 311				     loff_t start, loff_t end)
 312{
 313	struct fuse_inode *fi = get_fuse_inode(inode);
 314	struct fuse_dax_mapping *dmap, *n;
 315	int err, num = 0;
 316	LIST_HEAD(to_remove);
 317	unsigned long start_idx = start >> FUSE_DAX_SHIFT;
 318	unsigned long end_idx = end >> FUSE_DAX_SHIFT;
 319	struct interval_tree_node *node;
 320
 321	while (1) {
 322		node = interval_tree_iter_first(&fi->dax->tree, start_idx,
 323						end_idx);
 324		if (!node)
 325			break;
 326		dmap = node_to_dmap(node);
 327		/* inode is going away. There should not be any users of dmap */
 328		WARN_ON(refcount_read(&dmap->refcnt) > 1);
 329		interval_tree_remove(&dmap->itn, &fi->dax->tree);
 330		num++;
 331		list_add(&dmap->list, &to_remove);
 332	}
 333
 334	/* Nothing to remove */
 335	if (list_empty(&to_remove))
 336		return;
 337
 338	WARN_ON(fi->dax->nr < num);
 339	fi->dax->nr -= num;
 340	err = dmap_removemapping_list(inode, num, &to_remove);
 341	if (err && err != -ENOTCONN) {
 342		pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n",
 343			start, end);
 344	}
 345	spin_lock(&fcd->lock);
 346	list_for_each_entry_safe(dmap, n, &to_remove, list) {
 347		list_del_init(&dmap->list);
 348		dmap_reinit_add_to_free_pool(fcd, dmap);
 349	}
 350	spin_unlock(&fcd->lock);
 351}
 352
 353static int dmap_removemapping_one(struct inode *inode,
 354				  struct fuse_dax_mapping *dmap)
 355{
 356	struct fuse_removemapping_one forget_one;
 357	struct fuse_removemapping_in inarg;
 358
 359	memset(&inarg, 0, sizeof(inarg));
 360	inarg.count = 1;
 361	memset(&forget_one, 0, sizeof(forget_one));
 362	forget_one.moffset = dmap->window_offset;
 363	forget_one.len = dmap->length;
 364
 365	return fuse_send_removemapping(inode, &inarg, &forget_one);
 366}
 367
 368/*
 369 * It is called from evict_inode() and by that time inode is going away. So
 370 * this function does not take any locks like fi->dax->sem for traversing
 371 * that fuse inode interval tree. If that lock is taken then lock validator
 372 * complains of deadlock situation w.r.t fs_reclaim lock.
 373 */
 374void fuse_dax_inode_cleanup(struct inode *inode)
 375{
 376	struct fuse_conn *fc = get_fuse_conn(inode);
 377	struct fuse_inode *fi = get_fuse_inode(inode);
 378
 379	/*
 380	 * fuse_evict_inode() has already called truncate_inode_pages_final()
 381	 * before we arrive here. So we should not have to worry about any
 382	 * pages/exception entries still associated with inode.
 383	 */
 384	inode_reclaim_dmap_range(fc->dax, inode, 0, -1);
 385	WARN_ON(fi->dax->nr);
 386}
 387
 388static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length)
 389{
 390	iomap->addr = IOMAP_NULL_ADDR;
 391	iomap->length = length;
 392	iomap->type = IOMAP_HOLE;
 393}
 394
 395static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length,
 396			    struct iomap *iomap, struct fuse_dax_mapping *dmap,
 397			    unsigned int flags)
 398{
 399	loff_t offset, len;
 400	loff_t i_size = i_size_read(inode);
 401
 402	offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT);
 403	len = min(length, dmap->length - offset);
 404
 405	/* If length is beyond end of file, truncate further */
 406	if (pos + len > i_size)
 407		len = i_size - pos;
 408
 409	if (len > 0) {
 410		iomap->addr = dmap->window_offset + offset;
 411		iomap->length = len;
 412		if (flags & IOMAP_FAULT)
 413			iomap->length = ALIGN(len, PAGE_SIZE);
 414		iomap->type = IOMAP_MAPPED;
 415		/*
 416		 * increace refcnt so that reclaim code knows this dmap is in
 417		 * use. This assumes fi->dax->sem mutex is held either
 418		 * shared/exclusive.
 419		 */
 420		refcount_inc(&dmap->refcnt);
 421
 422		/* iomap->private should be NULL */
 423		WARN_ON_ONCE(iomap->private);
 424		iomap->private = dmap;
 425	} else {
 426		/* Mapping beyond end of file is hole */
 427		fuse_fill_iomap_hole(iomap, length);
 428	}
 429}
 430
 431static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
 432				      loff_t length, unsigned int flags,
 433				      struct iomap *iomap)
 434{
 435	struct fuse_inode *fi = get_fuse_inode(inode);
 436	struct fuse_conn *fc = get_fuse_conn(inode);
 437	struct fuse_conn_dax *fcd = fc->dax;
 438	struct fuse_dax_mapping *dmap, *alloc_dmap = NULL;
 439	int ret;
 440	bool writable = flags & IOMAP_WRITE;
 441	unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
 442	struct interval_tree_node *node;
 443
 444	/*
 445	 * Can't do inline reclaim in fault path. We call
 446	 * dax_layout_busy_page() before we free a range. And
 447	 * fuse_wait_dax_page() drops mapping->invalidate_lock and requires it.
 448	 * In fault path we enter with mapping->invalidate_lock held and can't
 449	 * drop it. Also in fault path we hold mapping->invalidate_lock shared
 450	 * and not exclusive, so that creates further issues with
 451	 * fuse_wait_dax_page().  Hence return -EAGAIN and fuse_dax_fault()
 452	 * will wait for a memory range to become free and retry.
 453	 */
 454	if (flags & IOMAP_FAULT) {
 455		alloc_dmap = alloc_dax_mapping(fcd);
 456		if (!alloc_dmap)
 457			return -EAGAIN;
 458	} else {
 459		alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode);
 460		if (IS_ERR(alloc_dmap))
 461			return PTR_ERR(alloc_dmap);
 462	}
 463
 464	/* If we are here, we should have memory allocated */
 465	if (WARN_ON(!alloc_dmap))
 466		return -EIO;
 467
 468	/*
 469	 * Take write lock so that only one caller can try to setup mapping
 470	 * and other waits.
 471	 */
 472	down_write(&fi->dax->sem);
 473	/*
 474	 * We dropped lock. Check again if somebody else setup
 475	 * mapping already.
 476	 */
 477	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
 478	if (node) {
 479		dmap = node_to_dmap(node);
 480		fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
 481		dmap_add_to_free_pool(fcd, alloc_dmap);
 482		up_write(&fi->dax->sem);
 483		return 0;
 484	}
 485
 486	/* Setup one mapping */
 487	ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap,
 488				     writable, false);
 489	if (ret < 0) {
 490		dmap_add_to_free_pool(fcd, alloc_dmap);
 491		up_write(&fi->dax->sem);
 492		return ret;
 493	}
 494	fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags);
 495	up_write(&fi->dax->sem);
 496	return 0;
 497}
 498
 499static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
 500				    loff_t length, unsigned int flags,
 501				    struct iomap *iomap)
 502{
 503	struct fuse_inode *fi = get_fuse_inode(inode);
 504	struct fuse_dax_mapping *dmap;
 505	int ret;
 506	unsigned long idx = pos >> FUSE_DAX_SHIFT;
 507	struct interval_tree_node *node;
 508
 509	/*
 510	 * Take exclusive lock so that only one caller can try to setup
 511	 * mapping and others wait.
 512	 */
 513	down_write(&fi->dax->sem);
 514	node = interval_tree_iter_first(&fi->dax->tree, idx, idx);
 515
 516	/* We are holding either inode lock or invalidate_lock, and that should
 517	 * ensure that dmap can't be truncated. We are holding a reference
 518	 * on dmap and that should make sure it can't be reclaimed. So dmap
 519	 * should still be there in tree despite the fact we dropped and
 520	 * re-acquired the fi->dax->sem lock.
 521	 */
 522	ret = -EIO;
 523	if (WARN_ON(!node))
 524		goto out_err;
 525
 526	dmap = node_to_dmap(node);
 527
 528	/* We took an extra reference on dmap to make sure its not reclaimd.
 529	 * Now we hold fi->dax->sem lock and that reference is not needed
 530	 * anymore. Drop it.
 531	 */
 532	if (refcount_dec_and_test(&dmap->refcnt)) {
 533		/* refcount should not hit 0. This object only goes
 534		 * away when fuse connection goes away
 535		 */
 536		WARN_ON_ONCE(1);
 537	}
 538
 539	/* Maybe another thread already upgraded mapping while we were not
 540	 * holding lock.
 541	 */
 542	if (dmap->writable) {
 543		ret = 0;
 544		goto out_fill_iomap;
 545	}
 546
 547	ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true,
 548				     true);
 549	if (ret < 0)
 550		goto out_err;
 551out_fill_iomap:
 552	fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
 553out_err:
 554	up_write(&fi->dax->sem);
 555	return ret;
 556}
 557
 558/* This is just for DAX and the mapping is ephemeral, do not use it for other
 559 * purposes since there is no block device with a permanent mapping.
 560 */
 561static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
 562			    unsigned int flags, struct iomap *iomap,
 563			    struct iomap *srcmap)
 564{
 565	struct fuse_inode *fi = get_fuse_inode(inode);
 566	struct fuse_conn *fc = get_fuse_conn(inode);
 567	struct fuse_dax_mapping *dmap;
 568	bool writable = flags & IOMAP_WRITE;
 569	unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
 570	struct interval_tree_node *node;
 571
 572	/* We don't support FIEMAP */
 573	if (WARN_ON(flags & IOMAP_REPORT))
 574		return -EIO;
 575
 576	iomap->offset = pos;
 577	iomap->flags = 0;
 578	iomap->bdev = NULL;
 579	iomap->dax_dev = fc->dax->dev;
 580
 581	/*
 582	 * Both read/write and mmap path can race here. So we need something
 583	 * to make sure if we are setting up mapping, then other path waits
 584	 *
 585	 * For now, use a semaphore for this. It probably needs to be
 586	 * optimized later.
 587	 */
 588	down_read(&fi->dax->sem);
 589	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
 590	if (node) {
 591		dmap = node_to_dmap(node);
 592		if (writable && !dmap->writable) {
 593			/* Upgrade read-only mapping to read-write. This will
 594			 * require exclusive fi->dax->sem lock as we don't want
 595			 * two threads to be trying to this simultaneously
 596			 * for same dmap. So drop shared lock and acquire
 597			 * exclusive lock.
 598			 *
 599			 * Before dropping fi->dax->sem lock, take reference
 600			 * on dmap so that its not freed by range reclaim.
 601			 */
 602			refcount_inc(&dmap->refcnt);
 603			up_read(&fi->dax->sem);
 604			pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n",
 605				 __func__, pos, length);
 606			return fuse_upgrade_dax_mapping(inode, pos, length,
 607							flags, iomap);
 608		} else {
 609			fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
 610			up_read(&fi->dax->sem);
 611			return 0;
 612		}
 613	} else {
 614		up_read(&fi->dax->sem);
 615		pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n",
 616				__func__, pos, length);
 617		if (pos >= i_size_read(inode))
 618			goto iomap_hole;
 619
 620		return fuse_setup_new_dax_mapping(inode, pos, length, flags,
 621						  iomap);
 622	}
 623
 624	/*
 625	 * If read beyond end of file happens, fs code seems to return
 626	 * it as hole
 627	 */
 628iomap_hole:
 629	fuse_fill_iomap_hole(iomap, length);
 630	pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n",
 631		 __func__, pos, length, iomap->length);
 632	return 0;
 633}
 634
 635static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 636			  ssize_t written, unsigned int flags,
 637			  struct iomap *iomap)
 638{
 639	struct fuse_dax_mapping *dmap = iomap->private;
 640
 641	if (dmap) {
 642		if (refcount_dec_and_test(&dmap->refcnt)) {
 643			/* refcount should not hit 0. This object only goes
 644			 * away when fuse connection goes away
 645			 */
 646			WARN_ON_ONCE(1);
 647		}
 648	}
 649
 650	/* DAX writes beyond end-of-file aren't handled using iomap, so the
 651	 * file size is unchanged and there is nothing to do here.
 652	 */
 653	return 0;
 654}
 655
 656static const struct iomap_ops fuse_iomap_ops = {
 657	.iomap_begin = fuse_iomap_begin,
 658	.iomap_end = fuse_iomap_end,
 659};
 660
 661static void fuse_wait_dax_page(struct inode *inode)
 662{
 663	filemap_invalidate_unlock(inode->i_mapping);
 664	schedule();
 665	filemap_invalidate_lock(inode->i_mapping);
 666}
 667
 668/* Should be called with mapping->invalidate_lock held exclusively. */
 669int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start,
 670				  u64 dmap_end)
 671{
 672	return dax_break_layout(inode, dmap_start, dmap_end,
 673				fuse_wait_dax_page);
 674}
 675
 676ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
 677{
 678	struct inode *inode = file_inode(iocb->ki_filp);
 679	ssize_t ret;
 680
 681	if (iocb->ki_flags & IOCB_NOWAIT) {
 682		if (!inode_trylock_shared(inode))
 683			return -EAGAIN;
 684	} else {
 685		inode_lock_shared(inode);
 686	}
 687
 688	ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops);
 689	inode_unlock_shared(inode);
 690
 691	/* TODO file_accessed(iocb->f_filp) */
 692	return ret;
 693}
 694
 695static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from)
 696{
 697	struct inode *inode = file_inode(iocb->ki_filp);
 698
 699	return (iov_iter_rw(from) == WRITE &&
 700		((iocb->ki_pos) >= i_size_read(inode) ||
 701		  (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode))));
 702}
 703
 704static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from)
 705{
 706	struct inode *inode = file_inode(iocb->ki_filp);
 707	struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
 708	ssize_t ret;
 709
 710	ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);
 711
 712	fuse_write_update_attr(inode, iocb->ki_pos, ret);
 713	return ret;
 714}
 715
 716ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
 717{
 718	struct inode *inode = file_inode(iocb->ki_filp);
 719	ssize_t ret;
 720
 721	if (iocb->ki_flags & IOCB_NOWAIT) {
 722		if (!inode_trylock(inode))
 723			return -EAGAIN;
 724	} else {
 725		inode_lock(inode);
 726	}
 727
 728	ret = generic_write_checks(iocb, from);
 729	if (ret <= 0)
 730		goto out;
 731
 732	ret = file_remove_privs(iocb->ki_filp);
 733	if (ret)
 734		goto out;
 735	/* TODO file_update_time() but we don't want metadata I/O */
 736
 737	/* Do not use dax for file extending writes as write and on
 738	 * disk i_size increase are not atomic otherwise.
 739	 */
 740	if (file_extending_write(iocb, from))
 741		ret = fuse_dax_direct_write(iocb, from);
 742	else
 743		ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops);
 744
 745out:
 746	inode_unlock(inode);
 747
 748	if (ret > 0)
 749		ret = generic_write_sync(iocb, ret);
 750	return ret;
 751}
 752
 753static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order,
 754		bool write)
 755{
 756	vm_fault_t ret;
 757	struct inode *inode = file_inode(vmf->vma->vm_file);
 758	struct super_block *sb = inode->i_sb;
 759	unsigned long pfn;
 760	int error = 0;
 761	struct fuse_conn *fc = get_fuse_conn(inode);
 762	struct fuse_conn_dax *fcd = fc->dax;
 763	bool retry = false;
 764
 765	if (write)
 766		sb_start_pagefault(sb);
 767retry:
 768	if (retry && !(fcd->nr_free_ranges > 0))
 769		wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0));
 770
 771	/*
 772	 * We need to serialize against not only truncate but also against
 773	 * fuse dax memory range reclaim. While a range is being reclaimed,
 774	 * we do not want any read/write/mmap to make progress and try
 775	 * to populate page cache or access memory we are trying to free.
 776	 */
 777	filemap_invalidate_lock_shared(inode->i_mapping);
 778	ret = dax_iomap_fault(vmf, order, &pfn, &error, &fuse_iomap_ops);
 779	if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) {
 780		error = 0;
 781		retry = true;
 782		filemap_invalidate_unlock_shared(inode->i_mapping);
 783		goto retry;
 784	}
 785
 786	if (ret & VM_FAULT_NEEDDSYNC)
 787		ret = dax_finish_sync_fault(vmf, order, pfn);
 788	filemap_invalidate_unlock_shared(inode->i_mapping);
 789
 790	if (write)
 791		sb_end_pagefault(sb);
 792
 793	return ret;
 794}
 795
 796static vm_fault_t fuse_dax_fault(struct vm_fault *vmf)
 797{
 798	return __fuse_dax_fault(vmf, 0, vmf->flags & FAULT_FLAG_WRITE);
 799}
 800
 801static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf, unsigned int order)
 802{
 803	return __fuse_dax_fault(vmf, order, vmf->flags & FAULT_FLAG_WRITE);
 804}
 805
 806static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf)
 807{
 808	return __fuse_dax_fault(vmf, 0, true);
 809}
 810
 811static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf)
 812{
 813	return __fuse_dax_fault(vmf, 0, true);
 814}
 815
 816static const struct vm_operations_struct fuse_dax_vm_ops = {
 817	.fault		= fuse_dax_fault,
 818	.huge_fault	= fuse_dax_huge_fault,
 819	.page_mkwrite	= fuse_dax_page_mkwrite,
 820	.pfn_mkwrite	= fuse_dax_pfn_mkwrite,
 821};
 822
 823int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma)
 824{
 825	file_accessed(file);
 826	vma->vm_ops = &fuse_dax_vm_ops;
 827	vm_flags_set(vma, VM_MIXEDMAP | VM_HUGEPAGE);
 828	return 0;
 829}
 830
 831static int dmap_writeback_invalidate(struct inode *inode,
 832				     struct fuse_dax_mapping *dmap)
 833{
 834	int ret;
 835	loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT;
 836	loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1);
 837
 838	ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos);
 839	if (ret) {
 840		pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n",
 841			 ret, start_pos, end_pos);
 842		return ret;
 843	}
 844
 845	ret = invalidate_inode_pages2_range(inode->i_mapping,
 846					    start_pos >> PAGE_SHIFT,
 847					    end_pos >> PAGE_SHIFT);
 848	if (ret)
 849		pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n",
 850			 ret);
 851
 852	return ret;
 853}
 854
 855static int reclaim_one_dmap_locked(struct inode *inode,
 856				   struct fuse_dax_mapping *dmap)
 857{
 858	int ret;
 859	struct fuse_inode *fi = get_fuse_inode(inode);
 860
 861	/*
 862	 * igrab() was done to make sure inode won't go under us, and this
 863	 * further avoids the race with evict().
 864	 */
 865	ret = dmap_writeback_invalidate(inode, dmap);
 866	if (ret)
 867		return ret;
 868
 869	/* Remove dax mapping from inode interval tree now */
 870	interval_tree_remove(&dmap->itn, &fi->dax->tree);
 871	fi->dax->nr--;
 872
 873	/* It is possible that umount/shutdown has killed the fuse connection
 874	 * and worker thread is trying to reclaim memory in parallel.  Don't
 875	 * warn in that case.
 876	 */
 877	ret = dmap_removemapping_one(inode, dmap);
 878	if (ret && ret != -ENOTCONN) {
 879		pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n",
 880			dmap->window_offset, dmap->length, ret);
 881	}
 882	return 0;
 883}
 884
 885/* Find first mapped dmap for an inode and return file offset. Caller needs
 886 * to hold fi->dax->sem lock either shared or exclusive.
 887 */
 888static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
 889{
 890	struct fuse_inode *fi = get_fuse_inode(inode);
 891	struct fuse_dax_mapping *dmap;
 892	struct interval_tree_node *node;
 893
 894	for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node;
 895	     node = interval_tree_iter_next(node, 0, -1)) {
 896		dmap = node_to_dmap(node);
 897		/* still in use. */
 898		if (refcount_read(&dmap->refcnt) > 1)
 899			continue;
 900
 901		return dmap;
 902	}
 903
 904	return NULL;
 905}
 906
 907/*
 908 * Find first mapping in the tree and free it and return it. Do not add
 909 * it back to free pool.
 910 */
 911static struct fuse_dax_mapping *
 912inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
 913			      bool *retry)
 914{
 915	struct fuse_inode *fi = get_fuse_inode(inode);
 916	struct fuse_dax_mapping *dmap;
 917	u64 dmap_start, dmap_end;
 918	unsigned long start_idx;
 919	int ret;
 920	struct interval_tree_node *node;
 921
 922	filemap_invalidate_lock(inode->i_mapping);
 923
 924	/* Lookup a dmap and corresponding file offset to reclaim. */
 925	down_read(&fi->dax->sem);
 926	dmap = inode_lookup_first_dmap(inode);
 927	if (dmap) {
 928		start_idx = dmap->itn.start;
 929		dmap_start = start_idx << FUSE_DAX_SHIFT;
 930		dmap_end = dmap_start + FUSE_DAX_SZ - 1;
 931	}
 932	up_read(&fi->dax->sem);
 933
 934	if (!dmap)
 935		goto out_mmap_sem;
 936	/*
 937	 * Make sure there are no references to inode pages using
 938	 * get_user_pages()
 939	 */
 940	ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
 941	if (ret) {
 942		pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n",
 943			 ret);
 944		dmap = ERR_PTR(ret);
 945		goto out_mmap_sem;
 946	}
 947
 948	down_write(&fi->dax->sem);
 949	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
 950	/* Range already got reclaimed by somebody else */
 951	if (!node) {
 952		if (retry)
 953			*retry = true;
 954		goto out_write_dmap_sem;
 955	}
 956
 957	dmap = node_to_dmap(node);
 958	/* still in use. */
 959	if (refcount_read(&dmap->refcnt) > 1) {
 960		dmap = NULL;
 961		if (retry)
 962			*retry = true;
 963		goto out_write_dmap_sem;
 964	}
 965
 966	ret = reclaim_one_dmap_locked(inode, dmap);
 967	if (ret < 0) {
 968		dmap = ERR_PTR(ret);
 969		goto out_write_dmap_sem;
 970	}
 971
 972	/* Clean up dmap. Do not add back to free list */
 973	dmap_remove_busy_list(fcd, dmap);
 974	dmap->inode = NULL;
 975	dmap->itn.start = dmap->itn.last = 0;
 976
 977	pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n",
 978		 __func__, inode, dmap->window_offset, dmap->length);
 979
 980out_write_dmap_sem:
 981	up_write(&fi->dax->sem);
 982out_mmap_sem:
 983	filemap_invalidate_unlock(inode->i_mapping);
 984	return dmap;
 985}
 986
 987static struct fuse_dax_mapping *
 988alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
 989{
 990	struct fuse_dax_mapping *dmap;
 991	struct fuse_inode *fi = get_fuse_inode(inode);
 992
 993	while (1) {
 994		bool retry = false;
 995
 996		dmap = alloc_dax_mapping(fcd);
 997		if (dmap)
 998			return dmap;
 999
1000		dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry);
1001		/*
1002		 * Either we got a mapping or it is an error, return in both
1003		 * the cases.
1004		 */
1005		if (dmap)
1006			return dmap;
1007
1008		/* If we could not reclaim a mapping because it
1009		 * had a reference or some other temporary failure,
1010		 * Try again. We want to give up inline reclaim only
1011		 * if there is no range assigned to this node. Otherwise
1012		 * if a deadlock is possible if we sleep with
1013		 * mapping->invalidate_lock held and worker to free memory
1014		 * can't make progress due to unavailability of
1015		 * mapping->invalidate_lock.  So sleep only if fi->dax->nr=0
1016		 */
1017		if (retry)
1018			continue;
1019		/*
1020		 * There are no mappings which can be reclaimed. Wait for one.
1021		 * We are not holding fi->dax->sem. So it is possible
1022		 * that range gets added now. But as we are not holding
1023		 * mapping->invalidate_lock, worker should still be able to
1024		 * free up a range and wake us up.
1025		 */
1026		if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {
1027			if (wait_event_killable_exclusive(fcd->range_waitq,
1028					(fcd->nr_free_ranges > 0))) {
1029				return ERR_PTR(-EINTR);
1030			}
1031		}
1032	}
1033}
1034
1035static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
1036					  struct inode *inode,
1037					  unsigned long start_idx)
1038{
1039	int ret;
1040	struct fuse_inode *fi = get_fuse_inode(inode);
1041	struct fuse_dax_mapping *dmap;
1042	struct interval_tree_node *node;
1043
1044	/* Find fuse dax mapping at file offset inode. */
1045	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
1046
1047	/* Range already got cleaned up by somebody else */
1048	if (!node)
1049		return 0;
1050	dmap = node_to_dmap(node);
1051
1052	/* still in use. */
1053	if (refcount_read(&dmap->refcnt) > 1)
1054		return 0;
1055
1056	ret = reclaim_one_dmap_locked(inode, dmap);
1057	if (ret < 0)
1058		return ret;
1059
1060	/* Cleanup dmap entry and add back to free list */
1061	spin_lock(&fcd->lock);
1062	dmap_reinit_add_to_free_pool(fcd, dmap);
1063	spin_unlock(&fcd->lock);
1064	return ret;
1065}
1066
1067/*
1068 * Free a range of memory.
1069 * Locking:
1070 * 1. Take mapping->invalidate_lock to block dax faults.
1071 * 2. Take fi->dax->sem to protect interval tree and also to make sure
1072 *    read/write can not reuse a dmap which we might be freeing.
1073 */
1074static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
1075				   struct inode *inode,
1076				   unsigned long start_idx,
1077				   unsigned long end_idx)
1078{
1079	int ret;
1080	struct fuse_inode *fi = get_fuse_inode(inode);
1081	loff_t dmap_start = start_idx << FUSE_DAX_SHIFT;
1082	loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1;
1083
1084	filemap_invalidate_lock(inode->i_mapping);
1085	ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
1086	if (ret) {
1087		pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n",
1088			 ret);
1089		goto out_mmap_sem;
1090	}
1091
1092	down_write(&fi->dax->sem);
1093	ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);
1094	up_write(&fi->dax->sem);
1095out_mmap_sem:
1096	filemap_invalidate_unlock(inode->i_mapping);
1097	return ret;
1098}
1099
1100static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
1101				   unsigned long nr_to_free)
1102{
1103	struct fuse_dax_mapping *dmap, *pos, *temp;
1104	int ret, nr_freed = 0;
1105	unsigned long start_idx = 0, end_idx = 0;
1106	struct inode *inode = NULL;
1107
1108	/* Pick first busy range and free it for now*/
1109	while (1) {
1110		if (nr_freed >= nr_to_free)
1111			break;
1112
1113		dmap = NULL;
1114		spin_lock(&fcd->lock);
1115
1116		if (!fcd->nr_busy_ranges) {
1117			spin_unlock(&fcd->lock);
1118			return 0;
1119		}
1120
1121		list_for_each_entry_safe(pos, temp, &fcd->busy_ranges,
1122						busy_list) {
1123			/* skip this range if it's in use. */
1124			if (refcount_read(&pos->refcnt) > 1)
1125				continue;
1126
1127			inode = igrab(pos->inode);
1128			/*
1129			 * This inode is going away. That will free
1130			 * up all the ranges anyway, continue to
1131			 * next range.
1132			 */
1133			if (!inode)
1134				continue;
1135			/*
1136			 * Take this element off list and add it tail. If
1137			 * this element can't be freed, it will help with
1138			 * selecting new element in next iteration of loop.
1139			 */
1140			dmap = pos;
1141			list_move_tail(&dmap->busy_list, &fcd->busy_ranges);
1142			start_idx = end_idx = dmap->itn.start;
1143			break;
1144		}
1145		spin_unlock(&fcd->lock);
1146		if (!dmap)
1147			return 0;
1148
1149		ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx);
1150		iput(inode);
1151		if (ret)
1152			return ret;
1153		nr_freed++;
1154	}
1155	return 0;
1156}
1157
1158static void fuse_dax_free_mem_worker(struct work_struct *work)
1159{
1160	int ret;
1161	struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,
1162						 free_work.work);
1163	ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);
1164	if (ret) {
1165		pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n",
1166			 ret);
1167	}
1168
1169	/* If number of free ranges are still below threshold, requeue */
1170	kick_dmap_free_worker(fcd, 1);
1171}
1172
1173static void fuse_free_dax_mem_ranges(struct list_head *mem_list)
1174{
1175	struct fuse_dax_mapping *range, *temp;
1176
1177	/* Free All allocated elements */
1178	list_for_each_entry_safe(range, temp, mem_list, list) {
1179		list_del(&range->list);
1180		if (!list_empty(&range->busy_list))
1181			list_del(&range->busy_list);
1182		kfree(range);
1183	}
1184}
1185
1186void fuse_dax_conn_free(struct fuse_conn *fc)
1187{
1188	if (fc->dax) {
1189		fuse_free_dax_mem_ranges(&fc->dax->free_ranges);
1190		kfree(fc->dax);
1191		fc->dax = NULL;
1192	}
1193}
1194
1195static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
1196{
1197	long nr_pages, nr_ranges;
1198	struct fuse_dax_mapping *range;
1199	int ret, id;
1200	size_t dax_size = -1;
1201	unsigned long i;
1202
1203	init_waitqueue_head(&fcd->range_waitq);
1204	INIT_LIST_HEAD(&fcd->free_ranges);
1205	INIT_LIST_HEAD(&fcd->busy_ranges);
1206	INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker);
1207
1208	id = dax_read_lock();
1209	nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size),
1210			DAX_ACCESS, NULL, NULL);
1211	dax_read_unlock(id);
1212	if (nr_pages < 0) {
1213		pr_debug("dax_direct_access() returned %ld\n", nr_pages);
1214		return nr_pages;
1215	}
1216
1217	nr_ranges = nr_pages/FUSE_DAX_PAGES;
1218	pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n",
1219		__func__, nr_pages, nr_ranges);
1220
1221	for (i = 0; i < nr_ranges; i++) {
1222		range = kzalloc(sizeof(struct fuse_dax_mapping), GFP_KERNEL);
1223		ret = -ENOMEM;
1224		if (!range)
1225			goto out_err;
1226
1227		/* TODO: This offset only works if virtio-fs driver is not
1228		 * having some memory hidden at the beginning. This needs
1229		 * better handling
1230		 */
1231		range->window_offset = i * FUSE_DAX_SZ;
1232		range->length = FUSE_DAX_SZ;
1233		INIT_LIST_HEAD(&range->busy_list);
1234		refcount_set(&range->refcnt, 1);
1235		list_add_tail(&range->list, &fcd->free_ranges);
1236	}
1237
1238	fcd->nr_free_ranges = nr_ranges;
1239	fcd->nr_ranges = nr_ranges;
1240	return 0;
1241out_err:
1242	/* Free All allocated elements */
1243	fuse_free_dax_mem_ranges(&fcd->free_ranges);
1244	return ret;
1245}
1246
1247int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode,
1248			struct dax_device *dax_dev)
1249{
1250	struct fuse_conn_dax *fcd;
1251	int err;
1252
1253	fc->dax_mode = dax_mode;
1254
1255	if (!dax_dev)
1256		return 0;
1257
1258	fcd = kzalloc(sizeof(*fcd), GFP_KERNEL);
1259	if (!fcd)
1260		return -ENOMEM;
1261
1262	spin_lock_init(&fcd->lock);
1263	fcd->dev = dax_dev;
1264	err = fuse_dax_mem_range_init(fcd);
1265	if (err) {
1266		kfree(fcd);
1267		return err;
1268	}
1269
1270	fc->dax = fcd;
1271	return 0;
1272}
1273
1274bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
1275{
1276	struct fuse_conn *fc = get_fuse_conn_super(sb);
1277
1278	fi->dax = NULL;
1279	if (fc->dax) {
1280		fi->dax = kzalloc(sizeof(*fi->dax), GFP_KERNEL_ACCOUNT);
1281		if (!fi->dax)
1282			return false;
1283
1284		init_rwsem(&fi->dax->sem);
1285		fi->dax->tree = RB_ROOT_CACHED;
1286	}
1287
1288	return true;
1289}
1290
1291static const struct address_space_operations fuse_dax_file_aops  = {
1292	.direct_IO	= noop_direct_IO,
1293	.dirty_folio	= noop_dirty_folio,
1294};
1295
1296static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags)
1297{
1298	struct fuse_conn *fc = get_fuse_conn(inode);
1299	enum fuse_dax_mode dax_mode = fc->dax_mode;
1300
1301	if (dax_mode == FUSE_DAX_NEVER)
1302		return false;
1303
1304	/*
1305	 * fc->dax may be NULL in 'inode' mode when filesystem device doesn't
1306	 * support DAX, in which case it will silently fallback to 'never' mode.
1307	 */
1308	if (!fc->dax)
1309		return false;
1310
1311	if (dax_mode == FUSE_DAX_ALWAYS)
1312		return true;
1313
1314	/* dax_mode is FUSE_DAX_INODE* */
1315	return fc->inode_dax && (flags & FUSE_ATTR_DAX);
1316}
1317
1318void fuse_dax_inode_init(struct inode *inode, unsigned int flags)
1319{
1320	if (!fuse_should_enable_dax(inode, flags))
1321		return;
1322
1323	inode->i_flags |= S_DAX;
1324	inode->i_data.a_ops = &fuse_dax_file_aops;
1325}
1326
1327void fuse_dax_dontcache(struct inode *inode, unsigned int flags)
1328{
1329	struct fuse_conn *fc = get_fuse_conn(inode);
1330
1331	if (fuse_is_inode_dax_mode(fc->dax_mode) &&
1332	    ((bool) IS_DAX(inode) != (bool) (flags & FUSE_ATTR_DAX)))
1333		d_mark_dontcache(inode);
1334}
1335
1336bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
1337{
1338	if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) {
1339		pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n",
1340			map_alignment, FUSE_DAX_SZ);
1341		return false;
1342	}
1343	return true;
1344}
1345
1346void fuse_dax_cancel_work(struct fuse_conn *fc)
1347{
1348	struct fuse_conn_dax *fcd = fc->dax;
1349
1350	if (fcd)
1351		cancel_delayed_work_sync(&fcd->free_work);
1352
1353}
1354EXPORT_SYMBOL_GPL(fuse_dax_cancel_work);