super.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright(c) 2017 Intel Corporation. All rights reserved.
  4 */
  5#include <linux/pagemap.h>
  6#include <linux/module.h>
  7#include <linux/mount.h>
  8#include <linux/pseudo_fs.h>
  9#include <linux/magic.h>
 10#include <linux/cdev.h>
 11#include <linux/slab.h>
 12#include <linux/uio.h>
 13#include <linux/dax.h>
 14#include <linux/fs.h>
 15#include <linux/cacheinfo.h>
 16#include "dax-private.h"
 17
 18/**
 19 * struct dax_device - anchor object for dax services
 20 * @inode: core vfs
 21 * @cdev: optional character interface for "device dax"
 22 * @private: dax driver private data
 23 * @flags: state and boolean properties
 24 * @ops: operations for this device
 25 * @holder_data: holder of a dax_device: could be filesystem or mapped device
 26 * @holder_ops: operations for the inner holder
 27 */
 28struct dax_device {
 29	struct inode inode;
 30	struct cdev cdev;
 31	void *private;
 32	unsigned long flags;
 33	const struct dax_operations *ops;
 34	void *holder_data;
 35	const struct dax_holder_operations *holder_ops;
 36};
 37
 38static dev_t dax_devt;
 39DEFINE_STATIC_SRCU(dax_srcu);
 40static struct vfsmount *dax_mnt;
 41static DEFINE_IDA(dax_minor_ida);
 42static struct kmem_cache *dax_cache __read_mostly;
 43static struct super_block *dax_superblock __read_mostly;
 44
 45int dax_read_lock(void)
 46{
 47	return srcu_read_lock(&dax_srcu);
 48}
 49EXPORT_SYMBOL_GPL(dax_read_lock);
 50
 51void dax_read_unlock(int id)
 52{
 53	srcu_read_unlock(&dax_srcu, id);
 54}
 55EXPORT_SYMBOL_GPL(dax_read_unlock);
 56
 57#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
 58#include <linux/blkdev.h>
 59
 60static DEFINE_XARRAY(dax_hosts);
 61
 62int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
 63{
 64	return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);
 65}
 66EXPORT_SYMBOL_GPL(dax_add_host);
 67
 68void dax_remove_host(struct gendisk *disk)
 69{
 70	xa_erase(&dax_hosts, (unsigned long)disk);
 71}
 72EXPORT_SYMBOL_GPL(dax_remove_host);
 73
 74/**
 75 * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax
 76 * @bdev: block device to find a dax_device for
 77 * @start_off: returns the byte offset into the dax_device that @bdev starts
 78 * @holder: filesystem or mapped device inside the dax_device
 79 * @ops: operations for the inner holder
 80 */
 81struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
 82		void *holder, const struct dax_holder_operations *ops)
 83{
 84	struct dax_device *dax_dev;
 85	u64 part_size;
 86	int id;
 87
 88	if (!blk_queue_dax(bdev->bd_disk->queue))
 89		return NULL;
 90
 91	*start_off = get_start_sect(bdev) * SECTOR_SIZE;
 92	part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE;
 93	if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) {
 94		pr_info("%pg: error: unaligned partition for dax\n", bdev);
 95		return NULL;
 96	}
 97
 98	id = dax_read_lock();
 99	dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);
100	if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))
101		dax_dev = NULL;
102	else if (holder) {
103		if (!cmpxchg(&dax_dev->holder_data, NULL, holder))
104			dax_dev->holder_ops = ops;
105		else
106			dax_dev = NULL;
107	}
108	dax_read_unlock(id);
109
110	return dax_dev;
111}
112EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
113
114void fs_put_dax(struct dax_device *dax_dev, void *holder)
115{
116	if (dax_dev && holder &&
117	    cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)
118		dax_dev->holder_ops = NULL;
119	put_dax(dax_dev);
120}
121EXPORT_SYMBOL_GPL(fs_put_dax);
122#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
123
124enum dax_device_flags {
125	/* !alive + rcu grace period == no new operations / mappings */
126	DAXDEV_ALIVE,
127	/* gate whether dax_flush() calls the low level flush routine */
128	DAXDEV_WRITE_CACHE,
129	/* flag to check if device supports synchronous flush */
130	DAXDEV_SYNC,
131	/* do not leave the caches dirty after writes */
132	DAXDEV_NOCACHE,
133	/* handle CPU fetch exceptions during reads */
134	DAXDEV_NOMC,
135};
136
137/**
138 * dax_direct_access() - translate a device pgoff to an absolute pfn
139 * @dax_dev: a dax_device instance representing the logical memory range
140 * @pgoff: offset in pages from the start of the device to translate
141 * @nr_pages: number of consecutive pages caller can handle relative to @pfn
142 * @mode: indicator on normal access or recovery write
143 * @kaddr: output parameter that returns a virtual address mapping of pfn
144 * @pfn: output parameter that returns an absolute pfn translation of @pgoff
145 *
146 * Return: negative errno if an error occurs, otherwise the number of
147 * pages accessible at the device relative @pgoff.
148 */
149long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
150		enum dax_access_mode mode, void **kaddr, unsigned long *pfn)
151{
152	long avail;
153
154	if (!dax_dev)
155		return -EOPNOTSUPP;
156
157	if (!dax_alive(dax_dev))
158		return -ENXIO;
159
160	if (nr_pages < 0)
161		return -EINVAL;
162
163	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,
164			mode, kaddr, pfn);
165	if (!avail)
166		return -ERANGE;
167	return min(avail, nr_pages);
168}
169EXPORT_SYMBOL_GPL(dax_direct_access);
170
171size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
172		size_t bytes, struct iov_iter *i)
173{
174	if (!dax_alive(dax_dev))
175		return 0;
176
177	/*
178	 * The userspace address for the memory copy has already been validated
179	 * via access_ok() in vfs_write, so use the 'no check' version to bypass
180	 * the HARDENED_USERCOPY overhead.
181	 */
182	if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))
183		return _copy_from_iter_flushcache(addr, bytes, i);
184	return _copy_from_iter(addr, bytes, i);
185}
186
187size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
188		size_t bytes, struct iov_iter *i)
189{
190	if (!dax_alive(dax_dev))
191		return 0;
192
193	/*
194	 * The userspace address for the memory copy has already been validated
195	 * via access_ok() in vfs_red, so use the 'no check' version to bypass
196	 * the HARDENED_USERCOPY overhead.
197	 */
198	if (test_bit(DAXDEV_NOMC, &dax_dev->flags))
199		return _copy_mc_to_iter(addr, bytes, i);
200	return _copy_to_iter(addr, bytes, i);
201}
202
203int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
204			size_t nr_pages)
205{
206	int ret;
207
208	if (!dax_alive(dax_dev))
209		return -ENXIO;
210	/*
211	 * There are no callers that want to zero more than one page as of now.
212	 * Once users are there, this check can be removed after the
213	 * device mapper code has been updated to split ranges across targets.
214	 */
215	if (nr_pages != 1)
216		return -EIO;
217
218	ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);
219	return dax_mem2blk_err(ret);
220}
221EXPORT_SYMBOL_GPL(dax_zero_page_range);
222
223size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
224		void *addr, size_t bytes, struct iov_iter *iter)
225{
226	if (!dax_dev->ops->recovery_write)
227		return 0;
228	return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);
229}
230EXPORT_SYMBOL_GPL(dax_recovery_write);
231
232int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,
233			      u64 len, int mf_flags)
234{
235	int rc, id;
236
237	id = dax_read_lock();
238	if (!dax_alive(dax_dev)) {
239		rc = -ENXIO;
240		goto out;
241	}
242
243	if (!dax_dev->holder_ops) {
244		rc = -EOPNOTSUPP;
245		goto out;
246	}
247
248	rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);
249out:
250	dax_read_unlock(id);
251	return rc;
252}
253EXPORT_SYMBOL_GPL(dax_holder_notify_failure);
254
255#ifdef CONFIG_ARCH_HAS_PMEM_API
256void arch_wb_cache_pmem(void *addr, size_t size);
257void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
258{
259	if (unlikely(!dax_write_cache_enabled(dax_dev)))
260		return;
261
262	arch_wb_cache_pmem(addr, size);
263}
264#else
265void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
266{
267}
268#endif
269EXPORT_SYMBOL_GPL(dax_flush);
270
271void dax_write_cache(struct dax_device *dax_dev, bool wc)
272{
273	if (wc)
274		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
275	else
276		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
277}
278EXPORT_SYMBOL_GPL(dax_write_cache);
279
280bool dax_write_cache_enabled(struct dax_device *dax_dev)
281{
282	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
283}
284EXPORT_SYMBOL_GPL(dax_write_cache_enabled);
285
286bool dax_synchronous(struct dax_device *dax_dev)
287{
288	return test_bit(DAXDEV_SYNC, &dax_dev->flags);
289}
290EXPORT_SYMBOL_GPL(dax_synchronous);
291
292void set_dax_synchronous(struct dax_device *dax_dev)
293{
294	set_bit(DAXDEV_SYNC, &dax_dev->flags);
295}
296EXPORT_SYMBOL_GPL(set_dax_synchronous);
297
298void set_dax_nocache(struct dax_device *dax_dev)
299{
300	set_bit(DAXDEV_NOCACHE, &dax_dev->flags);
301}
302EXPORT_SYMBOL_GPL(set_dax_nocache);
303
304void set_dax_nomc(struct dax_device *dax_dev)
305{
306	set_bit(DAXDEV_NOMC, &dax_dev->flags);
307}
308EXPORT_SYMBOL_GPL(set_dax_nomc);
309
310bool dax_alive(struct dax_device *dax_dev)
311{
312	lockdep_assert_held(&dax_srcu);
313	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);
314}
315EXPORT_SYMBOL_GPL(dax_alive);
316
317/*
318 * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring
319 * that any fault handlers or operations that might have seen
320 * dax_alive(), have completed.  Any operations that start after
321 * synchronize_srcu() has run will abort upon seeing !dax_alive().
322 *
323 * Note, because alloc_dax() returns an ERR_PTR() on error, callers
324 * typically store its result into a local variable in order to check
325 * the result. Therefore, care must be taken to populate the struct
326 * device dax_dev field make sure the dax_dev is not leaked.
327 */
328void kill_dax(struct dax_device *dax_dev)
329{
330	if (!dax_dev)
331		return;
332
333	if (dax_dev->holder_data != NULL)
334		dax_holder_notify_failure(dax_dev, 0, U64_MAX,
335				MF_MEM_PRE_REMOVE);
336
337	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);
338	synchronize_srcu(&dax_srcu);
339
340	/* clear holder data */
341	dax_dev->holder_ops = NULL;
342	dax_dev->holder_data = NULL;
343}
344EXPORT_SYMBOL_GPL(kill_dax);
345
346void run_dax(struct dax_device *dax_dev)
347{
348	set_bit(DAXDEV_ALIVE, &dax_dev->flags);
349}
350EXPORT_SYMBOL_GPL(run_dax);
351
352static struct inode *dax_alloc_inode(struct super_block *sb)
353{
354	struct dax_device *dax_dev;
355	struct inode *inode;
356
357	dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL);
358	if (!dax_dev)
359		return NULL;
360
361	inode = &dax_dev->inode;
362	inode->i_rdev = 0;
363	return inode;
364}
365
366static struct dax_device *to_dax_dev(struct inode *inode)
367{
368	return container_of(inode, struct dax_device, inode);
369}
370
371static void dax_free_inode(struct inode *inode)
372{
373	struct dax_device *dax_dev = to_dax_dev(inode);
374	if (inode->i_rdev)
375		ida_free(&dax_minor_ida, iminor(inode));
376	kmem_cache_free(dax_cache, dax_dev);
377}
378
379static void dax_destroy_inode(struct inode *inode)
380{
381	struct dax_device *dax_dev = to_dax_dev(inode);
382	WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),
383			"kill_dax() must be called before final iput()\n");
384}
385
386static const struct super_operations dax_sops = {
387	.statfs = simple_statfs,
388	.alloc_inode = dax_alloc_inode,
389	.destroy_inode = dax_destroy_inode,
390	.free_inode = dax_free_inode,
391	.drop_inode = inode_just_drop,
392};
393
394static int dax_init_fs_context(struct fs_context *fc)
395{
396	struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC);
397	if (!ctx)
398		return -ENOMEM;
399	ctx->ops = &dax_sops;
400	return 0;
401}
402
403static struct file_system_type dax_fs_type = {
404	.name		= "dax",
405	.init_fs_context = dax_init_fs_context,
406	.kill_sb	= kill_anon_super,
407};
408
409static int dax_test(struct inode *inode, void *data)
410{
411	dev_t devt = *(dev_t *) data;
412
413	return inode->i_rdev == devt;
414}
415
416static int dax_set(struct inode *inode, void *data)
417{
418	dev_t devt = *(dev_t *) data;
419
420	inode->i_rdev = devt;
421	return 0;
422}
423
424static struct dax_device *dax_dev_get(dev_t devt)
425{
426	struct dax_device *dax_dev;
427	struct inode *inode;
428
429	inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
430			dax_test, dax_set, &devt);
431
432	if (!inode)
433		return NULL;
434
435	dax_dev = to_dax_dev(inode);
436	if (inode_state_read_once(inode) & I_NEW) {
437		set_bit(DAXDEV_ALIVE, &dax_dev->flags);
438		inode->i_cdev = &dax_dev->cdev;
439		inode->i_mode = S_IFCHR;
440		inode->i_flags = S_DAX;
441		mapping_set_gfp_mask(&inode->i_data, GFP_USER);
442		unlock_new_inode(inode);
443	}
444
445	return dax_dev;
446}
447
448struct dax_device *alloc_dax(void *private, const struct dax_operations *ops)
449{
450	struct dax_device *dax_dev;
451	dev_t devt;
452	int minor;
453
454	/*
455	 * Unavailable on architectures with virtually aliased data caches,
456	 * except for device-dax (NULL operations pointer), which does
457	 * not use aliased mappings from the kernel.
458	 */
459	if (ops && cpu_dcache_is_aliasing())
460		return ERR_PTR(-EOPNOTSUPP);
461
462	if (WARN_ON_ONCE(ops && !ops->zero_page_range))
463		return ERR_PTR(-EINVAL);
464
465	minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL);
466	if (minor < 0)
467		return ERR_PTR(-ENOMEM);
468
469	devt = MKDEV(MAJOR(dax_devt), minor);
470	dax_dev = dax_dev_get(devt);
471	if (!dax_dev)
472		goto err_dev;
473
474	dax_dev->ops = ops;
475	dax_dev->private = private;
476	return dax_dev;
477
478 err_dev:
479	ida_free(&dax_minor_ida, minor);
480	return ERR_PTR(-ENOMEM);
481}
482EXPORT_SYMBOL_GPL(alloc_dax);
483
484void put_dax(struct dax_device *dax_dev)
485{
486	if (!dax_dev)
487		return;
488	iput(&dax_dev->inode);
489}
490EXPORT_SYMBOL_GPL(put_dax);
491
492/**
493 * dax_holder() - obtain the holder of a dax device
494 * @dax_dev: a dax_device instance
495 *
496 * Return: the holder's data which represents the holder if registered,
497 * otherwize NULL.
498 */
499void *dax_holder(struct dax_device *dax_dev)
500{
501	return dax_dev->holder_data;
502}
503EXPORT_SYMBOL_GPL(dax_holder);
504
505/**
506 * inode_dax: convert a public inode into its dax_dev
507 * @inode: An inode with i_cdev pointing to a dax_dev
508 *
509 * Note this is not equivalent to to_dax_dev() which is for private
510 * internal use where we know the inode filesystem type == dax_fs_type.
511 */
512struct dax_device *inode_dax(struct inode *inode)
513{
514	struct cdev *cdev = inode->i_cdev;
515
516	return container_of(cdev, struct dax_device, cdev);
517}
518EXPORT_SYMBOL_GPL(inode_dax);
519
520struct inode *dax_inode(struct dax_device *dax_dev)
521{
522	return &dax_dev->inode;
523}
524EXPORT_SYMBOL_GPL(dax_inode);
525
526void *dax_get_private(struct dax_device *dax_dev)
527{
528	if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))
529		return NULL;
530	return dax_dev->private;
531}
532EXPORT_SYMBOL_GPL(dax_get_private);
533
534static void init_once(void *_dax_dev)
535{
536	struct dax_device *dax_dev = _dax_dev;
537	struct inode *inode = &dax_dev->inode;
538
539	memset(dax_dev, 0, sizeof(*dax_dev));
540	inode_init_once(inode);
541}
542
543static int dax_fs_init(void)
544{
545	int rc;
546
547	dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,
548			SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
549			init_once);
550	if (!dax_cache)
551		return -ENOMEM;
552
553	dax_mnt = kern_mount(&dax_fs_type);
554	if (IS_ERR(dax_mnt)) {
555		rc = PTR_ERR(dax_mnt);
556		goto err_mount;
557	}
558	dax_superblock = dax_mnt->mnt_sb;
559
560	return 0;
561
562 err_mount:
563	kmem_cache_destroy(dax_cache);
564
565	return rc;
566}
567
568static void dax_fs_exit(void)
569{
570	kern_unmount(dax_mnt);
571	rcu_barrier();
572	kmem_cache_destroy(dax_cache);
573}
574
575static int __init dax_core_init(void)
576{
577	int rc;
578
579	rc = dax_fs_init();
580	if (rc)
581		return rc;
582
583	rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");
584	if (rc)
585		goto err_chrdev;
586
587	rc = dax_bus_init();
588	if (rc)
589		goto err_bus;
590	return 0;
591
592err_bus:
593	unregister_chrdev_region(dax_devt, MINORMASK+1);
594err_chrdev:
595	dax_fs_exit();
596	return 0;
597}
598
599static void __exit dax_core_exit(void)
600{
601	dax_bus_exit();
602	unregister_chrdev_region(dax_devt, MINORMASK+1);
603	ida_destroy(&dax_minor_ida);
604	dax_fs_exit();
605}
606
607MODULE_AUTHOR("Intel Corporation");
608MODULE_DESCRIPTION("DAX: direct access to differentiated memory");
609MODULE_LICENSE("GPL v2");
610subsys_initcall(dax_core_init);
611module_exit(dax_core_exit);