inode.c

   1/*
   2 * hugetlbpage-backed filesystem.  Based on ramfs.
   3 *
   4 * Nadia Yvette Chambers, 2002
   5 *
   6 * Copyright (C) 2002 Linus Torvalds.
   7 * License: GPL
   8 */
   9
  10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  11
  12#include <linux/thread_info.h>
  13#include <asm/current.h>
  14#include <linux/falloc.h>
  15#include <linux/fs.h>
  16#include <linux/mount.h>
  17#include <linux/file.h>
  18#include <linux/kernel.h>
  19#include <linux/writeback.h>
  20#include <linux/pagemap.h>
  21#include <linux/highmem.h>
  22#include <linux/init.h>
  23#include <linux/string.h>
  24#include <linux/capability.h>
  25#include <linux/ctype.h>
  26#include <linux/backing-dev.h>
  27#include <linux/hugetlb.h>
  28#include <linux/pagevec.h>
  29#include <linux/fs_parser.h>
  30#include <linux/mman.h>
  31#include <linux/slab.h>
  32#include <linux/dnotify.h>
  33#include <linux/statfs.h>
  34#include <linux/security.h>
  35#include <linux/magic.h>
  36#include <linux/migrate.h>
  37#include <linux/uio.h>
  38
  39#include <linux/uaccess.h>
  40#include <linux/sched/mm.h>
  41
  42#define CREATE_TRACE_POINTS
  43#include <trace/events/hugetlbfs.h>
  44
  45static const struct address_space_operations hugetlbfs_aops;
  46static const struct file_operations hugetlbfs_file_operations;
  47static const struct inode_operations hugetlbfs_dir_inode_operations;
  48static const struct inode_operations hugetlbfs_inode_operations;
  49
  50enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
  51
  52struct hugetlbfs_fs_context {
  53	struct hstate		*hstate;
  54	unsigned long long	max_size_opt;
  55	unsigned long long	min_size_opt;
  56	long			max_hpages;
  57	long			nr_inodes;
  58	long			min_hpages;
  59	enum hugetlbfs_size_type max_val_type;
  60	enum hugetlbfs_size_type min_val_type;
  61	kuid_t			uid;
  62	kgid_t			gid;
  63	umode_t			mode;
  64};
  65
  66int sysctl_hugetlb_shm_group;
  67
  68enum hugetlb_param {
  69	Opt_gid,
  70	Opt_min_size,
  71	Opt_mode,
  72	Opt_nr_inodes,
  73	Opt_pagesize,
  74	Opt_size,
  75	Opt_uid,
  76};
  77
  78static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
  79	fsparam_gid   ("gid",		Opt_gid),
  80	fsparam_string("min_size",	Opt_min_size),
  81	fsparam_u32oct("mode",		Opt_mode),
  82	fsparam_string("nr_inodes",	Opt_nr_inodes),
  83	fsparam_string("pagesize",	Opt_pagesize),
  84	fsparam_string("size",		Opt_size),
  85	fsparam_uid   ("uid",		Opt_uid),
  86	{}
  87};
  88
  89/*
  90 * Mask used when checking the page offset value passed in via system
  91 * calls.  This value will be converted to a loff_t which is signed.
  92 * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
  93 * value.  The extra bit (- 1 in the shift value) is to take the sign
  94 * bit into account.
  95 */
  96#define PGOFF_LOFFT_MAX \
  97	(((1UL << (PAGE_SHIFT + 1)) - 1) <<  (BITS_PER_LONG - (PAGE_SHIFT + 1)))
  98
  99static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
 100{
 101	/* Unfortunate we have to reassign vma->vm_private_data. */
 102	return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
 103}
 104
 105static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 106{
 107	struct file *file = desc->file;
 108	struct inode *inode = file_inode(file);
 109	loff_t len, vma_len;
 110	int ret;
 111	struct hstate *h = hstate_file(file);
 112	vm_flags_t vm_flags;
 113
 114	/*
 115	 * vma address alignment (but not the pgoff alignment) has
 116	 * already been checked by prepare_hugepage_range.  If you add
 117	 * any error returns here, do so after setting VM_HUGETLB, so
 118	 * is_vm_hugetlb_page tests below unmap_region go the right
 119	 * way when do_mmap unwinds (may be important on powerpc
 120	 * and ia64).
 121	 */
 122	desc->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
 123	desc->vm_ops = &hugetlb_vm_ops;
 124
 125	/*
 126	 * page based offset in vm_pgoff could be sufficiently large to
 127	 * overflow a loff_t when converted to byte offset.  This can
 128	 * only happen on architectures where sizeof(loff_t) ==
 129	 * sizeof(unsigned long).  So, only check in those instances.
 130	 */
 131	if (sizeof(unsigned long) == sizeof(loff_t)) {
 132		if (desc->pgoff & PGOFF_LOFFT_MAX)
 133			return -EINVAL;
 134	}
 135
 136	/* must be huge page aligned */
 137	if (desc->pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
 138		return -EINVAL;
 139
 140	vma_len = (loff_t)vma_desc_size(desc);
 141	len = vma_len + ((loff_t)desc->pgoff << PAGE_SHIFT);
 142	/* check for overflow */
 143	if (len < vma_len)
 144		return -EINVAL;
 145
 146	inode_lock(inode);
 147	file_accessed(file);
 148
 149	ret = -ENOMEM;
 150
 151	vm_flags = desc->vm_flags;
 152	/*
 153	 * for SHM_HUGETLB, the pages are reserved in the shmget() call so skip
 154	 * reserving here. Note: only for SHM hugetlbfs file, the inode
 155	 * flag S_PRIVATE is set.
 156	 */
 157	if (inode->i_flags & S_PRIVATE)
 158		vm_flags |= VM_NORESERVE;
 159
 160	if (hugetlb_reserve_pages(inode,
 161			desc->pgoff >> huge_page_order(h),
 162			len >> huge_page_shift(h), desc,
 163			vm_flags) < 0)
 164		goto out;
 165
 166	ret = 0;
 167	if ((desc->vm_flags & VM_WRITE) && inode->i_size < len)
 168		i_size_write(inode, len);
 169out:
 170	inode_unlock(inode);
 171
 172	if (!ret) {
 173		/* Allocate the VMA lock after we set it up. */
 174		desc->action.success_hook = hugetlb_file_mmap_prepare_success;
 175		/*
 176		 * We cannot permit the rmap finding this VMA in the time
 177		 * between the VMA being inserted into the VMA tree and the
 178		 * completion/success hook being invoked.
 179		 *
 180		 * This is because we establish a per-VMA hugetlb lock which can
 181		 * be raced by rmap.
 182		 */
 183		desc->action.hide_from_rmap_until_complete = true;
 184	}
 185	return ret;
 186}
 187
 188/*
 189 * Called under mmap_write_lock(mm).
 190 */
 191
 192unsigned long
 193hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 194			    unsigned long len, unsigned long pgoff,
 195			    unsigned long flags)
 196{
 197	unsigned long addr0 = 0;
 198	struct hstate *h = hstate_file(file);
 199
 200	if (len & ~huge_page_mask(h))
 201		return -EINVAL;
 202	if ((flags & MAP_FIXED) && (addr & ~huge_page_mask(h)))
 203		return -EINVAL;
 204	if (addr)
 205		addr0 = ALIGN(addr, huge_page_size(h));
 206
 207	return mm_get_unmapped_area_vmflags(file, addr0, len, pgoff, flags, 0);
 208}
 209
 210/*
 211 * Someone wants to read @bytes from a HWPOISON hugetlb @folio from @offset.
 212 * Returns the maximum number of bytes one can read without touching the 1st raw
 213 * HWPOISON page.
 214 */
 215static size_t adjust_range_hwpoison(struct folio *folio, size_t offset,
 216		size_t bytes)
 217{
 218	struct page *page = folio_page(folio, offset / PAGE_SIZE);
 219	size_t safe_bytes;
 220
 221	if (is_raw_hwpoison_page_in_hugepage(page))
 222		return 0;
 223	/* Safe to read the remaining bytes in this page. */
 224	safe_bytes = PAGE_SIZE - (offset % PAGE_SIZE);
 225	page++;
 226
 227	/* Check each remaining page as long as we are not done yet. */
 228	for (; safe_bytes < bytes; safe_bytes += PAGE_SIZE, page++)
 229		if (is_raw_hwpoison_page_in_hugepage(page))
 230			break;
 231
 232	return min(safe_bytes, bytes);
 233}
 234
 235/*
 236 * Support for read() - Find the page attached to f_mapping and copy out the
 237 * data. This provides functionality similar to filemap_read().
 238 */
 239static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
 240{
 241	struct file *file = iocb->ki_filp;
 242	struct hstate *h = hstate_file(file);
 243	struct address_space *mapping = file->f_mapping;
 244	struct inode *inode = mapping->host;
 245	unsigned long index = iocb->ki_pos >> huge_page_shift(h);
 246	unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
 247	unsigned long end_index;
 248	loff_t isize;
 249	ssize_t retval = 0;
 250
 251	while (iov_iter_count(to)) {
 252		struct folio *folio;
 253		size_t nr, copied, want;
 254
 255		/* nr is the maximum number of bytes to copy from this page */
 256		nr = huge_page_size(h);
 257		isize = i_size_read(inode);
 258		if (!isize)
 259			break;
 260		end_index = (isize - 1) >> huge_page_shift(h);
 261		if (index > end_index)
 262			break;
 263		if (index == end_index) {
 264			nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
 265			if (nr <= offset)
 266				break;
 267		}
 268		nr = nr - offset;
 269
 270		/* Find the folio */
 271		folio = filemap_lock_hugetlb_folio(h, mapping, index);
 272		if (IS_ERR(folio)) {
 273			/*
 274			 * We have a HOLE, zero out the user-buffer for the
 275			 * length of the hole or request.
 276			 */
 277			copied = iov_iter_zero(nr, to);
 278		} else {
 279			folio_unlock(folio);
 280
 281			if (!folio_test_hwpoison(folio))
 282				want = nr;
 283			else {
 284				/*
 285				 * Adjust how many bytes safe to read without
 286				 * touching the 1st raw HWPOISON page after
 287				 * offset.
 288				 */
 289				want = adjust_range_hwpoison(folio, offset, nr);
 290				if (want == 0) {
 291					folio_put(folio);
 292					retval = -EIO;
 293					break;
 294				}
 295			}
 296
 297			/*
 298			 * We have the folio, copy it to user space buffer.
 299			 */
 300			copied = copy_folio_to_iter(folio, offset, want, to);
 301			folio_put(folio);
 302		}
 303		offset += copied;
 304		retval += copied;
 305		if (copied != nr && iov_iter_count(to)) {
 306			if (!retval)
 307				retval = -EFAULT;
 308			break;
 309		}
 310		index += offset >> huge_page_shift(h);
 311		offset &= ~huge_page_mask(h);
 312	}
 313	iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
 314	return retval;
 315}
 316
 317static int hugetlbfs_write_begin(const struct kiocb *iocb,
 318			struct address_space *mapping,
 319			loff_t pos, unsigned len,
 320			struct folio **foliop, void **fsdata)
 321{
 322	return -EINVAL;
 323}
 324
 325static int hugetlbfs_write_end(const struct kiocb *iocb,
 326			       struct address_space *mapping,
 327			       loff_t pos, unsigned len, unsigned copied,
 328			       struct folio *folio, void *fsdata)
 329{
 330	BUG();
 331	return -EINVAL;
 332}
 333
 334static void hugetlb_delete_from_page_cache(struct folio *folio)
 335{
 336	folio_clear_dirty(folio);
 337	folio_clear_uptodate(folio);
 338	filemap_remove_folio(folio);
 339}
 340
 341/*
 342 * Called with i_mmap_rwsem held for inode based vma maps.  This makes
 343 * sure vma (and vm_mm) will not go away.  We also hold the hugetlb fault
 344 * mutex for the page in the mapping.  So, we can not race with page being
 345 * faulted into the vma.
 346 */
 347static bool hugetlb_vma_maps_pfn(struct vm_area_struct *vma,
 348				unsigned long addr, unsigned long pfn)
 349{
 350	pte_t *ptep, pte;
 351
 352	ptep = hugetlb_walk(vma, addr, huge_page_size(hstate_vma(vma)));
 353	if (!ptep)
 354		return false;
 355
 356	pte = huge_ptep_get(vma->vm_mm, addr, ptep);
 357	if (huge_pte_none(pte) || !pte_present(pte))
 358		return false;
 359
 360	if (pte_pfn(pte) == pfn)
 361		return true;
 362
 363	return false;
 364}
 365
 366/*
 367 * Can vma_offset_start/vma_offset_end overflow on 32-bit arches?
 368 * No, because the interval tree returns us only those vmas
 369 * which overlap the truncated area starting at pgoff,
 370 * and no vma on a 32-bit arch can span beyond the 4GB.
 371 */
 372static unsigned long vma_offset_start(struct vm_area_struct *vma, pgoff_t start)
 373{
 374	unsigned long offset = 0;
 375
 376	if (vma->vm_pgoff < start)
 377		offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
 378
 379	return vma->vm_start + offset;
 380}
 381
 382static unsigned long vma_offset_end(struct vm_area_struct *vma, pgoff_t end)
 383{
 384	unsigned long t_end;
 385
 386	if (!end)
 387		return vma->vm_end;
 388
 389	t_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) + vma->vm_start;
 390	if (t_end > vma->vm_end)
 391		t_end = vma->vm_end;
 392	return t_end;
 393}
 394
 395/*
 396 * Called with hugetlb fault mutex held.  Therefore, no more mappings to
 397 * this folio can be created while executing the routine.
 398 */
 399static void hugetlb_unmap_file_folio(struct hstate *h,
 400					struct address_space *mapping,
 401					struct folio *folio, pgoff_t index)
 402{
 403	struct rb_root_cached *root = &mapping->i_mmap;
 404	struct hugetlb_vma_lock *vma_lock;
 405	unsigned long pfn = folio_pfn(folio);
 406	struct vm_area_struct *vma;
 407	unsigned long v_start;
 408	unsigned long v_end;
 409	pgoff_t start, end;
 410
 411	start = index * pages_per_huge_page(h);
 412	end = (index + 1) * pages_per_huge_page(h);
 413
 414	i_mmap_lock_write(mapping);
 415retry:
 416	vma_lock = NULL;
 417	vma_interval_tree_foreach(vma, root, start, end - 1) {
 418		v_start = vma_offset_start(vma, start);
 419		v_end = vma_offset_end(vma, end);
 420
 421		if (!hugetlb_vma_maps_pfn(vma, v_start, pfn))
 422			continue;
 423
 424		if (!hugetlb_vma_trylock_write(vma)) {
 425			vma_lock = vma->vm_private_data;
 426			/*
 427			 * If we can not get vma lock, we need to drop
 428			 * immap_sema and take locks in order.  First,
 429			 * take a ref on the vma_lock structure so that
 430			 * we can be guaranteed it will not go away when
 431			 * dropping immap_sema.
 432			 */
 433			kref_get(&vma_lock->refs);
 434			break;
 435		}
 436
 437		unmap_hugepage_range(vma, v_start, v_end, NULL,
 438				     ZAP_FLAG_DROP_MARKER);
 439		hugetlb_vma_unlock_write(vma);
 440	}
 441
 442	i_mmap_unlock_write(mapping);
 443
 444	if (vma_lock) {
 445		/*
 446		 * Wait on vma_lock.  We know it is still valid as we have
 447		 * a reference.  We must 'open code' vma locking as we do
 448		 * not know if vma_lock is still attached to vma.
 449		 */
 450		down_write(&vma_lock->rw_sema);
 451		i_mmap_lock_write(mapping);
 452
 453		vma = vma_lock->vma;
 454		if (!vma) {
 455			/*
 456			 * If lock is no longer attached to vma, then just
 457			 * unlock, drop our reference and retry looking for
 458			 * other vmas.
 459			 */
 460			up_write(&vma_lock->rw_sema);
 461			kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
 462			goto retry;
 463		}
 464
 465		/*
 466		 * vma_lock is still attached to vma.  Check to see if vma
 467		 * still maps page and if so, unmap.
 468		 */
 469		v_start = vma_offset_start(vma, start);
 470		v_end = vma_offset_end(vma, end);
 471		if (hugetlb_vma_maps_pfn(vma, v_start, pfn))
 472			unmap_hugepage_range(vma, v_start, v_end, NULL,
 473					     ZAP_FLAG_DROP_MARKER);
 474
 475		kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
 476		hugetlb_vma_unlock_write(vma);
 477
 478		goto retry;
 479	}
 480}
 481
 482static void
 483hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
 484		      zap_flags_t zap_flags)
 485{
 486	struct vm_area_struct *vma;
 487
 488	/*
 489	 * end == 0 indicates that the entire range after start should be
 490	 * unmapped.  Note, end is exclusive, whereas the interval tree takes
 491	 * an inclusive "last".
 492	 */
 493	vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
 494		unsigned long v_start;
 495		unsigned long v_end;
 496
 497		if (!hugetlb_vma_trylock_write(vma))
 498			continue;
 499
 500		v_start = vma_offset_start(vma, start);
 501		v_end = vma_offset_end(vma, end);
 502
 503		unmap_hugepage_range(vma, v_start, v_end, NULL, zap_flags);
 504
 505		/*
 506		 * Note that vma lock only exists for shared/non-private
 507		 * vmas.  Therefore, lock is not held when calling
 508		 * unmap_hugepage_range for private vmas.
 509		 */
 510		hugetlb_vma_unlock_write(vma);
 511	}
 512}
 513
 514/*
 515 * Called with hugetlb fault mutex held.
 516 * Returns true if page was actually removed, false otherwise.
 517 */
 518static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
 519					struct address_space *mapping,
 520					struct folio *folio, pgoff_t index,
 521					bool truncate_op)
 522{
 523	bool ret = false;
 524
 525	/*
 526	 * If folio is mapped, it was faulted in after being
 527	 * unmapped in caller or hugetlb_vmdelete_list() skips
 528	 * unmapping it due to fail to grab lock.  Unmap (again)
 529	 * while holding the fault mutex.  The mutex will prevent
 530	 * faults until we finish removing the folio.  Hold folio
 531	 * lock to guarantee no concurrent migration.
 532	 */
 533	folio_lock(folio);
 534	if (unlikely(folio_mapped(folio)))
 535		hugetlb_unmap_file_folio(h, mapping, folio, index);
 536
 537	/*
 538	 * We must remove the folio from page cache before removing
 539	 * the region/ reserve map (hugetlb_unreserve_pages).  In
 540	 * rare out of memory conditions, removal of the region/reserve
 541	 * map could fail.  Correspondingly, the subpool and global
 542	 * reserve usage count can need to be adjusted.
 543	 */
 544	VM_BUG_ON_FOLIO(folio_test_hugetlb_restore_reserve(folio), folio);
 545	hugetlb_delete_from_page_cache(folio);
 546	ret = true;
 547	if (!truncate_op) {
 548		if (unlikely(hugetlb_unreserve_pages(inode, index,
 549							index + 1, 1)))
 550			hugetlb_fix_reserve_counts(inode);
 551	}
 552
 553	folio_unlock(folio);
 554	return ret;
 555}
 556
 557/*
 558 * remove_inode_hugepages handles two distinct cases: truncation and hole
 559 * punch.  There are subtle differences in operation for each case.
 560 *
 561 * truncation is indicated by end of range being LLONG_MAX
 562 *	In this case, we first scan the range and release found pages.
 563 *	After releasing pages, hugetlb_unreserve_pages cleans up region/reserve
 564 *	maps and global counts.  Page faults can race with truncation.
 565 *	During faults, hugetlb_no_page() checks i_size before page allocation,
 566 *	and again after obtaining page table lock.  It will 'back out'
 567 *	allocations in the truncated range.
 568 * hole punch is indicated if end is not LLONG_MAX
 569 *	In the hole punch case we scan the range and release found pages.
 570 *	Only when releasing a page is the associated region/reserve map
 571 *	deleted.  The region/reserve map for ranges without associated
 572 *	pages are not modified.  Page faults can race with hole punch.
 573 *	This is indicated if we find a mapped page.
 574 * Note: If the passed end of range value is beyond the end of file, but
 575 * not LLONG_MAX this routine still performs a hole punch operation.
 576 */
 577static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
 578				   loff_t lend)
 579{
 580	struct hstate *h = hstate_inode(inode);
 581	struct address_space *mapping = &inode->i_data;
 582	const pgoff_t end = lend >> PAGE_SHIFT;
 583	struct folio_batch fbatch;
 584	pgoff_t next, index;
 585	int i, freed = 0;
 586	bool truncate_op = (lend == LLONG_MAX);
 587
 588	folio_batch_init(&fbatch);
 589	next = lstart >> PAGE_SHIFT;
 590	while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) {
 591		for (i = 0; i < folio_batch_count(&fbatch); ++i) {
 592			struct folio *folio = fbatch.folios[i];
 593			u32 hash = 0;
 594
 595			index = folio->index >> huge_page_order(h);
 596			hash = hugetlb_fault_mutex_hash(mapping, index);
 597			mutex_lock(&hugetlb_fault_mutex_table[hash]);
 598
 599			/*
 600			 * Remove folio that was part of folio_batch.
 601			 */
 602			if (remove_inode_single_folio(h, inode, mapping, folio,
 603							index, truncate_op))
 604				freed++;
 605
 606			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 607		}
 608		folio_batch_release(&fbatch);
 609		cond_resched();
 610	}
 611
 612	if (truncate_op)
 613		(void)hugetlb_unreserve_pages(inode,
 614				lstart >> huge_page_shift(h),
 615				LONG_MAX, freed);
 616}
 617
 618static void hugetlbfs_evict_inode(struct inode *inode)
 619{
 620	struct resv_map *resv_map;
 621
 622	trace_hugetlbfs_evict_inode(inode);
 623	remove_inode_hugepages(inode, 0, LLONG_MAX);
 624
 625	/*
 626	 * Get the resv_map from the address space embedded in the inode.
 627	 * This is the address space which points to any resv_map allocated
 628	 * at inode creation time.  If this is a device special inode,
 629	 * i_mapping may not point to the original address space.
 630	 */
 631	resv_map = (struct resv_map *)(&inode->i_data)->i_private_data;
 632	/* Only regular and link inodes have associated reserve maps */
 633	if (resv_map)
 634		resv_map_release(&resv_map->refs);
 635	clear_inode(inode);
 636}
 637
 638static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
 639{
 640	pgoff_t pgoff;
 641	struct address_space *mapping = inode->i_mapping;
 642	struct hstate *h = hstate_inode(inode);
 643
 644	BUG_ON(offset & ~huge_page_mask(h));
 645	pgoff = offset >> PAGE_SHIFT;
 646
 647	i_size_write(inode, offset);
 648	i_mmap_lock_write(mapping);
 649	if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
 650		hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
 651				      ZAP_FLAG_DROP_MARKER);
 652	i_mmap_unlock_write(mapping);
 653	remove_inode_hugepages(inode, offset, LLONG_MAX);
 654}
 655
 656static void hugetlbfs_zero_partial_page(struct hstate *h,
 657					struct address_space *mapping,
 658					loff_t start,
 659					loff_t end)
 660{
 661	pgoff_t idx = start >> huge_page_shift(h);
 662	struct folio *folio;
 663
 664	folio = filemap_lock_hugetlb_folio(h, mapping, idx);
 665	if (IS_ERR(folio))
 666		return;
 667
 668	start = start & ~huge_page_mask(h);
 669	end = end & ~huge_page_mask(h);
 670	if (!end)
 671		end = huge_page_size(h);
 672
 673	folio_zero_segment(folio, (size_t)start, (size_t)end);
 674
 675	folio_unlock(folio);
 676	folio_put(folio);
 677}
 678
 679static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
 680{
 681	struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
 682	struct address_space *mapping = inode->i_mapping;
 683	struct hstate *h = hstate_inode(inode);
 684	loff_t hpage_size = huge_page_size(h);
 685	loff_t hole_start, hole_end;
 686
 687	/*
 688	 * hole_start and hole_end indicate the full pages within the hole.
 689	 */
 690	hole_start = round_up(offset, hpage_size);
 691	hole_end = round_down(offset + len, hpage_size);
 692
 693	inode_lock(inode);
 694
 695	/* protected by i_rwsem */
 696	if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
 697		inode_unlock(inode);
 698		return -EPERM;
 699	}
 700
 701	i_mmap_lock_write(mapping);
 702
 703	/* If range starts before first full page, zero partial page. */
 704	if (offset < hole_start)
 705		hugetlbfs_zero_partial_page(h, mapping,
 706				offset, min(offset + len, hole_start));
 707
 708	/* Unmap users of full pages in the hole. */
 709	if (hole_end > hole_start) {
 710		if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
 711			hugetlb_vmdelete_list(&mapping->i_mmap,
 712					      hole_start >> PAGE_SHIFT,
 713					      hole_end >> PAGE_SHIFT, 0);
 714	}
 715
 716	/* If range extends beyond last full page, zero partial page. */
 717	if ((offset + len) > hole_end && (offset + len) > hole_start)
 718		hugetlbfs_zero_partial_page(h, mapping,
 719				hole_end, offset + len);
 720
 721	i_mmap_unlock_write(mapping);
 722
 723	/* Remove full pages from the file. */
 724	if (hole_end > hole_start)
 725		remove_inode_hugepages(inode, hole_start, hole_end);
 726
 727	inode_unlock(inode);
 728
 729	return 0;
 730}
 731
 732static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
 733				loff_t len)
 734{
 735	struct inode *inode = file_inode(file);
 736	struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
 737	struct address_space *mapping = inode->i_mapping;
 738	struct hstate *h = hstate_inode(inode);
 739	struct vm_area_struct pseudo_vma;
 740	struct mm_struct *mm = current->mm;
 741	loff_t hpage_size = huge_page_size(h);
 742	unsigned long hpage_shift = huge_page_shift(h);
 743	pgoff_t start, index, end;
 744	int error;
 745	u32 hash;
 746
 747	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
 748		return -EOPNOTSUPP;
 749
 750	if (mode & FALLOC_FL_PUNCH_HOLE) {
 751		error = hugetlbfs_punch_hole(inode, offset, len);
 752		goto out_nolock;
 753	}
 754
 755	/*
 756	 * Default preallocate case.
 757	 * For this range, start is rounded down and end is rounded up
 758	 * as well as being converted to page offsets.
 759	 */
 760	start = offset >> hpage_shift;
 761	end = (offset + len + hpage_size - 1) >> hpage_shift;
 762
 763	inode_lock(inode);
 764
 765	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
 766	error = inode_newsize_ok(inode, offset + len);
 767	if (error)
 768		goto out;
 769
 770	if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
 771		error = -EPERM;
 772		goto out;
 773	}
 774
 775	/*
 776	 * Initialize a pseudo vma as this is required by the huge page
 777	 * allocation routines.
 778	 */
 779	vma_init(&pseudo_vma, mm);
 780	vm_flags_init(&pseudo_vma, VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
 781	pseudo_vma.vm_file = file;
 782
 783	for (index = start; index < end; index++) {
 784		/*
 785		 * This is supposed to be the vaddr where the page is being
 786		 * faulted in, but we have no vaddr here.
 787		 */
 788		struct folio *folio;
 789		unsigned long addr;
 790
 791		cond_resched();
 792
 793		/*
 794		 * fallocate(2) manpage permits EINTR; we may have been
 795		 * interrupted because we are using up too much memory.
 796		 */
 797		if (signal_pending(current)) {
 798			error = -EINTR;
 799			break;
 800		}
 801
 802		/* addr is the offset within the file (zero based) */
 803		addr = index * hpage_size;
 804
 805		/* mutex taken here, fault path and hole punch */
 806		hash = hugetlb_fault_mutex_hash(mapping, index);
 807		mutex_lock(&hugetlb_fault_mutex_table[hash]);
 808
 809		/* See if already present in mapping to avoid alloc/free */
 810		folio = filemap_get_folio(mapping, index << huge_page_order(h));
 811		if (!IS_ERR(folio)) {
 812			folio_put(folio);
 813			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 814			continue;
 815		}
 816
 817		/*
 818		 * Allocate folio without setting the avoid_reserve argument.
 819		 * There certainly are no reserves associated with the
 820		 * pseudo_vma.  However, there could be shared mappings with
 821		 * reserves for the file at the inode level.  If we fallocate
 822		 * folios in these areas, we need to consume the reserves
 823		 * to keep reservation accounting consistent.
 824		 */
 825		folio = alloc_hugetlb_folio(&pseudo_vma, addr, false);
 826		if (IS_ERR(folio)) {
 827			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 828			error = PTR_ERR(folio);
 829			goto out;
 830		}
 831		folio_zero_user(folio, addr);
 832		__folio_mark_uptodate(folio);
 833		error = hugetlb_add_to_page_cache(folio, mapping, index);
 834		if (unlikely(error)) {
 835			restore_reserve_on_error(h, &pseudo_vma, addr, folio);
 836			folio_put(folio);
 837			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 838			goto out;
 839		}
 840
 841		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 842
 843		folio_set_hugetlb_migratable(folio);
 844		/*
 845		 * folio_unlock because locked by hugetlb_add_to_page_cache()
 846		 * folio_put() due to reference from alloc_hugetlb_folio()
 847		 */
 848		folio_unlock(folio);
 849		folio_put(folio);
 850	}
 851
 852	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
 853		i_size_write(inode, offset + len);
 854	inode_set_ctime_current(inode);
 855out:
 856	inode_unlock(inode);
 857
 858out_nolock:
 859	trace_hugetlbfs_fallocate(inode, mode, offset, len, error);
 860	return error;
 861}
 862
 863static int hugetlbfs_setattr(struct mnt_idmap *idmap,
 864			     struct dentry *dentry, struct iattr *attr)
 865{
 866	struct inode *inode = d_inode(dentry);
 867	struct hstate *h = hstate_inode(inode);
 868	int error;
 869	unsigned int ia_valid = attr->ia_valid;
 870	struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
 871
 872	error = setattr_prepare(idmap, dentry, attr);
 873	if (error)
 874		return error;
 875
 876	trace_hugetlbfs_setattr(inode, dentry, attr);
 877
 878	if (ia_valid & ATTR_SIZE) {
 879		loff_t oldsize = inode->i_size;
 880		loff_t newsize = attr->ia_size;
 881
 882		if (newsize & ~huge_page_mask(h))
 883			return -EINVAL;
 884		/* protected by i_rwsem */
 885		if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
 886		    (newsize > oldsize && (info->seals & F_SEAL_GROW)))
 887			return -EPERM;
 888		hugetlb_vmtruncate(inode, newsize);
 889	}
 890
 891	setattr_copy(idmap, inode, attr);
 892	mark_inode_dirty(inode);
 893	return 0;
 894}
 895
 896static struct inode *hugetlbfs_get_root(struct super_block *sb,
 897					struct hugetlbfs_fs_context *ctx)
 898{
 899	struct inode *inode;
 900
 901	inode = new_inode(sb);
 902	if (inode) {
 903		inode->i_ino = get_next_ino();
 904		inode->i_mode = S_IFDIR | ctx->mode;
 905		inode->i_uid = ctx->uid;
 906		inode->i_gid = ctx->gid;
 907		simple_inode_init_ts(inode);
 908		inode->i_op = &hugetlbfs_dir_inode_operations;
 909		inode->i_fop = &simple_dir_operations;
 910		/* directory inodes start off with i_nlink == 2 (for "." entry) */
 911		inc_nlink(inode);
 912		lockdep_annotate_inode_mutex_key(inode);
 913	}
 914	return inode;
 915}
 916
 917/*
 918 * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
 919 * be taken from reclaim -- unlike regular filesystems. This needs an
 920 * annotation because huge_pmd_share() does an allocation under hugetlb's
 921 * i_mmap_rwsem.
 922 */
 923static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
 924
 925static struct inode *hugetlbfs_get_inode(struct super_block *sb,
 926					struct mnt_idmap *idmap,
 927					struct inode *dir,
 928					umode_t mode, dev_t dev)
 929{
 930	struct inode *inode;
 931	struct resv_map *resv_map = NULL;
 932
 933	/*
 934	 * Reserve maps are only needed for inodes that can have associated
 935	 * page allocations.
 936	 */
 937	if (S_ISREG(mode) || S_ISLNK(mode)) {
 938		resv_map = resv_map_alloc();
 939		if (!resv_map)
 940			return NULL;
 941	}
 942
 943	inode = new_inode(sb);
 944	if (inode) {
 945		struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
 946
 947		inode->i_ino = get_next_ino();
 948		inode_init_owner(idmap, inode, dir, mode);
 949		lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
 950				&hugetlbfs_i_mmap_rwsem_key);
 951		inode->i_mapping->a_ops = &hugetlbfs_aops;
 952		simple_inode_init_ts(inode);
 953		inode->i_mapping->i_private_data = resv_map;
 954		info->seals = F_SEAL_SEAL;
 955		switch (mode & S_IFMT) {
 956		default:
 957			init_special_inode(inode, mode, dev);
 958			break;
 959		case S_IFREG:
 960			inode->i_op = &hugetlbfs_inode_operations;
 961			inode->i_fop = &hugetlbfs_file_operations;
 962			break;
 963		case S_IFDIR:
 964			inode->i_op = &hugetlbfs_dir_inode_operations;
 965			inode->i_fop = &simple_dir_operations;
 966
 967			/* directory inodes start off with i_nlink == 2 (for "." entry) */
 968			inc_nlink(inode);
 969			break;
 970		case S_IFLNK:
 971			inode->i_op = &page_symlink_inode_operations;
 972			inode_nohighmem(inode);
 973			break;
 974		}
 975		lockdep_annotate_inode_mutex_key(inode);
 976		trace_hugetlbfs_alloc_inode(inode, dir, mode);
 977	} else {
 978		if (resv_map)
 979			kref_put(&resv_map->refs, resv_map_release);
 980	}
 981
 982	return inode;
 983}
 984
 985/*
 986 * File creation. Allocate an inode, and we're done..
 987 */
 988static int hugetlbfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 989			   struct dentry *dentry, umode_t mode, dev_t dev)
 990{
 991	struct inode *inode;
 992
 993	inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode, dev);
 994	if (!inode)
 995		return -ENOSPC;
 996	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
 997	d_make_persistent(dentry, inode);
 998	return 0;
 999}
1000
1001static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
1002				      struct dentry *dentry, umode_t mode)
1003{
1004	int retval = hugetlbfs_mknod(idmap, dir, dentry,
1005				     mode | S_IFDIR, 0);
1006	if (!retval)
1007		inc_nlink(dir);
1008	return ERR_PTR(retval);
1009}
1010
1011static int hugetlbfs_create(struct mnt_idmap *idmap,
1012			    struct inode *dir, struct dentry *dentry,
1013			    umode_t mode, bool excl)
1014{
1015	return hugetlbfs_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
1016}
1017
1018static int hugetlbfs_tmpfile(struct mnt_idmap *idmap,
1019			     struct inode *dir, struct file *file,
1020			     umode_t mode)
1021{
1022	struct inode *inode;
1023
1024	inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode | S_IFREG, 0);
1025	if (!inode)
1026		return -ENOSPC;
1027	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
1028	d_tmpfile(file, inode);
1029	return finish_open_simple(file, 0);
1030}
1031
1032static int hugetlbfs_symlink(struct mnt_idmap *idmap,
1033			     struct inode *dir, struct dentry *dentry,
1034			     const char *symname)
1035{
1036	const umode_t mode = S_IFLNK|S_IRWXUGO;
1037	struct inode *inode;
1038	int error = -ENOSPC;
1039
1040	inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode, 0);
1041	if (inode) {
1042		int l = strlen(symname)+1;
1043		error = page_symlink(inode, symname, l);
1044		if (!error)
1045			d_make_persistent(dentry, inode);
1046		else
1047			iput(inode);
1048	}
1049	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
1050
1051	return error;
1052}
1053
1054#ifdef CONFIG_MIGRATION
1055static int hugetlbfs_migrate_folio(struct address_space *mapping,
1056				struct folio *dst, struct folio *src,
1057				enum migrate_mode mode)
1058{
1059	int rc;
1060
1061	rc = migrate_huge_page_move_mapping(mapping, dst, src);
1062	if (rc)
1063		return rc;
1064
1065	if (hugetlb_folio_subpool(src)) {
1066		hugetlb_set_folio_subpool(dst,
1067					hugetlb_folio_subpool(src));
1068		hugetlb_set_folio_subpool(src, NULL);
1069	}
1070
1071	folio_migrate_flags(dst, src);
1072
1073	return 0;
1074}
1075#else
1076#define hugetlbfs_migrate_folio NULL
1077#endif
1078
1079static int hugetlbfs_error_remove_folio(struct address_space *mapping,
1080				struct folio *folio)
1081{
1082	return 0;
1083}
1084
1085/*
1086 * Display the mount options in /proc/mounts.
1087 */
1088static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
1089{
1090	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
1091	struct hugepage_subpool *spool = sbinfo->spool;
1092	unsigned long hpage_size = huge_page_size(sbinfo->hstate);
1093	unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
1094	char mod;
1095
1096	if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
1097		seq_printf(m, ",uid=%u",
1098			   from_kuid_munged(&init_user_ns, sbinfo->uid));
1099	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
1100		seq_printf(m, ",gid=%u",
1101			   from_kgid_munged(&init_user_ns, sbinfo->gid));
1102	if (sbinfo->mode != 0755)
1103		seq_printf(m, ",mode=%o", sbinfo->mode);
1104	if (sbinfo->max_inodes != -1)
1105		seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
1106
1107	hpage_size /= 1024;
1108	mod = 'K';
1109	if (hpage_size >= 1024) {
1110		hpage_size /= 1024;
1111		mod = 'M';
1112	}
1113	seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
1114	if (spool) {
1115		if (spool->max_hpages != -1)
1116			seq_printf(m, ",size=%llu",
1117				   (unsigned long long)spool->max_hpages << hpage_shift);
1118		if (spool->min_hpages != -1)
1119			seq_printf(m, ",min_size=%llu",
1120				   (unsigned long long)spool->min_hpages << hpage_shift);
1121	}
1122	return 0;
1123}
1124
1125static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
1126{
1127	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
1128	struct hstate *h = hstate_inode(d_inode(dentry));
1129	u64 id = huge_encode_dev(dentry->d_sb->s_dev);
1130
1131	buf->f_fsid = u64_to_fsid(id);
1132	buf->f_type = HUGETLBFS_MAGIC;
1133	buf->f_bsize = huge_page_size(h);
1134	if (sbinfo) {
1135		spin_lock(&sbinfo->stat_lock);
1136		/* If no limits set, just report 0 or -1 for max/free/used
1137		 * blocks, like simple_statfs() */
1138		if (sbinfo->spool) {
1139			long free_pages;
1140
1141			spin_lock_irq(&sbinfo->spool->lock);
1142			buf->f_blocks = sbinfo->spool->max_hpages;
1143			free_pages = sbinfo->spool->max_hpages
1144				- sbinfo->spool->used_hpages;
1145			buf->f_bavail = buf->f_bfree = free_pages;
1146			spin_unlock_irq(&sbinfo->spool->lock);
1147			buf->f_files = sbinfo->max_inodes;
1148			buf->f_ffree = sbinfo->free_inodes;
1149		}
1150		spin_unlock(&sbinfo->stat_lock);
1151	}
1152	buf->f_namelen = NAME_MAX;
1153	return 0;
1154}
1155
1156static void hugetlbfs_put_super(struct super_block *sb)
1157{
1158	struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
1159
1160	if (sbi) {
1161		sb->s_fs_info = NULL;
1162
1163		if (sbi->spool)
1164			hugepage_put_subpool(sbi->spool);
1165
1166		kfree(sbi);
1167	}
1168}
1169
1170static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
1171{
1172	if (sbinfo->free_inodes >= 0) {
1173		spin_lock(&sbinfo->stat_lock);
1174		if (unlikely(!sbinfo->free_inodes)) {
1175			spin_unlock(&sbinfo->stat_lock);
1176			return 0;
1177		}
1178		sbinfo->free_inodes--;
1179		spin_unlock(&sbinfo->stat_lock);
1180	}
1181
1182	return 1;
1183}
1184
1185static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
1186{
1187	if (sbinfo->free_inodes >= 0) {
1188		spin_lock(&sbinfo->stat_lock);
1189		sbinfo->free_inodes++;
1190		spin_unlock(&sbinfo->stat_lock);
1191	}
1192}
1193
1194
1195static struct kmem_cache *hugetlbfs_inode_cachep;
1196
1197static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
1198{
1199	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
1200	struct hugetlbfs_inode_info *p;
1201
1202	if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
1203		return NULL;
1204	p = alloc_inode_sb(sb, hugetlbfs_inode_cachep, GFP_KERNEL);
1205	if (unlikely(!p)) {
1206		hugetlbfs_inc_free_inodes(sbinfo);
1207		return NULL;
1208	}
1209	return &p->vfs_inode;
1210}
1211
1212static void hugetlbfs_free_inode(struct inode *inode)
1213{
1214	trace_hugetlbfs_free_inode(inode);
1215	kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
1216}
1217
1218static void hugetlbfs_destroy_inode(struct inode *inode)
1219{
1220	hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
1221}
1222
1223static const struct address_space_operations hugetlbfs_aops = {
1224	.write_begin	= hugetlbfs_write_begin,
1225	.write_end	= hugetlbfs_write_end,
1226	.dirty_folio	= noop_dirty_folio,
1227	.migrate_folio  = hugetlbfs_migrate_folio,
1228	.error_remove_folio	= hugetlbfs_error_remove_folio,
1229};
1230
1231
1232static void init_once(void *foo)
1233{
1234	struct hugetlbfs_inode_info *ei = foo;
1235
1236	inode_init_once(&ei->vfs_inode);
1237}
1238
1239static const struct file_operations hugetlbfs_file_operations = {
1240	.read_iter		= hugetlbfs_read_iter,
1241	.mmap_prepare		= hugetlbfs_file_mmap_prepare,
1242	.fsync			= noop_fsync,
1243	.get_unmapped_area	= hugetlb_get_unmapped_area,
1244	.llseek			= default_llseek,
1245	.fallocate		= hugetlbfs_fallocate,
1246	.fop_flags		= FOP_HUGE_PAGES,
1247};
1248
1249static const struct inode_operations hugetlbfs_dir_inode_operations = {
1250	.create		= hugetlbfs_create,
1251	.lookup		= simple_lookup,
1252	.link		= simple_link,
1253	.unlink		= simple_unlink,
1254	.symlink	= hugetlbfs_symlink,
1255	.mkdir		= hugetlbfs_mkdir,
1256	.rmdir		= simple_rmdir,
1257	.mknod		= hugetlbfs_mknod,
1258	.rename		= simple_rename,
1259	.setattr	= hugetlbfs_setattr,
1260	.tmpfile	= hugetlbfs_tmpfile,
1261};
1262
1263static const struct inode_operations hugetlbfs_inode_operations = {
1264	.setattr	= hugetlbfs_setattr,
1265};
1266
1267static const struct super_operations hugetlbfs_ops = {
1268	.alloc_inode    = hugetlbfs_alloc_inode,
1269	.free_inode     = hugetlbfs_free_inode,
1270	.destroy_inode  = hugetlbfs_destroy_inode,
1271	.evict_inode	= hugetlbfs_evict_inode,
1272	.statfs		= hugetlbfs_statfs,
1273	.put_super	= hugetlbfs_put_super,
1274	.show_options	= hugetlbfs_show_options,
1275};
1276
1277/*
1278 * Convert size option passed from command line to number of huge pages
1279 * in the pool specified by hstate.  Size option could be in bytes
1280 * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
1281 */
1282static long
1283hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
1284			 enum hugetlbfs_size_type val_type)
1285{
1286	if (val_type == NO_SIZE)
1287		return -1;
1288
1289	if (val_type == SIZE_PERCENT) {
1290		size_opt <<= huge_page_shift(h);
1291		size_opt *= h->max_huge_pages;
1292		do_div(size_opt, 100);
1293	}
1294
1295	size_opt >>= huge_page_shift(h);
1296	return size_opt;
1297}
1298
1299/*
1300 * Parse one mount parameter.
1301 */
1302static int hugetlbfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
1303{
1304	struct hugetlbfs_fs_context *ctx = fc->fs_private;
1305	struct fs_parse_result result;
1306	struct hstate *h;
1307	char *rest;
1308	unsigned long ps;
1309	int opt;
1310
1311	opt = fs_parse(fc, hugetlb_fs_parameters, param, &result);
1312	if (opt < 0)
1313		return opt;
1314
1315	switch (opt) {
1316	case Opt_uid:
1317		ctx->uid = result.uid;
1318		return 0;
1319
1320	case Opt_gid:
1321		ctx->gid = result.gid;
1322		return 0;
1323
1324	case Opt_mode:
1325		ctx->mode = result.uint_32 & 01777U;
1326		return 0;
1327
1328	case Opt_size:
1329		/* memparse() will accept a K/M/G without a digit */
1330		if (!param->string || !isdigit(param->string[0]))
1331			goto bad_val;
1332		ctx->max_size_opt = memparse(param->string, &rest);
1333		ctx->max_val_type = SIZE_STD;
1334		if (*rest == '%')
1335			ctx->max_val_type = SIZE_PERCENT;
1336		return 0;
1337
1338	case Opt_nr_inodes:
1339		/* memparse() will accept a K/M/G without a digit */
1340		if (!param->string || !isdigit(param->string[0]))
1341			goto bad_val;
1342		ctx->nr_inodes = memparse(param->string, &rest);
1343		return 0;
1344
1345	case Opt_pagesize:
1346		ps = memparse(param->string, &rest);
1347		h = size_to_hstate(ps);
1348		if (!h) {
1349			pr_err("Unsupported page size %lu MB\n", ps / SZ_1M);
1350			return -EINVAL;
1351		}
1352		ctx->hstate = h;
1353		return 0;
1354
1355	case Opt_min_size:
1356		/* memparse() will accept a K/M/G without a digit */
1357		if (!param->string || !isdigit(param->string[0]))
1358			goto bad_val;
1359		ctx->min_size_opt = memparse(param->string, &rest);
1360		ctx->min_val_type = SIZE_STD;
1361		if (*rest == '%')
1362			ctx->min_val_type = SIZE_PERCENT;
1363		return 0;
1364
1365	default:
1366		return -EINVAL;
1367	}
1368
1369bad_val:
1370	return invalfc(fc, "Bad value '%s' for mount option '%s'\n",
1371		      param->string, param->key);
1372}
1373
1374/*
1375 * Validate the parsed options.
1376 */
1377static int hugetlbfs_validate(struct fs_context *fc)
1378{
1379	struct hugetlbfs_fs_context *ctx = fc->fs_private;
1380
1381	/*
1382	 * Use huge page pool size (in hstate) to convert the size
1383	 * options to number of huge pages.  If NO_SIZE, -1 is returned.
1384	 */
1385	ctx->max_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
1386						   ctx->max_size_opt,
1387						   ctx->max_val_type);
1388	ctx->min_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
1389						   ctx->min_size_opt,
1390						   ctx->min_val_type);
1391
1392	/*
1393	 * If max_size was specified, then min_size must be smaller
1394	 */
1395	if (ctx->max_val_type > NO_SIZE &&
1396	    ctx->min_hpages > ctx->max_hpages) {
1397		pr_err("Minimum size can not be greater than maximum size\n");
1398		return -EINVAL;
1399	}
1400
1401	return 0;
1402}
1403
1404static int
1405hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc)
1406{
1407	struct hugetlbfs_fs_context *ctx = fc->fs_private;
1408	struct hugetlbfs_sb_info *sbinfo;
1409
1410	sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
1411	if (!sbinfo)
1412		return -ENOMEM;
1413	sb->s_fs_info = sbinfo;
1414	spin_lock_init(&sbinfo->stat_lock);
1415	sbinfo->hstate		= ctx->hstate;
1416	sbinfo->max_inodes	= ctx->nr_inodes;
1417	sbinfo->free_inodes	= ctx->nr_inodes;
1418	sbinfo->spool		= NULL;
1419	sbinfo->uid		= ctx->uid;
1420	sbinfo->gid		= ctx->gid;
1421	sbinfo->mode		= ctx->mode;
1422
1423	/*
1424	 * Allocate and initialize subpool if maximum or minimum size is
1425	 * specified.  Any needed reservations (for minimum size) are taken
1426	 * when the subpool is created.
1427	 */
1428	if (ctx->max_hpages != -1 || ctx->min_hpages != -1) {
1429		sbinfo->spool = hugepage_new_subpool(ctx->hstate,
1430						     ctx->max_hpages,
1431						     ctx->min_hpages);
1432		if (!sbinfo->spool)
1433			goto out_free;
1434	}
1435	sb->s_maxbytes = MAX_LFS_FILESIZE;
1436	sb->s_blocksize = huge_page_size(ctx->hstate);
1437	sb->s_blocksize_bits = huge_page_shift(ctx->hstate);
1438	sb->s_magic = HUGETLBFS_MAGIC;
1439	sb->s_op = &hugetlbfs_ops;
1440	sb->s_d_flags = DCACHE_DONTCACHE;
1441	sb->s_time_gran = 1;
1442
1443	/*
1444	 * Due to the special and limited functionality of hugetlbfs, it does
1445	 * not work well as a stacking filesystem.
1446	 */
1447	sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH;
1448	sb->s_root = d_make_root(hugetlbfs_get_root(sb, ctx));
1449	if (!sb->s_root)
1450		goto out_free;
1451	return 0;
1452out_free:
1453	kfree(sbinfo->spool);
1454	kfree(sbinfo);
1455	return -ENOMEM;
1456}
1457
1458static int hugetlbfs_get_tree(struct fs_context *fc)
1459{
1460	int err = hugetlbfs_validate(fc);
1461	if (err)
1462		return err;
1463	return get_tree_nodev(fc, hugetlbfs_fill_super);
1464}
1465
1466static void hugetlbfs_fs_context_free(struct fs_context *fc)
1467{
1468	kfree(fc->fs_private);
1469}
1470
1471static const struct fs_context_operations hugetlbfs_fs_context_ops = {
1472	.free		= hugetlbfs_fs_context_free,
1473	.parse_param	= hugetlbfs_parse_param,
1474	.get_tree	= hugetlbfs_get_tree,
1475};
1476
1477static int hugetlbfs_init_fs_context(struct fs_context *fc)
1478{
1479	struct hugetlbfs_fs_context *ctx;
1480
1481	ctx = kzalloc(sizeof(struct hugetlbfs_fs_context), GFP_KERNEL);
1482	if (!ctx)
1483		return -ENOMEM;
1484
1485	ctx->max_hpages	= -1; /* No limit on size by default */
1486	ctx->nr_inodes	= -1; /* No limit on number of inodes by default */
1487	ctx->uid	= current_fsuid();
1488	ctx->gid	= current_fsgid();
1489	ctx->mode	= 0755;
1490	ctx->hstate	= &default_hstate;
1491	ctx->min_hpages	= -1; /* No default minimum size */
1492	ctx->max_val_type = NO_SIZE;
1493	ctx->min_val_type = NO_SIZE;
1494	fc->fs_private = ctx;
1495	fc->ops	= &hugetlbfs_fs_context_ops;
1496	return 0;
1497}
1498
1499static struct file_system_type hugetlbfs_fs_type = {
1500	.name			= "hugetlbfs",
1501	.init_fs_context	= hugetlbfs_init_fs_context,
1502	.parameters		= hugetlb_fs_parameters,
1503	.kill_sb		= kill_anon_super,
1504	.fs_flags               = FS_ALLOW_IDMAP,
1505};
1506
1507static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
1508
1509static int can_do_hugetlb_shm(void)
1510{
1511	kgid_t shm_group;
1512	shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
1513	return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
1514}
1515
1516static int get_hstate_idx(int page_size_log)
1517{
1518	struct hstate *h = hstate_sizelog(page_size_log);
1519
1520	if (!h)
1521		return -1;
1522	return hstate_index(h);
1523}
1524
1525/*
1526 * Note that size should be aligned to proper hugepage size in caller side,
1527 * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
1528 */
1529struct file *hugetlb_file_setup(const char *name, size_t size,
1530				vm_flags_t acctflag, int creat_flags,
1531				int page_size_log)
1532{
1533	struct inode *inode;
1534	struct vfsmount *mnt;
1535	int hstate_idx;
1536	struct file *file;
1537
1538	hstate_idx = get_hstate_idx(page_size_log);
1539	if (hstate_idx < 0)
1540		return ERR_PTR(-ENODEV);
1541
1542	mnt = hugetlbfs_vfsmount[hstate_idx];
1543	if (!mnt)
1544		return ERR_PTR(-ENOENT);
1545
1546	if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
1547		struct ucounts *ucounts = current_ucounts();
1548
1549		if (user_shm_lock(size, ucounts)) {
1550			pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is obsolete\n",
1551				current->comm, current->pid);
1552			user_shm_unlock(size, ucounts);
1553		}
1554		return ERR_PTR(-EPERM);
1555	}
1556
1557	file = ERR_PTR(-ENOSPC);
1558	/* hugetlbfs_vfsmount[] mounts do not use idmapped mounts.  */
1559	inode = hugetlbfs_get_inode(mnt->mnt_sb, &nop_mnt_idmap, NULL,
1560				    S_IFREG | S_IRWXUGO, 0);
1561	if (!inode)
1562		goto out;
1563	if (creat_flags == HUGETLB_SHMFS_INODE)
1564		inode->i_flags |= S_PRIVATE;
1565
1566	inode->i_size = size;
1567	clear_nlink(inode);
1568
1569	if (hugetlb_reserve_pages(inode, 0,
1570			size >> huge_page_shift(hstate_inode(inode)), NULL,
1571			acctflag) < 0)
1572		file = ERR_PTR(-ENOMEM);
1573	else
1574		file = alloc_file_pseudo(inode, mnt, name, O_RDWR,
1575					&hugetlbfs_file_operations);
1576	if (!IS_ERR(file))
1577		return file;
1578
1579	iput(inode);
1580out:
1581	return file;
1582}
1583
1584static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h)
1585{
1586	struct fs_context *fc;
1587	struct vfsmount *mnt;
1588
1589	fc = fs_context_for_mount(&hugetlbfs_fs_type, SB_KERNMOUNT);
1590	if (IS_ERR(fc)) {
1591		mnt = ERR_CAST(fc);
1592	} else {
1593		struct hugetlbfs_fs_context *ctx = fc->fs_private;
1594		ctx->hstate = h;
1595		mnt = fc_mount_longterm(fc);
1596		put_fs_context(fc);
1597	}
1598	if (IS_ERR(mnt))
1599		pr_err("Cannot mount internal hugetlbfs for page size %luK",
1600		       huge_page_size(h) / SZ_1K);
1601	return mnt;
1602}
1603
1604static int __init init_hugetlbfs_fs(void)
1605{
1606	struct vfsmount *mnt;
1607	struct hstate *h;
1608	int error;
1609	int i;
1610
1611	if (!hugepages_supported()) {
1612		pr_info("disabling because there are no supported hugepage sizes\n");
1613		return -ENOTSUPP;
1614	}
1615
1616	error = -ENOMEM;
1617	hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
1618					sizeof(struct hugetlbfs_inode_info),
1619					0, SLAB_ACCOUNT, init_once);
1620	if (hugetlbfs_inode_cachep == NULL)
1621		goto out;
1622
1623	error = register_filesystem(&hugetlbfs_fs_type);
1624	if (error)
1625		goto out_free;
1626
1627	/* default hstate mount is required */
1628	mnt = mount_one_hugetlbfs(&default_hstate);
1629	if (IS_ERR(mnt)) {
1630		error = PTR_ERR(mnt);
1631		goto out_unreg;
1632	}
1633	hugetlbfs_vfsmount[default_hstate_idx] = mnt;
1634
1635	/* other hstates are optional */
1636	i = 0;
1637	for_each_hstate(h) {
1638		if (i == default_hstate_idx) {
1639			i++;
1640			continue;
1641		}
1642
1643		mnt = mount_one_hugetlbfs(h);
1644		if (IS_ERR(mnt))
1645			hugetlbfs_vfsmount[i] = NULL;
1646		else
1647			hugetlbfs_vfsmount[i] = mnt;
1648		i++;
1649	}
1650
1651	return 0;
1652
1653 out_unreg:
1654	(void)unregister_filesystem(&hugetlbfs_fs_type);
1655 out_free:
1656	kmem_cache_destroy(hugetlbfs_inode_cachep);
1657 out:
1658	return error;
1659}
1660fs_initcall(init_hugetlbfs_fs)