dcache.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * fs/dcache.c
   4 *
   5 * Complete reimplementation
   6 * (C) 1997 Thomas Schoebel-Theuer,
   7 * with heavy changes by Linus Torvalds
   8 */
   9
  10/*
  11 * Notes on the allocation strategy:
  12 *
  13 * The dcache is a master of the icache - whenever a dcache entry
  14 * exists, the inode will always exist. "iput()" is done either when
  15 * the dcache entry is deleted or garbage collected.
  16 */
  17
  18#include <linux/ratelimit.h>
  19#include <linux/string.h>
  20#include <linux/mm.h>
  21#include <linux/fs.h>
  22#include <linux/fscrypt.h>
  23#include <linux/fsnotify.h>
  24#include <linux/slab.h>
  25#include <linux/init.h>
  26#include <linux/hash.h>
  27#include <linux/cache.h>
  28#include <linux/export.h>
  29#include <linux/security.h>
  30#include <linux/seqlock.h>
  31#include <linux/memblock.h>
  32#include <linux/bit_spinlock.h>
  33#include <linux/rculist_bl.h>
  34#include <linux/list_lru.h>
  35#include "internal.h"
  36#include "mount.h"
  37
  38#include <asm/runtime-const.h>
  39
  40/*
  41 * Usage:
  42 * dcache->d_inode->i_lock protects:
  43 *   - i_dentry, d_u.d_alias, d_inode of aliases
  44 * dcache_hash_bucket lock protects:
  45 *   - the dcache hash table
  46 * s_roots bl list spinlock protects:
  47 *   - the s_roots list (see __d_drop)
  48 * dentry->d_sb->s_dentry_lru_lock protects:
  49 *   - the dcache lru lists and counters
  50 * d_lock protects:
  51 *   - d_flags
  52 *   - d_name
  53 *   - d_lru
  54 *   - d_count
  55 *   - d_unhashed()
  56 *   - d_parent and d_chilren
  57 *   - childrens' d_sib and d_parent
  58 *   - d_u.d_alias, d_inode
  59 *
  60 * Ordering:
  61 * dentry->d_inode->i_lock
  62 *   dentry->d_lock
  63 *     dentry->d_sb->s_dentry_lru_lock
  64 *     dcache_hash_bucket lock
  65 *     s_roots lock
  66 *
  67 * If there is an ancestor relationship:
  68 * dentry->d_parent->...->d_parent->d_lock
  69 *   ...
  70 *     dentry->d_parent->d_lock
  71 *       dentry->d_lock
  72 *
  73 * If no ancestor relationship:
  74 * arbitrary, since it's serialized on rename_lock
  75 */
  76static int sysctl_vfs_cache_pressure __read_mostly = 100;
  77static int sysctl_vfs_cache_pressure_denom __read_mostly = 100;
  78
  79unsigned long vfs_pressure_ratio(unsigned long val)
  80{
  81	return mult_frac(val, sysctl_vfs_cache_pressure, sysctl_vfs_cache_pressure_denom);
  82}
  83EXPORT_SYMBOL_GPL(vfs_pressure_ratio);
  84
  85__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
  86
  87EXPORT_SYMBOL(rename_lock);
  88
  89static struct kmem_cache *__dentry_cache __ro_after_init;
  90#define dentry_cache runtime_const_ptr(__dentry_cache)
  91
  92const struct qstr empty_name = QSTR_INIT("", 0);
  93EXPORT_SYMBOL(empty_name);
  94const struct qstr slash_name = QSTR_INIT("/", 1);
  95EXPORT_SYMBOL(slash_name);
  96const struct qstr dotdot_name = QSTR_INIT("..", 2);
  97EXPORT_SYMBOL(dotdot_name);
  98
  99/*
 100 * This is the single most critical data structure when it comes
 101 * to the dcache: the hashtable for lookups. Somebody should try
 102 * to make this good - I've just made it work.
 103 *
 104 * This hash-function tries to avoid losing too many bits of hash
 105 * information, yet avoid using a prime hash-size or similar.
 106 *
 107 * Marking the variables "used" ensures that the compiler doesn't
 108 * optimize them away completely on architectures with runtime
 109 * constant infrastructure, this allows debuggers to see their
 110 * values. But updating these values has no effect on those arches.
 111 */
 112
 113static unsigned int d_hash_shift __ro_after_init __used;
 114
 115static struct hlist_bl_head *dentry_hashtable __ro_after_init __used;
 116
 117static inline struct hlist_bl_head *d_hash(unsigned long hashlen)
 118{
 119	return runtime_const_ptr(dentry_hashtable) +
 120		runtime_const_shift_right_32(hashlen, d_hash_shift);
 121}
 122
 123#define IN_LOOKUP_SHIFT 10
 124static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
 125
 126static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
 127					unsigned int hash)
 128{
 129	hash += (unsigned long) parent / L1_CACHE_BYTES;
 130	return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
 131}
 132
 133struct dentry_stat_t {
 134	long nr_dentry;
 135	long nr_unused;
 136	long age_limit;		/* age in seconds */
 137	long want_pages;	/* pages requested by system */
 138	long nr_negative;	/* # of unused negative dentries */
 139	long dummy;		/* Reserved for future use */
 140};
 141
 142static DEFINE_PER_CPU(long, nr_dentry);
 143static DEFINE_PER_CPU(long, nr_dentry_unused);
 144static DEFINE_PER_CPU(long, nr_dentry_negative);
 145static int dentry_negative_policy;
 146
 147#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
 148/* Statistics gathering. */
 149static struct dentry_stat_t dentry_stat = {
 150	.age_limit = 45,
 151};
 152
 153/*
 154 * Here we resort to our own counters instead of using generic per-cpu counters
 155 * for consistency with what the vfs inode code does. We are expected to harvest
 156 * better code and performance by having our own specialized counters.
 157 *
 158 * Please note that the loop is done over all possible CPUs, not over all online
 159 * CPUs. The reason for this is that we don't want to play games with CPUs going
 160 * on and off. If one of them goes off, we will just keep their counters.
 161 *
 162 * glommer: See cffbc8a for details, and if you ever intend to change this,
 163 * please update all vfs counters to match.
 164 */
 165static long get_nr_dentry(void)
 166{
 167	int i;
 168	long sum = 0;
 169	for_each_possible_cpu(i)
 170		sum += per_cpu(nr_dentry, i);
 171	return sum < 0 ? 0 : sum;
 172}
 173
 174static long get_nr_dentry_unused(void)
 175{
 176	int i;
 177	long sum = 0;
 178	for_each_possible_cpu(i)
 179		sum += per_cpu(nr_dentry_unused, i);
 180	return sum < 0 ? 0 : sum;
 181}
 182
 183static long get_nr_dentry_negative(void)
 184{
 185	int i;
 186	long sum = 0;
 187
 188	for_each_possible_cpu(i)
 189		sum += per_cpu(nr_dentry_negative, i);
 190	return sum < 0 ? 0 : sum;
 191}
 192
 193static int proc_nr_dentry(const struct ctl_table *table, int write, void *buffer,
 194			  size_t *lenp, loff_t *ppos)
 195{
 196	dentry_stat.nr_dentry = get_nr_dentry();
 197	dentry_stat.nr_unused = get_nr_dentry_unused();
 198	dentry_stat.nr_negative = get_nr_dentry_negative();
 199	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
 200}
 201
 202static const struct ctl_table fs_dcache_sysctls[] = {
 203	{
 204		.procname	= "dentry-state",
 205		.data		= &dentry_stat,
 206		.maxlen		= 6*sizeof(long),
 207		.mode		= 0444,
 208		.proc_handler	= proc_nr_dentry,
 209	},
 210	{
 211		.procname	= "dentry-negative",
 212		.data		= &dentry_negative_policy,
 213		.maxlen		= sizeof(dentry_negative_policy),
 214		.mode		= 0644,
 215		.proc_handler	= proc_dointvec_minmax,
 216		.extra1		= SYSCTL_ZERO,
 217		.extra2		= SYSCTL_ONE,
 218	},
 219};
 220
 221static const struct ctl_table vm_dcache_sysctls[] = {
 222	{
 223		.procname	= "vfs_cache_pressure",
 224		.data		= &sysctl_vfs_cache_pressure,
 225		.maxlen		= sizeof(sysctl_vfs_cache_pressure),
 226		.mode		= 0644,
 227		.proc_handler	= proc_dointvec_minmax,
 228		.extra1		= SYSCTL_ZERO,
 229	},
 230	{
 231		.procname	= "vfs_cache_pressure_denom",
 232		.data		= &sysctl_vfs_cache_pressure_denom,
 233		.maxlen		= sizeof(sysctl_vfs_cache_pressure_denom),
 234		.mode		= 0644,
 235		.proc_handler	= proc_dointvec_minmax,
 236		.extra1		= SYSCTL_ONE_HUNDRED,
 237	},
 238};
 239
 240static int __init init_fs_dcache_sysctls(void)
 241{
 242	register_sysctl_init("vm", vm_dcache_sysctls);
 243	register_sysctl_init("fs", fs_dcache_sysctls);
 244	return 0;
 245}
 246fs_initcall(init_fs_dcache_sysctls);
 247#endif
 248
 249/*
 250 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
 251 * The strings are both count bytes long, and count is non-zero.
 252 */
 253#ifdef CONFIG_DCACHE_WORD_ACCESS
 254
 255#include <asm/word-at-a-time.h>
 256/*
 257 * NOTE! 'cs' and 'scount' come from a dentry, so it has a
 258 * aligned allocation for this particular component. We don't
 259 * strictly need the load_unaligned_zeropad() safety, but it
 260 * doesn't hurt either.
 261 *
 262 * In contrast, 'ct' and 'tcount' can be from a pathname, and do
 263 * need the careful unaligned handling.
 264 */
 265static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
 266{
 267	unsigned long a,b,mask;
 268
 269	for (;;) {
 270		a = read_word_at_a_time(cs);
 271		b = load_unaligned_zeropad(ct);
 272		if (tcount < sizeof(unsigned long))
 273			break;
 274		if (unlikely(a != b))
 275			return 1;
 276		cs += sizeof(unsigned long);
 277		ct += sizeof(unsigned long);
 278		tcount -= sizeof(unsigned long);
 279		if (!tcount)
 280			return 0;
 281	}
 282	mask = bytemask_from_count(tcount);
 283	return unlikely(!!((a ^ b) & mask));
 284}
 285
 286#else
 287
 288static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
 289{
 290	do {
 291		if (*cs != *ct)
 292			return 1;
 293		cs++;
 294		ct++;
 295		tcount--;
 296	} while (tcount);
 297	return 0;
 298}
 299
 300#endif
 301
 302static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
 303{
 304	/*
 305	 * Be careful about RCU walk racing with rename:
 306	 * use 'READ_ONCE' to fetch the name pointer.
 307	 *
 308	 * NOTE! Even if a rename will mean that the length
 309	 * was not loaded atomically, we don't care. The
 310	 * RCU walk will check the sequence count eventually,
 311	 * and catch it. And we won't overrun the buffer,
 312	 * because we're reading the name pointer atomically,
 313	 * and a dentry name is guaranteed to be properly
 314	 * terminated with a NUL byte.
 315	 *
 316	 * End result: even if 'len' is wrong, we'll exit
 317	 * early because the data cannot match (there can
 318	 * be no NUL in the ct/tcount data)
 319	 */
 320	const unsigned char *cs = READ_ONCE(dentry->d_name.name);
 321
 322	return dentry_string_cmp(cs, ct, tcount);
 323}
 324
 325/*
 326 * long names are allocated separately from dentry and never modified.
 327 * Refcounted, freeing is RCU-delayed.  See take_dentry_name_snapshot()
 328 * for the reason why ->count and ->head can't be combined into a union.
 329 * dentry_string_cmp() relies upon ->name[] being word-aligned.
 330 */
 331struct external_name {
 332	atomic_t count;
 333	struct rcu_head head;
 334	unsigned char name[] __aligned(sizeof(unsigned long));
 335};
 336
 337static inline struct external_name *external_name(struct dentry *dentry)
 338{
 339	return container_of(dentry->d_name.name, struct external_name, name[0]);
 340}
 341
 342static void __d_free(struct rcu_head *head)
 343{
 344	struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
 345
 346	kmem_cache_free(dentry_cache, dentry); 
 347}
 348
 349static void __d_free_external(struct rcu_head *head)
 350{
 351	struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
 352	kfree(external_name(dentry));
 353	kmem_cache_free(dentry_cache, dentry);
 354}
 355
 356static inline int dname_external(const struct dentry *dentry)
 357{
 358	return dentry->d_name.name != dentry->d_shortname.string;
 359}
 360
 361void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
 362{
 363	unsigned seq;
 364	const unsigned char *s;
 365
 366	rcu_read_lock();
 367retry:
 368	seq = read_seqcount_begin(&dentry->d_seq);
 369	s = READ_ONCE(dentry->d_name.name);
 370	name->name.hash_len = dentry->d_name.hash_len;
 371	name->name.name = name->inline_name.string;
 372	if (likely(s == dentry->d_shortname.string)) {
 373		name->inline_name = dentry->d_shortname;
 374	} else {
 375		struct external_name *p;
 376		p = container_of(s, struct external_name, name[0]);
 377		// get a valid reference
 378		if (unlikely(!atomic_inc_not_zero(&p->count)))
 379			goto retry;
 380		name->name.name = s;
 381	}
 382	if (read_seqcount_retry(&dentry->d_seq, seq)) {
 383		release_dentry_name_snapshot(name);
 384		goto retry;
 385	}
 386	rcu_read_unlock();
 387}
 388EXPORT_SYMBOL(take_dentry_name_snapshot);
 389
 390void release_dentry_name_snapshot(struct name_snapshot *name)
 391{
 392	if (unlikely(name->name.name != name->inline_name.string)) {
 393		struct external_name *p;
 394		p = container_of(name->name.name, struct external_name, name[0]);
 395		if (unlikely(atomic_dec_and_test(&p->count)))
 396			kfree_rcu(p, head);
 397	}
 398}
 399EXPORT_SYMBOL(release_dentry_name_snapshot);
 400
 401static inline void __d_set_inode_and_type(struct dentry *dentry,
 402					  struct inode *inode,
 403					  unsigned type_flags)
 404{
 405	unsigned flags;
 406
 407	dentry->d_inode = inode;
 408	flags = READ_ONCE(dentry->d_flags);
 409	flags &= ~DCACHE_ENTRY_TYPE;
 410	flags |= type_flags;
 411	smp_store_release(&dentry->d_flags, flags);
 412}
 413
 414static inline void __d_clear_type_and_inode(struct dentry *dentry)
 415{
 416	unsigned flags = READ_ONCE(dentry->d_flags);
 417
 418	flags &= ~DCACHE_ENTRY_TYPE;
 419	WRITE_ONCE(dentry->d_flags, flags);
 420	dentry->d_inode = NULL;
 421	/*
 422	 * The negative counter only tracks dentries on the LRU. Don't inc if
 423	 * d_lru is on another list.
 424	 */
 425	if ((flags & (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST)
 426		this_cpu_inc(nr_dentry_negative);
 427}
 428
 429static void dentry_free(struct dentry *dentry)
 430{
 431	WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
 432	if (unlikely(dname_external(dentry))) {
 433		struct external_name *p = external_name(dentry);
 434		if (likely(atomic_dec_and_test(&p->count))) {
 435			call_rcu(&dentry->d_u.d_rcu, __d_free_external);
 436			return;
 437		}
 438	}
 439	/* if dentry was never visible to RCU, immediate free is OK */
 440	if (dentry->d_flags & DCACHE_NORCU)
 441		__d_free(&dentry->d_u.d_rcu);
 442	else
 443		call_rcu(&dentry->d_u.d_rcu, __d_free);
 444}
 445
 446/*
 447 * Release the dentry's inode, using the filesystem
 448 * d_iput() operation if defined.
 449 */
 450static void dentry_unlink_inode(struct dentry * dentry)
 451	__releases(dentry->d_lock)
 452	__releases(dentry->d_inode->i_lock)
 453{
 454	struct inode *inode = dentry->d_inode;
 455
 456	raw_write_seqcount_begin(&dentry->d_seq);
 457	__d_clear_type_and_inode(dentry);
 458	hlist_del_init(&dentry->d_u.d_alias);
 459	raw_write_seqcount_end(&dentry->d_seq);
 460	spin_unlock(&dentry->d_lock);
 461	spin_unlock(&inode->i_lock);
 462	if (!inode->i_nlink)
 463		fsnotify_inoderemove(inode);
 464	if (dentry->d_op && dentry->d_op->d_iput)
 465		dentry->d_op->d_iput(dentry, inode);
 466	else
 467		iput(inode);
 468}
 469
 470/*
 471 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
 472 * is in use - which includes both the "real" per-superblock
 473 * LRU list _and_ the DCACHE_SHRINK_LIST use.
 474 *
 475 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
 476 * on the shrink list (ie not on the superblock LRU list).
 477 *
 478 * The per-cpu "nr_dentry_unused" counters are updated with
 479 * the DCACHE_LRU_LIST bit.
 480 *
 481 * The per-cpu "nr_dentry_negative" counters are only updated
 482 * when deleted from or added to the per-superblock LRU list, not
 483 * from/to the shrink list. That is to avoid an unneeded dec/inc
 484 * pair when moving from LRU to shrink list in select_collect().
 485 *
 486 * These helper functions make sure we always follow the
 487 * rules. d_lock must be held by the caller.
 488 */
 489#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
 490static void d_lru_add(struct dentry *dentry)
 491{
 492	D_FLAG_VERIFY(dentry, 0);
 493	dentry->d_flags |= DCACHE_LRU_LIST;
 494	this_cpu_inc(nr_dentry_unused);
 495	if (d_is_negative(dentry))
 496		this_cpu_inc(nr_dentry_negative);
 497	WARN_ON_ONCE(!list_lru_add_obj(
 498			&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
 499}
 500
 501static void d_lru_del(struct dentry *dentry)
 502{
 503	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
 504	dentry->d_flags &= ~DCACHE_LRU_LIST;
 505	this_cpu_dec(nr_dentry_unused);
 506	if (d_is_negative(dentry))
 507		this_cpu_dec(nr_dentry_negative);
 508	WARN_ON_ONCE(!list_lru_del_obj(
 509			&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
 510}
 511
 512static void d_shrink_del(struct dentry *dentry)
 513{
 514	D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
 515	list_del_init(&dentry->d_lru);
 516	dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
 517	this_cpu_dec(nr_dentry_unused);
 518}
 519
 520static void d_shrink_add(struct dentry *dentry, struct list_head *list)
 521{
 522	D_FLAG_VERIFY(dentry, 0);
 523	list_add(&dentry->d_lru, list);
 524	dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
 525	this_cpu_inc(nr_dentry_unused);
 526}
 527
 528/*
 529 * These can only be called under the global LRU lock, ie during the
 530 * callback for freeing the LRU list. "isolate" removes it from the
 531 * LRU lists entirely, while shrink_move moves it to the indicated
 532 * private list.
 533 */
 534static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
 535{
 536	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
 537	dentry->d_flags &= ~DCACHE_LRU_LIST;
 538	this_cpu_dec(nr_dentry_unused);
 539	if (d_is_negative(dentry))
 540		this_cpu_dec(nr_dentry_negative);
 541	list_lru_isolate(lru, &dentry->d_lru);
 542}
 543
 544static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
 545			      struct list_head *list)
 546{
 547	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
 548	dentry->d_flags |= DCACHE_SHRINK_LIST;
 549	if (d_is_negative(dentry))
 550		this_cpu_dec(nr_dentry_negative);
 551	list_lru_isolate_move(lru, &dentry->d_lru, list);
 552}
 553
 554static void ___d_drop(struct dentry *dentry)
 555{
 556	struct hlist_bl_head *b;
 557	/*
 558	 * Hashed dentries are normally on the dentry hashtable,
 559	 * with the exception of those newly allocated by
 560	 * d_obtain_root, which are always IS_ROOT:
 561	 */
 562	if (unlikely(IS_ROOT(dentry)))
 563		b = &dentry->d_sb->s_roots;
 564	else
 565		b = d_hash(dentry->d_name.hash);
 566
 567	hlist_bl_lock(b);
 568	__hlist_bl_del(&dentry->d_hash);
 569	hlist_bl_unlock(b);
 570}
 571
 572void __d_drop(struct dentry *dentry)
 573{
 574	if (!d_unhashed(dentry)) {
 575		___d_drop(dentry);
 576		dentry->d_hash.pprev = NULL;
 577		write_seqcount_invalidate(&dentry->d_seq);
 578	}
 579}
 580EXPORT_SYMBOL(__d_drop);
 581
 582/**
 583 * d_drop - drop a dentry
 584 * @dentry: dentry to drop
 585 *
 586 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
 587 * be found through a VFS lookup any more. Note that this is different from
 588 * deleting the dentry - d_delete will try to mark the dentry negative if
 589 * possible, giving a successful _negative_ lookup, while d_drop will
 590 * just make the cache lookup fail.
 591 *
 592 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
 593 * reason (NFS timeouts or autofs deletes).
 594 *
 595 * __d_drop requires dentry->d_lock
 596 *
 597 * ___d_drop doesn't mark dentry as "unhashed"
 598 * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
 599 */
 600void d_drop(struct dentry *dentry)
 601{
 602	spin_lock(&dentry->d_lock);
 603	__d_drop(dentry);
 604	spin_unlock(&dentry->d_lock);
 605}
 606EXPORT_SYMBOL(d_drop);
 607
 608static inline void dentry_unlist(struct dentry *dentry)
 609{
 610	struct dentry *next;
 611	/*
 612	 * Inform d_walk() and shrink_dentry_list() that we are no longer
 613	 * attached to the dentry tree
 614	 */
 615	dentry->d_flags |= DCACHE_DENTRY_KILLED;
 616	if (unlikely(hlist_unhashed(&dentry->d_sib)))
 617		return;
 618	__hlist_del(&dentry->d_sib);
 619	/*
 620	 * Cursors can move around the list of children.  While we'd been
 621	 * a normal list member, it didn't matter - ->d_sib.next would've
 622	 * been updated.  However, from now on it won't be and for the
 623	 * things like d_walk() it might end up with a nasty surprise.
 624	 * Normally d_walk() doesn't care about cursors moving around -
 625	 * ->d_lock on parent prevents that and since a cursor has no children
 626	 * of its own, we get through it without ever unlocking the parent.
 627	 * There is one exception, though - if we ascend from a child that
 628	 * gets killed as soon as we unlock it, the next sibling is found
 629	 * using the value left in its ->d_sib.next.  And if _that_
 630	 * pointed to a cursor, and cursor got moved (e.g. by lseek())
 631	 * before d_walk() regains parent->d_lock, we'll end up skipping
 632	 * everything the cursor had been moved past.
 633	 *
 634	 * Solution: make sure that the pointer left behind in ->d_sib.next
 635	 * points to something that won't be moving around.  I.e. skip the
 636	 * cursors.
 637	 */
 638	while (dentry->d_sib.next) {
 639		next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib);
 640		if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
 641			break;
 642		dentry->d_sib.next = next->d_sib.next;
 643	}
 644}
 645
 646static struct dentry *__dentry_kill(struct dentry *dentry)
 647{
 648	struct dentry *parent = NULL;
 649	bool can_free = true;
 650
 651	/*
 652	 * The dentry is now unrecoverably dead to the world.
 653	 */
 654	lockref_mark_dead(&dentry->d_lockref);
 655
 656	/*
 657	 * inform the fs via d_prune that this dentry is about to be
 658	 * unhashed and destroyed.
 659	 */
 660	if (dentry->d_flags & DCACHE_OP_PRUNE)
 661		dentry->d_op->d_prune(dentry);
 662
 663	if (dentry->d_flags & DCACHE_LRU_LIST) {
 664		if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
 665			d_lru_del(dentry);
 666	}
 667	/* if it was on the hash then remove it */
 668	__d_drop(dentry);
 669	if (dentry->d_inode)
 670		dentry_unlink_inode(dentry);
 671	else
 672		spin_unlock(&dentry->d_lock);
 673	this_cpu_dec(nr_dentry);
 674	if (dentry->d_op && dentry->d_op->d_release)
 675		dentry->d_op->d_release(dentry);
 676
 677	cond_resched();
 678	/* now that it's negative, ->d_parent is stable */
 679	if (!IS_ROOT(dentry)) {
 680		parent = dentry->d_parent;
 681		spin_lock(&parent->d_lock);
 682	}
 683	spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
 684	dentry_unlist(dentry);
 685	if (dentry->d_flags & DCACHE_SHRINK_LIST)
 686		can_free = false;
 687	spin_unlock(&dentry->d_lock);
 688	if (likely(can_free))
 689		dentry_free(dentry);
 690	if (parent && --parent->d_lockref.count) {
 691		spin_unlock(&parent->d_lock);
 692		return NULL;
 693	}
 694	return parent;
 695}
 696
 697/*
 698 * Lock a dentry for feeding it to __dentry_kill().
 699 * Called under rcu_read_lock() and dentry->d_lock; the former
 700 * guarantees that nothing we access will be freed under us.
 701 * Note that dentry is *not* protected from concurrent dentry_kill(),
 702 * d_delete(), etc.
 703 *
 704 * Return false if dentry is busy.  Otherwise, return true and have
 705 * that dentry's inode locked.
 706 */
 707
 708static bool lock_for_kill(struct dentry *dentry)
 709{
 710	struct inode *inode = dentry->d_inode;
 711
 712	if (unlikely(dentry->d_lockref.count))
 713		return false;
 714
 715	if (!inode || likely(spin_trylock(&inode->i_lock)))
 716		return true;
 717
 718	do {
 719		spin_unlock(&dentry->d_lock);
 720		spin_lock(&inode->i_lock);
 721		spin_lock(&dentry->d_lock);
 722		if (likely(inode == dentry->d_inode))
 723			break;
 724		spin_unlock(&inode->i_lock);
 725		inode = dentry->d_inode;
 726	} while (inode);
 727	if (likely(!dentry->d_lockref.count))
 728		return true;
 729	if (inode)
 730		spin_unlock(&inode->i_lock);
 731	return false;
 732}
 733
 734/*
 735 * Decide if dentry is worth retaining.  Usually this is called with dentry
 736 * locked; if not locked, we are more limited and might not be able to tell
 737 * without a lock.  False in this case means "punt to locked path and recheck".
 738 *
 739 * In case we aren't locked, these predicates are not "stable". However, it is
 740 * sufficient that at some point after we dropped the reference the dentry was
 741 * hashed and the flags had the proper value. Other dentry users may have
 742 * re-gotten a reference to the dentry and change that, but our work is done -
 743 * we can leave the dentry around with a zero refcount.
 744 */
 745static inline bool retain_dentry(struct dentry *dentry, bool locked)
 746{
 747	unsigned int d_flags;
 748
 749	smp_rmb();
 750	d_flags = READ_ONCE(dentry->d_flags);
 751
 752	// Unreachable? Nobody would be able to look it up, no point retaining
 753	if (unlikely(d_unhashed(dentry)))
 754		return false;
 755
 756	// Same if it's disconnected
 757	if (unlikely(d_flags & DCACHE_DISCONNECTED))
 758		return false;
 759
 760	// ->d_delete() might tell us not to bother, but that requires
 761	// ->d_lock; can't decide without it
 762	if (unlikely(d_flags & DCACHE_OP_DELETE)) {
 763		if (!locked || dentry->d_op->d_delete(dentry))
 764			return false;
 765	}
 766
 767	// Explicitly told not to bother
 768	if (unlikely(d_flags & DCACHE_DONTCACHE))
 769		return false;
 770
 771	// At this point it looks like we ought to keep it.  We also might
 772	// need to do something - put it on LRU if it wasn't there already
 773	// and mark it referenced if it was on LRU, but not marked yet.
 774	// Unfortunately, both actions require ->d_lock, so in lockless
 775	// case we'd have to punt rather than doing those.
 776	if (unlikely(!(d_flags & DCACHE_LRU_LIST))) {
 777		if (!locked)
 778			return false;
 779		d_lru_add(dentry);
 780	} else if (unlikely(!(d_flags & DCACHE_REFERENCED))) {
 781		if (!locked)
 782			return false;
 783		dentry->d_flags |= DCACHE_REFERENCED;
 784	}
 785	return true;
 786}
 787
 788void d_mark_dontcache(struct inode *inode)
 789{
 790	struct dentry *de;
 791
 792	spin_lock(&inode->i_lock);
 793	hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
 794		spin_lock(&de->d_lock);
 795		de->d_flags |= DCACHE_DONTCACHE;
 796		spin_unlock(&de->d_lock);
 797	}
 798	inode_state_set(inode, I_DONTCACHE);
 799	spin_unlock(&inode->i_lock);
 800}
 801EXPORT_SYMBOL(d_mark_dontcache);
 802
 803/*
 804 * Try to do a lockless dput(), and return whether that was successful.
 805 *
 806 * If unsuccessful, we return false, having already taken the dentry lock.
 807 * In that case refcount is guaranteed to be zero and we have already
 808 * decided that it's not worth keeping around.
 809 *
 810 * The caller needs to hold the RCU read lock, so that the dentry is
 811 * guaranteed to stay around even if the refcount goes down to zero!
 812 */
 813static inline bool fast_dput(struct dentry *dentry)
 814{
 815	int ret;
 816
 817	/*
 818	 * try to decrement the lockref optimistically.
 819	 */
 820	ret = lockref_put_return(&dentry->d_lockref);
 821
 822	/*
 823	 * If the lockref_put_return() failed due to the lock being held
 824	 * by somebody else, the fast path has failed. We will need to
 825	 * get the lock, and then check the count again.
 826	 */
 827	if (unlikely(ret < 0)) {
 828		spin_lock(&dentry->d_lock);
 829		if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) {
 830			spin_unlock(&dentry->d_lock);
 831			return true;
 832		}
 833		dentry->d_lockref.count--;
 834		goto locked;
 835	}
 836
 837	/*
 838	 * If we weren't the last ref, we're done.
 839	 */
 840	if (ret)
 841		return true;
 842
 843	/*
 844	 * Can we decide that decrement of refcount is all we needed without
 845	 * taking the lock?  There's a very common case when it's all we need -
 846	 * dentry looks like it ought to be retained and there's nothing else
 847	 * to do.
 848	 */
 849	if (retain_dentry(dentry, false))
 850		return true;
 851
 852	/*
 853	 * Either not worth retaining or we can't tell without the lock.
 854	 * Get the lock, then.  We've already decremented the refcount to 0,
 855	 * but we'll need to re-check the situation after getting the lock.
 856	 */
 857	spin_lock(&dentry->d_lock);
 858
 859	/*
 860	 * Did somebody else grab a reference to it in the meantime, and
 861	 * we're no longer the last user after all? Alternatively, somebody
 862	 * else could have killed it and marked it dead. Either way, we
 863	 * don't need to do anything else.
 864	 */
 865locked:
 866	if (dentry->d_lockref.count || retain_dentry(dentry, true)) {
 867		spin_unlock(&dentry->d_lock);
 868		return true;
 869	}
 870	return false;
 871}
 872
 873static void finish_dput(struct dentry *dentry)
 874	__releases(dentry->d_lock)
 875	__releases(RCU)
 876{
 877	while (lock_for_kill(dentry)) {
 878		rcu_read_unlock();
 879		dentry = __dentry_kill(dentry);
 880		if (!dentry)
 881			return;
 882		if (retain_dentry(dentry, true)) {
 883			spin_unlock(&dentry->d_lock);
 884			return;
 885		}
 886		rcu_read_lock();
 887	}
 888	rcu_read_unlock();
 889	spin_unlock(&dentry->d_lock);
 890}
 891
 892/* 
 893 * This is dput
 894 *
 895 * This is complicated by the fact that we do not want to put
 896 * dentries that are no longer on any hash chain on the unused
 897 * list: we'd much rather just get rid of them immediately.
 898 *
 899 * However, that implies that we have to traverse the dentry
 900 * tree upwards to the parents which might _also_ now be
 901 * scheduled for deletion (it may have been only waiting for
 902 * its last child to go away).
 903 *
 904 * This tail recursion is done by hand as we don't want to depend
 905 * on the compiler to always get this right (gcc generally doesn't).
 906 * Real recursion would eat up our stack space.
 907 */
 908
 909/*
 910 * dput - release a dentry
 911 * @dentry: dentry to release 
 912 *
 913 * Release a dentry. This will drop the usage count and if appropriate
 914 * call the dentry unlink method as well as removing it from the queues and
 915 * releasing its resources. If the parent dentries were scheduled for release
 916 * they too may now get deleted.
 917 */
 918void dput(struct dentry *dentry)
 919{
 920	if (!dentry)
 921		return;
 922	might_sleep();
 923	rcu_read_lock();
 924	if (likely(fast_dput(dentry))) {
 925		rcu_read_unlock();
 926		return;
 927	}
 928	finish_dput(dentry);
 929}
 930EXPORT_SYMBOL(dput);
 931
 932void d_make_discardable(struct dentry *dentry)
 933{
 934	spin_lock(&dentry->d_lock);
 935	WARN_ON(!(dentry->d_flags & DCACHE_PERSISTENT));
 936	dentry->d_flags &= ~DCACHE_PERSISTENT;
 937	dentry->d_lockref.count--;
 938	rcu_read_lock();
 939	finish_dput(dentry);
 940}
 941EXPORT_SYMBOL(d_make_discardable);
 942
 943static void to_shrink_list(struct dentry *dentry, struct list_head *list)
 944__must_hold(&dentry->d_lock)
 945{
 946	if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
 947		if (dentry->d_flags & DCACHE_LRU_LIST)
 948			d_lru_del(dentry);
 949		d_shrink_add(dentry, list);
 950	}
 951}
 952
 953void dput_to_list(struct dentry *dentry, struct list_head *list)
 954{
 955	rcu_read_lock();
 956	if (likely(fast_dput(dentry))) {
 957		rcu_read_unlock();
 958		return;
 959	}
 960	rcu_read_unlock();
 961	to_shrink_list(dentry, list);
 962	spin_unlock(&dentry->d_lock);
 963}
 964
 965struct dentry *dget_parent(struct dentry *dentry)
 966{
 967	int gotref;
 968	struct dentry *ret;
 969	unsigned seq;
 970
 971	/*
 972	 * Do optimistic parent lookup without any
 973	 * locking.
 974	 */
 975	rcu_read_lock();
 976	seq = raw_seqcount_begin(&dentry->d_seq);
 977	ret = READ_ONCE(dentry->d_parent);
 978	gotref = lockref_get_not_zero(&ret->d_lockref);
 979	rcu_read_unlock();
 980	if (likely(gotref)) {
 981		if (!read_seqcount_retry(&dentry->d_seq, seq))
 982			return ret;
 983		dput(ret);
 984	}
 985
 986repeat:
 987	/*
 988	 * Don't need rcu_dereference because we re-check it was correct under
 989	 * the lock.
 990	 */
 991	rcu_read_lock();
 992	ret = dentry->d_parent;
 993	spin_lock(&ret->d_lock);
 994	if (unlikely(ret != dentry->d_parent)) {
 995		spin_unlock(&ret->d_lock);
 996		rcu_read_unlock();
 997		goto repeat;
 998	}
 999	rcu_read_unlock();
1000	BUG_ON(!ret->d_lockref.count);
1001	ret->d_lockref.count++;
1002	spin_unlock(&ret->d_lock);
1003	return ret;
1004}
1005EXPORT_SYMBOL(dget_parent);
1006
1007static struct dentry * __d_find_any_alias(struct inode *inode)
1008{
1009	struct dentry *alias;
1010
1011	if (hlist_empty(&inode->i_dentry))
1012		return NULL;
1013	alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
1014	lockref_get(&alias->d_lockref);
1015	return alias;
1016}
1017
1018/**
1019 * d_find_any_alias - find any alias for a given inode
1020 * @inode: inode to find an alias for
1021 *
1022 * If any aliases exist for the given inode, take and return a
1023 * reference for one of them.  If no aliases exist, return %NULL.
1024 */
1025struct dentry *d_find_any_alias(struct inode *inode)
1026{
1027	struct dentry *de;
1028
1029	spin_lock(&inode->i_lock);
1030	de = __d_find_any_alias(inode);
1031	spin_unlock(&inode->i_lock);
1032	return de;
1033}
1034EXPORT_SYMBOL(d_find_any_alias);
1035
1036static struct dentry *__d_find_alias(struct inode *inode)
1037{
1038	struct dentry *alias;
1039
1040	if (S_ISDIR(inode->i_mode))
1041		return __d_find_any_alias(inode);
1042
1043	hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
1044		spin_lock(&alias->d_lock);
1045 		if (!d_unhashed(alias)) {
1046			dget_dlock(alias);
1047			spin_unlock(&alias->d_lock);
1048			return alias;
1049		}
1050		spin_unlock(&alias->d_lock);
1051	}
1052	return NULL;
1053}
1054
1055/**
1056 * d_find_alias - grab a hashed alias of inode
1057 * @inode: inode in question
1058 *
1059 * If inode has a hashed alias, or is a directory and has any alias,
1060 * acquire the reference to alias and return it. Otherwise return NULL.
1061 * Notice that if inode is a directory there can be only one alias and
1062 * it can be unhashed only if it has no children, or if it is the root
1063 * of a filesystem, or if the directory was renamed and d_revalidate
1064 * was the first vfs operation to notice.
1065 *
1066 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
1067 * any other hashed alias over that one.
1068 */
1069struct dentry *d_find_alias(struct inode *inode)
1070{
1071	struct dentry *de = NULL;
1072
1073	if (!hlist_empty(&inode->i_dentry)) {
1074		spin_lock(&inode->i_lock);
1075		de = __d_find_alias(inode);
1076		spin_unlock(&inode->i_lock);
1077	}
1078	return de;
1079}
1080EXPORT_SYMBOL(d_find_alias);
1081
1082/*
1083 *  Caller MUST be holding rcu_read_lock() and be guaranteed
1084 *  that inode won't get freed until rcu_read_unlock().
1085 */
1086struct dentry *d_find_alias_rcu(struct inode *inode)
1087{
1088	struct hlist_head *l = &inode->i_dentry;
1089	struct dentry *de = NULL;
1090
1091	spin_lock(&inode->i_lock);
1092	// ->i_dentry and ->i_rcu are colocated, but the latter won't be
1093	// used without having I_FREEING set, which means no aliases left
1094	if (likely(!(inode_state_read(inode) & I_FREEING) && !hlist_empty(l))) {
1095		if (S_ISDIR(inode->i_mode)) {
1096			de = hlist_entry(l->first, struct dentry, d_u.d_alias);
1097		} else {
1098			hlist_for_each_entry(de, l, d_u.d_alias)
1099				if (!d_unhashed(de))
1100					break;
1101		}
1102	}
1103	spin_unlock(&inode->i_lock);
1104	return de;
1105}
1106
1107/**
1108 * d_dispose_if_unused - move unreferenced dentries to shrink list
1109 * @dentry: dentry in question
1110 * @dispose: head of shrink list
1111 *
1112 * If dentry has no external references, move it to shrink list.
1113 *
1114 * NOTE!!! The caller is responsible for preventing eviction of the dentry by
1115 * holding dentry->d_inode->i_lock or equivalent.
1116 */
1117void d_dispose_if_unused(struct dentry *dentry, struct list_head *dispose)
1118{
1119	spin_lock(&dentry->d_lock);
1120	if (!dentry->d_lockref.count)
1121		to_shrink_list(dentry, dispose);
1122	spin_unlock(&dentry->d_lock);
1123}
1124EXPORT_SYMBOL(d_dispose_if_unused);
1125
1126/*
1127 *	Try to kill dentries associated with this inode.
1128 * WARNING: you must own a reference to inode.
1129 */
1130void d_prune_aliases(struct inode *inode)
1131{
1132	LIST_HEAD(dispose);
1133	struct dentry *dentry;
1134
1135	spin_lock(&inode->i_lock);
1136	hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias)
1137		d_dispose_if_unused(dentry, &dispose);
1138	spin_unlock(&inode->i_lock);
1139	shrink_dentry_list(&dispose);
1140}
1141EXPORT_SYMBOL(d_prune_aliases);
1142
1143static inline void shrink_kill(struct dentry *victim)
1144{
1145	do {
1146		rcu_read_unlock();
1147		victim = __dentry_kill(victim);
1148		rcu_read_lock();
1149	} while (victim && lock_for_kill(victim));
1150	rcu_read_unlock();
1151	if (victim)
1152		spin_unlock(&victim->d_lock);
1153}
1154
1155void shrink_dentry_list(struct list_head *list)
1156{
1157	while (!list_empty(list)) {
1158		struct dentry *dentry;
1159
1160		dentry = list_entry(list->prev, struct dentry, d_lru);
1161		spin_lock(&dentry->d_lock);
1162		rcu_read_lock();
1163		if (!lock_for_kill(dentry)) {
1164			bool can_free;
1165			rcu_read_unlock();
1166			d_shrink_del(dentry);
1167			can_free = dentry->d_flags & DCACHE_DENTRY_KILLED;
1168			spin_unlock(&dentry->d_lock);
1169			if (can_free)
1170				dentry_free(dentry);
1171			continue;
1172		}
1173		d_shrink_del(dentry);
1174		shrink_kill(dentry);
1175	}
1176}
1177EXPORT_SYMBOL(shrink_dentry_list);
1178
1179static enum lru_status dentry_lru_isolate(struct list_head *item,
1180		struct list_lru_one *lru, void *arg)
1181{
1182	struct list_head *freeable = arg;
1183	struct dentry	*dentry = container_of(item, struct dentry, d_lru);
1184
1185
1186	/*
1187	 * we are inverting the lru lock/dentry->d_lock here,
1188	 * so use a trylock. If we fail to get the lock, just skip
1189	 * it
1190	 */
1191	if (!spin_trylock(&dentry->d_lock))
1192		return LRU_SKIP;
1193
1194	/*
1195	 * Referenced dentries are still in use. If they have active
1196	 * counts, just remove them from the LRU. Otherwise give them
1197	 * another pass through the LRU.
1198	 */
1199	if (dentry->d_lockref.count) {
1200		d_lru_isolate(lru, dentry);
1201		spin_unlock(&dentry->d_lock);
1202		return LRU_REMOVED;
1203	}
1204
1205	if (dentry->d_flags & DCACHE_REFERENCED) {
1206		dentry->d_flags &= ~DCACHE_REFERENCED;
1207		spin_unlock(&dentry->d_lock);
1208
1209		/*
1210		 * The list move itself will be made by the common LRU code. At
1211		 * this point, we've dropped the dentry->d_lock but keep the
1212		 * lru lock. This is safe to do, since every list movement is
1213		 * protected by the lru lock even if both locks are held.
1214		 *
1215		 * This is guaranteed by the fact that all LRU management
1216		 * functions are intermediated by the LRU API calls like
1217		 * list_lru_add_obj and list_lru_del_obj. List movement in this file
1218		 * only ever occur through this functions or through callbacks
1219		 * like this one, that are called from the LRU API.
1220		 *
1221		 * The only exceptions to this are functions like
1222		 * shrink_dentry_list, and code that first checks for the
1223		 * DCACHE_SHRINK_LIST flag.  Those are guaranteed to be
1224		 * operating only with stack provided lists after they are
1225		 * properly isolated from the main list.  It is thus, always a
1226		 * local access.
1227		 */
1228		return LRU_ROTATE;
1229	}
1230
1231	d_lru_shrink_move(lru, dentry, freeable);
1232	spin_unlock(&dentry->d_lock);
1233
1234	return LRU_REMOVED;
1235}
1236
1237/**
1238 * prune_dcache_sb - shrink the dcache
1239 * @sb: superblock
1240 * @sc: shrink control, passed to list_lru_shrink_walk()
1241 *
1242 * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
1243 * is done when we need more memory and called from the superblock shrinker
1244 * function.
1245 *
1246 * This function may fail to free any resources if all the dentries are in
1247 * use.
1248 */
1249long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
1250{
1251	LIST_HEAD(dispose);
1252	long freed;
1253
1254	freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1255				     dentry_lru_isolate, &dispose);
1256	shrink_dentry_list(&dispose);
1257	return freed;
1258}
1259
1260static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
1261		struct list_lru_one *lru, void *arg)
1262{
1263	struct list_head *freeable = arg;
1264	struct dentry	*dentry = container_of(item, struct dentry, d_lru);
1265
1266	/*
1267	 * we are inverting the lru lock/dentry->d_lock here,
1268	 * so use a trylock. If we fail to get the lock, just skip
1269	 * it
1270	 */
1271	if (!spin_trylock(&dentry->d_lock))
1272		return LRU_SKIP;
1273
1274	d_lru_shrink_move(lru, dentry, freeable);
1275	spin_unlock(&dentry->d_lock);
1276
1277	return LRU_REMOVED;
1278}
1279
1280
1281/**
1282 * shrink_dcache_sb - shrink dcache for a superblock
1283 * @sb: superblock
1284 *
1285 * Shrink the dcache for the specified super block. This is used to free
1286 * the dcache before unmounting a file system.
1287 */
1288void shrink_dcache_sb(struct super_block *sb)
1289{
1290	do {
1291		LIST_HEAD(dispose);
1292
1293		list_lru_walk(&sb->s_dentry_lru,
1294			dentry_lru_isolate_shrink, &dispose, 1024);
1295		shrink_dentry_list(&dispose);
1296	} while (list_lru_count(&sb->s_dentry_lru) > 0);
1297}
1298EXPORT_SYMBOL(shrink_dcache_sb);
1299
1300/**
1301 * enum d_walk_ret - action to talke during tree walk
1302 * @D_WALK_CONTINUE:	contrinue walk
1303 * @D_WALK_QUIT:	quit walk
1304 * @D_WALK_NORETRY:	quit when retry is needed
1305 * @D_WALK_SKIP:	skip this dentry and its children
1306 */
1307enum d_walk_ret {
1308	D_WALK_CONTINUE,
1309	D_WALK_QUIT,
1310	D_WALK_NORETRY,
1311	D_WALK_SKIP,
1312};
1313
1314/**
1315 * d_walk - walk the dentry tree
1316 * @parent:	start of walk
1317 * @data:	data passed to @enter() and @finish()
1318 * @enter:	callback when first entering the dentry
1319 *
1320 * The @enter() callbacks are called with d_lock held.
1321 */
1322static void d_walk(struct dentry *parent, void *data,
1323		   enum d_walk_ret (*enter)(void *, struct dentry *))
1324{
1325	struct dentry *this_parent, *dentry;
1326	unsigned seq = 0;
1327	enum d_walk_ret ret;
1328	bool retry = true;
1329
1330again:
1331	read_seqbegin_or_lock(&rename_lock, &seq);
1332	this_parent = parent;
1333	spin_lock(&this_parent->d_lock);
1334
1335	ret = enter(data, this_parent);
1336	switch (ret) {
1337	case D_WALK_CONTINUE:
1338		break;
1339	case D_WALK_QUIT:
1340	case D_WALK_SKIP:
1341		goto out_unlock;
1342	case D_WALK_NORETRY:
1343		retry = false;
1344		break;
1345	}
1346repeat:
1347	dentry = d_first_child(this_parent);
1348resume:
1349	hlist_for_each_entry_from(dentry, d_sib) {
1350		if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1351			continue;
1352
1353		spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1354
1355		ret = enter(data, dentry);
1356		switch (ret) {
1357		case D_WALK_CONTINUE:
1358			break;
1359		case D_WALK_QUIT:
1360			spin_unlock(&dentry->d_lock);
1361			goto out_unlock;
1362		case D_WALK_NORETRY:
1363			retry = false;
1364			break;
1365		case D_WALK_SKIP:
1366			spin_unlock(&dentry->d_lock);
1367			continue;
1368		}
1369
1370		if (!hlist_empty(&dentry->d_children)) {
1371			spin_unlock(&this_parent->d_lock);
1372			spin_release(&dentry->d_lock.dep_map, _RET_IP_);
1373			this_parent = dentry;
1374			spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
1375			goto repeat;
1376		}
1377		spin_unlock(&dentry->d_lock);
1378	}
1379	/*
1380	 * All done at this level ... ascend and resume the search.
1381	 */
1382	rcu_read_lock();
1383ascend:
1384	if (this_parent != parent) {
1385		dentry = this_parent;
1386		this_parent = dentry->d_parent;
1387
1388		spin_unlock(&dentry->d_lock);
1389		spin_lock(&this_parent->d_lock);
1390
1391		/* might go back up the wrong parent if we have had a rename. */
1392		if (need_seqretry(&rename_lock, seq))
1393			goto rename_retry;
1394		/* go into the first sibling still alive */
1395		hlist_for_each_entry_continue(dentry, d_sib) {
1396			if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) {
1397				rcu_read_unlock();
1398				goto resume;
1399			}
1400		}
1401		goto ascend;
1402	}
1403	if (need_seqretry(&rename_lock, seq))
1404		goto rename_retry;
1405	rcu_read_unlock();
1406
1407out_unlock:
1408	spin_unlock(&this_parent->d_lock);
1409	done_seqretry(&rename_lock, seq);
1410	return;
1411
1412rename_retry:
1413	spin_unlock(&this_parent->d_lock);
1414	rcu_read_unlock();
1415	BUG_ON(seq & 1);
1416	if (!retry)
1417		return;
1418	seq = 1;
1419	goto again;
1420}
1421
1422struct check_mount {
1423	struct vfsmount *mnt;
1424	unsigned int mounted;
1425};
1426
1427/* locks: mount_locked_reader && dentry->d_lock */
1428static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
1429{
1430	struct check_mount *info = data;
1431	struct path path = { .mnt = info->mnt, .dentry = dentry };
1432
1433	if (likely(!d_mountpoint(dentry)))
1434		return D_WALK_CONTINUE;
1435	if (__path_is_mountpoint(&path)) {
1436		info->mounted = 1;
1437		return D_WALK_QUIT;
1438	}
1439	return D_WALK_CONTINUE;
1440}
1441
1442/**
1443 * path_has_submounts - check for mounts over a dentry in the
1444 *                      current namespace.
1445 * @parent: path to check.
1446 *
1447 * Return true if the parent or its subdirectories contain
1448 * a mount point in the current namespace.
1449 */
1450int path_has_submounts(const struct path *parent)
1451{
1452	struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
1453
1454	guard(mount_locked_reader)();
1455	d_walk(parent->dentry, &data, path_check_mount);
1456
1457	return data.mounted;
1458}
1459EXPORT_SYMBOL(path_has_submounts);
1460
1461/*
1462 * Called by mount code to set a mountpoint and check if the mountpoint is
1463 * reachable (e.g. NFS can unhash a directory dentry and then the complete
1464 * subtree can become unreachable).
1465 *
1466 * Only one of d_invalidate() and d_set_mounted() must succeed.  For
1467 * this reason take rename_lock and d_lock on dentry and ancestors.
1468 */
1469int d_set_mounted(struct dentry *dentry)
1470{
1471	struct dentry *p;
1472	int ret = -ENOENT;
1473	read_seqlock_excl(&rename_lock);
1474	for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
1475		/* Need exclusion wrt. d_invalidate() */
1476		spin_lock(&p->d_lock);
1477		if (unlikely(d_unhashed(p))) {
1478			spin_unlock(&p->d_lock);
1479			goto out;
1480		}
1481		spin_unlock(&p->d_lock);
1482	}
1483	spin_lock(&dentry->d_lock);
1484	if (!d_unlinked(dentry)) {
1485		ret = -EBUSY;
1486		if (!d_mountpoint(dentry)) {
1487			dentry->d_flags |= DCACHE_MOUNTED;
1488			ret = 0;
1489		}
1490	}
1491 	spin_unlock(&dentry->d_lock);
1492out:
1493	read_sequnlock_excl(&rename_lock);
1494	return ret;
1495}
1496
1497/*
1498 * Search the dentry child list of the specified parent,
1499 * and move any unused dentries to the end of the unused
1500 * list for prune_dcache(). We descend to the next level
1501 * whenever the d_children list is non-empty and continue
1502 * searching.
1503 *
1504 * It returns zero iff there are no unused children,
1505 * otherwise  it returns the number of children moved to
1506 * the end of the unused list. This may not be the total
1507 * number of unused children, because select_parent can
1508 * drop the lock and return early due to latency
1509 * constraints.
1510 */
1511
1512struct select_data {
1513	struct dentry *start;
1514	union {
1515		long found;
1516		struct dentry *victim;
1517	};
1518	struct list_head dispose;
1519};
1520
1521static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
1522{
1523	struct select_data *data = _data;
1524	enum d_walk_ret ret = D_WALK_CONTINUE;
1525
1526	if (data->start == dentry)
1527		goto out;
1528
1529	if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1530		data->found++;
1531	} else if (!dentry->d_lockref.count) {
1532		to_shrink_list(dentry, &data->dispose);
1533		data->found++;
1534	} else if (dentry->d_lockref.count < 0) {
1535		data->found++;
1536	}
1537	/*
1538	 * We can return to the caller if we have found some (this
1539	 * ensures forward progress). We'll be coming back to find
1540	 * the rest.
1541	 */
1542	if (!list_empty(&data->dispose))
1543		ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1544out:
1545	return ret;
1546}
1547
1548static enum d_walk_ret select_collect_umount(void *_data, struct dentry *dentry)
1549{
1550	if (dentry->d_flags & DCACHE_PERSISTENT) {
1551		dentry->d_flags &= ~DCACHE_PERSISTENT;
1552		dentry->d_lockref.count--;
1553	}
1554	return select_collect(_data, dentry);
1555}
1556
1557static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
1558{
1559	struct select_data *data = _data;
1560	enum d_walk_ret ret = D_WALK_CONTINUE;
1561
1562	if (data->start == dentry)
1563		goto out;
1564
1565	if (!dentry->d_lockref.count) {
1566		if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1567			rcu_read_lock();
1568			data->victim = dentry;
1569			return D_WALK_QUIT;
1570		}
1571		to_shrink_list(dentry, &data->dispose);
1572	}
1573	/*
1574	 * We can return to the caller if we have found some (this
1575	 * ensures forward progress). We'll be coming back to find
1576	 * the rest.
1577	 */
1578	if (!list_empty(&data->dispose))
1579		ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1580out:
1581	return ret;
1582}
1583
1584/**
1585 * shrink_dcache_tree - prune dcache
1586 * @parent: parent of entries to prune
1587 * @for_umount: true if we want to unpin the persistent ones
1588 *
1589 * Prune the dcache to remove unused children of the parent dentry.
1590 */
1591static void shrink_dcache_tree(struct dentry *parent, bool for_umount)
1592{
1593	for (;;) {
1594		struct select_data data = {.start = parent};
1595
1596		INIT_LIST_HEAD(&data.dispose);
1597		d_walk(parent, &data,
1598			for_umount ? select_collect_umount : select_collect);
1599
1600		if (!list_empty(&data.dispose)) {
1601			shrink_dentry_list(&data.dispose);
1602			continue;
1603		}
1604
1605		cond_resched();
1606		if (!data.found)
1607			break;
1608		data.victim = NULL;
1609		d_walk(parent, &data, select_collect2);
1610		if (data.victim) {
1611			spin_lock(&data.victim->d_lock);
1612			if (!lock_for_kill(data.victim)) {
1613				spin_unlock(&data.victim->d_lock);
1614				rcu_read_unlock();
1615			} else {
1616				shrink_kill(data.victim);
1617			}
1618		}
1619		if (!list_empty(&data.dispose))
1620			shrink_dentry_list(&data.dispose);
1621	}
1622}
1623
1624void shrink_dcache_parent(struct dentry *parent)
1625{
1626	shrink_dcache_tree(parent, false);
1627}
1628EXPORT_SYMBOL(shrink_dcache_parent);
1629
1630static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
1631{
1632	/* it has busy descendents; complain about those instead */
1633	if (!hlist_empty(&dentry->d_children))
1634		return D_WALK_CONTINUE;
1635
1636	/* root with refcount 1 is fine */
1637	if (dentry == _data && dentry->d_lockref.count == 1)
1638		return D_WALK_CONTINUE;
1639
1640	WARN(1, "BUG: Dentry %p{i=%lx,n=%pd} "
1641			" still in use (%d) [unmount of %s %s]\n",
1642		       dentry,
1643		       dentry->d_inode ?
1644		       dentry->d_inode->i_ino : 0UL,
1645		       dentry,
1646		       dentry->d_lockref.count,
1647		       dentry->d_sb->s_type->name,
1648		       dentry->d_sb->s_id);
1649	return D_WALK_CONTINUE;
1650}
1651
1652static void do_one_tree(struct dentry *dentry)
1653{
1654	shrink_dcache_tree(dentry, true);
1655	d_walk(dentry, dentry, umount_check);
1656	d_drop(dentry);
1657	dput(dentry);
1658}
1659
1660/*
1661 * destroy the dentries attached to a superblock on unmounting
1662 */
1663void shrink_dcache_for_umount(struct super_block *sb)
1664{
1665	struct dentry *dentry;
1666
1667	rwsem_assert_held_write(&sb->s_umount);
1668
1669	dentry = sb->s_root;
1670	sb->s_root = NULL;
1671	do_one_tree(dentry);
1672
1673	while (!hlist_bl_empty(&sb->s_roots)) {
1674		dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
1675		do_one_tree(dentry);
1676	}
1677}
1678
1679static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
1680{
1681	struct dentry **victim = _data;
1682	if (d_mountpoint(dentry)) {
1683		*victim = dget_dlock(dentry);
1684		return D_WALK_QUIT;
1685	}
1686	return D_WALK_CONTINUE;
1687}
1688
1689/**
1690 * d_invalidate - detach submounts, prune dcache, and drop
1691 * @dentry: dentry to invalidate (aka detach, prune and drop)
1692 */
1693void d_invalidate(struct dentry *dentry)
1694{
1695	bool had_submounts = false;
1696	spin_lock(&dentry->d_lock);
1697	if (d_unhashed(dentry)) {
1698		spin_unlock(&dentry->d_lock);
1699		return;
1700	}
1701	__d_drop(dentry);
1702	spin_unlock(&dentry->d_lock);
1703
1704	/* Negative dentries can be dropped without further checks */
1705	if (!dentry->d_inode)
1706		return;
1707
1708	shrink_dcache_parent(dentry);
1709	for (;;) {
1710		struct dentry *victim = NULL;
1711		d_walk(dentry, &victim, find_submount);
1712		if (!victim) {
1713			if (had_submounts)
1714				shrink_dcache_parent(dentry);
1715			return;
1716		}
1717		had_submounts = true;
1718		detach_mounts(victim);
1719		dput(victim);
1720	}
1721}
1722EXPORT_SYMBOL(d_invalidate);
1723
1724/**
1725 * __d_alloc	-	allocate a dcache entry
1726 * @sb: filesystem it will belong to
1727 * @name: qstr of the name
1728 *
1729 * Allocates a dentry. It returns %NULL if there is insufficient memory
1730 * available. On a success the dentry is returned. The name passed in is
1731 * copied and the copy passed in may be reused after this call.
1732 */
1733 
1734static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1735{
1736	struct dentry *dentry;
1737	char *dname;
1738	int err;
1739
1740	dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru,
1741				      GFP_KERNEL);
1742	if (!dentry)
1743		return NULL;
1744
1745	/*
1746	 * We guarantee that the inline name is always NUL-terminated.
1747	 * This way the memcpy() done by the name switching in rename
1748	 * will still always have a NUL at the end, even if we might
1749	 * be overwriting an internal NUL character
1750	 */
1751	dentry->d_shortname.string[DNAME_INLINE_LEN-1] = 0;
1752	if (unlikely(!name)) {
1753		name = &slash_name;
1754		dname = dentry->d_shortname.string;
1755	} else if (name->len > DNAME_INLINE_LEN-1) {
1756		size_t size = offsetof(struct external_name, name[1]);
1757		struct external_name *p = kmalloc(size + name->len,
1758						  GFP_KERNEL_ACCOUNT |
1759						  __GFP_RECLAIMABLE);
1760		if (!p) {
1761			kmem_cache_free(dentry_cache, dentry); 
1762			return NULL;
1763		}
1764		atomic_set(&p->count, 1);
1765		dname = p->name;
1766	} else  {
1767		dname = dentry->d_shortname.string;
1768	}	
1769
1770	dentry->__d_name.len = name->len;
1771	dentry->__d_name.hash = name->hash;
1772	memcpy(dname, name->name, name->len);
1773	dname[name->len] = 0;
1774
1775	/* Make sure we always see the terminating NUL character */
1776	smp_store_release(&dentry->__d_name.name, dname); /* ^^^ */
1777
1778	dentry->d_flags = 0;
1779	lockref_init(&dentry->d_lockref);
1780	seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
1781	dentry->d_inode = NULL;
1782	dentry->d_parent = dentry;
1783	dentry->d_sb = sb;
1784	dentry->d_op = sb->__s_d_op;
1785	dentry->d_flags = sb->s_d_flags;
1786	dentry->d_fsdata = NULL;
1787	INIT_HLIST_BL_NODE(&dentry->d_hash);
1788	INIT_LIST_HEAD(&dentry->d_lru);
1789	INIT_HLIST_HEAD(&dentry->d_children);
1790	INIT_HLIST_NODE(&dentry->d_u.d_alias);
1791	INIT_HLIST_NODE(&dentry->d_sib);
1792
1793	if (dentry->d_op && dentry->d_op->d_init) {
1794		err = dentry->d_op->d_init(dentry);
1795		if (err) {
1796			if (dname_external(dentry))
1797				kfree(external_name(dentry));
1798			kmem_cache_free(dentry_cache, dentry);
1799			return NULL;
1800		}
1801	}
1802
1803	this_cpu_inc(nr_dentry);
1804
1805	return dentry;
1806}
1807
1808/**
1809 * d_alloc	-	allocate a dcache entry
1810 * @parent: parent of entry to allocate
1811 * @name: qstr of the name
1812 *
1813 * Allocates a dentry. It returns %NULL if there is insufficient memory
1814 * available. On a success the dentry is returned. The name passed in is
1815 * copied and the copy passed in may be reused after this call.
1816 */
1817struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1818{
1819	struct dentry *dentry = __d_alloc(parent->d_sb, name);
1820	if (!dentry)
1821		return NULL;
1822	spin_lock(&parent->d_lock);
1823	/*
1824	 * don't need child lock because it is not subject
1825	 * to concurrency here
1826	 */
1827	dentry->d_parent = dget_dlock(parent);
1828	hlist_add_head(&dentry->d_sib, &parent->d_children);
1829	spin_unlock(&parent->d_lock);
1830
1831	return dentry;
1832}
1833EXPORT_SYMBOL(d_alloc);
1834
1835struct dentry *d_alloc_anon(struct super_block *sb)
1836{
1837	return __d_alloc(sb, NULL);
1838}
1839EXPORT_SYMBOL(d_alloc_anon);
1840
1841struct dentry *d_alloc_cursor(struct dentry * parent)
1842{
1843	struct dentry *dentry = d_alloc_anon(parent->d_sb);
1844	if (dentry) {
1845		dentry->d_flags |= DCACHE_DENTRY_CURSOR;
1846		dentry->d_parent = dget(parent);
1847	}
1848	return dentry;
1849}
1850
1851/**
1852 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
1853 * @sb: the superblock
1854 * @name: qstr of the name
1855 *
1856 * For a filesystem that just pins its dentries in memory and never
1857 * performs lookups at all, return an unhashed IS_ROOT dentry.
1858 * This is used for pipes, sockets et.al. - the stuff that should
1859 * never be anyone's children or parents.  Unlike all other
1860 * dentries, these will not have RCU delay between dropping the
1861 * last reference and freeing them.
1862 *
1863 * The only user is alloc_file_pseudo() and that's what should
1864 * be considered a public interface.  Don't use directly.
1865 */
1866struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1867{
1868	static const struct dentry_operations anon_ops = {
1869		.d_dname = simple_dname
1870	};
1871	struct dentry *dentry = __d_alloc(sb, name);
1872	if (likely(dentry)) {
1873		dentry->d_flags |= DCACHE_NORCU;
1874		/* d_op_flags(&anon_ops) is 0 */
1875		if (!dentry->d_op)
1876			dentry->d_op = &anon_ops;
1877	}
1878	return dentry;
1879}
1880
1881struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1882{
1883	struct qstr q;
1884
1885	q.name = name;
1886	q.hash_len = hashlen_string(parent, name);
1887	return d_alloc(parent, &q);
1888}
1889EXPORT_SYMBOL(d_alloc_name);
1890
1891#define DCACHE_OP_FLAGS \
1892	(DCACHE_OP_HASH | DCACHE_OP_COMPARE | DCACHE_OP_REVALIDATE | \
1893	 DCACHE_OP_WEAK_REVALIDATE | DCACHE_OP_DELETE | DCACHE_OP_PRUNE | \
1894	 DCACHE_OP_REAL)
1895
1896static unsigned int d_op_flags(const struct dentry_operations *op)
1897{
1898	unsigned int flags = 0;
1899	if (op) {
1900		if (op->d_hash)
1901			flags |= DCACHE_OP_HASH;
1902		if (op->d_compare)
1903			flags |= DCACHE_OP_COMPARE;
1904		if (op->d_revalidate)
1905			flags |= DCACHE_OP_REVALIDATE;
1906		if (op->d_weak_revalidate)
1907			flags |= DCACHE_OP_WEAK_REVALIDATE;
1908		if (op->d_delete)
1909			flags |= DCACHE_OP_DELETE;
1910		if (op->d_prune)
1911			flags |= DCACHE_OP_PRUNE;
1912		if (op->d_real)
1913			flags |= DCACHE_OP_REAL;
1914	}
1915	return flags;
1916}
1917
1918static void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1919{
1920	unsigned int flags = d_op_flags(op);
1921	WARN_ON_ONCE(dentry->d_op);
1922	WARN_ON_ONCE(dentry->d_flags & DCACHE_OP_FLAGS);
1923	dentry->d_op = op;
1924	if (flags)
1925		dentry->d_flags |= flags;
1926}
1927
1928void set_default_d_op(struct super_block *s, const struct dentry_operations *ops)
1929{
1930	unsigned int flags = d_op_flags(ops);
1931	s->__s_d_op = ops;
1932	s->s_d_flags = (s->s_d_flags & ~DCACHE_OP_FLAGS) | flags;
1933}
1934EXPORT_SYMBOL(set_default_d_op);
1935
1936static unsigned d_flags_for_inode(struct inode *inode)
1937{
1938	unsigned add_flags = DCACHE_REGULAR_TYPE;
1939
1940	if (!inode)
1941		return DCACHE_MISS_TYPE;
1942
1943	if (S_ISDIR(inode->i_mode)) {
1944		add_flags = DCACHE_DIRECTORY_TYPE;
1945		if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1946			if (unlikely(!inode->i_op->lookup))
1947				add_flags = DCACHE_AUTODIR_TYPE;
1948			else
1949				inode->i_opflags |= IOP_LOOKUP;
1950		}
1951		goto type_determined;
1952	}
1953
1954	if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
1955		if (unlikely(inode->i_op->get_link)) {
1956			add_flags = DCACHE_SYMLINK_TYPE;
1957			goto type_determined;
1958		}
1959		inode->i_opflags |= IOP_NOFOLLOW;
1960	}
1961
1962	if (unlikely(!S_ISREG(inode->i_mode)))
1963		add_flags = DCACHE_SPECIAL_TYPE;
1964
1965type_determined:
1966	if (unlikely(IS_AUTOMOUNT(inode)))
1967		add_flags |= DCACHE_NEED_AUTOMOUNT;
1968	return add_flags;
1969}
1970
1971static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1972{
1973	unsigned add_flags = d_flags_for_inode(inode);
1974	WARN_ON(d_in_lookup(dentry));
1975
1976	/*
1977	 * The negative counter only tracks dentries on the LRU. Don't dec if
1978	 * d_lru is on another list.
1979	 */
1980	if ((dentry->d_flags &
1981	     (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST)
1982		this_cpu_dec(nr_dentry_negative);
1983	hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
1984	raw_write_seqcount_begin(&dentry->d_seq);
1985	__d_set_inode_and_type(dentry, inode, add_flags);
1986	raw_write_seqcount_end(&dentry->d_seq);
1987	fsnotify_update_flags(dentry);
1988}
1989
1990/**
1991 * d_instantiate - fill in inode information for a dentry
1992 * @entry: dentry to complete
1993 * @inode: inode to attach to this dentry
1994 *
1995 * Fill in inode information in the entry.
1996 *
1997 * This turns negative dentries into productive full members
1998 * of society.
1999 *
2000 * NOTE! This assumes that the inode count has been incremented
2001 * (or otherwise set) by the caller to indicate that it is now
2002 * in use by the dcache.
2003 */
2004 
2005void d_instantiate(struct dentry *entry, struct inode * inode)
2006{
2007	BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2008	if (inode) {
2009		security_d_instantiate(entry, inode);
2010		spin_lock(&inode->i_lock);
2011		spin_lock(&entry->d_lock);
2012		__d_instantiate(entry, inode);
2013		spin_unlock(&entry->d_lock);
2014		spin_unlock(&inode->i_lock);
2015	}
2016}
2017EXPORT_SYMBOL(d_instantiate);
2018
2019/*
2020 * This should be equivalent to d_instantiate() + unlock_new_inode(),
2021 * with lockdep-related part of unlock_new_inode() done before
2022 * anything else.  Use that instead of open-coding d_instantiate()/
2023 * unlock_new_inode() combinations.
2024 */
2025void d_instantiate_new(struct dentry *entry, struct inode *inode)
2026{
2027	BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2028	BUG_ON(!inode);
2029	lockdep_annotate_inode_mutex_key(inode);
2030	security_d_instantiate(entry, inode);
2031	spin_lock(&inode->i_lock);
2032	spin_lock(&entry->d_lock);
2033	__d_instantiate(entry, inode);
2034	spin_unlock(&entry->d_lock);
2035	WARN_ON(!(inode_state_read(inode) & I_NEW));
2036	inode_state_clear(inode, I_NEW | I_CREATING);
2037	inode_wake_up_bit(inode, __I_NEW);
2038	spin_unlock(&inode->i_lock);
2039}
2040EXPORT_SYMBOL(d_instantiate_new);
2041
2042struct dentry *d_make_root(struct inode *root_inode)
2043{
2044	struct dentry *res = NULL;
2045
2046	if (root_inode) {
2047		res = d_alloc_anon(root_inode->i_sb);
2048		if (res)
2049			d_instantiate(res, root_inode);
2050		else
2051			iput(root_inode);
2052	}
2053	return res;
2054}
2055EXPORT_SYMBOL(d_make_root);
2056
2057static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
2058{
2059	struct super_block *sb;
2060	struct dentry *new, *res;
2061
2062	if (!inode)
2063		return ERR_PTR(-ESTALE);
2064	if (IS_ERR(inode))
2065		return ERR_CAST(inode);
2066
2067	sb = inode->i_sb;
2068
2069	res = d_find_any_alias(inode); /* existing alias? */
2070	if (res)
2071		goto out;
2072
2073	new = d_alloc_anon(sb);
2074	if (!new) {
2075		res = ERR_PTR(-ENOMEM);
2076		goto out;
2077	}
2078
2079	security_d_instantiate(new, inode);
2080	spin_lock(&inode->i_lock);
2081	res = __d_find_any_alias(inode); /* recheck under lock */
2082	if (likely(!res)) { /* still no alias, attach a disconnected dentry */
2083		unsigned add_flags = d_flags_for_inode(inode);
2084
2085		if (disconnected)
2086			add_flags |= DCACHE_DISCONNECTED;
2087
2088		spin_lock(&new->d_lock);
2089		__d_set_inode_and_type(new, inode, add_flags);
2090		hlist_add_head(&new->d_u.d_alias, &inode->i_dentry);
2091		if (!disconnected) {
2092			hlist_bl_lock(&sb->s_roots);
2093			hlist_bl_add_head(&new->d_hash, &sb->s_roots);
2094			hlist_bl_unlock(&sb->s_roots);
2095		}
2096		spin_unlock(&new->d_lock);
2097		spin_unlock(&inode->i_lock);
2098		inode = NULL; /* consumed by new->d_inode */
2099		res = new;
2100	} else {
2101		spin_unlock(&inode->i_lock);
2102		dput(new);
2103	}
2104
2105 out:
2106	iput(inode);
2107	return res;
2108}
2109
2110/**
2111 * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
2112 * @inode: inode to allocate the dentry for
2113 *
2114 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
2115 * similar open by handle operations.  The returned dentry may be anonymous,
2116 * or may have a full name (if the inode was already in the cache).
2117 *
2118 * When called on a directory inode, we must ensure that the inode only ever
2119 * has one dentry.  If a dentry is found, that is returned instead of
2120 * allocating a new one.
2121 *
2122 * On successful return, the reference to the inode has been transferred
2123 * to the dentry.  In case of an error the reference on the inode is released.
2124 * To make it easier to use in export operations a %NULL or IS_ERR inode may
2125 * be passed in and the error will be propagated to the return value,
2126 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
2127 */
2128struct dentry *d_obtain_alias(struct inode *inode)
2129{
2130	return __d_obtain_alias(inode, true);
2131}
2132EXPORT_SYMBOL(d_obtain_alias);
2133
2134/**
2135 * d_obtain_root - find or allocate a dentry for a given inode
2136 * @inode: inode to allocate the dentry for
2137 *
2138 * Obtain an IS_ROOT dentry for the root of a filesystem.
2139 *
2140 * We must ensure that directory inodes only ever have one dentry.  If a
2141 * dentry is found, that is returned instead of allocating a new one.
2142 *
2143 * On successful return, the reference to the inode has been transferred
2144 * to the dentry.  In case of an error the reference on the inode is
2145 * released.  A %NULL or IS_ERR inode may be passed in and will be the
2146 * error will be propagate to the return value, with a %NULL @inode
2147 * replaced by ERR_PTR(-ESTALE).
2148 */
2149struct dentry *d_obtain_root(struct inode *inode)
2150{
2151	return __d_obtain_alias(inode, false);
2152}
2153EXPORT_SYMBOL(d_obtain_root);
2154
2155/**
2156 * d_add_ci - lookup or allocate new dentry with case-exact name
2157 * @dentry: the negative dentry that was passed to the parent's lookup func
2158 * @inode:  the inode case-insensitive lookup has found
2159 * @name:   the case-exact name to be associated with the returned dentry
2160 *
2161 * This is to avoid filling the dcache with case-insensitive names to the
2162 * same inode, only the actual correct case is stored in the dcache for
2163 * case-insensitive filesystems.
2164 *
2165 * For a case-insensitive lookup match and if the case-exact dentry
2166 * already exists in the dcache, use it and return it.
2167 *
2168 * If no entry exists with the exact case name, allocate new dentry with
2169 * the exact case, and return the spliced entry.
2170 */
2171struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
2172			struct qstr *name)
2173{
2174	struct dentry *found, *res;
2175
2176	/*
2177	 * First check if a dentry matching the name already exists,
2178	 * if not go ahead and create it now.
2179	 */
2180	found = d_hash_and_lookup(dentry->d_parent, name);
2181	if (found) {
2182		iput(inode);
2183		return found;
2184	}
2185	if (d_in_lookup(dentry)) {
2186		found = d_alloc_parallel(dentry->d_parent, name,
2187					dentry->d_wait);
2188		if (IS_ERR(found) || !d_in_lookup(found)) {
2189			iput(inode);
2190			return found;
2191		}
2192	} else {
2193		found = d_alloc(dentry->d_parent, name);
2194		if (!found) {
2195			iput(inode);
2196			return ERR_PTR(-ENOMEM);
2197		} 
2198	}
2199	res = d_splice_alias(inode, found);
2200	if (res) {
2201		d_lookup_done(found);
2202		dput(found);
2203		return res;
2204	}
2205	return found;
2206}
2207EXPORT_SYMBOL(d_add_ci);
2208
2209/**
2210 * d_same_name - compare dentry name with case-exact name
2211 * @dentry: the negative dentry that was passed to the parent's lookup func
2212 * @parent: parent dentry
2213 * @name:   the case-exact name to be associated with the returned dentry
2214 *
2215 * Return: true if names are same, or false
2216 */
2217bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
2218		 const struct qstr *name)
2219{
2220	if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2221		if (dentry->d_name.len != name->len)
2222			return false;
2223		return dentry_cmp(dentry, name->name, name->len) == 0;
2224	}
2225	return parent->d_op->d_compare(dentry,
2226				       dentry->d_name.len, dentry->d_name.name,
2227				       name) == 0;
2228}
2229EXPORT_SYMBOL_GPL(d_same_name);
2230
2231/*
2232 * This is __d_lookup_rcu() when the parent dentry has
2233 * DCACHE_OP_COMPARE, which makes things much nastier.
2234 */
2235static noinline struct dentry *__d_lookup_rcu_op_compare(
2236	const struct dentry *parent,
2237	const struct qstr *name,
2238	unsigned *seqp)
2239{
2240	u64 hashlen = name->hash_len;
2241	struct hlist_bl_head *b = d_hash(hashlen);
2242	struct hlist_bl_node *node;
2243	struct dentry *dentry;
2244
2245	hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2246		int tlen;
2247		const char *tname;
2248		unsigned seq;
2249
2250seqretry:
2251		seq = raw_seqcount_begin(&dentry->d_seq);
2252		if (dentry->d_parent != parent)
2253			continue;
2254		if (d_unhashed(dentry))
2255			continue;
2256		if (dentry->d_name.hash != hashlen_hash(hashlen))
2257			continue;
2258		tlen = dentry->d_name.len;
2259		tname = dentry->d_name.name;
2260		/* we want a consistent (name,len) pair */
2261		if (read_seqcount_retry(&dentry->d_seq, seq)) {
2262			cpu_relax();
2263			goto seqretry;
2264		}
2265		if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0)
2266			continue;
2267		*seqp = seq;
2268		return dentry;
2269	}
2270	return NULL;
2271}
2272
2273/**
2274 * __d_lookup_rcu - search for a dentry (racy, store-free)
2275 * @parent: parent dentry
2276 * @name: qstr of name we wish to find
2277 * @seqp: returns d_seq value at the point where the dentry was found
2278 * Returns: dentry, or NULL
2279 *
2280 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
2281 * resolution (store-free path walking) design described in
2282 * Documentation/filesystems/path-lookup.txt.
2283 *
2284 * This is not to be used outside core vfs.
2285 *
2286 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
2287 * held, and rcu_read_lock held. The returned dentry must not be stored into
2288 * without taking d_lock and checking d_seq sequence count against @seq
2289 * returned here.
2290 *
2291 * Alternatively, __d_lookup_rcu may be called again to look up the child of
2292 * the returned dentry, so long as its parent's seqlock is checked after the
2293 * child is looked up. Thus, an interlocking stepping of sequence lock checks
2294 * is formed, giving integrity down the path walk.
2295 *
2296 * NOTE! The caller *has* to check the resulting dentry against the sequence
2297 * number we've returned before using any of the resulting dentry state!
2298 */
2299struct dentry *__d_lookup_rcu(const struct dentry *parent,
2300				const struct qstr *name,
2301				unsigned *seqp)
2302{
2303	u64 hashlen = name->hash_len;
2304	const unsigned char *str = name->name;
2305	struct hlist_bl_head *b = d_hash(hashlen);
2306	struct hlist_bl_node *node;
2307	struct dentry *dentry;
2308
2309	/*
2310	 * Note: There is significant duplication with __d_lookup_rcu which is
2311	 * required to prevent single threaded performance regressions
2312	 * especially on architectures where smp_rmb (in seqcounts) are costly.
2313	 * Keep the two functions in sync.
2314	 */
2315
2316	if (unlikely(parent->d_flags & DCACHE_OP_COMPARE))
2317		return __d_lookup_rcu_op_compare(parent, name, seqp);
2318
2319	/*
2320	 * The hash list is protected using RCU.
2321	 *
2322	 * Carefully use d_seq when comparing a candidate dentry, to avoid
2323	 * races with d_move().
2324	 *
2325	 * It is possible that concurrent renames can mess up our list
2326	 * walk here and result in missing our dentry, resulting in the
2327	 * false-negative result. d_lookup() protects against concurrent
2328	 * renames using rename_lock seqlock.
2329	 *
2330	 * See Documentation/filesystems/path-lookup.txt for more details.
2331	 */
2332	hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2333		unsigned seq;
2334
2335		/*
2336		 * The dentry sequence count protects us from concurrent
2337		 * renames, and thus protects parent and name fields.
2338		 *
2339		 * The caller must perform a seqcount check in order
2340		 * to do anything useful with the returned dentry.
2341		 *
2342		 * NOTE! We do a "raw" seqcount_begin here. That means that
2343		 * we don't wait for the sequence count to stabilize if it
2344		 * is in the middle of a sequence change. If we do the slow
2345		 * dentry compare, we will do seqretries until it is stable,
2346		 * and if we end up with a successful lookup, we actually
2347		 * want to exit RCU lookup anyway.
2348		 *
2349		 * Note that raw_seqcount_begin still *does* smp_rmb(), so
2350		 * we are still guaranteed NUL-termination of ->d_name.name.
2351		 */
2352		seq = raw_seqcount_begin(&dentry->d_seq);
2353		if (dentry->d_parent != parent)
2354			continue;
2355		if (dentry->d_name.hash_len != hashlen)
2356			continue;
2357		if (unlikely(dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0))
2358			continue;
2359		/*
2360		 * Check for the dentry being unhashed.
2361		 *
2362		 * As tempting as it is, we *can't* skip it because of a race window
2363		 * between us finding the dentry before it gets unhashed and loading
2364		 * the sequence counter after unhashing is finished.
2365		 *
2366		 * We can at least predict on it.
2367		 */
2368		if (unlikely(d_unhashed(dentry)))
2369			continue;
2370		*seqp = seq;
2371		return dentry;
2372	}
2373	return NULL;
2374}
2375
2376/**
2377 * d_lookup - search for a dentry
2378 * @parent: parent dentry
2379 * @name: qstr of name we wish to find
2380 * Returns: dentry, or NULL
2381 *
2382 * d_lookup searches the children of the parent dentry for the name in
2383 * question. If the dentry is found its reference count is incremented and the
2384 * dentry is returned. The caller must use dput to free the entry when it has
2385 * finished using it. %NULL is returned if the dentry does not exist.
2386 */
2387struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
2388{
2389	struct dentry *dentry;
2390	unsigned seq;
2391
2392	do {
2393		seq = read_seqbegin(&rename_lock);
2394		dentry = __d_lookup(parent, name);
2395		if (dentry)
2396			break;
2397	} while (read_seqretry(&rename_lock, seq));
2398	return dentry;
2399}
2400EXPORT_SYMBOL(d_lookup);
2401
2402/**
2403 * __d_lookup - search for a dentry (racy)
2404 * @parent: parent dentry
2405 * @name: qstr of name we wish to find
2406 * Returns: dentry, or NULL
2407 *
2408 * __d_lookup is like d_lookup, however it may (rarely) return a
2409 * false-negative result due to unrelated rename activity.
2410 *
2411 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
2412 * however it must be used carefully, eg. with a following d_lookup in
2413 * the case of failure.
2414 *
2415 * __d_lookup callers must be commented.
2416 */
2417struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
2418{
2419	unsigned int hash = name->hash;
2420	struct hlist_bl_head *b = d_hash(hash);
2421	struct hlist_bl_node *node;
2422	struct dentry *found = NULL;
2423	struct dentry *dentry;
2424
2425	/*
2426	 * Note: There is significant duplication with __d_lookup_rcu which is
2427	 * required to prevent single threaded performance regressions
2428	 * especially on architectures where smp_rmb (in seqcounts) are costly.
2429	 * Keep the two functions in sync.
2430	 */
2431
2432	/*
2433	 * The hash list is protected using RCU.
2434	 *
2435	 * Take d_lock when comparing a candidate dentry, to avoid races
2436	 * with d_move().
2437	 *
2438	 * It is possible that concurrent renames can mess up our list
2439	 * walk here and result in missing our dentry, resulting in the
2440	 * false-negative result. d_lookup() protects against concurrent
2441	 * renames using rename_lock seqlock.
2442	 *
2443	 * See Documentation/filesystems/path-lookup.txt for more details.
2444	 */
2445	rcu_read_lock();
2446	
2447	hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2448
2449		if (dentry->d_name.hash != hash)
2450			continue;
2451
2452		spin_lock(&dentry->d_lock);
2453		if (dentry->d_parent != parent)
2454			goto next;
2455		if (d_unhashed(dentry))
2456			goto next;
2457
2458		if (!d_same_name(dentry, parent, name))
2459			goto next;
2460
2461		dentry->d_lockref.count++;
2462		found = dentry;
2463		spin_unlock(&dentry->d_lock);
2464		break;
2465next:
2466		spin_unlock(&dentry->d_lock);
2467 	}
2468 	rcu_read_unlock();
2469
2470 	return found;
2471}
2472
2473/**
2474 * d_hash_and_lookup - hash the qstr then search for a dentry
2475 * @dir: Directory to search in
2476 * @name: qstr of name we wish to find
2477 *
2478 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
2479 */
2480struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2481{
2482	/*
2483	 * Check for a fs-specific hash function. Note that we must
2484	 * calculate the standard hash first, as the d_op->d_hash()
2485	 * routine may choose to leave the hash value unchanged.
2486	 */
2487	name->hash = full_name_hash(dir, name->name, name->len);
2488	if (dir->d_flags & DCACHE_OP_HASH) {
2489		int err = dir->d_op->d_hash(dir, name);
2490		if (unlikely(err < 0))
2491			return ERR_PTR(err);
2492	}
2493	return d_lookup(dir, name);
2494}
2495
2496/*
2497 * When a file is deleted, we have two options:
2498 * - turn this dentry into a negative dentry
2499 * - unhash this dentry and free it.
2500 *
2501 * Usually, we want to just turn this into
2502 * a negative dentry, but if anybody else is
2503 * currently using the dentry or the inode
2504 * we can't do that and we fall back on removing
2505 * it from the hash queues and waiting for
2506 * it to be deleted later when it has no users
2507 */
2508 
2509/**
2510 * d_delete - delete a dentry
2511 * @dentry: The dentry to delete
2512 *
2513 * Turn the dentry into a negative dentry if possible, otherwise
2514 * remove it from the hash queues so it can be deleted later
2515 */
2516 
2517void d_delete(struct dentry * dentry)
2518{
2519	struct inode *inode = dentry->d_inode;
2520
2521	spin_lock(&inode->i_lock);
2522	spin_lock(&dentry->d_lock);
2523	/*
2524	 * Are we the only user?
2525	 */
2526	if (dentry->d_lockref.count == 1) {
2527		if (dentry_negative_policy)
2528			__d_drop(dentry);
2529		dentry->d_flags &= ~DCACHE_CANT_MOUNT;
2530		dentry_unlink_inode(dentry);
2531	} else {
2532		__d_drop(dentry);
2533		spin_unlock(&dentry->d_lock);
2534		spin_unlock(&inode->i_lock);
2535	}
2536}
2537EXPORT_SYMBOL(d_delete);
2538
2539static void __d_rehash(struct dentry *entry)
2540{
2541	struct hlist_bl_head *b = d_hash(entry->d_name.hash);
2542
2543	hlist_bl_lock(b);
2544	hlist_bl_add_head_rcu(&entry->d_hash, b);
2545	hlist_bl_unlock(b);
2546}
2547
2548/**
2549 * d_rehash	- add an entry back to the hash
2550 * @entry: dentry to add to the hash
2551 *
2552 * Adds a dentry to the hash according to its name.
2553 */
2554 
2555void d_rehash(struct dentry * entry)
2556{
2557	spin_lock(&entry->d_lock);
2558	__d_rehash(entry);
2559	spin_unlock(&entry->d_lock);
2560}
2561EXPORT_SYMBOL(d_rehash);
2562
2563static inline unsigned start_dir_add(struct inode *dir)
2564{
2565	preempt_disable_nested();
2566	for (;;) {
2567		unsigned n = READ_ONCE(dir->i_dir_seq);
2568		if (!(n & 1) && try_cmpxchg(&dir->i_dir_seq, &n, n + 1))
2569			return n;
2570		cpu_relax();
2571	}
2572}
2573
2574static inline void end_dir_add(struct inode *dir, unsigned int n,
2575			       wait_queue_head_t *d_wait)
2576{
2577	smp_store_release(&dir->i_dir_seq, n + 2);
2578	preempt_enable_nested();
2579	if (wq_has_sleeper(d_wait))
2580		wake_up_all(d_wait);
2581}
2582
2583static void d_wait_lookup(struct dentry *dentry)
2584{
2585	if (d_in_lookup(dentry)) {
2586		DECLARE_WAITQUEUE(wait, current);
2587		add_wait_queue(dentry->d_wait, &wait);
2588		do {
2589			set_current_state(TASK_UNINTERRUPTIBLE);
2590			spin_unlock(&dentry->d_lock);
2591			schedule();
2592			spin_lock(&dentry->d_lock);
2593		} while (d_in_lookup(dentry));
2594	}
2595}
2596
2597struct dentry *d_alloc_parallel(struct dentry *parent,
2598				const struct qstr *name,
2599				wait_queue_head_t *wq)
2600{
2601	unsigned int hash = name->hash;
2602	struct hlist_bl_head *b = in_lookup_hash(parent, hash);
2603	struct hlist_bl_node *node;
2604	struct dentry *new = __d_alloc(parent->d_sb, name);
2605	struct dentry *dentry;
2606	unsigned seq, r_seq, d_seq;
2607
2608	if (unlikely(!new))
2609		return ERR_PTR(-ENOMEM);
2610
2611	new->d_flags |= DCACHE_PAR_LOOKUP;
2612	spin_lock(&parent->d_lock);
2613	new->d_parent = dget_dlock(parent);
2614	hlist_add_head(&new->d_sib, &parent->d_children);
2615	if (parent->d_flags & DCACHE_DISCONNECTED)
2616		new->d_flags |= DCACHE_DISCONNECTED;
2617	spin_unlock(&parent->d_lock);
2618
2619retry:
2620	rcu_read_lock();
2621	seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
2622	r_seq = read_seqbegin(&rename_lock);
2623	dentry = __d_lookup_rcu(parent, name, &d_seq);
2624	if (unlikely(dentry)) {
2625		if (!lockref_get_not_dead(&dentry->d_lockref)) {
2626			rcu_read_unlock();
2627			goto retry;
2628		}
2629		if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
2630			rcu_read_unlock();
2631			dput(dentry);
2632			goto retry;
2633		}
2634		rcu_read_unlock();
2635		dput(new);
2636		return dentry;
2637	}
2638	if (unlikely(read_seqretry(&rename_lock, r_seq))) {
2639		rcu_read_unlock();
2640		goto retry;
2641	}
2642
2643	if (unlikely(seq & 1)) {
2644		rcu_read_unlock();
2645		goto retry;
2646	}
2647
2648	hlist_bl_lock(b);
2649	if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
2650		hlist_bl_unlock(b);
2651		rcu_read_unlock();
2652		goto retry;
2653	}
2654	/*
2655	 * No changes for the parent since the beginning of d_lookup().
2656	 * Since all removals from the chain happen with hlist_bl_lock(),
2657	 * any potential in-lookup matches are going to stay here until
2658	 * we unlock the chain.  All fields are stable in everything
2659	 * we encounter.
2660	 */
2661	hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
2662		if (dentry->d_name.hash != hash)
2663			continue;
2664		if (dentry->d_parent != parent)
2665			continue;
2666		if (!d_same_name(dentry, parent, name))
2667			continue;
2668		hlist_bl_unlock(b);
2669		/* now we can try to grab a reference */
2670		if (!lockref_get_not_dead(&dentry->d_lockref)) {
2671			rcu_read_unlock();
2672			goto retry;
2673		}
2674
2675		rcu_read_unlock();
2676		/*
2677		 * somebody is likely to be still doing lookup for it;
2678		 * wait for them to finish
2679		 */
2680		spin_lock(&dentry->d_lock);
2681		d_wait_lookup(dentry);
2682		/*
2683		 * it's not in-lookup anymore; in principle we should repeat
2684		 * everything from dcache lookup, but it's likely to be what
2685		 * d_lookup() would've found anyway.  If it is, just return it;
2686		 * otherwise we really have to repeat the whole thing.
2687		 */
2688		if (unlikely(dentry->d_name.hash != hash))
2689			goto mismatch;
2690		if (unlikely(dentry->d_parent != parent))
2691			goto mismatch;
2692		if (unlikely(d_unhashed(dentry)))
2693			goto mismatch;
2694		if (unlikely(!d_same_name(dentry, parent, name)))
2695			goto mismatch;
2696		/* OK, it *is* a hashed match; return it */
2697		spin_unlock(&dentry->d_lock);
2698		dput(new);
2699		return dentry;
2700	}
2701	rcu_read_unlock();
2702	new->d_wait = wq;
2703	hlist_bl_add_head(&new->d_u.d_in_lookup_hash, b);
2704	hlist_bl_unlock(b);
2705	return new;
2706mismatch:
2707	spin_unlock(&dentry->d_lock);
2708	dput(dentry);
2709	goto retry;
2710}
2711EXPORT_SYMBOL(d_alloc_parallel);
2712
2713/*
2714 * - Unhash the dentry
2715 * - Retrieve and clear the waitqueue head in dentry
2716 * - Return the waitqueue head
2717 */
2718static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry)
2719{
2720	wait_queue_head_t *d_wait;
2721	struct hlist_bl_head *b;
2722
2723	lockdep_assert_held(&dentry->d_lock);
2724
2725	b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash);
2726	hlist_bl_lock(b);
2727	dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
2728	__hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
2729	d_wait = dentry->d_wait;
2730	dentry->d_wait = NULL;
2731	hlist_bl_unlock(b);
2732	INIT_HLIST_NODE(&dentry->d_u.d_alias);
2733	INIT_LIST_HEAD(&dentry->d_lru);
2734	return d_wait;
2735}
2736
2737void __d_lookup_unhash_wake(struct dentry *dentry)
2738{
2739	spin_lock(&dentry->d_lock);
2740	wake_up_all(__d_lookup_unhash(dentry));
2741	spin_unlock(&dentry->d_lock);
2742}
2743EXPORT_SYMBOL(__d_lookup_unhash_wake);
2744
2745/* inode->i_lock held if inode is non-NULL */
2746
2747static inline void __d_add(struct dentry *dentry, struct inode *inode,
2748			   const struct dentry_operations *ops)
2749{
2750	wait_queue_head_t *d_wait;
2751	struct inode *dir = NULL;
2752	unsigned n;
2753	spin_lock(&dentry->d_lock);
2754	if (unlikely(d_in_lookup(dentry))) {
2755		dir = dentry->d_parent->d_inode;
2756		n = start_dir_add(dir);
2757		d_wait = __d_lookup_unhash(dentry);
2758	}
2759	if (unlikely(ops))
2760		d_set_d_op(dentry, ops);
2761	if (inode) {
2762		unsigned add_flags = d_flags_for_inode(inode);
2763		hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2764		raw_write_seqcount_begin(&dentry->d_seq);
2765		__d_set_inode_and_type(dentry, inode, add_flags);
2766		raw_write_seqcount_end(&dentry->d_seq);
2767		fsnotify_update_flags(dentry);
2768	}
2769	__d_rehash(dentry);
2770	if (dir)
2771		end_dir_add(dir, n, d_wait);
2772	spin_unlock(&dentry->d_lock);
2773	if (inode)
2774		spin_unlock(&inode->i_lock);
2775}
2776
2777/**
2778 * d_add - add dentry to hash queues
2779 * @entry: dentry to add
2780 * @inode: The inode to attach to this dentry
2781 *
2782 * This adds the entry to the hash queues and initializes @inode.
2783 * The entry was actually filled in earlier during d_alloc().
2784 */
2785
2786void d_add(struct dentry *entry, struct inode *inode)
2787{
2788	if (inode) {
2789		security_d_instantiate(entry, inode);
2790		spin_lock(&inode->i_lock);
2791	}
2792	__d_add(entry, inode, NULL);
2793}
2794EXPORT_SYMBOL(d_add);
2795
2796struct dentry *d_make_persistent(struct dentry *dentry, struct inode *inode)
2797{
2798	WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
2799	WARN_ON(!inode);
2800	security_d_instantiate(dentry, inode);
2801	spin_lock(&inode->i_lock);
2802	spin_lock(&dentry->d_lock);
2803	__d_instantiate(dentry, inode);
2804	dentry->d_flags |= DCACHE_PERSISTENT;
2805	dget_dlock(dentry);
2806	if (d_unhashed(dentry))
2807		__d_rehash(dentry);
2808	spin_unlock(&dentry->d_lock);
2809	spin_unlock(&inode->i_lock);
2810	return dentry;
2811}
2812EXPORT_SYMBOL(d_make_persistent);
2813
2814static void swap_names(struct dentry *dentry, struct dentry *target)
2815{
2816	if (unlikely(dname_external(target))) {
2817		if (unlikely(dname_external(dentry))) {
2818			/*
2819			 * Both external: swap the pointers
2820			 */
2821			swap(target->__d_name.name, dentry->__d_name.name);
2822		} else {
2823			/*
2824			 * dentry:internal, target:external.  Steal target's
2825			 * storage and make target internal.
2826			 */
2827			dentry->__d_name.name = target->__d_name.name;
2828			target->d_shortname = dentry->d_shortname;
2829			target->__d_name.name = target->d_shortname.string;
2830		}
2831	} else {
2832		if (unlikely(dname_external(dentry))) {
2833			/*
2834			 * dentry:external, target:internal.  Give dentry's
2835			 * storage to target and make dentry internal
2836			 */
2837			target->__d_name.name = dentry->__d_name.name;
2838			dentry->d_shortname = target->d_shortname;
2839			dentry->__d_name.name = dentry->d_shortname.string;
2840		} else {
2841			/*
2842			 * Both are internal.
2843			 */
2844			for (int i = 0; i < DNAME_INLINE_WORDS; i++)
2845				swap(dentry->d_shortname.words[i],
2846				     target->d_shortname.words[i]);
2847		}
2848	}
2849	swap(dentry->__d_name.hash_len, target->__d_name.hash_len);
2850}
2851
2852static void copy_name(struct dentry *dentry, struct dentry *target)
2853{
2854	struct external_name *old_name = NULL;
2855	if (unlikely(dname_external(dentry)))
2856		old_name = external_name(dentry);
2857	if (unlikely(dname_external(target))) {
2858		atomic_inc(&external_name(target)->count);
2859		dentry->__d_name = target->__d_name;
2860	} else {
2861		dentry->d_shortname = target->d_shortname;
2862		dentry->__d_name.name = dentry->d_shortname.string;
2863		dentry->__d_name.hash_len = target->__d_name.hash_len;
2864	}
2865	if (old_name && likely(atomic_dec_and_test(&old_name->count)))
2866		kfree_rcu(old_name, head);
2867}
2868
2869/*
2870 * __d_move - move a dentry
2871 * @dentry: entry to move
2872 * @target: new dentry
2873 * @exchange: exchange the two dentries
2874 *
2875 * Update the dcache to reflect the move of a file name. Negative dcache
2876 * entries should not be moved in this way. Caller must hold rename_lock, the
2877 * i_rwsem of the source and target directories (exclusively), and the sb->
2878 * s_vfs_rename_mutex if they differ. See lock_rename().
2879 */
2880static void __d_move(struct dentry *dentry, struct dentry *target,
2881		     bool exchange)
2882{
2883	struct dentry *old_parent, *p;
2884	wait_queue_head_t *d_wait;
2885	struct inode *dir = NULL;
2886	unsigned n;
2887
2888	WARN_ON(!dentry->d_inode);
2889	if (WARN_ON(dentry == target))
2890		return;
2891
2892	BUG_ON(d_ancestor(target, dentry));
2893	old_parent = dentry->d_parent;
2894	p = d_ancestor(old_parent, target);
2895	if (IS_ROOT(dentry)) {
2896		BUG_ON(p);
2897		spin_lock(&target->d_parent->d_lock);
2898	} else if (!p) {
2899		/* target is not a descendent of dentry->d_parent */
2900		spin_lock(&target->d_parent->d_lock);
2901		spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
2902	} else {
2903		BUG_ON(p == dentry);
2904		spin_lock(&old_parent->d_lock);
2905		if (p != target)
2906			spin_lock_nested(&target->d_parent->d_lock,
2907					DENTRY_D_LOCK_NESTED);
2908	}
2909	spin_lock_nested(&dentry->d_lock, 2);
2910	spin_lock_nested(&target->d_lock, 3);
2911
2912	if (unlikely(d_in_lookup(target))) {
2913		dir = target->d_parent->d_inode;
2914		n = start_dir_add(dir);
2915		d_wait = __d_lookup_unhash(target);
2916	}
2917
2918	write_seqcount_begin(&dentry->d_seq);
2919	write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2920
2921	/* unhash both */
2922	if (!d_unhashed(dentry))
2923		___d_drop(dentry);
2924	if (!d_unhashed(target))
2925		___d_drop(target);
2926
2927	/* ... and switch them in the tree */
2928	dentry->d_parent = target->d_parent;
2929	if (!exchange) {
2930		copy_name(dentry, target);
2931		target->d_hash.pprev = NULL;
2932		dentry->d_parent->d_lockref.count++;
2933		if (dentry != old_parent) /* wasn't IS_ROOT */
2934			WARN_ON(!--old_parent->d_lockref.count);
2935	} else {
2936		target->d_parent = old_parent;
2937		swap_names(dentry, target);
2938		if (!hlist_unhashed(&target->d_sib))
2939			__hlist_del(&target->d_sib);
2940		hlist_add_head(&target->d_sib, &target->d_parent->d_children);
2941		__d_rehash(target);
2942		fsnotify_update_flags(target);
2943	}
2944	if (!hlist_unhashed(&dentry->d_sib))
2945		__hlist_del(&dentry->d_sib);
2946	hlist_add_head(&dentry->d_sib, &dentry->d_parent->d_children);
2947	__d_rehash(dentry);
2948	fsnotify_update_flags(dentry);
2949	fscrypt_handle_d_move(dentry);
2950
2951	write_seqcount_end(&target->d_seq);
2952	write_seqcount_end(&dentry->d_seq);
2953
2954	if (dir)
2955		end_dir_add(dir, n, d_wait);
2956
2957	if (dentry->d_parent != old_parent)
2958		spin_unlock(&dentry->d_parent->d_lock);
2959	if (dentry != old_parent)
2960		spin_unlock(&old_parent->d_lock);
2961	spin_unlock(&target->d_lock);
2962	spin_unlock(&dentry->d_lock);
2963}
2964
2965/*
2966 * d_move - move a dentry
2967 * @dentry: entry to move
2968 * @target: new dentry
2969 *
2970 * Update the dcache to reflect the move of a file name. Negative
2971 * dcache entries should not be moved in this way. See the locking
2972 * requirements for __d_move.
2973 */
2974void d_move(struct dentry *dentry, struct dentry *target)
2975{
2976	write_seqlock(&rename_lock);
2977	__d_move(dentry, target, false);
2978	write_sequnlock(&rename_lock);
2979}
2980EXPORT_SYMBOL(d_move);
2981
2982/*
2983 * d_exchange - exchange two dentries
2984 * @dentry1: first dentry
2985 * @dentry2: second dentry
2986 */
2987void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
2988{
2989	write_seqlock(&rename_lock);
2990
2991	WARN_ON(!dentry1->d_inode);
2992	WARN_ON(!dentry2->d_inode);
2993	WARN_ON(IS_ROOT(dentry1));
2994	WARN_ON(IS_ROOT(dentry2));
2995
2996	__d_move(dentry1, dentry2, true);
2997
2998	write_sequnlock(&rename_lock);
2999}
3000EXPORT_SYMBOL(d_exchange);
3001
3002/**
3003 * d_ancestor - search for an ancestor
3004 * @p1: ancestor dentry
3005 * @p2: child dentry
3006 *
3007 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
3008 * an ancestor of p2, else NULL.
3009 */
3010struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
3011{
3012	struct dentry *p;
3013
3014	for (p = p2; !IS_ROOT(p); p = p->d_parent) {
3015		if (p->d_parent == p1)
3016			return p;
3017	}
3018	return NULL;
3019}
3020
3021/*
3022 * This helper attempts to cope with remotely renamed directories
3023 *
3024 * It assumes that the caller is already holding
3025 * dentry->d_parent->d_inode->i_rwsem, and rename_lock
3026 *
3027 * Note: If ever the locking in lock_rename() changes, then please
3028 * remember to update this too...
3029 */
3030static int __d_unalias(struct dentry *dentry, struct dentry *alias)
3031{
3032	struct mutex *m1 = NULL;
3033	struct rw_semaphore *m2 = NULL;
3034	int ret = -ESTALE;
3035
3036	/* If alias and dentry share a parent, then no extra locks required */
3037	if (alias->d_parent == dentry->d_parent)
3038		goto out_unalias;
3039
3040	/* See lock_rename() */
3041	if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
3042		goto out_err;
3043	m1 = &dentry->d_sb->s_vfs_rename_mutex;
3044	if (!inode_trylock_shared(alias->d_parent->d_inode))
3045		goto out_err;
3046	m2 = &alias->d_parent->d_inode->i_rwsem;
3047out_unalias:
3048	if (alias->d_op && alias->d_op->d_unalias_trylock &&
3049	    !alias->d_op->d_unalias_trylock(alias))
3050		goto out_err;
3051	__d_move(alias, dentry, false);
3052	if (alias->d_op && alias->d_op->d_unalias_unlock)
3053		alias->d_op->d_unalias_unlock(alias);
3054	ret = 0;
3055out_err:
3056	if (m2)
3057		up_read(m2);
3058	if (m1)
3059		mutex_unlock(m1);
3060	return ret;
3061}
3062
3063struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry,
3064				  const struct dentry_operations *ops)
3065{
3066	if (IS_ERR(inode))
3067		return ERR_CAST(inode);
3068
3069	BUG_ON(!d_unhashed(dentry));
3070
3071	if (!inode)
3072		goto out;
3073
3074	security_d_instantiate(dentry, inode);
3075	spin_lock(&inode->i_lock);
3076	if (S_ISDIR(inode->i_mode)) {
3077		struct dentry *new = __d_find_any_alias(inode);
3078		if (unlikely(new)) {
3079			/* The reference to new ensures it remains an alias */
3080			spin_unlock(&inode->i_lock);
3081			write_seqlock(&rename_lock);
3082			if (unlikely(d_ancestor(new, dentry))) {
3083				write_sequnlock(&rename_lock);
3084				dput(new);
3085				new = ERR_PTR(-ELOOP);
3086				pr_warn_ratelimited(
3087					"VFS: Lookup of '%s' in %s %s"
3088					" would have caused loop\n",
3089					dentry->d_name.name,
3090					inode->i_sb->s_type->name,
3091					inode->i_sb->s_id);
3092			} else if (!IS_ROOT(new)) {
3093				struct dentry *old_parent = dget(new->d_parent);
3094				int err = __d_unalias(dentry, new);
3095				write_sequnlock(&rename_lock);
3096				if (err) {
3097					dput(new);
3098					new = ERR_PTR(err);
3099				}
3100				dput(old_parent);
3101			} else {
3102				__d_move(new, dentry, false);
3103				write_sequnlock(&rename_lock);
3104			}
3105			iput(inode);
3106			return new;
3107		}
3108	}
3109out:
3110	__d_add(dentry, inode, ops);
3111	return NULL;
3112}
3113
3114/**
3115 * d_splice_alias - splice a disconnected dentry into the tree if one exists
3116 * @inode:  the inode which may have a disconnected dentry
3117 * @dentry: a negative dentry which we want to point to the inode.
3118 *
3119 * If inode is a directory and has an IS_ROOT alias, then d_move that in
3120 * place of the given dentry and return it, else simply d_add the inode
3121 * to the dentry and return NULL.
3122 *
3123 * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
3124 * we should error out: directories can't have multiple aliases.
3125 *
3126 * This is needed in the lookup routine of any filesystem that is exportable
3127 * (via knfsd) so that we can build dcache paths to directories effectively.
3128 *
3129 * If a dentry was found and moved, then it is returned.  Otherwise NULL
3130 * is returned.  This matches the expected return value of ->lookup.
3131 *
3132 * Cluster filesystems may call this function with a negative, hashed dentry.
3133 * In that case, we know that the inode will be a regular file, and also this
3134 * will only occur during atomic_open. So we need to check for the dentry
3135 * being already hashed only in the final case.
3136 */
3137struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
3138{
3139	return d_splice_alias_ops(inode, dentry, NULL);
3140}
3141EXPORT_SYMBOL(d_splice_alias);
3142
3143/*
3144 * Test whether new_dentry is a subdirectory of old_dentry.
3145 *
3146 * Trivially implemented using the dcache structure
3147 */
3148
3149/**
3150 * is_subdir - is new dentry a subdirectory of old_dentry
3151 * @new_dentry: new dentry
3152 * @old_dentry: old dentry
3153 *
3154 * Returns true if new_dentry is a subdirectory of the parent (at any depth).
3155 * Returns false otherwise.
3156 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
3157 */
3158  
3159bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
3160{
3161	bool subdir;
3162	unsigned seq;
3163
3164	if (new_dentry == old_dentry)
3165		return true;
3166
3167	/* Access d_parent under rcu as d_move() may change it. */
3168	rcu_read_lock();
3169	seq = read_seqbegin(&rename_lock);
3170	subdir = d_ancestor(old_dentry, new_dentry);
3171	 /* Try lockless once... */
3172	if (read_seqretry(&rename_lock, seq)) {
3173		/* ...else acquire lock for progress even on deep chains. */
3174		read_seqlock_excl(&rename_lock);
3175		subdir = d_ancestor(old_dentry, new_dentry);
3176		read_sequnlock_excl(&rename_lock);
3177	}
3178	rcu_read_unlock();
3179	return subdir;
3180}
3181EXPORT_SYMBOL(is_subdir);
3182
3183void d_mark_tmpfile(struct file *file, struct inode *inode)
3184{
3185	struct dentry *dentry = file->f_path.dentry;
3186
3187	BUG_ON(dname_external(dentry) ||
3188		!hlist_unhashed(&dentry->d_u.d_alias) ||
3189		!d_unlinked(dentry));
3190	spin_lock(&dentry->d_parent->d_lock);
3191	spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3192	dentry->__d_name.len = sprintf(dentry->d_shortname.string, "#%llu",
3193				(unsigned long long)inode->i_ino);
3194	spin_unlock(&dentry->d_lock);
3195	spin_unlock(&dentry->d_parent->d_lock);
3196}
3197EXPORT_SYMBOL(d_mark_tmpfile);
3198
3199void d_tmpfile(struct file *file, struct inode *inode)
3200{
3201	struct dentry *dentry = file->f_path.dentry;
3202
3203	inode_dec_link_count(inode);
3204	d_mark_tmpfile(file, inode);
3205	d_instantiate(dentry, inode);
3206}
3207EXPORT_SYMBOL(d_tmpfile);
3208
3209/*
3210 * Obtain inode number of the parent dentry.
3211 */
3212ino_t d_parent_ino(struct dentry *dentry)
3213{
3214	struct dentry *parent;
3215	struct inode *iparent;
3216	unsigned seq;
3217	ino_t ret;
3218
3219	scoped_guard(rcu) {
3220		seq = raw_seqcount_begin(&dentry->d_seq);
3221		parent = READ_ONCE(dentry->d_parent);
3222		iparent = d_inode_rcu(parent);
3223		if (likely(iparent)) {
3224			ret = iparent->i_ino;
3225			if (!read_seqcount_retry(&dentry->d_seq, seq))
3226				return ret;
3227		}
3228	}
3229
3230	spin_lock(&dentry->d_lock);
3231	ret = dentry->d_parent->d_inode->i_ino;
3232	spin_unlock(&dentry->d_lock);
3233	return ret;
3234}
3235EXPORT_SYMBOL(d_parent_ino);
3236
3237static __initdata unsigned long dhash_entries;
3238static int __init set_dhash_entries(char *str)
3239{
3240	if (!str)
3241		return 0;
3242	dhash_entries = simple_strtoul(str, &str, 0);
3243	return 1;
3244}
3245__setup("dhash_entries=", set_dhash_entries);
3246
3247static void __init dcache_init_early(void)
3248{
3249	/* If hashes are distributed across NUMA nodes, defer
3250	 * hash allocation until vmalloc space is available.
3251	 */
3252	if (hashdist)
3253		return;
3254
3255	dentry_hashtable =
3256		alloc_large_system_hash("Dentry cache",
3257					sizeof(struct hlist_bl_head),
3258					dhash_entries,
3259					13,
3260					HASH_EARLY | HASH_ZERO,
3261					&d_hash_shift,
3262					NULL,
3263					2,
3264					0);
3265	d_hash_shift = 32 - d_hash_shift;
3266
3267	runtime_const_init(shift, d_hash_shift);
3268	runtime_const_init(ptr, dentry_hashtable);
3269}
3270
3271static void __init dcache_init(void)
3272{
3273	/*
3274	 * A constructor could be added for stable state like the lists,
3275	 * but it is probably not worth it because of the cache nature
3276	 * of the dcache.
3277	 */
3278	__dentry_cache = KMEM_CACHE_USERCOPY(dentry,
3279		SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT,
3280		d_shortname.string);
3281	runtime_const_init(ptr, __dentry_cache);
3282
3283	/* Hash may have been set up in dcache_init_early */
3284	if (!hashdist)
3285		return;
3286
3287	dentry_hashtable =
3288		alloc_large_system_hash("Dentry cache",
3289					sizeof(struct hlist_bl_head),
3290					dhash_entries,
3291					13,
3292					HASH_ZERO,
3293					&d_hash_shift,
3294					NULL,
3295					2,
3296					0);
3297	d_hash_shift = 32 - d_hash_shift;
3298
3299	runtime_const_init(shift, d_hash_shift);
3300	runtime_const_init(ptr, dentry_hashtable);
3301}
3302
3303/* SLAB cache for __getname() consumers */
3304struct kmem_cache *names_cachep __ro_after_init;
3305EXPORT_SYMBOL(names_cachep);
3306
3307void __init vfs_caches_init_early(void)
3308{
3309	int i;
3310
3311	for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
3312		INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
3313
3314	dcache_init_early();
3315	inode_init_early();
3316}
3317
3318void __init vfs_caches_init(void)
3319{
3320	names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
3321			SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
3322
3323	dcache_init();
3324	inode_init();
3325	files_init();
3326	files_maxfiles_init();
3327	mnt_init();
3328	bdev_cache_init();
3329	chrdev_init();
3330}