glock.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) Sistina Software, Inc.  1997-2003 All rights reserved.
   4 * Copyright (C) 2004-2008 Red Hat, Inc.  All rights reserved.
   5 */
   6
   7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9#include <linux/sched.h>
  10#include <linux/slab.h>
  11#include <linux/spinlock.h>
  12#include <linux/buffer_head.h>
  13#include <linux/delay.h>
  14#include <linux/sort.h>
  15#include <linux/hash.h>
  16#include <linux/jhash.h>
  17#include <linux/kallsyms.h>
  18#include <linux/gfs2_ondisk.h>
  19#include <linux/list.h>
  20#include <linux/wait.h>
  21#include <linux/module.h>
  22#include <linux/uaccess.h>
  23#include <linux/seq_file.h>
  24#include <linux/debugfs.h>
  25#include <linux/kthread.h>
  26#include <linux/freezer.h>
  27#include <linux/workqueue.h>
  28#include <linux/jiffies.h>
  29#include <linux/rcupdate.h>
  30#include <linux/rculist_bl.h>
  31#include <linux/bit_spinlock.h>
  32#include <linux/percpu.h>
  33#include <linux/list_sort.h>
  34#include <linux/lockref.h>
  35#include <linux/rhashtable.h>
  36#include <linux/pid_namespace.h>
  37#include <linux/file.h>
  38#include <linux/random.h>
  39
  40#include "gfs2.h"
  41#include "incore.h"
  42#include "glock.h"
  43#include "glops.h"
  44#include "inode.h"
  45#include "lops.h"
  46#include "meta_io.h"
  47#include "quota.h"
  48#include "super.h"
  49#include "util.h"
  50#include "bmap.h"
  51#define CREATE_TRACE_POINTS
  52#include "trace_gfs2.h"
  53
  54struct gfs2_glock_iter {
  55	struct gfs2_sbd *sdp;		/* incore superblock           */
  56	struct rhashtable_iter hti;	/* rhashtable iterator         */
  57	struct gfs2_glock *gl;		/* current glock struct        */
  58	loff_t last_pos;		/* last position               */
  59};
  60
  61typedef void (*glock_examiner) (struct gfs2_glock * gl);
  62
  63static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
  64static void request_demote(struct gfs2_glock *gl, unsigned int state,
  65			   unsigned long delay, bool remote);
  66
  67static struct dentry *gfs2_root;
  68static LIST_HEAD(lru_list);
  69static atomic_t lru_count = ATOMIC_INIT(0);
  70static DEFINE_SPINLOCK(lru_lock);
  71
  72#define GFS2_GL_HASH_SHIFT      15
  73#define GFS2_GL_HASH_SIZE       BIT(GFS2_GL_HASH_SHIFT)
  74
  75static const struct rhashtable_params ht_parms = {
  76	.nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
  77	.key_len = offsetofend(struct lm_lockname, ln_type),
  78	.key_offset = offsetof(struct gfs2_glock, gl_name),
  79	.head_offset = offsetof(struct gfs2_glock, gl_node),
  80};
  81
  82static struct rhashtable gl_hash_table;
  83
  84#define GLOCK_WAIT_TABLE_BITS 12
  85#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
  86static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
  87
  88struct wait_glock_queue {
  89	struct lm_lockname *name;
  90	wait_queue_entry_t wait;
  91};
  92
  93static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
  94			       int sync, void *key)
  95{
  96	struct wait_glock_queue *wait_glock =
  97		container_of(wait, struct wait_glock_queue, wait);
  98	struct lm_lockname *wait_name = wait_glock->name;
  99	struct lm_lockname *wake_name = key;
 100
 101	if (wake_name->ln_sbd != wait_name->ln_sbd ||
 102	    wake_name->ln_number != wait_name->ln_number ||
 103	    wake_name->ln_type != wait_name->ln_type)
 104		return 0;
 105	return autoremove_wake_function(wait, mode, sync, key);
 106}
 107
 108static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
 109{
 110	u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
 111
 112	return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
 113}
 114
 115/**
 116 * wake_up_glock  -  Wake up waiters on a glock
 117 * @gl: the glock
 118 */
 119static void wake_up_glock(struct gfs2_glock *gl)
 120{
 121	wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
 122
 123	if (waitqueue_active(wq))
 124		__wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
 125}
 126
 127static void gfs2_glock_dealloc(struct rcu_head *rcu)
 128{
 129	struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
 130
 131	kfree(gl->gl_lksb.sb_lvbptr);
 132	if (gl->gl_ops->go_flags & GLOF_ASPACE) {
 133		struct gfs2_glock_aspace *gla =
 134			container_of(gl, struct gfs2_glock_aspace, glock);
 135		kmem_cache_free(gfs2_glock_aspace_cachep, gla);
 136	} else
 137		kmem_cache_free(gfs2_glock_cachep, gl);
 138}
 139
 140static void __gfs2_glock_free(struct gfs2_glock *gl)
 141{
 142	rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
 143	smp_mb();
 144	wake_up_glock(gl);
 145	call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
 146}
 147
 148void gfs2_glock_free(struct gfs2_glock *gl) {
 149	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 150
 151	__gfs2_glock_free(gl);
 152	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
 153		wake_up(&sdp->sd_kill_wait);
 154}
 155
 156void gfs2_glock_free_later(struct gfs2_glock *gl) {
 157	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 158
 159	spin_lock(&lru_lock);
 160	list_add(&gl->gl_lru, &sdp->sd_dead_glocks);
 161	spin_unlock(&lru_lock);
 162	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
 163		wake_up(&sdp->sd_kill_wait);
 164}
 165
 166static void gfs2_free_dead_glocks(struct gfs2_sbd *sdp)
 167{
 168	struct list_head *list = &sdp->sd_dead_glocks;
 169
 170	while(!list_empty(list)) {
 171		struct gfs2_glock *gl;
 172
 173		gl = list_first_entry(list, struct gfs2_glock, gl_lru);
 174		list_del_init(&gl->gl_lru);
 175		__gfs2_glock_free(gl);
 176	}
 177}
 178
 179/**
 180 * gfs2_glock_hold() - increment reference count on glock
 181 * @gl: The glock to hold
 182 *
 183 */
 184
 185struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl)
 186{
 187	GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
 188	lockref_get(&gl->gl_lockref);
 189	return gl;
 190}
 191
 192static void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
 193{
 194	spin_lock(&lru_lock);
 195	list_move_tail(&gl->gl_lru, &lru_list);
 196
 197	if (!test_bit(GLF_LRU, &gl->gl_flags)) {
 198		set_bit(GLF_LRU, &gl->gl_flags);
 199		atomic_inc(&lru_count);
 200	}
 201
 202	spin_unlock(&lru_lock);
 203}
 204
 205static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
 206{
 207	spin_lock(&lru_lock);
 208	if (test_bit(GLF_LRU, &gl->gl_flags)) {
 209		list_del_init(&gl->gl_lru);
 210		atomic_dec(&lru_count);
 211		clear_bit(GLF_LRU, &gl->gl_flags);
 212	}
 213	spin_unlock(&lru_lock);
 214}
 215
 216/*
 217 * Enqueue the glock on the work queue.  Passes one glock reference on to the
 218 * work queue.
 219 */
 220static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
 221	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 222
 223	if (!queue_delayed_work(sdp->sd_glock_wq, &gl->gl_work, delay)) {
 224		/*
 225		 * We are holding the lockref spinlock, and the work was still
 226		 * queued above.  The queued work (glock_work_func) takes that
 227		 * spinlock before dropping its glock reference(s), so it
 228		 * cannot have dropped them in the meantime.
 229		 */
 230		GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
 231		gl->gl_lockref.count--;
 232	}
 233}
 234
 235static void __gfs2_glock_put(struct gfs2_glock *gl)
 236{
 237	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 238	struct address_space *mapping = gfs2_glock2aspace(gl);
 239
 240	lockref_mark_dead(&gl->gl_lockref);
 241	spin_unlock(&gl->gl_lockref.lock);
 242	gfs2_glock_remove_from_lru(gl);
 243	GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
 244	if (mapping) {
 245		truncate_inode_pages_final(mapping);
 246		if (!gfs2_withdrawn(sdp))
 247			GLOCK_BUG_ON(gl, !mapping_empty(mapping));
 248	}
 249	trace_gfs2_glock_put(gl);
 250	sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
 251}
 252
 253static bool __gfs2_glock_put_or_lock(struct gfs2_glock *gl)
 254{
 255	if (lockref_put_or_lock(&gl->gl_lockref))
 256		return true;
 257	GLOCK_BUG_ON(gl, gl->gl_lockref.count != 1);
 258	if (gl->gl_state != LM_ST_UNLOCKED) {
 259		gl->gl_lockref.count--;
 260		gfs2_glock_add_to_lru(gl);
 261		spin_unlock(&gl->gl_lockref.lock);
 262		return true;
 263	}
 264	return false;
 265}
 266
 267/**
 268 * gfs2_glock_put() - Decrement reference count on glock
 269 * @gl: The glock to put
 270 *
 271 */
 272
 273void gfs2_glock_put(struct gfs2_glock *gl)
 274{
 275	if (__gfs2_glock_put_or_lock(gl))
 276		return;
 277
 278	__gfs2_glock_put(gl);
 279}
 280
 281/*
 282 * gfs2_glock_put_async - Decrement reference count without sleeping
 283 * @gl: The glock to put
 284 *
 285 * Decrement the reference count on glock immediately unless it is the last
 286 * reference.  Defer putting the last reference to work queue context.
 287 */
 288void gfs2_glock_put_async(struct gfs2_glock *gl)
 289{
 290	if (__gfs2_glock_put_or_lock(gl))
 291		return;
 292
 293	gfs2_glock_queue_work(gl, 0);
 294	spin_unlock(&gl->gl_lockref.lock);
 295}
 296
 297/**
 298 * may_grant - check if it's ok to grant a new lock
 299 * @gl: The glock
 300 * @current_gh: One of the current holders of @gl
 301 * @gh: The lock request which we wish to grant
 302 *
 303 * With our current compatibility rules, if a glock has one or more active
 304 * holders (HIF_HOLDER flag set), any of those holders can be passed in as
 305 * @current_gh; they are all the same as far as compatibility with the new @gh
 306 * goes.
 307 *
 308 * Returns true if it's ok to grant the lock.
 309 */
 310
 311static inline bool may_grant(struct gfs2_glock *gl,
 312			     struct gfs2_holder *current_gh,
 313			     struct gfs2_holder *gh)
 314{
 315	if (current_gh) {
 316		GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, &current_gh->gh_iflags));
 317
 318		switch(current_gh->gh_state) {
 319		case LM_ST_EXCLUSIVE:
 320			/*
 321			 * Here we make a special exception to grant holders
 322			 * who agree to share the EX lock with other holders
 323			 * who also have the bit set. If the original holder
 324			 * has the LM_FLAG_NODE_SCOPE bit set, we grant more
 325			 * holders with the bit set.
 326			 */
 327			return gh->gh_state == LM_ST_EXCLUSIVE &&
 328			       (current_gh->gh_flags & LM_FLAG_NODE_SCOPE) &&
 329			       (gh->gh_flags & LM_FLAG_NODE_SCOPE);
 330
 331		case LM_ST_SHARED:
 332		case LM_ST_DEFERRED:
 333			return gh->gh_state == current_gh->gh_state;
 334
 335		default:
 336			return false;
 337		}
 338	}
 339
 340	if (gl->gl_state == gh->gh_state)
 341		return true;
 342	if (gh->gh_flags & GL_EXACT)
 343		return false;
 344	if (gl->gl_state == LM_ST_EXCLUSIVE) {
 345		return gh->gh_state == LM_ST_SHARED ||
 346		       gh->gh_state == LM_ST_DEFERRED;
 347	}
 348	if (gh->gh_flags & LM_FLAG_ANY)
 349		return gl->gl_state != LM_ST_UNLOCKED;
 350	return false;
 351}
 352
 353static void gfs2_holder_wake(struct gfs2_holder *gh)
 354{
 355	clear_bit(HIF_WAIT, &gh->gh_iflags);
 356	smp_mb__after_atomic();
 357	wake_up_bit(&gh->gh_iflags, HIF_WAIT);
 358	if (gh->gh_flags & GL_ASYNC) {
 359		struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
 360
 361		wake_up(&sdp->sd_async_glock_wait);
 362	}
 363}
 364
 365/**
 366 * do_error - Something unexpected has happened during a lock request
 367 * @gl: The glock
 368 * @ret: The status from the DLM
 369 */
 370
 371static void do_error(struct gfs2_glock *gl, const int ret)
 372{
 373	struct gfs2_holder *gh, *tmp;
 374
 375	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
 376		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
 377			continue;
 378		if (ret & LM_OUT_ERROR)
 379			gh->gh_error = -EIO;
 380		else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
 381			gh->gh_error = GLR_TRYFAILED;
 382		else
 383			continue;
 384		list_del_init(&gh->gh_list);
 385		trace_gfs2_glock_queue(gh, 0);
 386		gfs2_holder_wake(gh);
 387	}
 388}
 389
 390/**
 391 * find_first_holder - find the first "holder" gh
 392 * @gl: the glock
 393 */
 394
 395static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
 396{
 397	struct gfs2_holder *gh;
 398
 399	if (!list_empty(&gl->gl_holders)) {
 400		gh = list_first_entry(&gl->gl_holders, struct gfs2_holder,
 401				      gh_list);
 402		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
 403			return gh;
 404	}
 405	return NULL;
 406}
 407
 408/*
 409 * gfs2_instantiate - Call the glops instantiate function
 410 * @gh: The glock holder
 411 *
 412 * Returns: 0 if instantiate was successful, or error.
 413 */
 414int gfs2_instantiate(struct gfs2_holder *gh)
 415{
 416	struct gfs2_glock *gl = gh->gh_gl;
 417	const struct gfs2_glock_operations *glops = gl->gl_ops;
 418	int ret;
 419
 420again:
 421	if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags))
 422		goto done;
 423
 424	/*
 425	 * Since we unlock the lockref lock, we set a flag to indicate
 426	 * instantiate is in progress.
 427	 */
 428	if (test_and_set_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags)) {
 429		wait_on_bit(&gl->gl_flags, GLF_INSTANTIATE_IN_PROG,
 430			    TASK_UNINTERRUPTIBLE);
 431		/*
 432		 * Here we just waited for a different instantiate to finish.
 433		 * But that may not have been successful, as when a process
 434		 * locks an inode glock _before_ it has an actual inode to
 435		 * instantiate into. So we check again. This process might
 436		 * have an inode to instantiate, so might be successful.
 437		 */
 438		goto again;
 439	}
 440
 441	ret = glops->go_instantiate(gl);
 442	if (!ret)
 443		clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags);
 444	clear_and_wake_up_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags);
 445	if (ret)
 446		return ret;
 447
 448done:
 449	if (glops->go_held)
 450		return glops->go_held(gh);
 451	return 0;
 452}
 453
 454/**
 455 * do_promote - promote as many requests as possible on the current queue
 456 * @gl: The glock
 457 */
 458
 459static void do_promote(struct gfs2_glock *gl)
 460{
 461	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 462	struct gfs2_holder *gh, *current_gh;
 463
 464	if (gfs2_withdrawn(sdp)) {
 465		do_error(gl, LM_OUT_ERROR);
 466		return;
 467	}
 468
 469	current_gh = find_first_holder(gl);
 470	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
 471		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
 472			continue;
 473		if (!may_grant(gl, current_gh, gh)) {
 474			/*
 475			 * If we get here, it means we may not grant this
 476			 * holder for some reason.
 477			 */
 478			if (current_gh)
 479				do_error(gl, 0); /* Fail queued try locks */
 480			break;
 481		}
 482		set_bit(HIF_HOLDER, &gh->gh_iflags);
 483		trace_gfs2_promote(gh);
 484		gfs2_holder_wake(gh);
 485		if (!current_gh)
 486			current_gh = gh;
 487	}
 488}
 489
 490/**
 491 * find_first_waiter - find the first gh that's waiting for the glock
 492 * @gl: the glock
 493 */
 494
 495static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
 496{
 497	struct gfs2_holder *gh;
 498
 499	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
 500		if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
 501			return gh;
 502	}
 503	return NULL;
 504}
 505
 506/**
 507 * find_last_waiter - find the last gh that's waiting for the glock
 508 * @gl: the glock
 509 *
 510 * This also is a fast way of finding out if there are any waiters.
 511 */
 512
 513static inline struct gfs2_holder *find_last_waiter(const struct gfs2_glock *gl)
 514{
 515	struct gfs2_holder *gh;
 516
 517	if (list_empty(&gl->gl_holders))
 518		return NULL;
 519	gh = list_last_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
 520	return test_bit(HIF_HOLDER, &gh->gh_iflags) ? NULL : gh;
 521}
 522
 523/**
 524 * state_change - record that the glock is now in a different state
 525 * @gl: the glock
 526 * @new_state: the new state
 527 */
 528
 529static void state_change(struct gfs2_glock *gl, unsigned int new_state)
 530{
 531	if (new_state != gl->gl_target)
 532		/* shorten our minimum hold time */
 533		gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
 534				       GL_GLOCK_MIN_HOLD);
 535	gl->gl_state = new_state;
 536	gl->gl_tchange = jiffies;
 537}
 538
 539static void gfs2_set_demote(int nr, struct gfs2_glock *gl)
 540{
 541	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 542
 543	set_bit(nr, &gl->gl_flags);
 544	smp_mb();
 545	wake_up(&sdp->sd_async_glock_wait);
 546}
 547
 548static void gfs2_demote_wake(struct gfs2_glock *gl)
 549{
 550	gl->gl_demote_state = LM_ST_EXCLUSIVE;
 551	clear_bit(GLF_DEMOTE, &gl->gl_flags);
 552	smp_mb__after_atomic();
 553	wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
 554}
 555
 556/**
 557 * finish_xmote - The DLM has replied to one of our lock requests
 558 * @gl: The glock
 559 * @ret: The status from the DLM
 560 *
 561 */
 562
 563static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
 564{
 565	const struct gfs2_glock_operations *glops = gl->gl_ops;
 566
 567	if (!(ret & ~LM_OUT_ST_MASK)) {
 568		unsigned state = ret & LM_OUT_ST_MASK;
 569
 570		trace_gfs2_glock_state_change(gl, state);
 571		state_change(gl, state);
 572	}
 573
 574	/* Demote to UN request arrived during demote to SH or DF */
 575	if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
 576	    gl->gl_state != LM_ST_UNLOCKED &&
 577	    gl->gl_demote_state == LM_ST_UNLOCKED)
 578		gl->gl_target = LM_ST_UNLOCKED;
 579
 580	/* Check for state != intended state */
 581	if (unlikely(gl->gl_state != gl->gl_target)) {
 582		struct gfs2_holder *gh = find_first_waiter(gl);
 583
 584		if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
 585			if (ret & LM_OUT_CANCELED) {
 586				list_del_init(&gh->gh_list);
 587				trace_gfs2_glock_queue(gh, 0);
 588				gfs2_holder_wake(gh);
 589				gl->gl_target = gl->gl_state;
 590				goto out;
 591			}
 592			/* Some error or failed "try lock" - report it */
 593			if ((ret & LM_OUT_ERROR) ||
 594			    (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
 595				gl->gl_target = gl->gl_state;
 596				do_error(gl, ret);
 597				goto out;
 598			}
 599		}
 600		switch(gl->gl_state) {
 601		/* Unlocked due to conversion deadlock, try again */
 602		case LM_ST_UNLOCKED:
 603			do_xmote(gl, gh, gl->gl_target);
 604			break;
 605		/* Conversion fails, unlock and try again */
 606		case LM_ST_SHARED:
 607		case LM_ST_DEFERRED:
 608			do_xmote(gl, gh, LM_ST_UNLOCKED);
 609			break;
 610		default: /* Everything else */
 611			fs_err(gl->gl_name.ln_sbd,
 612			       "glock %u:%llu requested=%u ret=%u\n",
 613			       gl->gl_name.ln_type, gl->gl_name.ln_number,
 614			       gl->gl_req, ret);
 615			GLOCK_BUG_ON(gl, 1);
 616		}
 617		return;
 618	}
 619
 620	/* Fast path - we got what we asked for */
 621	if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
 622		clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
 623		gfs2_demote_wake(gl);
 624	}
 625	if (gl->gl_state != LM_ST_UNLOCKED) {
 626		if (glops->go_xmote_bh) {
 627			int rv;
 628
 629			spin_unlock(&gl->gl_lockref.lock);
 630			rv = glops->go_xmote_bh(gl);
 631			spin_lock(&gl->gl_lockref.lock);
 632			if (rv) {
 633				do_error(gl, rv);
 634				goto out;
 635			}
 636		}
 637		do_promote(gl);
 638	}
 639out:
 640	if (!test_bit(GLF_CANCELING, &gl->gl_flags))
 641		clear_bit(GLF_LOCK, &gl->gl_flags);
 642}
 643
 644/**
 645 * do_xmote - Calls the DLM to change the state of a lock
 646 * @gl: The lock state
 647 * @gh: The holder (only for promotes)
 648 * @target: The target lock state
 649 *
 650 */
 651
 652static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh,
 653					 unsigned int target)
 654__releases(&gl->gl_lockref.lock)
 655__acquires(&gl->gl_lockref.lock)
 656{
 657	const struct gfs2_glock_operations *glops = gl->gl_ops;
 658	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 659	struct lm_lockstruct *ls = &sdp->sd_lockstruct;
 660	int ret;
 661
 662	/*
 663	 * When a filesystem is withdrawing, the remaining cluster nodes will
 664	 * take care of recovering the withdrawing node's journal.  We only
 665	 * need to make sure that once we trigger remote recovery, we won't
 666	 * write to the shared block device anymore.  This means that here,
 667	 *
 668	 * - no new writes to the filesystem must be triggered (->go_sync()).
 669	 *
 670	 * - any cached data should be discarded by calling ->go_inval(), dirty
 671	 *   or not and journaled or unjournaled.
 672	 *
 673	 * - no more dlm locking operations should be issued (->lm_lock()).
 674	 */
 675
 676	GLOCK_BUG_ON(gl, gl->gl_state == target);
 677	GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
 678
 679	if (!glops->go_inval || !glops->go_sync)
 680		goto skip_inval;
 681
 682	spin_unlock(&gl->gl_lockref.lock);
 683	if (!gfs2_withdrawn(sdp)) {
 684		ret = glops->go_sync(gl);
 685		if (ret) {
 686			if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
 687				fs_err(sdp, "Error %d syncing glock\n", ret);
 688				gfs2_dump_glock(NULL, gl, true);
 689				gfs2_withdraw(sdp);
 690			}
 691		}
 692	}
 693
 694	if (target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED)
 695		glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
 696	spin_lock(&gl->gl_lockref.lock);
 697
 698skip_inval:
 699	if (gfs2_withdrawn(sdp)) {
 700		if (target != LM_ST_UNLOCKED)
 701			target = LM_OUT_ERROR;
 702		goto out;
 703	}
 704
 705	if (ls->ls_ops->lm_lock) {
 706		set_bit(GLF_PENDING_REPLY, &gl->gl_flags);
 707		spin_unlock(&gl->gl_lockref.lock);
 708		ret = ls->ls_ops->lm_lock(gl, target, gh ? gh->gh_flags : 0);
 709		spin_lock(&gl->gl_lockref.lock);
 710
 711		if (!ret) {
 712			/* The operation will be completed asynchronously. */
 713			gl->gl_lockref.count++;
 714			return;
 715		}
 716		clear_bit(GLF_PENDING_REPLY, &gl->gl_flags);
 717
 718		if (ret == -ENODEV) {
 719			/*
 720			 * The lockspace has been released and the lock has
 721			 * been unlocked implicitly.
 722			 */
 723			if (target != LM_ST_UNLOCKED) {
 724				target = LM_OUT_ERROR;
 725				goto out;
 726			}
 727		} else {
 728			fs_err(sdp, "lm_lock ret %d\n", ret);
 729			GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp));
 730			return;
 731		}
 732	}
 733
 734out:
 735	/* Complete the operation now. */
 736	finish_xmote(gl, target);
 737	gl->gl_lockref.count++;
 738	gfs2_glock_queue_work(gl, 0);
 739}
 740
 741/**
 742 * run_queue - do all outstanding tasks related to a glock
 743 * @gl: The glock in question
 744 * @nonblock: True if we must not block in run_queue
 745 *
 746 */
 747
 748static void run_queue(struct gfs2_glock *gl, const int nonblock)
 749__releases(&gl->gl_lockref.lock)
 750__acquires(&gl->gl_lockref.lock)
 751{
 752	struct gfs2_holder *gh;
 753
 754	if (test_bit(GLF_LOCK, &gl->gl_flags))
 755		return;
 756	set_bit(GLF_LOCK, &gl->gl_flags);
 757
 758	/*
 759	 * The GLF_DEMOTE_IN_PROGRESS flag is only set intermittently during
 760	 * locking operations.  We have just started a locking operation by
 761	 * setting the GLF_LOCK flag, so the GLF_DEMOTE_IN_PROGRESS flag must
 762	 * be cleared.
 763	 */
 764	GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
 765
 766	if (test_bit(GLF_DEMOTE, &gl->gl_flags)) {
 767		if (gl->gl_demote_state == gl->gl_state) {
 768			gfs2_demote_wake(gl);
 769			goto promote;
 770		}
 771
 772		if (find_first_holder(gl))
 773			goto out_unlock;
 774		if (nonblock)
 775			goto out_sched;
 776		set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
 777		GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
 778		gl->gl_target = gl->gl_demote_state;
 779		do_xmote(gl, NULL, gl->gl_target);
 780		return;
 781	}
 782
 783promote:
 784	do_promote(gl);
 785	if (find_first_holder(gl))
 786		goto out_unlock;
 787	gh = find_first_waiter(gl);
 788	if (!gh)
 789		goto out_unlock;
 790	if (nonblock)
 791		goto out_sched;
 792	gl->gl_target = gh->gh_state;
 793	if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
 794		do_error(gl, 0); /* Fail queued try locks */
 795	do_xmote(gl, gh, gl->gl_target);
 796	return;
 797
 798out_sched:
 799	clear_bit(GLF_LOCK, &gl->gl_flags);
 800	gl->gl_lockref.count++;
 801	gfs2_glock_queue_work(gl, 0);
 802	return;
 803
 804out_unlock:
 805	clear_bit(GLF_LOCK, &gl->gl_flags);
 806}
 807
 808/**
 809 * glock_set_object - set the gl_object field of a glock
 810 * @gl: the glock
 811 * @object: the object
 812 */
 813void glock_set_object(struct gfs2_glock *gl, void *object)
 814{
 815	void *prev_object;
 816
 817	spin_lock(&gl->gl_lockref.lock);
 818	prev_object = gl->gl_object;
 819	gl->gl_object = object;
 820	spin_unlock(&gl->gl_lockref.lock);
 821	if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == NULL))
 822		gfs2_dump_glock(NULL, gl, true);
 823}
 824
 825/**
 826 * glock_clear_object - clear the gl_object field of a glock
 827 * @gl: the glock
 828 * @object: object the glock currently points at
 829 */
 830void glock_clear_object(struct gfs2_glock *gl, void *object)
 831{
 832	void *prev_object;
 833
 834	spin_lock(&gl->gl_lockref.lock);
 835	prev_object = gl->gl_object;
 836	gl->gl_object = NULL;
 837	spin_unlock(&gl->gl_lockref.lock);
 838	if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == object))
 839		gfs2_dump_glock(NULL, gl, true);
 840}
 841
 842void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
 843{
 844	struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
 845
 846	if (ri->ri_magic == 0)
 847		ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
 848	if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
 849		ri->ri_generation_deleted = cpu_to_be64(generation);
 850}
 851
 852bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
 853{
 854	struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
 855
 856	if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
 857		return false;
 858	return generation <= be64_to_cpu(ri->ri_generation_deleted);
 859}
 860
 861static void gfs2_glock_poke(struct gfs2_glock *gl)
 862{
 863	int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
 864	struct gfs2_holder gh;
 865	int error;
 866
 867	__gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh, _RET_IP_);
 868	error = gfs2_glock_nq(&gh);
 869	if (!error)
 870		gfs2_glock_dq(&gh);
 871	gfs2_holder_uninit(&gh);
 872}
 873
 874static struct gfs2_inode *gfs2_grab_existing_inode(struct gfs2_glock *gl)
 875{
 876	struct gfs2_inode *ip;
 877
 878	spin_lock(&gl->gl_lockref.lock);
 879	ip = gl->gl_object;
 880	if (ip && !igrab(&ip->i_inode))
 881		ip = NULL;
 882	spin_unlock(&gl->gl_lockref.lock);
 883	if (ip) {
 884		wait_on_new_inode(&ip->i_inode);
 885		if (is_bad_inode(&ip->i_inode)) {
 886			iput(&ip->i_inode);
 887			ip = NULL;
 888		}
 889	}
 890	return ip;
 891}
 892
 893static void gfs2_try_to_evict(struct gfs2_glock *gl)
 894{
 895	struct gfs2_inode *ip;
 896
 897	/*
 898	 * If there is contention on the iopen glock and we have an inode, try
 899	 * to grab and release the inode so that it can be evicted.  The
 900	 * GLF_DEFER_DELETE flag indicates to gfs2_evict_inode() that the inode
 901	 * should not be deleted locally.  This will allow the remote node to
 902	 * go ahead and delete the inode without us having to do it, which will
 903	 * avoid rgrp glock thrashing.
 904	 *
 905	 * The remote node is likely still holding the corresponding inode
 906	 * glock, so it will run before we get to verify that the delete has
 907	 * happened below.  (Verification is triggered by the call to
 908	 * gfs2_queue_verify_delete() in gfs2_evict_inode().)
 909	 */
 910	ip = gfs2_grab_existing_inode(gl);
 911	if (ip) {
 912		set_bit(GLF_DEFER_DELETE, &gl->gl_flags);
 913		d_prune_aliases(&ip->i_inode);
 914		iput(&ip->i_inode);
 915		clear_bit(GLF_DEFER_DELETE, &gl->gl_flags);
 916
 917		/* If the inode was evicted, gl->gl_object will now be NULL. */
 918		ip = gfs2_grab_existing_inode(gl);
 919		if (ip) {
 920			gfs2_glock_poke(ip->i_gl);
 921			iput(&ip->i_inode);
 922		}
 923	}
 924}
 925
 926bool gfs2_queue_try_to_evict(struct gfs2_glock *gl)
 927{
 928	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 929
 930	if (test_and_set_bit(GLF_TRY_TO_EVICT, &gl->gl_flags))
 931		return false;
 932	return !mod_delayed_work(sdp->sd_delete_wq, &gl->gl_delete, 0);
 933}
 934
 935bool gfs2_queue_verify_delete(struct gfs2_glock *gl, bool later)
 936{
 937	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 938	unsigned long delay;
 939
 940	if (test_and_set_bit(GLF_VERIFY_DELETE, &gl->gl_flags))
 941		return false;
 942	delay = later ? HZ + get_random_long() % (HZ * 9) : 0;
 943	return queue_delayed_work(sdp->sd_delete_wq, &gl->gl_delete, delay);
 944}
 945
 946static void delete_work_func(struct work_struct *work)
 947{
 948	struct delayed_work *dwork = to_delayed_work(work);
 949	struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
 950	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 951	bool verify_delete = test_and_clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags);
 952
 953	/*
 954	 * Check for the GLF_VERIFY_DELETE above: this ensures that we won't
 955	 * immediately process GLF_VERIFY_DELETE work that the below call to
 956	 * gfs2_try_to_evict() queues.
 957	 */
 958
 959	if (test_and_clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags))
 960		gfs2_try_to_evict(gl);
 961
 962	if (verify_delete) {
 963		u64 no_addr = gl->gl_name.ln_number;
 964		struct inode *inode;
 965
 966		inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
 967					    GFS2_BLKST_UNLINKED);
 968		if (IS_ERR(inode)) {
 969			if (PTR_ERR(inode) == -EAGAIN &&
 970			    !test_bit(SDF_KILL, &sdp->sd_flags) &&
 971			    gfs2_queue_verify_delete(gl, true))
 972				return;
 973		} else {
 974			d_prune_aliases(inode);
 975			iput(inode);
 976		}
 977	}
 978
 979	gfs2_glock_put(gl);
 980}
 981
 982static void glock_work_func(struct work_struct *work)
 983{
 984	unsigned long delay = 0;
 985	struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
 986	unsigned int drop_refs = 1;
 987
 988	spin_lock(&gl->gl_lockref.lock);
 989	if (test_bit(GLF_HAVE_REPLY, &gl->gl_flags)) {
 990		clear_bit(GLF_HAVE_REPLY, &gl->gl_flags);
 991		finish_xmote(gl, gl->gl_reply);
 992		drop_refs++;
 993	}
 994	if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
 995	    gl->gl_state != LM_ST_UNLOCKED &&
 996	    gl->gl_demote_state != LM_ST_EXCLUSIVE) {
 997		if (gl->gl_name.ln_type == LM_TYPE_INODE) {
 998			unsigned long holdtime, now = jiffies;
 999
1000			holdtime = gl->gl_tchange + gl->gl_hold_time;
1001			if (time_before(now, holdtime))
1002				delay = holdtime - now;
1003		}
1004
1005		if (!delay) {
1006			clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1007			gfs2_set_demote(GLF_DEMOTE, gl);
1008		}
1009	}
1010	run_queue(gl, 0);
1011	if (delay) {
1012		/* Keep one glock reference for the work we requeue. */
1013		drop_refs--;
1014		gfs2_glock_queue_work(gl, delay);
1015	}
1016
1017	/* Drop the remaining glock references manually. */
1018	GLOCK_BUG_ON(gl, gl->gl_lockref.count < drop_refs);
1019	gl->gl_lockref.count -= drop_refs;
1020	if (!gl->gl_lockref.count) {
1021		if (gl->gl_state == LM_ST_UNLOCKED) {
1022			__gfs2_glock_put(gl);
1023			return;
1024		}
1025		gfs2_glock_add_to_lru(gl);
1026	}
1027	spin_unlock(&gl->gl_lockref.lock);
1028}
1029
1030static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
1031					    struct gfs2_glock *new)
1032{
1033	struct wait_glock_queue wait;
1034	wait_queue_head_t *wq = glock_waitqueue(name);
1035	struct gfs2_glock *gl;
1036
1037	wait.name = name;
1038	init_wait(&wait.wait);
1039	wait.wait.func = glock_wake_function;
1040
1041again:
1042	prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
1043	rcu_read_lock();
1044	if (new) {
1045		gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
1046			&new->gl_node, ht_parms);
1047		if (IS_ERR(gl))
1048			goto out;
1049	} else {
1050		gl = rhashtable_lookup_fast(&gl_hash_table,
1051			name, ht_parms);
1052	}
1053	if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
1054		rcu_read_unlock();
1055		schedule();
1056		goto again;
1057	}
1058out:
1059	rcu_read_unlock();
1060	finish_wait(wq, &wait.wait);
1061	if (gl)
1062		gfs2_glock_remove_from_lru(gl);
1063	return gl;
1064}
1065
1066/**
1067 * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
1068 * @sdp: The GFS2 superblock
1069 * @number: the lock number
1070 * @glops: The glock_operations to use
1071 * @create: If 0, don't create the glock if it doesn't exist
1072 * @glp: the glock is returned here
1073 *
1074 * This does not lock a glock, just finds/creates structures for one.
1075 *
1076 * Returns: errno
1077 */
1078
1079int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
1080		   const struct gfs2_glock_operations *glops, int create,
1081		   struct gfs2_glock **glp)
1082{
1083	struct lm_lockname name = { .ln_number = number,
1084				    .ln_type = glops->go_type,
1085				    .ln_sbd = sdp };
1086	struct gfs2_glock *gl, *tmp;
1087	struct address_space *mapping;
1088
1089	gl = find_insert_glock(&name, NULL);
1090	if (gl)
1091		goto found;
1092	if (!create)
1093		return -ENOENT;
1094
1095	if (glops->go_flags & GLOF_ASPACE) {
1096		struct gfs2_glock_aspace *gla =
1097			kmem_cache_alloc(gfs2_glock_aspace_cachep, GFP_NOFS);
1098		if (!gla)
1099			return -ENOMEM;
1100		gl = &gla->glock;
1101	} else {
1102		gl = kmem_cache_alloc(gfs2_glock_cachep, GFP_NOFS);
1103		if (!gl)
1104			return -ENOMEM;
1105	}
1106	memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1107	gl->gl_ops = glops;
1108
1109	if (glops->go_flags & GLOF_LVB) {
1110		gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1111		if (!gl->gl_lksb.sb_lvbptr) {
1112			gfs2_glock_dealloc(&gl->gl_rcu);
1113			return -ENOMEM;
1114		}
1115	}
1116
1117	atomic_inc(&sdp->sd_glock_disposal);
1118	gl->gl_node.next = NULL;
1119	gl->gl_flags = BIT(GLF_INITIAL);
1120	if (glops->go_instantiate)
1121		gl->gl_flags |= BIT(GLF_INSTANTIATE_NEEDED);
1122	gl->gl_name = name;
1123	lockref_init(&gl->gl_lockref);
1124	lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass);
1125	gl->gl_state = LM_ST_UNLOCKED;
1126	gl->gl_target = LM_ST_UNLOCKED;
1127	gl->gl_demote_state = LM_ST_EXCLUSIVE;
1128	gl->gl_dstamp = 0;
1129	preempt_disable();
1130	/* We use the global stats to estimate the initial per-glock stats */
1131	gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1132	preempt_enable();
1133	gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1134	gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1135	gl->gl_tchange = jiffies;
1136	gl->gl_object = NULL;
1137	gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1138	INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1139	if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
1140		INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1141
1142	mapping = gfs2_glock2aspace(gl);
1143	if (mapping) {
1144		gfp_t gfp_mask;
1145
1146                mapping->a_ops = &gfs2_meta_aops;
1147		mapping->host = sdp->sd_inode;
1148		mapping->flags = 0;
1149		gfp_mask = mapping_gfp_mask(sdp->sd_inode->i_mapping);
1150		mapping_set_gfp_mask(mapping, gfp_mask);
1151		mapping->i_private_data = NULL;
1152		mapping->writeback_index = 0;
1153	}
1154
1155	tmp = find_insert_glock(&name, gl);
1156	if (tmp) {
1157		gfs2_glock_dealloc(&gl->gl_rcu);
1158		if (atomic_dec_and_test(&sdp->sd_glock_disposal))
1159			wake_up(&sdp->sd_kill_wait);
1160
1161		if (IS_ERR(tmp))
1162			return PTR_ERR(tmp);
1163		gl = tmp;
1164	}
1165
1166found:
1167	*glp = gl;
1168	return 0;
1169}
1170
1171/**
1172 * __gfs2_holder_init - initialize a struct gfs2_holder in the default way
1173 * @gl: the glock
1174 * @state: the state we're requesting
1175 * @flags: the modifier flags
1176 * @gh: the holder structure
1177 * @ip: caller's return address for debugging
1178 */
1179
1180void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1181			struct gfs2_holder *gh, unsigned long ip)
1182{
1183	INIT_LIST_HEAD(&gh->gh_list);
1184	gh->gh_gl = gfs2_glock_hold(gl);
1185	gh->gh_ip = ip;
1186	gh->gh_owner_pid = get_pid(task_pid(current));
1187	gh->gh_state = state;
1188	gh->gh_flags = flags;
1189	gh->gh_iflags = 0;
1190}
1191
1192/**
1193 * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
1194 * @state: the state we're requesting
1195 * @flags: the modifier flags
1196 * @gh: the holder structure
1197 *
1198 * Don't mess with the glock.
1199 *
1200 */
1201
1202void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1203{
1204	gh->gh_state = state;
1205	gh->gh_flags = flags;
1206	gh->gh_iflags = 0;
1207	gh->gh_ip = _RET_IP_;
1208	put_pid(gh->gh_owner_pid);
1209	gh->gh_owner_pid = get_pid(task_pid(current));
1210}
1211
1212/**
1213 * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
1214 * @gh: the holder structure
1215 *
1216 */
1217
1218void gfs2_holder_uninit(struct gfs2_holder *gh)
1219{
1220	put_pid(gh->gh_owner_pid);
1221	gfs2_glock_put(gh->gh_gl);
1222	gfs2_holder_mark_uninitialized(gh);
1223	gh->gh_ip = 0;
1224}
1225
1226static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1227					unsigned long start_time)
1228{
1229	/* Have we waited longer that a second? */
1230	if (time_after(jiffies, start_time + HZ)) {
1231		/* Lengthen the minimum hold time. */
1232		gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1233				       GL_GLOCK_MAX_HOLD);
1234	}
1235}
1236
1237/**
1238 * gfs2_glock_holder_ready - holder is ready and its error code can be collected
1239 * @gh: the glock holder
1240 *
1241 * Called when a glock holder no longer needs to be waited for because it is
1242 * now either held (HIF_HOLDER set; gh_error == 0), or acquiring the lock has
1243 * failed (gh_error != 0).
1244 */
1245
1246int gfs2_glock_holder_ready(struct gfs2_holder *gh)
1247{
1248	if (gh->gh_error || (gh->gh_flags & GL_SKIP))
1249		return gh->gh_error;
1250	gh->gh_error = gfs2_instantiate(gh);
1251	if (gh->gh_error)
1252		gfs2_glock_dq(gh);
1253	return gh->gh_error;
1254}
1255
1256/**
1257 * gfs2_glock_wait - wait on a glock acquisition
1258 * @gh: the glock holder
1259 *
1260 * Returns: 0 on success
1261 */
1262
1263int gfs2_glock_wait(struct gfs2_holder *gh)
1264{
1265	unsigned long start_time = jiffies;
1266
1267	might_sleep();
1268	wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1269	gfs2_glock_update_hold_time(gh->gh_gl, start_time);
1270	return gfs2_glock_holder_ready(gh);
1271}
1272
1273static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1274{
1275	int i;
1276
1277	for (i = 0; i < num_gh; i++)
1278		if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1279			return 1;
1280	return 0;
1281}
1282
1283/**
1284 * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions
1285 * @num_gh: the number of holders in the array
1286 * @ghs: the glock holder array
1287 * @retries: number of retries attempted so far
1288 *
1289 * Returns: 0 on success, meaning all glocks have been granted and are held.
1290 *          -ESTALE if the request timed out, meaning all glocks were released,
1291 *          and the caller should retry the operation.
1292 */
1293
1294int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs,
1295			  unsigned int retries)
1296{
1297	struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1298	unsigned long start_time = jiffies;
1299	int i, ret = 0;
1300	long timeout;
1301
1302	might_sleep();
1303
1304	timeout = GL_GLOCK_MIN_HOLD;
1305	if (retries) {
1306		unsigned int max_shift;
1307		long incr;
1308
1309		/* Add a random delay and increase the timeout exponentially. */
1310		max_shift = BITS_PER_LONG - 2 - __fls(GL_GLOCK_HOLD_INCR);
1311		incr = min(GL_GLOCK_HOLD_INCR << min(retries - 1, max_shift),
1312			   10 * HZ - GL_GLOCK_MIN_HOLD);
1313		schedule_timeout_interruptible(get_random_long() % (incr / 3));
1314		if (signal_pending(current))
1315			goto interrupted;
1316		timeout += (incr / 3) + get_random_long() % (incr / 3);
1317	}
1318
1319	if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait,
1320				!glocks_pending(num_gh, ghs), timeout)) {
1321		ret = -ESTALE; /* request timed out. */
1322		goto out;
1323	}
1324	if (signal_pending(current))
1325		goto interrupted;
1326
1327	for (i = 0; i < num_gh; i++) {
1328		struct gfs2_holder *gh = &ghs[i];
1329		int ret2;
1330
1331		if (test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1332			gfs2_glock_update_hold_time(gh->gh_gl,
1333						    start_time);
1334		}
1335		ret2 = gfs2_glock_holder_ready(gh);
1336		if (!ret)
1337			ret = ret2;
1338	}
1339
1340out:
1341	if (ret) {
1342		for (i = 0; i < num_gh; i++) {
1343			struct gfs2_holder *gh = &ghs[i];
1344
1345			gfs2_glock_dq(gh);
1346		}
1347	}
1348	return ret;
1349
1350interrupted:
1351	ret = -EINTR;
1352	goto out;
1353}
1354
1355/**
1356 * request_demote - process a demote request
1357 * @gl: the glock
1358 * @state: the state the caller wants us to change to
1359 * @delay: zero to demote immediately; otherwise pending demote
1360 * @remote: true if this came from a different cluster node
1361 *
1362 * There are only two requests that we are going to see in actual
1363 * practise: LM_ST_SHARED and LM_ST_UNLOCKED
1364 */
1365
1366static void request_demote(struct gfs2_glock *gl, unsigned int state,
1367			   unsigned long delay, bool remote)
1368{
1369	gfs2_set_demote(delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE, gl);
1370	if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1371		gl->gl_demote_state = state;
1372		gl->gl_demote_time = jiffies;
1373	} else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1374			gl->gl_demote_state != state) {
1375		gl->gl_demote_state = LM_ST_UNLOCKED;
1376	}
1377	if (gl->gl_ops->go_callback)
1378		gl->gl_ops->go_callback(gl, remote);
1379	trace_gfs2_demote_rq(gl, remote);
1380}
1381
1382void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1383{
1384	struct va_format vaf;
1385	va_list args;
1386
1387	va_start(args, fmt);
1388
1389	if (seq) {
1390		seq_vprintf(seq, fmt, args);
1391	} else {
1392		vaf.fmt = fmt;
1393		vaf.va = &args;
1394
1395		pr_err("%pV", &vaf);
1396	}
1397
1398	va_end(args);
1399}
1400
1401static bool gfs2_should_queue_trylock(struct gfs2_glock *gl,
1402				      struct gfs2_holder *gh)
1403{
1404	struct gfs2_holder *current_gh, *gh2;
1405
1406	current_gh = find_first_holder(gl);
1407	if (current_gh && !may_grant(gl, current_gh, gh))
1408		return false;
1409
1410	list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1411		if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1412			continue;
1413		if (!(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
1414			return false;
1415	}
1416	return true;
1417}
1418
1419static inline bool pid_is_meaningful(const struct gfs2_holder *gh)
1420{
1421        if (!(gh->gh_flags & GL_NOPID))
1422                return true;
1423	return !test_bit(HIF_HOLDER, &gh->gh_iflags);
1424}
1425
1426/**
1427 * add_to_queue - Add a holder to the wait queue (but look for recursion)
1428 * @gh: the holder structure to add
1429 *
1430 * Eventually we should move the recursive locking trap to a
1431 * debugging option or something like that. This is the fast
1432 * path and needs to have the minimum number of distractions.
1433 * 
1434 */
1435
1436static inline void add_to_queue(struct gfs2_holder *gh)
1437{
1438	struct gfs2_glock *gl = gh->gh_gl;
1439	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1440	struct gfs2_holder *gh2;
1441
1442	GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1443	if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1444		GLOCK_BUG_ON(gl, true);
1445
1446	if ((gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) &&
1447	    !gfs2_should_queue_trylock(gl, gh)) {
1448		gh->gh_error = GLR_TRYFAILED;
1449		gfs2_holder_wake(gh);
1450		return;
1451	}
1452
1453	list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1454		if (likely(gh2->gh_owner_pid != gh->gh_owner_pid))
1455			continue;
1456		if (gh->gh_gl->gl_ops->go_type == LM_TYPE_FLOCK)
1457			continue;
1458		if (!pid_is_meaningful(gh2))
1459			continue;
1460		goto trap_recursive;
1461	}
1462	trace_gfs2_glock_queue(gh, 1);
1463	gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1464	gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1465	list_add_tail(&gh->gh_list, &gl->gl_holders);
1466	return;
1467
1468trap_recursive:
1469	fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1470	fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1471	fs_err(sdp, "lock type: %d req lock state : %d\n",
1472	       gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1473	fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1474	fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1475	fs_err(sdp, "lock type: %d req lock state : %d\n",
1476	       gh->gh_gl->gl_name.ln_type, gh->gh_state);
1477	gfs2_dump_glock(NULL, gl, true);
1478	BUG();
1479}
1480
1481/**
1482 * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1483 * @gh: the holder structure
1484 *
1485 * if (gh->gh_flags & GL_ASYNC), this never returns an error
1486 *
1487 * Returns: 0, GLR_TRYFAILED, or errno on failure
1488 */
1489
1490int gfs2_glock_nq(struct gfs2_holder *gh)
1491{
1492	struct gfs2_glock *gl = gh->gh_gl;
1493	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1494	int error;
1495
1496	if (gfs2_withdrawn(sdp))
1497		return -EIO;
1498
1499	if (gh->gh_flags & GL_NOBLOCK) {
1500		struct gfs2_holder *current_gh;
1501
1502		error = -ECHILD;
1503		spin_lock(&gl->gl_lockref.lock);
1504		if (find_last_waiter(gl))
1505			goto unlock;
1506		current_gh = find_first_holder(gl);
1507		if (!may_grant(gl, current_gh, gh))
1508			goto unlock;
1509		set_bit(HIF_HOLDER, &gh->gh_iflags);
1510		list_add_tail(&gh->gh_list, &gl->gl_holders);
1511		trace_gfs2_promote(gh);
1512		error = 0;
1513unlock:
1514		spin_unlock(&gl->gl_lockref.lock);
1515		return error;
1516	}
1517
1518	gh->gh_error = 0;
1519	spin_lock(&gl->gl_lockref.lock);
1520	add_to_queue(gh);
1521	if (unlikely((LM_FLAG_RECOVER & gh->gh_flags) &&
1522		     test_and_clear_bit(GLF_HAVE_FROZEN_REPLY, &gl->gl_flags))) {
1523		set_bit(GLF_HAVE_REPLY, &gl->gl_flags);
1524		gl->gl_lockref.count++;
1525		gfs2_glock_queue_work(gl, 0);
1526	}
1527	run_queue(gl, 1);
1528	spin_unlock(&gl->gl_lockref.lock);
1529
1530	error = 0;
1531	if (!(gh->gh_flags & GL_ASYNC))
1532		error = gfs2_glock_wait(gh);
1533
1534	return error;
1535}
1536
1537/**
1538 * gfs2_glock_poll - poll to see if an async request has been completed
1539 * @gh: the holder
1540 *
1541 * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1542 */
1543
1544int gfs2_glock_poll(struct gfs2_holder *gh)
1545{
1546	return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1547}
1548
1549static void __gfs2_glock_dq(struct gfs2_holder *gh)
1550{
1551	struct gfs2_glock *gl = gh->gh_gl;
1552	unsigned delay = 0;
1553	int fast_path = 0;
1554
1555	/*
1556	 * This holder should not be cached, so mark it for demote.
1557	 * Note: this should be done before the glock_needs_demote
1558	 * check below.
1559	 */
1560	if (gh->gh_flags & GL_NOCACHE)
1561		request_demote(gl, LM_ST_UNLOCKED, 0, false);
1562
1563	list_del_init(&gh->gh_list);
1564	clear_bit(HIF_HOLDER, &gh->gh_iflags);
1565	trace_gfs2_glock_queue(gh, 0);
1566
1567	/*
1568	 * If there hasn't been a demote request we are done.
1569	 * (Let the remaining holders, if any, keep holding it.)
1570	 */
1571	if (!glock_needs_demote(gl)) {
1572		if (list_empty(&gl->gl_holders))
1573			fast_path = 1;
1574	}
1575
1576	if (unlikely(!fast_path)) {
1577		gl->gl_lockref.count++;
1578		if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1579		    !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1580		    gl->gl_name.ln_type == LM_TYPE_INODE)
1581			delay = gl->gl_hold_time;
1582		gfs2_glock_queue_work(gl, delay);
1583	}
1584}
1585
1586/**
1587 * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1588 * @gh: the glock holder
1589 *
1590 */
1591void gfs2_glock_dq(struct gfs2_holder *gh)
1592{
1593	struct gfs2_glock *gl = gh->gh_gl;
1594
1595	spin_lock(&gl->gl_lockref.lock);
1596	if (!gfs2_holder_queued(gh)) {
1597		/*
1598		 * May have already been dequeued because the locking request
1599		 * was GL_ASYNC and it has failed in the meantime.
1600		 */
1601		goto out;
1602	}
1603
1604	if (list_is_first(&gh->gh_list, &gl->gl_holders) &&
1605	    !test_bit(HIF_HOLDER, &gh->gh_iflags) &&
1606	    test_bit(GLF_LOCK, &gl->gl_flags) &&
1607	    !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
1608	    !test_bit(GLF_CANCELING, &gl->gl_flags)) {
1609		set_bit(GLF_CANCELING, &gl->gl_flags);
1610		spin_unlock(&gl->gl_lockref.lock);
1611		gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl);
1612		wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1613		spin_lock(&gl->gl_lockref.lock);
1614		clear_bit(GLF_CANCELING, &gl->gl_flags);
1615		clear_bit(GLF_LOCK, &gl->gl_flags);
1616		if (!gfs2_holder_queued(gh))
1617			goto out;
1618	}
1619
1620	__gfs2_glock_dq(gh);
1621out:
1622	spin_unlock(&gl->gl_lockref.lock);
1623}
1624
1625void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1626{
1627	struct gfs2_glock *gl = gh->gh_gl;
1628	gfs2_glock_dq(gh);
1629	might_sleep();
1630	wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1631}
1632
1633/**
1634 * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1635 * @gh: the holder structure
1636 *
1637 */
1638
1639void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1640{
1641	gfs2_glock_dq(gh);
1642	gfs2_holder_uninit(gh);
1643}
1644
1645/**
1646 * gfs2_glock_nq_num - acquire a glock based on lock number
1647 * @sdp: the filesystem
1648 * @number: the lock number
1649 * @glops: the glock operations for the type of glock
1650 * @state: the state to acquire the glock in
1651 * @flags: modifier flags for the acquisition
1652 * @gh: the struct gfs2_holder
1653 *
1654 * Returns: errno
1655 */
1656
1657int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1658		      const struct gfs2_glock_operations *glops,
1659		      unsigned int state, u16 flags, struct gfs2_holder *gh)
1660{
1661	struct gfs2_glock *gl;
1662	int error;
1663
1664	error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1665	if (!error) {
1666		error = gfs2_glock_nq_init(gl, state, flags, gh);
1667		gfs2_glock_put(gl);
1668	}
1669
1670	return error;
1671}
1672
1673/**
1674 * glock_compare - Compare two struct gfs2_glock structures for sorting
1675 * @arg_a: the first structure
1676 * @arg_b: the second structure
1677 *
1678 */
1679
1680static int glock_compare(const void *arg_a, const void *arg_b)
1681{
1682	const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1683	const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1684	const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1685	const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1686
1687	if (a->ln_number > b->ln_number)
1688		return 1;
1689	if (a->ln_number < b->ln_number)
1690		return -1;
1691	BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1692	return 0;
1693}
1694
1695/**
1696 * nq_m_sync - synchronously acquire more than one glock in deadlock free order
1697 * @num_gh: the number of structures
1698 * @ghs: an array of struct gfs2_holder structures
1699 * @p: placeholder for the holder structure to pass back
1700 *
1701 * Returns: 0 on success (all glocks acquired),
1702 *          errno on failure (no glocks acquired)
1703 */
1704
1705static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1706		     struct gfs2_holder **p)
1707{
1708	unsigned int x;
1709	int error = 0;
1710
1711	for (x = 0; x < num_gh; x++)
1712		p[x] = &ghs[x];
1713
1714	sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1715
1716	for (x = 0; x < num_gh; x++) {
1717		error = gfs2_glock_nq(p[x]);
1718		if (error) {
1719			while (x--)
1720				gfs2_glock_dq(p[x]);
1721			break;
1722		}
1723	}
1724
1725	return error;
1726}
1727
1728/**
1729 * gfs2_glock_nq_m - acquire multiple glocks
1730 * @num_gh: the number of structures
1731 * @ghs: an array of struct gfs2_holder structures
1732 *
1733 * Returns: 0 on success (all glocks acquired),
1734 *          errno on failure (no glocks acquired)
1735 */
1736
1737int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1738{
1739	struct gfs2_holder *tmp[4];
1740	struct gfs2_holder **pph = tmp;
1741	int error = 0;
1742
1743	switch(num_gh) {
1744	case 0:
1745		return 0;
1746	case 1:
1747		return gfs2_glock_nq(ghs);
1748	default:
1749		if (num_gh <= 4)
1750			break;
1751		pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1752				    GFP_NOFS);
1753		if (!pph)
1754			return -ENOMEM;
1755	}
1756
1757	error = nq_m_sync(num_gh, ghs, pph);
1758
1759	if (pph != tmp)
1760		kfree(pph);
1761
1762	return error;
1763}
1764
1765/**
1766 * gfs2_glock_dq_m - release multiple glocks
1767 * @num_gh: the number of structures
1768 * @ghs: an array of struct gfs2_holder structures
1769 *
1770 */
1771
1772void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1773{
1774	while (num_gh--)
1775		gfs2_glock_dq(&ghs[num_gh]);
1776}
1777
1778void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1779{
1780	unsigned long delay = 0;
1781
1782	gfs2_glock_hold(gl);
1783	spin_lock(&gl->gl_lockref.lock);
1784	if (!list_empty(&gl->gl_holders) &&
1785	    gl->gl_name.ln_type == LM_TYPE_INODE) {
1786		unsigned long now = jiffies;
1787		unsigned long holdtime;
1788
1789		holdtime = gl->gl_tchange + gl->gl_hold_time;
1790
1791		if (time_before(now, holdtime))
1792			delay = holdtime - now;
1793		if (test_bit(GLF_HAVE_REPLY, &gl->gl_flags))
1794			delay = gl->gl_hold_time;
1795	}
1796	request_demote(gl, state, delay, true);
1797	gfs2_glock_queue_work(gl, delay);
1798	spin_unlock(&gl->gl_lockref.lock);
1799}
1800
1801/**
1802 * gfs2_should_freeze - Figure out if glock should be frozen
1803 * @gl: The glock in question
1804 *
1805 * Glocks are not frozen if (a) the result of the dlm operation is
1806 * an error, (b) the locking operation was an unlock operation or
1807 * (c) if there is a "recover" flagged request anywhere in the queue
1808 *
1809 * Returns: 1 if freezing should occur, 0 otherwise
1810 */
1811
1812static int gfs2_should_freeze(const struct gfs2_glock *gl)
1813{
1814	const struct gfs2_holder *gh;
1815
1816	if (gl->gl_reply & ~LM_OUT_ST_MASK)
1817		return 0;
1818	if (gl->gl_target == LM_ST_UNLOCKED)
1819		return 0;
1820
1821	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1822		if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1823			continue;
1824		if (LM_FLAG_RECOVER & gh->gh_flags)
1825			return 0;
1826	}
1827
1828	return 1;
1829}
1830
1831/**
1832 * gfs2_glock_complete - Callback used by locking
1833 * @gl: Pointer to the glock
1834 * @ret: The return value from the dlm
1835 *
1836 * The gl_reply field is under the gl_lockref.lock lock so that it is ok
1837 * to use a bitfield shared with other glock state fields.
1838 */
1839
1840void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1841{
1842	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1843
1844	spin_lock(&gl->gl_lockref.lock);
1845	clear_bit(GLF_PENDING_REPLY, &gl->gl_flags);
1846	gl->gl_reply = ret;
1847
1848	if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1849		if (gfs2_should_freeze(gl)) {
1850			set_bit(GLF_HAVE_FROZEN_REPLY, &gl->gl_flags);
1851			spin_unlock(&gl->gl_lockref.lock);
1852			return;
1853		}
1854	}
1855
1856	gl->gl_lockref.count++;
1857	set_bit(GLF_HAVE_REPLY, &gl->gl_flags);
1858	gfs2_glock_queue_work(gl, 0);
1859	spin_unlock(&gl->gl_lockref.lock);
1860}
1861
1862static int glock_cmp(void *priv, const struct list_head *a,
1863		     const struct list_head *b)
1864{
1865	struct gfs2_glock *gla, *glb;
1866
1867	gla = list_entry(a, struct gfs2_glock, gl_lru);
1868	glb = list_entry(b, struct gfs2_glock, gl_lru);
1869
1870	if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1871		return 1;
1872	if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1873		return -1;
1874
1875	return 0;
1876}
1877
1878static bool can_free_glock(struct gfs2_glock *gl)
1879{
1880	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1881
1882	return !test_bit(GLF_LOCK, &gl->gl_flags) &&
1883	       !gl->gl_lockref.count &&
1884	       (!test_bit(GLF_LFLUSH, &gl->gl_flags) ||
1885		test_bit(SDF_KILL, &sdp->sd_flags));
1886}
1887
1888/**
1889 * gfs2_dispose_glock_lru - Demote a list of glocks
1890 * @list: The list to dispose of
1891 *
1892 * Disposing of glocks may involve disk accesses, so that here we sort
1893 * the glocks by number (i.e. disk location of the inodes) so that if
1894 * there are any such accesses, they'll be sent in order (mostly).
1895 *
1896 * Must be called under the lru_lock, but may drop and retake this
1897 * lock. While the lru_lock is dropped, entries may vanish from the
1898 * list, but no new entries will appear on the list (since it is
1899 * private)
1900 */
1901
1902static unsigned long gfs2_dispose_glock_lru(struct list_head *list)
1903__releases(&lru_lock)
1904__acquires(&lru_lock)
1905{
1906	struct gfs2_glock *gl;
1907	unsigned long freed = 0;
1908
1909	list_sort(NULL, list, glock_cmp);
1910
1911	while(!list_empty(list)) {
1912		gl = list_first_entry(list, struct gfs2_glock, gl_lru);
1913		if (!spin_trylock(&gl->gl_lockref.lock)) {
1914add_back_to_lru:
1915			list_move(&gl->gl_lru, &lru_list);
1916			continue;
1917		}
1918		if (!can_free_glock(gl)) {
1919			spin_unlock(&gl->gl_lockref.lock);
1920			goto add_back_to_lru;
1921		}
1922		list_del_init(&gl->gl_lru);
1923		atomic_dec(&lru_count);
1924		clear_bit(GLF_LRU, &gl->gl_flags);
1925		freed++;
1926		gl->gl_lockref.count++;
1927		if (gl->gl_state != LM_ST_UNLOCKED)
1928			request_demote(gl, LM_ST_UNLOCKED, 0, false);
1929		gfs2_glock_queue_work(gl, 0);
1930		spin_unlock(&gl->gl_lockref.lock);
1931		cond_resched_lock(&lru_lock);
1932	}
1933	return freed;
1934}
1935
1936/**
1937 * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
1938 * @nr: The number of entries to scan
1939 *
1940 * This function selects the entries on the LRU which are able to
1941 * be demoted, and then kicks off the process by calling
1942 * gfs2_dispose_glock_lru() above.
1943 */
1944
1945static unsigned long gfs2_scan_glock_lru(unsigned long nr)
1946{
1947	struct gfs2_glock *gl, *next;
1948	LIST_HEAD(dispose);
1949	unsigned long freed = 0;
1950
1951	spin_lock(&lru_lock);
1952	list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
1953		if (!nr--)
1954			break;
1955		if (can_free_glock(gl))
1956			list_move(&gl->gl_lru, &dispose);
1957	}
1958	if (!list_empty(&dispose))
1959		freed = gfs2_dispose_glock_lru(&dispose);
1960	spin_unlock(&lru_lock);
1961
1962	return freed;
1963}
1964
1965static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
1966					    struct shrink_control *sc)
1967{
1968	if (!(sc->gfp_mask & __GFP_FS))
1969		return SHRINK_STOP;
1970	return gfs2_scan_glock_lru(sc->nr_to_scan);
1971}
1972
1973static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
1974					     struct shrink_control *sc)
1975{
1976	return vfs_pressure_ratio(atomic_read(&lru_count));
1977}
1978
1979static struct shrinker *glock_shrinker;
1980
1981/**
1982 * glock_hash_walk - Call a function for glock in a hash bucket
1983 * @examiner: the function
1984 * @sdp: the filesystem
1985 *
1986 * Note that the function can be called multiple times on the same
1987 * object.  So the user must ensure that the function can cope with
1988 * that.
1989 */
1990
1991static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
1992{
1993	struct gfs2_glock *gl;
1994	struct rhashtable_iter iter;
1995
1996	rhashtable_walk_enter(&gl_hash_table, &iter);
1997
1998	do {
1999		rhashtable_walk_start(&iter);
2000
2001		while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) {
2002			if (gl->gl_name.ln_sbd == sdp)
2003				examiner(gl);
2004		}
2005
2006		rhashtable_walk_stop(&iter);
2007	} while (cond_resched(), gl == ERR_PTR(-EAGAIN));
2008
2009	rhashtable_walk_exit(&iter);
2010}
2011
2012void gfs2_cancel_delete_work(struct gfs2_glock *gl)
2013{
2014	clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags);
2015	clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags);
2016	if (cancel_delayed_work(&gl->gl_delete))
2017		gfs2_glock_put(gl);
2018}
2019
2020static void flush_delete_work(struct gfs2_glock *gl)
2021{
2022	if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
2023		struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2024
2025		if (cancel_delayed_work(&gl->gl_delete)) {
2026			queue_delayed_work(sdp->sd_delete_wq,
2027					   &gl->gl_delete, 0);
2028		}
2029	}
2030}
2031
2032void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
2033{
2034	glock_hash_walk(flush_delete_work, sdp);
2035	flush_workqueue(sdp->sd_delete_wq);
2036}
2037
2038/**
2039 * thaw_glock - thaw out a glock which has an unprocessed reply waiting
2040 * @gl: The glock to thaw
2041 *
2042 */
2043
2044static void thaw_glock(struct gfs2_glock *gl)
2045{
2046	if (!test_and_clear_bit(GLF_HAVE_FROZEN_REPLY, &gl->gl_flags))
2047		return;
2048	if (!lockref_get_not_dead(&gl->gl_lockref))
2049		return;
2050
2051	gfs2_glock_remove_from_lru(gl);
2052	spin_lock(&gl->gl_lockref.lock);
2053	set_bit(GLF_HAVE_REPLY, &gl->gl_flags);
2054	gfs2_glock_queue_work(gl, 0);
2055	spin_unlock(&gl->gl_lockref.lock);
2056}
2057
2058/**
2059 * clear_glock - look at a glock and see if we can free it from glock cache
2060 * @gl: the glock to look at
2061 *
2062 */
2063
2064static void clear_glock(struct gfs2_glock *gl)
2065{
2066	gfs2_glock_remove_from_lru(gl);
2067
2068	spin_lock(&gl->gl_lockref.lock);
2069	if (!__lockref_is_dead(&gl->gl_lockref)) {
2070		gl->gl_lockref.count++;
2071		if (gl->gl_state != LM_ST_UNLOCKED)
2072			request_demote(gl, LM_ST_UNLOCKED, 0, false);
2073		gfs2_glock_queue_work(gl, 0);
2074	}
2075	spin_unlock(&gl->gl_lockref.lock);
2076}
2077
2078/**
2079 * gfs2_glock_thaw - Thaw any frozen glocks
2080 * @sdp: The super block
2081 *
2082 */
2083
2084void gfs2_glock_thaw(struct gfs2_sbd *sdp)
2085{
2086	glock_hash_walk(thaw_glock, sdp);
2087}
2088
2089static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2090{
2091	spin_lock(&gl->gl_lockref.lock);
2092	gfs2_dump_glock(seq, gl, fsid);
2093	spin_unlock(&gl->gl_lockref.lock);
2094}
2095
2096static void dump_glock_func(struct gfs2_glock *gl)
2097{
2098	dump_glock(NULL, gl, true);
2099}
2100
2101static void withdraw_glock(struct gfs2_glock *gl)
2102{
2103	spin_lock(&gl->gl_lockref.lock);
2104	if (!__lockref_is_dead(&gl->gl_lockref)) {
2105		/*
2106		 * We don't want to write back any more dirty data.  Unlock the
2107		 * remaining inode and resource group glocks; this will cause
2108		 * their ->go_inval() hooks to toss out all the remaining
2109		 * cached data, dirty or not.
2110		 */
2111		if (gl->gl_ops->go_inval && gl->gl_state != LM_ST_UNLOCKED)
2112			request_demote(gl, LM_ST_UNLOCKED, 0, false);
2113		do_error(gl, LM_OUT_ERROR); /* remove pending waiters */
2114	}
2115	spin_unlock(&gl->gl_lockref.lock);
2116}
2117
2118void gfs2_withdraw_glocks(struct gfs2_sbd *sdp)
2119{
2120	glock_hash_walk(withdraw_glock, sdp);
2121}
2122
2123/**
2124 * gfs2_gl_hash_clear - Empty out the glock hash table
2125 * @sdp: the filesystem
2126 *
2127 * Called when unmounting the filesystem.
2128 */
2129
2130void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
2131{
2132	unsigned long start = jiffies;
2133	bool timed_out = false;
2134
2135	set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
2136	flush_workqueue(sdp->sd_glock_wq);
2137	glock_hash_walk(clear_glock, sdp);
2138	flush_workqueue(sdp->sd_glock_wq);
2139
2140	while (!timed_out) {
2141		wait_event_timeout(sdp->sd_kill_wait,
2142				   !atomic_read(&sdp->sd_glock_disposal),
2143				   HZ * 60);
2144		if (!atomic_read(&sdp->sd_glock_disposal))
2145			break;
2146		timed_out = time_after(jiffies, start + (HZ * 600));
2147		fs_warn(sdp, "%u glocks left after %u seconds%s\n",
2148			atomic_read(&sdp->sd_glock_disposal),
2149			jiffies_to_msecs(jiffies - start) / 1000,
2150			timed_out ? ":" : "; still waiting");
2151	}
2152	gfs2_lm_unmount(sdp);
2153	gfs2_free_dead_glocks(sdp);
2154	glock_hash_walk(dump_glock_func, sdp);
2155	destroy_workqueue(sdp->sd_glock_wq);
2156	sdp->sd_glock_wq = NULL;
2157}
2158
2159static const char *state2str(unsigned state)
2160{
2161	switch(state) {
2162	case LM_ST_UNLOCKED:
2163		return "UN";
2164	case LM_ST_SHARED:
2165		return "SH";
2166	case LM_ST_DEFERRED:
2167		return "DF";
2168	case LM_ST_EXCLUSIVE:
2169		return "EX";
2170	}
2171	return "??";
2172}
2173
2174static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2175{
2176	char *p = buf;
2177	if (flags & LM_FLAG_TRY)
2178		*p++ = 't';
2179	if (flags & LM_FLAG_TRY_1CB)
2180		*p++ = 'T';
2181	if (flags & LM_FLAG_RECOVER)
2182		*p++ = 'e';
2183	if (flags & LM_FLAG_ANY)
2184		*p++ = 'A';
2185	if (flags & LM_FLAG_NODE_SCOPE)
2186		*p++ = 'n';
2187	if (flags & GL_ASYNC)
2188		*p++ = 'a';
2189	if (flags & GL_EXACT)
2190		*p++ = 'E';
2191	if (flags & GL_NOCACHE)
2192		*p++ = 'c';
2193	if (test_bit(HIF_HOLDER, &iflags))
2194		*p++ = 'H';
2195	if (test_bit(HIF_WAIT, &iflags))
2196		*p++ = 'W';
2197	if (flags & GL_SKIP)
2198		*p++ = 's';
2199	*p = 0;
2200	return buf;
2201}
2202
2203/**
2204 * dump_holder - print information about a glock holder
2205 * @seq: the seq_file struct
2206 * @gh: the glock holder
2207 * @fs_id_buf: pointer to file system id (if requested)
2208 *
2209 */
2210
2211static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2212			const char *fs_id_buf)
2213{
2214	const char *comm = "(none)";
2215	pid_t owner_pid = 0;
2216	char flags_buf[32];
2217
2218	rcu_read_lock();
2219	if (pid_is_meaningful(gh)) {
2220		struct task_struct *gh_owner;
2221
2222		comm = "(ended)";
2223		owner_pid = pid_nr(gh->gh_owner_pid);
2224		gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
2225		if (gh_owner)
2226			comm = gh_owner->comm;
2227	}
2228	gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2229		       fs_id_buf, state2str(gh->gh_state),
2230		       hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
2231		       gh->gh_error, (long)owner_pid, comm, (void *)gh->gh_ip);
2232	rcu_read_unlock();
2233}
2234
2235static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2236{
2237	const unsigned long *gflags = &gl->gl_flags;
2238	char *p = buf;
2239
2240	if (test_bit(GLF_LOCK, gflags))
2241		*p++ = 'l';
2242	if (test_bit(GLF_DEMOTE, gflags))
2243		*p++ = 'D';
2244	if (test_bit(GLF_PENDING_DEMOTE, gflags))
2245		*p++ = 'd';
2246	if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2247		*p++ = 'p';
2248	if (test_bit(GLF_DIRTY, gflags))
2249		*p++ = 'y';
2250	if (test_bit(GLF_LFLUSH, gflags))
2251		*p++ = 'f';
2252	if (test_bit(GLF_PENDING_REPLY, gflags))
2253		*p++ = 'R';
2254	if (test_bit(GLF_HAVE_REPLY, gflags))
2255		*p++ = 'r';
2256	if (test_bit(GLF_INITIAL, gflags))
2257		*p++ = 'a';
2258	if (test_bit(GLF_HAVE_FROZEN_REPLY, gflags))
2259		*p++ = 'F';
2260	if (!list_empty(&gl->gl_holders))
2261		*p++ = 'q';
2262	if (test_bit(GLF_LRU, gflags))
2263		*p++ = 'L';
2264	if (gl->gl_object)
2265		*p++ = 'o';
2266	if (test_bit(GLF_BLOCKING, gflags))
2267		*p++ = 'b';
2268	if (test_bit(GLF_INSTANTIATE_NEEDED, gflags))
2269		*p++ = 'n';
2270	if (test_bit(GLF_INSTANTIATE_IN_PROG, gflags))
2271		*p++ = 'N';
2272	if (test_bit(GLF_TRY_TO_EVICT, gflags))
2273		*p++ = 'e';
2274	if (test_bit(GLF_VERIFY_DELETE, gflags))
2275		*p++ = 'E';
2276	if (test_bit(GLF_DEFER_DELETE, gflags))
2277		*p++ = 's';
2278	if (test_bit(GLF_CANCELING, gflags))
2279		*p++ = 'C';
2280	*p = 0;
2281	return buf;
2282}
2283
2284/**
2285 * gfs2_dump_glock - print information about a glock
2286 * @seq: The seq_file struct
2287 * @gl: the glock
2288 * @fsid: If true, also dump the file system id
2289 *
2290 * The file format is as follows:
2291 * One line per object, capital letters are used to indicate objects
2292 * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
2293 * other objects are indented by a single space and follow the glock to
2294 * which they are related. Fields are indicated by lower case letters
2295 * followed by a colon and the field value, except for strings which are in
2296 * [] so that its possible to see if they are composed of spaces for
2297 * example. The field's are n = number (id of the object), f = flags,
2298 * t = type, s = state, r = refcount, e = error, p = pid.
2299 *
2300 */
2301
2302void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2303{
2304	const struct gfs2_glock_operations *glops = gl->gl_ops;
2305	unsigned long long dtime;
2306	const struct gfs2_holder *gh;
2307	char gflags_buf[32];
2308	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2309	char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2310	unsigned long nrpages = 0;
2311
2312	if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2313		struct address_space *mapping = gfs2_glock2aspace(gl);
2314
2315		nrpages = mapping->nrpages;
2316	}
2317	memset(fs_id_buf, 0, sizeof(fs_id_buf));
2318	if (fsid && sdp) /* safety precaution */
2319		sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
2320	dtime = jiffies - gl->gl_demote_time;
2321	dtime *= 1000000/HZ; /* demote time in uSec */
2322	if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2323		dtime = 0;
2324	gfs2_print_dbg(seq, "%sG:  s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2325		       "v:%d r:%d m:%ld p:%lu\n",
2326		       fs_id_buf, state2str(gl->gl_state),
2327		       gl->gl_name.ln_type,
2328		       (unsigned long long)gl->gl_name.ln_number,
2329		       gflags2str(gflags_buf, gl),
2330		       state2str(gl->gl_target),
2331		       state2str(gl->gl_demote_state), dtime,
2332		       atomic_read(&gl->gl_ail_count),
2333		       atomic_read(&gl->gl_revokes),
2334		       (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2335
2336	list_for_each_entry(gh, &gl->gl_holders, gh_list)
2337		dump_holder(seq, gh, fs_id_buf);
2338
2339	if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2340		glops->go_dump(seq, gl, fs_id_buf);
2341}
2342
2343static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2344{
2345	struct gfs2_glock *gl = iter_ptr;
2346
2347	seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2348		   gl->gl_name.ln_type,
2349		   (unsigned long long)gl->gl_name.ln_number,
2350		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2351		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2352		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2353		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2354		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2355		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2356		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2357		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2358	return 0;
2359}
2360
2361static const char *gfs2_gltype[] = {
2362	"type",
2363	"reserved",
2364	"nondisk",
2365	"inode",
2366	"rgrp",
2367	"meta",
2368	"iopen",
2369	"flock",
2370	"plock",
2371	"quota",
2372	"journal",
2373};
2374
2375static const char *gfs2_stype[] = {
2376	[GFS2_LKS_SRTT]		= "srtt",
2377	[GFS2_LKS_SRTTVAR]	= "srttvar",
2378	[GFS2_LKS_SRTTB]	= "srttb",
2379	[GFS2_LKS_SRTTVARB]	= "srttvarb",
2380	[GFS2_LKS_SIRT]		= "sirt",
2381	[GFS2_LKS_SIRTVAR]	= "sirtvar",
2382	[GFS2_LKS_DCOUNT]	= "dlm",
2383	[GFS2_LKS_QCOUNT]	= "queue",
2384};
2385
2386#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2387
2388static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2389{
2390	struct gfs2_sbd *sdp = seq->private;
2391	loff_t pos = *(loff_t *)iter_ptr;
2392	unsigned index = pos >> 3;
2393	unsigned subindex = pos & 0x07;
2394	int i;
2395
2396	if (index == 0 && subindex != 0)
2397		return 0;
2398
2399	seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
2400		   (index == 0) ? "cpu": gfs2_stype[subindex]);
2401
2402	for_each_possible_cpu(i) {
2403                const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2404
2405		if (index == 0)
2406			seq_printf(seq, " %15u", i);
2407		else
2408			seq_printf(seq, " %15llu", (unsigned long long)lkstats->
2409				   lkstats[index - 1].stats[subindex]);
2410	}
2411	seq_putc(seq, '\n');
2412	return 0;
2413}
2414
2415int __init gfs2_glock_init(void)
2416{
2417	int i, ret;
2418
2419	ret = rhashtable_init(&gl_hash_table, &ht_parms);
2420	if (ret < 0)
2421		return ret;
2422
2423	glock_shrinker = shrinker_alloc(0, "gfs2-glock");
2424	if (!glock_shrinker) {
2425		rhashtable_destroy(&gl_hash_table);
2426		return -ENOMEM;
2427	}
2428
2429	glock_shrinker->count_objects = gfs2_glock_shrink_count;
2430	glock_shrinker->scan_objects = gfs2_glock_shrink_scan;
2431
2432	shrinker_register(glock_shrinker);
2433
2434	for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2435		init_waitqueue_head(glock_wait_table + i);
2436
2437	return 0;
2438}
2439
2440void gfs2_glock_exit(void)
2441{
2442	shrinker_free(glock_shrinker);
2443	rhashtable_destroy(&gl_hash_table);
2444}
2445
2446static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2447{
2448	struct gfs2_glock *gl = gi->gl;
2449
2450	if (gl) {
2451		if (n == 0)
2452			return;
2453		gfs2_glock_put_async(gl);
2454	}
2455	for (;;) {
2456		gl = rhashtable_walk_next(&gi->hti);
2457		if (IS_ERR_OR_NULL(gl)) {
2458			if (gl == ERR_PTR(-EAGAIN)) {
2459				n = 1;
2460				continue;
2461			}
2462			gl = NULL;
2463			break;
2464		}
2465		if (gl->gl_name.ln_sbd != gi->sdp)
2466			continue;
2467		if (n <= 1) {
2468			if (!lockref_get_not_dead(&gl->gl_lockref))
2469				continue;
2470			break;
2471		} else {
2472			if (__lockref_is_dead(&gl->gl_lockref))
2473				continue;
2474			n--;
2475		}
2476	}
2477	gi->gl = gl;
2478}
2479
2480static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2481	__acquires(RCU)
2482{
2483	struct gfs2_glock_iter *gi = seq->private;
2484	loff_t n;
2485
2486	/*
2487	 * We can either stay where we are, skip to the next hash table
2488	 * entry, or start from the beginning.
2489	 */
2490	if (*pos < gi->last_pos) {
2491		rhashtable_walk_exit(&gi->hti);
2492		rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2493		n = *pos + 1;
2494	} else {
2495		n = *pos - gi->last_pos;
2496	}
2497
2498	rhashtable_walk_start(&gi->hti);
2499
2500	gfs2_glock_iter_next(gi, n);
2501	gi->last_pos = *pos;
2502	return gi->gl;
2503}
2504
2505static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2506				 loff_t *pos)
2507{
2508	struct gfs2_glock_iter *gi = seq->private;
2509
2510	(*pos)++;
2511	gi->last_pos = *pos;
2512	gfs2_glock_iter_next(gi, 1);
2513	return gi->gl;
2514}
2515
2516static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2517	__releases(RCU)
2518{
2519	struct gfs2_glock_iter *gi = seq->private;
2520
2521	rhashtable_walk_stop(&gi->hti);
2522}
2523
2524static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2525{
2526	dump_glock(seq, iter_ptr, false);
2527	return 0;
2528}
2529
2530static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2531{
2532	preempt_disable();
2533	if (*pos >= GFS2_NR_SBSTATS)
2534		return NULL;
2535	return pos;
2536}
2537
2538static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2539				   loff_t *pos)
2540{
2541	(*pos)++;
2542	if (*pos >= GFS2_NR_SBSTATS)
2543		return NULL;
2544	return pos;
2545}
2546
2547static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2548{
2549	preempt_enable();
2550}
2551
2552static const struct seq_operations gfs2_glock_seq_ops = {
2553	.start = gfs2_glock_seq_start,
2554	.next  = gfs2_glock_seq_next,
2555	.stop  = gfs2_glock_seq_stop,
2556	.show  = gfs2_glock_seq_show,
2557};
2558
2559static const struct seq_operations gfs2_glstats_seq_ops = {
2560	.start = gfs2_glock_seq_start,
2561	.next  = gfs2_glock_seq_next,
2562	.stop  = gfs2_glock_seq_stop,
2563	.show  = gfs2_glstats_seq_show,
2564};
2565
2566static const struct seq_operations gfs2_sbstats_sops = {
2567	.start = gfs2_sbstats_seq_start,
2568	.next  = gfs2_sbstats_seq_next,
2569	.stop  = gfs2_sbstats_seq_stop,
2570	.show  = gfs2_sbstats_seq_show,
2571};
2572
2573#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2574
2575static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2576			      const struct seq_operations *ops)
2577{
2578	int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2579	if (ret == 0) {
2580		struct seq_file *seq = file->private_data;
2581		struct gfs2_glock_iter *gi = seq->private;
2582
2583		gi->sdp = inode->i_private;
2584		seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2585		if (seq->buf)
2586			seq->size = GFS2_SEQ_GOODSIZE;
2587		/*
2588		 * Initially, we are "before" the first hash table entry; the
2589		 * first call to rhashtable_walk_next gets us the first entry.
2590		 */
2591		gi->last_pos = -1;
2592		gi->gl = NULL;
2593		rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2594	}
2595	return ret;
2596}
2597
2598static int gfs2_glocks_open(struct inode *inode, struct file *file)
2599{
2600	return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2601}
2602
2603static int gfs2_glocks_release(struct inode *inode, struct file *file)
2604{
2605	struct seq_file *seq = file->private_data;
2606	struct gfs2_glock_iter *gi = seq->private;
2607
2608	if (gi->gl)
2609		gfs2_glock_put(gi->gl);
2610	rhashtable_walk_exit(&gi->hti);
2611	return seq_release_private(inode, file);
2612}
2613
2614static int gfs2_glstats_open(struct inode *inode, struct file *file)
2615{
2616	return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2617}
2618
2619static const struct file_operations gfs2_glocks_fops = {
2620	.owner   = THIS_MODULE,
2621	.open    = gfs2_glocks_open,
2622	.read    = seq_read,
2623	.llseek  = seq_lseek,
2624	.release = gfs2_glocks_release,
2625};
2626
2627static const struct file_operations gfs2_glstats_fops = {
2628	.owner   = THIS_MODULE,
2629	.open    = gfs2_glstats_open,
2630	.read    = seq_read,
2631	.llseek  = seq_lseek,
2632	.release = gfs2_glocks_release,
2633};
2634
2635struct gfs2_glockfd_iter {
2636	struct super_block *sb;
2637	unsigned int tgid;
2638	struct task_struct *task;
2639	unsigned int fd;
2640	struct file *file;
2641};
2642
2643static struct task_struct *gfs2_glockfd_next_task(struct gfs2_glockfd_iter *i)
2644{
2645	struct pid_namespace *ns = task_active_pid_ns(current);
2646	struct pid *pid;
2647
2648	if (i->task)
2649		put_task_struct(i->task);
2650
2651	rcu_read_lock();
2652retry:
2653	i->task = NULL;
2654	pid = find_ge_pid(i->tgid, ns);
2655	if (pid) {
2656		i->tgid = pid_nr_ns(pid, ns);
2657		i->task = pid_task(pid, PIDTYPE_TGID);
2658		if (!i->task) {
2659			i->tgid++;
2660			goto retry;
2661		}
2662		get_task_struct(i->task);
2663	}
2664	rcu_read_unlock();
2665	return i->task;
2666}
2667
2668static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i)
2669{
2670	if (i->file) {
2671		fput(i->file);
2672		i->file = NULL;
2673	}
2674
2675	for(;; i->fd++) {
2676		i->file = fget_task_next(i->task, &i->fd);
2677		if (!i->file) {
2678			i->fd = 0;
2679			break;
2680		}
2681
2682		if (file_inode(i->file)->i_sb == i->sb)
2683			break;
2684
2685		fput(i->file);
2686	}
2687	return i->file;
2688}
2689
2690static void *gfs2_glockfd_seq_start(struct seq_file *seq, loff_t *pos)
2691{
2692	struct gfs2_glockfd_iter *i = seq->private;
2693
2694	if (*pos)
2695		return NULL;
2696	while (gfs2_glockfd_next_task(i)) {
2697		if (gfs2_glockfd_next_file(i))
2698			return i;
2699		i->tgid++;
2700	}
2701	return NULL;
2702}
2703
2704static void *gfs2_glockfd_seq_next(struct seq_file *seq, void *iter_ptr,
2705				   loff_t *pos)
2706{
2707	struct gfs2_glockfd_iter *i = seq->private;
2708
2709	(*pos)++;
2710	i->fd++;
2711	do {
2712		if (gfs2_glockfd_next_file(i))
2713			return i;
2714		i->tgid++;
2715	} while (gfs2_glockfd_next_task(i));
2716	return NULL;
2717}
2718
2719static void gfs2_glockfd_seq_stop(struct seq_file *seq, void *iter_ptr)
2720{
2721	struct gfs2_glockfd_iter *i = seq->private;
2722
2723	if (i->file)
2724		fput(i->file);
2725	if (i->task)
2726		put_task_struct(i->task);
2727}
2728
2729static void gfs2_glockfd_seq_show_flock(struct seq_file *seq,
2730					struct gfs2_glockfd_iter *i)
2731{
2732	struct gfs2_file *fp = i->file->private_data;
2733	struct gfs2_holder *fl_gh = &fp->f_fl_gh;
2734	struct lm_lockname gl_name = { .ln_type = LM_TYPE_RESERVED };
2735
2736	if (!READ_ONCE(fl_gh->gh_gl))
2737		return;
2738
2739	spin_lock(&i->file->f_lock);
2740	if (gfs2_holder_initialized(fl_gh))
2741		gl_name = fl_gh->gh_gl->gl_name;
2742	spin_unlock(&i->file->f_lock);
2743
2744	if (gl_name.ln_type != LM_TYPE_RESERVED) {
2745		seq_printf(seq, "%d %u %u/%llx\n",
2746			   i->tgid, i->fd, gl_name.ln_type,
2747			   (unsigned long long)gl_name.ln_number);
2748	}
2749}
2750
2751static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr)
2752{
2753	struct gfs2_glockfd_iter *i = seq->private;
2754	struct inode *inode = file_inode(i->file);
2755	struct gfs2_glock *gl;
2756
2757	inode_lock_shared(inode);
2758	gl = GFS2_I(inode)->i_iopen_gh.gh_gl;
2759	if (gl) {
2760		seq_printf(seq, "%d %u %u/%llx\n",
2761			   i->tgid, i->fd, gl->gl_name.ln_type,
2762			   (unsigned long long)gl->gl_name.ln_number);
2763	}
2764	gfs2_glockfd_seq_show_flock(seq, i);
2765	inode_unlock_shared(inode);
2766	return 0;
2767}
2768
2769static const struct seq_operations gfs2_glockfd_seq_ops = {
2770	.start = gfs2_glockfd_seq_start,
2771	.next  = gfs2_glockfd_seq_next,
2772	.stop  = gfs2_glockfd_seq_stop,
2773	.show  = gfs2_glockfd_seq_show,
2774};
2775
2776static int gfs2_glockfd_open(struct inode *inode, struct file *file)
2777{
2778	struct gfs2_glockfd_iter *i;
2779	struct gfs2_sbd *sdp = inode->i_private;
2780
2781	i = __seq_open_private(file, &gfs2_glockfd_seq_ops,
2782			       sizeof(struct gfs2_glockfd_iter));
2783	if (!i)
2784		return -ENOMEM;
2785	i->sb = sdp->sd_vfs;
2786	return 0;
2787}
2788
2789static const struct file_operations gfs2_glockfd_fops = {
2790	.owner   = THIS_MODULE,
2791	.open    = gfs2_glockfd_open,
2792	.read    = seq_read,
2793	.llseek  = seq_lseek,
2794	.release = seq_release_private,
2795};
2796
2797DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
2798
2799void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2800{
2801	sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2802
2803	debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2804			    &gfs2_glocks_fops);
2805
2806	debugfs_create_file("glockfd", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2807			    &gfs2_glockfd_fops);
2808
2809	debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2810			    &gfs2_glstats_fops);
2811
2812	debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2813			    &gfs2_sbstats_fops);
2814}
2815
2816void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2817{
2818	debugfs_remove_recursive(sdp->debugfs_dir);
2819	sdp->debugfs_dir = NULL;
2820}
2821
2822void gfs2_register_debugfs(void)
2823{
2824	gfs2_root = debugfs_create_dir("gfs2", NULL);
2825}
2826
2827void gfs2_unregister_debugfs(void)
2828{
2829	debugfs_remove(gfs2_root);
2830	gfs2_root = NULL;
2831}