xfs_log_cil.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2010 Red Hat, Inc. All Rights Reserved.
   4 */
   5
   6#include "xfs.h"
   7#include "xfs_fs.h"
   8#include "xfs_format.h"
   9#include "xfs_log_format.h"
  10#include "xfs_shared.h"
  11#include "xfs_trans_resv.h"
  12#include "xfs_mount.h"
  13#include "xfs_extent_busy.h"
  14#include "xfs_trans.h"
  15#include "xfs_trans_priv.h"
  16#include "xfs_log.h"
  17#include "xfs_log_priv.h"
  18#include "xfs_trace.h"
  19#include "xfs_discard.h"
  20
  21/*
  22 * Allocate a new ticket. Failing to get a new ticket makes it really hard to
  23 * recover, so we don't allow failure here. Also, we allocate in a context that
  24 * we don't want to be issuing transactions from, so we need to tell the
  25 * allocation code this as well.
  26 *
  27 * We don't reserve any space for the ticket - we are going to steal whatever
  28 * space we require from transactions as they commit. To ensure we reserve all
  29 * the space required, we need to set the current reservation of the ticket to
  30 * zero so that we know to steal the initial transaction overhead from the
  31 * first transaction commit.
  32 */
  33static struct xlog_ticket *
  34xlog_cil_ticket_alloc(
  35	struct xlog	*log)
  36{
  37	struct xlog_ticket *tic;
  38
  39	tic = xlog_ticket_alloc(log, 0, 1, 0);
  40
  41	/*
  42	 * set the current reservation to zero so we know to steal the basic
  43	 * transaction overhead reservation from the first transaction commit.
  44	 */
  45	tic->t_curr_res = 0;
  46	tic->t_iclog_hdrs = 0;
  47	return tic;
  48}
  49
  50static inline void
  51xlog_cil_set_iclog_hdr_count(struct xfs_cil *cil)
  52{
  53	struct xlog	*log = cil->xc_log;
  54
  55	atomic_set(&cil->xc_iclog_hdrs,
  56		   (XLOG_CIL_BLOCKING_SPACE_LIMIT(log) /
  57			(log->l_iclog_size - log->l_iclog_hsize)));
  58}
  59
  60/*
  61 * Check if the current log item was first committed in this sequence.
  62 * We can't rely on just the log item being in the CIL, we have to check
  63 * the recorded commit sequence number.
  64 *
  65 * Note: for this to be used in a non-racy manner, it has to be called with
  66 * CIL flushing locked out. As a result, it should only be used during the
  67 * transaction commit process when deciding what to format into the item.
  68 */
  69static bool
  70xlog_item_in_current_chkpt(
  71	struct xfs_cil		*cil,
  72	struct xfs_log_item	*lip)
  73{
  74	if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
  75		return false;
  76
  77	/*
  78	 * li_seq is written on the first commit of a log item to record the
  79	 * first checkpoint it is written to. Hence if it is different to the
  80	 * current sequence, we're in a new checkpoint.
  81	 */
  82	return lip->li_seq == READ_ONCE(cil->xc_current_sequence);
  83}
  84
  85bool
  86xfs_log_item_in_current_chkpt(
  87	struct xfs_log_item *lip)
  88{
  89	return xlog_item_in_current_chkpt(lip->li_log->l_cilp, lip);
  90}
  91
  92/*
  93 * Unavoidable forward declaration - xlog_cil_push_work() calls
  94 * xlog_cil_ctx_alloc() itself.
  95 */
  96static void xlog_cil_push_work(struct work_struct *work);
  97
  98static struct xfs_cil_ctx *
  99xlog_cil_ctx_alloc(void)
 100{
 101	struct xfs_cil_ctx	*ctx;
 102
 103	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL | __GFP_NOFAIL);
 104	INIT_LIST_HEAD(&ctx->committing);
 105	INIT_LIST_HEAD(&ctx->busy_extents.extent_list);
 106	INIT_LIST_HEAD(&ctx->log_items);
 107	INIT_LIST_HEAD(&ctx->lv_chain);
 108	INIT_WORK(&ctx->push_work, xlog_cil_push_work);
 109	return ctx;
 110}
 111
 112/*
 113 * Aggregate the CIL per cpu structures into global counts, lists, etc and
 114 * clear the percpu state ready for the next context to use. This is called
 115 * from the push code with the context lock held exclusively, hence nothing else
 116 * will be accessing or modifying the per-cpu counters.
 117 */
 118static void
 119xlog_cil_push_pcp_aggregate(
 120	struct xfs_cil		*cil,
 121	struct xfs_cil_ctx	*ctx)
 122{
 123	struct xlog_cil_pcp	*cilpcp;
 124	int			cpu;
 125
 126	for_each_cpu(cpu, &ctx->cil_pcpmask) {
 127		cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
 128
 129		ctx->ticket->t_curr_res += cilpcp->space_reserved;
 130		cilpcp->space_reserved = 0;
 131
 132		if (!list_empty(&cilpcp->busy_extents)) {
 133			list_splice_init(&cilpcp->busy_extents,
 134					&ctx->busy_extents.extent_list);
 135		}
 136		if (!list_empty(&cilpcp->log_items))
 137			list_splice_init(&cilpcp->log_items, &ctx->log_items);
 138
 139		/*
 140		 * We're in the middle of switching cil contexts.  Reset the
 141		 * counter we use to detect when the current context is nearing
 142		 * full.
 143		 */
 144		cilpcp->space_used = 0;
 145	}
 146}
 147
 148/*
 149 * Aggregate the CIL per-cpu space used counters into the global atomic value.
 150 * This is called when the per-cpu counter aggregation will first pass the soft
 151 * limit threshold so we can switch to atomic counter aggregation for accurate
 152 * detection of hard limit traversal.
 153 */
 154static void
 155xlog_cil_insert_pcp_aggregate(
 156	struct xfs_cil		*cil,
 157	struct xfs_cil_ctx	*ctx)
 158{
 159	int			cpu;
 160	int			count = 0;
 161
 162	/* Trigger atomic updates then aggregate only for the first caller */
 163	if (!test_and_clear_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags))
 164		return;
 165
 166	/*
 167	 * We can race with other cpus setting cil_pcpmask.  However, we've
 168	 * atomically cleared PCP_SPACE which forces other threads to add to
 169	 * the global space used count.  cil_pcpmask is a superset of cilpcp
 170	 * structures that could have a nonzero space_used.
 171	 */
 172	for_each_cpu(cpu, &ctx->cil_pcpmask) {
 173		struct xlog_cil_pcp	*cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
 174
 175		count += xchg(&cilpcp->space_used, 0);
 176	}
 177	atomic_add(count, &ctx->space_used);
 178}
 179
 180static void
 181xlog_cil_ctx_switch(
 182	struct xfs_cil		*cil,
 183	struct xfs_cil_ctx	*ctx)
 184{
 185	xlog_cil_set_iclog_hdr_count(cil);
 186	set_bit(XLOG_CIL_EMPTY, &cil->xc_flags);
 187	set_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags);
 188	ctx->sequence = ++cil->xc_current_sequence;
 189	ctx->cil = cil;
 190	cil->xc_ctx = ctx;
 191}
 192
 193/*
 194 * After the first stage of log recovery is done, we know where the head and
 195 * tail of the log are. We need this log initialisation done before we can
 196 * initialise the first CIL checkpoint context.
 197 *
 198 * Here we allocate a log ticket to track space usage during a CIL push.  This
 199 * ticket is passed to xlog_write() directly so that we don't slowly leak log
 200 * space by failing to account for space used by log headers and additional
 201 * region headers for split regions.
 202 */
 203void
 204xlog_cil_init_post_recovery(
 205	struct xlog	*log)
 206{
 207	log->l_cilp->xc_ctx->ticket = xlog_cil_ticket_alloc(log);
 208	log->l_cilp->xc_ctx->sequence = 1;
 209	xlog_cil_set_iclog_hdr_count(log->l_cilp);
 210}
 211
 212static inline int
 213xlog_cil_iovec_space(
 214	uint	niovecs)
 215{
 216	return round_up((sizeof(struct xfs_log_vec) +
 217					niovecs * sizeof(struct xfs_log_iovec)),
 218			sizeof(uint64_t));
 219}
 220
 221/*
 222 * Allocate or pin log vector buffers for CIL insertion.
 223 *
 224 * The CIL currently uses disposable buffers for copying a snapshot of the
 225 * modified items into the log during a push. The biggest problem with this is
 226 * the requirement to allocate the disposable buffer during the commit if:
 227 *	a) does not exist; or
 228 *	b) it is too small
 229 *
 230 * If we do this allocation within xlog_cil_insert_format_items(), it is done
 231 * under the xc_ctx_lock, which means that a CIL push cannot occur during
 232 * the memory allocation. This means that we have a potential deadlock situation
 233 * under low memory conditions when we have lots of dirty metadata pinned in
 234 * the CIL and we need a CIL commit to occur to free memory.
 235 *
 236 * To avoid this, we need to move the memory allocation outside the
 237 * xc_ctx_lock, but because the log vector buffers are disposable, that opens
 238 * up a TOCTOU race condition w.r.t. the CIL committing and removing the log
 239 * vector buffers between the check and the formatting of the item into the
 240 * log vector buffer within the xc_ctx_lock.
 241 *
 242 * Because the log vector buffer needs to be unchanged during the CIL push
 243 * process, we cannot share the buffer between the transaction commit (which
 244 * modifies the buffer) and the CIL push context that is writing the changes
 245 * into the log. This means skipping preallocation of buffer space is
 246 * unreliable, but we most definitely do not want to be allocating and freeing
 247 * buffers unnecessarily during commits when overwrites can be done safely.
 248 *
 249 * The simplest solution to this problem is to allocate a shadow buffer when a
 250 * log item is committed for the second time, and then to only use this buffer
 251 * if necessary. The buffer can remain attached to the log item until such time
 252 * it is needed, and this is the buffer that is reallocated to match the size of
 253 * the incoming modification. Then during the formatting of the item we can swap
 254 * the active buffer with the new one if we can't reuse the existing buffer. We
 255 * don't free the old buffer as it may be reused on the next modification if
 256 * it's size is right, otherwise we'll free and reallocate it at that point.
 257 *
 258 * This function builds a vector for the changes in each log item in the
 259 * transaction. It then works out the length of the buffer needed for each log
 260 * item, allocates them and attaches the vector to the log item in preparation
 261 * for the formatting step which occurs under the xc_ctx_lock.
 262 *
 263 * While this means the memory footprint goes up, it avoids the repeated
 264 * alloc/free pattern that repeated modifications of an item would otherwise
 265 * cause, and hence minimises the CPU overhead of such behaviour.
 266 */
 267static void
 268xlog_cil_alloc_shadow_bufs(
 269	struct xlog		*log,
 270	struct xfs_trans	*tp)
 271{
 272	struct xfs_log_item	*lip;
 273
 274	list_for_each_entry(lip, &tp->t_items, li_trans) {
 275		struct xfs_log_vec *lv;
 276		int	niovecs = 0;
 277		int	nbytes = 0;
 278		int	alloc_size;
 279		bool	ordered = false;
 280
 281		/* Skip items which aren't dirty in this transaction. */
 282		if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
 283			continue;
 284
 285		/* get number of vecs and size of data to be stored */
 286		lip->li_ops->iop_size(lip, &niovecs, &nbytes);
 287
 288		/*
 289		 * Ordered items need to be tracked but we do not wish to write
 290		 * them. We need a logvec to track the object, but we do not
 291		 * need an iovec or buffer to be allocated for copying data.
 292		 */
 293		if (niovecs == XFS_LOG_VEC_ORDERED) {
 294			ordered = true;
 295			niovecs = 0;
 296			nbytes = 0;
 297		}
 298
 299		/*
 300		 * We 64-bit align the length of each iovec so that the start of
 301		 * the next one is naturally aligned.  We'll need to account for
 302		 * that slack space here.
 303		 *
 304		 * We also add the xlog_op_header to each region when
 305		 * formatting, but that's not accounted to the size of the item
 306		 * at this point. Hence we'll need an addition number of bytes
 307		 * for each vector to hold an opheader.
 308		 *
 309		 * Then round nbytes up to 64-bit alignment so that the initial
 310		 * buffer alignment is easy to calculate and verify.
 311		 */
 312		nbytes = xlog_item_space(niovecs, nbytes);
 313
 314		/*
 315		 * The data buffer needs to start 64-bit aligned, so round up
 316		 * that space to ensure we can align it appropriately and not
 317		 * overrun the buffer.
 318		 */
 319		alloc_size = nbytes + xlog_cil_iovec_space(niovecs);
 320
 321		/*
 322		 * if we have no shadow buffer, or it is too small, we need to
 323		 * reallocate it.
 324		 */
 325		if (!lip->li_lv_shadow ||
 326		    alloc_size > lip->li_lv_shadow->lv_alloc_size) {
 327			/*
 328			 * We free and allocate here as a realloc would copy
 329			 * unnecessary data. We don't use kvzalloc() for the
 330			 * same reason - we don't need to zero the data area in
 331			 * the buffer, only the log vector header and the iovec
 332			 * storage.
 333			 */
 334			kvfree(lip->li_lv_shadow);
 335			lv = xlog_kvmalloc(alloc_size);
 336
 337			memset(lv, 0, xlog_cil_iovec_space(niovecs));
 338
 339			INIT_LIST_HEAD(&lv->lv_list);
 340			lv->lv_item = lip;
 341			lv->lv_alloc_size = alloc_size;
 342			if (ordered)
 343				lv->lv_buf_used = XFS_LOG_VEC_ORDERED;
 344			else
 345				lv->lv_iovecp = (struct xfs_log_iovec *)&lv[1];
 346			lip->li_lv_shadow = lv;
 347		} else {
 348			/* same or smaller, optimise common overwrite case */
 349			lv = lip->li_lv_shadow;
 350			if (ordered)
 351				lv->lv_buf_used = XFS_LOG_VEC_ORDERED;
 352			else
 353				lv->lv_buf_used = 0;
 354			lv->lv_bytes = 0;
 355		}
 356
 357		/* Ensure the lv is set up according to ->iop_size */
 358		lv->lv_niovecs = niovecs;
 359
 360		/* The allocated data region lies beyond the iovec region */
 361		lv->lv_buf = (char *)lv + xlog_cil_iovec_space(niovecs);
 362	}
 363
 364}
 365
 366/*
 367 * Prepare the log item for insertion into the CIL. Calculate the difference in
 368 * log space it will consume, and if it is a new item pin it as well.
 369 */
 370STATIC void
 371xfs_cil_prepare_item(
 372	struct xlog		*log,
 373	struct xfs_log_item	*lip,
 374	struct xfs_log_vec	*lv,
 375	int			*diff_len)
 376{
 377	/* Account for the new LV being passed in */
 378	if (lv->lv_buf_used != XFS_LOG_VEC_ORDERED)
 379		*diff_len += lv->lv_bytes;
 380
 381	/*
 382	 * If there is no old LV, this is the first time we've seen the item in
 383	 * this CIL context and so we need to pin it. If we are replacing the
 384	 * old lv, then remove the space it accounts for and make it the shadow
 385	 * buffer for later freeing. In both cases we are now switching to the
 386	 * shadow buffer, so update the pointer to it appropriately.
 387	 */
 388	if (!lip->li_lv) {
 389		if (lv->lv_item->li_ops->iop_pin)
 390			lv->lv_item->li_ops->iop_pin(lv->lv_item);
 391		lv->lv_item->li_lv_shadow = NULL;
 392	} else if (lip->li_lv != lv) {
 393		ASSERT(lv->lv_buf_used != XFS_LOG_VEC_ORDERED);
 394
 395		*diff_len -= lip->li_lv->lv_bytes;
 396		lv->lv_item->li_lv_shadow = lip->li_lv;
 397	}
 398
 399	/* attach new log vector to log item */
 400	lv->lv_item->li_lv = lv;
 401
 402	/*
 403	 * If this is the first time the item is being committed to the
 404	 * CIL, store the sequence number on the log item so we can
 405	 * tell in future commits whether this is the first checkpoint
 406	 * the item is being committed into.
 407	 */
 408	if (!lv->lv_item->li_seq)
 409		lv->lv_item->li_seq = log->l_cilp->xc_ctx->sequence;
 410}
 411
 412/*
 413 * Format log item into a flat buffers
 414 *
 415 * For delayed logging, we need to hold a formatted buffer containing all the
 416 * changes on the log item. This enables us to relog the item in memory and
 417 * write it out asynchronously without needing to relock the object that was
 418 * modified at the time it gets written into the iclog.
 419 *
 420 * This function takes the prepared log vectors attached to each log item, and
 421 * formats the changes into the log vector buffer. The buffer it uses is
 422 * dependent on the current state of the vector in the CIL - the shadow lv is
 423 * guaranteed to be large enough for the current modification, but we will only
 424 * use that if we can't reuse the existing lv. If we can't reuse the existing
 425 * lv, then simple swap it out for the shadow lv. We don't free it - that is
 426 * done lazily either by th enext modification or the freeing of the log item.
 427 *
 428 * We don't set up region headers during this process; we simply copy the
 429 * regions into the flat buffer. We can do this because we still have to do a
 430 * formatting step to write the regions into the iclog buffer.  Writing the
 431 * ophdrs during the iclog write means that we can support splitting large
 432 * regions across iclog boundares without needing a change in the format of the
 433 * item/region encapsulation.
 434 *
 435 * Hence what we need to do now is change the rewrite the vector array to point
 436 * to the copied region inside the buffer we just allocated. This allows us to
 437 * format the regions into the iclog as though they are being formatted
 438 * directly out of the objects themselves.
 439 */
 440static void
 441xlog_cil_insert_format_items(
 442	struct xlog		*log,
 443	struct xfs_trans	*tp,
 444	int			*diff_len)
 445{
 446	struct xfs_log_item	*lip;
 447
 448	/* Bail out if we didn't find a log item.  */
 449	if (list_empty(&tp->t_items)) {
 450		ASSERT(0);
 451		return;
 452	}
 453
 454	list_for_each_entry(lip, &tp->t_items, li_trans) {
 455		struct xfs_log_vec *lv = lip->li_lv;
 456		struct xfs_log_vec *shadow = lip->li_lv_shadow;
 457
 458		/* Skip items which aren't dirty in this transaction. */
 459		if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
 460			continue;
 461
 462		/*
 463		 * The formatting size information is already attached to
 464		 * the shadow lv on the log item.
 465		 */
 466		if (shadow->lv_buf_used == XFS_LOG_VEC_ORDERED) {
 467			if (!lv) {
 468				lv = shadow;
 469				lv->lv_item = lip;
 470			}
 471			ASSERT(shadow->lv_alloc_size == lv->lv_alloc_size);
 472			xfs_cil_prepare_item(log, lip, lv, diff_len);
 473			continue;
 474		}
 475
 476		/* Skip items that do not have any vectors for writing */
 477		if (!shadow->lv_niovecs)
 478			continue;
 479
 480		/* compare to existing item size */
 481		if (lv && shadow->lv_alloc_size <= lv->lv_alloc_size) {
 482			/* same or smaller, optimise common overwrite case */
 483
 484			/*
 485			 * set the item up as though it is a new insertion so
 486			 * that the space reservation accounting is correct.
 487			 */
 488			*diff_len -= lv->lv_bytes;
 489
 490			/* Ensure the lv is set up according to ->iop_size */
 491			lv->lv_niovecs = shadow->lv_niovecs;
 492
 493			/* reset the lv buffer information for new formatting */
 494			lv->lv_buf_used = 0;
 495			lv->lv_bytes = 0;
 496			lv->lv_buf = (char *)lv +
 497					xlog_cil_iovec_space(lv->lv_niovecs);
 498		} else {
 499			/* switch to shadow buffer! */
 500			lv = shadow;
 501			lv->lv_item = lip;
 502		}
 503
 504		ASSERT(IS_ALIGNED((unsigned long)lv->lv_buf, sizeof(uint64_t)));
 505		lip->li_ops->iop_format(lip, lv);
 506		xfs_cil_prepare_item(log, lip, lv, diff_len);
 507	}
 508}
 509
 510/*
 511 * The use of lockless waitqueue_active() requires that the caller has
 512 * serialised itself against the wakeup call in xlog_cil_push_work(). That
 513 * can be done by either holding the push lock or the context lock.
 514 */
 515static inline bool
 516xlog_cil_over_hard_limit(
 517	struct xlog	*log,
 518	int32_t		space_used)
 519{
 520	if (waitqueue_active(&log->l_cilp->xc_push_wait))
 521		return true;
 522	if (space_used >= XLOG_CIL_BLOCKING_SPACE_LIMIT(log))
 523		return true;
 524	return false;
 525}
 526
 527/*
 528 * Insert the log items into the CIL and calculate the difference in space
 529 * consumed by the item. Add the space to the checkpoint ticket and calculate
 530 * if the change requires additional log metadata. If it does, take that space
 531 * as well. Remove the amount of space we added to the checkpoint ticket from
 532 * the current transaction ticket so that the accounting works out correctly.
 533 */
 534static void
 535xlog_cil_insert_items(
 536	struct xlog		*log,
 537	struct xfs_trans	*tp,
 538	uint32_t		released_space)
 539{
 540	struct xfs_cil		*cil = log->l_cilp;
 541	struct xfs_cil_ctx	*ctx = cil->xc_ctx;
 542	struct xfs_log_item	*lip;
 543	int			len = 0;
 544	int			iovhdr_res = 0, split_res = 0, ctx_res = 0;
 545	int			space_used;
 546	int			order;
 547	unsigned int		cpu_nr;
 548	struct xlog_cil_pcp	*cilpcp;
 549
 550	ASSERT(tp);
 551
 552	/*
 553	 * We can do this safely because the context can't checkpoint until we
 554	 * are done so it doesn't matter exactly how we update the CIL.
 555	 */
 556	xlog_cil_insert_format_items(log, tp, &len);
 557
 558	/*
 559	 * Subtract the space released by intent cancelation from the space we
 560	 * consumed so that we remove it from the CIL space and add it back to
 561	 * the current transaction reservation context.
 562	 */
 563	len -= released_space;
 564
 565	/*
 566	 * Grab the per-cpu pointer for the CIL before we start any accounting.
 567	 * That ensures that we are running with pre-emption disabled and so we
 568	 * can't be scheduled away between split sample/update operations that
 569	 * are done without outside locking to serialise them.
 570	 */
 571	cpu_nr = get_cpu();
 572	cilpcp = this_cpu_ptr(cil->xc_pcp);
 573
 574	/* Tell the future push that there was work added by this CPU. */
 575	if (!cpumask_test_cpu(cpu_nr, &ctx->cil_pcpmask))
 576		cpumask_test_and_set_cpu(cpu_nr, &ctx->cil_pcpmask);
 577
 578	/*
 579	 * We need to take the CIL checkpoint unit reservation on the first
 580	 * commit into the CIL. Test the XLOG_CIL_EMPTY bit first so we don't
 581	 * unnecessarily do an atomic op in the fast path here. We can clear the
 582	 * XLOG_CIL_EMPTY bit as we are under the xc_ctx_lock here and that
 583	 * needs to be held exclusively to reset the XLOG_CIL_EMPTY bit.
 584	 */
 585	if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) &&
 586	    test_and_clear_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
 587		ctx_res = ctx->ticket->t_unit_res;
 588
 589	/*
 590	 * Check if we need to steal iclog headers. atomic_read() is not a
 591	 * locked atomic operation, so we can check the value before we do any
 592	 * real atomic ops in the fast path. If we've already taken the CIL unit
 593	 * reservation from this commit, we've already got one iclog header
 594	 * space reserved so we have to account for that otherwise we risk
 595	 * overrunning the reservation on this ticket.
 596	 *
 597	 * If the CIL is already at the hard limit, we might need more header
 598	 * space that originally reserved. So steal more header space from every
 599	 * commit that occurs once we are over the hard limit to ensure the CIL
 600	 * push won't run out of reservation space.
 601	 *
 602	 * This can steal more than we need, but that's OK.
 603	 *
 604	 * The cil->xc_ctx_lock provides the serialisation necessary for safely
 605	 * calling xlog_cil_over_hard_limit() in this context.
 606	 */
 607	space_used = atomic_read(&ctx->space_used) + cilpcp->space_used + len;
 608	if (atomic_read(&cil->xc_iclog_hdrs) > 0 ||
 609	    xlog_cil_over_hard_limit(log, space_used)) {
 610		split_res = log->l_iclog_hsize +
 611					sizeof(struct xlog_op_header);
 612		if (ctx_res)
 613			ctx_res += split_res * (tp->t_ticket->t_iclog_hdrs - 1);
 614		else
 615			ctx_res = split_res * tp->t_ticket->t_iclog_hdrs;
 616		atomic_sub(tp->t_ticket->t_iclog_hdrs, &cil->xc_iclog_hdrs);
 617	}
 618	cilpcp->space_reserved += ctx_res;
 619
 620	/*
 621	 * Accurately account when over the soft limit, otherwise fold the
 622	 * percpu count into the global count if over the per-cpu threshold.
 623	 */
 624	if (!test_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) {
 625		atomic_add(len, &ctx->space_used);
 626	} else if (cilpcp->space_used + len >
 627			(XLOG_CIL_SPACE_LIMIT(log) / num_online_cpus())) {
 628		space_used = atomic_add_return(cilpcp->space_used + len,
 629						&ctx->space_used);
 630		cilpcp->space_used = 0;
 631
 632		/*
 633		 * If we just transitioned over the soft limit, we need to
 634		 * transition to the global atomic counter.
 635		 */
 636		if (space_used >= XLOG_CIL_SPACE_LIMIT(log))
 637			xlog_cil_insert_pcp_aggregate(cil, ctx);
 638	} else {
 639		cilpcp->space_used += len;
 640	}
 641	/* attach the transaction to the CIL if it has any busy extents */
 642	if (!list_empty(&tp->t_busy))
 643		list_splice_init(&tp->t_busy, &cilpcp->busy_extents);
 644
 645	/*
 646	 * Now update the order of everything modified in the transaction
 647	 * and insert items into the CIL if they aren't already there.
 648	 * We do this here so we only need to take the CIL lock once during
 649	 * the transaction commit.
 650	 */
 651	order = atomic_inc_return(&ctx->order_id);
 652	list_for_each_entry(lip, &tp->t_items, li_trans) {
 653		/* Skip items which aren't dirty in this transaction. */
 654		if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
 655			continue;
 656
 657		lip->li_order_id = order;
 658		if (!list_empty(&lip->li_cil))
 659			continue;
 660		list_add_tail(&lip->li_cil, &cilpcp->log_items);
 661	}
 662	put_cpu();
 663
 664	/*
 665	 * If we've overrun the reservation, dump the tx details before we move
 666	 * the log items. Shutdown is imminent...
 667	 */
 668	tp->t_ticket->t_curr_res -= ctx_res + len;
 669	if (WARN_ON(tp->t_ticket->t_curr_res < 0)) {
 670		xfs_warn(log->l_mp, "Transaction log reservation overrun:");
 671		xfs_warn(log->l_mp,
 672			 "  log items: %d bytes (iov hdrs: %d bytes)",
 673			 len, iovhdr_res);
 674		xfs_warn(log->l_mp, "  split region headers: %d bytes",
 675			 split_res);
 676		xfs_warn(log->l_mp, "  ctx ticket: %d bytes", ctx_res);
 677		xlog_print_trans(tp);
 678		xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
 679	}
 680}
 681
 682static inline void
 683xlog_cil_ail_insert_batch(
 684	struct xfs_ail		*ailp,
 685	struct xfs_ail_cursor	*cur,
 686	struct xfs_log_item	**log_items,
 687	int			nr_items,
 688	xfs_lsn_t		commit_lsn)
 689{
 690	int	i;
 691
 692	spin_lock(&ailp->ail_lock);
 693	/* xfs_trans_ail_update_bulk drops ailp->ail_lock */
 694	xfs_trans_ail_update_bulk(ailp, cur, log_items, nr_items, commit_lsn);
 695
 696	for (i = 0; i < nr_items; i++) {
 697		struct xfs_log_item *lip = log_items[i];
 698
 699		if (lip->li_ops->iop_unpin)
 700			lip->li_ops->iop_unpin(lip, 0);
 701	}
 702}
 703
 704/*
 705 * Take the checkpoint's log vector chain of items and insert the attached log
 706 * items into the AIL. This uses bulk insertion techniques to minimise AIL lock
 707 * traffic.
 708 *
 709 * The AIL tracks log items via the start record LSN of the checkpoint,
 710 * not the commit record LSN. This is because we can pipeline multiple
 711 * checkpoints, and so the start record of checkpoint N+1 can be
 712 * written before the commit record of checkpoint N. i.e:
 713 *
 714 *   start N			commit N
 715 *	+-------------+------------+----------------+
 716 *		  start N+1			commit N+1
 717 *
 718 * The tail of the log cannot be moved to the LSN of commit N when all
 719 * the items of that checkpoint are written back, because then the
 720 * start record for N+1 is no longer in the active portion of the log
 721 * and recovery will fail/corrupt the filesystem.
 722 *
 723 * Hence when all the log items in checkpoint N are written back, the
 724 * tail of the log most now only move as far forwards as the start LSN
 725 * of checkpoint N+1.
 726 *
 727 * If we are called with the aborted flag set, it is because a log write during
 728 * a CIL checkpoint commit has failed. In this case, all the items in the
 729 * checkpoint have already gone through iop_committed and iop_committing, which
 730 * means that checkpoint commit abort handling is treated exactly the same as an
 731 * iclog write error even though we haven't started any IO yet. Hence in this
 732 * case all we need to do is iop_committed processing, followed by an
 733 * iop_unpin(aborted) call.
 734 *
 735 * The AIL cursor is used to optimise the insert process. If commit_lsn is not
 736 * at the end of the AIL, the insert cursor avoids the need to walk the AIL to
 737 * find the insertion point on every xfs_log_item_batch_insert() call. This
 738 * saves a lot of needless list walking and is a net win, even though it
 739 * slightly increases that amount of AIL lock traffic to set it up and tear it
 740 * down.
 741 */
 742static void
 743xlog_cil_ail_insert(
 744	struct xfs_cil_ctx	*ctx,
 745	bool			aborted)
 746{
 747#define LOG_ITEM_BATCH_SIZE	32
 748	struct xfs_ail		*ailp = ctx->cil->xc_log->l_ailp;
 749	struct xfs_log_item	*log_items[LOG_ITEM_BATCH_SIZE];
 750	struct xfs_log_vec	*lv;
 751	struct xfs_ail_cursor	cur;
 752	xfs_lsn_t		old_head;
 753	int			i = 0;
 754
 755	/*
 756	 * Update the AIL head LSN with the commit record LSN of this
 757	 * checkpoint. As iclogs are always completed in order, this should
 758	 * always be the same (as iclogs can contain multiple commit records) or
 759	 * higher LSN than the current head. We do this before insertion of the
 760	 * items so that log space checks during insertion will reflect the
 761	 * space that this checkpoint has already consumed.  We call
 762	 * xfs_ail_update_finish() so that tail space and space-based wakeups
 763	 * will be recalculated appropriately.
 764	 */
 765	ASSERT(XFS_LSN_CMP(ctx->commit_lsn, ailp->ail_head_lsn) >= 0 ||
 766			aborted);
 767	spin_lock(&ailp->ail_lock);
 768	xfs_trans_ail_cursor_last(ailp, &cur, ctx->start_lsn);
 769	old_head = ailp->ail_head_lsn;
 770	ailp->ail_head_lsn = ctx->commit_lsn;
 771	/* xfs_ail_update_finish() drops the ail_lock */
 772	xfs_ail_update_finish(ailp, NULLCOMMITLSN);
 773
 774	/*
 775	 * We move the AIL head forwards to account for the space used in the
 776	 * log before we remove that space from the grant heads. This prevents a
 777	 * transient condition where reservation space appears to become
 778	 * available on return, only for it to disappear again immediately as
 779	 * the AIL head update accounts in the log tail space.
 780	 */
 781	smp_wmb();	/* paired with smp_rmb in xlog_grant_space_left */
 782	xlog_grant_return_space(ailp->ail_log, old_head, ailp->ail_head_lsn);
 783
 784	/* unpin all the log items */
 785	list_for_each_entry(lv, &ctx->lv_chain, lv_list) {
 786		struct xfs_log_item	*lip = lv->lv_item;
 787		xfs_lsn_t		item_lsn;
 788
 789		if (aborted) {
 790			trace_xlog_ail_insert_abort(lip);
 791			set_bit(XFS_LI_ABORTED, &lip->li_flags);
 792		}
 793
 794		if (lip->li_ops->flags & XFS_ITEM_RELEASE_WHEN_COMMITTED) {
 795			lip->li_ops->iop_release(lip);
 796			continue;
 797		}
 798
 799		if (lip->li_ops->iop_committed)
 800			item_lsn = lip->li_ops->iop_committed(lip,
 801					ctx->start_lsn);
 802		else
 803			item_lsn = ctx->start_lsn;
 804
 805		/* item_lsn of -1 means the item needs no further processing */
 806		if (XFS_LSN_CMP(item_lsn, (xfs_lsn_t)-1) == 0)
 807			continue;
 808
 809		/*
 810		 * if we are aborting the operation, no point in inserting the
 811		 * object into the AIL as we are in a shutdown situation.
 812		 */
 813		if (aborted) {
 814			ASSERT(xlog_is_shutdown(ailp->ail_log));
 815			if (lip->li_ops->iop_unpin)
 816				lip->li_ops->iop_unpin(lip, 1);
 817			continue;
 818		}
 819
 820		if (item_lsn != ctx->start_lsn) {
 821
 822			/*
 823			 * Not a bulk update option due to unusual item_lsn.
 824			 * Push into AIL immediately, rechecking the lsn once
 825			 * we have the ail lock. Then unpin the item. This does
 826			 * not affect the AIL cursor the bulk insert path is
 827			 * using.
 828			 */
 829			spin_lock(&ailp->ail_lock);
 830			if (XFS_LSN_CMP(item_lsn, lip->li_lsn) > 0)
 831				xfs_trans_ail_update(ailp, lip, item_lsn);
 832			else
 833				spin_unlock(&ailp->ail_lock);
 834			if (lip->li_ops->iop_unpin)
 835				lip->li_ops->iop_unpin(lip, 0);
 836			continue;
 837		}
 838
 839		/* Item is a candidate for bulk AIL insert.  */
 840		log_items[i++] = lv->lv_item;
 841		if (i >= LOG_ITEM_BATCH_SIZE) {
 842			xlog_cil_ail_insert_batch(ailp, &cur, log_items,
 843					LOG_ITEM_BATCH_SIZE, ctx->start_lsn);
 844			i = 0;
 845		}
 846	}
 847
 848	/* make sure we insert the remainder! */
 849	if (i)
 850		xlog_cil_ail_insert_batch(ailp, &cur, log_items, i,
 851				ctx->start_lsn);
 852
 853	spin_lock(&ailp->ail_lock);
 854	xfs_trans_ail_cursor_done(&cur);
 855	spin_unlock(&ailp->ail_lock);
 856}
 857
 858static void
 859xlog_cil_free_logvec(
 860	struct list_head	*lv_chain)
 861{
 862	struct xfs_log_vec	*lv;
 863
 864	while (!list_empty(lv_chain)) {
 865		lv = list_first_entry(lv_chain, struct xfs_log_vec, lv_list);
 866		list_del_init(&lv->lv_list);
 867		kvfree(lv);
 868	}
 869}
 870
 871/*
 872 * Mark all items committed and clear busy extents. We free the log vector
 873 * chains in a separate pass so that we unpin the log items as quickly as
 874 * possible.
 875 */
 876static void
 877xlog_cil_committed(
 878	struct xfs_cil_ctx	*ctx)
 879{
 880	struct xfs_mount	*mp = ctx->cil->xc_log->l_mp;
 881	bool			abort = xlog_is_shutdown(ctx->cil->xc_log);
 882
 883	/*
 884	 * If the I/O failed, we're aborting the commit and already shutdown.
 885	 * Wake any commit waiters before aborting the log items so we don't
 886	 * block async log pushers on callbacks. Async log pushers explicitly do
 887	 * not wait on log force completion because they may be holding locks
 888	 * required to unpin items.
 889	 */
 890	if (abort) {
 891		spin_lock(&ctx->cil->xc_push_lock);
 892		wake_up_all(&ctx->cil->xc_start_wait);
 893		wake_up_all(&ctx->cil->xc_commit_wait);
 894		spin_unlock(&ctx->cil->xc_push_lock);
 895	}
 896
 897	xlog_cil_ail_insert(ctx, abort);
 898
 899	xfs_extent_busy_sort(&ctx->busy_extents.extent_list);
 900	xfs_extent_busy_clear(&ctx->busy_extents.extent_list,
 901			      xfs_has_discard(mp) && !abort);
 902
 903	spin_lock(&ctx->cil->xc_push_lock);
 904	list_del(&ctx->committing);
 905	spin_unlock(&ctx->cil->xc_push_lock);
 906
 907	xlog_cil_free_logvec(&ctx->lv_chain);
 908
 909	if (!list_empty(&ctx->busy_extents.extent_list)) {
 910		ctx->busy_extents.owner = ctx;
 911		xfs_discard_extents(mp, &ctx->busy_extents);
 912		return;
 913	}
 914
 915	kfree(ctx);
 916}
 917
 918void
 919xlog_cil_process_committed(
 920	struct list_head	*list)
 921{
 922	struct xfs_cil_ctx	*ctx;
 923
 924	while ((ctx = list_first_entry_or_null(list,
 925			struct xfs_cil_ctx, iclog_entry))) {
 926		list_del(&ctx->iclog_entry);
 927		xlog_cil_committed(ctx);
 928	}
 929}
 930
 931/*
 932* Record the LSN of the iclog we were just granted space to start writing into.
 933* If the context doesn't have a start_lsn recorded, then this iclog will
 934* contain the start record for the checkpoint. Otherwise this write contains
 935* the commit record for the checkpoint.
 936*/
 937void
 938xlog_cil_set_ctx_write_state(
 939	struct xfs_cil_ctx	*ctx,
 940	struct xlog_in_core	*iclog)
 941{
 942	struct xfs_cil		*cil = ctx->cil;
 943	xfs_lsn_t		lsn = be64_to_cpu(iclog->ic_header->h_lsn);
 944
 945	ASSERT(!ctx->commit_lsn);
 946	if (!ctx->start_lsn) {
 947		spin_lock(&cil->xc_push_lock);
 948		/*
 949		 * The LSN we need to pass to the log items on transaction
 950		 * commit is the LSN reported by the first log vector write, not
 951		 * the commit lsn. If we use the commit record lsn then we can
 952		 * move the grant write head beyond the tail LSN and overwrite
 953		 * it.
 954		 */
 955		ctx->start_lsn = lsn;
 956		wake_up_all(&cil->xc_start_wait);
 957		spin_unlock(&cil->xc_push_lock);
 958
 959		/*
 960		 * Make sure the metadata we are about to overwrite in the log
 961		 * has been flushed to stable storage before this iclog is
 962		 * issued.
 963		 */
 964		spin_lock(&cil->xc_log->l_icloglock);
 965		iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
 966		spin_unlock(&cil->xc_log->l_icloglock);
 967		return;
 968	}
 969
 970	/*
 971	 * Take a reference to the iclog for the context so that we still hold
 972	 * it when xlog_write is done and has released it. This means the
 973	 * context controls when the iclog is released for IO.
 974	 */
 975	atomic_inc(&iclog->ic_refcnt);
 976
 977	/*
 978	 * xlog_state_get_iclog_space() guarantees there is enough space in the
 979	 * iclog for an entire commit record, so we can attach the context
 980	 * callbacks now.  This needs to be done before we make the commit_lsn
 981	 * visible to waiters so that checkpoints with commit records in the
 982	 * same iclog order their IO completion callbacks in the same order that
 983	 * the commit records appear in the iclog.
 984	 */
 985	spin_lock(&cil->xc_log->l_icloglock);
 986	list_add_tail(&ctx->iclog_entry, &iclog->ic_callbacks);
 987	spin_unlock(&cil->xc_log->l_icloglock);
 988
 989	/*
 990	 * Now we can record the commit LSN and wake anyone waiting for this
 991	 * sequence to have the ordered commit record assigned to a physical
 992	 * location in the log.
 993	 */
 994	spin_lock(&cil->xc_push_lock);
 995	ctx->commit_iclog = iclog;
 996	ctx->commit_lsn = lsn;
 997	wake_up_all(&cil->xc_commit_wait);
 998	spin_unlock(&cil->xc_push_lock);
 999}
1000
1001
1002/*
1003 * Ensure that the order of log writes follows checkpoint sequence order. This
1004 * relies on the context LSN being zero until the log write has guaranteed the
1005 * LSN that the log write will start at via xlog_state_get_iclog_space().
1006 */
1007enum _record_type {
1008	_START_RECORD,
1009	_COMMIT_RECORD,
1010};
1011
1012static int
1013xlog_cil_order_write(
1014	struct xfs_cil		*cil,
1015	xfs_csn_t		sequence,
1016	enum _record_type	record)
1017{
1018	struct xfs_cil_ctx	*ctx;
1019
1020restart:
1021	spin_lock(&cil->xc_push_lock);
1022	list_for_each_entry(ctx, &cil->xc_committing, committing) {
1023		/*
1024		 * Avoid getting stuck in this loop because we were woken by the
1025		 * shutdown, but then went back to sleep once already in the
1026		 * shutdown state.
1027		 */
1028		if (xlog_is_shutdown(cil->xc_log)) {
1029			spin_unlock(&cil->xc_push_lock);
1030			return -EIO;
1031		}
1032
1033		/*
1034		 * Higher sequences will wait for this one so skip them.
1035		 * Don't wait for our own sequence, either.
1036		 */
1037		if (ctx->sequence >= sequence)
1038			continue;
1039
1040		/* Wait until the LSN for the record has been recorded. */
1041		switch (record) {
1042		case _START_RECORD:
1043			if (!ctx->start_lsn) {
1044				xlog_wait(&cil->xc_start_wait, &cil->xc_push_lock);
1045				goto restart;
1046			}
1047			break;
1048		case _COMMIT_RECORD:
1049			if (!ctx->commit_lsn) {
1050				xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock);
1051				goto restart;
1052			}
1053			break;
1054		}
1055	}
1056	spin_unlock(&cil->xc_push_lock);
1057	return 0;
1058}
1059
1060/*
1061 * Write out the log vector change now attached to the CIL context. This will
1062 * write a start record that needs to be strictly ordered in ascending CIL
1063 * sequence order so that log recovery will always use in-order start LSNs when
1064 * replaying checkpoints.
1065 */
1066static int
1067xlog_cil_write_chain(
1068	struct xfs_cil_ctx	*ctx,
1069	uint32_t		chain_len)
1070{
1071	struct xlog		*log = ctx->cil->xc_log;
1072	int			error;
1073
1074	error = xlog_cil_order_write(ctx->cil, ctx->sequence, _START_RECORD);
1075	if (error)
1076		return error;
1077	return xlog_write(log, ctx, &ctx->lv_chain, ctx->ticket, chain_len);
1078}
1079
1080/*
1081 * Write out the commit record of a checkpoint transaction to close off a
1082 * running log write. These commit records are strictly ordered in ascending CIL
1083 * sequence order so that log recovery will always replay the checkpoints in the
1084 * correct order.
1085 */
1086static int
1087xlog_cil_write_commit_record(
1088	struct xfs_cil_ctx	*ctx)
1089{
1090	struct xlog		*log = ctx->cil->xc_log;
1091	struct xlog_op_header	ophdr = {
1092		.oh_clientid = XFS_TRANSACTION,
1093		.oh_tid = cpu_to_be32(ctx->ticket->t_tid),
1094		.oh_flags = XLOG_COMMIT_TRANS,
1095	};
1096	struct xfs_log_iovec	reg = {
1097		.i_addr = &ophdr,
1098		.i_len = sizeof(struct xlog_op_header),
1099		.i_type = XLOG_REG_TYPE_COMMIT,
1100	};
1101	struct xfs_log_vec	vec = {
1102		.lv_niovecs = 1,
1103		.lv_iovecp = &reg,
1104	};
1105	int			error;
1106	LIST_HEAD(lv_chain);
1107	list_add(&vec.lv_list, &lv_chain);
1108
1109	if (xlog_is_shutdown(log))
1110		return -EIO;
1111
1112	error = xlog_cil_order_write(ctx->cil, ctx->sequence, _COMMIT_RECORD);
1113	if (error)
1114		return error;
1115
1116	/* account for space used by record data */
1117	ctx->ticket->t_curr_res -= reg.i_len;
1118	error = xlog_write(log, ctx, &lv_chain, ctx->ticket, reg.i_len);
1119	if (error)
1120		xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
1121	return error;
1122}
1123
1124struct xlog_cil_trans_hdr {
1125	struct xlog_op_header	oph[2];
1126	struct xfs_trans_header	thdr;
1127	struct xfs_log_iovec	lhdr[2];
1128};
1129
1130/*
1131 * Build a checkpoint transaction header to begin the journal transaction.  We
1132 * need to account for the space used by the transaction header here as it is
1133 * not accounted for in xlog_write().
1134 *
1135 * This is the only place we write a transaction header, so we also build the
1136 * log opheaders that indicate the start of a log transaction and wrap the
1137 * transaction header. We keep the start record in it's own log vector rather
1138 * than compacting them into a single region as this ends up making the logic
1139 * in xlog_write() for handling empty opheaders for start, commit and unmount
1140 * records much simpler.
1141 */
1142static void
1143xlog_cil_build_trans_hdr(
1144	struct xfs_cil_ctx	*ctx,
1145	struct xlog_cil_trans_hdr *hdr,
1146	struct xfs_log_vec	*lvhdr,
1147	int			num_iovecs)
1148{
1149	struct xlog_ticket	*tic = ctx->ticket;
1150	__be32			tid = cpu_to_be32(tic->t_tid);
1151
1152	memset(hdr, 0, sizeof(*hdr));
1153
1154	/* Log start record */
1155	hdr->oph[0].oh_tid = tid;
1156	hdr->oph[0].oh_clientid = XFS_TRANSACTION;
1157	hdr->oph[0].oh_flags = XLOG_START_TRANS;
1158
1159	/* log iovec region pointer */
1160	hdr->lhdr[0].i_addr = &hdr->oph[0];
1161	hdr->lhdr[0].i_len = sizeof(struct xlog_op_header);
1162	hdr->lhdr[0].i_type = XLOG_REG_TYPE_LRHEADER;
1163
1164	/* log opheader */
1165	hdr->oph[1].oh_tid = tid;
1166	hdr->oph[1].oh_clientid = XFS_TRANSACTION;
1167	hdr->oph[1].oh_len = cpu_to_be32(sizeof(struct xfs_trans_header));
1168
1169	/* transaction header in host byte order format */
1170	hdr->thdr.th_magic = XFS_TRANS_HEADER_MAGIC;
1171	hdr->thdr.th_type = XFS_TRANS_CHECKPOINT;
1172	hdr->thdr.th_tid = tic->t_tid;
1173	hdr->thdr.th_num_items = num_iovecs;
1174
1175	/* log iovec region pointer */
1176	hdr->lhdr[1].i_addr = &hdr->oph[1];
1177	hdr->lhdr[1].i_len = sizeof(struct xlog_op_header) +
1178				sizeof(struct xfs_trans_header);
1179	hdr->lhdr[1].i_type = XLOG_REG_TYPE_TRANSHDR;
1180
1181	lvhdr->lv_niovecs = 2;
1182	lvhdr->lv_iovecp = &hdr->lhdr[0];
1183	lvhdr->lv_bytes = hdr->lhdr[0].i_len + hdr->lhdr[1].i_len;
1184
1185	tic->t_curr_res -= lvhdr->lv_bytes;
1186}
1187
1188/*
1189 * CIL item reordering compare function. We want to order in ascending ID order,
1190 * but we want to leave items with the same ID in the order they were added to
1191 * the list. This is important for operations like reflink where we log 4 order
1192 * dependent intents in a single transaction when we overwrite an existing
1193 * shared extent with a new shared extent. i.e. BUI(unmap), CUI(drop),
1194 * CUI (inc), BUI(remap)...
1195 */
1196static int
1197xlog_cil_order_cmp(
1198	void			*priv,
1199	const struct list_head	*a,
1200	const struct list_head	*b)
1201{
1202	struct xfs_log_vec	*l1 = container_of(a, struct xfs_log_vec, lv_list);
1203	struct xfs_log_vec	*l2 = container_of(b, struct xfs_log_vec, lv_list);
1204
1205	return l1->lv_order_id > l2->lv_order_id;
1206}
1207
1208/*
1209 * Pull all the log vectors off the items in the CIL, and remove the items from
1210 * the CIL. We don't need the CIL lock here because it's only needed on the
1211 * transaction commit side which is currently locked out by the flush lock.
1212 *
1213 * If a log item is marked with a whiteout, we do not need to write it to the
1214 * journal and so we just move them to the whiteout list for the caller to
1215 * dispose of appropriately.
1216 */
1217static void
1218xlog_cil_build_lv_chain(
1219	struct xfs_cil_ctx	*ctx,
1220	struct list_head	*whiteouts,
1221	uint32_t		*num_iovecs,
1222	uint32_t		*num_bytes)
1223{
1224	while (!list_empty(&ctx->log_items)) {
1225		struct xfs_log_item	*item;
1226		struct xfs_log_vec	*lv;
1227
1228		item = list_first_entry(&ctx->log_items,
1229					struct xfs_log_item, li_cil);
1230
1231		if (test_bit(XFS_LI_WHITEOUT, &item->li_flags)) {
1232			list_move(&item->li_cil, whiteouts);
1233			trace_xfs_cil_whiteout_skip(item);
1234			continue;
1235		}
1236
1237		lv = item->li_lv;
1238		lv->lv_order_id = item->li_order_id;
1239
1240		/* we don't write ordered log vectors */
1241		if (lv->lv_buf_used != XFS_LOG_VEC_ORDERED)
1242			*num_bytes += lv->lv_bytes;
1243		*num_iovecs += lv->lv_niovecs;
1244		list_add_tail(&lv->lv_list, &ctx->lv_chain);
1245
1246		list_del_init(&item->li_cil);
1247		item->li_order_id = 0;
1248		item->li_lv = NULL;
1249	}
1250}
1251
1252static void
1253xlog_cil_cleanup_whiteouts(
1254	struct list_head	*whiteouts)
1255{
1256	while (!list_empty(whiteouts)) {
1257		struct xfs_log_item *item = list_first_entry(whiteouts,
1258						struct xfs_log_item, li_cil);
1259		list_del_init(&item->li_cil);
1260		trace_xfs_cil_whiteout_unpin(item);
1261		item->li_ops->iop_unpin(item, 1);
1262	}
1263}
1264
1265/*
1266 * Push the Committed Item List to the log.
1267 *
1268 * If the current sequence is the same as xc_push_seq we need to do a flush. If
1269 * xc_push_seq is less than the current sequence, then it has already been
1270 * flushed and we don't need to do anything - the caller will wait for it to
1271 * complete if necessary.
1272 *
1273 * xc_push_seq is checked unlocked against the sequence number for a match.
1274 * Hence we can allow log forces to run racily and not issue pushes for the
1275 * same sequence twice.  If we get a race between multiple pushes for the same
1276 * sequence they will block on the first one and then abort, hence avoiding
1277 * needless pushes.
1278 *
1279 * This runs from a workqueue so it does not inherent any specific memory
1280 * allocation context. However, we do not want to block on memory reclaim
1281 * recursing back into the filesystem because this push may have been triggered
1282 * by memory reclaim itself. Hence we really need to run under full GFP_NOFS
1283 * contraints here.
1284 */
1285static void
1286xlog_cil_push_work(
1287	struct work_struct	*work)
1288{
1289	unsigned int		nofs_flags = memalloc_nofs_save();
1290	struct xfs_cil_ctx	*ctx =
1291		container_of(work, struct xfs_cil_ctx, push_work);
1292	struct xfs_cil		*cil = ctx->cil;
1293	struct xlog		*log = cil->xc_log;
1294	struct xfs_cil_ctx	*new_ctx;
1295	int			num_iovecs = 0;
1296	int			num_bytes = 0;
1297	int			error = 0;
1298	struct xlog_cil_trans_hdr thdr;
1299	struct xfs_log_vec	lvhdr = {};
1300	xfs_csn_t		push_seq;
1301	bool			push_commit_stable;
1302	LIST_HEAD		(whiteouts);
1303	struct xlog_ticket	*ticket;
1304
1305	new_ctx = xlog_cil_ctx_alloc();
1306	new_ctx->ticket = xlog_cil_ticket_alloc(log);
1307
1308	down_write(&cil->xc_ctx_lock);
1309
1310	spin_lock(&cil->xc_push_lock);
1311	push_seq = cil->xc_push_seq;
1312	ASSERT(push_seq <= ctx->sequence);
1313	push_commit_stable = cil->xc_push_commit_stable;
1314	cil->xc_push_commit_stable = false;
1315
1316	/*
1317	 * As we are about to switch to a new, empty CIL context, we no longer
1318	 * need to throttle tasks on CIL space overruns. Wake any waiters that
1319	 * the hard push throttle may have caught so they can start committing
1320	 * to the new context. The ctx->xc_push_lock provides the serialisation
1321	 * necessary for safely using the lockless waitqueue_active() check in
1322	 * this context.
1323	 */
1324	if (waitqueue_active(&cil->xc_push_wait))
1325		wake_up_all(&cil->xc_push_wait);
1326
1327	xlog_cil_push_pcp_aggregate(cil, ctx);
1328
1329	/*
1330	 * Check if we've anything to push. If there is nothing, then we don't
1331	 * move on to a new sequence number and so we have to be able to push
1332	 * this sequence again later.
1333	 */
1334	if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) {
1335		cil->xc_push_seq = 0;
1336		spin_unlock(&cil->xc_push_lock);
1337		goto out_skip;
1338	}
1339
1340
1341	/* check for a previously pushed sequence */
1342	if (push_seq < ctx->sequence) {
1343		spin_unlock(&cil->xc_push_lock);
1344		goto out_skip;
1345	}
1346
1347	/*
1348	 * We are now going to push this context, so add it to the committing
1349	 * list before we do anything else. This ensures that anyone waiting on
1350	 * this push can easily detect the difference between a "push in
1351	 * progress" and "CIL is empty, nothing to do".
1352	 *
1353	 * IOWs, a wait loop can now check for:
1354	 *	the current sequence not being found on the committing list;
1355	 *	an empty CIL; and
1356	 *	an unchanged sequence number
1357	 * to detect a push that had nothing to do and therefore does not need
1358	 * waiting on. If the CIL is not empty, we get put on the committing
1359	 * list before emptying the CIL and bumping the sequence number. Hence
1360	 * an empty CIL and an unchanged sequence number means we jumped out
1361	 * above after doing nothing.
1362	 *
1363	 * Hence the waiter will either find the commit sequence on the
1364	 * committing list or the sequence number will be unchanged and the CIL
1365	 * still dirty. In that latter case, the push has not yet started, and
1366	 * so the waiter will have to continue trying to check the CIL
1367	 * committing list until it is found. In extreme cases of delay, the
1368	 * sequence may fully commit between the attempts the wait makes to wait
1369	 * on the commit sequence.
1370	 */
1371	list_add(&ctx->committing, &cil->xc_committing);
1372	spin_unlock(&cil->xc_push_lock);
1373
1374	xlog_cil_build_lv_chain(ctx, &whiteouts, &num_iovecs, &num_bytes);
1375
1376	/*
1377	 * Switch the contexts so we can drop the context lock and move out
1378	 * of a shared context. We can't just go straight to the commit record,
1379	 * though - we need to synchronise with previous and future commits so
1380	 * that the commit records are correctly ordered in the log to ensure
1381	 * that we process items during log IO completion in the correct order.
1382	 *
1383	 * For example, if we get an EFI in one checkpoint and the EFD in the
1384	 * next (e.g. due to log forces), we do not want the checkpoint with
1385	 * the EFD to be committed before the checkpoint with the EFI.  Hence
1386	 * we must strictly order the commit records of the checkpoints so
1387	 * that: a) the checkpoint callbacks are attached to the iclogs in the
1388	 * correct order; and b) the checkpoints are replayed in correct order
1389	 * in log recovery.
1390	 *
1391	 * Hence we need to add this context to the committing context list so
1392	 * that higher sequences will wait for us to write out a commit record
1393	 * before they do.
1394	 *
1395	 * xfs_log_force_seq requires us to mirror the new sequence into the cil
1396	 * structure atomically with the addition of this sequence to the
1397	 * committing list. This also ensures that we can do unlocked checks
1398	 * against the current sequence in log forces without risking
1399	 * deferencing a freed context pointer.
1400	 */
1401	spin_lock(&cil->xc_push_lock);
1402	xlog_cil_ctx_switch(cil, new_ctx);
1403	spin_unlock(&cil->xc_push_lock);
1404	up_write(&cil->xc_ctx_lock);
1405
1406	/*
1407	 * Sort the log vector chain before we add the transaction headers.
1408	 * This ensures we always have the transaction headers at the start
1409	 * of the chain.
1410	 */
1411	list_sort(NULL, &ctx->lv_chain, xlog_cil_order_cmp);
1412
1413	/*
1414	 * Build a checkpoint transaction header and write it to the log to
1415	 * begin the transaction. We need to account for the space used by the
1416	 * transaction header here as it is not accounted for in xlog_write().
1417	 * Add the lvhdr to the head of the lv chain we pass to xlog_write() so
1418	 * it gets written into the iclog first.
1419	 */
1420	xlog_cil_build_trans_hdr(ctx, &thdr, &lvhdr, num_iovecs);
1421	num_bytes += lvhdr.lv_bytes;
1422	list_add(&lvhdr.lv_list, &ctx->lv_chain);
1423
1424	/*
1425	 * Take the lvhdr back off the lv_chain immediately after calling
1426	 * xlog_cil_write_chain() as it should not be passed to log IO
1427	 * completion.
1428	 */
1429	error = xlog_cil_write_chain(ctx, num_bytes);
1430	list_del(&lvhdr.lv_list);
1431	if (error)
1432		goto out_abort_free_ticket;
1433
1434	error = xlog_cil_write_commit_record(ctx);
1435	if (error)
1436		goto out_abort_free_ticket;
1437
1438	/*
1439	 * Grab the ticket from the ctx so we can ungrant it after releasing the
1440	 * commit_iclog. The ctx may be freed by the time we return from
1441	 * releasing the commit_iclog (i.e. checkpoint has been completed and
1442	 * callback run) so we can't reference the ctx after the call to
1443	 * xlog_state_release_iclog().
1444	 */
1445	ticket = ctx->ticket;
1446
1447	/*
1448	 * If the checkpoint spans multiple iclogs, wait for all previous iclogs
1449	 * to complete before we submit the commit_iclog. We can't use state
1450	 * checks for this - ACTIVE can be either a past completed iclog or a
1451	 * future iclog being filled, while WANT_SYNC through SYNC_DONE can be a
1452	 * past or future iclog awaiting IO or ordered IO completion to be run.
1453	 * In the latter case, if it's a future iclog and we wait on it, the we
1454	 * will hang because it won't get processed through to ic_force_wait
1455	 * wakeup until this commit_iclog is written to disk.  Hence we use the
1456	 * iclog header lsn and compare it to the commit lsn to determine if we
1457	 * need to wait on iclogs or not.
1458	 */
1459	spin_lock(&log->l_icloglock);
1460	if (ctx->start_lsn != ctx->commit_lsn) {
1461		xfs_lsn_t	plsn = be64_to_cpu(
1462			ctx->commit_iclog->ic_prev->ic_header->h_lsn);
1463
1464		if (plsn && XFS_LSN_CMP(plsn, ctx->commit_lsn) < 0) {
1465			/*
1466			 * Waiting on ic_force_wait orders the completion of
1467			 * iclogs older than ic_prev. Hence we only need to wait
1468			 * on the most recent older iclog here.
1469			 */
1470			xlog_wait_on_iclog(ctx->commit_iclog->ic_prev);
1471			spin_lock(&log->l_icloglock);
1472		}
1473
1474		/*
1475		 * We need to issue a pre-flush so that the ordering for this
1476		 * checkpoint is correctly preserved down to stable storage.
1477		 */
1478		ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
1479	}
1480
1481	/*
1482	 * The commit iclog must be written to stable storage to guarantee
1483	 * journal IO vs metadata writeback IO is correctly ordered on stable
1484	 * storage.
1485	 *
1486	 * If the push caller needs the commit to be immediately stable and the
1487	 * commit_iclog is not yet marked as XLOG_STATE_WANT_SYNC to indicate it
1488	 * will be written when released, switch it's state to WANT_SYNC right
1489	 * now.
1490	 */
1491	ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FUA;
1492	if (push_commit_stable &&
1493	    ctx->commit_iclog->ic_state == XLOG_STATE_ACTIVE)
1494		xlog_state_switch_iclogs(log, ctx->commit_iclog, 0);
1495	ticket = ctx->ticket;
1496	xlog_state_release_iclog(log, ctx->commit_iclog, ticket);
1497
1498	/* Not safe to reference ctx now! */
1499
1500	spin_unlock(&log->l_icloglock);
1501	xlog_cil_cleanup_whiteouts(&whiteouts);
1502	xfs_log_ticket_ungrant(log, ticket);
1503	memalloc_nofs_restore(nofs_flags);
1504	return;
1505
1506out_skip:
1507	up_write(&cil->xc_ctx_lock);
1508	xfs_log_ticket_put(new_ctx->ticket);
1509	kfree(new_ctx);
1510	memalloc_nofs_restore(nofs_flags);
1511	return;
1512
1513out_abort_free_ticket:
1514	ASSERT(xlog_is_shutdown(log));
1515	xlog_cil_cleanup_whiteouts(&whiteouts);
1516	if (!ctx->commit_iclog) {
1517		xfs_log_ticket_ungrant(log, ctx->ticket);
1518		xlog_cil_committed(ctx);
1519		memalloc_nofs_restore(nofs_flags);
1520		return;
1521	}
1522	spin_lock(&log->l_icloglock);
1523	ticket = ctx->ticket;
1524	xlog_state_release_iclog(log, ctx->commit_iclog, ticket);
1525	/* Not safe to reference ctx now! */
1526	spin_unlock(&log->l_icloglock);
1527	xfs_log_ticket_ungrant(log, ticket);
1528	memalloc_nofs_restore(nofs_flags);
1529}
1530
1531/*
1532 * We need to push CIL every so often so we don't cache more than we can fit in
1533 * the log. The limit really is that a checkpoint can't be more than half the
1534 * log (the current checkpoint is not allowed to overwrite the previous
1535 * checkpoint), but commit latency and memory usage limit this to a smaller
1536 * size.
1537 */
1538static void
1539xlog_cil_push_background(
1540	struct xlog	*log)
1541{
1542	struct xfs_cil	*cil = log->l_cilp;
1543	int		space_used = atomic_read(&cil->xc_ctx->space_used);
1544
1545	/*
1546	 * The cil won't be empty because we are called while holding the
1547	 * context lock so whatever we added to the CIL will still be there.
1548	 */
1549	ASSERT(!test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
1550
1551	/*
1552	 * We are done if:
1553	 * - we haven't used up all the space available yet; or
1554	 * - we've already queued up a push; and
1555	 * - we're not over the hard limit; and
1556	 * - nothing has been over the hard limit.
1557	 *
1558	 * If so, we don't need to take the push lock as there's nothing to do.
1559	 */
1560	if (space_used < XLOG_CIL_SPACE_LIMIT(log) ||
1561	    (cil->xc_push_seq == cil->xc_current_sequence &&
1562	     space_used < XLOG_CIL_BLOCKING_SPACE_LIMIT(log) &&
1563	     !waitqueue_active(&cil->xc_push_wait))) {
1564		up_read(&cil->xc_ctx_lock);
1565		return;
1566	}
1567
1568	spin_lock(&cil->xc_push_lock);
1569	if (cil->xc_push_seq < cil->xc_current_sequence) {
1570		cil->xc_push_seq = cil->xc_current_sequence;
1571		queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work);
1572	}
1573
1574	/*
1575	 * Drop the context lock now, we can't hold that if we need to sleep
1576	 * because we are over the blocking threshold. The push_lock is still
1577	 * held, so blocking threshold sleep/wakeup is still correctly
1578	 * serialised here.
1579	 */
1580	up_read(&cil->xc_ctx_lock);
1581
1582	/*
1583	 * If we are well over the space limit, throttle the work that is being
1584	 * done until the push work on this context has begun. Enforce the hard
1585	 * throttle on all transaction commits once it has been activated, even
1586	 * if the committing transactions have resulted in the space usage
1587	 * dipping back down under the hard limit.
1588	 *
1589	 * The ctx->xc_push_lock provides the serialisation necessary for safely
1590	 * calling xlog_cil_over_hard_limit() in this context.
1591	 */
1592	if (xlog_cil_over_hard_limit(log, space_used)) {
1593		trace_xfs_log_cil_wait(log, cil->xc_ctx->ticket);
1594		ASSERT(space_used < log->l_logsize);
1595		xlog_wait(&cil->xc_push_wait, &cil->xc_push_lock);
1596		return;
1597	}
1598
1599	spin_unlock(&cil->xc_push_lock);
1600
1601}
1602
1603/*
1604 * xlog_cil_push_now() is used to trigger an immediate CIL push to the sequence
1605 * number that is passed. When it returns, the work will be queued for
1606 * @push_seq, but it won't be completed.
1607 *
1608 * If the caller is performing a synchronous force, we will flush the workqueue
1609 * to get previously queued work moving to minimise the wait time they will
1610 * undergo waiting for all outstanding pushes to complete. The caller is
1611 * expected to do the required waiting for push_seq to complete.
1612 *
1613 * If the caller is performing an async push, we need to ensure that the
1614 * checkpoint is fully flushed out of the iclogs when we finish the push. If we
1615 * don't do this, then the commit record may remain sitting in memory in an
1616 * ACTIVE iclog. This then requires another full log force to push to disk,
1617 * which defeats the purpose of having an async, non-blocking CIL force
1618 * mechanism. Hence in this case we need to pass a flag to the push work to
1619 * indicate it needs to flush the commit record itself.
1620 */
1621static void
1622xlog_cil_push_now(
1623	struct xlog	*log,
1624	xfs_lsn_t	push_seq,
1625	bool		async)
1626{
1627	struct xfs_cil	*cil = log->l_cilp;
1628
1629	if (!cil)
1630		return;
1631
1632	ASSERT(push_seq && push_seq <= cil->xc_current_sequence);
1633
1634	/* start on any pending background push to minimise wait time on it */
1635	if (!async)
1636		flush_workqueue(cil->xc_push_wq);
1637
1638	spin_lock(&cil->xc_push_lock);
1639
1640	/*
1641	 * If this is an async flush request, we always need to set the
1642	 * xc_push_commit_stable flag even if something else has already queued
1643	 * a push. The flush caller is asking for the CIL to be on stable
1644	 * storage when the next push completes, so regardless of who has queued
1645	 * the push, the flush requires stable semantics from it.
1646	 */
1647	cil->xc_push_commit_stable = async;
1648
1649	/*
1650	 * If the CIL is empty or we've already pushed the sequence then
1651	 * there's no more work that we need to do.
1652	 */
1653	if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) ||
1654	    push_seq <= cil->xc_push_seq) {
1655		spin_unlock(&cil->xc_push_lock);
1656		return;
1657	}
1658
1659	cil->xc_push_seq = push_seq;
1660	queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work);
1661	spin_unlock(&cil->xc_push_lock);
1662}
1663
1664bool
1665xlog_cil_empty(
1666	struct xlog	*log)
1667{
1668	struct xfs_cil	*cil = log->l_cilp;
1669	bool		empty = false;
1670
1671	spin_lock(&cil->xc_push_lock);
1672	if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags))
1673		empty = true;
1674	spin_unlock(&cil->xc_push_lock);
1675	return empty;
1676}
1677
1678/*
1679 * If there are intent done items in this transaction and the related intent was
1680 * committed in the current (same) CIL checkpoint, we don't need to write either
1681 * the intent or intent done item to the journal as the change will be
1682 * journalled atomically within this checkpoint. As we cannot remove items from
1683 * the CIL here, mark the related intent with a whiteout so that the CIL push
1684 * can remove it rather than writing it to the journal. Then remove the intent
1685 * done item from the current transaction and release it so it doesn't get put
1686 * into the CIL at all.
1687 */
1688static uint32_t
1689xlog_cil_process_intents(
1690	struct xfs_cil		*cil,
1691	struct xfs_trans	*tp)
1692{
1693	struct xfs_log_item	*lip, *ilip, *next;
1694	uint32_t		len = 0;
1695
1696	list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
1697		if (!(lip->li_ops->flags & XFS_ITEM_INTENT_DONE))
1698			continue;
1699
1700		ilip = lip->li_ops->iop_intent(lip);
1701		if (!ilip || !xlog_item_in_current_chkpt(cil, ilip))
1702			continue;
1703		set_bit(XFS_LI_WHITEOUT, &ilip->li_flags);
1704		trace_xfs_cil_whiteout_mark(ilip);
1705		len += ilip->li_lv->lv_bytes;
1706		kvfree(ilip->li_lv);
1707		ilip->li_lv = NULL;
1708
1709		xfs_trans_del_item(lip);
1710		lip->li_ops->iop_release(lip);
1711	}
1712	return len;
1713}
1714
1715/*
1716 * Commit a transaction with the given vector to the Committed Item List.
1717 *
1718 * To do this, we need to format the item, pin it in memory if required and
1719 * account for the space used by the transaction. Once we have done that we
1720 * need to release the unused reservation for the transaction, attach the
1721 * transaction to the checkpoint context so we carry the busy extents through
1722 * to checkpoint completion, and then unlock all the items in the transaction.
1723 *
1724 * Called with the context lock already held in read mode to lock out
1725 * background commit, returns without it held once background commits are
1726 * allowed again.
1727 */
1728void
1729xlog_cil_commit(
1730	struct xlog		*log,
1731	struct xfs_trans	*tp,
1732	xfs_csn_t		*commit_seq,
1733	bool			regrant)
1734{
1735	struct xfs_cil		*cil = log->l_cilp;
1736	struct xfs_log_item	*lip, *next;
1737	uint32_t		released_space = 0;
1738
1739	/*
1740	 * Do all necessary memory allocation before we lock the CIL.
1741	 * This ensures the allocation does not deadlock with a CIL
1742	 * push in memory reclaim (e.g. from kswapd).
1743	 */
1744	xlog_cil_alloc_shadow_bufs(log, tp);
1745
1746	/* lock out background commit */
1747	down_read(&cil->xc_ctx_lock);
1748
1749	if (tp->t_flags & XFS_TRANS_HAS_INTENT_DONE)
1750		released_space = xlog_cil_process_intents(cil, tp);
1751
1752	xlog_cil_insert_items(log, tp, released_space);
1753
1754	if (regrant && !xlog_is_shutdown(log))
1755		xfs_log_ticket_regrant(log, tp->t_ticket);
1756	else
1757		xfs_log_ticket_ungrant(log, tp->t_ticket);
1758	tp->t_ticket = NULL;
1759	xfs_trans_unreserve_and_mod_sb(tp);
1760
1761	/*
1762	 * Once all the items of the transaction have been copied to the CIL,
1763	 * the items can be unlocked and possibly freed.
1764	 *
1765	 * This needs to be done before we drop the CIL context lock because we
1766	 * have to update state in the log items and unlock them before they go
1767	 * to disk. If we don't, then the CIL checkpoint can race with us and
1768	 * we can run checkpoint completion before we've updated and unlocked
1769	 * the log items. This affects (at least) processing of stale buffers,
1770	 * inodes and EFIs.
1771	 */
1772	trace_xfs_trans_commit_items(tp, _RET_IP_);
1773	list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) {
1774		xfs_trans_del_item(lip);
1775		if (lip->li_ops->iop_committing)
1776			lip->li_ops->iop_committing(lip, cil->xc_ctx->sequence);
1777	}
1778	if (commit_seq)
1779		*commit_seq = cil->xc_ctx->sequence;
1780
1781	/* xlog_cil_push_background() releases cil->xc_ctx_lock */
1782	xlog_cil_push_background(log);
1783}
1784
1785/*
1786 * Flush the CIL to stable storage but don't wait for it to complete. This
1787 * requires the CIL push to ensure the commit record for the push hits the disk,
1788 * but otherwise is no different to a push done from a log force.
1789 */
1790void
1791xlog_cil_flush(
1792	struct xlog	*log)
1793{
1794	xfs_csn_t	seq = log->l_cilp->xc_current_sequence;
1795
1796	trace_xfs_log_force(log->l_mp, seq, _RET_IP_);
1797	xlog_cil_push_now(log, seq, true);
1798
1799	/*
1800	 * If the CIL is empty, make sure that any previous checkpoint that may
1801	 * still be in an active iclog is pushed to stable storage.
1802	 */
1803	if (test_bit(XLOG_CIL_EMPTY, &log->l_cilp->xc_flags))
1804		xfs_log_force(log->l_mp, 0);
1805}
1806
1807/*
1808 * Conditionally push the CIL based on the sequence passed in.
1809 *
1810 * We only need to push if we haven't already pushed the sequence number given.
1811 * Hence the only time we will trigger a push here is if the push sequence is
1812 * the same as the current context.
1813 *
1814 * We return the current commit lsn to allow the callers to determine if a
1815 * iclog flush is necessary following this call.
1816 */
1817xfs_lsn_t
1818xlog_cil_force_seq(
1819	struct xlog	*log,
1820	xfs_csn_t	sequence)
1821{
1822	struct xfs_cil		*cil = log->l_cilp;
1823	struct xfs_cil_ctx	*ctx;
1824	xfs_lsn_t		commit_lsn = NULLCOMMITLSN;
1825
1826	ASSERT(sequence <= cil->xc_current_sequence);
1827
1828	if (!sequence)
1829		sequence = cil->xc_current_sequence;
1830	trace_xfs_log_force(log->l_mp, sequence, _RET_IP_);
1831
1832	/*
1833	 * check to see if we need to force out the current context.
1834	 * xlog_cil_push() handles racing pushes for the same sequence,
1835	 * so no need to deal with it here.
1836	 */
1837restart:
1838	xlog_cil_push_now(log, sequence, false);
1839
1840	/*
1841	 * See if we can find a previous sequence still committing.
1842	 * We need to wait for all previous sequence commits to complete
1843	 * before allowing the force of push_seq to go ahead. Hence block
1844	 * on commits for those as well.
1845	 */
1846	spin_lock(&cil->xc_push_lock);
1847	list_for_each_entry(ctx, &cil->xc_committing, committing) {
1848		/*
1849		 * Avoid getting stuck in this loop because we were woken by the
1850		 * shutdown, but then went back to sleep once already in the
1851		 * shutdown state.
1852		 */
1853		if (xlog_is_shutdown(log))
1854			goto out_shutdown;
1855		if (ctx->sequence > sequence)
1856			continue;
1857		if (!ctx->commit_lsn) {
1858			/*
1859			 * It is still being pushed! Wait for the push to
1860			 * complete, then start again from the beginning.
1861			 */
1862			XFS_STATS_INC(log->l_mp, xs_log_force_sleep);
1863			xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock);
1864			goto restart;
1865		}
1866		if (ctx->sequence != sequence)
1867			continue;
1868		/* found it! */
1869		commit_lsn = ctx->commit_lsn;
1870	}
1871
1872	/*
1873	 * The call to xlog_cil_push_now() executes the push in the background.
1874	 * Hence by the time we have got here it our sequence may not have been
1875	 * pushed yet. This is true if the current sequence still matches the
1876	 * push sequence after the above wait loop and the CIL still contains
1877	 * dirty objects. This is guaranteed by the push code first adding the
1878	 * context to the committing list before emptying the CIL.
1879	 *
1880	 * Hence if we don't find the context in the committing list and the
1881	 * current sequence number is unchanged then the CIL contents are
1882	 * significant.  If the CIL is empty, if means there was nothing to push
1883	 * and that means there is nothing to wait for. If the CIL is not empty,
1884	 * it means we haven't yet started the push, because if it had started
1885	 * we would have found the context on the committing list.
1886	 */
1887	if (sequence == cil->xc_current_sequence &&
1888	    !test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) {
1889		spin_unlock(&cil->xc_push_lock);
1890		goto restart;
1891	}
1892
1893	spin_unlock(&cil->xc_push_lock);
1894	return commit_lsn;
1895
1896	/*
1897	 * We detected a shutdown in progress. We need to trigger the log force
1898	 * to pass through it's iclog state machine error handling, even though
1899	 * we are already in a shutdown state. Hence we can't return
1900	 * NULLCOMMITLSN here as that has special meaning to log forces (i.e.
1901	 * LSN is already stable), so we return a zero LSN instead.
1902	 */
1903out_shutdown:
1904	spin_unlock(&cil->xc_push_lock);
1905	return 0;
1906}
1907
1908/*
1909 * Perform initial CIL structure initialisation.
1910 */
1911int
1912xlog_cil_init(
1913	struct xlog		*log)
1914{
1915	struct xfs_cil		*cil;
1916	struct xfs_cil_ctx	*ctx;
1917	struct xlog_cil_pcp	*cilpcp;
1918	int			cpu;
1919
1920	cil = kzalloc(sizeof(*cil), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1921	if (!cil)
1922		return -ENOMEM;
1923	/*
1924	 * Limit the CIL pipeline depth to 4 concurrent works to bound the
1925	 * concurrency the log spinlocks will be exposed to.
1926	 */
1927	cil->xc_push_wq = alloc_workqueue("xfs-cil/%s",
1928			XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_UNBOUND),
1929			4, log->l_mp->m_super->s_id);
1930	if (!cil->xc_push_wq)
1931		goto out_destroy_cil;
1932
1933	cil->xc_log = log;
1934	cil->xc_pcp = alloc_percpu(struct xlog_cil_pcp);
1935	if (!cil->xc_pcp)
1936		goto out_destroy_wq;
1937
1938	for_each_possible_cpu(cpu) {
1939		cilpcp = per_cpu_ptr(cil->xc_pcp, cpu);
1940		INIT_LIST_HEAD(&cilpcp->busy_extents);
1941		INIT_LIST_HEAD(&cilpcp->log_items);
1942	}
1943
1944	INIT_LIST_HEAD(&cil->xc_committing);
1945	spin_lock_init(&cil->xc_push_lock);
1946	init_waitqueue_head(&cil->xc_push_wait);
1947	init_rwsem(&cil->xc_ctx_lock);
1948	init_waitqueue_head(&cil->xc_start_wait);
1949	init_waitqueue_head(&cil->xc_commit_wait);
1950	log->l_cilp = cil;
1951
1952	ctx = xlog_cil_ctx_alloc();
1953	xlog_cil_ctx_switch(cil, ctx);
1954	return 0;
1955
1956out_destroy_wq:
1957	destroy_workqueue(cil->xc_push_wq);
1958out_destroy_cil:
1959	kfree(cil);
1960	return -ENOMEM;
1961}
1962
1963void
1964xlog_cil_destroy(
1965	struct xlog	*log)
1966{
1967	struct xfs_cil	*cil = log->l_cilp;
1968
1969	if (cil->xc_ctx) {
1970		if (cil->xc_ctx->ticket)
1971			xfs_log_ticket_put(cil->xc_ctx->ticket);
1972		kfree(cil->xc_ctx);
1973	}
1974
1975	ASSERT(test_bit(XLOG_CIL_EMPTY, &cil->xc_flags));
1976	free_percpu(cil->xc_pcp);
1977	destroy_workqueue(cil->xc_push_wq);
1978	kfree(cil);
1979}
1980