xfs_trans_ail.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
  4 * Copyright (c) 2008 Dave Chinner
  5 * All Rights Reserved.
  6 */
  7#include "xfs.h"
  8#include "xfs_fs.h"
  9#include "xfs_shared.h"
 10#include "xfs_format.h"
 11#include "xfs_log_format.h"
 12#include "xfs_trans_resv.h"
 13#include "xfs_mount.h"
 14#include "xfs_trans.h"
 15#include "xfs_trans_priv.h"
 16#include "xfs_trace.h"
 17#include "xfs_errortag.h"
 18#include "xfs_error.h"
 19#include "xfs_log.h"
 20#include "xfs_log_priv.h"
 21
 22#ifdef DEBUG
 23/*
 24 * Check that the list is sorted as it should be.
 25 *
 26 * Called with the ail lock held, but we don't want to assert fail with it
 27 * held otherwise we'll lock everything up and won't be able to debug the
 28 * cause. Hence we sample and check the state under the AIL lock and return if
 29 * everything is fine, otherwise we drop the lock and run the ASSERT checks.
 30 * Asserts may not be fatal, so pick the lock back up and continue onwards.
 31 */
 32STATIC void
 33xfs_ail_check(
 34	struct xfs_ail		*ailp,
 35	struct xfs_log_item	*lip)
 36	__must_hold(&ailp->ail_lock)
 37{
 38	struct xfs_log_item	*prev_lip;
 39	struct xfs_log_item	*next_lip;
 40	xfs_lsn_t		prev_lsn = NULLCOMMITLSN;
 41	xfs_lsn_t		next_lsn = NULLCOMMITLSN;
 42	xfs_lsn_t		lsn;
 43	bool			in_ail;
 44
 45
 46	if (list_empty(&ailp->ail_head))
 47		return;
 48
 49	/*
 50	 * Sample then check the next and previous entries are valid.
 51	 */
 52	in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags);
 53	prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail);
 54	if (&prev_lip->li_ail != &ailp->ail_head)
 55		prev_lsn = prev_lip->li_lsn;
 56	next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail);
 57	if (&next_lip->li_ail != &ailp->ail_head)
 58		next_lsn = next_lip->li_lsn;
 59	lsn = lip->li_lsn;
 60
 61	if (in_ail &&
 62	    (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) &&
 63	    (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0))
 64		return;
 65
 66	spin_unlock(&ailp->ail_lock);
 67	ASSERT(in_ail);
 68	ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0);
 69	ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0);
 70	spin_lock(&ailp->ail_lock);
 71}
 72#else /* !DEBUG */
 73#define	xfs_ail_check(a,l)
 74#endif /* DEBUG */
 75
 76/*
 77 * Return a pointer to the last item in the AIL.  If the AIL is empty, then
 78 * return NULL.
 79 */
 80static struct xfs_log_item *
 81xfs_ail_max(
 82	struct xfs_ail  *ailp)
 83{
 84	if (list_empty(&ailp->ail_head))
 85		return NULL;
 86
 87	return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail);
 88}
 89
 90/*
 91 * Return a pointer to the item which follows the given item in the AIL.  If
 92 * the given item is the last item in the list, then return NULL.
 93 */
 94static struct xfs_log_item *
 95xfs_ail_next(
 96	struct xfs_ail		*ailp,
 97	struct xfs_log_item	*lip)
 98{
 99	if (lip->li_ail.next == &ailp->ail_head)
100		return NULL;
101
102	return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail);
103}
104
105/*
106 * This is called by the log manager code to determine the LSN of the tail of
107 * the log.  This is exactly the LSN of the first item in the AIL.  If the AIL
108 * is empty, then this function returns 0.
109 *
110 * We need the AIL lock in order to get a coherent read of the lsn of the last
111 * item in the AIL.
112 */
113static xfs_lsn_t
114__xfs_ail_min_lsn(
115	struct xfs_ail		*ailp)
116{
117	struct xfs_log_item	*lip = xfs_ail_min(ailp);
118
119	if (lip)
120		return lip->li_lsn;
121	return 0;
122}
123
124xfs_lsn_t
125xfs_ail_min_lsn(
126	struct xfs_ail		*ailp)
127{
128	xfs_lsn_t		lsn;
129
130	spin_lock(&ailp->ail_lock);
131	lsn = __xfs_ail_min_lsn(ailp);
132	spin_unlock(&ailp->ail_lock);
133
134	return lsn;
135}
136
137/*
138 * The cursor keeps track of where our current traversal is up to by tracking
139 * the next item in the list for us. However, for this to be safe, removing an
140 * object from the AIL needs to invalidate any cursor that points to it. hence
141 * the traversal cursor needs to be linked to the struct xfs_ail so that
142 * deletion can search all the active cursors for invalidation.
143 */
144STATIC void
145xfs_trans_ail_cursor_init(
146	struct xfs_ail		*ailp,
147	struct xfs_ail_cursor	*cur)
148{
149	cur->item = NULL;
150	list_add_tail(&cur->list, &ailp->ail_cursors);
151}
152
153/*
154 * Get the next item in the traversal and advance the cursor.  If the cursor
155 * was invalidated (indicated by a lip of 1), restart the traversal.
156 */
157struct xfs_log_item *
158xfs_trans_ail_cursor_next(
159	struct xfs_ail		*ailp,
160	struct xfs_ail_cursor	*cur)
161{
162	struct xfs_log_item	*lip = cur->item;
163
164	if ((uintptr_t)lip & 1)
165		lip = xfs_ail_min(ailp);
166	if (lip)
167		cur->item = xfs_ail_next(ailp, lip);
168	return lip;
169}
170
171/*
172 * When the traversal is complete, we need to remove the cursor from the list
173 * of traversing cursors.
174 */
175void
176xfs_trans_ail_cursor_done(
177	struct xfs_ail_cursor	*cur)
178{
179	cur->item = NULL;
180	list_del_init(&cur->list);
181}
182
183/*
184 * Invalidate any cursor that is pointing to this item. This is called when an
185 * item is removed from the AIL. Any cursor pointing to this object is now
186 * invalid and the traversal needs to be terminated so it doesn't reference a
187 * freed object. We set the low bit of the cursor item pointer so we can
188 * distinguish between an invalidation and the end of the list when getting the
189 * next item from the cursor.
190 */
191STATIC void
192xfs_trans_ail_cursor_clear(
193	struct xfs_ail		*ailp,
194	struct xfs_log_item	*lip)
195{
196	struct xfs_ail_cursor	*cur;
197
198	list_for_each_entry(cur, &ailp->ail_cursors, list) {
199		if (cur->item == lip)
200			cur->item = (struct xfs_log_item *)
201					((uintptr_t)cur->item | 1);
202	}
203}
204
205/*
206 * Find the first item in the AIL with the given @lsn by searching in ascending
207 * LSN order and initialise the cursor to point to the next item for a
208 * ascending traversal.  Pass a @lsn of zero to initialise the cursor to the
209 * first item in the AIL. Returns NULL if the list is empty.
210 */
211struct xfs_log_item *
212xfs_trans_ail_cursor_first(
213	struct xfs_ail		*ailp,
214	struct xfs_ail_cursor	*cur,
215	xfs_lsn_t		lsn)
216{
217	struct xfs_log_item	*lip;
218
219	xfs_trans_ail_cursor_init(ailp, cur);
220
221	if (lsn == 0) {
222		lip = xfs_ail_min(ailp);
223		goto out;
224	}
225
226	list_for_each_entry(lip, &ailp->ail_head, li_ail) {
227		if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
228			goto out;
229	}
230	return NULL;
231
232out:
233	if (lip)
234		cur->item = xfs_ail_next(ailp, lip);
235	return lip;
236}
237
238static struct xfs_log_item *
239__xfs_trans_ail_cursor_last(
240	struct xfs_ail		*ailp,
241	xfs_lsn_t		lsn)
242{
243	struct xfs_log_item	*lip;
244
245	list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) {
246		if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
247			return lip;
248	}
249	return NULL;
250}
251
252/*
253 * Find the last item in the AIL with the given @lsn by searching in descending
254 * LSN order and initialise the cursor to point to that item.  If there is no
255 * item with the value of @lsn, then it sets the cursor to the last item with an
256 * LSN lower than @lsn.  Returns NULL if the list is empty.
257 */
258struct xfs_log_item *
259xfs_trans_ail_cursor_last(
260	struct xfs_ail		*ailp,
261	struct xfs_ail_cursor	*cur,
262	xfs_lsn_t		lsn)
263{
264	xfs_trans_ail_cursor_init(ailp, cur);
265	cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
266	return cur->item;
267}
268
269/*
270 * Splice the log item list into the AIL at the given LSN. We splice to the
271 * tail of the given LSN to maintain insert order for push traversals. The
272 * cursor is optional, allowing repeated updates to the same LSN to avoid
273 * repeated traversals.  This should not be called with an empty list.
274 */
275static void
276xfs_ail_splice(
277	struct xfs_ail		*ailp,
278	struct xfs_ail_cursor	*cur,
279	struct list_head	*list,
280	xfs_lsn_t		lsn)
281{
282	struct xfs_log_item	*lip;
283
284	ASSERT(!list_empty(list));
285
286	/*
287	 * Use the cursor to determine the insertion point if one is
288	 * provided.  If not, or if the one we got is not valid,
289	 * find the place in the AIL where the items belong.
290	 */
291	lip = cur ? cur->item : NULL;
292	if (!lip || (uintptr_t)lip & 1)
293		lip = __xfs_trans_ail_cursor_last(ailp, lsn);
294
295	/*
296	 * If a cursor is provided, we know we're processing the AIL
297	 * in lsn order, and future items to be spliced in will
298	 * follow the last one being inserted now.  Update the
299	 * cursor to point to that last item, now while we have a
300	 * reliable pointer to it.
301	 */
302	if (cur)
303		cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
304
305	/*
306	 * Finally perform the splice.  Unless the AIL was empty,
307	 * lip points to the item in the AIL _after_ which the new
308	 * items should go.  If lip is null the AIL was empty, so
309	 * the new items go at the head of the AIL.
310	 */
311	if (lip)
312		list_splice(list, &lip->li_ail);
313	else
314		list_splice(list, &ailp->ail_head);
315}
316
317/*
318 * Delete the given item from the AIL.
319 */
320static void
321xfs_ail_delete(
322	struct xfs_ail		*ailp,
323	struct xfs_log_item	*lip)
324{
325	xfs_ail_check(ailp, lip);
326	list_del(&lip->li_ail);
327	xfs_trans_ail_cursor_clear(ailp, lip);
328}
329
330/*
331 * Requeue a failed buffer for writeback.
332 *
333 * We clear the log item failed state here as well, but we have to be careful
334 * about reference counts because the only active reference counts on the buffer
335 * may be the failed log items. Hence if we clear the log item failed state
336 * before queuing the buffer for IO we can release all active references to
337 * the buffer and free it, leading to use after free problems in
338 * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which
339 * order we process them in - the buffer is locked, and we own the buffer list
340 * so nothing on them is going to change while we are performing this action.
341 *
342 * Hence we can safely queue the buffer for IO before we clear the failed log
343 * item state, therefore  always having an active reference to the buffer and
344 * avoiding the transient zero-reference state that leads to use-after-free.
345 */
346static inline int
347xfsaild_resubmit_item(
348	struct xfs_log_item	*lip,
349	struct list_head	*buffer_list)
350{
351	struct xfs_buf		*bp = lip->li_buf;
352
353	if (!xfs_buf_trylock(bp))
354		return XFS_ITEM_LOCKED;
355
356	if (!xfs_buf_delwri_queue(bp, buffer_list)) {
357		xfs_buf_unlock(bp);
358		return XFS_ITEM_FLUSHING;
359	}
360
361	/* protected by ail_lock */
362	list_for_each_entry(lip, &bp->b_li_list, li_bio_list)
363		clear_bit(XFS_LI_FAILED, &lip->li_flags);
364	xfs_buf_unlock(bp);
365	return XFS_ITEM_SUCCESS;
366}
367
368/*
369 * Push a single log item from the AIL.
370 *
371 * @lip may have been released and freed by the time this function returns,
372 * so callers must not dereference the log item afterwards.
373 */
374static inline uint
375xfsaild_push_item(
376	struct xfs_ail		*ailp,
377	struct xfs_log_item	*lip)
378{
379	/*
380	 * If log item pinning is enabled, skip the push and track the item as
381	 * pinned. This can help induce head-behind-tail conditions.
382	 */
383	if (XFS_TEST_ERROR(ailp->ail_log->l_mp, XFS_ERRTAG_LOG_ITEM_PIN))
384		return XFS_ITEM_PINNED;
385
386	/*
387	 * Consider the item pinned if a push callback is not defined so the
388	 * caller will force the log. This should only happen for intent items
389	 * as they are unpinned once the associated done item is committed to
390	 * the on-disk log.
391	 */
392	if (!lip->li_ops->iop_push)
393		return XFS_ITEM_PINNED;
394	if (test_bit(XFS_LI_FAILED, &lip->li_flags))
395		return xfsaild_resubmit_item(lip, &ailp->ail_buf_list);
396	return lip->li_ops->iop_push(lip, &ailp->ail_buf_list);
397}
398
399/*
400 * Compute the LSN that we'd need to push the log tail towards in order to have
401 * at least 25% of the log space free.  If the log free space already meets this
402 * threshold, this function returns the lowest LSN in the AIL to slowly keep
403 * writeback ticking over and the tail of the log moving forward.
404 */
405static xfs_lsn_t
406xfs_ail_calc_push_target(
407	struct xfs_ail		*ailp)
408{
409	struct xlog		*log = ailp->ail_log;
410	struct xfs_log_item	*lip;
411	xfs_lsn_t		target_lsn;
412	xfs_lsn_t		max_lsn;
413	xfs_lsn_t		min_lsn;
414	int32_t			free_bytes;
415	uint32_t		target_block;
416	uint32_t		target_cycle;
417
418	lockdep_assert_held(&ailp->ail_lock);
419
420	lip = xfs_ail_max(ailp);
421	if (!lip)
422		return NULLCOMMITLSN;
423
424	max_lsn = lip->li_lsn;
425	min_lsn = __xfs_ail_min_lsn(ailp);
426
427	/*
428	 * If we are supposed to push all the items in the AIL, we want to push
429	 * to the current head. We then clear the push flag so that we don't
430	 * keep pushing newly queued items beyond where the push all command was
431	 * run. If the push waiter wants to empty the ail, it should queue
432	 * itself on the ail_empty wait queue.
433	 */
434	if (test_and_clear_bit(XFS_AIL_OPSTATE_PUSH_ALL, &ailp->ail_opstate))
435		return max_lsn;
436
437	/* If someone wants the AIL empty, keep pushing everything we have. */
438	if (waitqueue_active(&ailp->ail_empty))
439		return max_lsn;
440
441	/*
442	 * Background pushing - attempt to keep 25% of the log free and if we
443	 * have that much free retain the existing target.
444	 */
445	free_bytes = log->l_logsize - xlog_lsn_sub(log, max_lsn, min_lsn);
446	if (free_bytes >= log->l_logsize >> 2)
447		return ailp->ail_target;
448
449	target_cycle = CYCLE_LSN(min_lsn);
450	target_block = BLOCK_LSN(min_lsn) + (log->l_logBBsize >> 2);
451	if (target_block >= log->l_logBBsize) {
452		target_block -= log->l_logBBsize;
453		target_cycle += 1;
454	}
455	target_lsn = xlog_assign_lsn(target_cycle, target_block);
456
457	/* Cap the target to the highest LSN known to be in the AIL. */
458	if (XFS_LSN_CMP(target_lsn, max_lsn) > 0)
459		return max_lsn;
460
461	/* If the existing target is higher than the new target, keep it. */
462	if (XFS_LSN_CMP(ailp->ail_target, target_lsn) >= 0)
463		return ailp->ail_target;
464	return target_lsn;
465}
466
467static long
468xfsaild_push(
469	struct xfs_ail		*ailp)
470{
471	struct xfs_mount	*mp = ailp->ail_log->l_mp;
472	struct xfs_ail_cursor	cur;
473	struct xfs_log_item	*lip;
474	xfs_lsn_t		lsn;
475	long			tout;
476	int			stuck = 0;
477	int			flushing = 0;
478	int			count = 0;
479
480	/*
481	 * If we encountered pinned items or did not finish writing out all
482	 * buffers the last time we ran, force a background CIL push to get the
483	 * items unpinned in the near future. We do not wait on the CIL push as
484	 * that could stall us for seconds if there is enough background IO
485	 * load. Stalling for that long when the tail of the log is pinned and
486	 * needs flushing will hard stop the transaction subsystem when log
487	 * space runs out.
488	 */
489	if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 &&
490	    (!list_empty_careful(&ailp->ail_buf_list) ||
491	     xfs_ail_min_lsn(ailp))) {
492		ailp->ail_log_flush = 0;
493
494		XFS_STATS_INC(mp, xs_push_ail_flush);
495		xlog_cil_flush(ailp->ail_log);
496	}
497
498	spin_lock(&ailp->ail_lock);
499	WRITE_ONCE(ailp->ail_target, xfs_ail_calc_push_target(ailp));
500	if (ailp->ail_target == NULLCOMMITLSN)
501		goto out_done;
502
503	/* we're done if the AIL is empty or our push has reached the end */
504	lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);
505	if (!lip)
506		goto out_done_cursor;
507
508	XFS_STATS_INC(mp, xs_push_ail);
509
510	ASSERT(ailp->ail_target != NULLCOMMITLSN);
511
512	lsn = lip->li_lsn;
513	while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) {
514		int		lock_result;
515		uint		type = lip->li_type;
516		unsigned long	flags = lip->li_flags;
517		xfs_lsn_t	item_lsn = lip->li_lsn;
518
519		if (test_bit(XFS_LI_FLUSHING, &lip->li_flags))
520			goto next_item;
521
522		/*
523		 * Note that iop_push may unlock and reacquire the AIL lock.  We
524		 * rely on the AIL cursor implementation to be able to deal with
525		 * the dropped lock.
526		 *
527		 * The log item may have been freed by the push, so it must not
528		 * be accessed or dereferenced below this line.
529		 */
530		lock_result = xfsaild_push_item(ailp, lip);
531		switch (lock_result) {
532		case XFS_ITEM_SUCCESS:
533			XFS_STATS_INC(mp, xs_push_ail_success);
534			trace_xfs_ail_push(ailp, type, flags, item_lsn);
535
536			ailp->ail_last_pushed_lsn = item_lsn;
537			break;
538
539		case XFS_ITEM_FLUSHING:
540			/*
541			 * The item or its backing buffer is already being
542			 * flushed.  The typical reason for that is that an
543			 * inode buffer is locked because we already pushed the
544			 * updates to it as part of inode clustering.
545			 *
546			 * We do not want to stop flushing just because lots
547			 * of items are already being flushed, but we need to
548			 * re-try the flushing relatively soon if most of the
549			 * AIL is being flushed.
550			 */
551			XFS_STATS_INC(mp, xs_push_ail_flushing);
552			trace_xfs_ail_flushing(ailp, type, flags, item_lsn);
553
554			flushing++;
555			ailp->ail_last_pushed_lsn = item_lsn;
556			break;
557
558		case XFS_ITEM_PINNED:
559			XFS_STATS_INC(mp, xs_push_ail_pinned);
560			trace_xfs_ail_pinned(ailp, type, flags, item_lsn);
561
562			stuck++;
563			ailp->ail_log_flush++;
564			break;
565		case XFS_ITEM_LOCKED:
566			XFS_STATS_INC(mp, xs_push_ail_locked);
567			trace_xfs_ail_locked(ailp, type, flags, item_lsn);
568
569			stuck++;
570			break;
571		default:
572			ASSERT(0);
573			break;
574		}
575
576		count++;
577
578		/*
579		 * Are there too many items we can't do anything with?
580		 *
581		 * If we are skipping too many items because we can't flush
582		 * them or they are already being flushed, we back off and
583		 * given them time to complete whatever operation is being
584		 * done. i.e. remove pressure from the AIL while we can't make
585		 * progress so traversals don't slow down further inserts and
586		 * removals to/from the AIL.
587		 *
588		 * The value of 100 is an arbitrary magic number based on
589		 * observation.
590		 */
591		if (stuck > 100)
592			break;
593
594next_item:
595		lip = xfs_trans_ail_cursor_next(ailp, &cur);
596		if (lip == NULL)
597			break;
598		if (lip->li_lsn != lsn && count > 1000)
599			break;
600		lsn = lip->li_lsn;
601	}
602
603out_done_cursor:
604	xfs_trans_ail_cursor_done(&cur);
605out_done:
606	spin_unlock(&ailp->ail_lock);
607
608	if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list))
609		ailp->ail_log_flush++;
610
611	if (!count || XFS_LSN_CMP(lsn, ailp->ail_target) >= 0) {
612		/*
613		 * We reached the target or the AIL is empty, so wait a bit
614		 * longer for I/O to complete and remove pushed items from the
615		 * AIL before we start the next scan from the start of the AIL.
616		 */
617		tout = 50;
618		ailp->ail_last_pushed_lsn = 0;
619	} else if (((stuck + flushing) * 100) / count > 90) {
620		/*
621		 * Either there is a lot of contention on the AIL or we are
622		 * stuck due to operations in progress. "Stuck" in this case
623		 * is defined as >90% of the items we tried to push were stuck.
624		 *
625		 * Backoff a bit more to allow some I/O to complete before
626		 * restarting from the start of the AIL. This prevents us from
627		 * spinning on the same items, and if they are pinned will all
628		 * the restart to issue a log force to unpin the stuck items.
629		 */
630		tout = 20;
631		ailp->ail_last_pushed_lsn = 0;
632	} else {
633		/*
634		 * Assume we have more work to do in a short while.
635		 */
636		tout = 0;
637	}
638
639	return tout;
640}
641
642static int
643xfsaild(
644	void		*data)
645{
646	struct xfs_ail	*ailp = data;
647	long		tout = 0;	/* milliseconds */
648	unsigned int	noreclaim_flag;
649
650	noreclaim_flag = memalloc_noreclaim_save();
651	set_freezable();
652
653	while (1) {
654		/*
655		 * Long waits of 50ms or more occur when we've run out of items
656		 * to push, so we only want uninterruptible state if we're
657		 * actually blocked on something.
658		 */
659		if (tout && tout <= 20)
660			set_current_state(TASK_KILLABLE|TASK_FREEZABLE);
661		else
662			set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
663
664		/*
665		 * Check kthread_should_stop() after we set the task state to
666		 * guarantee that we either see the stop bit and exit or the
667		 * task state is reset to runnable such that it's not scheduled
668		 * out indefinitely and detects the stop bit at next iteration.
669		 * A memory barrier is included in above task state set to
670		 * serialize again kthread_stop().
671		 */
672		if (kthread_should_stop()) {
673			__set_current_state(TASK_RUNNING);
674
675			/*
676			 * The caller forces out the AIL before stopping the
677			 * thread in the common case, which means the delwri
678			 * queue is drained. In the shutdown case, the queue may
679			 * still hold relogged buffers that haven't been
680			 * submitted because they were pinned since added to the
681			 * queue.
682			 *
683			 * Log I/O error processing stales the underlying buffer
684			 * and clears the delwri state, expecting the buf to be
685			 * removed on the next submission attempt. That won't
686			 * happen if we're shutting down, so this is the last
687			 * opportunity to release such buffers from the queue.
688			 */
689			ASSERT(list_empty(&ailp->ail_buf_list) ||
690			       xlog_is_shutdown(ailp->ail_log));
691			xfs_buf_delwri_cancel(&ailp->ail_buf_list);
692			break;
693		}
694
695		/* Idle if the AIL is empty. */
696		spin_lock(&ailp->ail_lock);
697		if (!xfs_ail_min(ailp) && list_empty(&ailp->ail_buf_list)) {
698			spin_unlock(&ailp->ail_lock);
699			schedule();
700			tout = 0;
701			continue;
702		}
703		spin_unlock(&ailp->ail_lock);
704
705		if (tout)
706			schedule_timeout(msecs_to_jiffies(tout));
707
708		__set_current_state(TASK_RUNNING);
709
710		try_to_freeze();
711
712		tout = xfsaild_push(ailp);
713	}
714
715	memalloc_noreclaim_restore(noreclaim_flag);
716	return 0;
717}
718
719/*
720 * Push out all items in the AIL immediately and wait until the AIL is empty.
721 */
722void
723xfs_ail_push_all_sync(
724	struct xfs_ail  *ailp)
725{
726	DEFINE_WAIT(wait);
727
728	spin_lock(&ailp->ail_lock);
729	while (xfs_ail_max(ailp) != NULL) {
730		prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE);
731		wake_up_process(ailp->ail_task);
732		spin_unlock(&ailp->ail_lock);
733		schedule();
734		spin_lock(&ailp->ail_lock);
735	}
736	spin_unlock(&ailp->ail_lock);
737
738	finish_wait(&ailp->ail_empty, &wait);
739}
740
741void
742__xfs_ail_assign_tail_lsn(
743	struct xfs_ail		*ailp)
744{
745	struct xlog		*log = ailp->ail_log;
746	xfs_lsn_t		tail_lsn;
747
748	assert_spin_locked(&ailp->ail_lock);
749
750	if (xlog_is_shutdown(log))
751		return;
752
753	tail_lsn = __xfs_ail_min_lsn(ailp);
754	if (!tail_lsn)
755		tail_lsn = ailp->ail_head_lsn;
756
757	WRITE_ONCE(log->l_tail_space,
758			xlog_lsn_sub(log, ailp->ail_head_lsn, tail_lsn));
759	trace_xfs_log_assign_tail_lsn(log, tail_lsn);
760	atomic64_set(&log->l_tail_lsn, tail_lsn);
761}
762
763/*
764 * Callers should pass the original tail lsn so that we can detect if the tail
765 * has moved as a result of the operation that was performed. If the caller
766 * needs to force a tail space update, it should pass NULLCOMMITLSN to bypass
767 * the "did the tail LSN change?" checks. If the caller wants to avoid a tail
768 * update (e.g. it knows the tail did not change) it should pass an @old_lsn of
769 * 0.
770 */
771void
772xfs_ail_update_finish(
773	struct xfs_ail		*ailp,
774	xfs_lsn_t		old_lsn) __releases(ailp->ail_lock)
775{
776	struct xlog		*log = ailp->ail_log;
777
778	/* If the tail lsn hasn't changed, don't do updates or wakeups. */
779	if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) {
780		spin_unlock(&ailp->ail_lock);
781		return;
782	}
783
784	__xfs_ail_assign_tail_lsn(ailp);
785	if (list_empty(&ailp->ail_head))
786		wake_up_all(&ailp->ail_empty);
787	spin_unlock(&ailp->ail_lock);
788	xfs_log_space_wake(log->l_mp);
789}
790
791/*
792 * xfs_trans_ail_update_bulk - bulk AIL insertion operation.
793 *
794 * @xfs_trans_ail_update_bulk takes an array of log items that all need to be
795 * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
796 * be added. Otherwise, it will be repositioned by removing it and re-adding
797 * it to the AIL.
798 *
799 * If we move the first item in the AIL, update the log tail to match the new
800 * minimum LSN in the AIL.
801 *
802 * This function should be called with the AIL lock held.
803 *
804 * To optimise the insert operation, we add all items to a temporary list, then
805 * splice this list into the correct position in the AIL.
806 *
807 * Items that are already in the AIL are first deleted from their current
808 * location before being added to the temporary list.
809 *
810 * This avoids needing to do an insert operation on every item.
811 *
812 * The AIL lock is dropped by xfs_ail_update_finish() before returning to
813 * the caller.
814 */
815void
816xfs_trans_ail_update_bulk(
817	struct xfs_ail		*ailp,
818	struct xfs_ail_cursor	*cur,
819	struct xfs_log_item	**log_items,
820	int			nr_items,
821	xfs_lsn_t		lsn) __releases(ailp->ail_lock)
822{
823	struct xfs_log_item	*mlip;
824	xfs_lsn_t		tail_lsn = 0;
825	int			i;
826	LIST_HEAD(tmp);
827
828	ASSERT(nr_items > 0);		/* Not required, but true. */
829	mlip = xfs_ail_min(ailp);
830
831	for (i = 0; i < nr_items; i++) {
832		struct xfs_log_item *lip = log_items[i];
833		if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
834			/* check if we really need to move the item */
835			if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
836				continue;
837
838			trace_xfs_ail_move(lip, lip->li_lsn, lsn);
839			if (mlip == lip && !tail_lsn)
840				tail_lsn = lip->li_lsn;
841
842			xfs_ail_delete(ailp, lip);
843		} else {
844			trace_xfs_ail_insert(lip, 0, lsn);
845		}
846		lip->li_lsn = lsn;
847		list_add_tail(&lip->li_ail, &tmp);
848	}
849
850	if (!list_empty(&tmp))
851		xfs_ail_splice(ailp, cur, &tmp, lsn);
852
853	/*
854	 * If this is the first insert, wake up the push daemon so it can
855	 * actively scan for items to push. We also need to do a log tail
856	 * LSN update to ensure that it is correctly tracked by the log, so
857	 * set the tail_lsn to NULLCOMMITLSN so that xfs_ail_update_finish()
858	 * will see that the tail lsn has changed and will update the tail
859	 * appropriately.
860	 */
861	if (!mlip) {
862		wake_up_process(ailp->ail_task);
863		tail_lsn = NULLCOMMITLSN;
864	}
865
866	xfs_ail_update_finish(ailp, tail_lsn);
867}
868
869/* Insert a log item into the AIL. */
870void
871xfs_trans_ail_insert(
872	struct xfs_ail		*ailp,
873	struct xfs_log_item	*lip,
874	xfs_lsn_t		lsn)
875{
876	spin_lock(&ailp->ail_lock);
877	xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn);
878}
879
880/*
881 * Delete one log item from the AIL.
882 *
883 * If this item was at the tail of the AIL, return the LSN of the log item so
884 * that we can use it to check if the LSN of the tail of the log has moved
885 * when finishing up the AIL delete process in xfs_ail_update_finish().
886 */
887xfs_lsn_t
888xfs_ail_delete_one(
889	struct xfs_ail		*ailp,
890	struct xfs_log_item	*lip)
891{
892	struct xfs_log_item	*mlip = xfs_ail_min(ailp);
893	xfs_lsn_t		lsn = lip->li_lsn;
894
895	trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
896	xfs_ail_delete(ailp, lip);
897	clear_bit(XFS_LI_IN_AIL, &lip->li_flags);
898	lip->li_lsn = 0;
899
900	if (mlip == lip)
901		return lsn;
902	return 0;
903}
904
905void
906xfs_trans_ail_delete(
907	struct xfs_log_item	*lip,
908	int			shutdown_type)
909{
910	struct xfs_ail		*ailp = lip->li_ailp;
911	struct xlog		*log = ailp->ail_log;
912	xfs_lsn_t		tail_lsn;
913
914	spin_lock(&ailp->ail_lock);
915	if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) {
916		spin_unlock(&ailp->ail_lock);
917		if (shutdown_type && !xlog_is_shutdown(log)) {
918			xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE,
919	"%s: attempting to delete a log item that is not in the AIL",
920					__func__);
921			xlog_force_shutdown(log, shutdown_type);
922		}
923		return;
924	}
925
926	clear_bit(XFS_LI_FAILED, &lip->li_flags);
927	tail_lsn = xfs_ail_delete_one(ailp, lip);
928	xfs_ail_update_finish(ailp, tail_lsn);	/* drops the AIL lock */
929}
930
931int
932xfs_trans_ail_init(
933	xfs_mount_t	*mp)
934{
935	struct xfs_ail	*ailp;
936
937	ailp = kzalloc(sizeof(struct xfs_ail),
938			GFP_KERNEL | __GFP_RETRY_MAYFAIL);
939	if (!ailp)
940		return -ENOMEM;
941
942	ailp->ail_log = mp->m_log;
943	INIT_LIST_HEAD(&ailp->ail_head);
944	INIT_LIST_HEAD(&ailp->ail_cursors);
945	spin_lock_init(&ailp->ail_lock);
946	INIT_LIST_HEAD(&ailp->ail_buf_list);
947	init_waitqueue_head(&ailp->ail_empty);
948
949	ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
950				mp->m_super->s_id);
951	if (IS_ERR(ailp->ail_task))
952		goto out_free_ailp;
953
954	mp->m_ail = ailp;
955	return 0;
956
957out_free_ailp:
958	kfree(ailp);
959	return -ENOMEM;
960}
961
962void
963xfs_trans_ail_destroy(
964	xfs_mount_t	*mp)
965{
966	struct xfs_ail	*ailp = mp->m_ail;
967
968	kthread_stop(ailp->ail_task);
969	kfree(ailp);
970}