lops.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) Sistina Software, Inc.  1997-2003 All rights reserved.
   4 * Copyright (C) 2004-2006 Red Hat, Inc.  All rights reserved.
   5 */
   6
   7#include <linux/sched.h>
   8#include <linux/slab.h>
   9#include <linux/spinlock.h>
  10#include <linux/completion.h>
  11#include <linux/buffer_head.h>
  12#include <linux/mempool.h>
  13#include <linux/gfs2_ondisk.h>
  14#include <linux/bio.h>
  15#include <linux/fs.h>
  16#include <linux/list_sort.h>
  17#include <linux/blkdev.h>
  18
  19#include "bmap.h"
  20#include "dir.h"
  21#include "gfs2.h"
  22#include "incore.h"
  23#include "inode.h"
  24#include "glock.h"
  25#include "glops.h"
  26#include "log.h"
  27#include "lops.h"
  28#include "meta_io.h"
  29#include "recovery.h"
  30#include "rgrp.h"
  31#include "trans.h"
  32#include "util.h"
  33#include "trace_gfs2.h"
  34
  35/**
  36 * gfs2_pin - Pin a buffer in memory
  37 * @sdp: The superblock
  38 * @bh: The buffer to be pinned
  39 *
  40 * The log lock must be held when calling this function
  41 */
  42void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh)
  43{
  44	struct gfs2_bufdata *bd;
  45
  46	BUG_ON(!current->journal_info);
  47
  48	clear_buffer_dirty(bh);
  49	if (test_set_buffer_pinned(bh))
  50		gfs2_assert_withdraw(sdp, 0);
  51	if (!buffer_uptodate(bh))
  52		gfs2_io_error_bh(sdp, bh);
  53	bd = bh->b_private;
  54	/* If this buffer is in the AIL and it has already been written
  55	 * to in-place disk block, remove it from the AIL.
  56	 */
  57	spin_lock(&sdp->sd_ail_lock);
  58	if (bd->bd_tr)
  59		list_move(&bd->bd_ail_st_list, &bd->bd_tr->tr_ail2_list);
  60	spin_unlock(&sdp->sd_ail_lock);
  61	get_bh(bh);
  62	atomic_inc(&sdp->sd_log_pinned);
  63	trace_gfs2_pin(bd, 1);
  64}
  65
  66static bool buffer_is_rgrp(const struct gfs2_bufdata *bd)
  67{
  68	return bd->bd_gl->gl_name.ln_type == LM_TYPE_RGRP;
  69}
  70
  71static void maybe_release_space(struct gfs2_bufdata *bd)
  72{
  73	struct gfs2_glock *gl = bd->bd_gl;
  74	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
  75	struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl);
  76	unsigned int index = bd->bd_bh->b_blocknr - gl->gl_name.ln_number;
  77	struct gfs2_bitmap *bi = rgd->rd_bits + index;
  78
  79	rgrp_lock_local(rgd);
  80	if (bi->bi_clone == NULL)
  81		goto out;
  82	if (sdp->sd_args.ar_discard)
  83		gfs2_rgrp_send_discards(sdp, rgd->rd_data0, bd->bd_bh, bi, 1, NULL);
  84	memcpy(bi->bi_clone + bi->bi_offset,
  85	       bd->bd_bh->b_data + bi->bi_offset, bi->bi_bytes);
  86	clear_bit(GBF_FULL, &bi->bi_flags);
  87	rgd->rd_free_clone = rgd->rd_free;
  88	BUG_ON(rgd->rd_free_clone < rgd->rd_reserved);
  89	rgd->rd_extfail_pt = rgd->rd_free;
  90
  91out:
  92	rgrp_unlock_local(rgd);
  93}
  94
  95/**
  96 * gfs2_unpin - Unpin a buffer
  97 * @sdp: the filesystem the buffer belongs to
  98 * @bh: The buffer to unpin
  99 * @tr: The system transaction being flushed
 100 */
 101
 102static void gfs2_unpin(struct gfs2_sbd *sdp, struct buffer_head *bh,
 103		       struct gfs2_trans *tr)
 104{
 105	struct gfs2_bufdata *bd = bh->b_private;
 106
 107	BUG_ON(!buffer_uptodate(bh));
 108	BUG_ON(!buffer_pinned(bh));
 109
 110	lock_buffer(bh);
 111	mark_buffer_dirty(bh);
 112	clear_buffer_pinned(bh);
 113
 114	if (buffer_is_rgrp(bd))
 115		maybe_release_space(bd);
 116
 117	spin_lock(&sdp->sd_ail_lock);
 118	if (bd->bd_tr) {
 119		list_del(&bd->bd_ail_st_list);
 120		brelse(bh);
 121	} else {
 122		struct gfs2_glock *gl = bd->bd_gl;
 123		list_add(&bd->bd_ail_gl_list, &gl->gl_ail_list);
 124		atomic_inc(&gl->gl_ail_count);
 125	}
 126	bd->bd_tr = tr;
 127	list_add(&bd->bd_ail_st_list, &tr->tr_ail1_list);
 128	spin_unlock(&sdp->sd_ail_lock);
 129
 130	clear_bit(GLF_LFLUSH, &bd->bd_gl->gl_flags);
 131	trace_gfs2_pin(bd, 0);
 132	unlock_buffer(bh);
 133	atomic_dec(&sdp->sd_log_pinned);
 134}
 135
 136void gfs2_log_incr_head(struct gfs2_sbd *sdp)
 137{
 138	BUG_ON((sdp->sd_log_flush_head == sdp->sd_log_tail) &&
 139	       (sdp->sd_log_flush_head != sdp->sd_log_head));
 140
 141	if (++sdp->sd_log_flush_head == sdp->sd_jdesc->jd_blocks)
 142		sdp->sd_log_flush_head = 0;
 143}
 144
 145u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lblock)
 146{
 147	struct gfs2_journal_extent *je;
 148
 149	list_for_each_entry(je, &jd->extent_list, list) {
 150		if (lblock >= je->lblock && lblock < je->lblock + je->blocks)
 151			return je->dblock + lblock - je->lblock;
 152	}
 153
 154	return -1;
 155}
 156
 157/**
 158 * gfs2_end_log_write_bh - end log write of pagecache data with buffers
 159 * @sdp: The superblock
 160 * @folio: The folio
 161 * @offset: The first byte within the folio that completed
 162 * @size: The number of bytes that completed
 163 * @error: The i/o status
 164 *
 165 * This finds the relevant buffers and unlocks them and sets the
 166 * error flag according to the status of the i/o request. This is
 167 * used when the log is writing data which has an in-place version
 168 * that is pinned in the pagecache.
 169 */
 170
 171static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct folio *folio,
 172		size_t offset, size_t size, blk_status_t error)
 173{
 174	struct buffer_head *bh, *next;
 175
 176	bh = folio_buffers(folio);
 177	while (bh_offset(bh) < offset)
 178		bh = bh->b_this_page;
 179	do {
 180		if (error)
 181			mark_buffer_write_io_error(bh);
 182		unlock_buffer(bh);
 183		next = bh->b_this_page;
 184		size -= bh->b_size;
 185		brelse(bh);
 186		bh = next;
 187	} while (bh && size);
 188}
 189
 190/**
 191 * gfs2_end_log_write - end of i/o to the log
 192 * @bio: The bio
 193 *
 194 * Each bio_vec contains either data from the pagecache or data
 195 * relating to the log itself. Here we iterate over the bio_vec
 196 * array, processing both kinds of data.
 197 *
 198 */
 199
 200static void gfs2_end_log_write(struct bio *bio)
 201{
 202	struct gfs2_sbd *sdp = bio->bi_private;
 203	struct bio_vec *bvec;
 204	struct bvec_iter_all iter_all;
 205
 206	if (bio->bi_status) {
 207		int err = blk_status_to_errno(bio->bi_status);
 208
 209		if (!cmpxchg(&sdp->sd_log_error, 0, err))
 210			fs_err(sdp, "Error %d writing to journal, jid=%u\n",
 211			       err, sdp->sd_jdesc->jd_jid);
 212		gfs2_withdraw(sdp);
 213	}
 214
 215	bio_for_each_segment_all(bvec, bio, iter_all) {
 216		struct page *page = bvec->bv_page;
 217		struct folio *folio = page_folio(page);
 218
 219		if (folio && folio_buffers(folio))
 220			gfs2_end_log_write_bh(sdp, folio, bvec->bv_offset,
 221					bvec->bv_len, bio->bi_status);
 222		else
 223			mempool_free(page, gfs2_page_pool);
 224	}
 225
 226	bio_put(bio);
 227	if (atomic_dec_and_test(&sdp->sd_log_in_flight))
 228		wake_up(&sdp->sd_log_flush_wait);
 229}
 230
 231/**
 232 * gfs2_log_submit_write - Submit a pending log write bio
 233 * @biop: Address of the bio pointer
 234 *
 235 * Submit any pending part-built or full bio to the block device. If
 236 * there is no pending bio, then this is a no-op.
 237 */
 238
 239void gfs2_log_submit_write(struct bio **biop)
 240{
 241	struct bio *bio = *biop;
 242	if (bio) {
 243		struct gfs2_sbd *sdp = bio->bi_private;
 244		atomic_inc(&sdp->sd_log_in_flight);
 245		submit_bio(bio);
 246		*biop = NULL;
 247	}
 248}
 249
 250/**
 251 * gfs2_log_alloc_bio - Allocate a bio
 252 * @sdp: The super block
 253 * @blkno: The device block number we want to write to
 254 * @end_io: The bi_end_io callback
 255 * @opf: REQ_OP | op_flags
 256 *
 257 * Allocate a new bio, initialize it with the given parameters and return it.
 258 *
 259 * Returns: The newly allocated bio
 260 */
 261
 262static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno,
 263				      bio_end_io_t *end_io, blk_opf_t opf)
 264{
 265	struct super_block *sb = sdp->sd_vfs;
 266	struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, opf, GFP_NOIO);
 267
 268	bio->bi_iter.bi_sector = blkno << sdp->sd_fsb2bb_shift;
 269	bio->bi_end_io = end_io;
 270	bio->bi_private = sdp;
 271
 272	return bio;
 273}
 274
 275/**
 276 * gfs2_log_get_bio - Get cached log bio, or allocate a new one
 277 * @sdp: The super block
 278 * @blkno: The device block number we want to write to
 279 * @biop: The bio to get or allocate
 280 * @op: REQ_OP
 281 * @end_io: The bi_end_io callback
 282 * @flush: Always flush the current bio and allocate a new one?
 283 *
 284 * If there is a cached bio, then if the next block number is sequential
 285 * with the previous one, return it, otherwise flush the bio to the
 286 * device. If there is no cached bio, or we just flushed it, then
 287 * allocate a new one.
 288 *
 289 * Returns: The bio to use for log writes
 290 */
 291
 292static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
 293				    struct bio **biop, enum req_op op,
 294				    bio_end_io_t *end_io, bool flush)
 295{
 296	struct bio *bio = *biop;
 297
 298	if (bio) {
 299		u64 nblk;
 300
 301		nblk = bio_end_sector(bio);
 302		nblk >>= sdp->sd_fsb2bb_shift;
 303		if (blkno == nblk && !flush)
 304			return bio;
 305		gfs2_log_submit_write(biop);
 306	}
 307
 308	*biop = gfs2_log_alloc_bio(sdp, blkno, end_io, op);
 309	return *biop;
 310}
 311
 312/**
 313 * gfs2_log_write - write to log
 314 * @sdp: the filesystem
 315 * @jd: The journal descriptor
 316 * @page: the page to write
 317 * @size: the size of the data to write
 318 * @offset: the offset within the page 
 319 * @blkno: block number of the log entry
 320 * @opf: REQ_OP | op_flags
 321 *
 322 * Try and add the page segment to the current bio. If that fails,
 323 * submit the current bio to the device and create a new one, and
 324 * then add the page segment to that.
 325 */
 326
 327void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 328		    struct page *page, unsigned size, unsigned offset,
 329		    u64 blkno, blk_opf_t opf)
 330{
 331	struct bio *bio;
 332	int ret;
 333
 334	bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, opf,
 335			       gfs2_end_log_write, false);
 336	ret = bio_add_page(bio, page, size, offset);
 337	if (ret == 0) {
 338		bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio,
 339				       opf, gfs2_end_log_write, true);
 340		ret = bio_add_page(bio, page, size, offset);
 341		WARN_ON(ret == 0);
 342	}
 343}
 344
 345/**
 346 * gfs2_log_write_bh - write a buffer's content to the log
 347 * @sdp: The super block
 348 * @bh: The buffer pointing to the in-place location
 349 * 
 350 * This writes the content of the buffer to the next available location
 351 * in the log. The buffer will be unlocked once the i/o to the log has
 352 * completed.
 353 */
 354
 355static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh)
 356{
 357	u64 dblock;
 358
 359	dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head);
 360	gfs2_log_incr_head(sdp);
 361	gfs2_log_write(sdp, sdp->sd_jdesc, folio_page(bh->b_folio, 0),
 362			bh->b_size, bh_offset(bh), dblock, REQ_OP_WRITE);
 363}
 364
 365/**
 366 * gfs2_log_write_page - write one block stored in a page, into the log
 367 * @sdp: The superblock
 368 * @page: The struct page
 369 *
 370 * This writes the first block-sized part of the page into the log. Note
 371 * that the page must have been allocated from the gfs2_page_pool mempool
 372 * and that after this has been called, ownership has been transferred and
 373 * the page may be freed at any time.
 374 */
 375
 376static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)
 377{
 378	struct super_block *sb = sdp->sd_vfs;
 379	u64 dblock;
 380
 381	dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head);
 382	gfs2_log_incr_head(sdp);
 383	gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock,
 384		       REQ_OP_WRITE);
 385}
 386
 387/**
 388 * gfs2_end_log_read - end I/O callback for reads from the log
 389 * @bio: The bio
 390 *
 391 * Simply unlock the pages in the bio. The main thread will wait on them and
 392 * process them in order as necessary.
 393 */
 394static void gfs2_end_log_read(struct bio *bio)
 395{
 396	int error = blk_status_to_errno(bio->bi_status);
 397	struct folio_iter fi;
 398
 399	bio_for_each_folio_all(fi, bio) {
 400		/* We're abusing wb_err to get the error to gfs2_find_jhead */
 401		filemap_set_wb_err(fi.folio->mapping, error);
 402		folio_end_read(fi.folio, !error);
 403	}
 404
 405	bio_put(bio);
 406}
 407
 408/**
 409 * gfs2_jhead_folio_search - Look for the journal head in a given page.
 410 * @jd: The journal descriptor
 411 * @head: The journal head to start from
 412 * @folio: The folio to look in
 413 *
 414 * Returns: 1 if found, 0 otherwise.
 415 */
 416static bool gfs2_jhead_folio_search(struct gfs2_jdesc *jd,
 417				    struct gfs2_log_header_host *head,
 418				    struct folio *folio)
 419{
 420	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 421	struct gfs2_log_header_host lh;
 422	void *kaddr;
 423	unsigned int offset;
 424	bool ret = false;
 425
 426	VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
 427	kaddr = kmap_local_folio(folio, 0);
 428	for (offset = 0; offset < PAGE_SIZE; offset += sdp->sd_sb.sb_bsize) {
 429		if (!__get_log_header(sdp, kaddr + offset, 0, &lh)) {
 430			if (lh.lh_sequence >= head->lh_sequence)
 431				*head = lh;
 432			else {
 433				ret = true;
 434				break;
 435			}
 436		}
 437	}
 438	kunmap_local(kaddr);
 439	return ret;
 440}
 441
 442/**
 443 * gfs2_jhead_process_page - Search/cleanup a page
 444 * @jd: The journal descriptor
 445 * @index: Index of the page to look into
 446 * @head: The journal head to start from
 447 * @done: If set, perform only cleanup, else search and set if found.
 448 *
 449 * Find the folio with 'index' in the journal's mapping. Search the folio for
 450 * the journal head if requested (cleanup == false). Release refs on the
 451 * folio so the page cache can reclaim it. We grabbed a
 452 * reference on this folio twice, first when we did a filemap_grab_folio()
 453 * to obtain the folio to add it to the bio and second when we do a
 454 * filemap_get_folio() here to get the folio to wait on while I/O on it is being
 455 * completed.
 456 * This function is also used to free up a folio we might've grabbed but not
 457 * used. Maybe we added it to a bio, but not submitted it for I/O. Or we
 458 * submitted the I/O, but we already found the jhead so we only need to drop
 459 * our references to the folio.
 460 */
 461
 462static void gfs2_jhead_process_page(struct gfs2_jdesc *jd, unsigned long index,
 463				    struct gfs2_log_header_host *head,
 464				    bool *done)
 465{
 466	struct folio *folio;
 467
 468	folio = filemap_get_folio(jd->jd_inode->i_mapping, index);
 469
 470	folio_wait_locked(folio);
 471	if (!folio_test_uptodate(folio))
 472		*done = true;
 473
 474	if (!*done)
 475		*done = gfs2_jhead_folio_search(jd, head, folio);
 476
 477	/* filemap_get_folio() and the earlier filemap_grab_folio() */
 478	folio_put_refs(folio, 2);
 479}
 480
 481static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs,
 482				  blk_opf_t opf)
 483{
 484	struct bio *new;
 485
 486	new = bio_alloc(prev->bi_bdev, nr_iovecs, opf, GFP_NOIO);
 487	bio_clone_blkg_association(new, prev);
 488	new->bi_iter.bi_sector = bio_end_sector(prev);
 489	bio_chain(new, prev);
 490	submit_bio(prev);
 491	return new;
 492}
 493
 494/**
 495 * gfs2_find_jhead - find the head of a log
 496 * @jd: The journal descriptor
 497 * @head: The log descriptor for the head of the log is returned here
 498 *
 499 * Do a search of a journal by reading it in large chunks using bios and find
 500 * the valid log entry with the highest sequence number.  (i.e. the log head)
 501 *
 502 * Returns: 0 on success, errno otherwise
 503 */
 504int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head)
 505{
 506	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 507	struct address_space *mapping = jd->jd_inode->i_mapping;
 508	unsigned int block = 0, blocks_submitted = 0, blocks_read = 0;
 509	unsigned int bsize = sdp->sd_sb.sb_bsize, off;
 510	unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
 511	unsigned int shift = PAGE_SHIFT - bsize_shift;
 512	unsigned int max_blocks = 2 * 1024 * 1024 >> bsize_shift;
 513	struct gfs2_journal_extent *je;
 514	int ret = 0;
 515	struct bio *bio = NULL;
 516	struct folio *folio = NULL;
 517	bool done = false;
 518	errseq_t since;
 519
 520	memset(head, 0, sizeof(*head));
 521	if (list_empty(&jd->extent_list))
 522		gfs2_map_journal_extents(sdp, jd);
 523
 524	since = filemap_sample_wb_err(mapping);
 525	list_for_each_entry(je, &jd->extent_list, list) {
 526		u64 dblock = je->dblock;
 527
 528		for (; block < je->lblock + je->blocks; block++, dblock++) {
 529			if (!folio) {
 530				folio = filemap_grab_folio(mapping,
 531						block >> shift);
 532				if (IS_ERR(folio)) {
 533					ret = PTR_ERR(folio);
 534					done = true;
 535					goto out;
 536				}
 537				off = 0;
 538			}
 539
 540			if (bio && (off || block < blocks_submitted + max_blocks)) {
 541				sector_t sector = dblock << sdp->sd_fsb2bb_shift;
 542
 543				if (bio_end_sector(bio) == sector) {
 544					if (bio_add_folio(bio, folio, bsize, off))
 545						goto block_added;
 546				}
 547				if (off) {
 548					unsigned int blocks =
 549						(PAGE_SIZE - off) >> bsize_shift;
 550
 551					bio = gfs2_chain_bio(bio, blocks,
 552							     REQ_OP_READ);
 553					goto add_block_to_new_bio;
 554				}
 555			}
 556
 557			if (bio) {
 558				blocks_submitted = block;
 559				submit_bio(bio);
 560			}
 561
 562			bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read,
 563						 REQ_OP_READ);
 564add_block_to_new_bio:
 565			bio_add_folio_nofail(bio, folio, bsize, off);
 566block_added:
 567			off += bsize;
 568			if (off == folio_size(folio))
 569				folio = NULL;
 570			if (blocks_submitted <= blocks_read + max_blocks) {
 571				/* Keep at least one bio in flight */
 572				continue;
 573			}
 574
 575			gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done);
 576			blocks_read += PAGE_SIZE >> bsize_shift;
 577			if (done)
 578				goto out;  /* found */
 579		}
 580	}
 581
 582out:
 583	if (bio)
 584		submit_bio(bio);
 585	while (blocks_read < block) {
 586		gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done);
 587		blocks_read += PAGE_SIZE >> bsize_shift;
 588	}
 589
 590	if (!ret)
 591		ret = filemap_check_wb_err(mapping, since);
 592
 593	truncate_inode_pages(mapping, 0);
 594
 595	return ret;
 596}
 597
 598static struct page *gfs2_get_log_desc(struct gfs2_sbd *sdp, u32 ld_type,
 599				      u32 ld_length, u32 ld_data1)
 600{
 601	struct page *page = mempool_alloc(gfs2_page_pool, GFP_NOIO);
 602	struct gfs2_log_descriptor *ld = page_address(page);
 603	clear_page(ld);
 604	ld->ld_header.mh_magic = cpu_to_be32(GFS2_MAGIC);
 605	ld->ld_header.mh_type = cpu_to_be32(GFS2_METATYPE_LD);
 606	ld->ld_header.mh_format = cpu_to_be32(GFS2_FORMAT_LD);
 607	ld->ld_type = cpu_to_be32(ld_type);
 608	ld->ld_length = cpu_to_be32(ld_length);
 609	ld->ld_data1 = cpu_to_be32(ld_data1);
 610	ld->ld_data2 = 0;
 611	return page;
 612}
 613
 614static void gfs2_check_magic(struct buffer_head *bh)
 615{
 616	__be32 *ptr;
 617
 618	clear_buffer_escaped(bh);
 619	ptr = kmap_local_folio(bh->b_folio, bh_offset(bh));
 620	if (*ptr == cpu_to_be32(GFS2_MAGIC))
 621		set_buffer_escaped(bh);
 622	kunmap_local(ptr);
 623}
 624
 625static int blocknr_cmp(void *priv, const struct list_head *a,
 626		       const struct list_head *b)
 627{
 628	struct gfs2_bufdata *bda, *bdb;
 629
 630	bda = list_entry(a, struct gfs2_bufdata, bd_list);
 631	bdb = list_entry(b, struct gfs2_bufdata, bd_list);
 632
 633	if (bda->bd_bh->b_blocknr < bdb->bd_bh->b_blocknr)
 634		return -1;
 635	if (bda->bd_bh->b_blocknr > bdb->bd_bh->b_blocknr)
 636		return 1;
 637	return 0;
 638}
 639
 640static void gfs2_before_commit(struct gfs2_sbd *sdp, unsigned int limit,
 641				unsigned int total, struct list_head *blist,
 642				bool is_databuf)
 643{
 644	struct gfs2_log_descriptor *ld;
 645	struct gfs2_bufdata *bd1 = NULL, *bd2;
 646	struct page *page;
 647	unsigned int num;
 648	unsigned n;
 649	__be64 *ptr;
 650
 651	gfs2_log_lock(sdp);
 652	list_sort(NULL, blist, blocknr_cmp);
 653	bd1 = bd2 = list_prepare_entry(bd1, blist, bd_list);
 654	while(total) {
 655		num = total;
 656		if (total > limit)
 657			num = limit;
 658		gfs2_log_unlock(sdp);
 659		page = gfs2_get_log_desc(sdp,
 660					 is_databuf ? GFS2_LOG_DESC_JDATA :
 661					 GFS2_LOG_DESC_METADATA, num + 1, num);
 662		ld = page_address(page);
 663		gfs2_log_lock(sdp);
 664		ptr = (__be64 *)(ld + 1);
 665
 666		n = 0;
 667		list_for_each_entry_continue(bd1, blist, bd_list) {
 668			*ptr++ = cpu_to_be64(bd1->bd_bh->b_blocknr);
 669			if (is_databuf) {
 670				gfs2_check_magic(bd1->bd_bh);
 671				*ptr++ = cpu_to_be64(buffer_escaped(bd1->bd_bh) ? 1 : 0);
 672			}
 673			if (++n >= num)
 674				break;
 675		}
 676
 677		gfs2_log_unlock(sdp);
 678		gfs2_log_write_page(sdp, page);
 679		gfs2_log_lock(sdp);
 680
 681		n = 0;
 682		list_for_each_entry_continue(bd2, blist, bd_list) {
 683			get_bh(bd2->bd_bh);
 684			gfs2_log_unlock(sdp);
 685			lock_buffer(bd2->bd_bh);
 686
 687			if (buffer_escaped(bd2->bd_bh)) {
 688				void *p;
 689
 690				page = mempool_alloc(gfs2_page_pool, GFP_NOIO);
 691				p = page_address(page);
 692				memcpy_from_page(p, page, bh_offset(bd2->bd_bh), bd2->bd_bh->b_size);
 693				*(__be32 *)p = 0;
 694				clear_buffer_escaped(bd2->bd_bh);
 695				unlock_buffer(bd2->bd_bh);
 696				brelse(bd2->bd_bh);
 697				gfs2_log_write_page(sdp, page);
 698			} else {
 699				gfs2_log_write_bh(sdp, bd2->bd_bh);
 700			}
 701			gfs2_log_lock(sdp);
 702			if (++n >= num)
 703				break;
 704		}
 705
 706		BUG_ON(total < num);
 707		total -= num;
 708	}
 709	gfs2_log_unlock(sdp);
 710}
 711
 712static void buf_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 713{
 714	unsigned int limit = buf_limit(sdp); /* 503 for 4k blocks */
 715	unsigned int nbuf;
 716	if (tr == NULL)
 717		return;
 718	nbuf = tr->tr_num_buf_new - tr->tr_num_buf_rm;
 719	gfs2_before_commit(sdp, limit, nbuf, &tr->tr_buf, 0);
 720}
 721
 722static void buf_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 723{
 724	struct list_head *head;
 725	struct gfs2_bufdata *bd;
 726
 727	if (tr == NULL)
 728		return;
 729
 730	head = &tr->tr_buf;
 731	while (!list_empty(head)) {
 732		bd = list_first_entry(head, struct gfs2_bufdata, bd_list);
 733		list_del_init(&bd->bd_list);
 734		gfs2_unpin(sdp, bd->bd_bh, tr);
 735	}
 736}
 737
 738static void buf_lo_before_scan(struct gfs2_jdesc *jd,
 739			       struct gfs2_log_header_host *head, int pass)
 740{
 741	if (pass != 0)
 742		return;
 743
 744	jd->jd_found_blocks = 0;
 745	jd->jd_replayed_blocks = 0;
 746}
 747
 748#define obsolete_rgrp_replay \
 749"Replaying 0x%llx from jid=%d/0x%llx but we already have a bh!\n"
 750#define obsolete_rgrp_replay2 \
 751"busy:%d, pinned:%d rg_gen:0x%llx, j_gen:0x%llx\n"
 752
 753static void obsolete_rgrp(struct gfs2_jdesc *jd, struct buffer_head *bh_log,
 754			  u64 blkno)
 755{
 756	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 757	struct gfs2_rgrpd *rgd;
 758	struct gfs2_rgrp *jrgd = (struct gfs2_rgrp *)bh_log->b_data;
 759
 760	rgd = gfs2_blk2rgrpd(sdp, blkno, false);
 761	if (rgd && rgd->rd_addr == blkno &&
 762	    rgd->rd_bits && rgd->rd_bits->bi_bh) {
 763		fs_info(sdp, obsolete_rgrp_replay, (unsigned long long)blkno,
 764			jd->jd_jid, bh_log->b_blocknr);
 765		fs_info(sdp, obsolete_rgrp_replay2,
 766			buffer_busy(rgd->rd_bits->bi_bh) ? 1 : 0,
 767			buffer_pinned(rgd->rd_bits->bi_bh),
 768			rgd->rd_igeneration,
 769			be64_to_cpu(jrgd->rg_igeneration));
 770		gfs2_dump_glock(NULL, rgd->rd_gl, true);
 771	}
 772}
 773
 774static int buf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 775				struct gfs2_log_descriptor *ld, __be64 *ptr,
 776				int pass)
 777{
 778	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
 779	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 780	struct gfs2_glock *gl = ip->i_gl;
 781	unsigned int blks = be32_to_cpu(ld->ld_data1);
 782	struct buffer_head *bh_log, *bh_ip;
 783	u64 blkno;
 784	int error = 0;
 785
 786	if (pass != 1 || be32_to_cpu(ld->ld_type) != GFS2_LOG_DESC_METADATA)
 787		return 0;
 788
 789	gfs2_replay_incr_blk(jd, &start);
 790
 791	for (; blks; gfs2_replay_incr_blk(jd, &start), blks--) {
 792		blkno = be64_to_cpu(*ptr++);
 793
 794		jd->jd_found_blocks++;
 795
 796		if (gfs2_revoke_check(jd, blkno, start))
 797			continue;
 798
 799		error = gfs2_replay_read_block(jd, start, &bh_log);
 800		if (error)
 801			return error;
 802
 803		bh_ip = gfs2_meta_new(gl, blkno);
 804		memcpy(bh_ip->b_data, bh_log->b_data, bh_log->b_size);
 805
 806		if (gfs2_meta_check(sdp, bh_ip))
 807			error = -EIO;
 808		else {
 809			struct gfs2_meta_header *mh =
 810				(struct gfs2_meta_header *)bh_ip->b_data;
 811
 812			if (mh->mh_type == cpu_to_be32(GFS2_METATYPE_RG))
 813				obsolete_rgrp(jd, bh_log, blkno);
 814
 815			mark_buffer_dirty(bh_ip);
 816		}
 817		brelse(bh_log);
 818		brelse(bh_ip);
 819
 820		if (error)
 821			break;
 822
 823		jd->jd_replayed_blocks++;
 824	}
 825
 826	return error;
 827}
 828
 829static void buf_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass)
 830{
 831	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
 832	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 833
 834	if (error) {
 835		gfs2_inode_metasync(ip->i_gl);
 836		return;
 837	}
 838	if (pass != 1)
 839		return;
 840
 841	gfs2_inode_metasync(ip->i_gl);
 842
 843	fs_info(sdp, "jid=%u: Replayed %u of %u blocks\n",
 844	        jd->jd_jid, jd->jd_replayed_blocks, jd->jd_found_blocks);
 845}
 846
 847static void revoke_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 848{
 849	struct gfs2_meta_header *mh;
 850	unsigned int offset;
 851	struct list_head *head = &sdp->sd_log_revokes;
 852	struct gfs2_bufdata *bd;
 853	struct page *page;
 854	unsigned int length;
 855
 856	gfs2_flush_revokes(sdp);
 857	if (!sdp->sd_log_num_revoke)
 858		return;
 859
 860	length = gfs2_struct2blk(sdp, sdp->sd_log_num_revoke);
 861	page = gfs2_get_log_desc(sdp, GFS2_LOG_DESC_REVOKE, length, sdp->sd_log_num_revoke);
 862	offset = sizeof(struct gfs2_log_descriptor);
 863
 864	list_for_each_entry(bd, head, bd_list) {
 865		sdp->sd_log_num_revoke--;
 866
 867		if (offset + sizeof(u64) > sdp->sd_sb.sb_bsize) {
 868			gfs2_log_write_page(sdp, page);
 869			page = mempool_alloc(gfs2_page_pool, GFP_NOIO);
 870			mh = page_address(page);
 871			clear_page(mh);
 872			mh->mh_magic = cpu_to_be32(GFS2_MAGIC);
 873			mh->mh_type = cpu_to_be32(GFS2_METATYPE_LB);
 874			mh->mh_format = cpu_to_be32(GFS2_FORMAT_LB);
 875			offset = sizeof(struct gfs2_meta_header);
 876		}
 877
 878		*(__be64 *)(page_address(page) + offset) = cpu_to_be64(bd->bd_blkno);
 879		offset += sizeof(u64);
 880	}
 881	gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke);
 882
 883	gfs2_log_write_page(sdp, page);
 884}
 885
 886void gfs2_drain_revokes(struct gfs2_sbd *sdp)
 887{
 888	struct list_head *head = &sdp->sd_log_revokes;
 889	struct gfs2_bufdata *bd;
 890	struct gfs2_glock *gl;
 891
 892	while (!list_empty(head)) {
 893		bd = list_first_entry(head, struct gfs2_bufdata, bd_list);
 894		list_del_init(&bd->bd_list);
 895		gl = bd->bd_gl;
 896		gfs2_glock_remove_revoke(gl);
 897		kmem_cache_free(gfs2_bufdata_cachep, bd);
 898	}
 899}
 900
 901static void revoke_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 902{
 903	gfs2_drain_revokes(sdp);
 904}
 905
 906static void revoke_lo_before_scan(struct gfs2_jdesc *jd,
 907				  struct gfs2_log_header_host *head, int pass)
 908{
 909	if (pass != 0)
 910		return;
 911
 912	jd->jd_found_revokes = 0;
 913	jd->jd_replay_tail = head->lh_tail;
 914}
 915
 916static int revoke_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 917				   struct gfs2_log_descriptor *ld, __be64 *ptr,
 918				   int pass)
 919{
 920	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 921	unsigned int blks = be32_to_cpu(ld->ld_length);
 922	unsigned int revokes = be32_to_cpu(ld->ld_data1);
 923	struct buffer_head *bh;
 924	unsigned int offset;
 925	u64 blkno;
 926	int first = 1;
 927	int error;
 928
 929	if (pass != 0 || be32_to_cpu(ld->ld_type) != GFS2_LOG_DESC_REVOKE)
 930		return 0;
 931
 932	offset = sizeof(struct gfs2_log_descriptor);
 933
 934	for (; blks; gfs2_replay_incr_blk(jd, &start), blks--) {
 935		error = gfs2_replay_read_block(jd, start, &bh);
 936		if (error)
 937			return error;
 938
 939		if (!first)
 940			gfs2_metatype_check(sdp, bh, GFS2_METATYPE_LB);
 941
 942		while (offset + sizeof(u64) <= sdp->sd_sb.sb_bsize) {
 943			blkno = be64_to_cpu(*(__be64 *)(bh->b_data + offset));
 944
 945			error = gfs2_revoke_add(jd, blkno, start);
 946			if (error < 0) {
 947				brelse(bh);
 948				return error;
 949			}
 950			else if (error)
 951				jd->jd_found_revokes++;
 952
 953			if (!--revokes)
 954				break;
 955			offset += sizeof(u64);
 956		}
 957
 958		brelse(bh);
 959		offset = sizeof(struct gfs2_meta_header);
 960		first = 0;
 961	}
 962
 963	return 0;
 964}
 965
 966static void revoke_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass)
 967{
 968	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 969
 970	if (error) {
 971		gfs2_revoke_clean(jd);
 972		return;
 973	}
 974	if (pass != 1)
 975		return;
 976
 977	fs_info(sdp, "jid=%u: Found %u revoke tags\n",
 978	        jd->jd_jid, jd->jd_found_revokes);
 979
 980	gfs2_revoke_clean(jd);
 981}
 982
 983/**
 984 * databuf_lo_before_commit - Scan the data buffers, writing as we go
 985 * @sdp: The filesystem
 986 * @tr: The system transaction being flushed
 987 */
 988
 989static void databuf_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 990{
 991	unsigned int limit = databuf_limit(sdp);
 992	unsigned int nbuf;
 993	if (tr == NULL)
 994		return;
 995	nbuf = tr->tr_num_databuf_new - tr->tr_num_databuf_rm;
 996	gfs2_before_commit(sdp, limit, nbuf, &tr->tr_databuf, 1);
 997}
 998
 999static int databuf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
1000				    struct gfs2_log_descriptor *ld,
1001				    __be64 *ptr, int pass)
1002{
1003	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
1004	struct gfs2_glock *gl = ip->i_gl;
1005	unsigned int blks = be32_to_cpu(ld->ld_data1);
1006	struct buffer_head *bh_log, *bh_ip;
1007	u64 blkno;
1008	u64 esc;
1009	int error = 0;
1010
1011	if (pass != 1 || be32_to_cpu(ld->ld_type) != GFS2_LOG_DESC_JDATA)
1012		return 0;
1013
1014	gfs2_replay_incr_blk(jd, &start);
1015	for (; blks; gfs2_replay_incr_blk(jd, &start), blks--) {
1016		blkno = be64_to_cpu(*ptr++);
1017		esc = be64_to_cpu(*ptr++);
1018
1019		jd->jd_found_blocks++;
1020
1021		if (gfs2_revoke_check(jd, blkno, start))
1022			continue;
1023
1024		error = gfs2_replay_read_block(jd, start, &bh_log);
1025		if (error)
1026			return error;
1027
1028		bh_ip = gfs2_meta_new(gl, blkno);
1029		memcpy(bh_ip->b_data, bh_log->b_data, bh_log->b_size);
1030
1031		/* Unescape */
1032		if (esc) {
1033			__be32 *eptr = (__be32 *)bh_ip->b_data;
1034			*eptr = cpu_to_be32(GFS2_MAGIC);
1035		}
1036		mark_buffer_dirty(bh_ip);
1037
1038		brelse(bh_log);
1039		brelse(bh_ip);
1040
1041		jd->jd_replayed_blocks++;
1042	}
1043
1044	return error;
1045}
1046
1047/* FIXME: sort out accounting for log blocks etc. */
1048
1049static void databuf_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass)
1050{
1051	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
1052	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
1053
1054	if (error) {
1055		gfs2_inode_metasync(ip->i_gl);
1056		return;
1057	}
1058	if (pass != 1)
1059		return;
1060
1061	/* data sync? */
1062	gfs2_inode_metasync(ip->i_gl);
1063
1064	fs_info(sdp, "jid=%u: Replayed %u of %u data blocks\n",
1065		jd->jd_jid, jd->jd_replayed_blocks, jd->jd_found_blocks);
1066}
1067
1068static void databuf_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
1069{
1070	struct list_head *head;
1071	struct gfs2_bufdata *bd;
1072
1073	if (tr == NULL)
1074		return;
1075
1076	head = &tr->tr_databuf;
1077	while (!list_empty(head)) {
1078		bd = list_first_entry(head, struct gfs2_bufdata, bd_list);
1079		list_del_init(&bd->bd_list);
1080		gfs2_unpin(sdp, bd->bd_bh, tr);
1081	}
1082}
1083
1084
1085static const struct gfs2_log_operations gfs2_buf_lops = {
1086	.lo_before_commit = buf_lo_before_commit,
1087	.lo_after_commit = buf_lo_after_commit,
1088	.lo_before_scan = buf_lo_before_scan,
1089	.lo_scan_elements = buf_lo_scan_elements,
1090	.lo_after_scan = buf_lo_after_scan,
1091	.lo_name = "buf",
1092};
1093
1094static const struct gfs2_log_operations gfs2_revoke_lops = {
1095	.lo_before_commit = revoke_lo_before_commit,
1096	.lo_after_commit = revoke_lo_after_commit,
1097	.lo_before_scan = revoke_lo_before_scan,
1098	.lo_scan_elements = revoke_lo_scan_elements,
1099	.lo_after_scan = revoke_lo_after_scan,
1100	.lo_name = "revoke",
1101};
1102
1103static const struct gfs2_log_operations gfs2_databuf_lops = {
1104	.lo_before_commit = databuf_lo_before_commit,
1105	.lo_after_commit = databuf_lo_after_commit,
1106	.lo_scan_elements = databuf_lo_scan_elements,
1107	.lo_after_scan = databuf_lo_after_scan,
1108	.lo_name = "databuf",
1109};
1110
1111const struct gfs2_log_operations *gfs2_log_ops[] = {
1112	&gfs2_databuf_lops,
1113	&gfs2_buf_lops,
1114	&gfs2_revoke_lops,
1115	NULL,
1116};
1117