jfs_metapage.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 *   Copyright (C) International Business Machines Corp., 2000-2005
  4 *   Portions Copyright (C) Christoph Hellwig, 2001-2002
  5 */
  6
  7#include <linux/blkdev.h>
  8#include <linux/fs.h>
  9#include <linux/mm.h>
 10#include <linux/module.h>
 11#include <linux/bio.h>
 12#include <linux/slab.h>
 13#include <linux/init.h>
 14#include <linux/buffer_head.h>
 15#include <linux/mempool.h>
 16#include <linux/seq_file.h>
 17#include <linux/writeback.h>
 18#include <linux/migrate.h>
 19#include "jfs_incore.h"
 20#include "jfs_superblock.h"
 21#include "jfs_filsys.h"
 22#include "jfs_metapage.h"
 23#include "jfs_txnmgr.h"
 24#include "jfs_debug.h"
 25
 26#ifdef CONFIG_JFS_STATISTICS
 27static struct {
 28	uint	pagealloc;	/* # of page allocations */
 29	uint	pagefree;	/* # of page frees */
 30	uint	lockwait;	/* # of sleeping lock_metapage() calls */
 31} mpStat;
 32#endif
 33
 34#define metapage_locked(mp) test_bit(META_locked, &(mp)->flag)
 35#define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag)
 36
 37static inline void unlock_metapage(struct metapage *mp)
 38{
 39	clear_bit_unlock(META_locked, &mp->flag);
 40	wake_up(&mp->wait);
 41}
 42
 43static inline void __lock_metapage(struct metapage *mp)
 44{
 45	DECLARE_WAITQUEUE(wait, current);
 46	INCREMENT(mpStat.lockwait);
 47	add_wait_queue_exclusive(&mp->wait, &wait);
 48	do {
 49		set_current_state(TASK_UNINTERRUPTIBLE);
 50		if (metapage_locked(mp)) {
 51			folio_unlock(mp->folio);
 52			io_schedule();
 53			folio_lock(mp->folio);
 54		}
 55	} while (trylock_metapage(mp));
 56	__set_current_state(TASK_RUNNING);
 57	remove_wait_queue(&mp->wait, &wait);
 58}
 59
 60/*
 61 * Must have mp->folio locked
 62 */
 63static inline void lock_metapage(struct metapage *mp)
 64{
 65	if (trylock_metapage(mp))
 66		__lock_metapage(mp);
 67}
 68
 69#define METAPOOL_MIN_PAGES 32
 70static struct kmem_cache *metapage_cache;
 71static mempool_t *metapage_mempool;
 72
 73#define MPS_PER_PAGE (PAGE_SIZE >> L2PSIZE)
 74
 75#if MPS_PER_PAGE > 1
 76
 77struct meta_anchor {
 78	int mp_count;
 79	atomic_t io_count;
 80	blk_status_t status;
 81	struct metapage *mp[MPS_PER_PAGE];
 82};
 83
 84static inline struct metapage *folio_to_mp(struct folio *folio, int offset)
 85{
 86	struct meta_anchor *anchor = folio->private;
 87
 88	if (!anchor)
 89		return NULL;
 90	return anchor->mp[offset >> L2PSIZE];
 91}
 92
 93static inline int insert_metapage(struct folio *folio, struct metapage *mp)
 94{
 95	struct meta_anchor *a;
 96	int index;
 97	int l2mp_blocks;	/* log2 blocks per metapage */
 98
 99	a = folio->private;
100	if (!a) {
101		a = kzalloc(sizeof(struct meta_anchor), GFP_NOFS);
102		if (!a)
103			return -ENOMEM;
104		folio_attach_private(folio, a);
105		kmap(&folio->page);
106	}
107
108	if (mp) {
109		l2mp_blocks = L2PSIZE - folio->mapping->host->i_blkbits;
110		index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1);
111		a->mp_count++;
112		a->mp[index] = mp;
113	}
114
115	return 0;
116}
117
118static inline void remove_metapage(struct folio *folio, struct metapage *mp)
119{
120	struct meta_anchor *a = folio->private;
121	int l2mp_blocks = L2PSIZE - folio->mapping->host->i_blkbits;
122	int index;
123
124	index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1);
125
126	BUG_ON(a->mp[index] != mp);
127
128	a->mp[index] = NULL;
129	if (--a->mp_count == 0) {
130		kfree(a);
131		folio_detach_private(folio);
132		kunmap(&folio->page);
133	}
134}
135
136static inline void inc_io(struct folio *folio)
137{
138	struct meta_anchor *anchor = folio->private;
139
140	atomic_inc(&anchor->io_count);
141}
142
143static inline void dec_io(struct folio *folio, blk_status_t status,
144		void (*handler)(struct folio *, blk_status_t))
145{
146	struct meta_anchor *anchor = folio->private;
147
148	if (anchor->status == BLK_STS_OK)
149		anchor->status = status;
150
151	if (atomic_dec_and_test(&anchor->io_count))
152		handler(folio, anchor->status);
153}
154
155#ifdef CONFIG_MIGRATION
156static int __metapage_migrate_folio(struct address_space *mapping,
157				    struct folio *dst, struct folio *src,
158				    enum migrate_mode mode)
159{
160	struct meta_anchor *src_anchor = src->private;
161	struct metapage *mps[MPS_PER_PAGE] = {0};
162	struct metapage *mp;
163	int i, rc;
164
165	for (i = 0; i < MPS_PER_PAGE; i++) {
166		mp = src_anchor->mp[i];
167		if (mp && metapage_locked(mp))
168			return -EAGAIN;
169	}
170
171	rc = filemap_migrate_folio(mapping, dst, src, mode);
172	if (rc)
173		return rc;
174
175	for (i = 0; i < MPS_PER_PAGE; i++) {
176		mp = src_anchor->mp[i];
177		if (!mp)
178			continue;
179		if (unlikely(insert_metapage(dst, mp))) {
180			/* If error, roll-back previosly inserted pages */
181			for (int j = 0 ; j < i; j++) {
182				if (mps[j])
183					remove_metapage(dst, mps[j]);
184			}
185			return -EAGAIN;
186		}
187		mps[i] = mp;
188	}
189
190	/* Update the metapage and remove it from src */
191	for (i = 0; i < MPS_PER_PAGE; i++) {
192		mp = mps[i];
193		if (mp) {
194			int page_offset = mp->data - folio_address(src);
195
196			mp->data = folio_address(dst) + page_offset;
197			mp->folio = dst;
198			remove_metapage(src, mp);
199		}
200	}
201
202	return 0;
203}
204#endif	/* CONFIG_MIGRATION */
205
206#else
207
208static inline struct metapage *folio_to_mp(struct folio *folio, int offset)
209{
210	return folio->private;
211}
212
213static inline int insert_metapage(struct folio *folio, struct metapage *mp)
214{
215	if (mp) {
216		folio_attach_private(folio, mp);
217		kmap(&folio->page);
218	}
219	return 0;
220}
221
222static inline void remove_metapage(struct folio *folio, struct metapage *mp)
223{
224	folio_detach_private(folio);
225	kunmap(&folio->page);
226}
227
228#define inc_io(folio) do {} while(0)
229#define dec_io(folio, status, handler) handler(folio, status)
230
231#ifdef CONFIG_MIGRATION
232static int __metapage_migrate_folio(struct address_space *mapping,
233				    struct folio *dst, struct folio *src,
234				    enum migrate_mode mode)
235{
236	struct metapage *mp;
237	int page_offset;
238	int rc;
239
240	mp = folio_to_mp(src, 0);
241	if (metapage_locked(mp))
242		return -EAGAIN;
243
244	rc = filemap_migrate_folio(mapping, dst, src, mode);
245	if (rc)
246		return rc;
247
248	if (unlikely(insert_metapage(dst, mp)))
249		return -EAGAIN;
250
251	page_offset = mp->data - folio_address(src);
252	mp->data = folio_address(dst) + page_offset;
253	mp->folio = dst;
254	remove_metapage(src, mp);
255
256	return 0;
257}
258#endif	/* CONFIG_MIGRATION */
259
260#endif
261
262static inline struct metapage *alloc_metapage(gfp_t gfp_mask)
263{
264	struct metapage *mp = mempool_alloc(metapage_mempool, gfp_mask);
265
266	if (mp) {
267		mp->lid = 0;
268		mp->lsn = 0;
269		mp->data = NULL;
270		mp->clsn = 0;
271		mp->log = NULL;
272		init_waitqueue_head(&mp->wait);
273	}
274	return mp;
275}
276
277static inline void free_metapage(struct metapage *mp)
278{
279	mempool_free(mp, metapage_mempool);
280}
281
282int __init metapage_init(void)
283{
284	/*
285	 * Allocate the metapage structures
286	 */
287	metapage_cache = kmem_cache_create("jfs_mp", sizeof(struct metapage),
288					   0, 0, NULL);
289	if (metapage_cache == NULL)
290		return -ENOMEM;
291
292	metapage_mempool = mempool_create_slab_pool(METAPOOL_MIN_PAGES,
293						    metapage_cache);
294
295	if (metapage_mempool == NULL) {
296		kmem_cache_destroy(metapage_cache);
297		return -ENOMEM;
298	}
299
300	return 0;
301}
302
303void metapage_exit(void)
304{
305	mempool_destroy(metapage_mempool);
306	kmem_cache_destroy(metapage_cache);
307}
308
309static inline void drop_metapage(struct folio *folio, struct metapage *mp)
310{
311	if (mp->count || mp->nohomeok || test_bit(META_dirty, &mp->flag) ||
312	    test_bit(META_io, &mp->flag))
313		return;
314	remove_metapage(folio, mp);
315	INCREMENT(mpStat.pagefree);
316	free_metapage(mp);
317}
318
319/*
320 * Metapage address space operations
321 */
322
323static sector_t metapage_get_blocks(struct inode *inode, sector_t lblock,
324				    int *len)
325{
326	int rc = 0;
327	int xflag;
328	s64 xaddr;
329	sector_t file_blocks = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
330			       inode->i_blkbits;
331
332	if (lblock >= file_blocks)
333		return 0;
334	if (lblock + *len > file_blocks)
335		*len = file_blocks - lblock;
336
337	if (inode->i_ino) {
338		rc = xtLookup(inode, (s64)lblock, *len, &xflag, &xaddr, len, 0);
339		if ((rc == 0) && *len)
340			lblock = (sector_t)xaddr;
341		else
342			lblock = 0;
343	} /* else no mapping */
344
345	return lblock;
346}
347
348static void last_read_complete(struct folio *folio, blk_status_t status)
349{
350	if (status)
351		printk(KERN_ERR "Read error %d at %#llx\n", status,
352				folio_pos(folio));
353
354	folio_end_read(folio, status == 0);
355}
356
357static void metapage_read_end_io(struct bio *bio)
358{
359	struct folio *folio = bio->bi_private;
360
361	dec_io(folio, bio->bi_status, last_read_complete);
362	bio_put(bio);
363}
364
365static void remove_from_logsync(struct metapage *mp)
366{
367	struct jfs_log *log = mp->log;
368	unsigned long flags;
369/*
370 * This can race.  Recheck that log hasn't been set to null, and after
371 * acquiring logsync lock, recheck lsn
372 */
373	if (!log)
374		return;
375
376	LOGSYNC_LOCK(log, flags);
377	if (mp->lsn) {
378		mp->log = NULL;
379		mp->lsn = 0;
380		mp->clsn = 0;
381		log->count--;
382		list_del(&mp->synclist);
383	}
384	LOGSYNC_UNLOCK(log, flags);
385}
386
387static void last_write_complete(struct folio *folio, blk_status_t status)
388{
389	struct metapage *mp;
390	unsigned int offset;
391
392	if (status) {
393		int err = blk_status_to_errno(status);
394		printk(KERN_ERR "metapage_write_end_io: I/O error\n");
395		mapping_set_error(folio->mapping, err);
396	}
397
398	for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
399		mp = folio_to_mp(folio, offset);
400		if (mp && test_bit(META_io, &mp->flag)) {
401			if (mp->lsn)
402				remove_from_logsync(mp);
403			clear_bit(META_io, &mp->flag);
404		}
405		/*
406		 * I'd like to call drop_metapage here, but I don't think it's
407		 * safe unless I have the page locked
408		 */
409	}
410	folio_end_writeback(folio);
411}
412
413static void metapage_write_end_io(struct bio *bio)
414{
415	struct folio *folio = bio->bi_private;
416
417	BUG_ON(!folio->private);
418
419	dec_io(folio, bio->bi_status, last_write_complete);
420	bio_put(bio);
421}
422
423static int metapage_write_folio(struct folio *folio,
424		struct writeback_control *wbc)
425{
426	struct bio *bio = NULL;
427	int block_offset;	/* block offset of mp within page */
428	struct inode *inode = folio->mapping->host;
429	int blocks_per_mp = JFS_SBI(inode->i_sb)->nbperpage;
430	int len;
431	int xlen;
432	struct metapage *mp;
433	int redirty = 0;
434	sector_t lblock;
435	int nr_underway = 0;
436	sector_t pblock;
437	sector_t next_block = 0;
438	sector_t page_start;
439	unsigned long bio_bytes = 0;
440	unsigned long bio_offset = 0;
441	int offset;
442	int bad_blocks = 0;
443
444	page_start = folio_pos(folio) >> inode->i_blkbits;
445	BUG_ON(!folio_test_locked(folio));
446	BUG_ON(folio_test_writeback(folio));
447	folio_start_writeback(folio);
448
449	for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
450		mp = folio_to_mp(folio, offset);
451
452		if (!mp || !test_bit(META_dirty, &mp->flag))
453			continue;
454
455		if (mp->nohomeok && !test_bit(META_forcewrite, &mp->flag)) {
456			redirty = 1;
457			/*
458			 * Make sure this page isn't blocked indefinitely.
459			 * If the journal isn't undergoing I/O, push it
460			 */
461			if (mp->log && !(mp->log->cflag & logGC_PAGEOUT))
462				jfs_flush_journal(mp->log, 0);
463			continue;
464		}
465
466		clear_bit(META_dirty, &mp->flag);
467		set_bit(META_io, &mp->flag);
468		block_offset = offset >> inode->i_blkbits;
469		lblock = page_start + block_offset;
470		if (bio) {
471			if (xlen && lblock == next_block) {
472				/* Contiguous, in memory & on disk */
473				len = min(xlen, blocks_per_mp);
474				xlen -= len;
475				bio_bytes += len << inode->i_blkbits;
476				continue;
477			}
478			/* Not contiguous */
479			bio_add_folio_nofail(bio, folio, bio_bytes, bio_offset);
480			/*
481			 * Increment counter before submitting i/o to keep
482			 * count from hitting zero before we're through
483			 */
484			inc_io(folio);
485			if (!bio->bi_iter.bi_size)
486				goto dump_bio;
487			submit_bio(bio);
488			nr_underway++;
489			bio = NULL;
490		} else
491			inc_io(folio);
492		xlen = (folio_size(folio) - offset) >> inode->i_blkbits;
493		pblock = metapage_get_blocks(inode, lblock, &xlen);
494		if (!pblock) {
495			printk(KERN_ERR "JFS: metapage_get_blocks failed\n");
496			/*
497			 * We already called inc_io(), but can't cancel it
498			 * with dec_io() until we're done with the page
499			 */
500			bad_blocks++;
501			continue;
502		}
503		len = min(xlen, (int)JFS_SBI(inode->i_sb)->nbperpage);
504
505		bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOFS);
506		bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9);
507		bio->bi_end_io = metapage_write_end_io;
508		bio->bi_private = folio;
509
510		/* Don't call bio_add_page yet, we may add to this vec */
511		bio_offset = offset;
512		bio_bytes = len << inode->i_blkbits;
513
514		xlen -= len;
515		next_block = lblock + len;
516	}
517	if (bio) {
518		bio_add_folio_nofail(bio, folio, bio_bytes, bio_offset);
519		if (!bio->bi_iter.bi_size)
520			goto dump_bio;
521
522		submit_bio(bio);
523		nr_underway++;
524	}
525	if (redirty)
526		folio_redirty_for_writepage(wbc, folio);
527
528	folio_unlock(folio);
529
530	if (bad_blocks)
531		goto err_out;
532
533	if (nr_underway == 0)
534		folio_end_writeback(folio);
535
536	return 0;
537dump_bio:
538	print_hex_dump(KERN_ERR, "JFS: dump of bio: ", DUMP_PREFIX_ADDRESS, 16,
539		       4, bio, sizeof(*bio), 0);
540	bio_put(bio);
541	folio_unlock(folio);
542	dec_io(folio, BLK_STS_OK, last_write_complete);
543err_out:
544	while (bad_blocks--)
545		dec_io(folio, BLK_STS_OK, last_write_complete);
546	return -EIO;
547}
548
549static int metapage_writepages(struct address_space *mapping,
550		struct writeback_control *wbc)
551{
552	struct blk_plug plug;
553	struct folio *folio = NULL;
554	int err;
555
556	blk_start_plug(&plug);
557	while ((folio = writeback_iter(mapping, wbc, folio, &err)))
558		err = metapage_write_folio(folio, wbc);
559	blk_finish_plug(&plug);
560
561	return err;
562}
563
564static int metapage_read_folio(struct file *fp, struct folio *folio)
565{
566	struct inode *inode = folio->mapping->host;
567	struct bio *bio = NULL;
568	int block_offset;
569	int blocks_per_page = i_blocks_per_folio(inode, folio);
570	sector_t page_start;	/* address of page in fs blocks */
571	sector_t pblock;
572	int xlen;
573	unsigned int len;
574	int offset;
575
576	BUG_ON(!folio_test_locked(folio));
577	page_start = folio_pos(folio) >> inode->i_blkbits;
578
579	block_offset = 0;
580	while (block_offset < blocks_per_page) {
581		xlen = blocks_per_page - block_offset;
582		pblock = metapage_get_blocks(inode, page_start + block_offset,
583					     &xlen);
584		if (pblock) {
585			if (!folio->private)
586				insert_metapage(folio, NULL);
587			inc_io(folio);
588			if (bio)
589				submit_bio(bio);
590
591			bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_READ,
592					GFP_NOFS);
593			bio->bi_iter.bi_sector =
594				pblock << (inode->i_blkbits - 9);
595			bio->bi_end_io = metapage_read_end_io;
596			bio->bi_private = folio;
597			len = xlen << inode->i_blkbits;
598			offset = block_offset << inode->i_blkbits;
599			bio_add_folio_nofail(bio, folio, len, offset);
600			block_offset += xlen;
601		} else
602			block_offset++;
603	}
604	if (bio)
605		submit_bio(bio);
606	else
607		folio_unlock(folio);
608
609	return 0;
610}
611
612static bool metapage_release_folio(struct folio *folio, gfp_t gfp_mask)
613{
614	struct metapage *mp;
615	bool ret = true;
616	int offset;
617
618	for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
619		mp = folio_to_mp(folio, offset);
620
621		if (!mp)
622			continue;
623
624		jfs_info("metapage_release_folio: mp = 0x%p", mp);
625		if (mp->count || mp->nohomeok ||
626		    test_bit(META_dirty, &mp->flag)) {
627			jfs_info("count = %ld, nohomeok = %d", mp->count,
628				 mp->nohomeok);
629			ret = false;
630			continue;
631		}
632		if (mp->lsn)
633			remove_from_logsync(mp);
634		remove_metapage(folio, mp);
635		INCREMENT(mpStat.pagefree);
636		free_metapage(mp);
637	}
638	return ret;
639}
640
641#ifdef CONFIG_MIGRATION
642/*
643 * metapage_migrate_folio - Migration function for JFS metapages
644 */
645static int metapage_migrate_folio(struct address_space *mapping,
646				  struct folio *dst, struct folio *src,
647				  enum migrate_mode mode)
648{
649	int expected_count;
650
651	if (!src->private)
652		return filemap_migrate_folio(mapping, dst, src, mode);
653
654	/* Check whether page does not have extra refs before we do more work */
655	expected_count = folio_expected_ref_count(src) + 1;
656	if (folio_ref_count(src) != expected_count)
657		return -EAGAIN;
658	return __metapage_migrate_folio(mapping, dst, src, mode);
659}
660#else
661#define metapage_migrate_folio NULL
662#endif	/* CONFIG_MIGRATION */
663
664static void metapage_invalidate_folio(struct folio *folio, size_t offset,
665				    size_t length)
666{
667	BUG_ON(offset || length < folio_size(folio));
668
669	BUG_ON(folio_test_writeback(folio));
670
671	metapage_release_folio(folio, 0);
672}
673
674const struct address_space_operations jfs_metapage_aops = {
675	.read_folio	= metapage_read_folio,
676	.writepages	= metapage_writepages,
677	.release_folio	= metapage_release_folio,
678	.invalidate_folio = metapage_invalidate_folio,
679	.dirty_folio	= filemap_dirty_folio,
680	.migrate_folio	= metapage_migrate_folio,
681};
682
683struct metapage *__get_metapage(struct inode *inode, unsigned long lblock,
684				unsigned int size, int absolute,
685				unsigned long new)
686{
687	int l2BlocksPerPage;
688	int l2bsize;
689	struct address_space *mapping;
690	struct metapage *mp = NULL;
691	struct folio *folio;
692	unsigned long page_index;
693	unsigned long page_offset;
694
695	jfs_info("__get_metapage: ino = %ld, lblock = 0x%lx, abs=%d",
696		 inode->i_ino, lblock, absolute);
697
698	l2bsize = inode->i_blkbits;
699	l2BlocksPerPage = PAGE_SHIFT - l2bsize;
700	page_index = lblock >> l2BlocksPerPage;
701	page_offset = (lblock - (page_index << l2BlocksPerPage)) << l2bsize;
702	if ((page_offset + size) > PAGE_SIZE) {
703		jfs_err("MetaData crosses page boundary!!");
704		jfs_err("lblock = %lx, size  = %d", lblock, size);
705		dump_stack();
706		return NULL;
707	}
708	if (absolute)
709		mapping = JFS_SBI(inode->i_sb)->direct_inode->i_mapping;
710	else {
711		/*
712		 * If an nfs client tries to read an inode that is larger
713		 * than any existing inodes, we may try to read past the
714		 * end of the inode map
715		 */
716		if ((lblock << inode->i_blkbits) >= inode->i_size)
717			return NULL;
718		mapping = inode->i_mapping;
719	}
720
721	if (new && (PSIZE == PAGE_SIZE)) {
722		folio = filemap_grab_folio(mapping, page_index);
723		if (IS_ERR(folio)) {
724			jfs_err("filemap_grab_folio failed!");
725			return NULL;
726		}
727		folio_mark_uptodate(folio);
728	} else {
729		folio = read_mapping_folio(mapping, page_index, NULL);
730		if (IS_ERR(folio)) {
731			jfs_err("read_mapping_page failed!");
732			return NULL;
733		}
734		folio_lock(folio);
735	}
736
737	mp = folio_to_mp(folio, page_offset);
738	if (mp) {
739		if (mp->logical_size != size) {
740			jfs_error(inode->i_sb,
741				  "get_mp->logical_size != size\n");
742			jfs_err("logical_size = %d, size = %d",
743				mp->logical_size, size);
744			dump_stack();
745			goto unlock;
746		}
747		mp->count++;
748		lock_metapage(mp);
749		if (test_bit(META_discard, &mp->flag)) {
750			if (!new) {
751				jfs_error(inode->i_sb,
752					  "using a discarded metapage\n");
753				discard_metapage(mp);
754				goto unlock;
755			}
756			clear_bit(META_discard, &mp->flag);
757		}
758	} else {
759		INCREMENT(mpStat.pagealloc);
760		mp = alloc_metapage(GFP_NOFS);
761		if (!mp)
762			goto unlock;
763		mp->folio = folio;
764		mp->sb = inode->i_sb;
765		mp->flag = 0;
766		mp->xflag = COMMIT_PAGE;
767		mp->count = 1;
768		mp->nohomeok = 0;
769		mp->logical_size = size;
770		mp->data = folio_address(folio) + page_offset;
771		mp->index = lblock;
772		if (unlikely(insert_metapage(folio, mp))) {
773			free_metapage(mp);
774			goto unlock;
775		}
776		lock_metapage(mp);
777	}
778
779	if (new) {
780		jfs_info("zeroing mp = 0x%p", mp);
781		memset(mp->data, 0, PSIZE);
782	}
783
784	folio_unlock(folio);
785	jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp, mp->data);
786	return mp;
787
788unlock:
789	folio_unlock(folio);
790	return NULL;
791}
792
793void grab_metapage(struct metapage * mp)
794{
795	jfs_info("grab_metapage: mp = 0x%p", mp);
796	folio_get(mp->folio);
797	folio_lock(mp->folio);
798	mp->count++;
799	lock_metapage(mp);
800	folio_unlock(mp->folio);
801}
802
803static int metapage_write_one(struct folio *folio)
804{
805	struct address_space *mapping = folio->mapping;
806	struct writeback_control wbc = {
807		.sync_mode = WB_SYNC_ALL,
808		.nr_to_write = folio_nr_pages(folio),
809	};
810	int ret = 0;
811
812	BUG_ON(!folio_test_locked(folio));
813
814	folio_wait_writeback(folio);
815
816	if (folio_clear_dirty_for_io(folio)) {
817		folio_get(folio);
818		ret = metapage_write_folio(folio, &wbc);
819		if (ret == 0)
820			folio_wait_writeback(folio);
821		folio_put(folio);
822	} else {
823		folio_unlock(folio);
824	}
825
826	if (!ret)
827		ret = filemap_check_errors(mapping);
828	return ret;
829}
830
831void force_metapage(struct metapage *mp)
832{
833	struct folio *folio = mp->folio;
834	jfs_info("force_metapage: mp = 0x%p", mp);
835	set_bit(META_forcewrite, &mp->flag);
836	clear_bit(META_sync, &mp->flag);
837	folio_get(folio);
838	folio_lock(folio);
839	folio_mark_dirty(folio);
840	if (metapage_write_one(folio))
841		jfs_error(mp->sb, "metapage_write_one() failed\n");
842	clear_bit(META_forcewrite, &mp->flag);
843	folio_put(folio);
844}
845
846void hold_metapage(struct metapage *mp)
847{
848	folio_lock(mp->folio);
849}
850
851void put_metapage(struct metapage *mp)
852{
853	if (mp->count || mp->nohomeok) {
854		/* Someone else will release this */
855		folio_unlock(mp->folio);
856		return;
857	}
858	folio_get(mp->folio);
859	mp->count++;
860	lock_metapage(mp);
861	folio_unlock(mp->folio);
862	release_metapage(mp);
863}
864
865void release_metapage(struct metapage * mp)
866{
867	struct folio *folio = mp->folio;
868	jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag);
869
870	folio_lock(folio);
871	unlock_metapage(mp);
872
873	assert(mp->count);
874	if (--mp->count || mp->nohomeok) {
875		folio_unlock(folio);
876		folio_put(folio);
877		return;
878	}
879
880	if (test_bit(META_dirty, &mp->flag)) {
881		folio_mark_dirty(folio);
882		if (test_bit(META_sync, &mp->flag)) {
883			clear_bit(META_sync, &mp->flag);
884			if (metapage_write_one(folio))
885				jfs_error(mp->sb, "metapage_write_one() failed\n");
886			folio_lock(folio);
887		}
888	} else if (mp->lsn)	/* discard_metapage doesn't remove it */
889		remove_from_logsync(mp);
890
891	/* Try to keep metapages from using up too much memory */
892	drop_metapage(folio, mp);
893
894	folio_unlock(folio);
895	folio_put(folio);
896}
897
898void __invalidate_metapages(struct inode *ip, s64 addr, int len)
899{
900	sector_t lblock;
901	int l2BlocksPerPage = PAGE_SHIFT - ip->i_blkbits;
902	int BlocksPerPage = 1 << l2BlocksPerPage;
903	/* All callers are interested in block device's mapping */
904	struct address_space *mapping =
905		JFS_SBI(ip->i_sb)->direct_inode->i_mapping;
906	struct metapage *mp;
907	unsigned int offset;
908
909	/*
910	 * Mark metapages to discard.  They will eventually be
911	 * released, but should not be written.
912	 */
913	for (lblock = addr & ~(BlocksPerPage - 1); lblock < addr + len;
914	     lblock += BlocksPerPage) {
915		struct folio *folio = filemap_lock_folio(mapping,
916				lblock >> l2BlocksPerPage);
917		if (IS_ERR(folio))
918			continue;
919		for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
920			mp = folio_to_mp(folio, offset);
921			if (!mp)
922				continue;
923			if (mp->index < addr)
924				continue;
925			if (mp->index >= addr + len)
926				break;
927
928			clear_bit(META_dirty, &mp->flag);
929			set_bit(META_discard, &mp->flag);
930			if (mp->lsn)
931				remove_from_logsync(mp);
932		}
933		folio_unlock(folio);
934		folio_put(folio);
935	}
936}
937
938#ifdef CONFIG_JFS_STATISTICS
939int jfs_mpstat_proc_show(struct seq_file *m, void *v)
940{
941	seq_printf(m,
942		       "JFS Metapage statistics\n"
943		       "=======================\n"
944		       "page allocations = %d\n"
945		       "page frees = %d\n"
946		       "lock waits = %d\n",
947		       mpStat.pagealloc,
948		       mpStat.pagefree,
949		       mpStat.lockwait);
950	return 0;
951}
952#endif