xfs_fsmap.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Copyright (C) 2017 Oracle.  All Rights Reserved.
   4 * Author: Darrick J. Wong <darrick.wong@oracle.com>
   5 */
   6#include "xfs.h"
   7#include "xfs_fs.h"
   8#include "xfs_shared.h"
   9#include "xfs_format.h"
  10#include "xfs_log_format.h"
  11#include "xfs_trans_resv.h"
  12#include "xfs_mount.h"
  13#include "xfs_inode.h"
  14#include "xfs_trans.h"
  15#include "xfs_btree.h"
  16#include "xfs_rmap_btree.h"
  17#include "xfs_trace.h"
  18#include "xfs_rmap.h"
  19#include "xfs_alloc.h"
  20#include "xfs_bit.h"
  21#include <linux/fsmap.h>
  22#include "xfs_fsmap.h"
  23#include "xfs_refcount.h"
  24#include "xfs_refcount_btree.h"
  25#include "xfs_alloc_btree.h"
  26#include "xfs_rtbitmap.h"
  27#include "xfs_ag.h"
  28#include "xfs_rtgroup.h"
  29#include "xfs_rtrmap_btree.h"
  30#include "xfs_rtrefcount_btree.h"
  31
  32/* Convert an xfs_fsmap to an fsmap. */
  33static void
  34xfs_fsmap_from_internal(
  35	struct fsmap		*dest,
  36	struct xfs_fsmap	*src)
  37{
  38	dest->fmr_device = src->fmr_device;
  39	dest->fmr_flags = src->fmr_flags;
  40	dest->fmr_physical = BBTOB(src->fmr_physical);
  41	dest->fmr_owner = src->fmr_owner;
  42	dest->fmr_offset = BBTOB(src->fmr_offset);
  43	dest->fmr_length = BBTOB(src->fmr_length);
  44	dest->fmr_reserved[0] = 0;
  45	dest->fmr_reserved[1] = 0;
  46	dest->fmr_reserved[2] = 0;
  47}
  48
  49/* Convert an fsmap to an xfs_fsmap. */
  50static void
  51xfs_fsmap_to_internal(
  52	struct xfs_fsmap	*dest,
  53	struct fsmap		*src)
  54{
  55	dest->fmr_device = src->fmr_device;
  56	dest->fmr_flags = src->fmr_flags;
  57	dest->fmr_physical = BTOBBT(src->fmr_physical);
  58	dest->fmr_owner = src->fmr_owner;
  59	dest->fmr_offset = BTOBBT(src->fmr_offset);
  60	dest->fmr_length = BTOBBT(src->fmr_length);
  61}
  62
  63/* Convert an fsmap owner into an rmapbt owner. */
  64static int
  65xfs_fsmap_owner_to_rmap(
  66	struct xfs_rmap_irec	*dest,
  67	const struct xfs_fsmap	*src)
  68{
  69	if (!(src->fmr_flags & FMR_OF_SPECIAL_OWNER)) {
  70		dest->rm_owner = src->fmr_owner;
  71		return 0;
  72	}
  73
  74	switch (src->fmr_owner) {
  75	case 0:			/* "lowest owner id possible" */
  76	case -1ULL:		/* "highest owner id possible" */
  77		dest->rm_owner = src->fmr_owner;
  78		break;
  79	case XFS_FMR_OWN_FREE:
  80		dest->rm_owner = XFS_RMAP_OWN_NULL;
  81		break;
  82	case XFS_FMR_OWN_UNKNOWN:
  83		dest->rm_owner = XFS_RMAP_OWN_UNKNOWN;
  84		break;
  85	case XFS_FMR_OWN_FS:
  86		dest->rm_owner = XFS_RMAP_OWN_FS;
  87		break;
  88	case XFS_FMR_OWN_LOG:
  89		dest->rm_owner = XFS_RMAP_OWN_LOG;
  90		break;
  91	case XFS_FMR_OWN_AG:
  92		dest->rm_owner = XFS_RMAP_OWN_AG;
  93		break;
  94	case XFS_FMR_OWN_INOBT:
  95		dest->rm_owner = XFS_RMAP_OWN_INOBT;
  96		break;
  97	case XFS_FMR_OWN_INODES:
  98		dest->rm_owner = XFS_RMAP_OWN_INODES;
  99		break;
 100	case XFS_FMR_OWN_REFC:
 101		dest->rm_owner = XFS_RMAP_OWN_REFC;
 102		break;
 103	case XFS_FMR_OWN_COW:
 104		dest->rm_owner = XFS_RMAP_OWN_COW;
 105		break;
 106	case XFS_FMR_OWN_DEFECTIVE:	/* not implemented */
 107		/* fall through */
 108	default:
 109		return -EINVAL;
 110	}
 111	return 0;
 112}
 113
 114/* Convert an rmapbt owner into an fsmap owner. */
 115static int
 116xfs_fsmap_owner_from_frec(
 117	struct xfs_fsmap		*dest,
 118	const struct xfs_fsmap_irec	*frec)
 119{
 120	dest->fmr_flags = 0;
 121	if (!XFS_RMAP_NON_INODE_OWNER(frec->owner)) {
 122		dest->fmr_owner = frec->owner;
 123		return 0;
 124	}
 125	dest->fmr_flags |= FMR_OF_SPECIAL_OWNER;
 126
 127	switch (frec->owner) {
 128	case XFS_RMAP_OWN_FS:
 129		dest->fmr_owner = XFS_FMR_OWN_FS;
 130		break;
 131	case XFS_RMAP_OWN_LOG:
 132		dest->fmr_owner = XFS_FMR_OWN_LOG;
 133		break;
 134	case XFS_RMAP_OWN_AG:
 135		dest->fmr_owner = XFS_FMR_OWN_AG;
 136		break;
 137	case XFS_RMAP_OWN_INOBT:
 138		dest->fmr_owner = XFS_FMR_OWN_INOBT;
 139		break;
 140	case XFS_RMAP_OWN_INODES:
 141		dest->fmr_owner = XFS_FMR_OWN_INODES;
 142		break;
 143	case XFS_RMAP_OWN_REFC:
 144		dest->fmr_owner = XFS_FMR_OWN_REFC;
 145		break;
 146	case XFS_RMAP_OWN_COW:
 147		dest->fmr_owner = XFS_FMR_OWN_COW;
 148		break;
 149	case XFS_RMAP_OWN_NULL:	/* "free" */
 150		dest->fmr_owner = XFS_FMR_OWN_FREE;
 151		break;
 152	default:
 153		ASSERT(0);
 154		return -EFSCORRUPTED;
 155	}
 156	return 0;
 157}
 158
 159/* getfsmap query state */
 160struct xfs_getfsmap_info {
 161	struct xfs_fsmap_head	*head;
 162	struct fsmap		*fsmap_recs;	/* mapping records */
 163	struct xfs_buf		*agf_bp;	/* AGF, for refcount queries */
 164	struct xfs_group	*group;		/* group info, if applicable */
 165	xfs_daddr_t		next_daddr;	/* next daddr we expect */
 166	/* daddr of low fsmap key when we're using the rtbitmap */
 167	xfs_daddr_t		low_daddr;
 168	/* daddr of high fsmap key, or the last daddr on the device */
 169	xfs_daddr_t		end_daddr;
 170	u64			missing_owner;	/* owner of holes */
 171	u32			dev;		/* device id */
 172	/*
 173	 * Low rmap key for the query.  If low.rm_blockcount is nonzero, this
 174	 * is the second (or later) call to retrieve the recordset in pieces.
 175	 * xfs_getfsmap_rec_before_start will compare all records retrieved
 176	 * by the rmapbt query to filter out any records that start before
 177	 * the last record.
 178	 */
 179	struct xfs_rmap_irec	low;
 180	struct xfs_rmap_irec	high;		/* high rmap key */
 181	bool			last;		/* last extent? */
 182};
 183
 184/* Associate a device with a getfsmap handler. */
 185struct xfs_getfsmap_dev {
 186	u32			dev;
 187	int			(*fn)(struct xfs_trans *tp,
 188				      const struct xfs_fsmap *keys,
 189				      struct xfs_getfsmap_info *info);
 190	sector_t		nr_sectors;
 191};
 192
 193/* Compare two getfsmap device handlers. */
 194static int
 195xfs_getfsmap_dev_compare(
 196	const void			*p1,
 197	const void			*p2)
 198{
 199	const struct xfs_getfsmap_dev	*d1 = p1;
 200	const struct xfs_getfsmap_dev	*d2 = p2;
 201
 202	return d1->dev - d2->dev;
 203}
 204
 205/* Decide if this mapping is shared. */
 206STATIC int
 207xfs_getfsmap_is_shared(
 208	struct xfs_trans		*tp,
 209	struct xfs_getfsmap_info	*info,
 210	const struct xfs_fsmap_irec	*frec,
 211	bool				*stat)
 212{
 213	struct xfs_mount		*mp = tp->t_mountp;
 214	struct xfs_btree_cur		*cur;
 215	xfs_agblock_t			fbno;
 216	xfs_extlen_t			flen = 0;
 217	int				error;
 218
 219	*stat = false;
 220	if (!xfs_has_reflink(mp) || !info->group)
 221		return 0;
 222
 223	if (info->group->xg_type == XG_TYPE_RTG)
 224		cur = xfs_rtrefcountbt_init_cursor(tp, to_rtg(info->group));
 225	else
 226		cur = xfs_refcountbt_init_cursor(mp, tp, info->agf_bp,
 227				to_perag(info->group));
 228
 229	/* Are there any shared blocks here? */
 230	error = xfs_refcount_find_shared(cur, frec->rec_key,
 231			XFS_BB_TO_FSBT(mp, frec->len_daddr), &fbno, &flen,
 232			false);
 233
 234	xfs_btree_del_cursor(cur, error);
 235	if (error)
 236		return error;
 237
 238	*stat = flen > 0;
 239	return 0;
 240}
 241
 242static inline void
 243xfs_getfsmap_format(
 244	struct xfs_mount		*mp,
 245	struct xfs_fsmap		*xfm,
 246	struct xfs_getfsmap_info	*info)
 247{
 248	struct fsmap			*rec;
 249
 250	trace_xfs_getfsmap_mapping(mp, xfm);
 251
 252	rec = &info->fsmap_recs[info->head->fmh_entries++];
 253	xfs_fsmap_from_internal(rec, xfm);
 254}
 255
 256static inline bool
 257xfs_getfsmap_frec_before_start(
 258	struct xfs_getfsmap_info	*info,
 259	const struct xfs_fsmap_irec	*frec)
 260{
 261	if (info->low_daddr != XFS_BUF_DADDR_NULL)
 262		return frec->start_daddr < info->low_daddr;
 263	if (info->low.rm_blockcount) {
 264		struct xfs_rmap_irec	rec = {
 265			.rm_startblock	= frec->rec_key,
 266			.rm_owner	= frec->owner,
 267			.rm_flags	= frec->rm_flags,
 268		};
 269
 270		return xfs_rmap_compare(&rec, &info->low) < 0;
 271	}
 272
 273	return false;
 274}
 275
 276/*
 277 * Format a reverse mapping for getfsmap, having translated rm_startblock
 278 * into the appropriate daddr units.  Pass in a nonzero @len_daddr if the
 279 * length could be larger than rm_blockcount in struct xfs_rmap_irec.
 280 */
 281STATIC int
 282xfs_getfsmap_helper(
 283	struct xfs_trans		*tp,
 284	struct xfs_getfsmap_info	*info,
 285	const struct xfs_fsmap_irec	*frec)
 286{
 287	struct xfs_fsmap		fmr;
 288	struct xfs_mount		*mp = tp->t_mountp;
 289	bool				shared;
 290	int				error = 0;
 291
 292	if (fatal_signal_pending(current))
 293		return -EINTR;
 294
 295	/*
 296	 * Filter out records that start before our startpoint, if the
 297	 * caller requested that.
 298	 */
 299	if (xfs_getfsmap_frec_before_start(info, frec))
 300		goto out;
 301
 302	/* Are we just counting mappings? */
 303	if (info->head->fmh_count == 0) {
 304		if (info->head->fmh_entries == UINT_MAX)
 305			return -ECANCELED;
 306
 307		if (frec->start_daddr > info->next_daddr)
 308			info->head->fmh_entries++;
 309
 310		if (info->last)
 311			return 0;
 312
 313		info->head->fmh_entries++;
 314		goto out;
 315	}
 316
 317	/*
 318	 * If the record starts past the last physical block we saw,
 319	 * then we've found a gap.  Report the gap as being owned by
 320	 * whatever the caller specified is the missing owner.
 321	 */
 322	if (frec->start_daddr > info->next_daddr) {
 323		if (info->head->fmh_entries >= info->head->fmh_count)
 324			return -ECANCELED;
 325
 326		fmr.fmr_device = info->dev;
 327		fmr.fmr_physical = info->next_daddr;
 328		fmr.fmr_owner = info->missing_owner;
 329		fmr.fmr_offset = 0;
 330		fmr.fmr_length = frec->start_daddr - info->next_daddr;
 331		fmr.fmr_flags = FMR_OF_SPECIAL_OWNER;
 332		xfs_getfsmap_format(mp, &fmr, info);
 333	}
 334
 335	if (info->last)
 336		goto out;
 337
 338	/* Fill out the extent we found */
 339	if (info->head->fmh_entries >= info->head->fmh_count)
 340		return -ECANCELED;
 341
 342	trace_xfs_fsmap_mapping(mp, info->dev,
 343			info->group ? info->group->xg_gno : NULLAGNUMBER,
 344			frec);
 345
 346	fmr.fmr_device = info->dev;
 347	fmr.fmr_physical = frec->start_daddr;
 348	error = xfs_fsmap_owner_from_frec(&fmr, frec);
 349	if (error)
 350		return error;
 351	fmr.fmr_offset = XFS_FSB_TO_BB(mp, frec->offset);
 352	fmr.fmr_length = frec->len_daddr;
 353	if (frec->rm_flags & XFS_RMAP_UNWRITTEN)
 354		fmr.fmr_flags |= FMR_OF_PREALLOC;
 355	if (frec->rm_flags & XFS_RMAP_ATTR_FORK)
 356		fmr.fmr_flags |= FMR_OF_ATTR_FORK;
 357	if (frec->rm_flags & XFS_RMAP_BMBT_BLOCK)
 358		fmr.fmr_flags |= FMR_OF_EXTENT_MAP;
 359	if (fmr.fmr_flags == 0) {
 360		error = xfs_getfsmap_is_shared(tp, info, frec, &shared);
 361		if (error)
 362			return error;
 363		if (shared)
 364			fmr.fmr_flags |= FMR_OF_SHARED;
 365	}
 366
 367	xfs_getfsmap_format(mp, &fmr, info);
 368out:
 369	info->next_daddr = max(info->next_daddr,
 370			       frec->start_daddr + frec->len_daddr);
 371	return 0;
 372}
 373
 374static inline int
 375xfs_getfsmap_group_helper(
 376	struct xfs_getfsmap_info	*info,
 377	struct xfs_trans		*tp,
 378	struct xfs_group		*xg,
 379	xfs_agblock_t			startblock,
 380	xfs_extlen_t			blockcount,
 381	struct xfs_fsmap_irec		*frec)
 382{
 383	/*
 384	 * For an info->last query, we're looking for a gap between the last
 385	 * mapping emitted and the high key specified by userspace.  If the
 386	 * user's query spans less than 1 fsblock, then info->high and
 387	 * info->low will have the same rm_startblock, which causes rec_daddr
 388	 * and next_daddr to be the same.  Therefore, use the end_daddr that
 389	 * we calculated from userspace's high key to synthesize the record.
 390	 * Note that if the btree query found a mapping, there won't be a gap.
 391	 */
 392	if (info->last)
 393		frec->start_daddr = info->end_daddr + 1;
 394	else
 395		frec->start_daddr = xfs_gbno_to_daddr(xg, startblock);
 396
 397	frec->len_daddr = XFS_FSB_TO_BB(xg->xg_mount, blockcount);
 398	return xfs_getfsmap_helper(tp, info, frec);
 399}
 400
 401/* Transform a rmapbt irec into a fsmap */
 402STATIC int
 403xfs_getfsmap_rmapbt_helper(
 404	struct xfs_btree_cur		*cur,
 405	const struct xfs_rmap_irec	*rec,
 406	void				*priv)
 407{
 408	struct xfs_fsmap_irec		frec = {
 409		.owner			= rec->rm_owner,
 410		.offset			= rec->rm_offset,
 411		.rm_flags		= rec->rm_flags,
 412		.rec_key		= rec->rm_startblock,
 413	};
 414	struct xfs_getfsmap_info	*info = priv;
 415
 416	return xfs_getfsmap_group_helper(info, cur->bc_tp, cur->bc_group,
 417			rec->rm_startblock, rec->rm_blockcount, &frec);
 418}
 419
 420/* Transform a bnobt irec into a fsmap */
 421STATIC int
 422xfs_getfsmap_datadev_bnobt_helper(
 423	struct xfs_btree_cur		*cur,
 424	const struct xfs_alloc_rec_incore *rec,
 425	void				*priv)
 426{
 427	struct xfs_fsmap_irec		frec = {
 428		.owner			= XFS_RMAP_OWN_NULL, /* "free" */
 429		.rec_key		= rec->ar_startblock,
 430	};
 431	struct xfs_getfsmap_info	*info = priv;
 432
 433	return xfs_getfsmap_group_helper(info, cur->bc_tp, cur->bc_group,
 434			rec->ar_startblock, rec->ar_blockcount, &frec);
 435}
 436
 437/* Set rmap flags based on the getfsmap flags */
 438static void
 439xfs_getfsmap_set_irec_flags(
 440	struct xfs_rmap_irec	*irec,
 441	const struct xfs_fsmap	*fmr)
 442{
 443	irec->rm_flags = 0;
 444	if (fmr->fmr_flags & FMR_OF_ATTR_FORK)
 445		irec->rm_flags |= XFS_RMAP_ATTR_FORK;
 446	if (fmr->fmr_flags & FMR_OF_EXTENT_MAP)
 447		irec->rm_flags |= XFS_RMAP_BMBT_BLOCK;
 448	if (fmr->fmr_flags & FMR_OF_PREALLOC)
 449		irec->rm_flags |= XFS_RMAP_UNWRITTEN;
 450}
 451
 452static inline bool
 453rmap_not_shareable(struct xfs_mount *mp, const struct xfs_rmap_irec *r)
 454{
 455	if (!xfs_has_reflink(mp))
 456		return true;
 457	if (XFS_RMAP_NON_INODE_OWNER(r->rm_owner))
 458		return true;
 459	if (r->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK |
 460			   XFS_RMAP_UNWRITTEN))
 461		return true;
 462	return false;
 463}
 464
 465/* Execute a getfsmap query against the regular data device. */
 466STATIC int
 467__xfs_getfsmap_datadev(
 468	struct xfs_trans		*tp,
 469	const struct xfs_fsmap		*keys,
 470	struct xfs_getfsmap_info	*info,
 471	int				(*query_fn)(struct xfs_trans *,
 472						    struct xfs_getfsmap_info *,
 473						    struct xfs_btree_cur **,
 474						    void *),
 475	void				*priv)
 476{
 477	struct xfs_mount		*mp = tp->t_mountp;
 478	struct xfs_perag		*pag = NULL;
 479	struct xfs_btree_cur		*bt_cur = NULL;
 480	xfs_fsblock_t			start_fsb;
 481	xfs_fsblock_t			end_fsb;
 482	xfs_agnumber_t			start_ag, end_ag;
 483	uint64_t			eofs;
 484	int				error = 0;
 485
 486	eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks);
 487	if (keys[0].fmr_physical >= eofs)
 488		return 0;
 489	start_fsb = XFS_DADDR_TO_FSB(mp, keys[0].fmr_physical);
 490	end_fsb = XFS_DADDR_TO_FSB(mp, min(eofs - 1, keys[1].fmr_physical));
 491
 492	/*
 493	 * Convert the fsmap low/high keys to AG based keys.  Initialize
 494	 * low to the fsmap low key and max out the high key to the end
 495	 * of the AG.
 496	 */
 497	info->low.rm_offset = XFS_BB_TO_FSBT(mp, keys[0].fmr_offset);
 498	error = xfs_fsmap_owner_to_rmap(&info->low, &keys[0]);
 499	if (error)
 500		return error;
 501	info->low.rm_blockcount = XFS_BB_TO_FSBT(mp, keys[0].fmr_length);
 502	xfs_getfsmap_set_irec_flags(&info->low, &keys[0]);
 503
 504	/* Adjust the low key if we are continuing from where we left off. */
 505	if (info->low.rm_blockcount == 0) {
 506		/* No previous record from which to continue */
 507	} else if (rmap_not_shareable(mp, &info->low)) {
 508		/* Last record seen was an unshareable extent */
 509		info->low.rm_owner = 0;
 510		info->low.rm_offset = 0;
 511
 512		start_fsb += info->low.rm_blockcount;
 513		if (XFS_FSB_TO_DADDR(mp, start_fsb) >= eofs)
 514			return 0;
 515	} else {
 516		/* Last record seen was a shareable file data extent */
 517		info->low.rm_offset += info->low.rm_blockcount;
 518	}
 519	info->low.rm_startblock = XFS_FSB_TO_AGBNO(mp, start_fsb);
 520
 521	info->high.rm_startblock = -1U;
 522	info->high.rm_owner = ULLONG_MAX;
 523	info->high.rm_offset = ULLONG_MAX;
 524	info->high.rm_blockcount = 0;
 525	info->high.rm_flags = XFS_RMAP_KEY_FLAGS | XFS_RMAP_REC_FLAGS;
 526
 527	start_ag = XFS_FSB_TO_AGNO(mp, start_fsb);
 528	end_ag = XFS_FSB_TO_AGNO(mp, end_fsb);
 529
 530	while ((pag = xfs_perag_next_range(mp, pag, start_ag, end_ag))) {
 531		/*
 532		 * Set the AG high key from the fsmap high key if this
 533		 * is the last AG that we're querying.
 534		 */
 535		info->group = pag_group(pag);
 536		if (pag_agno(pag) == end_ag) {
 537			info->high.rm_startblock = XFS_FSB_TO_AGBNO(mp,
 538					end_fsb);
 539			info->high.rm_offset = XFS_BB_TO_FSBT(mp,
 540					keys[1].fmr_offset);
 541			error = xfs_fsmap_owner_to_rmap(&info->high, &keys[1]);
 542			if (error)
 543				break;
 544			xfs_getfsmap_set_irec_flags(&info->high, &keys[1]);
 545		}
 546
 547		if (bt_cur) {
 548			xfs_btree_del_cursor(bt_cur, XFS_BTREE_NOERROR);
 549			bt_cur = NULL;
 550			xfs_trans_brelse(tp, info->agf_bp);
 551			info->agf_bp = NULL;
 552		}
 553
 554		error = xfs_alloc_read_agf(pag, tp, 0, &info->agf_bp);
 555		if (error)
 556			break;
 557
 558		trace_xfs_fsmap_low_group_key(mp, info->dev, pag_agno(pag),
 559				&info->low);
 560		trace_xfs_fsmap_high_group_key(mp, info->dev, pag_agno(pag),
 561				&info->high);
 562
 563		error = query_fn(tp, info, &bt_cur, priv);
 564		if (error)
 565			break;
 566
 567		/*
 568		 * Set the AG low key to the start of the AG prior to
 569		 * moving on to the next AG.
 570		 */
 571		if (pag_agno(pag) == start_ag)
 572			memset(&info->low, 0, sizeof(info->low));
 573
 574		/*
 575		 * If this is the last AG, report any gap at the end of it
 576		 * before we drop the reference to the perag when the loop
 577		 * terminates.
 578		 */
 579		if (pag_agno(pag) == end_ag) {
 580			info->last = true;
 581			error = query_fn(tp, info, &bt_cur, priv);
 582			if (error)
 583				break;
 584		}
 585		info->group = NULL;
 586	}
 587
 588	if (bt_cur)
 589		xfs_btree_del_cursor(bt_cur, error < 0 ? XFS_BTREE_ERROR :
 590							 XFS_BTREE_NOERROR);
 591	if (info->agf_bp) {
 592		xfs_trans_brelse(tp, info->agf_bp);
 593		info->agf_bp = NULL;
 594	}
 595	if (info->group) {
 596		xfs_perag_rele(pag);
 597		info->group = NULL;
 598	} else if (pag) {
 599		/* loop termination case */
 600		xfs_perag_rele(pag);
 601	}
 602
 603	return error;
 604}
 605
 606/* Actually query the rmap btree. */
 607STATIC int
 608xfs_getfsmap_datadev_rmapbt_query(
 609	struct xfs_trans		*tp,
 610	struct xfs_getfsmap_info	*info,
 611	struct xfs_btree_cur		**curpp,
 612	void				*priv)
 613{
 614	/* Report any gap at the end of the last AG. */
 615	if (info->last)
 616		return xfs_getfsmap_rmapbt_helper(*curpp, &info->high, info);
 617
 618	/* Allocate cursor for this AG and query_range it. */
 619	*curpp = xfs_rmapbt_init_cursor(tp->t_mountp, tp, info->agf_bp,
 620			to_perag(info->group));
 621	return xfs_rmap_query_range(*curpp, &info->low, &info->high,
 622			xfs_getfsmap_rmapbt_helper, info);
 623}
 624
 625/* Execute a getfsmap query against the regular data device rmapbt. */
 626STATIC int
 627xfs_getfsmap_datadev_rmapbt(
 628	struct xfs_trans		*tp,
 629	const struct xfs_fsmap		*keys,
 630	struct xfs_getfsmap_info	*info)
 631{
 632	info->missing_owner = XFS_FMR_OWN_FREE;
 633	return __xfs_getfsmap_datadev(tp, keys, info,
 634			xfs_getfsmap_datadev_rmapbt_query, NULL);
 635}
 636
 637/* Actually query the bno btree. */
 638STATIC int
 639xfs_getfsmap_datadev_bnobt_query(
 640	struct xfs_trans		*tp,
 641	struct xfs_getfsmap_info	*info,
 642	struct xfs_btree_cur		**curpp,
 643	void				*priv)
 644{
 645	struct xfs_alloc_rec_incore	*key = priv;
 646
 647	/* Report any gap at the end of the last AG. */
 648	if (info->last)
 649		return xfs_getfsmap_datadev_bnobt_helper(*curpp, &key[1], info);
 650
 651	/* Allocate cursor for this AG and query_range it. */
 652	*curpp = xfs_bnobt_init_cursor(tp->t_mountp, tp, info->agf_bp,
 653			to_perag(info->group));
 654	key->ar_startblock = info->low.rm_startblock;
 655	key[1].ar_startblock = info->high.rm_startblock;
 656	return xfs_alloc_query_range(*curpp, key, &key[1],
 657			xfs_getfsmap_datadev_bnobt_helper, info);
 658}
 659
 660/* Execute a getfsmap query against the regular data device's bnobt. */
 661STATIC int
 662xfs_getfsmap_datadev_bnobt(
 663	struct xfs_trans		*tp,
 664	const struct xfs_fsmap		*keys,
 665	struct xfs_getfsmap_info	*info)
 666{
 667	struct xfs_alloc_rec_incore	akeys[2];
 668
 669	memset(akeys, 0, sizeof(akeys));
 670	info->missing_owner = XFS_FMR_OWN_UNKNOWN;
 671	return __xfs_getfsmap_datadev(tp, keys, info,
 672			xfs_getfsmap_datadev_bnobt_query, &akeys[0]);
 673}
 674
 675/* Execute a getfsmap query against the log device. */
 676STATIC int
 677xfs_getfsmap_logdev(
 678	struct xfs_trans		*tp,
 679	const struct xfs_fsmap		*keys,
 680	struct xfs_getfsmap_info	*info)
 681{
 682	struct xfs_fsmap_irec		frec = {
 683		.start_daddr		= 0,
 684		.rec_key		= 0,
 685		.owner			= XFS_RMAP_OWN_LOG,
 686	};
 687	struct xfs_mount		*mp = tp->t_mountp;
 688	xfs_fsblock_t			start_fsb, end_fsb;
 689	uint64_t			eofs;
 690
 691	eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks);
 692	if (keys[0].fmr_physical >= eofs)
 693		return 0;
 694	start_fsb = XFS_BB_TO_FSBT(mp,
 695				keys[0].fmr_physical + keys[0].fmr_length);
 696	end_fsb = XFS_BB_TO_FSB(mp, min(eofs - 1, keys[1].fmr_physical));
 697
 698	/* Adjust the low key if we are continuing from where we left off. */
 699	if (keys[0].fmr_length > 0)
 700		info->low_daddr = XFS_FSB_TO_BB(mp, start_fsb);
 701
 702	trace_xfs_fsmap_low_linear_key(mp, info->dev, start_fsb);
 703	trace_xfs_fsmap_high_linear_key(mp, info->dev, end_fsb);
 704
 705	if (start_fsb > 0)
 706		return 0;
 707
 708	/* Fabricate an rmap entry for the external log device. */
 709	frec.len_daddr = XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks);
 710	return xfs_getfsmap_helper(tp, info, &frec);
 711}
 712
 713#ifdef CONFIG_XFS_RT
 714/* Transform a rtbitmap "record" into a fsmap */
 715STATIC int
 716xfs_getfsmap_rtdev_rtbitmap_helper(
 717	struct xfs_rtgroup		*rtg,
 718	struct xfs_trans		*tp,
 719	const struct xfs_rtalloc_rec	*rec,
 720	void				*priv)
 721{
 722	struct xfs_fsmap_irec		frec = {
 723		.owner			= XFS_RMAP_OWN_NULL, /* "free" */
 724	};
 725	struct xfs_mount		*mp = rtg_mount(rtg);
 726	struct xfs_getfsmap_info	*info = priv;
 727	xfs_rtblock_t			start_rtb =
 728				xfs_rtx_to_rtb(rtg, rec->ar_startext);
 729	uint64_t			rtbcount =
 730				xfs_rtbxlen_to_blen(mp, rec->ar_extcount);
 731
 732	/*
 733	 * For an info->last query, we're looking for a gap between the last
 734	 * mapping emitted and the high key specified by userspace.  If the
 735	 * user's query spans less than 1 fsblock, then info->high and
 736	 * info->low will have the same rm_startblock, which causes rec_daddr
 737	 * and next_daddr to be the same.  Therefore, use the end_daddr that
 738	 * we calculated from userspace's high key to synthesize the record.
 739	 * Note that if the btree query found a mapping, there won't be a gap.
 740	 */
 741	if (info->last)
 742		frec.start_daddr = info->end_daddr + 1;
 743	else
 744		frec.start_daddr = xfs_rtb_to_daddr(mp, start_rtb);
 745
 746	frec.len_daddr = XFS_FSB_TO_BB(mp, rtbcount);
 747	return xfs_getfsmap_helper(tp, info, &frec);
 748}
 749
 750/* Execute a getfsmap query against the realtime device rtbitmap. */
 751STATIC int
 752xfs_getfsmap_rtdev_rtbitmap(
 753	struct xfs_trans		*tp,
 754	const struct xfs_fsmap		*keys,
 755	struct xfs_getfsmap_info	*info)
 756{
 757	struct xfs_mount		*mp = tp->t_mountp;
 758	xfs_rtblock_t			start_rtbno, end_rtbno;
 759	xfs_rtxnum_t			start_rtx, end_rtx;
 760	xfs_rgnumber_t			start_rgno, end_rgno;
 761	struct xfs_rtgroup		*rtg = NULL;
 762	uint64_t			eofs;
 763	int				error;
 764
 765	eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
 766	if (keys[0].fmr_physical >= eofs)
 767		return 0;
 768
 769	info->missing_owner = XFS_FMR_OWN_UNKNOWN;
 770
 771	/* Adjust the low key if we are continuing from where we left off. */
 772	start_rtbno = xfs_daddr_to_rtb(mp,
 773			keys[0].fmr_physical + keys[0].fmr_length);
 774	if (keys[0].fmr_length > 0) {
 775		info->low_daddr = xfs_rtb_to_daddr(mp, start_rtbno);
 776		if (info->low_daddr >= eofs)
 777			return 0;
 778	}
 779	start_rtx = xfs_rtb_to_rtx(mp, start_rtbno);
 780	start_rgno = xfs_rtb_to_rgno(mp, start_rtbno);
 781
 782	end_rtbno = xfs_daddr_to_rtb(mp, min(eofs - 1, keys[1].fmr_physical));
 783	end_rgno = xfs_rtb_to_rgno(mp, end_rtbno);
 784
 785	trace_xfs_fsmap_low_linear_key(mp, info->dev, start_rtbno);
 786	trace_xfs_fsmap_high_linear_key(mp, info->dev, end_rtbno);
 787
 788	end_rtx = -1ULL;
 789
 790	while ((rtg = xfs_rtgroup_next_range(mp, rtg, start_rgno, end_rgno))) {
 791		if (rtg_rgno(rtg) == end_rgno)
 792			end_rtx = xfs_rtb_to_rtx(mp,
 793					end_rtbno + mp->m_sb.sb_rextsize - 1);
 794
 795		info->group = rtg_group(rtg);
 796		xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
 797		error = xfs_rtalloc_query_range(rtg, tp, start_rtx, end_rtx,
 798				xfs_getfsmap_rtdev_rtbitmap_helper, info);
 799		if (error)
 800			break;
 801
 802		/*
 803		 * Report any gaps at the end of the rtbitmap by simulating a
 804		 * zero-length free extent starting at the rtx after the end
 805		 * of the query range.
 806		 */
 807		if (rtg_rgno(rtg) == end_rgno) {
 808			struct xfs_rtalloc_rec	ahigh = {
 809				.ar_startext	= min(end_rtx + 1,
 810						      rtg->rtg_extents),
 811			};
 812
 813			info->last = true;
 814			error = xfs_getfsmap_rtdev_rtbitmap_helper(rtg, tp,
 815					&ahigh, info);
 816			if (error)
 817				break;
 818		}
 819
 820		xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
 821		info->group = NULL;
 822		start_rtx = 0;
 823	}
 824
 825	/* loop termination case */
 826	if (rtg) {
 827		if (info->group) {
 828			xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
 829			info->group = NULL;
 830		}
 831		xfs_rtgroup_rele(rtg);
 832	}
 833
 834	return error;
 835}
 836
 837/* Transform a realtime rmapbt record into a fsmap */
 838STATIC int
 839xfs_getfsmap_rtdev_rmapbt_helper(
 840	struct xfs_btree_cur		*cur,
 841	const struct xfs_rmap_irec	*rec,
 842	void				*priv)
 843{
 844	struct xfs_fsmap_irec		frec = {
 845		.owner			= rec->rm_owner,
 846		.offset			= rec->rm_offset,
 847		.rm_flags		= rec->rm_flags,
 848		.rec_key		= rec->rm_startblock,
 849	};
 850	struct xfs_getfsmap_info	*info = priv;
 851
 852	return xfs_getfsmap_group_helper(info, cur->bc_tp, cur->bc_group,
 853			rec->rm_startblock, rec->rm_blockcount, &frec);
 854}
 855
 856/* Actually query the rtrmap btree. */
 857STATIC int
 858xfs_getfsmap_rtdev_rmapbt_query(
 859	struct xfs_trans		*tp,
 860	struct xfs_getfsmap_info	*info,
 861	struct xfs_btree_cur		**curpp)
 862{
 863	struct xfs_rtgroup		*rtg = to_rtg(info->group);
 864
 865	/* Query the rtrmapbt */
 866	xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP | XFS_RTGLOCK_REFCOUNT);
 867	*curpp = xfs_rtrmapbt_init_cursor(tp, rtg);
 868	return xfs_rmap_query_range(*curpp, &info->low, &info->high,
 869			xfs_getfsmap_rtdev_rmapbt_helper, info);
 870}
 871
 872/* Execute a getfsmap query against the realtime device rmapbt. */
 873STATIC int
 874xfs_getfsmap_rtdev_rmapbt(
 875	struct xfs_trans		*tp,
 876	const struct xfs_fsmap		*keys,
 877	struct xfs_getfsmap_info	*info)
 878{
 879	struct xfs_fsmap		key0 = *keys; /* struct copy */
 880	struct xfs_mount		*mp = tp->t_mountp;
 881	struct xfs_rtgroup		*rtg = NULL;
 882	struct xfs_btree_cur		*bt_cur = NULL;
 883	xfs_daddr_t			rtstart_daddr;
 884	xfs_rtblock_t			start_rtb;
 885	xfs_rtblock_t			end_rtb;
 886	xfs_rgnumber_t			start_rg, end_rg;
 887	uint64_t			eofs;
 888	int				error = 0;
 889
 890	eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart + mp->m_sb.sb_rblocks);
 891	if (key0.fmr_physical >= eofs)
 892		return 0;
 893
 894	/*
 895	 * On zoned filesystems with an internal rt volume, the volume comes
 896	 * immediately after the end of the data volume.  However, the
 897	 * xfs_rtblock_t address space is relative to the start of the data
 898	 * device, which means that the first @rtstart fsblocks do not actually
 899	 * point anywhere.  If a fsmap query comes in with the low key starting
 900	 * below @rtstart, report it as "owned by filesystem".
 901	 */
 902	rtstart_daddr = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart);
 903	if (xfs_has_zoned(mp) && key0.fmr_physical < rtstart_daddr) {
 904		struct xfs_fsmap_irec		frec = {
 905			.owner			= XFS_RMAP_OWN_FS,
 906			.len_daddr		= rtstart_daddr,
 907		};
 908
 909		/*
 910		 * Adjust the start of the query range if we're picking up from
 911		 * a previous round, and only emit the record if we haven't
 912		 * already gone past.
 913		 */
 914		key0.fmr_physical += key0.fmr_length;
 915		if (key0.fmr_physical < rtstart_daddr) {
 916			error = xfs_getfsmap_helper(tp, info, &frec);
 917			if (error)
 918				return error;
 919
 920			key0.fmr_physical = rtstart_daddr;
 921		}
 922
 923		/* Zero the other fields to avoid further adjustments. */
 924		key0.fmr_owner = 0;
 925		key0.fmr_offset = 0;
 926		key0.fmr_length = 0;
 927	}
 928
 929	start_rtb = xfs_daddr_to_rtb(mp, key0.fmr_physical);
 930	end_rtb = xfs_daddr_to_rtb(mp, min(eofs - 1, keys[1].fmr_physical));
 931	info->missing_owner = XFS_FMR_OWN_FREE;
 932
 933	/*
 934	 * Convert the fsmap low/high keys to rtgroup based keys.  Initialize
 935	 * low to the fsmap low key and max out the high key to the end
 936	 * of the rtgroup.
 937	 */
 938	info->low.rm_offset = XFS_BB_TO_FSBT(mp, key0.fmr_offset);
 939	error = xfs_fsmap_owner_to_rmap(&info->low, &key0);
 940	if (error)
 941		return error;
 942	info->low.rm_blockcount = XFS_BB_TO_FSBT(mp, key0.fmr_length);
 943	xfs_getfsmap_set_irec_flags(&info->low, &key0);
 944
 945	/* Adjust the low key if we are continuing from where we left off. */
 946	if (info->low.rm_blockcount == 0) {
 947		/* No previous record from which to continue */
 948	} else if (rmap_not_shareable(mp, &info->low)) {
 949		/* Last record seen was an unshareable extent */
 950		info->low.rm_owner = 0;
 951		info->low.rm_offset = 0;
 952
 953		start_rtb += info->low.rm_blockcount;
 954		if (xfs_rtb_to_daddr(mp, start_rtb) >= eofs)
 955			return 0;
 956	} else {
 957		/* Last record seen was a shareable file data extent */
 958		info->low.rm_offset += info->low.rm_blockcount;
 959	}
 960	info->low.rm_startblock = xfs_rtb_to_rgbno(mp, start_rtb);
 961
 962	info->high.rm_startblock = -1U;
 963	info->high.rm_owner = ULLONG_MAX;
 964	info->high.rm_offset = ULLONG_MAX;
 965	info->high.rm_blockcount = 0;
 966	info->high.rm_flags = XFS_RMAP_KEY_FLAGS | XFS_RMAP_REC_FLAGS;
 967
 968	start_rg = xfs_rtb_to_rgno(mp, start_rtb);
 969	end_rg = xfs_rtb_to_rgno(mp, end_rtb);
 970
 971	while ((rtg = xfs_rtgroup_next_range(mp, rtg, start_rg, end_rg))) {
 972		/*
 973		 * Set the rtgroup high key from the fsmap high key if this
 974		 * is the last rtgroup that we're querying.
 975		 */
 976		info->group = rtg_group(rtg);
 977		if (rtg_rgno(rtg) == end_rg) {
 978			info->high.rm_startblock =
 979				xfs_rtb_to_rgbno(mp, end_rtb);
 980			info->high.rm_offset =
 981				XFS_BB_TO_FSBT(mp, keys[1].fmr_offset);
 982			error = xfs_fsmap_owner_to_rmap(&info->high, &keys[1]);
 983			if (error)
 984				break;
 985			xfs_getfsmap_set_irec_flags(&info->high, &keys[1]);
 986		}
 987
 988		if (bt_cur) {
 989			xfs_rtgroup_unlock(to_rtg(bt_cur->bc_group),
 990					XFS_RTGLOCK_RMAP |
 991					XFS_RTGLOCK_REFCOUNT);
 992			xfs_btree_del_cursor(bt_cur, XFS_BTREE_NOERROR);
 993			bt_cur = NULL;
 994		}
 995
 996		trace_xfs_fsmap_low_group_key(mp, info->dev, rtg_rgno(rtg),
 997				&info->low);
 998		trace_xfs_fsmap_high_group_key(mp, info->dev, rtg_rgno(rtg),
 999				&info->high);
1000
1001		error = xfs_getfsmap_rtdev_rmapbt_query(tp, info, &bt_cur);
1002		if (error)
1003			break;
1004
1005		/*
1006		 * Set the rtgroup low key to the start of the rtgroup prior to
1007		 * moving on to the next rtgroup.
1008		 */
1009		if (rtg_rgno(rtg) == start_rg)
1010			memset(&info->low, 0, sizeof(info->low));
1011
1012		/*
1013		 * If this is the last rtgroup, report any gap at the end of it
1014		 * before we drop the reference to the perag when the loop
1015		 * terminates.
1016		 */
1017		if (rtg_rgno(rtg) == end_rg) {
1018			info->last = true;
1019			error = xfs_getfsmap_rtdev_rmapbt_helper(bt_cur,
1020					&info->high, info);
1021			if (error)
1022				break;
1023		}
1024		info->group = NULL;
1025	}
1026
1027	if (bt_cur) {
1028		xfs_rtgroup_unlock(to_rtg(bt_cur->bc_group),
1029				XFS_RTGLOCK_RMAP | XFS_RTGLOCK_REFCOUNT);
1030		xfs_btree_del_cursor(bt_cur, error < 0 ? XFS_BTREE_ERROR :
1031							 XFS_BTREE_NOERROR);
1032	}
1033
1034	/* loop termination case */
1035	if (rtg) {
1036		info->group = NULL;
1037		xfs_rtgroup_rele(rtg);
1038	}
1039
1040	return error;
1041}
1042#endif /* CONFIG_XFS_RT */
1043
1044static uint32_t
1045xfs_getfsmap_device(
1046	struct xfs_mount	*mp,
1047	enum xfs_device		dev)
1048{
1049	if (mp->m_sb.sb_rtstart)
1050		return dev;
1051
1052	switch (dev) {
1053	case XFS_DEV_DATA:
1054		return new_encode_dev(mp->m_ddev_targp->bt_dev);
1055	case XFS_DEV_LOG:
1056		return new_encode_dev(mp->m_logdev_targp->bt_dev);
1057	case XFS_DEV_RT:
1058		if (!mp->m_rtdev_targp)
1059			break;
1060		return new_encode_dev(mp->m_rtdev_targp->bt_dev);
1061	}
1062
1063	return -1;
1064}
1065
1066/* Do we recognize the device? */
1067STATIC bool
1068xfs_getfsmap_is_valid_device(
1069	struct xfs_mount	*mp,
1070	struct xfs_fsmap	*fm)
1071{
1072	return fm->fmr_device == 0 ||
1073		fm->fmr_device == UINT_MAX ||
1074		fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_DATA) ||
1075		fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_LOG) ||
1076		(mp->m_rtdev_targp &&
1077		 fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_RT));
1078}
1079
1080/* Ensure that the low key is less than the high key. */
1081STATIC bool
1082xfs_getfsmap_check_keys(
1083	struct xfs_fsmap		*low_key,
1084	struct xfs_fsmap		*high_key)
1085{
1086	if (low_key->fmr_flags & (FMR_OF_SPECIAL_OWNER | FMR_OF_EXTENT_MAP)) {
1087		if (low_key->fmr_offset)
1088			return false;
1089	}
1090	if (high_key->fmr_flags != -1U &&
1091	    (high_key->fmr_flags & (FMR_OF_SPECIAL_OWNER |
1092				    FMR_OF_EXTENT_MAP))) {
1093		if (high_key->fmr_offset && high_key->fmr_offset != -1ULL)
1094			return false;
1095	}
1096	if (high_key->fmr_length && high_key->fmr_length != -1ULL)
1097		return false;
1098
1099	if (low_key->fmr_device > high_key->fmr_device)
1100		return false;
1101	if (low_key->fmr_device < high_key->fmr_device)
1102		return true;
1103
1104	if (low_key->fmr_physical > high_key->fmr_physical)
1105		return false;
1106	if (low_key->fmr_physical < high_key->fmr_physical)
1107		return true;
1108
1109	if (low_key->fmr_owner > high_key->fmr_owner)
1110		return false;
1111	if (low_key->fmr_owner < high_key->fmr_owner)
1112		return true;
1113
1114	if (low_key->fmr_offset > high_key->fmr_offset)
1115		return false;
1116	if (low_key->fmr_offset < high_key->fmr_offset)
1117		return true;
1118
1119	return false;
1120}
1121
1122/*
1123 * There are only two devices if we didn't configure RT devices at build time.
1124 */
1125#ifdef CONFIG_XFS_RT
1126#define XFS_GETFSMAP_DEVS	3
1127#else
1128#define XFS_GETFSMAP_DEVS	2
1129#endif /* CONFIG_XFS_RT */
1130
1131/*
1132 * Get filesystem's extents as described in head, and format for output. Fills
1133 * in the supplied records array until there are no more reverse mappings to
1134 * return or head.fmh_entries == head.fmh_count.  In the second case, this
1135 * function returns -ECANCELED to indicate that more records would have been
1136 * returned.
1137 *
1138 * Key to Confusion
1139 * ----------------
1140 * There are multiple levels of keys and counters at work here:
1141 * xfs_fsmap_head.fmh_keys	-- low and high fsmap keys passed in;
1142 *				   these reflect fs-wide sector addrs.
1143 * dkeys			-- fmh_keys used to query each device;
1144 *				   these are fmh_keys but w/ the low key
1145 *				   bumped up by fmr_length.
1146 * xfs_getfsmap_info.next_daddr	-- next disk addr we expect to see; this
1147 *				   is how we detect gaps in the fsmap
1148				   records and report them.
1149 * xfs_getfsmap_info.low/high	-- per-AG low/high keys computed from
1150 *				   dkeys; used to query the metadata.
1151 */
1152STATIC int
1153xfs_getfsmap(
1154	struct xfs_mount		*mp,
1155	struct xfs_fsmap_head		*head,
1156	struct fsmap			*fsmap_recs)
1157{
1158	struct xfs_trans		*tp = NULL;
1159	struct xfs_fsmap		dkeys[2];	/* per-dev keys */
1160	struct xfs_getfsmap_dev		handlers[XFS_GETFSMAP_DEVS];
1161	struct xfs_getfsmap_info	info = {
1162		.fsmap_recs		= fsmap_recs,
1163		.head			= head,
1164	};
1165	bool				use_rmap;
1166	int				i;
1167	int				error = 0;
1168
1169	if (head->fmh_iflags & ~FMH_IF_VALID)
1170		return -EINVAL;
1171	if (!xfs_getfsmap_is_valid_device(mp, &head->fmh_keys[0]) ||
1172	    !xfs_getfsmap_is_valid_device(mp, &head->fmh_keys[1]))
1173		return -EINVAL;
1174	if (!xfs_getfsmap_check_keys(&head->fmh_keys[0], &head->fmh_keys[1]))
1175		return -EINVAL;
1176
1177	use_rmap = xfs_has_rmapbt(mp) &&
1178		   has_capability_noaudit(current, CAP_SYS_ADMIN);
1179	head->fmh_entries = 0;
1180
1181	/* Set up our device handlers. */
1182	memset(handlers, 0, sizeof(handlers));
1183	handlers[0].nr_sectors = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks);
1184	handlers[0].dev = xfs_getfsmap_device(mp, XFS_DEV_DATA);
1185	if (use_rmap)
1186		handlers[0].fn = xfs_getfsmap_datadev_rmapbt;
1187	else
1188		handlers[0].fn = xfs_getfsmap_datadev_bnobt;
1189	if (mp->m_logdev_targp != mp->m_ddev_targp) {
1190		handlers[1].nr_sectors = XFS_FSB_TO_BB(mp,
1191						       mp->m_sb.sb_logblocks);
1192		handlers[1].dev = xfs_getfsmap_device(mp, XFS_DEV_LOG);
1193		handlers[1].fn = xfs_getfsmap_logdev;
1194	}
1195#ifdef CONFIG_XFS_RT
1196	/*
1197	 * For zoned file systems there is no rtbitmap, so only support fsmap
1198	 * if the callers is privileged enough to use the full rmap version.
1199	 */
1200	if (mp->m_rtdev_targp && (use_rmap || !xfs_has_zoned(mp))) {
1201		handlers[2].nr_sectors = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
1202		handlers[2].dev = xfs_getfsmap_device(mp, XFS_DEV_RT);
1203		if (use_rmap)
1204			handlers[2].fn = xfs_getfsmap_rtdev_rmapbt;
1205		else
1206			handlers[2].fn = xfs_getfsmap_rtdev_rtbitmap;
1207	}
1208#endif /* CONFIG_XFS_RT */
1209
1210	xfs_sort(handlers, XFS_GETFSMAP_DEVS, sizeof(struct xfs_getfsmap_dev),
1211			xfs_getfsmap_dev_compare);
1212
1213	/*
1214	 * To continue where we left off, we allow userspace to use the
1215	 * last mapping from a previous call as the low key of the next.
1216	 * This is identified by a non-zero length in the low key. We
1217	 * have to increment the low key in this scenario to ensure we
1218	 * don't return the same mapping again, and instead return the
1219	 * very next mapping.
1220	 *
1221	 * If the low key mapping refers to file data, the same physical
1222	 * blocks could be mapped to several other files/offsets.
1223	 * According to rmapbt record ordering, the minimal next
1224	 * possible record for the block range is the next starting
1225	 * offset in the same inode. Therefore, each fsmap backend bumps
1226	 * the file offset to continue the search appropriately.  For
1227	 * all other low key mapping types (attr blocks, metadata), each
1228	 * fsmap backend bumps the physical offset as there can be no
1229	 * other mapping for the same physical block range.
1230	 */
1231	dkeys[0] = head->fmh_keys[0];
1232	memset(&dkeys[1], 0xFF, sizeof(struct xfs_fsmap));
1233
1234	info.next_daddr = head->fmh_keys[0].fmr_physical +
1235			  head->fmh_keys[0].fmr_length;
1236
1237	/* For each device we support... */
1238	for (i = 0; i < XFS_GETFSMAP_DEVS; i++) {
1239		/* Is this device within the range the user asked for? */
1240		if (!handlers[i].fn)
1241			continue;
1242		if (head->fmh_keys[0].fmr_device > handlers[i].dev)
1243			continue;
1244		if (head->fmh_keys[1].fmr_device < handlers[i].dev)
1245			break;
1246
1247		/*
1248		 * If this device number matches the high key, we have to pass
1249		 * the high key to the handler to limit the query results, and
1250		 * set the end_daddr so that we can synthesize records at the
1251		 * end of the query range or device.
1252		 */
1253		if (handlers[i].dev == head->fmh_keys[1].fmr_device) {
1254			dkeys[1] = head->fmh_keys[1];
1255			info.end_daddr = min(handlers[i].nr_sectors - 1,
1256					     dkeys[1].fmr_physical);
1257		} else {
1258			info.end_daddr = handlers[i].nr_sectors - 1;
1259		}
1260
1261		/*
1262		 * If the device number exceeds the low key, zero out the low
1263		 * key so that we get everything from the beginning.
1264		 */
1265		if (handlers[i].dev > head->fmh_keys[0].fmr_device)
1266			memset(&dkeys[0], 0, sizeof(struct xfs_fsmap));
1267
1268		/*
1269		 * Grab an empty transaction so that we can use its recursive
1270		 * buffer locking abilities to detect cycles in the rmapbt
1271		 * without deadlocking.
1272		 */
1273		tp = xfs_trans_alloc_empty(mp);
1274
1275		info.dev = handlers[i].dev;
1276		info.last = false;
1277		info.group = NULL;
1278		info.low_daddr = XFS_BUF_DADDR_NULL;
1279		info.low.rm_blockcount = 0;
1280		error = handlers[i].fn(tp, dkeys, &info);
1281		if (error)
1282			break;
1283		xfs_trans_cancel(tp);
1284		tp = NULL;
1285		info.next_daddr = 0;
1286	}
1287
1288	if (tp)
1289		xfs_trans_cancel(tp);
1290
1291	/*
1292	 * For internal RT device we need to report different synthetic devices
1293	 * for a single physical device, and thus can't report the actual dev_t.
1294	 */
1295	if (!mp->m_sb.sb_rtstart)
1296		head->fmh_oflags = FMH_OF_DEV_T;
1297	return error;
1298}
1299
1300int
1301xfs_ioc_getfsmap(
1302	struct xfs_inode	*ip,
1303	struct fsmap_head	__user *arg)
1304{
1305	struct xfs_fsmap_head	xhead = {0};
1306	struct fsmap_head	head;
1307	struct fsmap		*recs;
1308	unsigned int		count;
1309	__u32			last_flags = 0;
1310	bool			done = false;
1311	int			error;
1312
1313	if (copy_from_user(&head, arg, sizeof(struct fsmap_head)))
1314		return -EFAULT;
1315	if (memchr_inv(head.fmh_reserved, 0, sizeof(head.fmh_reserved)) ||
1316	    memchr_inv(head.fmh_keys[0].fmr_reserved, 0,
1317		       sizeof(head.fmh_keys[0].fmr_reserved)) ||
1318	    memchr_inv(head.fmh_keys[1].fmr_reserved, 0,
1319		       sizeof(head.fmh_keys[1].fmr_reserved)))
1320		return -EINVAL;
1321
1322	/*
1323	 * Use an internal memory buffer so that we don't have to copy fsmap
1324	 * data to userspace while holding locks.  Start by trying to allocate
1325	 * up to 128k for the buffer, but fall back to a single page if needed.
1326	 */
1327	count = min_t(unsigned int, head.fmh_count,
1328			131072 / sizeof(struct fsmap));
1329	recs = kvcalloc(count, sizeof(struct fsmap), GFP_KERNEL);
1330	if (!recs) {
1331		count = min_t(unsigned int, head.fmh_count,
1332				PAGE_SIZE / sizeof(struct fsmap));
1333		recs = kvcalloc(count, sizeof(struct fsmap), GFP_KERNEL);
1334		if (!recs)
1335			return -ENOMEM;
1336	}
1337
1338	xhead.fmh_iflags = head.fmh_iflags;
1339	xfs_fsmap_to_internal(&xhead.fmh_keys[0], &head.fmh_keys[0]);
1340	xfs_fsmap_to_internal(&xhead.fmh_keys[1], &head.fmh_keys[1]);
1341
1342	trace_xfs_getfsmap_low_key(ip->i_mount, &xhead.fmh_keys[0]);
1343	trace_xfs_getfsmap_high_key(ip->i_mount, &xhead.fmh_keys[1]);
1344
1345	head.fmh_entries = 0;
1346	do {
1347		struct fsmap __user	*user_recs;
1348		struct fsmap		*last_rec;
1349
1350		user_recs = &arg->fmh_recs[head.fmh_entries];
1351		xhead.fmh_entries = 0;
1352		xhead.fmh_count = min_t(unsigned int, count,
1353					head.fmh_count - head.fmh_entries);
1354
1355		/* Run query, record how many entries we got. */
1356		error = xfs_getfsmap(ip->i_mount, &xhead, recs);
1357		switch (error) {
1358		case 0:
1359			/*
1360			 * There are no more records in the result set.  Copy
1361			 * whatever we got to userspace and break out.
1362			 */
1363			done = true;
1364			break;
1365		case -ECANCELED:
1366			/*
1367			 * The internal memory buffer is full.  Copy whatever
1368			 * records we got to userspace and go again if we have
1369			 * not yet filled the userspace buffer.
1370			 */
1371			error = 0;
1372			break;
1373		default:
1374			goto out_free;
1375		}
1376		head.fmh_entries += xhead.fmh_entries;
1377		head.fmh_oflags = xhead.fmh_oflags;
1378
1379		/*
1380		 * If the caller wanted a record count or there aren't any
1381		 * new records to return, we're done.
1382		 */
1383		if (head.fmh_count == 0 || xhead.fmh_entries == 0)
1384			break;
1385
1386		/* Copy all the records we got out to userspace. */
1387		if (copy_to_user(user_recs, recs,
1388				 xhead.fmh_entries * sizeof(struct fsmap))) {
1389			error = -EFAULT;
1390			goto out_free;
1391		}
1392
1393		/* Remember the last record flags we copied to userspace. */
1394		last_rec = &recs[xhead.fmh_entries - 1];
1395		last_flags = last_rec->fmr_flags;
1396
1397		/* Set up the low key for the next iteration. */
1398		xfs_fsmap_to_internal(&xhead.fmh_keys[0], last_rec);
1399		trace_xfs_getfsmap_low_key(ip->i_mount, &xhead.fmh_keys[0]);
1400	} while (!done && head.fmh_entries < head.fmh_count);
1401
1402	/*
1403	 * If there are no more records in the query result set and we're not
1404	 * in counting mode, mark the last record returned with the LAST flag.
1405	 */
1406	if (done && head.fmh_count > 0 && head.fmh_entries > 0) {
1407		struct fsmap __user	*user_rec;
1408
1409		last_flags |= FMR_OF_LAST;
1410		user_rec = &arg->fmh_recs[head.fmh_entries - 1];
1411
1412		if (copy_to_user(&user_rec->fmr_flags, &last_flags,
1413					sizeof(last_flags))) {
1414			error = -EFAULT;
1415			goto out_free;
1416		}
1417	}
1418
1419	/* copy back header */
1420	if (copy_to_user(arg, &head, sizeof(struct fsmap_head))) {
1421		error = -EFAULT;
1422		goto out_free;
1423	}
1424
1425out_free:
1426	kvfree(recs);
1427	return error;
1428}