xfs_rtalloc.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
   4 * All Rights Reserved.
   5 */
   6#include "xfs.h"
   7#include "xfs_fs.h"
   8#include "xfs_shared.h"
   9#include "xfs_format.h"
  10#include "xfs_log_format.h"
  11#include "xfs_trans_resv.h"
  12#include "xfs_bit.h"
  13#include "xfs_mount.h"
  14#include "xfs_inode.h"
  15#include "xfs_alloc.h"
  16#include "xfs_bmap.h"
  17#include "xfs_bmap_btree.h"
  18#include "xfs_bmap_util.h"
  19#include "xfs_trans.h"
  20#include "xfs_trans_space.h"
  21#include "xfs_icache.h"
  22#include "xfs_rtalloc.h"
  23#include "xfs_sb.h"
  24#include "xfs_rtbitmap.h"
  25#include "xfs_rtrmap_btree.h"
  26#include "xfs_quota.h"
  27#include "xfs_log_priv.h"
  28#include "xfs_health.h"
  29#include "xfs_da_format.h"
  30#include "xfs_metafile.h"
  31#include "xfs_rtgroup.h"
  32#include "xfs_error.h"
  33#include "xfs_trace.h"
  34#include "xfs_rtrefcount_btree.h"
  35#include "xfs_reflink.h"
  36#include "xfs_zone_alloc.h"
  37
  38/*
  39 * Return whether there are any free extents in the size range given
  40 * by low and high, for the bitmap block bbno.
  41 */
  42STATIC int
  43xfs_rtany_summary(
  44	struct xfs_rtalloc_args	*args,
  45	int			low,	/* low log2 extent size */
  46	int			high,	/* high log2 extent size */
  47	xfs_fileoff_t		bbno,	/* bitmap block number */
  48	int			*maxlog) /* out: max log2 extent size free */
  49{
  50	uint8_t			*rsum_cache = args->rtg->rtg_rsum_cache;
  51	int			error;
  52	int			log;	/* loop counter, log2 of ext. size */
  53	xfs_suminfo_t		sum;	/* summary data */
  54
  55	/* There are no extents at levels >= rsum_cache[bbno]. */
  56	if (rsum_cache) {
  57		high = min(high, rsum_cache[bbno] - 1);
  58		if (low > high) {
  59			*maxlog = -1;
  60			return 0;
  61		}
  62	}
  63
  64	/*
  65	 * Loop over logs of extent sizes.
  66	 */
  67	for (log = high; log >= low; log--) {
  68		/*
  69		 * Get one summary datum.
  70		 */
  71		error = xfs_rtget_summary(args, log, bbno, &sum);
  72		if (error) {
  73			return error;
  74		}
  75		/*
  76		 * If there are any, return success.
  77		 */
  78		if (sum) {
  79			*maxlog = log;
  80			goto out;
  81		}
  82	}
  83	/*
  84	 * Found nothing, return failure.
  85	 */
  86	*maxlog = -1;
  87out:
  88	/* There were no extents at levels > log. */
  89	if (rsum_cache && log + 1 < rsum_cache[bbno])
  90		rsum_cache[bbno] = log + 1;
  91	return 0;
  92}
  93
  94/*
  95 * Copy and transform the summary file, given the old and new
  96 * parameters in the mount structures.
  97 */
  98STATIC int
  99xfs_rtcopy_summary(
 100	struct xfs_rtalloc_args	*oargs,
 101	struct xfs_rtalloc_args	*nargs)
 102{
 103	xfs_fileoff_t		bbno;	/* bitmap block number */
 104	int			error;
 105	int			log;	/* summary level number (log length) */
 106	xfs_suminfo_t		sum;	/* summary data */
 107
 108	for (log = oargs->mp->m_rsumlevels - 1; log >= 0; log--) {
 109		for (bbno = oargs->mp->m_sb.sb_rbmblocks - 1;
 110		     (xfs_srtblock_t)bbno >= 0;
 111		     bbno--) {
 112			error = xfs_rtget_summary(oargs, log, bbno, &sum);
 113			if (error)
 114				goto out;
 115			if (sum == 0)
 116				continue;
 117			error = xfs_rtmodify_summary(oargs, log, bbno, -sum);
 118			if (error)
 119				goto out;
 120			error = xfs_rtmodify_summary(nargs, log, bbno, sum);
 121			if (error)
 122				goto out;
 123			ASSERT(sum > 0);
 124		}
 125	}
 126	error = 0;
 127out:
 128	xfs_rtbuf_cache_relse(oargs);
 129	return error;
 130}
 131/*
 132 * Mark an extent specified by start and len allocated.
 133 * Updates all the summary information as well as the bitmap.
 134 */
 135STATIC int
 136xfs_rtallocate_range(
 137	struct xfs_rtalloc_args	*args,
 138	xfs_rtxnum_t		start,	/* start rtext to allocate */
 139	xfs_rtxlen_t		len)	/* in/out: summary block number */
 140{
 141	struct xfs_mount	*mp = args->mp;
 142	xfs_rtxnum_t		end;	/* end of the allocated rtext */
 143	int			error;
 144	xfs_rtxnum_t		postblock = 0; /* first rtext allocated > end */
 145	xfs_rtxnum_t		preblock = 0; /* first rtext allocated < start */
 146
 147	end = start + len - 1;
 148	/*
 149	 * Assume we're allocating out of the middle of a free extent.
 150	 * We need to find the beginning and end of the extent so we can
 151	 * properly update the summary.
 152	 */
 153	error = xfs_rtfind_back(args, start, &preblock);
 154	if (error)
 155		return error;
 156
 157	/*
 158	 * Find the next allocated block (end of free extent).
 159	 */
 160	error = xfs_rtfind_forw(args, end, args->rtg->rtg_extents - 1,
 161			&postblock);
 162	if (error)
 163		return error;
 164
 165	/*
 166	 * Decrement the summary information corresponding to the entire
 167	 * (old) free extent.
 168	 */
 169	error = xfs_rtmodify_summary(args,
 170			xfs_highbit64(postblock + 1 - preblock),
 171			xfs_rtx_to_rbmblock(mp, preblock), -1);
 172	if (error)
 173		return error;
 174
 175	/*
 176	 * If there are blocks not being allocated at the front of the
 177	 * old extent, add summary data for them to be free.
 178	 */
 179	if (preblock < start) {
 180		error = xfs_rtmodify_summary(args,
 181				xfs_highbit64(start - preblock),
 182				xfs_rtx_to_rbmblock(mp, preblock), 1);
 183		if (error)
 184			return error;
 185	}
 186
 187	/*
 188	 * If there are blocks not being allocated at the end of the
 189	 * old extent, add summary data for them to be free.
 190	 */
 191	if (postblock > end) {
 192		error = xfs_rtmodify_summary(args,
 193				xfs_highbit64(postblock - end),
 194				xfs_rtx_to_rbmblock(mp, end + 1), 1);
 195		if (error)
 196			return error;
 197	}
 198
 199	/*
 200	 * Modify the bitmap to mark this extent allocated.
 201	 */
 202	return xfs_rtmodify_range(args, start, len, 0);
 203}
 204
 205/* Reduce @rtxlen until it is a multiple of @prod. */
 206static inline xfs_rtxlen_t
 207xfs_rtalloc_align_len(
 208	xfs_rtxlen_t	rtxlen,
 209	xfs_rtxlen_t	prod)
 210{
 211	if (unlikely(prod > 1))
 212		return rounddown(rtxlen, prod);
 213	return rtxlen;
 214}
 215
 216/*
 217 * Make sure we don't run off the end of the rt volume.  Be careful that
 218 * adjusting maxlen downwards doesn't cause us to fail the alignment checks.
 219 */
 220static inline xfs_rtxlen_t
 221xfs_rtallocate_clamp_len(
 222	struct xfs_rtgroup	*rtg,
 223	xfs_rtxnum_t		startrtx,
 224	xfs_rtxlen_t		rtxlen,
 225	xfs_rtxlen_t		prod)
 226{
 227	xfs_rtxlen_t		ret;
 228
 229	ret = min(rtg->rtg_extents, startrtx + rtxlen) - startrtx;
 230	return xfs_rtalloc_align_len(ret, prod);
 231}
 232
 233/*
 234 * Attempt to allocate an extent minlen<=len<=maxlen starting from
 235 * bitmap block bbno.  If we don't get maxlen then use prod to trim
 236 * the length, if given.  Returns error; returns starting block in *rtx.
 237 * The lengths are all in rtextents.
 238 */
 239STATIC int
 240xfs_rtallocate_extent_block(
 241	struct xfs_rtalloc_args	*args,
 242	xfs_fileoff_t		bbno,	/* bitmap block number */
 243	xfs_rtxlen_t		minlen,	/* minimum length to allocate */
 244	xfs_rtxlen_t		maxlen,	/* maximum length to allocate */
 245	xfs_rtxlen_t		*len,	/* out: actual length allocated */
 246	xfs_rtxnum_t		*nextp,	/* out: next rtext to try */
 247	xfs_rtxlen_t		prod,	/* extent product factor */
 248	xfs_rtxnum_t		*rtx)	/* out: start rtext allocated */
 249{
 250	struct xfs_mount	*mp = args->mp;
 251	xfs_rtxnum_t		besti = -1; /* best rtext found so far */
 252	xfs_rtxnum_t		end;	/* last rtext in chunk */
 253	xfs_rtxnum_t		i;	/* current rtext trying */
 254	xfs_rtxnum_t		next;	/* next rtext to try */
 255	xfs_rtxlen_t		scanlen; /* number of free rtx to look for */
 256	xfs_rtxlen_t		bestlen = 0; /* best length found so far */
 257	int			stat;	/* status from internal calls */
 258	int			error;
 259
 260	/*
 261	 * Loop over all the extents starting in this bitmap block up to the
 262	 * end of the rt volume, looking for one that's long enough.
 263	 */
 264	end = min(args->rtg->rtg_extents, xfs_rbmblock_to_rtx(mp, bbno + 1)) -
 265		1;
 266	for (i = xfs_rbmblock_to_rtx(mp, bbno); i <= end; i++) {
 267		/* Make sure we don't scan off the end of the rt volume. */
 268		scanlen = xfs_rtallocate_clamp_len(args->rtg, i, maxlen, prod);
 269		if (scanlen < minlen)
 270			break;
 271
 272		/*
 273		 * See if there's a free extent of scanlen starting at i.
 274		 * If it's not so then next will contain the first non-free.
 275		 */
 276		error = xfs_rtcheck_range(args, i, scanlen, 1, &next, &stat);
 277		if (error)
 278			return error;
 279		if (stat) {
 280			/*
 281			 * i to scanlen is all free, allocate and return that.
 282			 */
 283			*len = scanlen;
 284			*rtx = i;
 285			return 0;
 286		}
 287
 288		/*
 289		 * In the case where we have a variable-sized allocation
 290		 * request, figure out how big this free piece is,
 291		 * and if it's big enough for the minimum, and the best
 292		 * so far, remember it.
 293		 */
 294		if (minlen < maxlen) {
 295			xfs_rtxnum_t	thislen;	/* this extent size */
 296
 297			thislen = next - i;
 298			if (thislen >= minlen && thislen > bestlen) {
 299				besti = i;
 300				bestlen = thislen;
 301			}
 302		}
 303		/*
 304		 * If not done yet, find the start of the next free space.
 305		 */
 306		if (next >= end)
 307			break;
 308		error = xfs_rtfind_forw(args, next, end, &i);
 309		if (error)
 310			return error;
 311	}
 312
 313	/* Searched the whole thing & didn't find a maxlen free extent. */
 314	if (besti == -1)
 315		goto nospace;
 316
 317	/*
 318	 * Ensure bestlen is a multiple of prod, but don't return a too-short
 319	 * extent.
 320	 */
 321	bestlen = xfs_rtalloc_align_len(bestlen, prod);
 322	if (bestlen < minlen)
 323		goto nospace;
 324
 325	/*
 326	 * Pick besti for bestlen & return that.
 327	 */
 328	*len = bestlen;
 329	*rtx = besti;
 330	return 0;
 331nospace:
 332	/* Allocation failed.  Set *nextp to the next block to try. */
 333	*nextp = next;
 334	return -ENOSPC;
 335}
 336
 337/*
 338 * Allocate an extent of length minlen<=len<=maxlen, starting at block
 339 * bno.  If we don't get maxlen then use prod to trim the length, if given.
 340 * Returns error; returns starting block in *rtx.
 341 * The lengths are all in rtextents.
 342 */
 343STATIC int
 344xfs_rtallocate_extent_exact(
 345	struct xfs_rtalloc_args	*args,
 346	xfs_rtxnum_t		start,	/* starting rtext number to allocate */
 347	xfs_rtxlen_t		minlen,	/* minimum length to allocate */
 348	xfs_rtxlen_t		maxlen,	/* maximum length to allocate */
 349	xfs_rtxlen_t		*len,	/* out: actual length allocated */
 350	xfs_rtxlen_t		prod,	/* extent product factor */
 351	xfs_rtxnum_t		*rtx)	/* out: start rtext allocated */
 352{
 353	xfs_rtxnum_t		next;	/* next rtext to try (dummy) */
 354	xfs_rtxlen_t		alloclen; /* candidate length */
 355	xfs_rtxlen_t		scanlen; /* number of free rtx to look for */
 356	int			isfree;	/* extent is free */
 357	int			error;
 358
 359	ASSERT(minlen % prod == 0);
 360	ASSERT(maxlen % prod == 0);
 361
 362	/* Make sure we don't run off the end of the rt volume. */
 363	scanlen = xfs_rtallocate_clamp_len(args->rtg, start, maxlen, prod);
 364	if (scanlen < minlen)
 365		return -ENOSPC;
 366
 367	/* Check if the range in question (for scanlen) is free. */
 368	error = xfs_rtcheck_range(args, start, scanlen, 1, &next, &isfree);
 369	if (error)
 370		return error;
 371
 372	if (isfree) {
 373		/* start to scanlen is all free; allocate it. */
 374		*len = scanlen;
 375		*rtx = start;
 376		return 0;
 377	}
 378
 379	/*
 380	 * If not, allocate what there is, if it's at least minlen.
 381	 */
 382	alloclen = next - start;
 383	if (alloclen < minlen)
 384		return -ENOSPC;
 385
 386	/* Ensure alloclen is a multiple of prod. */
 387	alloclen = xfs_rtalloc_align_len(alloclen, prod);
 388	if (alloclen < minlen)
 389		return -ENOSPC;
 390
 391	*len = alloclen;
 392	*rtx = start;
 393	return 0;
 394}
 395
 396/*
 397 * Allocate an extent of length minlen<=len<=maxlen, starting as near
 398 * to start as possible.  If we don't get maxlen then use prod to trim
 399 * the length, if given.  The lengths are all in rtextents.
 400 */
 401STATIC int
 402xfs_rtallocate_extent_near(
 403	struct xfs_rtalloc_args	*args,
 404	xfs_rtxnum_t		start,	/* starting rtext number to allocate */
 405	xfs_rtxlen_t		minlen,	/* minimum length to allocate */
 406	xfs_rtxlen_t		maxlen,	/* maximum length to allocate */
 407	xfs_rtxlen_t		*len,	/* out: actual length allocated */
 408	xfs_rtxlen_t		prod,	/* extent product factor */
 409	xfs_rtxnum_t		*rtx)	/* out: start rtext allocated */
 410{
 411	struct xfs_mount	*mp = args->mp;
 412	int			maxlog;	/* max useful extent from summary */
 413	xfs_fileoff_t		bbno;	/* bitmap block number */
 414	int			error;
 415	int			i;	/* bitmap block offset (loop control) */
 416	int			j;	/* secondary loop control */
 417	int			log2len; /* log2 of minlen */
 418	xfs_rtxnum_t		n;	/* next rtext to try */
 419
 420	ASSERT(minlen % prod == 0);
 421	ASSERT(maxlen % prod == 0);
 422
 423	/*
 424	 * If the block number given is off the end, silently set it to the last
 425	 * block.
 426	 */
 427	start = min(start, args->rtg->rtg_extents - 1);
 428
 429	/*
 430	 * Try the exact allocation first.
 431	 */
 432	error = xfs_rtallocate_extent_exact(args, start, minlen, maxlen, len,
 433			prod, rtx);
 434	if (error != -ENOSPC)
 435		return error;
 436
 437	bbno = xfs_rtx_to_rbmblock(mp, start);
 438	i = 0;
 439	j = -1;
 440	ASSERT(minlen != 0);
 441	log2len = xfs_highbit32(minlen);
 442	/*
 443	 * Loop over all bitmap blocks (bbno + i is current block).
 444	 */
 445	for (;;) {
 446		/*
 447		 * Get summary information of extents of all useful levels
 448		 * starting in this bitmap block.
 449		 */
 450		error = xfs_rtany_summary(args, log2len, mp->m_rsumlevels - 1,
 451				bbno + i, &maxlog);
 452		if (error)
 453			return error;
 454
 455		/*
 456		 * If there are any useful extents starting here, try
 457		 * allocating one.
 458		 */
 459		if (maxlog >= 0) {
 460			xfs_extlen_t maxavail =
 461				min_t(xfs_rtblock_t, maxlen,
 462				      (1ULL << (maxlog + 1)) - 1);
 463			/*
 464			 * On the positive side of the starting location.
 465			 */
 466			if (i >= 0) {
 467				/*
 468				 * Try to allocate an extent starting in
 469				 * this block.
 470				 */
 471				error = xfs_rtallocate_extent_block(args,
 472						bbno + i, minlen, maxavail, len,
 473						&n, prod, rtx);
 474				if (error != -ENOSPC)
 475					return error;
 476			}
 477			/*
 478			 * On the negative side of the starting location.
 479			 */
 480			else {		/* i < 0 */
 481				int	maxblocks;
 482
 483				/*
 484				 * Loop backwards to find the end of the extent
 485				 * we found in the realtime summary.
 486				 *
 487				 * maxblocks is the maximum possible number of
 488				 * bitmap blocks from the start of the extent
 489				 * to the end of the extent.
 490				 */
 491				if (maxlog == 0)
 492					maxblocks = 0;
 493				else if (maxlog < mp->m_blkbit_log)
 494					maxblocks = 1;
 495				else
 496					maxblocks = 2 << (maxlog - mp->m_blkbit_log);
 497
 498				/*
 499				 * We need to check bbno + i + maxblocks down to
 500				 * bbno + i. We already checked bbno down to
 501				 * bbno + j + 1, so we don't need to check those
 502				 * again.
 503				 */
 504				j = min(i + maxblocks, j);
 505				for (; j >= i; j--) {
 506					error = xfs_rtallocate_extent_block(args,
 507							bbno + j, minlen,
 508							maxavail, len, &n, prod,
 509							rtx);
 510					if (error != -ENOSPC)
 511						return error;
 512				}
 513			}
 514		}
 515		/*
 516		 * Loop control.  If we were on the positive side, and there's
 517		 * still more blocks on the negative side, go there.
 518		 */
 519		if (i > 0 && (int)bbno - i >= 0)
 520			i = -i;
 521		/*
 522		 * If positive, and no more negative, but there are more
 523		 * positive, go there.
 524		 */
 525		else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)
 526			i++;
 527		/*
 528		 * If negative or 0 (just started), and there are positive
 529		 * blocks to go, go there.  The 0 case moves to block 1.
 530		 */
 531		else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)
 532			i = 1 - i;
 533		/*
 534		 * If negative or 0 and there are more negative blocks,
 535		 * go there.
 536		 */
 537		else if (i <= 0 && (int)bbno + i > 0)
 538			i--;
 539		/*
 540		 * Must be done.  Return failure.
 541		 */
 542		else
 543			break;
 544	}
 545	return -ENOSPC;
 546}
 547
 548static int
 549xfs_rtalloc_sumlevel(
 550	struct xfs_rtalloc_args	*args,
 551	int			l,	/* level number */
 552	xfs_rtxlen_t		minlen,	/* minimum length to allocate */
 553	xfs_rtxlen_t		maxlen,	/* maximum length to allocate */
 554	xfs_rtxlen_t		prod,	/* extent product factor */
 555	xfs_rtxlen_t		*len,	/* out: actual length allocated */
 556	xfs_rtxnum_t		*rtx)	/* out: start rtext allocated */
 557{
 558	xfs_fileoff_t		i;	/* bitmap block number */
 559	int			error;
 560
 561	for (i = 0; i < args->mp->m_sb.sb_rbmblocks; i++) {
 562		xfs_suminfo_t	sum;	/* summary information for extents */
 563		xfs_rtxnum_t	n;	/* next rtext to be tried */
 564
 565		error = xfs_rtget_summary(args, l, i, &sum);
 566		if (error)
 567			return error;
 568
 569		/*
 570		 * Nothing there, on to the next block.
 571		 */
 572		if (!sum)
 573			continue;
 574
 575		/*
 576		 * Try allocating the extent.
 577		 */
 578		error = xfs_rtallocate_extent_block(args, i, minlen, maxlen,
 579				len, &n, prod, rtx);
 580		if (error != -ENOSPC)
 581			return error;
 582
 583		/*
 584		 * If the "next block to try" returned from the allocator is
 585		 * beyond the next bitmap block, skip to that bitmap block.
 586		 */
 587		if (xfs_rtx_to_rbmblock(args->mp, n) > i + 1)
 588			i = xfs_rtx_to_rbmblock(args->mp, n) - 1;
 589	}
 590
 591	return -ENOSPC;
 592}
 593
 594/*
 595 * Allocate an extent of length minlen<=len<=maxlen, with no position
 596 * specified.  If we don't get maxlen then use prod to trim
 597 * the length, if given.  The lengths are all in rtextents.
 598 */
 599static int
 600xfs_rtallocate_extent_size(
 601	struct xfs_rtalloc_args	*args,
 602	xfs_rtxlen_t		minlen,	/* minimum length to allocate */
 603	xfs_rtxlen_t		maxlen,	/* maximum length to allocate */
 604	xfs_rtxlen_t		*len,	/* out: actual length allocated */
 605	xfs_rtxlen_t		prod,	/* extent product factor */
 606	xfs_rtxnum_t		*rtx)	/* out: start rtext allocated */
 607{
 608	int			error;
 609	int			l;	/* level number (loop control) */
 610
 611	ASSERT(minlen % prod == 0);
 612	ASSERT(maxlen % prod == 0);
 613	ASSERT(maxlen != 0);
 614
 615	/*
 616	 * Loop over all the levels starting with maxlen.
 617	 *
 618	 * At each level, look at all the bitmap blocks, to see if there are
 619	 * extents starting there that are long enough (>= maxlen).
 620	 *
 621	 * Note, only on the initial level can the allocation fail if the
 622	 * summary says there's an extent.
 623	 */
 624	for (l = xfs_highbit32(maxlen); l < args->mp->m_rsumlevels; l++) {
 625		error = xfs_rtalloc_sumlevel(args, l, minlen, maxlen, prod, len,
 626				rtx);
 627		if (error != -ENOSPC)
 628			return error;
 629	}
 630
 631	/*
 632	 * Didn't find any maxlen blocks.  Try smaller ones, unless we are
 633	 * looking for a fixed size extent.
 634	 */
 635	if (minlen > --maxlen)
 636		return -ENOSPC;
 637	ASSERT(minlen != 0);
 638	ASSERT(maxlen != 0);
 639
 640	/*
 641	 * Loop over sizes, from maxlen down to minlen.
 642	 *
 643	 * This time, when we do the allocations, allow smaller ones to succeed,
 644	 * but make sure the specified minlen/maxlen are in the possible range
 645	 * for this summary level.
 646	 */
 647	for (l = xfs_highbit32(maxlen); l >= xfs_highbit32(minlen); l--) {
 648		error = xfs_rtalloc_sumlevel(args, l,
 649				max_t(xfs_rtxlen_t, minlen, 1 << l),
 650				min_t(xfs_rtxlen_t, maxlen, (1 << (l + 1)) - 1),
 651				prod, len, rtx);
 652		if (error != -ENOSPC)
 653			return error;
 654	}
 655
 656	return -ENOSPC;
 657}
 658
 659static void
 660xfs_rtunmount_rtg(
 661	struct xfs_rtgroup	*rtg)
 662{
 663	int			i;
 664
 665	for (i = 0; i < XFS_RTGI_MAX; i++)
 666		xfs_rtginode_irele(&rtg->rtg_inodes[i]);
 667	if (!xfs_has_zoned(rtg_mount(rtg)))
 668		kvfree(rtg->rtg_rsum_cache);
 669}
 670
 671static int
 672xfs_alloc_rsum_cache(
 673	struct xfs_rtgroup	*rtg,
 674	xfs_extlen_t		rbmblocks)
 675{
 676	/*
 677	 * The rsum cache is initialized to the maximum value, which is
 678	 * trivially an upper bound on the maximum level with any free extents.
 679	 */
 680	rtg->rtg_rsum_cache = kvmalloc(rbmblocks, GFP_KERNEL);
 681	if (!rtg->rtg_rsum_cache)
 682		return -ENOMEM;
 683	memset(rtg->rtg_rsum_cache, -1, rbmblocks);
 684	return 0;
 685}
 686
 687/*
 688 * If we changed the rt extent size (meaning there was no rt volume previously)
 689 * and the root directory had EXTSZINHERIT and RTINHERIT set, it's possible
 690 * that the extent size hint on the root directory is no longer congruent with
 691 * the new rt extent size.  Log the rootdir inode to fix this.
 692 */
 693static int
 694xfs_growfs_rt_fixup_extsize(
 695	struct xfs_mount	*mp)
 696{
 697	struct xfs_inode	*ip = mp->m_rootip;
 698	struct xfs_trans	*tp;
 699	int			error = 0;
 700
 701	xfs_ilock(ip, XFS_IOLOCK_EXCL);
 702	if (!(ip->i_diflags & XFS_DIFLAG_RTINHERIT) ||
 703	    !(ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT))
 704		goto out_iolock;
 705
 706	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_ichange, 0, 0, false,
 707			&tp);
 708	if (error)
 709		goto out_iolock;
 710
 711	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 712	error = xfs_trans_commit(tp);
 713	xfs_iunlock(ip, XFS_ILOCK_EXCL);
 714
 715out_iolock:
 716	xfs_iunlock(ip, XFS_IOLOCK_EXCL);
 717	return error;
 718}
 719
 720/* Ensure that the rtgroup metadata inode is loaded, creating it if neeeded. */
 721static int
 722xfs_rtginode_ensure(
 723	struct xfs_rtgroup	*rtg,
 724	enum xfs_rtg_inodes	type)
 725{
 726	struct xfs_trans	*tp;
 727	int			error;
 728
 729	if (rtg->rtg_inodes[type])
 730		return 0;
 731
 732	tp = xfs_trans_alloc_empty(rtg_mount(rtg));
 733	error = xfs_rtginode_load(rtg, type, tp);
 734	xfs_trans_cancel(tp);
 735
 736	if (error != -ENOENT)
 737		return 0;
 738	return xfs_rtginode_create(rtg, type, true);
 739}
 740
 741static struct xfs_mount *
 742xfs_growfs_rt_alloc_fake_mount(
 743	const struct xfs_mount	*mp,
 744	xfs_rfsblock_t		rblocks,
 745	xfs_agblock_t		rextsize)
 746{
 747	struct xfs_mount	*nmp;
 748
 749	nmp = kmemdup(mp, sizeof(*mp), GFP_KERNEL);
 750	if (!nmp)
 751		return NULL;
 752	xfs_mount_sb_set_rextsize(nmp, &nmp->m_sb, rextsize);
 753	nmp->m_sb.sb_rblocks = rblocks;
 754	nmp->m_sb.sb_rextents = xfs_blen_to_rtbxlen(nmp, nmp->m_sb.sb_rblocks);
 755	nmp->m_sb.sb_rbmblocks = xfs_rtbitmap_blockcount(nmp);
 756	nmp->m_sb.sb_rextslog = xfs_compute_rextslog(nmp->m_sb.sb_rextents);
 757	if (xfs_has_rtgroups(nmp))
 758		nmp->m_sb.sb_rgcount = howmany_64(nmp->m_sb.sb_rextents,
 759						  nmp->m_sb.sb_rgextents);
 760	else
 761		nmp->m_sb.sb_rgcount = 1;
 762	nmp->m_rsumblocks = xfs_rtsummary_blockcount(nmp, &nmp->m_rsumlevels);
 763
 764	if (rblocks > 0)
 765		nmp->m_features |= XFS_FEAT_REALTIME;
 766
 767	/* recompute growfsrt reservation from new rsumsize */
 768	xfs_trans_resv_calc(nmp, &nmp->m_resv);
 769	return nmp;
 770}
 771
 772/* Free all the new space and return the number of extents actually freed. */
 773static int
 774xfs_growfs_rt_free_new(
 775	struct xfs_rtgroup	*rtg,
 776	struct xfs_rtalloc_args	*nargs,
 777	xfs_rtbxlen_t		*freed_rtx)
 778{
 779	struct xfs_mount	*mp = rtg_mount(rtg);
 780	xfs_rgnumber_t		rgno = rtg_rgno(rtg);
 781	xfs_rtxnum_t		start_rtx = 0, end_rtx;
 782
 783	if (rgno < mp->m_sb.sb_rgcount)
 784		start_rtx = xfs_rtgroup_extents(mp, rgno);
 785	end_rtx = xfs_rtgroup_extents(nargs->mp, rgno);
 786
 787	/*
 788	 * Compute the first new extent that we want to free, being careful to
 789	 * skip past a realtime superblock at the start of the realtime volume.
 790	 */
 791	if (xfs_has_rtsb(nargs->mp) && rgno == 0 && start_rtx == 0)
 792		start_rtx++;
 793	*freed_rtx = end_rtx - start_rtx;
 794	return xfs_rtfree_range(nargs, start_rtx, *freed_rtx);
 795}
 796
 797static xfs_rfsblock_t
 798xfs_growfs_rt_nrblocks(
 799	struct xfs_rtgroup	*rtg,
 800	xfs_rfsblock_t		nrblocks,
 801	xfs_agblock_t		rextsize,
 802	xfs_fileoff_t		bmbno)
 803{
 804	struct xfs_mount	*mp = rtg_mount(rtg);
 805	xfs_rfsblock_t		step;
 806
 807	step = (bmbno + 1) * mp->m_rtx_per_rbmblock * rextsize;
 808	if (xfs_has_rtgroups(mp)) {
 809		xfs_rfsblock_t	rgblocks = mp->m_sb.sb_rgextents * rextsize;
 810
 811		step = min(rgblocks, step) + rgblocks * rtg_rgno(rtg);
 812	}
 813
 814	return min(nrblocks, step);
 815}
 816
 817/*
 818 * If the post-grow filesystem will have an rtsb; we're initializing the first
 819 * rtgroup; and the filesystem didn't have a realtime section, write the rtsb
 820 * now, and attach the rtsb buffer to the real mount.
 821 */
 822static int
 823xfs_growfs_rt_init_rtsb(
 824	const struct xfs_rtalloc_args	*nargs,
 825	const struct xfs_rtgroup	*rtg,
 826	const struct xfs_rtalloc_args	*args)
 827{
 828	struct xfs_mount		*mp = args->mp;
 829	struct xfs_buf			*rtsb_bp;
 830	int				error;
 831
 832	if (!xfs_has_rtsb(nargs->mp))
 833		return 0;
 834	if (rtg_rgno(rtg) > 0)
 835		return 0;
 836	if (mp->m_sb.sb_rblocks)
 837		return 0;
 838
 839	error = xfs_buf_get_uncached(mp->m_rtdev_targp, XFS_FSB_TO_BB(mp, 1),
 840			&rtsb_bp);
 841	if (error)
 842		return error;
 843
 844	rtsb_bp->b_maps[0].bm_bn = XFS_RTSB_DADDR;
 845	rtsb_bp->b_ops = &xfs_rtsb_buf_ops;
 846
 847	xfs_update_rtsb(rtsb_bp, mp->m_sb_bp);
 848	mp->m_rtsb_bp = rtsb_bp;
 849	error = xfs_bwrite(rtsb_bp);
 850	xfs_buf_unlock(rtsb_bp);
 851	if (error)
 852		return error;
 853
 854	/* Initialize the rtrmap to reflect the rtsb. */
 855	if (rtg_rmap(args->rtg) != NULL)
 856		error = xfs_rtrmapbt_init_rtsb(nargs->mp, args->rtg, args->tp);
 857
 858	return error;
 859}
 860
 861static void
 862xfs_growfs_rt_sb_fields(
 863	struct xfs_trans	*tp,
 864	const struct xfs_mount	*nmp)
 865{
 866	struct xfs_mount	*mp = tp->t_mountp;
 867
 868	if (nmp->m_sb.sb_rextsize != mp->m_sb.sb_rextsize)
 869		xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSIZE,
 870			nmp->m_sb.sb_rextsize - mp->m_sb.sb_rextsize);
 871	if (nmp->m_sb.sb_rbmblocks != mp->m_sb.sb_rbmblocks)
 872		xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS,
 873			nmp->m_sb.sb_rbmblocks - mp->m_sb.sb_rbmblocks);
 874	if (nmp->m_sb.sb_rblocks != mp->m_sb.sb_rblocks)
 875		xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBLOCKS,
 876			nmp->m_sb.sb_rblocks - mp->m_sb.sb_rblocks);
 877	if (nmp->m_sb.sb_rextents != mp->m_sb.sb_rextents)
 878		xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTENTS,
 879			nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents);
 880	if (nmp->m_sb.sb_rextslog != mp->m_sb.sb_rextslog)
 881		xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG,
 882			nmp->m_sb.sb_rextslog - mp->m_sb.sb_rextslog);
 883	if (nmp->m_sb.sb_rgcount != mp->m_sb.sb_rgcount)
 884		xfs_trans_mod_sb(tp, XFS_TRANS_SB_RGCOUNT,
 885			nmp->m_sb.sb_rgcount - mp->m_sb.sb_rgcount);
 886}
 887
 888static int
 889xfs_growfs_rt_zoned(
 890	struct xfs_rtgroup	*rtg,
 891	xfs_rfsblock_t		nrblocks)
 892{
 893	struct xfs_mount	*mp = rtg_mount(rtg);
 894	struct xfs_mount	*nmp;
 895	struct xfs_trans	*tp;
 896	xfs_rtbxlen_t		freed_rtx;
 897	int			error;
 898
 899	/*
 900	 * Calculate new sb and mount fields for this round.  Also ensure the
 901	 * rtg_extents value is uptodate as the rtbitmap code relies on it.
 902	 */
 903	nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks,
 904			mp->m_sb.sb_rextsize);
 905	if (!nmp)
 906		return -ENOMEM;
 907	freed_rtx = nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents;
 908
 909	xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg),
 910			nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents);
 911
 912	error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0, &tp);
 913	if (error)
 914		goto out_free;
 915
 916	xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
 917	xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP);
 918
 919	xfs_growfs_rt_sb_fields(tp, nmp);
 920	xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, freed_rtx);
 921
 922	error = xfs_trans_commit(tp);
 923	if (error)
 924		goto out_free;
 925
 926	/*
 927	 * Ensure the mount RT feature flag is now set, and compute new
 928	 * maxlevels for rt btrees.
 929	 */
 930	mp->m_features |= XFS_FEAT_REALTIME;
 931	xfs_rtrmapbt_compute_maxlevels(mp);
 932	xfs_rtrefcountbt_compute_maxlevels(mp);
 933	xfs_zoned_add_available(mp, freed_rtx);
 934out_free:
 935	kfree(nmp);
 936	return error;
 937}
 938
 939static int
 940xfs_growfs_rt_bmblock(
 941	struct xfs_rtgroup	*rtg,
 942	xfs_rfsblock_t		nrblocks,
 943	xfs_agblock_t		rextsize,
 944	xfs_fileoff_t		bmbno)
 945{
 946	struct xfs_mount	*mp = rtg_mount(rtg);
 947	struct xfs_inode	*rbmip = rtg_bitmap(rtg);
 948	struct xfs_inode	*rsumip = rtg_summary(rtg);
 949	struct xfs_rtalloc_args	args = {
 950		.mp		= mp,
 951		.rtg		= rtg,
 952	};
 953	struct xfs_rtalloc_args	nargs = {
 954		.rtg		= rtg,
 955	};
 956	struct xfs_mount	*nmp;
 957	xfs_rtbxlen_t		freed_rtx;
 958	int			error;
 959
 960	/*
 961	 * Calculate new sb and mount fields for this round.  Also ensure the
 962	 * rtg_extents value is uptodate as the rtbitmap code relies on it.
 963	 */
 964	nmp = nargs.mp = xfs_growfs_rt_alloc_fake_mount(mp,
 965			xfs_growfs_rt_nrblocks(rtg, nrblocks, rextsize, bmbno),
 966			rextsize);
 967	if (!nmp)
 968		return -ENOMEM;
 969
 970	xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg),
 971			nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents);
 972
 973	/*
 974	 * Recompute the growfsrt reservation from the new rsumsize, so that the
 975	 * transaction below use the new, potentially larger value.
 976	 * */
 977	xfs_trans_resv_calc(nmp, &nmp->m_resv);
 978	error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0,
 979			&args.tp);
 980	if (error)
 981		goto out_free;
 982	nargs.tp = args.tp;
 983
 984	xfs_rtgroup_lock(args.rtg, XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP);
 985	xfs_rtgroup_trans_join(args.tp, args.rtg,
 986			XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP);
 987
 988	/*
 989	 * Update the bitmap inode's size ondisk and incore.  We need to update
 990	 * the incore size so that inode inactivation won't punch what it thinks
 991	 * are "posteof" blocks.
 992	 */
 993	rbmip->i_disk_size = nmp->m_sb.sb_rbmblocks * nmp->m_sb.sb_blocksize;
 994	i_size_write(VFS_I(rbmip), rbmip->i_disk_size);
 995	xfs_trans_log_inode(args.tp, rbmip, XFS_ILOG_CORE);
 996
 997	/*
 998	 * Update the summary inode's size.  We need to update the incore size
 999	 * so that inode inactivation won't punch what it thinks are "posteof"
1000	 * blocks.
1001	 */
1002	rsumip->i_disk_size = nmp->m_rsumblocks * nmp->m_sb.sb_blocksize;
1003	i_size_write(VFS_I(rsumip), rsumip->i_disk_size);
1004	xfs_trans_log_inode(args.tp, rsumip, XFS_ILOG_CORE);
1005
1006	/*
1007	 * Copy summary data from old to new sizes when the real size (not
1008	 * block-aligned) changes.
1009	 */
1010	if (mp->m_sb.sb_rbmblocks != nmp->m_sb.sb_rbmblocks ||
1011	    mp->m_rsumlevels != nmp->m_rsumlevels) {
1012		error = xfs_rtcopy_summary(&args, &nargs);
1013		if (error)
1014			goto out_cancel;
1015	}
1016
1017	error = xfs_growfs_rt_init_rtsb(&nargs, rtg, &args);
1018	if (error)
1019		goto out_cancel;
1020
1021	/*
1022	 * Update superblock fields.
1023	 */
1024	xfs_growfs_rt_sb_fields(args.tp, nmp);
1025
1026	/*
1027	 * Free the new extent.
1028	 */
1029	error = xfs_growfs_rt_free_new(rtg, &nargs, &freed_rtx);
1030	xfs_rtbuf_cache_relse(&nargs);
1031	if (error)
1032		goto out_cancel;
1033
1034	/*
1035	 * Mark more blocks free in the superblock.
1036	 */
1037	xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_FREXTENTS, freed_rtx);
1038
1039	/*
1040	 * Update the calculated values in the real mount structure.
1041	 */
1042	mp->m_rsumlevels = nmp->m_rsumlevels;
1043	mp->m_rsumblocks = nmp->m_rsumblocks;
1044
1045	/*
1046	 * Recompute the growfsrt reservation from the new rsumsize.
1047	 */
1048	xfs_trans_resv_calc(mp, &mp->m_resv);
1049
1050	error = xfs_trans_commit(args.tp);
1051	if (error)
1052		goto out_free;
1053
1054	/*
1055	 * Ensure the mount RT feature flag is now set, and compute new
1056	 * maxlevels for rt btrees.
1057	 */
1058	mp->m_features |= XFS_FEAT_REALTIME;
1059	xfs_rtrmapbt_compute_maxlevels(mp);
1060	xfs_rtrefcountbt_compute_maxlevels(mp);
1061
1062	kfree(nmp);
1063	return 0;
1064
1065out_cancel:
1066	xfs_trans_cancel(args.tp);
1067out_free:
1068	kfree(nmp);
1069	return error;
1070}
1071
1072static xfs_rtxnum_t
1073xfs_last_rtgroup_extents(
1074	struct xfs_mount	*mp)
1075{
1076	return mp->m_sb.sb_rextents -
1077		((xfs_rtxnum_t)(mp->m_sb.sb_rgcount - 1) *
1078		 mp->m_sb.sb_rgextents);
1079}
1080
1081/*
1082 * Calculate the last rbmblock currently used.
1083 *
1084 * This also deals with the case where there were no rtextents before.
1085 */
1086static xfs_fileoff_t
1087xfs_last_rt_bmblock(
1088	struct xfs_rtgroup	*rtg)
1089{
1090	struct xfs_mount	*mp = rtg_mount(rtg);
1091	xfs_rgnumber_t		rgno = rtg_rgno(rtg);
1092	xfs_fileoff_t		bmbno = 0;
1093
1094	ASSERT(!mp->m_sb.sb_rgcount || rgno >= mp->m_sb.sb_rgcount - 1);
1095
1096	if (mp->m_sb.sb_rgcount && rgno == mp->m_sb.sb_rgcount - 1) {
1097		xfs_rtxnum_t	nrext = xfs_last_rtgroup_extents(mp);
1098
1099		/* Also fill up the previous block if not entirely full. */
1100		bmbno = xfs_rtbitmap_blockcount_len(mp, nrext);
1101		if (xfs_rtx_to_rbmword(mp, nrext) != 0)
1102			bmbno--;
1103	}
1104
1105	return bmbno;
1106}
1107
1108/*
1109 * Allocate space to the bitmap and summary files, as necessary.
1110 */
1111static int
1112xfs_growfs_rt_alloc_blocks(
1113	struct xfs_rtgroup	*rtg,
1114	xfs_rfsblock_t		nrblocks,
1115	xfs_agblock_t		rextsize,
1116	xfs_extlen_t		*nrbmblocks)
1117{
1118	struct xfs_mount	*mp = rtg_mount(rtg);
1119	struct xfs_inode	*rbmip = rtg_bitmap(rtg);
1120	struct xfs_inode	*rsumip = rtg_summary(rtg);
1121	xfs_extlen_t		orbmblocks = 0;
1122	xfs_extlen_t		orsumblocks = 0;
1123	struct xfs_mount	*nmp;
1124	int			error = 0;
1125
1126	nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks, rextsize);
1127	if (!nmp)
1128		return -ENOMEM;
1129	*nrbmblocks = nmp->m_sb.sb_rbmblocks;
1130
1131	if (xfs_has_rtgroups(mp)) {
1132		/*
1133		 * For file systems with the rtgroups feature, the RT bitmap and
1134		 * summary are always fully allocated, which means that we never
1135		 * need to grow the existing files.
1136		 *
1137		 * But we have to be careful to only fill the bitmap until the
1138		 * end of the actually used range.
1139		 */
1140		if (rtg_rgno(rtg) == nmp->m_sb.sb_rgcount - 1)
1141			*nrbmblocks = xfs_rtbitmap_blockcount_len(nmp,
1142					xfs_last_rtgroup_extents(nmp));
1143
1144		if (mp->m_sb.sb_rgcount &&
1145		    rtg_rgno(rtg) == mp->m_sb.sb_rgcount - 1)
1146			goto out_free;
1147	} else {
1148		/*
1149		 * Get the old block counts for bitmap and summary inodes.
1150		 * These can't change since other growfs callers are locked out.
1151		 */
1152		orbmblocks = XFS_B_TO_FSB(mp, rbmip->i_disk_size);
1153		orsumblocks = XFS_B_TO_FSB(mp, rsumip->i_disk_size);
1154	}
1155
1156	error = xfs_rtfile_initialize_blocks(rtg, XFS_RTGI_BITMAP, orbmblocks,
1157			nmp->m_sb.sb_rbmblocks, NULL);
1158	if (error)
1159		goto out_free;
1160	error = xfs_rtfile_initialize_blocks(rtg, XFS_RTGI_SUMMARY, orsumblocks,
1161			nmp->m_rsumblocks, NULL);
1162out_free:
1163	kfree(nmp);
1164	return error;
1165}
1166
1167static int
1168xfs_growfs_rtg(
1169	struct xfs_mount	*mp,
1170	xfs_rgnumber_t		rgno,
1171	xfs_rfsblock_t		nrblocks,
1172	xfs_agblock_t		rextsize)
1173{
1174	uint8_t			*old_rsum_cache = NULL;
1175	xfs_extlen_t		bmblocks;
1176	xfs_fileoff_t		bmbno;
1177	struct xfs_rtgroup	*rtg;
1178	unsigned int		i;
1179	int			error;
1180
1181	rtg = xfs_rtgroup_grab(mp, rgno);
1182	if (!rtg)
1183		return -EINVAL;
1184
1185	for (i = 0; i < XFS_RTGI_MAX; i++) {
1186		error = xfs_rtginode_ensure(rtg, i);
1187		if (error)
1188			goto out_rele;
1189	}
1190
1191	if (xfs_has_zoned(mp)) {
1192		error = xfs_growfs_rt_zoned(rtg, nrblocks);
1193		goto out_rele;
1194	}
1195
1196	error = xfs_growfs_rt_alloc_blocks(rtg, nrblocks, rextsize, &bmblocks);
1197	if (error)
1198		goto out_rele;
1199
1200	if (bmblocks != rtg_mount(rtg)->m_sb.sb_rbmblocks) {
1201		old_rsum_cache = rtg->rtg_rsum_cache;
1202		error = xfs_alloc_rsum_cache(rtg, bmblocks);
1203		if (error)
1204			goto out_rele;
1205	}
1206
1207	for (bmbno = xfs_last_rt_bmblock(rtg); bmbno < bmblocks; bmbno++) {
1208		error = xfs_growfs_rt_bmblock(rtg, nrblocks, rextsize, bmbno);
1209		if (error)
1210			goto out_error;
1211	}
1212
1213	kvfree(old_rsum_cache);
1214	goto out_rele;
1215
1216out_error:
1217	/*
1218	 * Reset rtg_extents to the old value if adding more blocks failed.
1219	 */
1220	xfs_rtgroup_calc_geometry(mp, rtg, rtg_rgno(rtg), mp->m_sb.sb_rgcount,
1221			mp->m_sb.sb_rextents);
1222	if (old_rsum_cache) {
1223		kvfree(rtg->rtg_rsum_cache);
1224		rtg->rtg_rsum_cache = old_rsum_cache;
1225	}
1226out_rele:
1227	xfs_rtgroup_rele(rtg);
1228	return error;
1229}
1230
1231int
1232xfs_growfs_check_rtgeom(
1233	const struct xfs_mount	*mp,
1234	xfs_rfsblock_t		dblocks,
1235	xfs_rfsblock_t		rblocks,
1236	xfs_extlen_t		rextsize)
1237{
1238	xfs_extlen_t		min_logfsbs;
1239	struct xfs_mount	*nmp;
1240
1241	nmp = xfs_growfs_rt_alloc_fake_mount(mp, rblocks, rextsize);
1242	if (!nmp)
1243		return -ENOMEM;
1244	nmp->m_sb.sb_dblocks = dblocks;
1245
1246	xfs_rtrmapbt_compute_maxlevels(nmp);
1247	xfs_rtrefcountbt_compute_maxlevels(nmp);
1248	xfs_trans_resv_calc(nmp, M_RES(nmp));
1249
1250	/*
1251	 * New summary size can't be more than half the size of the log.  This
1252	 * prevents us from getting a log overflow, since we'll log basically
1253	 * the whole summary file at once.
1254	 */
1255	min_logfsbs = min_t(xfs_extlen_t, xfs_log_calc_minimum_size(nmp),
1256			nmp->m_rsumblocks * 2);
1257
1258	trace_xfs_growfs_check_rtgeom(mp, min_logfsbs);
1259
1260	if (min_logfsbs > mp->m_sb.sb_logblocks)
1261		goto out_inval;
1262
1263	if (xfs_has_zoned(mp)) {
1264		uint32_t	gblocks = mp->m_groups[XG_TYPE_RTG].blocks;
1265		uint32_t	rem;
1266
1267		if (rextsize != 1)
1268			goto out_inval;
1269		div_u64_rem(nmp->m_sb.sb_rblocks, gblocks, &rem);
1270		if (rem) {
1271			xfs_warn(mp,
1272"new RT volume size (%lld) not aligned to RT group size (%d)",
1273				nmp->m_sb.sb_rblocks, gblocks);
1274			goto out_inval;
1275		}
1276	}
1277
1278	kfree(nmp);
1279	return 0;
1280out_inval:
1281	kfree(nmp);
1282	return -EINVAL;
1283}
1284
1285/*
1286 * Compute the new number of rt groups and ensure that /rtgroups exists.
1287 *
1288 * Changing the rtgroup size is not allowed (even if the rt volume hasn't yet
1289 * been initialized) because the userspace ABI doesn't support it.
1290 */
1291static int
1292xfs_growfs_rt_prep_groups(
1293	struct xfs_mount	*mp,
1294	xfs_rfsblock_t		rblocks,
1295	xfs_extlen_t		rextsize,
1296	xfs_rgnumber_t		*new_rgcount)
1297{
1298	int			error;
1299
1300	*new_rgcount = howmany_64(rblocks, mp->m_sb.sb_rgextents * rextsize);
1301	if (*new_rgcount > XFS_MAX_RGNUMBER)
1302		return -EINVAL;
1303
1304	/* Make sure the /rtgroups dir has been created */
1305	if (!mp->m_rtdirip) {
1306		struct xfs_trans	*tp;
1307
1308		tp = xfs_trans_alloc_empty(mp);
1309		error = xfs_rtginode_load_parent(tp);
1310		xfs_trans_cancel(tp);
1311
1312		if (error == -ENOENT)
1313			error = xfs_rtginode_mkdir_parent(mp);
1314		if (error)
1315			return error;
1316	}
1317
1318	return 0;
1319}
1320
1321static bool
1322xfs_grow_last_rtg(
1323	struct xfs_mount	*mp)
1324{
1325	if (!xfs_has_rtgroups(mp))
1326		return true;
1327	if (mp->m_sb.sb_rgcount == 0)
1328		return false;
1329	return xfs_rtgroup_extents(mp, mp->m_sb.sb_rgcount - 1) <
1330			mp->m_sb.sb_rgextents;
1331}
1332
1333/*
1334 * Read in the last block of the RT device to make sure it is accessible.
1335 */
1336static int
1337xfs_rt_check_size(
1338	struct xfs_mount	*mp,
1339	xfs_rfsblock_t		last_block)
1340{
1341	xfs_daddr_t		daddr = XFS_FSB_TO_BB(mp, last_block);
1342	struct xfs_buf		*bp;
1343	int			error;
1344
1345	if (XFS_BB_TO_FSB(mp, daddr) != last_block) {
1346		xfs_warn(mp, "RT device size overflow: %llu != %llu",
1347			XFS_BB_TO_FSB(mp, daddr), last_block);
1348		return -EFBIG;
1349	}
1350
1351	error = xfs_buf_read_uncached(mp->m_rtdev_targp,
1352			XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart) + daddr,
1353			XFS_FSB_TO_BB(mp, 1), &bp, NULL);
1354	if (error)
1355		xfs_warn(mp, "cannot read last RT device sector (%lld)",
1356				last_block);
1357	else
1358		xfs_buf_relse(bp);
1359	return error;
1360}
1361
1362/*
1363 * Grow the realtime area of the filesystem.
1364 */
1365int
1366xfs_growfs_rt(
1367	struct xfs_mount	*mp,
1368	struct xfs_growfs_rt	*in)
1369{
1370	xfs_rgnumber_t		old_rgcount = mp->m_sb.sb_rgcount;
1371	xfs_rgnumber_t		new_rgcount = 1;
1372	xfs_rgnumber_t		rgno;
1373	xfs_agblock_t		old_rextsize = mp->m_sb.sb_rextsize;
1374	int			error;
1375
1376	if (!capable(CAP_SYS_ADMIN))
1377		return -EPERM;
1378
1379	/* Needs to have been mounted with an rt device. */
1380	if (!XFS_IS_REALTIME_MOUNT(mp))
1381		return -EINVAL;
1382
1383	if (!mutex_trylock(&mp->m_growlock))
1384		return -EWOULDBLOCK;
1385
1386	/* Shrink not supported. */
1387	error = -EINVAL;
1388	if (in->newblocks <= mp->m_sb.sb_rblocks)
1389		goto out_unlock;
1390	/* Can only change rt extent size when adding rt volume. */
1391	if (mp->m_sb.sb_rblocks > 0 && in->extsize != mp->m_sb.sb_rextsize)
1392		goto out_unlock;
1393
1394	/* Range check the extent size. */
1395	if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||
1396	    XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)
1397		goto out_unlock;
1398
1399	/* Check for features supported only on rtgroups filesystems. */
1400	error = -EOPNOTSUPP;
1401	if (!xfs_has_rtgroups(mp)) {
1402		if (xfs_has_rmapbt(mp))
1403			goto out_unlock;
1404		if (xfs_has_quota(mp))
1405			goto out_unlock;
1406		if (xfs_has_reflink(mp))
1407			goto out_unlock;
1408	} else if (xfs_has_reflink(mp) &&
1409		   !xfs_reflink_supports_rextsize(mp, in->extsize))
1410		goto out_unlock;
1411
1412	error = xfs_sb_validate_fsb_count(&mp->m_sb, in->newblocks);
1413	if (error)
1414		goto out_unlock;
1415
1416	error = xfs_rt_check_size(mp, in->newblocks - 1);
1417	if (error)
1418		goto out_unlock;
1419
1420	/*
1421	 * Calculate new parameters.  These are the final values to be reached.
1422	 */
1423	error = -EINVAL;
1424	if (in->newblocks < in->extsize)
1425		goto out_unlock;
1426
1427	/* Make sure the new fs size won't cause problems with the log. */
1428	error = xfs_growfs_check_rtgeom(mp, mp->m_sb.sb_dblocks, in->newblocks,
1429			in->extsize);
1430	if (error)
1431		goto out_unlock;
1432
1433	if (xfs_has_rtgroups(mp)) {
1434		error = xfs_growfs_rt_prep_groups(mp, in->newblocks,
1435				in->extsize, &new_rgcount);
1436		if (error)
1437			goto out_unlock;
1438	}
1439
1440	if (xfs_grow_last_rtg(mp)) {
1441		error = xfs_growfs_rtg(mp, old_rgcount - 1, in->newblocks,
1442				in->extsize);
1443		if (error)
1444			goto out_unlock;
1445	}
1446
1447	for (rgno = old_rgcount; rgno < new_rgcount; rgno++) {
1448		xfs_rtbxlen_t	rextents = div_u64(in->newblocks, in->extsize);
1449
1450		error = xfs_rtgroup_alloc(mp, rgno, new_rgcount, rextents);
1451		if (error)
1452			goto out_unlock;
1453
1454		error = xfs_growfs_rtg(mp, rgno, in->newblocks, in->extsize);
1455		if (error) {
1456			struct xfs_rtgroup	*rtg;
1457
1458			rtg = xfs_rtgroup_grab(mp, rgno);
1459			if (!WARN_ON_ONCE(!rtg)) {
1460				xfs_rtunmount_rtg(rtg);
1461				xfs_rtgroup_rele(rtg);
1462				xfs_rtgroup_free(mp, rgno);
1463			}
1464			break;
1465		}
1466	}
1467
1468	if (!error && old_rextsize != in->extsize)
1469		error = xfs_growfs_rt_fixup_extsize(mp);
1470
1471	/*
1472	 * Update secondary superblocks now the physical grow has completed.
1473	 *
1474	 * Also do this in case of an error as we might have already
1475	 * successfully updated one or more RTGs and incremented sb_rgcount.
1476	 */
1477	if (!xfs_is_shutdown(mp)) {
1478		int error2 = xfs_update_secondary_sbs(mp);
1479
1480		if (!error)
1481			error = error2;
1482
1483		/* Reset the rt metadata btree space reservations. */
1484		error2 = xfs_metafile_resv_init(mp);
1485		if (error2 && error2 != -ENOSPC)
1486			error = error2;
1487	}
1488
1489out_unlock:
1490	mutex_unlock(&mp->m_growlock);
1491	return error;
1492}
1493
1494/* Read the realtime superblock and attach it to the mount. */
1495int
1496xfs_rtmount_readsb(
1497	struct xfs_mount	*mp)
1498{
1499	struct xfs_buf		*bp;
1500	int			error;
1501
1502	if (!xfs_has_rtsb(mp))
1503		return 0;
1504	if (mp->m_sb.sb_rblocks == 0)
1505		return 0;
1506	if (mp->m_rtdev_targp == NULL) {
1507		xfs_warn(mp,
1508	"Filesystem has a realtime volume, use rtdev=device option");
1509		return -ENODEV;
1510	}
1511
1512	/* m_blkbb_log is not set up yet */
1513	error = xfs_buf_read_uncached(mp->m_rtdev_targp, XFS_RTSB_DADDR,
1514			mp->m_sb.sb_blocksize >> BBSHIFT, &bp,
1515			&xfs_rtsb_buf_ops);
1516	if (error) {
1517		xfs_warn(mp, "rt sb validate failed with error %d.", error);
1518		/* bad CRC means corrupted metadata */
1519		if (error == -EFSBADCRC)
1520			error = -EFSCORRUPTED;
1521		return error;
1522	}
1523
1524	mp->m_rtsb_bp = bp;
1525	xfs_buf_unlock(bp);
1526	return 0;
1527}
1528
1529/* Detach the realtime superblock from the mount and free it. */
1530void
1531xfs_rtmount_freesb(
1532	struct xfs_mount	*mp)
1533{
1534	struct xfs_buf		*bp = mp->m_rtsb_bp;
1535
1536	if (!bp)
1537		return;
1538
1539	xfs_buf_lock(bp);
1540	mp->m_rtsb_bp = NULL;
1541	xfs_buf_relse(bp);
1542}
1543
1544/*
1545 * Initialize realtime fields in the mount structure.
1546 */
1547int				/* error */
1548xfs_rtmount_init(
1549	struct xfs_mount	*mp)	/* file system mount structure */
1550{
1551	if (mp->m_sb.sb_rblocks == 0)
1552		return 0;
1553	if (mp->m_rtdev_targp == NULL) {
1554		xfs_warn(mp,
1555	"Filesystem has a realtime volume, use rtdev=device option");
1556		return -ENODEV;
1557	}
1558
1559	mp->m_rsumblocks = xfs_rtsummary_blockcount(mp, &mp->m_rsumlevels);
1560
1561	return xfs_rt_check_size(mp, mp->m_sb.sb_rblocks - 1);
1562}
1563
1564static int
1565xfs_rtalloc_count_frextent(
1566	struct xfs_rtgroup		*rtg,
1567	struct xfs_trans		*tp,
1568	const struct xfs_rtalloc_rec	*rec,
1569	void				*priv)
1570{
1571	uint64_t			*valp = priv;
1572
1573	*valp += rec->ar_extcount;
1574	return 0;
1575}
1576
1577/*
1578 * Reinitialize the number of free realtime extents from the realtime bitmap.
1579 * Callers must ensure that there is no other activity in the filesystem.
1580 */
1581int
1582xfs_rtalloc_reinit_frextents(
1583	struct xfs_mount	*mp)
1584{
1585	uint64_t		val = 0;
1586	int			error;
1587
1588	struct xfs_rtgroup	*rtg = NULL;
1589
1590	while ((rtg = xfs_rtgroup_next(mp, rtg))) {
1591		xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
1592		error = xfs_rtalloc_query_all(rtg, NULL,
1593				xfs_rtalloc_count_frextent, &val);
1594		xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
1595		if (error) {
1596			xfs_rtgroup_rele(rtg);
1597			return error;
1598		}
1599	}
1600
1601	spin_lock(&mp->m_sb_lock);
1602	mp->m_sb.sb_frextents = val;
1603	spin_unlock(&mp->m_sb_lock);
1604	xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, mp->m_sb.sb_frextents);
1605	return 0;
1606}
1607
1608/*
1609 * Read in the bmbt of an rt metadata inode so that we never have to load them
1610 * at runtime.  This enables the use of shared ILOCKs for rtbitmap scans.  Use
1611 * an empty transaction to avoid deadlocking on loops in the bmbt.
1612 */
1613static inline int
1614xfs_rtmount_iread_extents(
1615	struct xfs_trans	*tp,
1616	struct xfs_inode	*ip)
1617{
1618	int			error;
1619
1620	xfs_ilock(ip, XFS_ILOCK_EXCL);
1621
1622	error = xfs_iread_extents(tp, ip, XFS_DATA_FORK);
1623	if (error)
1624		goto out_unlock;
1625
1626	if (xfs_inode_has_attr_fork(ip)) {
1627		error = xfs_iread_extents(tp, ip, XFS_ATTR_FORK);
1628		if (error)
1629			goto out_unlock;
1630	}
1631
1632out_unlock:
1633	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1634	return error;
1635}
1636
1637static int
1638xfs_rtmount_rtg(
1639	struct xfs_mount	*mp,
1640	struct xfs_trans	*tp,
1641	struct xfs_rtgroup	*rtg)
1642{
1643	int			error, i;
1644
1645	for (i = 0; i < XFS_RTGI_MAX; i++) {
1646		error = xfs_rtginode_load(rtg, i, tp);
1647		if (error)
1648			return error;
1649
1650		if (rtg->rtg_inodes[i]) {
1651			error = xfs_rtmount_iread_extents(tp,
1652					rtg->rtg_inodes[i]);
1653			if (error)
1654				return error;
1655		}
1656	}
1657
1658	if (xfs_has_zoned(mp))
1659		return 0;
1660	return xfs_alloc_rsum_cache(rtg, mp->m_sb.sb_rbmblocks);
1661}
1662
1663/*
1664 * Get the bitmap and summary inodes and the summary cache into the mount
1665 * structure at mount time.
1666 */
1667int
1668xfs_rtmount_inodes(
1669	struct xfs_mount	*mp)
1670{
1671	struct xfs_trans	*tp;
1672	struct xfs_rtgroup	*rtg = NULL;
1673	int			error;
1674
1675	tp = xfs_trans_alloc_empty(mp);
1676	if (xfs_has_rtgroups(mp) && mp->m_sb.sb_rgcount > 0) {
1677		error = xfs_rtginode_load_parent(tp);
1678		if (error)
1679			goto out_cancel;
1680	}
1681
1682	while ((rtg = xfs_rtgroup_next(mp, rtg))) {
1683		error = xfs_rtmount_rtg(mp, tp, rtg);
1684		if (error) {
1685			xfs_rtgroup_rele(rtg);
1686			xfs_rtunmount_inodes(mp);
1687			break;
1688		}
1689	}
1690
1691out_cancel:
1692	xfs_trans_cancel(tp);
1693	return error;
1694}
1695
1696void
1697xfs_rtunmount_inodes(
1698	struct xfs_mount	*mp)
1699{
1700	struct xfs_rtgroup	*rtg = NULL;
1701
1702	while ((rtg = xfs_rtgroup_next(mp, rtg)))
1703		xfs_rtunmount_rtg(rtg);
1704	xfs_rtginode_irele(&mp->m_rtdirip);
1705}
1706
1707/*
1708 * Pick an extent for allocation at the start of a new realtime file.
1709 * Use the sequence number stored in the atime field of the bitmap inode.
1710 * Translate this to a fraction of the rtextents, and return the product
1711 * of rtextents and the fraction.
1712 * The fraction sequence is 0, 1/2, 1/4, 3/4, 1/8, ..., 7/8, 1/16, ...
1713 */
1714static xfs_rtxnum_t
1715xfs_rtpick_extent(
1716	struct xfs_rtgroup	*rtg,
1717	struct xfs_trans	*tp,
1718	xfs_rtxlen_t		len)		/* allocation length (rtextents) */
1719{
1720	struct xfs_mount	*mp = rtg_mount(rtg);
1721	struct xfs_inode	*rbmip = rtg_bitmap(rtg);
1722	xfs_rtxnum_t		b = 0;		/* result rtext */
1723	int			log2;		/* log of sequence number */
1724	uint64_t		resid;		/* residual after log removed */
1725	uint64_t		seq;		/* sequence number of file creation */
1726	struct timespec64	ts;		/* timespec in inode */
1727
1728	xfs_assert_ilocked(rbmip, XFS_ILOCK_EXCL);
1729
1730	ts = inode_get_atime(VFS_I(rbmip));
1731	if (!(rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {
1732		rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;
1733		seq = 0;
1734	} else {
1735		seq = ts.tv_sec;
1736	}
1737	log2 = xfs_highbit64(seq);
1738	if (log2 != -1) {
1739		resid = seq - (1ULL << log2);
1740		b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>
1741		    (log2 + 1);
1742		if (b >= mp->m_sb.sb_rextents)
1743			div64_u64_rem(b, mp->m_sb.sb_rextents, &b);
1744		if (b + len > mp->m_sb.sb_rextents)
1745			b = mp->m_sb.sb_rextents - len;
1746	}
1747	ts.tv_sec = seq + 1;
1748	inode_set_atime_to_ts(VFS_I(rbmip), ts);
1749	xfs_trans_log_inode(tp, rbmip, XFS_ILOG_CORE);
1750	return b;
1751}
1752
1753static void
1754xfs_rtalloc_align_minmax(
1755	xfs_rtxlen_t		*raminlen,
1756	xfs_rtxlen_t		*ramaxlen,
1757	xfs_rtxlen_t		*prod)
1758{
1759	xfs_rtxlen_t		newmaxlen = *ramaxlen;
1760	xfs_rtxlen_t		newminlen = *raminlen;
1761	xfs_rtxlen_t		slack;
1762
1763	slack = newmaxlen % *prod;
1764	if (slack)
1765		newmaxlen -= slack;
1766	slack = newminlen % *prod;
1767	if (slack)
1768		newminlen += *prod - slack;
1769
1770	/*
1771	 * If adjusting for extent size hint alignment produces an invalid
1772	 * min/max len combination, go ahead without it.
1773	 */
1774	if (newmaxlen < newminlen) {
1775		*prod = 1;
1776		return;
1777	}
1778	*ramaxlen = newmaxlen;
1779	*raminlen = newminlen;
1780}
1781
1782/* Given a free extent, find any part of it that isn't busy, if possible. */
1783STATIC bool
1784xfs_rtalloc_check_busy(
1785	struct xfs_rtalloc_args	*args,
1786	xfs_rtxnum_t		start,
1787	xfs_rtxlen_t		minlen_rtx,
1788	xfs_rtxlen_t		maxlen_rtx,
1789	xfs_rtxlen_t		len_rtx,
1790	xfs_rtxlen_t		prod,
1791	xfs_rtxnum_t		rtx,
1792	xfs_rtxlen_t		*reslen,
1793	xfs_rtxnum_t		*resrtx,
1794	unsigned		*busy_gen)
1795{
1796	struct xfs_rtgroup	*rtg = args->rtg;
1797	struct xfs_mount	*mp = rtg_mount(rtg);
1798	xfs_agblock_t		rgbno = xfs_rtx_to_rgbno(rtg, rtx);
1799	xfs_rgblock_t		min_rgbno = xfs_rtx_to_rgbno(rtg, start);
1800	xfs_extlen_t		minlen = xfs_rtxlen_to_extlen(mp, minlen_rtx);
1801	xfs_extlen_t		len = xfs_rtxlen_to_extlen(mp, len_rtx);
1802	xfs_extlen_t		diff;
1803	bool			busy;
1804
1805	busy = xfs_extent_busy_trim(rtg_group(rtg), minlen,
1806			xfs_rtxlen_to_extlen(mp, maxlen_rtx), &rgbno, &len,
1807			busy_gen);
1808
1809	/*
1810	 * If we have a largish extent that happens to start before min_rgbno,
1811	 * see if we can shift it into range...
1812	 */
1813	if (rgbno < min_rgbno && rgbno + len > min_rgbno) {
1814		diff = min_rgbno - rgbno;
1815		if (len > diff) {
1816			rgbno += diff;
1817			len -= diff;
1818		}
1819	}
1820
1821	if (prod > 1 && len >= minlen) {
1822		xfs_rgblock_t	aligned_rgbno = roundup(rgbno, prod);
1823
1824		diff = aligned_rgbno - rgbno;
1825
1826		*resrtx = xfs_rgbno_to_rtx(mp, aligned_rgbno);
1827		*reslen = xfs_extlen_to_rtxlen(mp,
1828				diff >= len ? 0 : len - diff);
1829	} else {
1830		*resrtx = xfs_rgbno_to_rtx(mp, rgbno);
1831		*reslen = xfs_extlen_to_rtxlen(mp, len);
1832	}
1833
1834	return busy;
1835}
1836
1837/*
1838 * Adjust the given free extent so that it isn't busy, or flush the log and
1839 * wait for the space to become unbusy.  Only needed for rtgroups.
1840 */
1841STATIC int
1842xfs_rtallocate_adjust_for_busy(
1843	struct xfs_rtalloc_args	*args,
1844	xfs_rtxnum_t		start,
1845	xfs_rtxlen_t		minlen,
1846	xfs_rtxlen_t		maxlen,
1847	xfs_rtxlen_t		*len,
1848	xfs_rtxlen_t		prod,
1849	xfs_rtxnum_t		*rtx)
1850{
1851	xfs_rtxnum_t		resrtx;
1852	xfs_rtxlen_t		reslen;
1853	unsigned		busy_gen;
1854	bool			busy;
1855	int			error;
1856
1857again:
1858	busy = xfs_rtalloc_check_busy(args, start, minlen, maxlen, *len, prod,
1859			*rtx, &reslen, &resrtx, &busy_gen);
1860	if (!busy)
1861		return 0;
1862
1863	if (reslen < minlen || (start != 0 && resrtx != *rtx)) {
1864		/*
1865		 * Enough of the extent was busy that we cannot satisfy the
1866		 * allocation, or this is a near allocation and the start of
1867		 * the extent is busy.  Flush the log and wait for the busy
1868		 * situation to resolve.
1869		 */
1870		trace_xfs_rtalloc_extent_busy(args->rtg, start, minlen, maxlen,
1871				*len, prod, *rtx, busy_gen);
1872
1873		error = xfs_extent_busy_flush(args->tp, rtg_group(args->rtg),
1874				busy_gen, 0);
1875		if (error)
1876			return error;
1877
1878		goto again;
1879	}
1880
1881	/* Some of the free space wasn't busy, hand that back to the caller. */
1882	trace_xfs_rtalloc_extent_busy_trim(args->rtg, *rtx, *len, resrtx,
1883			reslen);
1884	*len = reslen;
1885	*rtx = resrtx;
1886
1887	return 0;
1888}
1889
1890static int
1891xfs_rtallocate_rtg(
1892	struct xfs_trans	*tp,
1893	xfs_rgnumber_t		rgno,
1894	xfs_rtblock_t		bno_hint,
1895	xfs_rtxlen_t		minlen,
1896	xfs_rtxlen_t		maxlen,
1897	xfs_rtxlen_t		prod,
1898	bool			wasdel,
1899	bool			initial_user_data,
1900	bool			*rtlocked,
1901	xfs_rtblock_t		*bno,
1902	xfs_extlen_t		*blen)
1903{
1904	struct xfs_rtalloc_args	args = {
1905		.mp		= tp->t_mountp,
1906		.tp		= tp,
1907	};
1908	xfs_rtxnum_t		start = 0;
1909	xfs_rtxnum_t		rtx;
1910	xfs_rtxlen_t		len = 0;
1911	int			error = 0;
1912
1913	args.rtg = xfs_rtgroup_grab(args.mp, rgno);
1914	if (!args.rtg)
1915		return -ENOSPC;
1916
1917	/*
1918	 * We need to lock out modifications to both the RT bitmap and summary
1919	 * inodes for finding free space in xfs_rtallocate_extent_{near,size}
1920	 * and join the bitmap and summary inodes for the actual allocation
1921	 * down in xfs_rtallocate_range.
1922	 *
1923	 * For RTG-enabled file system we don't want to join the inodes to the
1924	 * transaction until we are committed to allocate to allocate from this
1925	 * RTG so that only one inode of each type is locked at a time.
1926	 *
1927	 * But for pre-RTG file systems we need to already to join the bitmap
1928	 * inode to the transaction for xfs_rtpick_extent, which bumps the
1929	 * sequence number in it, so we'll have to join the inode to the
1930	 * transaction early here.
1931	 *
1932	 * This is all a bit messy, but at least the mess is contained in
1933	 * this function.
1934	 */
1935	if (!*rtlocked) {
1936		xfs_rtgroup_lock(args.rtg, XFS_RTGLOCK_BITMAP);
1937		if (!xfs_has_rtgroups(args.mp))
1938			xfs_rtgroup_trans_join(tp, args.rtg,
1939					XFS_RTGLOCK_BITMAP);
1940		*rtlocked = true;
1941	}
1942
1943	/*
1944	 * For an allocation to an empty file at offset 0, pick an extent that
1945	 * will space things out in the rt area.
1946	 */
1947	if (bno_hint != NULLFSBLOCK)
1948		start = xfs_rtb_to_rtx(args.mp, bno_hint);
1949	else if (!xfs_has_rtgroups(args.mp) && initial_user_data)
1950		start = xfs_rtpick_extent(args.rtg, tp, maxlen);
1951
1952	if (start) {
1953		error = xfs_rtallocate_extent_near(&args, start, minlen, maxlen,
1954				&len, prod, &rtx);
1955		/*
1956		 * If we can't allocate near a specific rt extent, try again
1957		 * without locality criteria.
1958		 */
1959		if (error == -ENOSPC) {
1960			xfs_rtbuf_cache_relse(&args);
1961			error = 0;
1962		}
1963	}
1964
1965	if (!error) {
1966		error = xfs_rtallocate_extent_size(&args, minlen, maxlen, &len,
1967				prod, &rtx);
1968	}
1969
1970	if (error) {
1971		if (xfs_has_rtgroups(args.mp))
1972			goto out_unlock;
1973		goto out_release;
1974	}
1975
1976	if (xfs_has_rtgroups(args.mp)) {
1977		error = xfs_rtallocate_adjust_for_busy(&args, start, minlen,
1978				maxlen, &len, prod, &rtx);
1979		if (error)
1980			goto out_unlock;
1981
1982		xfs_rtgroup_trans_join(tp, args.rtg, XFS_RTGLOCK_BITMAP);
1983	}
1984
1985	error = xfs_rtallocate_range(&args, rtx, len);
1986	if (error)
1987		goto out_release;
1988
1989	xfs_trans_mod_sb(tp, wasdel ?
1990			XFS_TRANS_SB_RES_FREXTENTS : XFS_TRANS_SB_FREXTENTS,
1991			-(long)len);
1992	*bno = xfs_rtx_to_rtb(args.rtg, rtx);
1993	*blen = xfs_rtxlen_to_extlen(args.mp, len);
1994
1995out_release:
1996	xfs_rtgroup_rele(args.rtg);
1997	xfs_rtbuf_cache_relse(&args);
1998	return error;
1999out_unlock:
2000	xfs_rtgroup_unlock(args.rtg, XFS_RTGLOCK_BITMAP);
2001	*rtlocked = false;
2002	goto out_release;
2003}
2004
2005int
2006xfs_rtallocate_rtgs(
2007	struct xfs_trans	*tp,
2008	xfs_fsblock_t		bno_hint,
2009	xfs_rtxlen_t		minlen,
2010	xfs_rtxlen_t		maxlen,
2011	xfs_rtxlen_t		prod,
2012	bool			wasdel,
2013	bool			initial_user_data,
2014	xfs_rtblock_t		*bno,
2015	xfs_extlen_t		*blen)
2016{
2017	struct xfs_mount	*mp = tp->t_mountp;
2018	xfs_rgnumber_t		start_rgno, rgno;
2019	int			error;
2020
2021	/*
2022	 * For now this just blindly iterates over the RTGs for an initial
2023	 * allocation.  We could try to keep an in-memory rtg_longest member
2024	 * to avoid the locking when just looking for big enough free space,
2025	 * but for now this keeps things simple.
2026	 */
2027	if (bno_hint != NULLFSBLOCK)
2028		start_rgno = xfs_rtb_to_rgno(mp, bno_hint);
2029	else
2030		start_rgno = (atomic_inc_return(&mp->m_rtgrotor) - 1) %
2031				mp->m_sb.sb_rgcount;
2032
2033	rgno = start_rgno;
2034	do {
2035		bool		rtlocked = false;
2036
2037		error = xfs_rtallocate_rtg(tp, rgno, bno_hint, minlen, maxlen,
2038				prod, wasdel, initial_user_data, &rtlocked,
2039				bno, blen);
2040		if (error != -ENOSPC)
2041			return error;
2042		ASSERT(!rtlocked);
2043
2044		if (++rgno == mp->m_sb.sb_rgcount)
2045			rgno = 0;
2046		bno_hint = NULLFSBLOCK;
2047	} while (rgno != start_rgno);
2048
2049	return -ENOSPC;
2050}
2051
2052static int
2053xfs_rtallocate_align(
2054	struct xfs_bmalloca	*ap,
2055	xfs_rtxlen_t		*ralen,
2056	xfs_rtxlen_t		*raminlen,
2057	xfs_rtxlen_t		*prod,
2058	bool			*noalign)
2059{
2060	struct xfs_mount	*mp = ap->ip->i_mount;
2061	xfs_fileoff_t		orig_offset = ap->offset;
2062	xfs_extlen_t		minlen = mp->m_sb.sb_rextsize;
2063	xfs_extlen_t            align;	/* minimum allocation alignment */
2064	xfs_extlen_t		mod;	/* product factor for allocators */
2065	int			error;
2066
2067	if (*noalign) {
2068		align = mp->m_sb.sb_rextsize;
2069	} else {
2070		if (ap->flags & XFS_BMAPI_COWFORK)
2071			align = xfs_get_cowextsz_hint(ap->ip);
2072		else
2073			align = xfs_get_extsz_hint(ap->ip);
2074		if (!align)
2075			align = 1;
2076		if (align == mp->m_sb.sb_rextsize)
2077			*noalign = true;
2078	}
2079
2080	error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 1,
2081			ap->eof, 0, ap->conv, &ap->offset, &ap->length);
2082	if (error)
2083		return error;
2084	ASSERT(ap->length);
2085	ASSERT(xfs_extlen_to_rtxmod(mp, ap->length) == 0);
2086
2087	/*
2088	 * If we shifted the file offset downward to satisfy an extent size
2089	 * hint, increase minlen by that amount so that the allocator won't
2090	 * give us an allocation that's too short to cover at least one of the
2091	 * blocks that the caller asked for.
2092	 */
2093	if (ap->offset != orig_offset)
2094		minlen += orig_offset - ap->offset;
2095
2096	/*
2097	 * Set ralen to be the actual requested length in rtextents.
2098	 *
2099	 * If the old value was close enough to XFS_BMBT_MAX_EXTLEN that
2100	 * we rounded up to it, cut it back so it's valid again.
2101	 * Note that if it's a really large request (bigger than
2102	 * XFS_BMBT_MAX_EXTLEN), we don't hear about that number, and can't
2103	 * adjust the starting point to match it.
2104	 */
2105	*ralen = xfs_extlen_to_rtxlen(mp, min(ap->length, XFS_MAX_BMBT_EXTLEN));
2106	*raminlen = max_t(xfs_rtxlen_t, 1, xfs_extlen_to_rtxlen(mp, minlen));
2107	ASSERT(*raminlen > 0);
2108	ASSERT(*raminlen <= *ralen);
2109
2110	/*
2111	 * Only bother calculating a real prod factor if offset & length are
2112	 * perfectly aligned, otherwise it will just get us in trouble.
2113	 */
2114	div_u64_rem(ap->offset, align, &mod);
2115	if (mod || ap->length % align)
2116		*prod = 1;
2117	else
2118		*prod = xfs_extlen_to_rtxlen(mp, align);
2119
2120	if (*prod > 1)
2121		xfs_rtalloc_align_minmax(raminlen, ralen, prod);
2122	return 0;
2123}
2124
2125int
2126xfs_bmap_rtalloc(
2127	struct xfs_bmalloca	*ap)
2128{
2129	xfs_fileoff_t		orig_offset = ap->offset;
2130	xfs_rtxlen_t		prod = 0;  /* product factor for allocators */
2131	xfs_rtxlen_t		ralen = 0; /* realtime allocation length */
2132	xfs_rtblock_t		bno_hint = NULLRTBLOCK;
2133	xfs_extlen_t		orig_length = ap->length;
2134	xfs_rtxlen_t		raminlen;
2135	bool			rtlocked = false;
2136	bool			noalign = false;
2137	bool			initial_user_data =
2138		ap->datatype & XFS_ALLOC_INITIAL_USER_DATA;
2139	int			error;
2140
2141	ASSERT(!xfs_has_zoned(ap->tp->t_mountp));
2142
2143retry:
2144	error = xfs_rtallocate_align(ap, &ralen, &raminlen, &prod, &noalign);
2145	if (error)
2146		return error;
2147
2148	if (xfs_bmap_adjacent(ap))
2149		bno_hint = ap->blkno;
2150
2151	if (xfs_has_rtgroups(ap->ip->i_mount)) {
2152		error = xfs_rtallocate_rtgs(ap->tp, bno_hint, raminlen, ralen,
2153				prod, ap->wasdel, initial_user_data,
2154				&ap->blkno, &ap->length);
2155	} else {
2156		error = xfs_rtallocate_rtg(ap->tp, 0, bno_hint, raminlen, ralen,
2157				prod, ap->wasdel, initial_user_data,
2158				&rtlocked, &ap->blkno, &ap->length);
2159	}
2160
2161	if (error == -ENOSPC) {
2162		if (!noalign) {
2163			/*
2164			 * We previously enlarged the request length to try to
2165			 * satisfy an extent size hint.  The allocator didn't
2166			 * return anything, so reset the parameters to the
2167			 * original values and try again without alignment
2168			 * criteria.
2169			 */
2170			ap->offset = orig_offset;
2171			ap->length = orig_length;
2172			noalign = true;
2173			goto retry;
2174		}
2175
2176		ap->blkno = NULLFSBLOCK;
2177		ap->length = 0;
2178		return 0;
2179	}
2180	if (error)
2181		return error;
2182
2183	xfs_bmap_alloc_account(ap);
2184	return 0;
2185}