vmm.c

   1/*
   2 * Copyright 2017 Red Hat Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22#define NVKM_VMM_LEVELS_MAX 6
  23#include "vmm.h"
  24
  25#include <subdev/fb.h>
  26
  27static void
  28nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
  29{
  30	struct nvkm_vmm_pt *pgt = *ppgt;
  31	if (pgt) {
  32		kvfree(pgt->pde);
  33		kfree(pgt);
  34		*ppgt = NULL;
  35	}
  36}
  37
  38
  39static struct nvkm_vmm_pt *
  40nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
  41		const struct nvkm_vmm_page *page)
  42{
  43	const u32 pten = 1 << desc->bits;
  44	struct nvkm_vmm_pt *pgt;
  45	u32 lpte = 0;
  46
  47	if (desc->type > PGT) {
  48		if (desc->type == SPT) {
  49			const struct nvkm_vmm_desc *pair = page[-1].desc;
  50			lpte = pten >> (desc->bits - pair->bits);
  51		} else {
  52			lpte = pten;
  53		}
  54	}
  55
  56	if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
  57		return NULL;
  58	pgt->page = page ? page->shift : 0;
  59	pgt->sparse = sparse;
  60
  61	if (desc->type == PGD) {
  62		pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
  63		if (!pgt->pde) {
  64			kfree(pgt);
  65			return NULL;
  66		}
  67	}
  68
  69	return pgt;
  70}
  71
  72struct nvkm_vmm_iter {
  73	const struct nvkm_vmm_page *page;
  74	const struct nvkm_vmm_desc *desc;
  75	struct nvkm_vmm *vmm;
  76	u64 cnt;
  77	u16 max, lvl;
  78	u32 pte[NVKM_VMM_LEVELS_MAX];
  79	struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
  80	int flush;
  81};
  82
  83#ifdef CONFIG_NOUVEAU_DEBUG_MMU
  84static const char *
  85nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
  86{
  87	switch (desc->type) {
  88	case PGD: return "PGD";
  89	case PGT: return "PGT";
  90	case SPT: return "SPT";
  91	case LPT: return "LPT";
  92	default:
  93		return "UNKNOWN";
  94	}
  95}
  96
  97static void
  98nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
  99{
 100	int lvl;
 101	for (lvl = it->max; lvl >= 0; lvl--) {
 102		if (lvl >= it->lvl)
 103			buf += sprintf(buf,  "%05x:", it->pte[lvl]);
 104		else
 105			buf += sprintf(buf, "xxxxx:");
 106	}
 107}
 108
 109#define TRA(i,f,a...) do {                                                     \
 110	char _buf[NVKM_VMM_LEVELS_MAX * 7];                                    \
 111	struct nvkm_vmm_iter *_it = (i);                                       \
 112	nvkm_vmm_trace(_it, _buf);                                             \
 113	VMM_TRACE(_it->vmm, "%s "f, _buf, ##a);                                \
 114} while(0)
 115#else
 116#define TRA(i,f,a...)
 117#endif
 118
 119static inline void
 120nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
 121{
 122	it->flush = min(it->flush, it->max - it->lvl);
 123}
 124
 125static inline void
 126nvkm_vmm_flush(struct nvkm_vmm_iter *it)
 127{
 128	if (it->flush != NVKM_VMM_LEVELS_MAX) {
 129		if (it->vmm->func->flush) {
 130			TRA(it, "flush: %d", it->flush);
 131			it->vmm->func->flush(it->vmm, it->flush);
 132		}
 133		it->flush = NVKM_VMM_LEVELS_MAX;
 134	}
 135}
 136
 137static void
 138nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
 139{
 140	const struct nvkm_vmm_desc *desc = it->desc;
 141	const int type = desc[it->lvl].type == SPT;
 142	struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
 143	struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
 144	struct nvkm_mmu_pt *pt = pgt->pt[type];
 145	struct nvkm_vmm *vmm = it->vmm;
 146	u32 pdei = it->pte[it->lvl + 1];
 147
 148	/* Recurse up the tree, unreferencing/destroying unneeded PDs. */
 149	it->lvl++;
 150	if (--pgd->refs[0]) {
 151		const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
 152		/* PD has other valid PDEs, so we need a proper update. */
 153		TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
 154		pgt->pt[type] = NULL;
 155		if (!pgt->refs[!type]) {
 156			/* PDE no longer required. */
 157			if (pgd->pt[0]) {
 158				if (pgt->sparse) {
 159					func->sparse(vmm, pgd->pt[0], pdei, 1);
 160					pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
 161				} else {
 162					func->unmap(vmm, pgd->pt[0], pdei, 1);
 163					pgd->pde[pdei] = NULL;
 164				}
 165			} else {
 166				/* Special handling for Tesla-class GPUs,
 167				 * where there's no central PD, but each
 168				 * instance has its own embedded PD.
 169				 */
 170				func->pde(vmm, pgd, pdei);
 171				pgd->pde[pdei] = NULL;
 172			}
 173		} else {
 174			/* PDE was pointing at dual-PTs and we're removing
 175			 * one of them, leaving the other in place.
 176			 */
 177			func->pde(vmm, pgd, pdei);
 178		}
 179
 180		/* GPU may have cached the PTs, flush before freeing. */
 181		nvkm_vmm_flush_mark(it);
 182		nvkm_vmm_flush(it);
 183	} else {
 184		/* PD has no valid PDEs left, so we can just destroy it. */
 185		nvkm_vmm_unref_pdes(it);
 186	}
 187
 188	/* Destroy PD/PT. */
 189	TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
 190	nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
 191	if (!pgt->refs[!type])
 192		nvkm_vmm_pt_del(&pgt);
 193	it->lvl--;
 194}
 195
 196static void
 197nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
 198		     const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
 199{
 200	const struct nvkm_vmm_desc *pair = it->page[-1].desc;
 201	const u32 sptb = desc->bits - pair->bits;
 202	const u32 sptn = 1 << sptb;
 203	struct nvkm_vmm *vmm = it->vmm;
 204	u32 spti = ptei & (sptn - 1), lpti, pteb;
 205
 206	/* Determine how many SPTEs are being touched under each LPTE,
 207	 * and drop reference counts.
 208	 */
 209	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
 210		const u32 pten = min(sptn - spti, ptes);
 211		pgt->pte[lpti] -= pten;
 212		ptes -= pten;
 213	}
 214
 215	/* We're done here if there's no corresponding LPT. */
 216	if (!pgt->refs[0])
 217		return;
 218
 219	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
 220		/* Skip over any LPTEs that still have valid SPTEs. */
 221		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
 222			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 223				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
 224					break;
 225			}
 226			continue;
 227		}
 228
 229		/* As there's no more non-UNMAPPED SPTEs left in the range
 230		 * covered by a number of LPTEs, the LPTEs once again take
 231		 * control over their address range.
 232		 *
 233		 * Determine how many LPTEs need to transition state.
 234		 */
 235		pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
 236		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 237			if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
 238				break;
 239			pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
 240		}
 241
 242		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
 243			TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
 244			pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
 245		} else
 246		if (pair->func->invalid) {
 247			/* If the MMU supports it, restore the LPTE to the
 248			 * INVALID state to tell the MMU there is no point
 249			 * trying to fetch the corresponding SPTEs.
 250			 */
 251			TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
 252			pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
 253		}
 254	}
 255}
 256
 257static bool
 258nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 259{
 260	const struct nvkm_vmm_desc *desc = it->desc;
 261	const int type = desc->type == SPT;
 262	struct nvkm_vmm_pt *pgt = it->pt[0];
 263	bool dma;
 264
 265	if (pfn) {
 266		/* Need to clear PTE valid bits before we dma_unmap_page(). */
 267		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
 268		if (dma) {
 269			/* GPU may have cached the PT, flush before unmap. */
 270			nvkm_vmm_flush_mark(it);
 271			nvkm_vmm_flush(it);
 272			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
 273		}
 274	}
 275
 276	/* Drop PTE references. */
 277	pgt->refs[type] -= ptes;
 278
 279	/* Dual-PTs need special handling, unless PDE becoming invalid. */
 280	if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
 281		nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
 282
 283	/* PT no longer needed? Destroy it. */
 284	if (!pgt->refs[type]) {
 285		it->lvl++;
 286		TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
 287		it->lvl--;
 288		nvkm_vmm_unref_pdes(it);
 289		return false; /* PTE writes for unmap() not necessary. */
 290	}
 291
 292	return true;
 293}
 294
 295static void
 296nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
 297		   const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
 298{
 299	const struct nvkm_vmm_desc *pair = it->page[-1].desc;
 300	const u32 sptb = desc->bits - pair->bits;
 301	const u32 sptn = 1 << sptb;
 302	struct nvkm_vmm *vmm = it->vmm;
 303	u32 spti = ptei & (sptn - 1), lpti, pteb;
 304
 305	/* Determine how many SPTEs are being touched under each LPTE,
 306	 * and increase reference counts.
 307	 */
 308	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
 309		const u32 pten = min(sptn - spti, ptes);
 310		pgt->pte[lpti] += pten;
 311		ptes -= pten;
 312	}
 313
 314	/* We're done here if there's no corresponding LPT. */
 315	if (!pgt->refs[0])
 316		return;
 317
 318	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
 319		/* Skip over any LPTEs that already have valid SPTEs. */
 320		if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
 321			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 322				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
 323					break;
 324			}
 325			continue;
 326		}
 327
 328		/* As there are now non-UNMAPPED SPTEs in the range covered
 329		 * by a number of LPTEs, we need to transfer control of the
 330		 * address range to the SPTEs.
 331		 *
 332		 * Determine how many LPTEs need to transition state.
 333		 */
 334		pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
 335		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 336			if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
 337				break;
 338			pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
 339		}
 340
 341		if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
 342			const u32 spti = pteb * sptn;
 343			const u32 sptc = ptes * sptn;
 344			/* The entire LPTE is marked as sparse, we need
 345			 * to make sure that the SPTEs are too.
 346			 */
 347			TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
 348			desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
 349			/* Sparse LPTEs prevent SPTEs from being accessed. */
 350			TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
 351			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
 352		} else
 353		if (pair->func->invalid) {
 354			/* MMU supports blocking SPTEs by marking an LPTE
 355			 * as INVALID.  We need to reverse that here.
 356			 */
 357			TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
 358			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
 359		}
 360	}
 361}
 362
 363static bool
 364nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 365{
 366	const struct nvkm_vmm_desc *desc = it->desc;
 367	const int type = desc->type == SPT;
 368	struct nvkm_vmm_pt *pgt = it->pt[0];
 369
 370	/* Take PTE references. */
 371	pgt->refs[type] += ptes;
 372
 373	/* Dual-PTs need special handling. */
 374	if (desc->type == SPT)
 375		nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
 376
 377	return true;
 378}
 379
 380static void
 381nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
 382		     struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
 383{
 384	if (desc->type == PGD) {
 385		while (ptes--)
 386			pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
 387	} else
 388	if (desc->type == LPT) {
 389		memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
 390	}
 391}
 392
 393static bool
 394nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 395{
 396	struct nvkm_vmm_pt *pt = it->pt[0];
 397	if (it->desc->type == PGD)
 398		memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
 399	else
 400	if (it->desc->type == LPT)
 401		memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
 402	return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
 403}
 404
 405static bool
 406nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 407{
 408	nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
 409	return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
 410}
 411
 412static bool
 413nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
 414{
 415	const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
 416	const int type = desc->type == SPT;
 417	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 418	const bool zero = !pgt->sparse && !desc->func->invalid;
 419	struct nvkm_vmm *vmm = it->vmm;
 420	struct nvkm_mmu *mmu = vmm->mmu;
 421	struct nvkm_mmu_pt *pt;
 422	u32 pten = 1 << desc->bits;
 423	u32 pteb, ptei, ptes;
 424	u32 size = desc->size * pten;
 425
 426	pgd->refs[0]++;
 427
 428	pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
 429	if (!pgt->pt[type]) {
 430		it->lvl--;
 431		nvkm_vmm_unref_pdes(it);
 432		return false;
 433	}
 434
 435	if (zero)
 436		goto done;
 437
 438	pt = pgt->pt[type];
 439
 440	if (desc->type == LPT && pgt->refs[1]) {
 441		/* SPT already exists covering the same range as this LPT,
 442		 * which means we need to be careful that any LPTEs which
 443		 * overlap valid SPTEs are unmapped as opposed to invalid
 444		 * or sparse, which would prevent the MMU from looking at
 445		 * the SPTEs on some GPUs.
 446		 */
 447		for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
 448			bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
 449			for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
 450				bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
 451				if (spte != next)
 452					break;
 453			}
 454
 455			if (!spte) {
 456				if (pgt->sparse)
 457					desc->func->sparse(vmm, pt, pteb, ptes);
 458				else
 459					desc->func->invalid(vmm, pt, pteb, ptes);
 460				memset(&pgt->pte[pteb], 0x00, ptes);
 461			} else {
 462				desc->func->unmap(vmm, pt, pteb, ptes);
 463				while (ptes--)
 464					pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
 465			}
 466		}
 467	} else {
 468		if (pgt->sparse) {
 469			nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
 470			desc->func->sparse(vmm, pt, 0, pten);
 471		} else {
 472			desc->func->invalid(vmm, pt, 0, pten);
 473		}
 474	}
 475
 476done:
 477	TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
 478	it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
 479	nvkm_vmm_flush_mark(it);
 480	return true;
 481}
 482
 483static bool
 484nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
 485{
 486	const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
 487	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 488
 489	pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
 490	if (!pgt) {
 491		if (!pgd->refs[0])
 492			nvkm_vmm_unref_pdes(it);
 493		return false;
 494	}
 495
 496	pgd->pde[pdei] = pgt;
 497	return true;
 498}
 499
 500static inline u64
 501nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 502	      u64 addr, u64 size, const char *name, bool ref, bool pfn,
 503	      bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
 504	      nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
 505	      nvkm_vmm_pxe_func CLR_PTES)
 506{
 507	const struct nvkm_vmm_desc *desc = page->desc;
 508	struct nvkm_vmm_iter it;
 509	u64 bits = addr >> page->shift;
 510
 511	it.page = page;
 512	it.desc = desc;
 513	it.vmm = vmm;
 514	it.cnt = size >> page->shift;
 515	it.flush = NVKM_VMM_LEVELS_MAX;
 516
 517	/* Deconstruct address into PTE indices for each mapping level. */
 518	for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
 519		it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
 520		bits >>= desc[it.lvl].bits;
 521	}
 522	it.max = --it.lvl;
 523	it.pt[it.max] = vmm->pd;
 524
 525	it.lvl = 0;
 526	TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
 527	         addr, size, page->shift, it.cnt);
 528	it.lvl = it.max;
 529
 530	/* Depth-first traversal of page tables. */
 531	while (it.cnt) {
 532		struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
 533		const int type = desc->type == SPT;
 534		const u32 pten = 1 << desc->bits;
 535		const u32 ptei = it.pte[0];
 536		const u32 ptes = min_t(u64, it.cnt, pten - ptei);
 537
 538		/* Walk down the tree, finding page tables for each level. */
 539		for (; it.lvl; it.lvl--) {
 540			const u32 pdei = it.pte[it.lvl];
 541			struct nvkm_vmm_pt *pgd = pgt;
 542
 543			/* Software PT. */
 544			if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
 545				if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
 546					goto fail;
 547			}
 548			it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
 549
 550			/* Hardware PT.
 551			 *
 552			 * This is a separate step from above due to GF100 and
 553			 * newer having dual page tables at some levels, which
 554			 * are refcounted independently.
 555			 */
 556			if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
 557				if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
 558					goto fail;
 559			}
 560		}
 561
 562		/* Handle PTE updates. */
 563		if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
 564			struct nvkm_mmu_pt *pt = pgt->pt[type];
 565			if (MAP_PTES || CLR_PTES) {
 566				if (MAP_PTES)
 567					MAP_PTES(vmm, pt, ptei, ptes, map);
 568				else
 569					CLR_PTES(vmm, pt, ptei, ptes);
 570				nvkm_vmm_flush_mark(&it);
 571			}
 572		}
 573
 574		/* Walk back up the tree to the next position. */
 575		it.pte[it.lvl] += ptes;
 576		it.cnt -= ptes;
 577		if (it.cnt) {
 578			while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
 579				it.pte[it.lvl++] = 0;
 580				it.pte[it.lvl]++;
 581			}
 582		}
 583	}
 584
 585	nvkm_vmm_flush(&it);
 586	return ~0ULL;
 587
 588fail:
 589	/* Reconstruct the failure address so the caller is able to
 590	 * reverse any partially completed operations.
 591	 */
 592	addr = it.pte[it.max--];
 593	do {
 594		addr  = addr << desc[it.max].bits;
 595		addr |= it.pte[it.max];
 596	} while (it.max--);
 597
 598	return addr << page->shift;
 599}
 600
 601static void
 602nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 603			 u64 addr, u64 size)
 604{
 605	nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
 606		      nvkm_vmm_sparse_unref_ptes, NULL, NULL,
 607		      page->desc->func->invalid ?
 608		      page->desc->func->invalid : page->desc->func->unmap);
 609}
 610
 611static int
 612nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 613			 u64 addr, u64 size)
 614{
 615	if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
 616		u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
 617					 true, false, nvkm_vmm_sparse_ref_ptes,
 618					 NULL, NULL, page->desc->func->sparse);
 619		if (fail != ~0ULL) {
 620			if ((size = fail - addr))
 621				nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
 622			return -ENOMEM;
 623		}
 624		return 0;
 625	}
 626	return -EINVAL;
 627}
 628
 629static int
 630nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
 631{
 632	const struct nvkm_vmm_page *page = vmm->func->page;
 633	int m = 0, i;
 634	u64 start = addr;
 635	u64 block;
 636
 637	while (size) {
 638		/* Limit maximum page size based on remaining size. */
 639		while (size < (1ULL << page[m].shift))
 640			m++;
 641		i = m;
 642
 643		/* Find largest page size suitable for alignment. */
 644		while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
 645			i++;
 646
 647		/* Determine number of PTEs at this page size. */
 648		if (i != m) {
 649			/* Limited to alignment boundary of next page size. */
 650			u64 next = 1ULL << page[i - 1].shift;
 651			u64 part = ALIGN(addr, next) - addr;
 652			if (size - part >= next)
 653				block = (part >> page[i].shift) << page[i].shift;
 654			else
 655				block = (size >> page[i].shift) << page[i].shift;
 656		} else {
 657			block = (size >> page[i].shift) << page[i].shift;
 658		}
 659
 660		/* Perform operation. */
 661		if (ref) {
 662			int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
 663			if (ret) {
 664				if ((size = addr - start))
 665					nvkm_vmm_ptes_sparse(vmm, start, size, false);
 666				return ret;
 667			}
 668		} else {
 669			nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
 670		}
 671
 672		size -= block;
 673		addr += block;
 674	}
 675
 676	return 0;
 677}
 678
 679static void
 680nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 681		    u64 addr, u64 size, bool sparse, bool pfn)
 682{
 683	const struct nvkm_vmm_desc_func *func = page->desc->func;
 684
 685	mutex_lock(&vmm->mutex.map);
 686	nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
 687		      NULL, NULL, NULL,
 688		      sparse ? func->sparse : func->invalid ? func->invalid :
 689							      func->unmap);
 690	mutex_unlock(&vmm->mutex.map);
 691}
 692
 693static void
 694nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 695		  u64 addr, u64 size, struct nvkm_vmm_map *map,
 696		  nvkm_vmm_pte_func func)
 697{
 698	mutex_lock(&vmm->mutex.map);
 699	nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
 700		      NULL, func, map, NULL);
 701	mutex_unlock(&vmm->mutex.map);
 702}
 703
 704static void
 705nvkm_vmm_ptes_put_locked(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 706			 u64 addr, u64 size)
 707{
 708	nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
 709		      nvkm_vmm_unref_ptes, NULL, NULL, NULL);
 710}
 711
 712static void
 713nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 714		  u64 addr, u64 size)
 715{
 716	mutex_lock(&vmm->mutex.ref);
 717	nvkm_vmm_ptes_put_locked(vmm, page, addr, size);
 718	mutex_unlock(&vmm->mutex.ref);
 719}
 720
 721static int
 722nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 723		  u64 addr, u64 size)
 724{
 725	u64 fail;
 726
 727	mutex_lock(&vmm->mutex.ref);
 728	fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
 729			     nvkm_vmm_ref_ptes, NULL, NULL, NULL);
 730	if (fail != ~0ULL) {
 731		if (fail != addr)
 732			nvkm_vmm_ptes_put_locked(vmm, page, addr, fail - addr);
 733		mutex_unlock(&vmm->mutex.ref);
 734		return -ENOMEM;
 735	}
 736	mutex_unlock(&vmm->mutex.ref);
 737	return 0;
 738}
 739
 740static void
 741__nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 742			  u64 addr, u64 size, bool sparse, bool pfn)
 743{
 744	const struct nvkm_vmm_desc_func *func = page->desc->func;
 745
 746	nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
 747		      false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
 748		      sparse ? func->sparse : func->invalid ? func->invalid :
 749							      func->unmap);
 750}
 751
 752static void
 753nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 754			u64 addr, u64 size, bool sparse, bool pfn)
 755{
 756	if (vmm->managed.raw) {
 757		nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, pfn);
 758		nvkm_vmm_ptes_put(vmm, page, addr, size);
 759	} else {
 760		__nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, sparse, pfn);
 761	}
 762}
 763
 764static int
 765__nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 766			u64 addr, u64 size, struct nvkm_vmm_map *map,
 767			nvkm_vmm_pte_func func)
 768{
 769	u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
 770				 false, nvkm_vmm_ref_ptes, func, map, NULL);
 771	if (fail != ~0ULL) {
 772		if ((size = fail - addr))
 773			nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
 774		return -ENOMEM;
 775	}
 776	return 0;
 777}
 778
 779static int
 780nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 781		      u64 addr, u64 size, struct nvkm_vmm_map *map,
 782		      nvkm_vmm_pte_func func)
 783{
 784	int ret;
 785
 786	if (vmm->managed.raw) {
 787		ret = nvkm_vmm_ptes_get(vmm, page, addr, size);
 788		if (ret)
 789			return ret;
 790
 791		nvkm_vmm_ptes_map(vmm, page, addr, size, map, func);
 792
 793		return 0;
 794	} else {
 795		return __nvkm_vmm_ptes_get_map(vmm, page, addr, size, map, func);
 796	}
 797}
 798
 799struct nvkm_vma *
 800nvkm_vma_new(u64 addr, u64 size)
 801{
 802	struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 803	if (vma) {
 804		vma->addr = addr;
 805		vma->size = size;
 806		vma->page = NVKM_VMA_PAGE_NONE;
 807		vma->refd = NVKM_VMA_PAGE_NONE;
 808	}
 809	return vma;
 810}
 811
 812struct nvkm_vma *
 813nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
 814{
 815	struct nvkm_vma *new;
 816
 817	BUG_ON(vma->size == tail);
 818
 819	if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
 820		return NULL;
 821	vma->size -= tail;
 822
 823	new->mapref = vma->mapref;
 824	new->sparse = vma->sparse;
 825	new->page = vma->page;
 826	new->refd = vma->refd;
 827	new->used = vma->used;
 828	new->part = vma->part;
 829	new->busy = vma->busy;
 830	new->mapped = vma->mapped;
 831	list_add(&new->head, &vma->head);
 832	return new;
 833}
 834
 835static inline void
 836nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 837{
 838	rb_erase(&vma->tree, &vmm->free);
 839}
 840
 841static inline void
 842nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 843{
 844	nvkm_vmm_free_remove(vmm, vma);
 845	list_del(&vma->head);
 846	kfree(vma);
 847}
 848
 849static void
 850nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 851{
 852	struct rb_node **ptr = &vmm->free.rb_node;
 853	struct rb_node *parent = NULL;
 854
 855	while (*ptr) {
 856		struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
 857		parent = *ptr;
 858		if (vma->size < this->size)
 859			ptr = &parent->rb_left;
 860		else
 861		if (vma->size > this->size)
 862			ptr = &parent->rb_right;
 863		else
 864		if (vma->addr < this->addr)
 865			ptr = &parent->rb_left;
 866		else
 867		if (vma->addr > this->addr)
 868			ptr = &parent->rb_right;
 869		else
 870			BUG();
 871	}
 872
 873	rb_link_node(&vma->tree, parent, ptr);
 874	rb_insert_color(&vma->tree, &vmm->free);
 875}
 876
 877static inline void
 878nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 879{
 880	rb_erase(&vma->tree, &vmm->root);
 881}
 882
 883static inline void
 884nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 885{
 886	nvkm_vmm_node_remove(vmm, vma);
 887	list_del(&vma->head);
 888	kfree(vma);
 889}
 890
 891static void
 892nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 893{
 894	struct rb_node **ptr = &vmm->root.rb_node;
 895	struct rb_node *parent = NULL;
 896
 897	while (*ptr) {
 898		struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
 899		parent = *ptr;
 900		if (vma->addr < this->addr)
 901			ptr = &parent->rb_left;
 902		else
 903		if (vma->addr > this->addr)
 904			ptr = &parent->rb_right;
 905		else
 906			BUG();
 907	}
 908
 909	rb_link_node(&vma->tree, parent, ptr);
 910	rb_insert_color(&vma->tree, &vmm->root);
 911}
 912
 913struct nvkm_vma *
 914nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
 915{
 916	struct rb_node *node = vmm->root.rb_node;
 917	while (node) {
 918		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
 919		if (addr < vma->addr)
 920			node = node->rb_left;
 921		else
 922		if (addr >= vma->addr + vma->size)
 923			node = node->rb_right;
 924		else
 925			return vma;
 926	}
 927	return NULL;
 928}
 929
 930#define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL :             \
 931	list_entry((root)->head.dir, struct nvkm_vma, head))
 932
 933static struct nvkm_vma *
 934nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
 935		    struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
 936{
 937	if (next) {
 938		if (vma->size == size) {
 939			vma->size += next->size;
 940			nvkm_vmm_node_delete(vmm, next);
 941			if (prev) {
 942				prev->size += vma->size;
 943				nvkm_vmm_node_delete(vmm, vma);
 944				return prev;
 945			}
 946			return vma;
 947		}
 948		BUG_ON(prev);
 949
 950		nvkm_vmm_node_remove(vmm, next);
 951		vma->size -= size;
 952		next->addr -= size;
 953		next->size += size;
 954		nvkm_vmm_node_insert(vmm, next);
 955		return next;
 956	}
 957
 958	if (prev) {
 959		if (vma->size != size) {
 960			nvkm_vmm_node_remove(vmm, vma);
 961			prev->size += size;
 962			vma->addr += size;
 963			vma->size -= size;
 964			nvkm_vmm_node_insert(vmm, vma);
 965		} else {
 966			prev->size += vma->size;
 967			nvkm_vmm_node_delete(vmm, vma);
 968		}
 969		return prev;
 970	}
 971
 972	return vma;
 973}
 974
 975struct nvkm_vma *
 976nvkm_vmm_node_split(struct nvkm_vmm *vmm,
 977		    struct nvkm_vma *vma, u64 addr, u64 size)
 978{
 979	struct nvkm_vma *prev = NULL;
 980
 981	if (vma->addr != addr) {
 982		prev = vma;
 983		if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
 984			return NULL;
 985		vma->part = true;
 986		nvkm_vmm_node_insert(vmm, vma);
 987	}
 988
 989	if (vma->size != size) {
 990		struct nvkm_vma *tmp;
 991		if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
 992			nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
 993			return NULL;
 994		}
 995		tmp->part = true;
 996		nvkm_vmm_node_insert(vmm, tmp);
 997	}
 998
 999	return vma;
1000}
1001
1002static void
1003nvkm_vma_dump(struct nvkm_vma *vma)
1004{
1005	printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n",
1006	       vma->addr, (u64)vma->size,
1007	       vma->used ? '-' : 'F',
1008	       vma->mapref ? 'R' : '-',
1009	       vma->sparse ? 'S' : '-',
1010	       vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
1011	       vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
1012	       vma->part ? 'P' : '-',
1013	       vma->busy ? 'B' : '-',
1014	       vma->mapped ? 'M' : '-',
1015	       vma->memory);
1016}
1017
1018static void
1019nvkm_vmm_dump(struct nvkm_vmm *vmm)
1020{
1021	struct nvkm_vma *vma;
1022	list_for_each_entry(vma, &vmm->list, head) {
1023		nvkm_vma_dump(vma);
1024	}
1025}
1026
1027static void
1028nvkm_vmm_dtor(struct nvkm_vmm *vmm)
1029{
1030	struct nvkm_vma *vma;
1031	struct rb_node *node;
1032
1033	if (vmm->rm.client.gsp)
1034		r535_mmu_vaspace_del(vmm);
1035
1036	if (0)
1037		nvkm_vmm_dump(vmm);
1038
1039	while ((node = rb_first(&vmm->root))) {
1040		struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
1041		nvkm_vmm_put(vmm, &vma);
1042	}
1043
1044	if (vmm->bootstrapped) {
1045		const struct nvkm_vmm_page *page = vmm->func->page;
1046		const u64 limit = vmm->limit - vmm->start;
1047
1048		while (page[1].shift)
1049			page++;
1050
1051		nvkm_mmu_ptc_dump(vmm->mmu);
1052		nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
1053	}
1054
1055	vma = list_first_entry(&vmm->list, typeof(*vma), head);
1056	list_del(&vma->head);
1057	kfree(vma);
1058	WARN_ON(!list_empty(&vmm->list));
1059
1060	if (vmm->nullp) {
1061		dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
1062				  vmm->nullp, vmm->null);
1063	}
1064
1065	if (vmm->pd) {
1066		nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1067		nvkm_vmm_pt_del(&vmm->pd);
1068	}
1069}
1070
1071static int
1072nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
1073{
1074	struct nvkm_vma *vma;
1075	if (!(vma = nvkm_vma_new(addr, size)))
1076		return -ENOMEM;
1077	vma->mapref = true;
1078	vma->sparse = false;
1079	vma->used = true;
1080	nvkm_vmm_node_insert(vmm, vma);
1081	list_add_tail(&vma->head, &vmm->list);
1082	return 0;
1083}
1084
1085static int
1086nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1087	      u32 pd_header, bool managed, u64 addr, u64 size,
1088	      struct lock_class_key *key, const char *name,
1089	      struct nvkm_vmm *vmm)
1090{
1091	static struct lock_class_key _key;
1092	const struct nvkm_vmm_page *page = func->page;
1093	const struct nvkm_vmm_desc *desc;
1094	struct nvkm_vma *vma;
1095	int levels, bits = 0, ret;
1096
1097	vmm->func = func;
1098	vmm->mmu = mmu;
1099	vmm->name = name;
1100	vmm->debug = mmu->subdev.debug;
1101	kref_init(&vmm->kref);
1102
1103	__mutex_init(&vmm->mutex.vmm, "&vmm->mutex.vmm", key ? key : &_key);
1104	mutex_init(&vmm->mutex.ref);
1105	mutex_init(&vmm->mutex.map);
1106
1107	/* Locate the smallest page size supported by the backend, it will
1108	 * have the deepest nesting of page tables.
1109	 */
1110	while (page[1].shift)
1111		page++;
1112
1113	/* Locate the structure that describes the layout of the top-level
1114	 * page table, and determine the number of valid bits in a virtual
1115	 * address.
1116	 */
1117	for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1118		bits += desc->bits;
1119	bits += page->shift;
1120	desc--;
1121
1122	if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1123		return -EINVAL;
1124
1125	/* Allocate top-level page table. */
1126	vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1127	if (!vmm->pd)
1128		return -ENOMEM;
1129	vmm->pd->refs[0] = 1;
1130	INIT_LIST_HEAD(&vmm->join);
1131
1132	/* ... and the GPU storage for it, except on Tesla-class GPUs that
1133	 * have the PD embedded in the instance structure.
1134	 */
1135	if (desc->size) {
1136		const u32 size = pd_header + desc->size * (1 << desc->bits);
1137		vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1138		if (!vmm->pd->pt[0])
1139			return -ENOMEM;
1140	}
1141
1142	/* Initialise address-space MM. */
1143	INIT_LIST_HEAD(&vmm->list);
1144	vmm->free = RB_ROOT;
1145	vmm->root = RB_ROOT;
1146
1147	if (managed) {
1148		/* Address-space will be managed by the client for the most
1149		 * part, except for a specified area where NVKM allocations
1150		 * are allowed to be placed.
1151		 */
1152		vmm->start = 0;
1153		vmm->limit = 1ULL << bits;
1154		if (addr + size < addr || addr + size > vmm->limit)
1155			return -EINVAL;
1156
1157		/* Client-managed area before the NVKM-managed area. */
1158		if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
1159			return ret;
1160
1161		vmm->managed.p.addr = 0;
1162		vmm->managed.p.size = addr;
1163
1164		/* NVKM-managed area. */
1165		if (size) {
1166			if (!(vma = nvkm_vma_new(addr, size)))
1167				return -ENOMEM;
1168			nvkm_vmm_free_insert(vmm, vma);
1169			list_add_tail(&vma->head, &vmm->list);
1170		}
1171
1172		/* Client-managed area after the NVKM-managed area. */
1173		addr = addr + size;
1174		size = vmm->limit - addr;
1175		if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
1176			return ret;
1177
1178		vmm->managed.n.addr = addr;
1179		vmm->managed.n.size = size;
1180	} else {
1181		/* Address-space fully managed by NVKM, requiring calls to
1182		 * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space.
1183		 */
1184		vmm->start = addr;
1185		vmm->limit = size ? (addr + size) : (1ULL << bits);
1186		if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
1187			return -EINVAL;
1188
1189		if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1190			return -ENOMEM;
1191
1192		nvkm_vmm_free_insert(vmm, vma);
1193		list_add(&vma->head, &vmm->list);
1194	}
1195
1196	return 0;
1197}
1198
1199int
1200nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1201	      u32 hdr, bool managed, u64 addr, u64 size,
1202	      struct lock_class_key *key, const char *name,
1203	      struct nvkm_vmm **pvmm)
1204{
1205	if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1206		return -ENOMEM;
1207	return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1208}
1209
1210static struct nvkm_vma *
1211nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1212			 u64 addr, u64 size, u8 page, bool map)
1213{
1214	struct nvkm_vma *prev = NULL;
1215	struct nvkm_vma *next = NULL;
1216
1217	if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1218		if (prev->memory || prev->mapped != map)
1219			prev = NULL;
1220	}
1221
1222	if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1223		if (!next->part ||
1224		    next->memory || next->mapped != map)
1225			next = NULL;
1226	}
1227
1228	if (prev || next)
1229		return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1230	return nvkm_vmm_node_split(vmm, vma, addr, size);
1231}
1232
1233int
1234nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1235{
1236	struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1237	struct nvkm_vma *next;
1238	u64 limit = addr + size;
1239	u64 start = addr;
1240
1241	if (!vma)
1242		return -EINVAL;
1243
1244	do {
1245		if (!vma->mapped || vma->memory)
1246			continue;
1247
1248		size = min(limit - start, vma->size - (start - vma->addr));
1249
1250		nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1251					start, size, false, true);
1252
1253		next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1254		if (!WARN_ON(!next)) {
1255			vma = next;
1256			vma->refd = NVKM_VMA_PAGE_NONE;
1257			vma->mapped = false;
1258		}
1259	} while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1260
1261	return 0;
1262}
1263
1264/*TODO:
1265 * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1266 *   with inside HMM, which would be a lot nicer for us to deal with.
1267 * - Support for systems without a 4KiB page size.
1268 */
1269int
1270nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1271{
1272	const struct nvkm_vmm_page *page = vmm->func->page;
1273	struct nvkm_vma *vma, *tmp;
1274	u64 limit = addr + size;
1275	u64 start = addr;
1276	int pm = size >> shift;
1277	int pi = 0;
1278
1279	/* Only support mapping where the page size of the incoming page
1280	 * array matches a page size available for direct mapping.
1281	 */
1282	while (page->shift && (page->shift != shift ||
1283	       page->desc->func->pfn == NULL))
1284		page++;
1285
1286	if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1287			    !IS_ALIGNED(size, 1ULL << shift) ||
1288	    addr + size < addr || addr + size > vmm->limit) {
1289		VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1290			  shift, page->shift, addr, size);
1291		return -EINVAL;
1292	}
1293
1294	if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1295		return -ENOENT;
1296
1297	do {
1298		bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1299		bool mapped = vma->mapped;
1300		u64 size = limit - start;
1301		u64 addr = start;
1302		int pn, ret = 0;
1303
1304		/* Narrow the operation window to cover a single action (page
1305		 * should be mapped or not) within a single VMA.
1306		 */
1307		for (pn = 0; pi + pn < pm; pn++) {
1308			if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1309				break;
1310		}
1311		size = min_t(u64, size, pn << page->shift);
1312		size = min_t(u64, size, vma->size + vma->addr - addr);
1313
1314		/* Reject any operation to unmanaged regions, and areas that
1315		 * have nvkm_memory objects mapped in them already.
1316		 */
1317		if (!vma->mapref || vma->memory) {
1318			ret = -EINVAL;
1319			goto next;
1320		}
1321
1322		/* In order to both properly refcount GPU page tables, and
1323		 * prevent "normal" mappings and these direct mappings from
1324		 * interfering with each other, we need to track contiguous
1325		 * ranges that have been mapped with this interface.
1326		 *
1327		 * Here we attempt to either split an existing VMA so we're
1328		 * able to flag the region as either unmapped/mapped, or to
1329		 * merge with adjacent VMAs that are already compatible.
1330		 *
1331		 * If the region is already compatible, nothing is required.
1332		 */
1333		if (map != mapped) {
1334			tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1335						       page -
1336						       vmm->func->page, map);
1337			if (WARN_ON(!tmp)) {
1338				ret = -ENOMEM;
1339				goto next;
1340			}
1341
1342			if ((tmp->mapped = map))
1343				tmp->refd = page - vmm->func->page;
1344			else
1345				tmp->refd = NVKM_VMA_PAGE_NONE;
1346			vma = tmp;
1347		}
1348
1349		/* Update HW page tables. */
1350		if (map) {
1351			struct nvkm_vmm_map args;
1352			args.page = page;
1353			args.pfn = &pfn[pi];
1354
1355			if (!mapped) {
1356				ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1357							    size, &args, page->
1358							    desc->func->pfn);
1359			} else {
1360				nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1361						  page->desc->func->pfn);
1362			}
1363		} else {
1364			if (mapped) {
1365				nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1366							false, true);
1367			}
1368		}
1369
1370next:
1371		/* Iterate to next operation. */
1372		if (vma->addr + vma->size == addr + size)
1373			vma = node(vma, next);
1374		start += size;
1375
1376		if (ret) {
1377			/* Failure is signalled by clearing the valid bit on
1378			 * any PFN that couldn't be modified as requested.
1379			 */
1380			while (size) {
1381				pfn[pi++] = NVKM_VMM_PFN_NONE;
1382				size -= 1 << page->shift;
1383			}
1384		} else {
1385			pi += size >> page->shift;
1386		}
1387	} while (vma && start < limit);
1388
1389	return 0;
1390}
1391
1392void
1393nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1394{
1395	struct nvkm_vma *prev = NULL;
1396	struct nvkm_vma *next;
1397
1398	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1399	nvkm_memory_unref(&vma->memory);
1400	vma->mapped = false;
1401
1402	if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1403		prev = NULL;
1404	if ((next = node(vma, next)) && (!next->part || next->mapped))
1405		next = NULL;
1406	nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1407}
1408
1409void
1410nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1411{
1412	const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1413
1414	if (vma->mapref) {
1415		nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1416		vma->refd = NVKM_VMA_PAGE_NONE;
1417	} else {
1418		nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1419	}
1420
1421	nvkm_vmm_unmap_region(vmm, vma);
1422}
1423
1424void
1425nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1426{
1427	if (vma->memory) {
1428		mutex_lock(&vmm->mutex.vmm);
1429		nvkm_vmm_unmap_locked(vmm, vma, false);
1430		mutex_unlock(&vmm->mutex.vmm);
1431	}
1432}
1433
1434static int
1435nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1436		   void *argv, u32 argc, struct nvkm_vmm_map *map)
1437{
1438	switch (nvkm_memory_target(map->memory)) {
1439	case NVKM_MEM_TARGET_VRAM:
1440		if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1441			VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1442			return -EINVAL;
1443		}
1444		break;
1445	case NVKM_MEM_TARGET_HOST:
1446	case NVKM_MEM_TARGET_NCOH:
1447		if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1448			VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1449			return -EINVAL;
1450		}
1451		break;
1452	default:
1453		WARN_ON(1);
1454		return -ENOSYS;
1455	}
1456
1457	if (!IS_ALIGNED(     vma->addr, 1ULL << map->page->shift) ||
1458	    !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1459	    !IS_ALIGNED(   map->offset, 1ULL << map->page->shift) ||
1460	    nvkm_memory_page(map->memory) < map->page->shift) {
1461		VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1462		    vma->addr, (u64)vma->size, map->offset, map->page->shift,
1463		    nvkm_memory_page(map->memory));
1464		return -EINVAL;
1465	}
1466
1467	return vmm->func->valid(vmm, argv, argc, map);
1468}
1469
1470static int
1471nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1472		    void *argv, u32 argc, struct nvkm_vmm_map *map)
1473{
1474	for (map->page = vmm->func->page; map->page->shift; map->page++) {
1475		VMM_DEBUG(vmm, "trying %d", map->page->shift);
1476		if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1477			return 0;
1478	}
1479	return -EINVAL;
1480}
1481
1482static int
1483nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1484		    void *argv, u32 argc, struct nvkm_vmm_map *map)
1485{
1486	nvkm_vmm_pte_func func;
1487	int ret;
1488
1489	map->no_comp = vma->no_comp;
1490
1491	/* Make sure we won't overrun the end of the memory object. */
1492	if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1493		VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1494			  nvkm_memory_size(map->memory),
1495			  map->offset, (u64)vma->size);
1496		return -EINVAL;
1497	}
1498
1499	/* Check remaining arguments for validity. */
1500	if (vma->page == NVKM_VMA_PAGE_NONE &&
1501	    vma->refd == NVKM_VMA_PAGE_NONE) {
1502		/* Find the largest page size we can perform the mapping at. */
1503		const u32 debug = vmm->debug;
1504		vmm->debug = 0;
1505		ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1506		vmm->debug = debug;
1507		if (ret) {
1508			VMM_DEBUG(vmm, "invalid at any page size");
1509			nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1510			return -EINVAL;
1511		}
1512	} else {
1513		/* Page size of the VMA is already pre-determined. */
1514		if (vma->refd != NVKM_VMA_PAGE_NONE)
1515			map->page = &vmm->func->page[vma->refd];
1516		else
1517			map->page = &vmm->func->page[vma->page];
1518
1519		ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1520		if (ret) {
1521			VMM_DEBUG(vmm, "invalid %d\n", ret);
1522			return ret;
1523		}
1524	}
1525
1526	/* Deal with the 'offset' argument, and fetch the backend function. */
1527	map->off = map->offset;
1528	if (map->mem) {
1529		for (; map->off; map->mem = map->mem->next) {
1530			u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1531			if (size > map->off)
1532				break;
1533			map->off -= size;
1534		}
1535		func = map->page->desc->func->mem;
1536	} else
1537	if (map->sgl) {
1538		for (; map->off; map->sgl = sg_next(map->sgl)) {
1539			u64 size = sg_dma_len(map->sgl);
1540			if (size > map->off)
1541				break;
1542			map->off -= size;
1543		}
1544		func = map->page->desc->func->sgl;
1545	} else {
1546		map->dma += map->offset >> PAGE_SHIFT;
1547		map->off  = map->offset & PAGE_MASK;
1548		func = map->page->desc->func->dma;
1549	}
1550
1551	/* Perform the map. */
1552	if (vma->refd == NVKM_VMA_PAGE_NONE) {
1553		ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1554		if (ret)
1555			return ret;
1556
1557		vma->refd = map->page - vmm->func->page;
1558	} else {
1559		nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1560	}
1561
1562	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1563	nvkm_memory_unref(&vma->memory);
1564	vma->memory = nvkm_memory_ref(map->memory);
1565	vma->mapped = true;
1566	vma->tags = map->tags;
1567	return 0;
1568}
1569
1570int
1571nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1572	     struct nvkm_vmm_map *map)
1573{
1574	int ret;
1575
1576	if (nvkm_vmm_in_managed_range(vmm, vma->addr, vma->size) &&
1577	    vmm->managed.raw)
1578		return nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1579
1580	mutex_lock(&vmm->mutex.vmm);
1581	ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1582	vma->busy = false;
1583	mutex_unlock(&vmm->mutex.vmm);
1584	return ret;
1585}
1586
1587static void
1588nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1589{
1590	struct nvkm_vma *prev, *next;
1591
1592	if ((prev = node(vma, prev)) && !prev->used) {
1593		vma->addr  = prev->addr;
1594		vma->size += prev->size;
1595		nvkm_vmm_free_delete(vmm, prev);
1596	}
1597
1598	if ((next = node(vma, next)) && !next->used) {
1599		vma->size += next->size;
1600		nvkm_vmm_free_delete(vmm, next);
1601	}
1602
1603	nvkm_vmm_free_insert(vmm, vma);
1604}
1605
1606void
1607nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1608{
1609	const struct nvkm_vmm_page *page = vmm->func->page;
1610	struct nvkm_vma *next = vma;
1611
1612	BUG_ON(vma->part);
1613
1614	if (vma->mapref || !vma->sparse) {
1615		do {
1616			const bool mem = next->memory != NULL;
1617			const bool map = next->mapped;
1618			const u8  refd = next->refd;
1619			const u64 addr = next->addr;
1620			u64 size = next->size;
1621
1622			/* Merge regions that are in the same state. */
1623			while ((next = node(next, next)) && next->part &&
1624			       (next->mapped == map) &&
1625			       (next->memory != NULL) == mem &&
1626			       (next->refd == refd))
1627				size += next->size;
1628
1629			if (map) {
1630				/* Region(s) are mapped, merge the unmap
1631				 * and dereference into a single walk of
1632				 * the page tree.
1633				 */
1634				nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1635							size, vma->sparse,
1636							!mem);
1637			} else
1638			if (refd != NVKM_VMA_PAGE_NONE) {
1639				/* Drop allocation-time PTE references. */
1640				nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1641			}
1642		} while (next && next->part);
1643	}
1644
1645	/* Merge any mapped regions that were split from the initial
1646	 * address-space allocation back into the allocated VMA, and
1647	 * release memory/compression resources.
1648	 */
1649	next = vma;
1650	do {
1651		if (next->mapped)
1652			nvkm_vmm_unmap_region(vmm, next);
1653	} while ((next = node(vma, next)) && next->part);
1654
1655	if (vma->sparse && !vma->mapref) {
1656		/* Sparse region that was allocated with a fixed page size,
1657		 * meaning all relevant PTEs were referenced once when the
1658		 * region was allocated, and remained that way, regardless
1659		 * of whether memory was mapped into it afterwards.
1660		 *
1661		 * The process of unmapping, unsparsing, and dereferencing
1662		 * PTEs can be done in a single page tree walk.
1663		 */
1664		nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1665	} else
1666	if (vma->sparse) {
1667		/* Sparse region that wasn't allocated with a fixed page size,
1668		 * PTE references were taken both at allocation time (to make
1669		 * the GPU see the region as sparse), and when mapping memory
1670		 * into the region.
1671		 *
1672		 * The latter was handled above, and the remaining references
1673		 * are dealt with here.
1674		 */
1675		nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1676	}
1677
1678	/* Remove VMA from the list of allocated nodes. */
1679	nvkm_vmm_node_remove(vmm, vma);
1680
1681	/* Merge VMA back into the free list. */
1682	vma->page = NVKM_VMA_PAGE_NONE;
1683	vma->refd = NVKM_VMA_PAGE_NONE;
1684	vma->used = false;
1685	nvkm_vmm_put_region(vmm, vma);
1686}
1687
1688void
1689nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1690{
1691	struct nvkm_vma *vma = *pvma;
1692	if (vma) {
1693		mutex_lock(&vmm->mutex.vmm);
1694		nvkm_vmm_put_locked(vmm, vma);
1695		mutex_unlock(&vmm->mutex.vmm);
1696		*pvma = NULL;
1697	}
1698}
1699
1700int
1701nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1702		    u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1703{
1704	const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1705	struct rb_node *node = NULL, *temp;
1706	struct nvkm_vma *vma = NULL, *tmp;
1707	u64 addr, tail;
1708	int ret;
1709
1710	VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1711		       "shift: %d align: %d size: %016llx",
1712		  getref, mapref, sparse, shift, align, size);
1713
1714	/* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1715	if (unlikely(!size || (!getref && !mapref && sparse))) {
1716		VMM_DEBUG(vmm, "args %016llx %d %d %d",
1717			  size, getref, mapref, sparse);
1718		return -EINVAL;
1719	}
1720
1721	/* Tesla-class GPUs can only select page size per-PDE, which means
1722	 * we're required to know the mapping granularity up-front to find
1723	 * a suitable region of address-space.
1724	 *
1725	 * The same goes if we're requesting up-front allocation of PTES.
1726	 */
1727	if (unlikely((getref || vmm->func->page_block) && !shift)) {
1728		VMM_DEBUG(vmm, "page size required: %d %016llx",
1729			  getref, vmm->func->page_block);
1730		return -EINVAL;
1731	}
1732
1733	/* If a specific page size was requested, determine its index and
1734	 * make sure the requested size is a multiple of the page size.
1735	 */
1736	if (shift) {
1737		for (page = vmm->func->page; page->shift; page++) {
1738			if (shift == page->shift)
1739				break;
1740		}
1741
1742		if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1743			VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1744			return -EINVAL;
1745		}
1746		align = max_t(u8, align, shift);
1747	} else {
1748		align = max_t(u8, align, 12);
1749	}
1750
1751	/* Locate smallest block that can possibly satisfy the allocation. */
1752	temp = vmm->free.rb_node;
1753	while (temp) {
1754		struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1755		if (this->size < size) {
1756			temp = temp->rb_right;
1757		} else {
1758			node = temp;
1759			temp = temp->rb_left;
1760		}
1761	}
1762
1763	if (unlikely(!node))
1764		return -ENOSPC;
1765
1766	/* Take into account alignment restrictions, trying larger blocks
1767	 * in turn until we find a suitable free block.
1768	 */
1769	do {
1770		struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1771		struct nvkm_vma *prev = node(this, prev);
1772		struct nvkm_vma *next = node(this, next);
1773		const int p = page - vmm->func->page;
1774
1775		addr = this->addr;
1776		if (vmm->func->page_block && prev && prev->page != p)
1777			addr = ALIGN(addr, vmm->func->page_block);
1778		addr = ALIGN(addr, 1ULL << align);
1779
1780		tail = this->addr + this->size;
1781		if (vmm->func->page_block && next && next->page != p)
1782			tail = ALIGN_DOWN(tail, vmm->func->page_block);
1783
1784		if (addr <= tail && tail - addr >= size) {
1785			nvkm_vmm_free_remove(vmm, this);
1786			vma = this;
1787			break;
1788		}
1789	} while ((node = rb_next(node)));
1790
1791	if (unlikely(!vma))
1792		return -ENOSPC;
1793
1794	/* If the VMA we found isn't already exactly the requested size,
1795	 * it needs to be split, and the remaining free blocks returned.
1796	 */
1797	if (addr != vma->addr) {
1798		if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1799			nvkm_vmm_put_region(vmm, vma);
1800			return -ENOMEM;
1801		}
1802		nvkm_vmm_free_insert(vmm, vma);
1803		vma = tmp;
1804	}
1805
1806	if (size != vma->size) {
1807		if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1808			nvkm_vmm_put_region(vmm, vma);
1809			return -ENOMEM;
1810		}
1811		nvkm_vmm_free_insert(vmm, tmp);
1812	}
1813
1814	/* Pre-allocate page tables and/or setup sparse mappings. */
1815	if (sparse && getref)
1816		ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1817	else if (sparse)
1818		ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1819	else if (getref)
1820		ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1821	else
1822		ret = 0;
1823	if (ret) {
1824		nvkm_vmm_put_region(vmm, vma);
1825		return ret;
1826	}
1827
1828	vma->mapref = mapref && !getref;
1829	vma->sparse = sparse;
1830	vma->page = page - vmm->func->page;
1831	vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1832	vma->used = true;
1833	nvkm_vmm_node_insert(vmm, vma);
1834	*pvma = vma;
1835	return 0;
1836}
1837
1838int
1839nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1840{
1841	int ret;
1842	mutex_lock(&vmm->mutex.vmm);
1843	ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1844	mutex_unlock(&vmm->mutex.vmm);
1845	return ret;
1846}
1847
1848void
1849nvkm_vmm_raw_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size,
1850		   bool sparse, u8 refd)
1851{
1852	const struct nvkm_vmm_page *page = &vmm->func->page[refd];
1853
1854	nvkm_vmm_ptes_unmap(vmm, page, addr, size, sparse, false);
1855}
1856
1857void
1858nvkm_vmm_raw_put(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1859{
1860	const struct nvkm_vmm_page *page = vmm->func->page;
1861
1862	nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1863}
1864
1865int
1866nvkm_vmm_raw_get(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd)
1867{
1868	const struct nvkm_vmm_page *page = vmm->func->page;
1869
1870	if (unlikely(!size))
1871		return -EINVAL;
1872
1873	return nvkm_vmm_ptes_get(vmm, &page[refd], addr, size);
1874}
1875
1876int
1877nvkm_vmm_raw_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
1878{
1879	int ret;
1880
1881	mutex_lock(&vmm->mutex.ref);
1882	ret = nvkm_vmm_ptes_sparse(vmm, addr, size, ref);
1883	mutex_unlock(&vmm->mutex.ref);
1884
1885	return ret;
1886}
1887
1888void
1889nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1890{
1891	if (inst && vmm && vmm->func->part) {
1892		mutex_lock(&vmm->mutex.vmm);
1893		vmm->func->part(vmm, inst);
1894		mutex_unlock(&vmm->mutex.vmm);
1895	}
1896}
1897
1898int
1899nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1900{
1901	int ret = 0;
1902	if (vmm->func->join) {
1903		mutex_lock(&vmm->mutex.vmm);
1904		ret = vmm->func->join(vmm, inst);
1905		mutex_unlock(&vmm->mutex.vmm);
1906	}
1907	return ret;
1908}
1909
1910static bool
1911nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1912{
1913	const struct nvkm_vmm_desc *desc = it->desc;
1914	const int type = desc->type == SPT;
1915	nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1916	return false;
1917}
1918
1919int
1920nvkm_vmm_boot(struct nvkm_vmm *vmm)
1921{
1922	const struct nvkm_vmm_page *page = vmm->func->page;
1923	const u64 limit = vmm->limit - vmm->start;
1924	int ret;
1925
1926	while (page[1].shift)
1927		page++;
1928
1929	ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1930	if (ret)
1931		return ret;
1932
1933	nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1934		      nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1935	vmm->bootstrapped = true;
1936	return 0;
1937}
1938
1939static void
1940nvkm_vmm_del(struct kref *kref)
1941{
1942	struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1943	nvkm_vmm_dtor(vmm);
1944	kfree(vmm);
1945}
1946
1947void
1948nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1949{
1950	struct nvkm_vmm *vmm = *pvmm;
1951	if (vmm) {
1952		kref_put(&vmm->kref, nvkm_vmm_del);
1953		*pvmm = NULL;
1954	}
1955}
1956
1957struct nvkm_vmm *
1958nvkm_vmm_ref(struct nvkm_vmm *vmm)
1959{
1960	if (vmm)
1961		kref_get(&vmm->kref);
1962	return vmm;
1963}
1964
1965int
1966nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1967	     u32 argc, struct lock_class_key *key, const char *name,
1968	     struct nvkm_vmm **pvmm)
1969{
1970	struct nvkm_mmu *mmu = device->mmu;
1971	struct nvkm_vmm *vmm = NULL;
1972	int ret;
1973	ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
1974				  key, name, &vmm);
1975	if (ret)
1976		nvkm_vmm_unref(&vmm);
1977	*pvmm = vmm;
1978	return ret;
1979}