nouveau_svm.c

   1/*
   2 * Copyright 2018 Red Hat Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22#include "nouveau_svm.h"
  23#include "nouveau_drv.h"
  24#include "nouveau_chan.h"
  25#include "nouveau_dmem.h"
  26
  27#include <nvif/event.h>
  28#include <nvif/object.h>
  29#include <nvif/vmm.h>
  30
  31#include <nvif/class.h>
  32#include <nvif/clb069.h>
  33#include <nvif/ifc00d.h>
  34
  35#include <linux/sched/mm.h>
  36#include <linux/sort.h>
  37#include <linux/hmm.h>
  38#include <linux/memremap.h>
  39#include <linux/rmap.h>
  40
  41struct nouveau_svm {
  42	struct nouveau_drm *drm;
  43	struct mutex mutex;
  44	struct list_head inst;
  45
  46	struct nouveau_svm_fault_buffer {
  47		int id;
  48		struct nvif_object object;
  49		u32 entries;
  50		u32 getaddr;
  51		u32 putaddr;
  52		u32 get;
  53		u32 put;
  54		struct nvif_event notify;
  55		struct work_struct work;
  56
  57		struct nouveau_svm_fault {
  58			u64 inst;
  59			u64 addr;
  60			u64 time;
  61			u32 engine;
  62			u8  gpc;
  63			u8  hub;
  64			u8  access;
  65			u8  client;
  66			u8  fault;
  67			struct nouveau_svmm *svmm;
  68		} **fault;
  69		int fault_nr;
  70	} buffer[];
  71};
  72
  73#define FAULT_ACCESS_READ 0
  74#define FAULT_ACCESS_WRITE 1
  75#define FAULT_ACCESS_ATOMIC 2
  76#define FAULT_ACCESS_PREFETCH 3
  77
  78#define SVM_DBG(s,f,a...) NV_DEBUG((s)->drm, "svm: "f"\n", ##a)
  79#define SVM_ERR(s,f,a...) NV_WARN((s)->drm, "svm: "f"\n", ##a)
  80
  81struct nouveau_pfnmap_args {
  82	struct nvif_ioctl_v0_hdr i;
  83	struct nvif_ioctl_mthd_v0_hdr m;
  84	struct nvif_vmm_pfnmap_v0 p;
  85};
  86
  87struct nouveau_ivmm {
  88	struct nouveau_svmm *svmm;
  89	u64 inst;
  90	struct list_head head;
  91};
  92
  93static struct nouveau_ivmm *
  94nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst)
  95{
  96	struct nouveau_ivmm *ivmm;
  97	list_for_each_entry(ivmm, &svm->inst, head) {
  98		if (ivmm->inst == inst)
  99			return ivmm;
 100	}
 101	return NULL;
 102}
 103
 104#define SVMM_DBG(s,f,a...)                                                     \
 105	NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)
 106#define SVMM_ERR(s,f,a...)                                                     \
 107	NV_WARN((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a)
 108
 109int
 110nouveau_svmm_bind(struct drm_device *dev, void *data,
 111		  struct drm_file *file_priv)
 112{
 113	struct nouveau_cli *cli = nouveau_cli(file_priv);
 114	struct drm_nouveau_svm_bind *args = data;
 115	unsigned target, cmd;
 116	unsigned long addr, end;
 117	struct mm_struct *mm;
 118
 119	args->va_start &= PAGE_MASK;
 120	args->va_end = ALIGN(args->va_end, PAGE_SIZE);
 121
 122	/* Sanity check arguments */
 123	if (args->reserved0 || args->reserved1)
 124		return -EINVAL;
 125	if (args->header & (~NOUVEAU_SVM_BIND_VALID_MASK))
 126		return -EINVAL;
 127	if (args->va_start >= args->va_end)
 128		return -EINVAL;
 129
 130	cmd = args->header >> NOUVEAU_SVM_BIND_COMMAND_SHIFT;
 131	cmd &= NOUVEAU_SVM_BIND_COMMAND_MASK;
 132	switch (cmd) {
 133	case NOUVEAU_SVM_BIND_COMMAND__MIGRATE:
 134		break;
 135	default:
 136		return -EINVAL;
 137	}
 138
 139	/* FIXME support CPU target ie all target value < GPU_VRAM */
 140	target = args->header >> NOUVEAU_SVM_BIND_TARGET_SHIFT;
 141	target &= NOUVEAU_SVM_BIND_TARGET_MASK;
 142	switch (target) {
 143	case NOUVEAU_SVM_BIND_TARGET__GPU_VRAM:
 144		break;
 145	default:
 146		return -EINVAL;
 147	}
 148
 149	/*
 150	 * FIXME: For now refuse non 0 stride, we need to change the migrate
 151	 * kernel function to handle stride to avoid to create a mess within
 152	 * each device driver.
 153	 */
 154	if (args->stride)
 155		return -EINVAL;
 156
 157	/*
 158	 * Ok we are ask to do something sane, for now we only support migrate
 159	 * commands but we will add things like memory policy (what to do on
 160	 * page fault) and maybe some other commands.
 161	 */
 162
 163	mm = get_task_mm(current);
 164	if (!mm) {
 165		return -EINVAL;
 166	}
 167	mmap_read_lock(mm);
 168
 169	if (!cli->svm.svmm) {
 170		mmap_read_unlock(mm);
 171		mmput(mm);
 172		return -EINVAL;
 173	}
 174
 175	for (addr = args->va_start, end = args->va_end; addr < end;) {
 176		struct vm_area_struct *vma;
 177		unsigned long next;
 178
 179		vma = find_vma_intersection(mm, addr, end);
 180		if (!vma)
 181			break;
 182
 183		addr = max(addr, vma->vm_start);
 184		next = min(vma->vm_end, end);
 185		/* This is a best effort so we ignore errors */
 186		nouveau_dmem_migrate_vma(cli->drm, cli->svm.svmm, vma, addr,
 187					 next);
 188		addr = next;
 189	}
 190
 191	/*
 192	 * FIXME Return the number of page we have migrated, again we need to
 193	 * update the migrate API to return that information so that we can
 194	 * report it to user space.
 195	 */
 196	args->result = 0;
 197
 198	mmap_read_unlock(mm);
 199	mmput(mm);
 200
 201	return 0;
 202}
 203
 204/* Unlink channel instance from SVMM. */
 205void
 206nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst)
 207{
 208	struct nouveau_ivmm *ivmm;
 209	if (svmm) {
 210		mutex_lock(&svmm->vmm->cli->drm->svm->mutex);
 211		ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst);
 212		if (ivmm) {
 213			list_del(&ivmm->head);
 214			kfree(ivmm);
 215		}
 216		mutex_unlock(&svmm->vmm->cli->drm->svm->mutex);
 217	}
 218}
 219
 220/* Link channel instance to SVMM. */
 221int
 222nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst)
 223{
 224	struct nouveau_ivmm *ivmm;
 225	if (svmm) {
 226		if (!(ivmm = kmalloc(sizeof(*ivmm), GFP_KERNEL)))
 227			return -ENOMEM;
 228		ivmm->svmm = svmm;
 229		ivmm->inst = inst;
 230
 231		mutex_lock(&svmm->vmm->cli->drm->svm->mutex);
 232		list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst);
 233		mutex_unlock(&svmm->vmm->cli->drm->svm->mutex);
 234	}
 235	return 0;
 236}
 237
 238/* Invalidate SVMM address-range on GPU. */
 239void
 240nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit)
 241{
 242	if (limit > start) {
 243		nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR,
 244				 &(struct nvif_vmm_pfnclr_v0) {
 245					.addr = start,
 246					.size = limit - start,
 247				 }, sizeof(struct nvif_vmm_pfnclr_v0));
 248	}
 249}
 250
 251static int
 252nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn,
 253				    const struct mmu_notifier_range *update)
 254{
 255	struct nouveau_svmm *svmm =
 256		container_of(mn, struct nouveau_svmm, notifier);
 257	unsigned long start = update->start;
 258	unsigned long limit = update->end;
 259
 260	if (!mmu_notifier_range_blockable(update))
 261		return -EAGAIN;
 262
 263	SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit);
 264
 265	mutex_lock(&svmm->mutex);
 266	if (unlikely(!svmm->vmm))
 267		goto out;
 268
 269	/*
 270	 * Ignore invalidation callbacks for device private pages since
 271	 * the invalidation is handled as part of the migration process.
 272	 */
 273	if (update->event == MMU_NOTIFY_MIGRATE &&
 274	    update->owner == svmm->vmm->cli->drm->dev)
 275		goto out;
 276
 277	if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) {
 278		if (start < svmm->unmanaged.start) {
 279			nouveau_svmm_invalidate(svmm, start,
 280						svmm->unmanaged.limit);
 281		}
 282		start = svmm->unmanaged.limit;
 283	}
 284
 285	nouveau_svmm_invalidate(svmm, start, limit);
 286
 287out:
 288	mutex_unlock(&svmm->mutex);
 289	return 0;
 290}
 291
 292static void nouveau_svmm_free_notifier(struct mmu_notifier *mn)
 293{
 294	kfree(container_of(mn, struct nouveau_svmm, notifier));
 295}
 296
 297static const struct mmu_notifier_ops nouveau_mn_ops = {
 298	.invalidate_range_start = nouveau_svmm_invalidate_range_start,
 299	.free_notifier = nouveau_svmm_free_notifier,
 300};
 301
 302void
 303nouveau_svmm_fini(struct nouveau_svmm **psvmm)
 304{
 305	struct nouveau_svmm *svmm = *psvmm;
 306	if (svmm) {
 307		mutex_lock(&svmm->mutex);
 308		svmm->vmm = NULL;
 309		mutex_unlock(&svmm->mutex);
 310		mmu_notifier_put(&svmm->notifier);
 311		*psvmm = NULL;
 312	}
 313}
 314
 315int
 316nouveau_svmm_init(struct drm_device *dev, void *data,
 317		  struct drm_file *file_priv)
 318{
 319	struct nouveau_cli *cli = nouveau_cli(file_priv);
 320	struct nouveau_svmm *svmm;
 321	struct drm_nouveau_svm_init *args = data;
 322	int ret;
 323
 324	/* We need to fail if svm is disabled */
 325	if (!cli->drm->svm)
 326		return -ENOSYS;
 327
 328	/* Allocate tracking for SVM-enabled VMM. */
 329	if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL)))
 330		return -ENOMEM;
 331	svmm->vmm = &cli->svm;
 332	svmm->unmanaged.start = args->unmanaged_addr;
 333	svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size;
 334	mutex_init(&svmm->mutex);
 335
 336	/* Check that SVM isn't already enabled for the client. */
 337	mutex_lock(&cli->mutex);
 338	if (cli->svm.cli) {
 339		ret = -EBUSY;
 340		goto out_free;
 341	}
 342
 343	/* Allocate a new GPU VMM that can support SVM (managed by the
 344	 * client, with replayable faults enabled).
 345	 *
 346	 * All future channel/memory allocations will make use of this
 347	 * VMM instead of the standard one.
 348	 */
 349	ret = nvif_vmm_ctor(&cli->mmu, "svmVmm",
 350			    cli->vmm.vmm.object.oclass, MANAGED,
 351			    args->unmanaged_addr, args->unmanaged_size,
 352			    &(struct gp100_vmm_v0) {
 353				.fault_replay = true,
 354			    }, sizeof(struct gp100_vmm_v0), &cli->svm.vmm);
 355	if (ret)
 356		goto out_free;
 357
 358	mmap_write_lock(current->mm);
 359	svmm->notifier.ops = &nouveau_mn_ops;
 360	ret = __mmu_notifier_register(&svmm->notifier, current->mm);
 361	if (ret)
 362		goto out_mm_unlock;
 363	/* Note, ownership of svmm transfers to mmu_notifier */
 364
 365	cli->svm.svmm = svmm;
 366	cli->svm.cli = cli;
 367	mmap_write_unlock(current->mm);
 368	mutex_unlock(&cli->mutex);
 369	return 0;
 370
 371out_mm_unlock:
 372	mmap_write_unlock(current->mm);
 373out_free:
 374	mutex_unlock(&cli->mutex);
 375	kfree(svmm);
 376	return ret;
 377}
 378
 379/* Issue fault replay for GPU to retry accesses that faulted previously. */
 380static void
 381nouveau_svm_fault_replay(struct nouveau_svm *svm)
 382{
 383	SVM_DBG(svm, "replay");
 384	WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object,
 385				 GP100_VMM_VN_FAULT_REPLAY,
 386				 &(struct gp100_vmm_fault_replay_vn) {},
 387				 sizeof(struct gp100_vmm_fault_replay_vn)));
 388}
 389
 390/* Cancel a replayable fault that could not be handled.
 391 *
 392 * Cancelling the fault will trigger recovery to reset the engine
 393 * and kill the offending channel (ie. GPU SIGSEGV).
 394 */
 395static void
 396nouveau_svm_fault_cancel(struct nouveau_svm *svm,
 397			 u64 inst, u8 hub, u8 gpc, u8 client)
 398{
 399	SVM_DBG(svm, "cancel %016llx %d %02x %02x", inst, hub, gpc, client);
 400	WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object,
 401				 GP100_VMM_VN_FAULT_CANCEL,
 402				 &(struct gp100_vmm_fault_cancel_v0) {
 403					.hub = hub,
 404					.gpc = gpc,
 405					.client = client,
 406					.inst = inst,
 407				 }, sizeof(struct gp100_vmm_fault_cancel_v0)));
 408}
 409
 410static void
 411nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm,
 412			       struct nouveau_svm_fault *fault)
 413{
 414	nouveau_svm_fault_cancel(svm, fault->inst,
 415				      fault->hub,
 416				      fault->gpc,
 417				      fault->client);
 418}
 419
 420static int
 421nouveau_svm_fault_priority(u8 fault)
 422{
 423	switch (fault) {
 424	case FAULT_ACCESS_PREFETCH:
 425		return 0;
 426	case FAULT_ACCESS_READ:
 427		return 1;
 428	case FAULT_ACCESS_WRITE:
 429		return 2;
 430	case FAULT_ACCESS_ATOMIC:
 431		return 3;
 432	default:
 433		WARN_ON_ONCE(1);
 434		return -1;
 435	}
 436}
 437
 438static int
 439nouveau_svm_fault_cmp(const void *a, const void *b)
 440{
 441	const struct nouveau_svm_fault *fa = *(struct nouveau_svm_fault **)a;
 442	const struct nouveau_svm_fault *fb = *(struct nouveau_svm_fault **)b;
 443	int ret;
 444	if ((ret = (s64)fa->inst - fb->inst))
 445		return ret;
 446	if ((ret = (s64)fa->addr - fb->addr))
 447		return ret;
 448	return nouveau_svm_fault_priority(fa->access) -
 449		nouveau_svm_fault_priority(fb->access);
 450}
 451
 452static void
 453nouveau_svm_fault_cache(struct nouveau_svm *svm,
 454			struct nouveau_svm_fault_buffer *buffer, u32 offset)
 455{
 456	struct nvif_object *memory = &buffer->object;
 457	const u32 instlo = nvif_rd32(memory, offset + 0x00);
 458	const u32 insthi = nvif_rd32(memory, offset + 0x04);
 459	const u32 addrlo = nvif_rd32(memory, offset + 0x08);
 460	const u32 addrhi = nvif_rd32(memory, offset + 0x0c);
 461	const u32 timelo = nvif_rd32(memory, offset + 0x10);
 462	const u32 timehi = nvif_rd32(memory, offset + 0x14);
 463	const u32 engine = nvif_rd32(memory, offset + 0x18);
 464	const u32   info = nvif_rd32(memory, offset + 0x1c);
 465	const u64   inst = (u64)insthi << 32 | instlo;
 466	const u8     gpc = (info & 0x1f000000) >> 24;
 467	const u8     hub = (info & 0x00100000) >> 20;
 468	const u8  client = (info & 0x00007f00) >> 8;
 469	struct nouveau_svm_fault *fault;
 470
 471	//XXX: i think we're supposed to spin waiting */
 472	if (WARN_ON(!(info & 0x80000000)))
 473		return;
 474
 475	nvif_mask(memory, offset + 0x1c, 0x80000000, 0x00000000);
 476
 477	if (!buffer->fault[buffer->fault_nr]) {
 478		fault = kmalloc(sizeof(*fault), GFP_KERNEL);
 479		if (WARN_ON(!fault)) {
 480			nouveau_svm_fault_cancel(svm, inst, hub, gpc, client);
 481			return;
 482		}
 483		buffer->fault[buffer->fault_nr] = fault;
 484	}
 485
 486	fault = buffer->fault[buffer->fault_nr++];
 487	fault->inst   = inst;
 488	fault->addr   = (u64)addrhi << 32 | addrlo;
 489	fault->time   = (u64)timehi << 32 | timelo;
 490	fault->engine = engine;
 491	fault->gpc    = gpc;
 492	fault->hub    = hub;
 493	fault->access = (info & 0x000f0000) >> 16;
 494	fault->client = client;
 495	fault->fault  = (info & 0x0000001f);
 496
 497	SVM_DBG(svm, "fault %016llx %016llx %02x",
 498		fault->inst, fault->addr, fault->access);
 499}
 500
 501struct svm_notifier {
 502	struct mmu_interval_notifier notifier;
 503	struct nouveau_svmm *svmm;
 504};
 505
 506static bool nouveau_svm_range_invalidate(struct mmu_interval_notifier *mni,
 507					 const struct mmu_notifier_range *range,
 508					 unsigned long cur_seq)
 509{
 510	struct svm_notifier *sn =
 511		container_of(mni, struct svm_notifier, notifier);
 512
 513	if (range->event == MMU_NOTIFY_EXCLUSIVE &&
 514	    range->owner == sn->svmm->vmm->cli->drm->dev)
 515		return true;
 516
 517	/*
 518	 * serializes the update to mni->invalidate_seq done by caller and
 519	 * prevents invalidation of the PTE from progressing while HW is being
 520	 * programmed. This is very hacky and only works because the normal
 521	 * notifier that does invalidation is always called after the range
 522	 * notifier.
 523	 */
 524	if (mmu_notifier_range_blockable(range))
 525		mutex_lock(&sn->svmm->mutex);
 526	else if (!mutex_trylock(&sn->svmm->mutex))
 527		return false;
 528	mmu_interval_set_seq(mni, cur_seq);
 529	mutex_unlock(&sn->svmm->mutex);
 530	return true;
 531}
 532
 533static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = {
 534	.invalidate = nouveau_svm_range_invalidate,
 535};
 536
 537static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm,
 538				    struct hmm_range *range,
 539				    struct nouveau_pfnmap_args *args)
 540{
 541	struct page *page;
 542
 543	/*
 544	 * The address prepared here is passed through nvif_object_ioctl()
 545	 * to an eventual DMA map in something like gp100_vmm_pgt_pfn()
 546	 *
 547	 * This is all just encoding the internal hmm representation into a
 548	 * different nouveau internal representation.
 549	 */
 550	if (!(range->hmm_pfns[0] & HMM_PFN_VALID)) {
 551		args->p.phys[0] = 0;
 552		return;
 553	}
 554
 555	page = hmm_pfn_to_page(range->hmm_pfns[0]);
 556	/*
 557	 * Only map compound pages to the GPU if the CPU is also mapping the
 558	 * page as a compound page. Otherwise, the PTE protections might not be
 559	 * consistent (e.g., CPU only maps part of a compound page).
 560	 * Note that the underlying page might still be larger than the
 561	 * CPU mapping (e.g., a PUD sized compound page partially mapped with
 562	 * a PMD sized page table entry).
 563	 */
 564	if (hmm_pfn_to_map_order(range->hmm_pfns[0])) {
 565		unsigned long addr = args->p.addr;
 566
 567		args->p.page = hmm_pfn_to_map_order(range->hmm_pfns[0]) +
 568				PAGE_SHIFT;
 569		args->p.size = 1UL << args->p.page;
 570		args->p.addr &= ~(args->p.size - 1);
 571		page -= (addr - args->p.addr) >> PAGE_SHIFT;
 572	}
 573	if (is_device_private_page(page))
 574		args->p.phys[0] = nouveau_dmem_page_addr(page) |
 575				NVIF_VMM_PFNMAP_V0_V |
 576				NVIF_VMM_PFNMAP_V0_VRAM;
 577	else
 578		args->p.phys[0] = page_to_phys(page) |
 579				NVIF_VMM_PFNMAP_V0_V |
 580				NVIF_VMM_PFNMAP_V0_HOST;
 581	if (range->hmm_pfns[0] & HMM_PFN_WRITE)
 582		args->p.phys[0] |= NVIF_VMM_PFNMAP_V0_W;
 583}
 584
 585static int nouveau_atomic_range_fault(struct nouveau_svmm *svmm,
 586			       struct nouveau_drm *drm,
 587			       struct nouveau_pfnmap_args *args, u32 size,
 588			       struct svm_notifier *notifier)
 589{
 590	unsigned long timeout =
 591		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 592	struct mm_struct *mm = svmm->notifier.mm;
 593	struct folio *folio;
 594	struct page *page;
 595	unsigned long start = args->p.addr;
 596	unsigned long notifier_seq;
 597	int ret = 0;
 598
 599	ret = mmu_interval_notifier_insert(&notifier->notifier, mm,
 600					args->p.addr, args->p.size,
 601					&nouveau_svm_mni_ops);
 602	if (ret)
 603		return ret;
 604
 605	while (true) {
 606		if (time_after(jiffies, timeout)) {
 607			ret = -EBUSY;
 608			goto out;
 609		}
 610
 611		notifier_seq = mmu_interval_read_begin(&notifier->notifier);
 612		mmap_read_lock(mm);
 613		page = make_device_exclusive(mm, start, drm->dev, &folio);
 614		mmap_read_unlock(mm);
 615		if (IS_ERR(page)) {
 616			ret = -EINVAL;
 617			goto out;
 618		}
 619		folio = page_folio(page);
 620
 621		mutex_lock(&svmm->mutex);
 622		if (!mmu_interval_read_retry(&notifier->notifier,
 623					     notifier_seq))
 624			break;
 625		mutex_unlock(&svmm->mutex);
 626
 627		folio_unlock(folio);
 628		folio_put(folio);
 629	}
 630
 631	/* Map the page on the GPU. */
 632	args->p.page = 12;
 633	args->p.size = PAGE_SIZE;
 634	args->p.addr = start;
 635	args->p.phys[0] = page_to_phys(page) |
 636		NVIF_VMM_PFNMAP_V0_V |
 637		NVIF_VMM_PFNMAP_V0_W |
 638		NVIF_VMM_PFNMAP_V0_A |
 639		NVIF_VMM_PFNMAP_V0_HOST;
 640
 641	ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL);
 642	mutex_unlock(&svmm->mutex);
 643
 644	folio_unlock(folio);
 645	folio_put(folio);
 646
 647out:
 648	mmu_interval_notifier_remove(&notifier->notifier);
 649	return ret;
 650}
 651
 652static int nouveau_range_fault(struct nouveau_svmm *svmm,
 653			       struct nouveau_drm *drm,
 654			       struct nouveau_pfnmap_args *args, u32 size,
 655			       unsigned long hmm_flags,
 656			       struct svm_notifier *notifier)
 657{
 658	unsigned long timeout =
 659		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 660	/* Have HMM fault pages within the fault window to the GPU. */
 661	unsigned long hmm_pfns[1];
 662	struct hmm_range range = {
 663		.notifier = &notifier->notifier,
 664		.default_flags = hmm_flags,
 665		.hmm_pfns = hmm_pfns,
 666		.dev_private_owner = drm->dev,
 667	};
 668	struct mm_struct *mm = svmm->notifier.mm;
 669	int ret;
 670
 671	ret = mmu_interval_notifier_insert(&notifier->notifier, mm,
 672					args->p.addr, args->p.size,
 673					&nouveau_svm_mni_ops);
 674	if (ret)
 675		return ret;
 676
 677	range.start = notifier->notifier.interval_tree.start;
 678	range.end = notifier->notifier.interval_tree.last + 1;
 679
 680	while (true) {
 681		if (time_after(jiffies, timeout)) {
 682			ret = -EBUSY;
 683			goto out;
 684		}
 685
 686		range.notifier_seq = mmu_interval_read_begin(range.notifier);
 687		mmap_read_lock(mm);
 688		ret = hmm_range_fault(&range);
 689		mmap_read_unlock(mm);
 690		if (ret) {
 691			if (ret == -EBUSY)
 692				continue;
 693			goto out;
 694		}
 695
 696		mutex_lock(&svmm->mutex);
 697		if (mmu_interval_read_retry(range.notifier,
 698					    range.notifier_seq)) {
 699			mutex_unlock(&svmm->mutex);
 700			continue;
 701		}
 702		break;
 703	}
 704
 705	nouveau_hmm_convert_pfn(drm, &range, args);
 706
 707	ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL);
 708	mutex_unlock(&svmm->mutex);
 709
 710out:
 711	mmu_interval_notifier_remove(&notifier->notifier);
 712
 713	return ret;
 714}
 715
 716static void
 717nouveau_svm_fault(struct work_struct *work)
 718{
 719	struct nouveau_svm_fault_buffer *buffer = container_of(work, typeof(*buffer), work);
 720	struct nouveau_svm *svm = container_of(buffer, typeof(*svm), buffer[buffer->id]);
 721	struct nvif_object *device = &svm->drm->client.device.object;
 722	struct nouveau_svmm *svmm;
 723	DEFINE_RAW_FLEX(struct nouveau_pfnmap_args, args, p.phys, 1);
 724	unsigned long hmm_flags;
 725	u64 inst, start, limit;
 726	int fi, fn;
 727	int replay = 0, atomic = 0, ret;
 728
 729	/* Parse available fault buffer entries into a cache, and update
 730	 * the GET pointer so HW can reuse the entries.
 731	 */
 732	SVM_DBG(svm, "fault handler");
 733	if (buffer->get == buffer->put) {
 734		buffer->put = nvif_rd32(device, buffer->putaddr);
 735		buffer->get = nvif_rd32(device, buffer->getaddr);
 736		if (buffer->get == buffer->put)
 737			return;
 738	}
 739	buffer->fault_nr = 0;
 740
 741	SVM_DBG(svm, "get %08x put %08x", buffer->get, buffer->put);
 742	while (buffer->get != buffer->put) {
 743		nouveau_svm_fault_cache(svm, buffer, buffer->get * 0x20);
 744		if (++buffer->get == buffer->entries)
 745			buffer->get = 0;
 746	}
 747	nvif_wr32(device, buffer->getaddr, buffer->get);
 748	SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr);
 749
 750	/* Sort parsed faults by instance pointer to prevent unnecessary
 751	 * instance to SVMM translations, followed by address and access
 752	 * type to reduce the amount of work when handling the faults.
 753	 */
 754	sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault),
 755	     nouveau_svm_fault_cmp, NULL);
 756
 757	/* Lookup SVMM structure for each unique instance pointer. */
 758	mutex_lock(&svm->mutex);
 759	for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) {
 760		if (!svmm || buffer->fault[fi]->inst != inst) {
 761			struct nouveau_ivmm *ivmm =
 762				nouveau_ivmm_find(svm, buffer->fault[fi]->inst);
 763			svmm = ivmm ? ivmm->svmm : NULL;
 764			inst = buffer->fault[fi]->inst;
 765			SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm);
 766		}
 767		buffer->fault[fi]->svmm = svmm;
 768	}
 769	mutex_unlock(&svm->mutex);
 770
 771	/* Process list of faults. */
 772	args->i.version = 0;
 773	args->i.type = NVIF_IOCTL_V0_MTHD;
 774	args->m.version = 0;
 775	args->m.method = NVIF_VMM_V0_PFNMAP;
 776	args->p.version = 0;
 777
 778	for (fi = 0; fn = fi + 1, fi < buffer->fault_nr; fi = fn) {
 779		struct svm_notifier notifier;
 780		struct mm_struct *mm;
 781
 782		/* Cancel any faults from non-SVM channels. */
 783		if (!(svmm = buffer->fault[fi]->svmm)) {
 784			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
 785			continue;
 786		}
 787		SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr);
 788
 789		/* We try and group handling of faults within a small
 790		 * window into a single update.
 791		 */
 792		start = buffer->fault[fi]->addr;
 793		limit = start + PAGE_SIZE;
 794		if (start < svmm->unmanaged.limit)
 795			limit = min_t(u64, limit, svmm->unmanaged.start);
 796
 797		/*
 798		 * Prepare the GPU-side update of all pages within the
 799		 * fault window, determining required pages and access
 800		 * permissions based on pending faults.
 801		 */
 802		args->p.addr = start;
 803		args->p.page = PAGE_SHIFT;
 804		args->p.size = PAGE_SIZE;
 805		/*
 806		 * Determine required permissions based on GPU fault
 807		 * access flags.
 808		 */
 809		switch (buffer->fault[fi]->access) {
 810		case 0: /* READ. */
 811			hmm_flags = HMM_PFN_REQ_FAULT;
 812			break;
 813		case 2: /* ATOMIC. */
 814			atomic = true;
 815			break;
 816		case 3: /* PREFETCH. */
 817			hmm_flags = 0;
 818			break;
 819		default:
 820			hmm_flags = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE;
 821			break;
 822		}
 823
 824		mm = svmm->notifier.mm;
 825		if (!mmget_not_zero(mm)) {
 826			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
 827			continue;
 828		}
 829
 830		notifier.svmm = svmm;
 831		if (atomic)
 832			ret = nouveau_atomic_range_fault(svmm, svm->drm, args,
 833							 __struct_size(args),
 834							 &notifier);
 835		else
 836			ret = nouveau_range_fault(svmm, svm->drm, args,
 837						  __struct_size(args),
 838						  hmm_flags, &notifier);
 839		mmput(mm);
 840
 841		limit = args->p.addr + args->p.size;
 842		for (fn = fi; ++fn < buffer->fault_nr; ) {
 843			/* It's okay to skip over duplicate addresses from the
 844			 * same SVMM as faults are ordered by access type such
 845			 * that only the first one needs to be handled.
 846			 *
 847			 * ie. WRITE faults appear first, thus any handling of
 848			 * pending READ faults will already be satisfied.
 849			 * But if a large page is mapped, make sure subsequent
 850			 * fault addresses have sufficient access permission.
 851			 */
 852			if (buffer->fault[fn]->svmm != svmm ||
 853			    buffer->fault[fn]->addr >= limit ||
 854			    (buffer->fault[fi]->access == FAULT_ACCESS_READ &&
 855			     !(args->p.phys[0] & NVIF_VMM_PFNMAP_V0_V)) ||
 856			    (buffer->fault[fi]->access != FAULT_ACCESS_READ &&
 857			     buffer->fault[fi]->access != FAULT_ACCESS_PREFETCH &&
 858			     !(args->p.phys[0] & NVIF_VMM_PFNMAP_V0_W)) ||
 859			    (buffer->fault[fi]->access != FAULT_ACCESS_READ &&
 860			     buffer->fault[fi]->access != FAULT_ACCESS_WRITE &&
 861			     buffer->fault[fi]->access != FAULT_ACCESS_PREFETCH &&
 862			     !(args->p.phys[0] & NVIF_VMM_PFNMAP_V0_A)))
 863				break;
 864		}
 865
 866		/* If handling failed completely, cancel all faults. */
 867		if (ret) {
 868			while (fi < fn) {
 869				struct nouveau_svm_fault *fault =
 870					buffer->fault[fi++];
 871
 872				nouveau_svm_fault_cancel_fault(svm, fault);
 873			}
 874		} else
 875			replay++;
 876	}
 877
 878	/* Issue fault replay to the GPU. */
 879	if (replay)
 880		nouveau_svm_fault_replay(svm);
 881}
 882
 883static int
 884nouveau_svm_event(struct nvif_event *event, void *argv, u32 argc)
 885{
 886	struct nouveau_svm_fault_buffer *buffer = container_of(event, typeof(*buffer), notify);
 887
 888	schedule_work(&buffer->work);
 889	return NVIF_EVENT_KEEP;
 890}
 891
 892static struct nouveau_pfnmap_args *
 893nouveau_pfns_to_args(void *pfns)
 894{
 895	return container_of(pfns, struct nouveau_pfnmap_args, p.phys);
 896}
 897
 898u64 *
 899nouveau_pfns_alloc(unsigned long npages)
 900{
 901	struct nouveau_pfnmap_args *args;
 902
 903	args = kzalloc(struct_size(args, p.phys, npages), GFP_KERNEL);
 904	if (!args)
 905		return NULL;
 906
 907	args->i.type = NVIF_IOCTL_V0_MTHD;
 908	args->m.method = NVIF_VMM_V0_PFNMAP;
 909	args->p.page = PAGE_SHIFT;
 910
 911	return args->p.phys;
 912}
 913
 914void
 915nouveau_pfns_free(u64 *pfns)
 916{
 917	struct nouveau_pfnmap_args *args = nouveau_pfns_to_args(pfns);
 918
 919	kfree(args);
 920}
 921
 922void
 923nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm,
 924		 unsigned long addr, u64 *pfns, unsigned long npages,
 925		 unsigned int page_shift)
 926{
 927	struct nouveau_pfnmap_args *args = nouveau_pfns_to_args(pfns);
 928
 929	args->p.addr = addr;
 930	args->p.size = npages << page_shift;
 931	args->p.page = page_shift;
 932
 933	mutex_lock(&svmm->mutex);
 934
 935	nvif_object_ioctl(&svmm->vmm->vmm.object, args,
 936			  struct_size(args, p.phys, npages), NULL);
 937
 938	mutex_unlock(&svmm->mutex);
 939}
 940
 941static void
 942nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id)
 943{
 944	struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
 945
 946	nvif_event_block(&buffer->notify);
 947	flush_work(&buffer->work);
 948}
 949
 950static int
 951nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id)
 952{
 953	struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
 954	struct nvif_object *device = &svm->drm->client.device.object;
 955
 956	buffer->get = nvif_rd32(device, buffer->getaddr);
 957	buffer->put = nvif_rd32(device, buffer->putaddr);
 958	SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put);
 959
 960	return nvif_event_allow(&buffer->notify);
 961}
 962
 963static void
 964nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id)
 965{
 966	struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
 967	int i;
 968
 969	if (!nvif_object_constructed(&buffer->object))
 970		return;
 971
 972	nouveau_svm_fault_buffer_fini(svm, id);
 973
 974	if (buffer->fault) {
 975		for (i = 0; buffer->fault[i] && i < buffer->entries; i++)
 976			kfree(buffer->fault[i]);
 977		kvfree(buffer->fault);
 978	}
 979
 980	nvif_event_dtor(&buffer->notify);
 981	nvif_object_dtor(&buffer->object);
 982}
 983
 984static int
 985nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id)
 986{
 987	struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id];
 988	struct nouveau_drm *drm = svm->drm;
 989	struct nvif_object *device = &drm->client.device.object;
 990	struct nvif_clb069_v0 args = {};
 991	int ret;
 992
 993	buffer->id = id;
 994
 995	ret = nvif_object_ctor(device, "svmFaultBuffer", 0, oclass, &args,
 996			       sizeof(args), &buffer->object);
 997	if (ret < 0) {
 998		SVM_ERR(svm, "Fault buffer allocation failed: %d", ret);
 999		return ret;
1000	}
1001
1002	nvif_object_map(&buffer->object, NULL, 0);
1003	buffer->entries = args.entries;
1004	buffer->getaddr = args.get;
1005	buffer->putaddr = args.put;
1006	INIT_WORK(&buffer->work, nouveau_svm_fault);
1007
1008	ret = nvif_event_ctor(&buffer->object, "svmFault", id, nouveau_svm_event, true, NULL, 0,
1009			      &buffer->notify);
1010	if (ret)
1011		return ret;
1012
1013	buffer->fault = kvcalloc(buffer->entries, sizeof(*buffer->fault), GFP_KERNEL);
1014	if (!buffer->fault)
1015		return -ENOMEM;
1016
1017	return nouveau_svm_fault_buffer_init(svm, id);
1018}
1019
1020void
1021nouveau_svm_resume(struct nouveau_drm *drm)
1022{
1023	struct nouveau_svm *svm = drm->svm;
1024	if (svm)
1025		nouveau_svm_fault_buffer_init(svm, 0);
1026}
1027
1028void
1029nouveau_svm_suspend(struct nouveau_drm *drm)
1030{
1031	struct nouveau_svm *svm = drm->svm;
1032	if (svm)
1033		nouveau_svm_fault_buffer_fini(svm, 0);
1034}
1035
1036void
1037nouveau_svm_fini(struct nouveau_drm *drm)
1038{
1039	struct nouveau_svm *svm = drm->svm;
1040	if (svm) {
1041		nouveau_svm_fault_buffer_dtor(svm, 0);
1042		kfree(drm->svm);
1043		drm->svm = NULL;
1044	}
1045}
1046
1047void
1048nouveau_svm_init(struct nouveau_drm *drm)
1049{
1050	static const struct nvif_mclass buffers[] = {
1051		{   VOLTA_FAULT_BUFFER_A, 0 },
1052		{ MAXWELL_FAULT_BUFFER_A, 0 },
1053		{}
1054	};
1055	struct nouveau_svm *svm;
1056	int ret;
1057
1058	/* Disable on Volta and newer until channel recovery is fixed,
1059	 * otherwise clients will have a trivial way to trash the GPU
1060	 * for everyone.
1061	 */
1062	if (drm->client.device.info.family > NV_DEVICE_INFO_V0_PASCAL)
1063		return;
1064
1065	drm->svm = svm = kzalloc(struct_size(drm->svm, buffer, 1), GFP_KERNEL);
1066	if (!drm->svm)
1067		return;
1068
1069	drm->svm->drm = drm;
1070	mutex_init(&drm->svm->mutex);
1071	INIT_LIST_HEAD(&drm->svm->inst);
1072
1073	ret = nvif_mclass(&drm->client.device.object, buffers);
1074	if (ret < 0) {
1075		SVM_DBG(svm, "No supported fault buffer class");
1076		nouveau_svm_fini(drm);
1077		return;
1078	}
1079
1080	ret = nouveau_svm_fault_buffer_ctor(svm, buffers[ret].oclass, 0);
1081	if (ret) {
1082		nouveau_svm_fini(drm);
1083		return;
1084	}
1085
1086	SVM_DBG(svm, "Initialised");
1087}