nouveau_uvmm.c

   1// SPDX-License-Identifier: MIT
   2
   3/*
   4 * Locking:
   5 *
   6 * The uvmm mutex protects any operations on the GPU VA space provided by the
   7 * DRM GPU VA manager.
   8 *
   9 * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a
  10 * mapping to it's backing GEM must be performed under this lock.
  11 *
  12 * Actual map/unmap operations within the fence signalling critical path are
  13 * protected by installing DMA fences to the corresponding GEMs DMA
  14 * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA
  15 * list in order to map/unmap it's entries, can't occur concurrently.
  16 *
  17 * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate
  18 * protection, since there are no accesses other than from BO move callbacks
  19 * and from the fence signalling critical path, which are already protected by
  20 * the corresponding GEMs DMA reservation fence.
  21 */
  22
  23#include "nouveau_drv.h"
  24#include "nouveau_gem.h"
  25#include "nouveau_mem.h"
  26#include "nouveau_uvmm.h"
  27
  28#include <nvif/vmm.h>
  29#include <nvif/mem.h>
  30
  31#include <nvif/class.h>
  32#include <nvif/if000c.h>
  33#include <nvif/if900d.h>
  34
  35#define NOUVEAU_VA_SPACE_BITS		47 /* FIXME */
  36#define NOUVEAU_VA_SPACE_START		0x0
  37#define NOUVEAU_VA_SPACE_END		(1ULL << NOUVEAU_VA_SPACE_BITS)
  38
  39#define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry)
  40#define list_prev_op(_op) list_prev_entry(_op, entry)
  41#define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry)
  42#define list_for_each_op_from_reverse(_op, _ops) \
  43	list_for_each_entry_from_reverse(_op, _ops, entry)
  44#define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry)
  45
  46enum vm_bind_op {
  47	OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP,
  48	OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP,
  49	OP_MAP_SPARSE,
  50	OP_UNMAP_SPARSE,
  51};
  52
  53struct nouveau_uvma_prealloc {
  54	struct nouveau_uvma *map;
  55	struct nouveau_uvma *prev;
  56	struct nouveau_uvma *next;
  57};
  58
  59struct bind_job_op {
  60	struct list_head entry;
  61
  62	enum vm_bind_op op;
  63	u32 flags;
  64
  65	struct drm_gpuvm_bo *vm_bo;
  66
  67	struct {
  68		u64 addr;
  69		u64 range;
  70	} va;
  71
  72	struct {
  73		u32 handle;
  74		u64 offset;
  75		struct drm_gem_object *obj;
  76	} gem;
  77
  78	struct nouveau_uvma_region *reg;
  79	struct nouveau_uvma_prealloc new;
  80	struct drm_gpuva_ops *ops;
  81};
  82
  83struct uvmm_map_args {
  84	struct nouveau_uvma_region *region;
  85	u64 addr;
  86	u64 range;
  87	u8 kind;
  88};
  89
  90static int
  91nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm,
  92			    u64 addr, u64 range)
  93{
  94	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
  95
  96	return nvif_vmm_raw_sparse(vmm, addr, range, true);
  97}
  98
  99static int
 100nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm,
 101			      u64 addr, u64 range)
 102{
 103	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 104
 105	return nvif_vmm_raw_sparse(vmm, addr, range, false);
 106}
 107
 108static int
 109nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm,
 110		     u64 addr, u64 range, u8 page_shift)
 111{
 112	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 113
 114	return nvif_vmm_raw_get(vmm, addr, range, page_shift);
 115}
 116
 117static int
 118nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm,
 119		     u64 addr, u64 range, u8 page_shift)
 120{
 121	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 122
 123	return nvif_vmm_raw_put(vmm, addr, range, page_shift);
 124}
 125
 126static int
 127nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm,
 128		       u64 addr, u64 range, u8 page_shift, bool sparse)
 129{
 130	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 131
 132	return nvif_vmm_raw_unmap(vmm, addr, range, page_shift, sparse);
 133}
 134
 135static int
 136nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm,
 137		     u64 addr, u64 range, u8 page_shift,
 138		     u64 bo_offset, u8 kind,
 139		     struct nouveau_mem *mem)
 140{
 141	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 142	union {
 143		struct gf100_vmm_map_v0 gf100;
 144	} args;
 145	u32 argc = 0;
 146
 147	switch (vmm->object.oclass) {
 148	case NVIF_CLASS_VMM_GF100:
 149	case NVIF_CLASS_VMM_GM200:
 150	case NVIF_CLASS_VMM_GP100:
 151		args.gf100.version = 0;
 152		if (mem->mem.type & NVIF_MEM_VRAM)
 153			args.gf100.vol = 0;
 154		else
 155			args.gf100.vol = 1;
 156		args.gf100.ro = 0;
 157		args.gf100.priv = 0;
 158		args.gf100.kind = kind;
 159		argc = sizeof(args.gf100);
 160		break;
 161	default:
 162		WARN_ON(1);
 163		return -ENOSYS;
 164	}
 165
 166	return nvif_vmm_raw_map(vmm, addr, range, page_shift,
 167				&args, argc,
 168				&mem->mem, bo_offset);
 169}
 170
 171static int
 172nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg)
 173{
 174	u64 addr = reg->va.addr;
 175	u64 range = reg->va.range;
 176
 177	return nouveau_uvmm_vmm_sparse_unref(reg->uvmm, addr, range);
 178}
 179
 180static int
 181nouveau_uvma_vmm_put(struct nouveau_uvma *uvma)
 182{
 183	u64 addr = uvma->va.va.addr;
 184	u64 range = uvma->va.va.range;
 185	u8 page_shift = uvma->page_shift;
 186
 187	return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range, page_shift);
 188}
 189
 190static int
 191nouveau_uvma_map(struct nouveau_uvma *uvma,
 192		 struct nouveau_mem *mem)
 193{
 194	u64 addr = uvma->va.va.addr;
 195	u64 offset = uvma->va.gem.offset;
 196	u64 range = uvma->va.va.range;
 197	u8 page_shift = uvma->page_shift;
 198
 199	return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range,
 200				    page_shift, offset, uvma->kind,
 201				    mem);
 202}
 203
 204static int
 205nouveau_uvma_unmap(struct nouveau_uvma *uvma)
 206{
 207	u64 addr = uvma->va.va.addr;
 208	u64 range = uvma->va.va.range;
 209	u8 page_shift = uvma->page_shift;
 210	bool sparse = !!uvma->region;
 211
 212	if (drm_gpuva_invalidated(&uvma->va))
 213		return 0;
 214
 215	return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, page_shift, sparse);
 216}
 217
 218static int
 219nouveau_uvma_alloc(struct nouveau_uvma **puvma)
 220{
 221	*puvma = kzalloc(sizeof(**puvma), GFP_KERNEL);
 222	if (!*puvma)
 223		return -ENOMEM;
 224
 225	return 0;
 226}
 227
 228static void
 229nouveau_uvma_free(struct nouveau_uvma *uvma)
 230{
 231	kfree(uvma);
 232}
 233
 234static void
 235nouveau_uvma_gem_get(struct nouveau_uvma *uvma)
 236{
 237	drm_gem_object_get(uvma->va.gem.obj);
 238}
 239
 240static void
 241nouveau_uvma_gem_put(struct nouveau_uvma *uvma)
 242{
 243	drm_gem_object_put(uvma->va.gem.obj);
 244}
 245
 246static int
 247nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg)
 248{
 249	*preg = kzalloc(sizeof(**preg), GFP_KERNEL);
 250	if (!*preg)
 251		return -ENOMEM;
 252
 253	kref_init(&(*preg)->kref);
 254
 255	return 0;
 256}
 257
 258static void
 259nouveau_uvma_region_free(struct kref *kref)
 260{
 261	struct nouveau_uvma_region *reg =
 262		container_of(kref, struct nouveau_uvma_region, kref);
 263
 264	kfree(reg);
 265}
 266
 267static void
 268nouveau_uvma_region_get(struct nouveau_uvma_region *reg)
 269{
 270	kref_get(&reg->kref);
 271}
 272
 273static void
 274nouveau_uvma_region_put(struct nouveau_uvma_region *reg)
 275{
 276	kref_put(&reg->kref, nouveau_uvma_region_free);
 277}
 278
 279static int
 280__nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,
 281			     struct nouveau_uvma_region *reg)
 282{
 283	u64 addr = reg->va.addr;
 284	u64 range = reg->va.range;
 285	u64 last = addr + range - 1;
 286	MA_STATE(mas, &uvmm->region_mt, addr, addr);
 287
 288	if (unlikely(mas_walk(&mas)))
 289		return -EEXIST;
 290
 291	if (unlikely(mas.last < last))
 292		return -EEXIST;
 293
 294	mas.index = addr;
 295	mas.last = last;
 296
 297	mas_store_gfp(&mas, reg, GFP_KERNEL);
 298
 299	reg->uvmm = uvmm;
 300
 301	return 0;
 302}
 303
 304static int
 305nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm,
 306			   struct nouveau_uvma_region *reg,
 307			   u64 addr, u64 range)
 308{
 309	int ret;
 310
 311	reg->uvmm = uvmm;
 312	reg->va.addr = addr;
 313	reg->va.range = range;
 314
 315	ret = __nouveau_uvma_region_insert(uvmm, reg);
 316	if (ret)
 317		return ret;
 318
 319	return 0;
 320}
 321
 322static void
 323nouveau_uvma_region_remove(struct nouveau_uvma_region *reg)
 324{
 325	struct nouveau_uvmm *uvmm = reg->uvmm;
 326	MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0);
 327
 328	mas_erase(&mas);
 329}
 330
 331static int
 332nouveau_uvma_region_create(struct nouveau_uvmm *uvmm,
 333			   u64 addr, u64 range)
 334{
 335	struct nouveau_uvma_region *reg;
 336	int ret;
 337
 338	if (!drm_gpuvm_interval_empty(&uvmm->base, addr, range))
 339		return -ENOSPC;
 340
 341	ret = nouveau_uvma_region_alloc(&reg);
 342	if (ret)
 343		return ret;
 344
 345	ret = nouveau_uvma_region_insert(uvmm, reg, addr, range);
 346	if (ret)
 347		goto err_free_region;
 348
 349	ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range);
 350	if (ret)
 351		goto err_region_remove;
 352
 353	return 0;
 354
 355err_region_remove:
 356	nouveau_uvma_region_remove(reg);
 357err_free_region:
 358	nouveau_uvma_region_put(reg);
 359	return ret;
 360}
 361
 362static struct nouveau_uvma_region *
 363nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm,
 364			       u64 addr, u64 range)
 365{
 366	MA_STATE(mas, &uvmm->region_mt, addr, 0);
 367
 368	return mas_find(&mas, addr + range - 1);
 369}
 370
 371static struct nouveau_uvma_region *
 372nouveau_uvma_region_find(struct nouveau_uvmm *uvmm,
 373			 u64 addr, u64 range)
 374{
 375	struct nouveau_uvma_region *reg;
 376
 377	reg = nouveau_uvma_region_find_first(uvmm, addr, range);
 378	if (!reg)
 379		return NULL;
 380
 381	if (reg->va.addr != addr ||
 382	    reg->va.range != range)
 383		return NULL;
 384
 385	return reg;
 386}
 387
 388static bool
 389nouveau_uvma_region_empty(struct nouveau_uvma_region *reg)
 390{
 391	struct nouveau_uvmm *uvmm = reg->uvmm;
 392
 393	return drm_gpuvm_interval_empty(&uvmm->base,
 394					reg->va.addr,
 395					reg->va.range);
 396}
 397
 398static int
 399__nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg)
 400{
 401	struct nouveau_uvmm *uvmm = reg->uvmm;
 402	u64 addr = reg->va.addr;
 403	u64 range = reg->va.range;
 404
 405	if (!nouveau_uvma_region_empty(reg))
 406		return -EBUSY;
 407
 408	nouveau_uvma_region_remove(reg);
 409	nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range);
 410	nouveau_uvma_region_put(reg);
 411
 412	return 0;
 413}
 414
 415static int
 416nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm,
 417			    u64 addr, u64 range)
 418{
 419	struct nouveau_uvma_region *reg;
 420
 421	reg = nouveau_uvma_region_find(uvmm, addr, range);
 422	if (!reg)
 423		return -ENOENT;
 424
 425	return __nouveau_uvma_region_destroy(reg);
 426}
 427
 428static void
 429nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg)
 430{
 431
 432	init_completion(&reg->complete);
 433	reg->dirty = true;
 434}
 435
 436static void
 437nouveau_uvma_region_complete(struct nouveau_uvma_region *reg)
 438{
 439	complete_all(&reg->complete);
 440}
 441
 442static void
 443op_map_prepare_unwind(struct nouveau_uvma *uvma)
 444{
 445	struct drm_gpuva *va = &uvma->va;
 446	nouveau_uvma_gem_put(uvma);
 447	drm_gpuva_remove(va);
 448	nouveau_uvma_free(uvma);
 449}
 450
 451static void
 452op_unmap_prepare_unwind(struct drm_gpuva *va)
 453{
 454	drm_gpuva_insert(va->vm, va);
 455}
 456
 457static bool
 458op_map_aligned_to_page_shift(const struct drm_gpuva_op_map *op, u8 page_shift)
 459{
 460	u64 non_page_bits = (1ULL << page_shift) - 1;
 461
 462	return (op->va.addr & non_page_bits) == 0 &&
 463	       (op->va.range & non_page_bits) == 0 &&
 464	       (op->gem.offset & non_page_bits) == 0;
 465}
 466
 467static u8
 468select_page_shift(struct nouveau_uvmm *uvmm, struct drm_gpuva_op_map *op)
 469{
 470	struct nouveau_bo *nvbo = nouveau_gem_object(op->gem.obj);
 471
 472	/* nouveau_bo_fixup_align() guarantees that the page size will be aligned
 473	 * for most cases, but it can't handle cases where userspace allocates with
 474	 * a size and then binds with a smaller granularity. So in order to avoid
 475	 * breaking old userspace, we need to ensure that the VA is actually
 476	 * aligned before using it, and if it isn't, then we downgrade to the first
 477	 * granularity that will fit, which is optimal from a correctness and
 478	 * performance perspective.
 479	 */
 480	if (op_map_aligned_to_page_shift(op, nvbo->page))
 481		return nvbo->page;
 482
 483	struct nouveau_mem *mem = nouveau_mem(nvbo->bo.resource);
 484	struct nvif_vmm *vmm = &uvmm->vmm.vmm;
 485	int i;
 486
 487	/* If the given granularity doesn't fit, let's find one that will fit. */
 488	for (i = 0; i < vmm->page_nr; i++) {
 489		/* Ignore anything that is bigger or identical to the BO preference. */
 490		if (vmm->page[i].shift >= nvbo->page)
 491			continue;
 492
 493		/* Skip incompatible domains. */
 494		if ((mem->mem.type & NVIF_MEM_VRAM) && !vmm->page[i].vram)
 495			continue;
 496		if ((mem->mem.type & NVIF_MEM_HOST) &&
 497		    (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
 498			continue;
 499
 500		/* If it fits, return the proposed shift. */
 501		if (op_map_aligned_to_page_shift(op, vmm->page[i].shift))
 502			return vmm->page[i].shift;
 503	}
 504
 505	/* If we get here then nothing can reconcile the requirements. This should never
 506	 * happen.
 507	 */
 508	drm_WARN_ONCE(op->gem.obj->dev, 1, "Could not find an appropriate page size.\n");
 509
 510	return PAGE_SHIFT;
 511}
 512
 513static void
 514nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm,
 515			       struct nouveau_uvma_prealloc *new,
 516			       struct drm_gpuva_ops *ops,
 517			       struct drm_gpuva_op *last,
 518			       struct uvmm_map_args *args)
 519{
 520	struct drm_gpuva_op *op = last;
 521	u64 vmm_get_start = args ? args->addr : 0;
 522	u64 vmm_get_end = args ? args->addr + args->range : 0;
 523
 524	/* Unwind GPUVA space. */
 525	drm_gpuva_for_each_op_from_reverse(op, ops) {
 526		switch (op->op) {
 527		case DRM_GPUVA_OP_MAP:
 528			op_map_prepare_unwind(new->map);
 529			break;
 530		case DRM_GPUVA_OP_REMAP: {
 531			struct drm_gpuva_op_remap *r = &op->remap;
 532			struct drm_gpuva *va = r->unmap->va;
 533
 534			if (r->next)
 535				op_map_prepare_unwind(new->next);
 536
 537			if (r->prev)
 538				op_map_prepare_unwind(new->prev);
 539
 540			op_unmap_prepare_unwind(va);
 541			break;
 542		}
 543		case DRM_GPUVA_OP_UNMAP:
 544			op_unmap_prepare_unwind(op->unmap.va);
 545			break;
 546		default:
 547			break;
 548		}
 549	}
 550
 551	/* Unmap operation don't allocate page tables, hence skip the following
 552	 * page table unwind.
 553	 */
 554	if (!args)
 555		return;
 556
 557	drm_gpuva_for_each_op(op, ops) {
 558		switch (op->op) {
 559		case DRM_GPUVA_OP_MAP: {
 560			u64 vmm_get_range = vmm_get_end - vmm_get_start;
 561
 562			if (vmm_get_range)
 563				nouveau_uvmm_vmm_put(uvmm, vmm_get_start,
 564						     vmm_get_range,
 565						     select_page_shift(uvmm, &op->map));
 566			break;
 567		}
 568		case DRM_GPUVA_OP_REMAP: {
 569			struct drm_gpuva_op_remap *r = &op->remap;
 570			struct drm_gpuva *va = r->unmap->va;
 571			u64 ustart = va->va.addr;
 572			u64 urange = va->va.range;
 573			u64 uend = ustart + urange;
 574
 575			if (r->prev)
 576				vmm_get_start = uend;
 577
 578			if (r->next)
 579				vmm_get_end = ustart;
 580
 581			if (r->prev && r->next)
 582				vmm_get_start = vmm_get_end = 0;
 583
 584			break;
 585		}
 586		case DRM_GPUVA_OP_UNMAP: {
 587			struct drm_gpuva_op_unmap *u = &op->unmap;
 588			struct drm_gpuva *va = u->va;
 589			u64 ustart = va->va.addr;
 590			u64 urange = va->va.range;
 591			u64 uend = ustart + urange;
 592			u8 page_shift = uvma_from_va(va)->page_shift;
 593
 594			/* Nothing to do for mappings we merge with. */
 595			if (uend == vmm_get_start ||
 596			    ustart == vmm_get_end)
 597				break;
 598
 599			if (ustart > vmm_get_start) {
 600				u64 vmm_get_range = ustart - vmm_get_start;
 601
 602				nouveau_uvmm_vmm_put(uvmm, vmm_get_start,
 603						     vmm_get_range,
 604						     page_shift);
 605			}
 606			vmm_get_start = uend;
 607			break;
 608		}
 609		default:
 610			break;
 611		}
 612
 613		if (op == last)
 614			break;
 615	}
 616}
 617
 618static void
 619nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm,
 620				   struct nouveau_uvma_prealloc *new,
 621				   struct drm_gpuva_ops *ops,
 622				   u64 addr, u64 range)
 623{
 624	struct drm_gpuva_op *last = drm_gpuva_last_op(ops);
 625	struct uvmm_map_args args = {
 626		.addr = addr,
 627		.range = range,
 628	};
 629
 630	nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, &args);
 631}
 632
 633static void
 634nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm,
 635				     struct nouveau_uvma_prealloc *new,
 636				     struct drm_gpuva_ops *ops)
 637{
 638	struct drm_gpuva_op *last = drm_gpuva_last_op(ops);
 639
 640	nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL);
 641}
 642
 643static int
 644op_map_prepare(struct nouveau_uvmm *uvmm,
 645	       struct nouveau_uvma **puvma,
 646	       struct drm_gpuva_op_map *op,
 647	       struct uvmm_map_args *args)
 648{
 649	struct nouveau_uvma *uvma;
 650	int ret;
 651
 652	ret = nouveau_uvma_alloc(&uvma);
 653	if (ret)
 654		return ret;
 655
 656	uvma->region = args->region;
 657	uvma->kind = args->kind;
 658	uvma->page_shift = select_page_shift(uvmm, op);
 659
 660	drm_gpuva_map(&uvmm->base, &uvma->va, op);
 661
 662	/* Keep a reference until this uvma is destroyed. */
 663	nouveau_uvma_gem_get(uvma);
 664
 665	*puvma = uvma;
 666	return 0;
 667}
 668
 669static void
 670op_unmap_prepare(struct drm_gpuva_op_unmap *u)
 671{
 672	drm_gpuva_unmap(u);
 673}
 674
 675/*
 676 * Note: @args should not be NULL when calling for a map operation.
 677 */
 678static int
 679nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm,
 680			struct nouveau_uvma_prealloc *new,
 681			struct drm_gpuva_ops *ops,
 682			struct uvmm_map_args *args)
 683{
 684	struct drm_gpuva_op *op;
 685	u64 vmm_get_start = args ? args->addr : 0;
 686	u64 vmm_get_end = args ? args->addr + args->range : 0;
 687	int ret;
 688
 689	drm_gpuva_for_each_op(op, ops) {
 690		switch (op->op) {
 691		case DRM_GPUVA_OP_MAP: {
 692			u64 vmm_get_range = vmm_get_end - vmm_get_start;
 693
 694			ret = op_map_prepare(uvmm, &new->map, &op->map, args);
 695			if (ret)
 696				goto unwind;
 697
 698			if (vmm_get_range) {
 699				ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,
 700							   vmm_get_range,
 701							   new->map->page_shift);
 702				if (ret) {
 703					op_map_prepare_unwind(new->map);
 704					goto unwind;
 705				}
 706			}
 707
 708			break;
 709		}
 710		case DRM_GPUVA_OP_REMAP: {
 711			struct drm_gpuva_op_remap *r = &op->remap;
 712			struct drm_gpuva *va = r->unmap->va;
 713			struct uvmm_map_args remap_args = {
 714				.kind = uvma_from_va(va)->kind,
 715				.region = uvma_from_va(va)->region,
 716			};
 717			u64 ustart = va->va.addr;
 718			u64 urange = va->va.range;
 719			u64 uend = ustart + urange;
 720
 721			op_unmap_prepare(r->unmap);
 722
 723			if (r->prev) {
 724				ret = op_map_prepare(uvmm, &new->prev, r->prev,
 725						     &remap_args);
 726				if (ret)
 727					goto unwind;
 728
 729				if (args)
 730					vmm_get_start = uend;
 731			}
 732
 733			if (r->next) {
 734				ret = op_map_prepare(uvmm, &new->next, r->next,
 735						     &remap_args);
 736				if (ret) {
 737					if (r->prev)
 738						op_map_prepare_unwind(new->prev);
 739					goto unwind;
 740				}
 741
 742				if (args)
 743					vmm_get_end = ustart;
 744			}
 745
 746			if (args && (r->prev && r->next))
 747				vmm_get_start = vmm_get_end = 0;
 748
 749			break;
 750		}
 751		case DRM_GPUVA_OP_UNMAP: {
 752			struct drm_gpuva_op_unmap *u = &op->unmap;
 753			struct drm_gpuva *va = u->va;
 754			u64 ustart = va->va.addr;
 755			u64 urange = va->va.range;
 756			u64 uend = ustart + urange;
 757			u8 page_shift = uvma_from_va(va)->page_shift;
 758
 759			op_unmap_prepare(u);
 760
 761			if (!args)
 762				break;
 763
 764			/* Nothing to do for mappings we merge with. */
 765			if (uend == vmm_get_start ||
 766			    ustart == vmm_get_end)
 767				break;
 768
 769			if (ustart > vmm_get_start) {
 770				u64 vmm_get_range = ustart - vmm_get_start;
 771
 772				ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start,
 773							   vmm_get_range, page_shift);
 774				if (ret) {
 775					op_unmap_prepare_unwind(va);
 776					goto unwind;
 777				}
 778			}
 779			vmm_get_start = uend;
 780
 781			break;
 782		}
 783		default:
 784			ret = -EINVAL;
 785			goto unwind;
 786		}
 787	}
 788
 789	return 0;
 790
 791unwind:
 792	if (op != drm_gpuva_first_op(ops))
 793		nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops,
 794					       drm_gpuva_prev_op(op),
 795					       args);
 796	return ret;
 797}
 798
 799static int
 800nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm,
 801			    struct nouveau_uvma_prealloc *new,
 802			    struct nouveau_uvma_region *region,
 803			    struct drm_gpuva_ops *ops,
 804			    u64 addr, u64 range, u8 kind)
 805{
 806	struct uvmm_map_args args = {
 807		.region = region,
 808		.addr = addr,
 809		.range = range,
 810		.kind = kind,
 811	};
 812
 813	return nouveau_uvmm_sm_prepare(uvmm, new, ops, &args);
 814}
 815
 816static int
 817nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm,
 818			      struct nouveau_uvma_prealloc *new,
 819			      struct drm_gpuva_ops *ops)
 820{
 821	return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL);
 822}
 823
 824static struct drm_gem_object *
 825op_gem_obj(struct drm_gpuva_op *op)
 826{
 827	switch (op->op) {
 828	case DRM_GPUVA_OP_MAP:
 829		return op->map.gem.obj;
 830	case DRM_GPUVA_OP_REMAP:
 831		/* Actually, we're looking for the GEMs backing remap.prev and
 832		 * remap.next, but since this is a remap they're identical to
 833		 * the GEM backing the unmapped GPUVA.
 834		 */
 835		return op->remap.unmap->va->gem.obj;
 836	case DRM_GPUVA_OP_UNMAP:
 837		return op->unmap.va->gem.obj;
 838	default:
 839		WARN(1, "Unknown operation.\n");
 840		return NULL;
 841	}
 842}
 843
 844static void
 845op_map(struct nouveau_uvma *uvma)
 846{
 847	struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj);
 848
 849	nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource));
 850}
 851
 852static void
 853op_unmap(struct drm_gpuva_op_unmap *u)
 854{
 855	struct drm_gpuva *va = u->va;
 856	struct nouveau_uvma *uvma = uvma_from_va(va);
 857
 858	/* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */
 859	if (!u->keep)
 860		nouveau_uvma_unmap(uvma);
 861}
 862
 863static void
 864op_unmap_range(struct drm_gpuva_op_unmap *u,
 865	       u64 addr, u64 range)
 866{
 867	struct nouveau_uvma *uvma = uvma_from_va(u->va);
 868	u8 page_shift = uvma->page_shift;
 869	bool sparse = !!uvma->region;
 870
 871	if (!drm_gpuva_invalidated(u->va))
 872		nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, page_shift, sparse);
 873}
 874
 875static void
 876op_remap(struct drm_gpuva_op_remap *r,
 877	 struct nouveau_uvma_prealloc *new)
 878{
 879	struct drm_gpuva_op_unmap *u = r->unmap;
 880	struct nouveau_uvma *uvma = uvma_from_va(u->va);
 881	u64 addr = uvma->va.va.addr;
 882	u64 end = uvma->va.va.addr + uvma->va.va.range;
 883
 884	if (r->prev)
 885		addr = r->prev->va.addr + r->prev->va.range;
 886
 887	if (r->next)
 888		end = r->next->va.addr;
 889
 890	op_unmap_range(u, addr, end - addr);
 891}
 892
 893static int
 894nouveau_uvmm_sm(struct nouveau_uvmm *uvmm,
 895		struct nouveau_uvma_prealloc *new,
 896		struct drm_gpuva_ops *ops)
 897{
 898	struct drm_gpuva_op *op;
 899
 900	drm_gpuva_for_each_op(op, ops) {
 901		switch (op->op) {
 902		case DRM_GPUVA_OP_MAP:
 903			op_map(new->map);
 904			break;
 905		case DRM_GPUVA_OP_REMAP:
 906			op_remap(&op->remap, new);
 907			break;
 908		case DRM_GPUVA_OP_UNMAP:
 909			op_unmap(&op->unmap);
 910			break;
 911		default:
 912			break;
 913		}
 914	}
 915
 916	return 0;
 917}
 918
 919static int
 920nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm,
 921		    struct nouveau_uvma_prealloc *new,
 922		    struct drm_gpuva_ops *ops)
 923{
 924	return nouveau_uvmm_sm(uvmm, new, ops);
 925}
 926
 927static int
 928nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm,
 929		      struct nouveau_uvma_prealloc *new,
 930		      struct drm_gpuva_ops *ops)
 931{
 932	return nouveau_uvmm_sm(uvmm, new, ops);
 933}
 934
 935static void
 936nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm,
 937			struct nouveau_uvma_prealloc *new,
 938			struct drm_gpuva_ops *ops, bool unmap)
 939{
 940	struct drm_gpuva_op *op;
 941
 942	drm_gpuva_for_each_op(op, ops) {
 943		switch (op->op) {
 944		case DRM_GPUVA_OP_MAP:
 945			break;
 946		case DRM_GPUVA_OP_REMAP: {
 947			struct drm_gpuva_op_remap *r = &op->remap;
 948			struct drm_gpuva_op_map *p = r->prev;
 949			struct drm_gpuva_op_map *n = r->next;
 950			struct drm_gpuva *va = r->unmap->va;
 951			struct nouveau_uvma *uvma = uvma_from_va(va);
 952			u8 page_shift = uvma->page_shift;
 953
 954			if (unmap) {
 955				u64 addr = va->va.addr;
 956				u64 end = addr + va->va.range;
 957
 958				if (p)
 959					addr = p->va.addr + p->va.range;
 960
 961				if (n)
 962					end = n->va.addr;
 963
 964				nouveau_uvmm_vmm_put(uvmm, addr, end - addr, page_shift);
 965			}
 966
 967			nouveau_uvma_gem_put(uvma);
 968			nouveau_uvma_free(uvma);
 969			break;
 970		}
 971		case DRM_GPUVA_OP_UNMAP: {
 972			struct drm_gpuva_op_unmap *u = &op->unmap;
 973			struct drm_gpuva *va = u->va;
 974			struct nouveau_uvma *uvma = uvma_from_va(va);
 975
 976			if (unmap)
 977				nouveau_uvma_vmm_put(uvma);
 978
 979			nouveau_uvma_gem_put(uvma);
 980			nouveau_uvma_free(uvma);
 981			break;
 982		}
 983		default:
 984			break;
 985		}
 986	}
 987}
 988
 989static void
 990nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm,
 991			    struct nouveau_uvma_prealloc *new,
 992			    struct drm_gpuva_ops *ops)
 993{
 994	nouveau_uvmm_sm_cleanup(uvmm, new, ops, false);
 995}
 996
 997static void
 998nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm,
 999			      struct nouveau_uvma_prealloc *new,
1000			      struct drm_gpuva_ops *ops)
1001{
1002	nouveau_uvmm_sm_cleanup(uvmm, new, ops, true);
1003}
1004
1005static int
1006nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range)
1007{
1008	if (addr & ~PAGE_MASK)
1009		return -EINVAL;
1010
1011	if (range & ~PAGE_MASK)
1012		return -EINVAL;
1013
1014	if (!drm_gpuvm_range_valid(&uvmm->base, addr, range))
1015		return -EINVAL;
1016
1017	return 0;
1018}
1019
1020static int
1021nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob)
1022{
1023	*pjob = kzalloc(sizeof(**pjob), GFP_KERNEL);
1024	if (!*pjob)
1025		return -ENOMEM;
1026
1027	kref_init(&(*pjob)->kref);
1028
1029	return 0;
1030}
1031
1032static void
1033nouveau_uvmm_bind_job_free(struct kref *kref)
1034{
1035	struct nouveau_uvmm_bind_job *job =
1036		container_of(kref, struct nouveau_uvmm_bind_job, kref);
1037	struct bind_job_op *op, *next;
1038
1039	list_for_each_op_safe(op, next, &job->ops) {
1040		list_del(&op->entry);
1041		kfree(op);
1042	}
1043
1044	nouveau_job_free(&job->base);
1045	kfree(job);
1046}
1047
1048static void
1049nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job)
1050{
1051	kref_get(&job->kref);
1052}
1053
1054static void
1055nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job)
1056{
1057	kref_put(&job->kref, nouveau_uvmm_bind_job_free);
1058}
1059
1060static int
1061bind_validate_op(struct nouveau_job *job,
1062		 struct bind_job_op *op)
1063{
1064	struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1065	struct drm_gem_object *obj = op->gem.obj;
1066
1067	if (op->op == OP_MAP) {
1068		if (op->gem.offset & ~PAGE_MASK)
1069			return -EINVAL;
1070
1071		if (obj->size <= op->gem.offset)
1072			return -EINVAL;
1073
1074		if (op->va.range > (obj->size - op->gem.offset))
1075			return -EINVAL;
1076	}
1077
1078	return nouveau_uvmm_validate_range(uvmm, op->va.addr, op->va.range);
1079}
1080
1081static void
1082bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range)
1083{
1084	struct nouveau_sched *sched = job->sched;
1085	struct nouveau_job *__job;
1086	struct bind_job_op *op;
1087	u64 end = addr + range;
1088
1089again:
1090	spin_lock(&sched->job.list.lock);
1091	list_for_each_entry(__job, &sched->job.list.head, entry) {
1092		struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(__job);
1093
1094		list_for_each_op(op, &bind_job->ops) {
1095			if (op->op == OP_UNMAP) {
1096				u64 op_addr = op->va.addr;
1097				u64 op_end = op_addr + op->va.range;
1098
1099				if (!(end <= op_addr || addr >= op_end)) {
1100					nouveau_uvmm_bind_job_get(bind_job);
1101					spin_unlock(&sched->job.list.lock);
1102					wait_for_completion(&bind_job->complete);
1103					nouveau_uvmm_bind_job_put(bind_job);
1104					goto again;
1105				}
1106			}
1107		}
1108	}
1109	spin_unlock(&sched->job.list.lock);
1110}
1111
1112static int
1113bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range,
1114			 bool sparse)
1115{
1116	struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1117	struct nouveau_uvma_region *reg;
1118	u64 reg_addr, reg_end;
1119	u64 end = addr + range;
1120
1121again:
1122	nouveau_uvmm_lock(uvmm);
1123	reg = nouveau_uvma_region_find_first(uvmm, addr, range);
1124	if (!reg) {
1125		nouveau_uvmm_unlock(uvmm);
1126		return 0;
1127	}
1128
1129	/* Generally, job submits are serialized, hence only
1130	 * dirty regions can be modified concurrently.
1131	 */
1132	if (reg->dirty) {
1133		nouveau_uvma_region_get(reg);
1134		nouveau_uvmm_unlock(uvmm);
1135		wait_for_completion(&reg->complete);
1136		nouveau_uvma_region_put(reg);
1137		goto again;
1138	}
1139	nouveau_uvmm_unlock(uvmm);
1140
1141	if (sparse)
1142		return -ENOSPC;
1143
1144	reg_addr = reg->va.addr;
1145	reg_end = reg_addr + reg->va.range;
1146
1147	/* Make sure the mapping is either outside of a
1148	 * region or fully enclosed by a region.
1149	 */
1150	if (reg_addr > addr || reg_end < end)
1151		return -ENOSPC;
1152
1153	return 0;
1154}
1155
1156static int
1157bind_validate_region(struct nouveau_job *job)
1158{
1159	struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1160	struct bind_job_op *op;
1161	int ret;
1162
1163	list_for_each_op(op, &bind_job->ops) {
1164		u64 op_addr = op->va.addr;
1165		u64 op_range = op->va.range;
1166		bool sparse = false;
1167
1168		switch (op->op) {
1169		case OP_MAP_SPARSE:
1170			sparse = true;
1171			bind_validate_map_sparse(job, op_addr, op_range);
1172			fallthrough;
1173		case OP_MAP:
1174			ret = bind_validate_map_common(job, op_addr, op_range,
1175						       sparse);
1176			if (ret)
1177				return ret;
1178			break;
1179		default:
1180			break;
1181		}
1182	}
1183
1184	return 0;
1185}
1186
1187static void
1188bind_link_gpuvas(struct bind_job_op *bop)
1189{
1190	struct nouveau_uvma_prealloc *new = &bop->new;
1191	struct drm_gpuvm_bo *vm_bo = bop->vm_bo;
1192	struct drm_gpuva_ops *ops = bop->ops;
1193	struct drm_gpuva_op *op;
1194
1195	drm_gpuva_for_each_op(op, ops) {
1196		switch (op->op) {
1197		case DRM_GPUVA_OP_MAP:
1198			drm_gpuva_link(&new->map->va, vm_bo);
1199			break;
1200		case DRM_GPUVA_OP_REMAP: {
1201			struct drm_gpuva *va = op->remap.unmap->va;
1202
1203			if (op->remap.prev)
1204				drm_gpuva_link(&new->prev->va, va->vm_bo);
1205			if (op->remap.next)
1206				drm_gpuva_link(&new->next->va, va->vm_bo);
1207			drm_gpuva_unlink(va);
1208			break;
1209		}
1210		case DRM_GPUVA_OP_UNMAP:
1211			drm_gpuva_unlink(op->unmap.va);
1212			break;
1213		default:
1214			break;
1215		}
1216	}
1217}
1218
1219static int
1220bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec,
1221		   unsigned int num_fences)
1222{
1223	struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1224	struct bind_job_op *op;
1225	int ret;
1226
1227	list_for_each_op(op, &bind_job->ops) {
1228		struct drm_gpuva_op *va_op;
1229
1230		if (!op->ops)
1231			continue;
1232
1233		drm_gpuva_for_each_op(va_op, op->ops) {
1234			struct drm_gem_object *obj = op_gem_obj(va_op);
1235
1236			if (unlikely(!obj))
1237				continue;
1238
1239			ret = drm_exec_prepare_obj(exec, obj, num_fences);
1240			if (ret)
1241				return ret;
1242
1243			/* Don't validate GEMs backing mappings we're about to
1244			 * unmap, it's not worth the effort.
1245			 */
1246			if (va_op->op == DRM_GPUVA_OP_UNMAP)
1247				continue;
1248
1249			ret = nouveau_bo_validate(nouveau_gem_object(obj),
1250						  true, false);
1251			if (ret)
1252				return ret;
1253		}
1254	}
1255
1256	return 0;
1257}
1258
1259static int
1260nouveau_uvmm_bind_job_submit(struct nouveau_job *job,
1261			     struct drm_gpuvm_exec *vme)
1262{
1263	struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1264	struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1265	struct drm_exec *exec = &vme->exec;
1266	struct bind_job_op *op;
1267	int ret;
1268
1269	list_for_each_op(op, &bind_job->ops) {
1270		if (op->op == OP_MAP) {
1271			struct drm_gem_object *obj = op->gem.obj =
1272				drm_gem_object_lookup(job->file_priv,
1273						      op->gem.handle);
1274			if (!obj)
1275				return -ENOENT;
1276
1277			dma_resv_lock(obj->resv, NULL);
1278			op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);
1279			dma_resv_unlock(obj->resv);
1280			if (IS_ERR(op->vm_bo))
1281				return PTR_ERR(op->vm_bo);
1282
1283			drm_gpuvm_bo_extobj_add(op->vm_bo);
1284		}
1285
1286		ret = bind_validate_op(job, op);
1287		if (ret)
1288			return ret;
1289	}
1290
1291	/* If a sparse region or mapping overlaps a dirty region, we need to
1292	 * wait for the region to complete the unbind process. This is due to
1293	 * how page table management is currently implemented. A future
1294	 * implementation might change this.
1295	 */
1296	ret = bind_validate_region(job);
1297	if (ret)
1298		return ret;
1299
1300	/* Once we start modifying the GPU VA space we need to keep holding the
1301	 * uvmm lock until we can't fail anymore. This is due to the set of GPU
1302	 * VA space changes must appear atomically and we need to be able to
1303	 * unwind all GPU VA space changes on failure.
1304	 */
1305	nouveau_uvmm_lock(uvmm);
1306
1307	list_for_each_op(op, &bind_job->ops) {
1308		switch (op->op) {
1309		case OP_MAP_SPARSE:
1310			ret = nouveau_uvma_region_create(uvmm,
1311							 op->va.addr,
1312							 op->va.range);
1313			if (ret)
1314				goto unwind_continue;
1315
1316			break;
1317		case OP_UNMAP_SPARSE:
1318			op->reg = nouveau_uvma_region_find(uvmm, op->va.addr,
1319							   op->va.range);
1320			if (!op->reg || op->reg->dirty) {
1321				ret = -ENOENT;
1322				goto unwind_continue;
1323			}
1324
1325			op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,
1326								op->va.addr,
1327								op->va.range);
1328			if (IS_ERR(op->ops)) {
1329				ret = PTR_ERR(op->ops);
1330				goto unwind_continue;
1331			}
1332
1333			ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,
1334							    op->ops);
1335			if (ret) {
1336				drm_gpuva_ops_free(&uvmm->base, op->ops);
1337				op->ops = NULL;
1338				op->reg = NULL;
1339				goto unwind_continue;
1340			}
1341
1342			nouveau_uvma_region_dirty(op->reg);
1343
1344			break;
1345		case OP_MAP: {
1346			struct nouveau_uvma_region *reg;
1347			struct drm_gpuvm_map_req map_req = {
1348				.map.va.addr = op->va.addr,
1349				.map.va.range = op->va.range,
1350				.map.gem.obj = op->gem.obj,
1351				.map.gem.offset = op->gem.offset,
1352			};
1353
1354			reg = nouveau_uvma_region_find_first(uvmm,
1355							     op->va.addr,
1356							     op->va.range);
1357			if (reg) {
1358				u64 reg_addr = reg->va.addr;
1359				u64 reg_end = reg_addr + reg->va.range;
1360				u64 op_addr = op->va.addr;
1361				u64 op_end = op_addr + op->va.range;
1362
1363				if (unlikely(reg->dirty)) {
1364					ret = -EINVAL;
1365					goto unwind_continue;
1366				}
1367
1368				/* Make sure the mapping is either outside of a
1369				 * region or fully enclosed by a region.
1370				 */
1371				if (reg_addr > op_addr || reg_end < op_end) {
1372					ret = -ENOSPC;
1373					goto unwind_continue;
1374				}
1375			}
1376
1377			op->ops = drm_gpuvm_sm_map_ops_create(&uvmm->base,
1378							      &map_req);
1379			if (IS_ERR(op->ops)) {
1380				ret = PTR_ERR(op->ops);
1381				goto unwind_continue;
1382			}
1383
1384			ret = nouveau_uvmm_sm_map_prepare(uvmm, &op->new,
1385							  reg, op->ops,
1386							  op->va.addr,
1387							  op->va.range,
1388							  op->flags & 0xff);
1389			if (ret) {
1390				drm_gpuva_ops_free(&uvmm->base, op->ops);
1391				op->ops = NULL;
1392				goto unwind_continue;
1393			}
1394
1395			break;
1396		}
1397		case OP_UNMAP:
1398			op->ops = drm_gpuvm_sm_unmap_ops_create(&uvmm->base,
1399								op->va.addr,
1400								op->va.range);
1401			if (IS_ERR(op->ops)) {
1402				ret = PTR_ERR(op->ops);
1403				goto unwind_continue;
1404			}
1405
1406			ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new,
1407							    op->ops);
1408			if (ret) {
1409				drm_gpuva_ops_free(&uvmm->base, op->ops);
1410				op->ops = NULL;
1411				goto unwind_continue;
1412			}
1413
1414			break;
1415		default:
1416			ret = -EINVAL;
1417			goto unwind_continue;
1418		}
1419	}
1420
1421	drm_exec_init(exec, vme->flags, 0);
1422	drm_exec_until_all_locked(exec) {
1423		ret = bind_lock_validate(job, exec, vme->num_fences);
1424		drm_exec_retry_on_contention(exec);
1425		if (ret) {
1426			op = list_last_op(&bind_job->ops);
1427			goto unwind;
1428		}
1429	}
1430
1431	/* Link and unlink GPUVAs while holding the dma_resv lock.
1432	 *
1433	 * As long as we validate() all GEMs and add fences to all GEMs DMA
1434	 * reservations backing map and remap operations we can be sure there
1435	 * won't be any concurrent (in)validations during job execution, hence
1436	 * we're safe to check drm_gpuva_invalidated() within the fence
1437	 * signalling critical path without holding a separate lock.
1438	 *
1439	 * GPUVAs about to be unmapped are safe as well, since they're unlinked
1440	 * already.
1441	 *
1442	 * GEMs from map and remap operations must be validated before linking
1443	 * their corresponding mappings to prevent the actual PT update to
1444	 * happen right away in validate() rather than asynchronously as
1445	 * intended.
1446	 *
1447	 * Note that after linking and unlinking the GPUVAs in this loop this
1448	 * function cannot fail anymore, hence there is no need for an unwind
1449	 * path.
1450	 */
1451	list_for_each_op(op, &bind_job->ops) {
1452		switch (op->op) {
1453		case OP_UNMAP_SPARSE:
1454		case OP_MAP:
1455		case OP_UNMAP:
1456			bind_link_gpuvas(op);
1457			break;
1458		default:
1459			break;
1460		}
1461	}
1462	nouveau_uvmm_unlock(uvmm);
1463
1464	return 0;
1465
1466unwind_continue:
1467	op = list_prev_op(op);
1468unwind:
1469	list_for_each_op_from_reverse(op, &bind_job->ops) {
1470		switch (op->op) {
1471		case OP_MAP_SPARSE:
1472			nouveau_uvma_region_destroy(uvmm, op->va.addr,
1473						    op->va.range);
1474			break;
1475		case OP_UNMAP_SPARSE:
1476			__nouveau_uvma_region_insert(uvmm, op->reg);
1477			nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,
1478							     op->ops);
1479			break;
1480		case OP_MAP:
1481			nouveau_uvmm_sm_map_prepare_unwind(uvmm, &op->new,
1482							   op->ops,
1483							   op->va.addr,
1484							   op->va.range);
1485			break;
1486		case OP_UNMAP:
1487			nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new,
1488							     op->ops);
1489			break;
1490		}
1491
1492		drm_gpuva_ops_free(&uvmm->base, op->ops);
1493		op->ops = NULL;
1494		op->reg = NULL;
1495	}
1496
1497	nouveau_uvmm_unlock(uvmm);
1498	drm_gpuvm_exec_unlock(vme);
1499	return ret;
1500}
1501
1502static void
1503nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job,
1504				   struct drm_gpuvm_exec *vme)
1505{
1506	drm_gpuvm_exec_resv_add_fence(vme, job->done_fence,
1507				      job->resv_usage, job->resv_usage);
1508	drm_gpuvm_exec_unlock(vme);
1509}
1510
1511static struct dma_fence *
1512nouveau_uvmm_bind_job_run(struct nouveau_job *job)
1513{
1514	struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1515	struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1516	struct bind_job_op *op;
1517	int ret = 0;
1518
1519	list_for_each_op(op, &bind_job->ops) {
1520		switch (op->op) {
1521		case OP_MAP_SPARSE:
1522			/* noop */
1523			break;
1524		case OP_MAP:
1525			ret = nouveau_uvmm_sm_map(uvmm, &op->new, op->ops);
1526			if (ret)
1527				goto out;
1528			break;
1529		case OP_UNMAP_SPARSE:
1530			fallthrough;
1531		case OP_UNMAP:
1532			ret = nouveau_uvmm_sm_unmap(uvmm, &op->new, op->ops);
1533			if (ret)
1534				goto out;
1535			break;
1536		}
1537	}
1538
1539out:
1540	if (ret)
1541		NV_PRINTK(err, job->cli, "bind job failed: %d\n", ret);
1542	return ERR_PTR(ret);
1543}
1544
1545static void
1546nouveau_uvmm_bind_job_cleanup(struct nouveau_job *job)
1547{
1548	struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job);
1549	struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli);
1550	struct bind_job_op *op;
1551
1552	list_for_each_op(op, &bind_job->ops) {
1553		struct drm_gem_object *obj = op->gem.obj;
1554
1555		/* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg
1556		 * will be NULL, hence skip the cleanup.
1557		 */
1558		switch (op->op) {
1559		case OP_MAP_SPARSE:
1560			/* noop */
1561			break;
1562		case OP_UNMAP_SPARSE:
1563			if (!IS_ERR_OR_NULL(op->ops))
1564				nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,
1565							      op->ops);
1566
1567			if (op->reg) {
1568				nouveau_uvma_region_sparse_unref(op->reg);
1569				nouveau_uvmm_lock(uvmm);
1570				nouveau_uvma_region_remove(op->reg);
1571				nouveau_uvmm_unlock(uvmm);
1572				nouveau_uvma_region_complete(op->reg);
1573				nouveau_uvma_region_put(op->reg);
1574			}
1575
1576			break;
1577		case OP_MAP:
1578			if (!IS_ERR_OR_NULL(op->ops))
1579				nouveau_uvmm_sm_map_cleanup(uvmm, &op->new,
1580							    op->ops);
1581			break;
1582		case OP_UNMAP:
1583			if (!IS_ERR_OR_NULL(op->ops))
1584				nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new,
1585							      op->ops);
1586			break;
1587		}
1588
1589		if (!IS_ERR_OR_NULL(op->ops))
1590			drm_gpuva_ops_free(&uvmm->base, op->ops);
1591
1592		if (!IS_ERR_OR_NULL(op->vm_bo)) {
1593			dma_resv_lock(obj->resv, NULL);
1594			drm_gpuvm_bo_put(op->vm_bo);
1595			dma_resv_unlock(obj->resv);
1596		}
1597
1598		if (obj)
1599			drm_gem_object_put(obj);
1600	}
1601
1602	nouveau_job_done(job);
1603	complete_all(&bind_job->complete);
1604
1605	nouveau_uvmm_bind_job_put(bind_job);
1606}
1607
1608static const struct nouveau_job_ops nouveau_bind_job_ops = {
1609	.submit = nouveau_uvmm_bind_job_submit,
1610	.armed_submit = nouveau_uvmm_bind_job_armed_submit,
1611	.run = nouveau_uvmm_bind_job_run,
1612	.free = nouveau_uvmm_bind_job_cleanup,
1613};
1614
1615static int
1616bind_job_op_from_uop(struct bind_job_op **pop,
1617		     struct drm_nouveau_vm_bind_op *uop)
1618{
1619	struct bind_job_op *op;
1620
1621	op = *pop = kzalloc(sizeof(*op), GFP_KERNEL);
1622	if (!op)
1623		return -ENOMEM;
1624
1625	switch (uop->op) {
1626	case OP_MAP:
1627		op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?
1628			 OP_MAP_SPARSE : OP_MAP;
1629		break;
1630	case OP_UNMAP:
1631		op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ?
1632			 OP_UNMAP_SPARSE : OP_UNMAP;
1633		break;
1634	default:
1635		op->op = uop->op;
1636		break;
1637	}
1638
1639	op->flags = uop->flags;
1640	op->va.addr = uop->addr;
1641	op->va.range = uop->range;
1642	op->gem.handle = uop->handle;
1643	op->gem.offset = uop->bo_offset;
1644
1645	return 0;
1646}
1647
1648static void
1649bind_job_ops_free(struct list_head *ops)
1650{
1651	struct bind_job_op *op, *next;
1652
1653	list_for_each_op_safe(op, next, ops) {
1654		list_del(&op->entry);
1655		kfree(op);
1656	}
1657}
1658
1659static int
1660nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob,
1661			   struct nouveau_uvmm_bind_job_args *__args)
1662{
1663	struct nouveau_uvmm_bind_job *job;
1664	struct nouveau_job_args args = {};
1665	struct bind_job_op *op;
1666	int i, ret;
1667
1668	ret = nouveau_uvmm_bind_job_alloc(&job);
1669	if (ret)
1670		return ret;
1671
1672	INIT_LIST_HEAD(&job->ops);
1673
1674	for (i = 0; i < __args->op.count; i++) {
1675		ret = bind_job_op_from_uop(&op, &__args->op.s[i]);
1676		if (ret)
1677			goto err_free;
1678
1679		list_add_tail(&op->entry, &job->ops);
1680	}
1681
1682	init_completion(&job->complete);
1683
1684	args.file_priv = __args->file_priv;
1685
1686	args.sched = __args->sched;
1687	args.credits = 1;
1688
1689	args.in_sync.count = __args->in_sync.count;
1690	args.in_sync.s = __args->in_sync.s;
1691
1692	args.out_sync.count = __args->out_sync.count;
1693	args.out_sync.s = __args->out_sync.s;
1694
1695	args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC);
1696	args.ops = &nouveau_bind_job_ops;
1697	args.resv_usage = DMA_RESV_USAGE_BOOKKEEP;
1698
1699	ret = nouveau_job_init(&job->base, &args);
1700	if (ret)
1701		goto err_free;
1702
1703	*pjob = job;
1704	return 0;
1705
1706err_free:
1707	bind_job_ops_free(&job->ops);
1708	kfree(job);
1709	*pjob = NULL;
1710
1711	return ret;
1712}
1713
1714static int
1715nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args)
1716{
1717	struct nouveau_uvmm_bind_job *job;
1718	int ret;
1719
1720	ret = nouveau_uvmm_bind_job_init(&job, args);
1721	if (ret)
1722		return ret;
1723
1724	ret = nouveau_job_submit(&job->base);
1725	if (ret)
1726		goto err_job_fini;
1727
1728	return 0;
1729
1730err_job_fini:
1731	nouveau_job_fini(&job->base);
1732	return ret;
1733}
1734
1735static int
1736nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args,
1737			   struct drm_nouveau_vm_bind *req)
1738{
1739	struct drm_nouveau_sync **s;
1740	u32 inc = req->wait_count;
1741	u64 ins = req->wait_ptr;
1742	u32 outc = req->sig_count;
1743	u64 outs = req->sig_ptr;
1744	u32 opc = req->op_count;
1745	u64 ops = req->op_ptr;
1746	int ret;
1747
1748	args->flags = req->flags;
1749
1750	if (opc) {
1751		args->op.count = opc;
1752		args->op.s = u_memcpya(ops, opc,
1753				       sizeof(*args->op.s));
1754		if (IS_ERR(args->op.s))
1755			return PTR_ERR(args->op.s);
1756	}
1757
1758	if (inc) {
1759		s = &args->in_sync.s;
1760
1761		args->in_sync.count = inc;
1762		*s = u_memcpya(ins, inc, sizeof(**s));
1763		if (IS_ERR(*s)) {
1764			ret = PTR_ERR(*s);
1765			goto err_free_ops;
1766		}
1767	}
1768
1769	if (outc) {
1770		s = &args->out_sync.s;
1771
1772		args->out_sync.count = outc;
1773		*s = u_memcpya(outs, outc, sizeof(**s));
1774		if (IS_ERR(*s)) {
1775			ret = PTR_ERR(*s);
1776			goto err_free_ins;
1777		}
1778	}
1779
1780	return 0;
1781
1782err_free_ops:
1783	u_free(args->op.s);
1784err_free_ins:
1785	u_free(args->in_sync.s);
1786	return ret;
1787}
1788
1789static void
1790nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args)
1791{
1792	u_free(args->op.s);
1793	u_free(args->in_sync.s);
1794	u_free(args->out_sync.s);
1795}
1796
1797int
1798nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev,
1799			   void *data,
1800			   struct drm_file *file_priv)
1801{
1802	struct nouveau_cli *cli = nouveau_cli(file_priv);
1803	struct nouveau_uvmm_bind_job_args args = {};
1804	struct drm_nouveau_vm_bind *req = data;
1805	int ret = 0;
1806
1807	if (unlikely(!nouveau_cli_uvmm_locked(cli)))
1808		return -ENOSYS;
1809
1810	ret = nouveau_uvmm_vm_bind_ucopy(&args, req);
1811	if (ret)
1812		return ret;
1813
1814	args.sched = cli->sched;
1815	args.file_priv = file_priv;
1816
1817	ret = nouveau_uvmm_vm_bind(&args);
1818	if (ret)
1819		goto out_free_args;
1820
1821out_free_args:
1822	nouveau_uvmm_vm_bind_ufree(&args);
1823	return ret;
1824}
1825
1826void
1827nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)
1828{
1829	struct drm_gem_object *obj = &nvbo->bo.base;
1830	struct drm_gpuvm_bo *vm_bo;
1831	struct drm_gpuva *va;
1832
1833	dma_resv_assert_held(obj->resv);
1834
1835	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1836		drm_gpuvm_bo_for_each_va(va, vm_bo) {
1837			struct nouveau_uvma *uvma = uvma_from_va(va);
1838
1839			nouveau_uvma_map(uvma, mem);
1840			drm_gpuva_invalidate(va, false);
1841		}
1842	}
1843}
1844
1845void
1846nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)
1847{
1848	struct drm_gem_object *obj = &nvbo->bo.base;
1849	struct drm_gpuvm_bo *vm_bo;
1850	struct drm_gpuva *va;
1851
1852	dma_resv_assert_held(obj->resv);
1853
1854	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1855		drm_gpuvm_bo_for_each_va(va, vm_bo) {
1856			struct nouveau_uvma *uvma = uvma_from_va(va);
1857
1858			nouveau_uvma_unmap(uvma);
1859			drm_gpuva_invalidate(va, true);
1860		}
1861	}
1862}
1863
1864static void
1865nouveau_uvmm_free(struct drm_gpuvm *gpuvm)
1866{
1867	struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm);
1868
1869	kfree(uvmm);
1870}
1871
1872static int
1873nouveau_uvmm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
1874{
1875	struct nouveau_bo *nvbo = nouveau_gem_object(vm_bo->obj);
1876
1877	nouveau_bo_placement_set(nvbo, nvbo->valid_domains, 0);
1878	return nouveau_bo_validate(nvbo, true, false);
1879}
1880
1881static const struct drm_gpuvm_ops gpuvm_ops = {
1882	.vm_free = nouveau_uvmm_free,
1883	.vm_bo_validate = nouveau_uvmm_bo_validate,
1884};
1885
1886int
1887nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
1888			   void *data,
1889			   struct drm_file *file_priv)
1890{
1891	struct nouveau_uvmm *uvmm;
1892	struct nouveau_cli *cli = nouveau_cli(file_priv);
1893	struct drm_device *drm = cli->drm->dev;
1894	struct drm_gem_object *r_obj;
1895	struct drm_nouveau_vm_init *init = data;
1896	u64 kernel_managed_end;
1897	int ret;
1898
1899	if (check_add_overflow(init->kernel_managed_addr,
1900			       init->kernel_managed_size,
1901			       &kernel_managed_end))
1902		return -EINVAL;
1903
1904	if (kernel_managed_end > NOUVEAU_VA_SPACE_END)
1905		return -EINVAL;
1906
1907	mutex_lock(&cli->mutex);
1908
1909	if (unlikely(cli->uvmm.disabled)) {
1910		ret = -ENOSYS;
1911		goto out_unlock;
1912	}
1913
1914	uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL);
1915	if (!uvmm) {
1916		ret = -ENOMEM;
1917		goto out_unlock;
1918	}
1919
1920	r_obj = drm_gpuvm_resv_object_alloc(drm);
1921	if (!r_obj) {
1922		kfree(uvmm);
1923		ret = -ENOMEM;
1924		goto out_unlock;
1925	}
1926
1927	mutex_init(&uvmm->mutex);
1928	mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
1929	mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
1930
1931	drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,
1932		       NOUVEAU_VA_SPACE_START,
1933		       NOUVEAU_VA_SPACE_END,
1934		       init->kernel_managed_addr,
1935		       init->kernel_managed_size,
1936		       &gpuvm_ops);
1937	/* GPUVM takes care from here on. */
1938	drm_gem_object_put(r_obj);
1939
1940	ret = nvif_vmm_ctor(&cli->mmu, "uvmm",
1941			    cli->vmm.vmm.object.oclass, RAW,
1942			    init->kernel_managed_addr,
1943			    init->kernel_managed_size,
1944			    NULL, 0, &uvmm->vmm.vmm);
1945	if (ret)
1946		goto out_gpuvm_fini;
1947
1948	uvmm->vmm.cli = cli;
1949	cli->uvmm.ptr = uvmm;
1950	mutex_unlock(&cli->mutex);
1951
1952	return 0;
1953
1954out_gpuvm_fini:
1955	drm_gpuvm_put(&uvmm->base);
1956out_unlock:
1957	mutex_unlock(&cli->mutex);
1958	return ret;
1959}
1960
1961void
1962nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
1963{
1964	MA_STATE(mas, &uvmm->region_mt, 0, 0);
1965	struct nouveau_uvma_region *reg;
1966	struct nouveau_cli *cli = uvmm->vmm.cli;
1967	struct drm_gpuva *va, *next;
1968
1969	nouveau_uvmm_lock(uvmm);
1970	drm_gpuvm_for_each_va_safe(va, next, &uvmm->base) {
1971		struct nouveau_uvma *uvma = uvma_from_va(va);
1972		struct drm_gem_object *obj = va->gem.obj;
1973
1974		if (unlikely(va == &uvmm->base.kernel_alloc_node))
1975			continue;
1976
1977		drm_gpuva_remove(va);
1978
1979		dma_resv_lock(obj->resv, NULL);
1980		drm_gpuva_unlink(va);
1981		dma_resv_unlock(obj->resv);
1982
1983		nouveau_uvma_unmap(uvma);
1984		nouveau_uvma_vmm_put(uvma);
1985
1986		nouveau_uvma_gem_put(uvma);
1987		nouveau_uvma_free(uvma);
1988	}
1989
1990	mas_for_each(&mas, reg, ULONG_MAX) {
1991		mas_erase(&mas);
1992		nouveau_uvma_region_sparse_unref(reg);
1993		nouveau_uvma_region_put(reg);
1994	}
1995
1996	WARN(!mtree_empty(&uvmm->region_mt),
1997	     "nouveau_uvma_region tree not empty, potentially leaking memory.");
1998	__mt_destroy(&uvmm->region_mt);
1999	nouveau_uvmm_unlock(uvmm);
2000
2001	mutex_lock(&cli->mutex);
2002	nouveau_vmm_fini(&uvmm->vmm);
2003	drm_gpuvm_put(&uvmm->base);
2004	mutex_unlock(&cli->mutex);
2005}