nouveau_bo.c

   1/*
   2 * Copyright 2007 Dave Airlied
   3 * All Rights Reserved.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice (including the next
  13 * paragraph) shall be included in all copies or substantial portions of the
  14 * Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 */
  24/*
  25 * Authors: Dave Airlied <airlied@linux.ie>
  26 *	    Ben Skeggs   <darktama@iinet.net.au>
  27 *	    Jeremy Kolb  <jkolb@brandeis.edu>
  28 */
  29
  30#include <linux/dma-mapping.h>
  31#include <drm/ttm/ttm_tt.h>
  32
  33#include "nouveau_drv.h"
  34#include "nouveau_chan.h"
  35#include "nouveau_fence.h"
  36
  37#include "nouveau_bo.h"
  38#include "nouveau_ttm.h"
  39#include "nouveau_gem.h"
  40#include "nouveau_mem.h"
  41#include "nouveau_vmm.h"
  42
  43#include <nvif/class.h>
  44#include <nvif/if500b.h>
  45#include <nvif/if900b.h>
  46
  47static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm,
  48			       struct ttm_resource *reg);
  49static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm);
  50
  51/*
  52 * NV10-NV40 tiling helpers
  53 */
  54
  55static void
  56nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
  57			   u32 addr, u32 size, u32 pitch, u32 flags)
  58{
  59	struct nouveau_drm *drm = nouveau_drm(dev);
  60	int i = reg - drm->tile.reg;
  61	struct nvkm_fb *fb = nvxx_fb(drm);
  62	struct nvkm_fb_tile *tile = &fb->tile.region[i];
  63
  64	nouveau_fence_unref(&reg->fence);
  65
  66	if (tile->pitch)
  67		nvkm_fb_tile_fini(fb, i, tile);
  68
  69	if (pitch)
  70		nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
  71
  72	nvkm_fb_tile_prog(fb, i, tile);
  73}
  74
  75static struct nouveau_drm_tile *
  76nv10_bo_get_tile_region(struct drm_device *dev, int i)
  77{
  78	struct nouveau_drm *drm = nouveau_drm(dev);
  79	struct nouveau_drm_tile *tile = &drm->tile.reg[i];
  80
  81	spin_lock(&drm->tile.lock);
  82
  83	if (!tile->used &&
  84	    (!tile->fence || nouveau_fence_done(tile->fence)))
  85		tile->used = true;
  86	else
  87		tile = NULL;
  88
  89	spin_unlock(&drm->tile.lock);
  90	return tile;
  91}
  92
  93static void
  94nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
  95			struct dma_fence *fence)
  96{
  97	struct nouveau_drm *drm = nouveau_drm(dev);
  98
  99	if (tile) {
 100		spin_lock(&drm->tile.lock);
 101		tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
 102		tile->used = false;
 103		spin_unlock(&drm->tile.lock);
 104	}
 105}
 106
 107static struct nouveau_drm_tile *
 108nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
 109		   u32 size, u32 pitch, u32 zeta)
 110{
 111	struct nouveau_drm *drm = nouveau_drm(dev);
 112	struct nvkm_fb *fb = nvxx_fb(drm);
 113	struct nouveau_drm_tile *tile, *found = NULL;
 114	int i;
 115
 116	for (i = 0; i < fb->tile.regions; i++) {
 117		tile = nv10_bo_get_tile_region(dev, i);
 118
 119		if (pitch && !found) {
 120			found = tile;
 121			continue;
 122
 123		} else if (tile && fb->tile.region[i].pitch) {
 124			/* Kill an unused tile region. */
 125			nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
 126		}
 127
 128		nv10_bo_put_tile_region(dev, tile, NULL);
 129	}
 130
 131	if (found)
 132		nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta);
 133	return found;
 134}
 135
 136static void
 137nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
 138{
 139	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 140	struct drm_device *dev = drm->dev;
 141	struct nouveau_bo *nvbo = nouveau_bo(bo);
 142
 143	WARN_ON(nvbo->bo.pin_count > 0);
 144	nouveau_bo_del_io_reserve_lru(bo);
 145	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
 146
 147	if (bo->base.import_attach)
 148		drm_prime_gem_destroy(&bo->base, bo->sg);
 149
 150	/*
 151	 * If nouveau_bo_new() allocated this buffer, the GEM object was never
 152	 * initialized, so don't attempt to release it.
 153	 */
 154	if (bo->base.dev) {
 155		/* Gem objects not being shared with other VMs get their
 156		 * dma_resv from a root GEM object.
 157		 */
 158		if (nvbo->no_share)
 159			drm_gem_object_put(nvbo->r_obj);
 160
 161		drm_gem_object_release(&bo->base);
 162	} else {
 163		dma_resv_fini(&bo->base._resv);
 164	}
 165
 166	kfree(nvbo);
 167}
 168
 169static inline u64
 170roundup_64(u64 x, u32 y)
 171{
 172	x += y - 1;
 173	do_div(x, y);
 174	return x * y;
 175}
 176
 177static void
 178nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size)
 179{
 180	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 181	struct nvif_device *device = &drm->client.device;
 182
 183	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
 184		if (nvbo->mode) {
 185			if (device->info.chipset >= 0x40) {
 186				*align = 65536;
 187				*size = roundup_64(*size, 64 * nvbo->mode);
 188
 189			} else if (device->info.chipset >= 0x30) {
 190				*align = 32768;
 191				*size = roundup_64(*size, 64 * nvbo->mode);
 192
 193			} else if (device->info.chipset >= 0x20) {
 194				*align = 16384;
 195				*size = roundup_64(*size, 64 * nvbo->mode);
 196
 197			} else if (device->info.chipset >= 0x10) {
 198				*align = 16384;
 199				*size = roundup_64(*size, 32 * nvbo->mode);
 200			}
 201		}
 202	} else {
 203		*size = roundup_64(*size, (1 << nvbo->page));
 204		*align = max((1 <<  nvbo->page), *align);
 205	}
 206
 207	*size = roundup_64(*size, PAGE_SIZE);
 208}
 209
 210struct nouveau_bo *
 211nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain,
 212		 u32 tile_mode, u32 tile_flags, bool internal)
 213{
 214	struct nouveau_drm *drm = cli->drm;
 215	struct nouveau_bo *nvbo;
 216	struct nvif_mmu *mmu = &cli->mmu;
 217	struct nvif_vmm *vmm = &nouveau_cli_vmm(cli)->vmm;
 218	int i, pi = -1;
 219
 220	if (!*size) {
 221		NV_WARN(drm, "skipped size %016llx\n", *size);
 222		return ERR_PTR(-EINVAL);
 223	}
 224
 225	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
 226	if (!nvbo)
 227		return ERR_PTR(-ENOMEM);
 228
 229	INIT_LIST_HEAD(&nvbo->head);
 230	INIT_LIST_HEAD(&nvbo->entry);
 231	INIT_LIST_HEAD(&nvbo->vma_list);
 232	nvbo->bo.bdev = &drm->ttm.bdev;
 233
 234	/* This is confusing, and doesn't actually mean we want an uncached
 235	 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated
 236	 * into in nouveau_gem_new().
 237	 */
 238	if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) {
 239		/* Determine if we can get a cache-coherent map, forcing
 240		 * uncached mapping if we can't.
 241		 */
 242		if (!nouveau_drm_use_coherent_gpu_mapping(drm))
 243			nvbo->force_coherent = true;
 244	}
 245
 246	nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);
 247
 248	if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
 249		nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
 250		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
 251			kfree(nvbo);
 252			return ERR_PTR(-EINVAL);
 253		}
 254
 255		nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
 256	} else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
 257		nvbo->kind = (tile_flags & 0x00007f00) >> 8;
 258		nvbo->comp = (tile_flags & 0x00030000) >> 16;
 259		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
 260			kfree(nvbo);
 261			return ERR_PTR(-EINVAL);
 262		}
 263	} else {
 264		nvbo->zeta = (tile_flags & 0x00000007);
 265	}
 266	nvbo->mode = tile_mode;
 267
 268	if (!nouveau_cli_uvmm(cli) || internal) {
 269		/* Determine the desirable target GPU page size for the buffer. */
 270		for (i = 0; i < vmm->page_nr; i++) {
 271			/* Because we cannot currently allow VMM maps to fail
 272			 * during buffer migration, we need to determine page
 273			 * size for the buffer up-front, and pre-allocate its
 274			 * page tables.
 275			 *
 276			 * Skip page sizes that can't support needed domains.
 277			 */
 278			if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE &&
 279			    (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram)
 280				continue;
 281			if ((domain & NOUVEAU_GEM_DOMAIN_GART) &&
 282			    (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
 283				continue;
 284
 285			/* Select this page size if it's the first that supports
 286			 * the potential memory domains, or when it's compatible
 287			 * with the requested compression settings.
 288			 */
 289			if (pi < 0 || !nvbo->comp || vmm->page[i].comp)
 290				pi = i;
 291
 292			/* Stop once the buffer is larger than the current page size. */
 293			if (*size >= 1ULL << vmm->page[i].shift)
 294				break;
 295		}
 296
 297		if (WARN_ON(pi < 0)) {
 298			kfree(nvbo);
 299			return ERR_PTR(-EINVAL);
 300		}
 301
 302		/* Disable compression if suitable settings couldn't be found. */
 303		if (nvbo->comp && !vmm->page[pi].comp) {
 304			if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100)
 305				nvbo->kind = mmu->kind[nvbo->kind];
 306			nvbo->comp = 0;
 307		}
 308		nvbo->page = vmm->page[pi].shift;
 309	} else {
 310		/* Determine the desirable target GPU page size for the buffer. */
 311		for (i = 0; i < vmm->page_nr; i++) {
 312			/* Because we cannot currently allow VMM maps to fail
 313			 * during buffer migration, we need to determine page
 314			 * size for the buffer up-front, and pre-allocate its
 315			 * page tables.
 316			 *
 317			 * Skip page sizes that can't support needed domains.
 318			 */
 319			if ((domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram)
 320				continue;
 321			if ((domain & NOUVEAU_GEM_DOMAIN_GART) &&
 322			    (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
 323				continue;
 324
 325			/* pick the last one as it will be smallest. */
 326			pi = i;
 327
 328			/* Stop once the buffer is larger than the current page size. */
 329			if (*size >= 1ULL << vmm->page[i].shift)
 330				break;
 331		}
 332		if (WARN_ON(pi < 0)) {
 333			kfree(nvbo);
 334			return ERR_PTR(-EINVAL);
 335		}
 336		nvbo->page = vmm->page[pi].shift;
 337	}
 338
 339	nouveau_bo_fixup_align(nvbo, align, size);
 340
 341	return nvbo;
 342}
 343
 344int
 345nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain,
 346		struct sg_table *sg, struct dma_resv *robj)
 347{
 348	int type = sg ? ttm_bo_type_sg : ttm_bo_type_device;
 349	int ret;
 350	struct ttm_operation_ctx ctx = {
 351		.interruptible = false,
 352		.no_wait_gpu = false,
 353		.resv = robj,
 354	};
 355
 356	nouveau_bo_placement_set(nvbo, domain, 0);
 357	INIT_LIST_HEAD(&nvbo->io_reserve_lru);
 358
 359	ret = ttm_bo_init_reserved(nvbo->bo.bdev, &nvbo->bo, type,
 360				   &nvbo->placement, align >> PAGE_SHIFT, &ctx,
 361				   sg, robj, nouveau_bo_del_ttm);
 362	if (ret) {
 363		/* ttm will call nouveau_bo_del_ttm if it fails.. */
 364		return ret;
 365	}
 366
 367	if (!robj)
 368		ttm_bo_unreserve(&nvbo->bo);
 369
 370	return 0;
 371}
 372
 373int
 374nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align,
 375	       uint32_t domain, uint32_t tile_mode, uint32_t tile_flags,
 376	       struct sg_table *sg, struct dma_resv *robj,
 377	       struct nouveau_bo **pnvbo)
 378{
 379	struct nouveau_bo *nvbo;
 380	int ret;
 381
 382	nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode,
 383				tile_flags, true);
 384	if (IS_ERR(nvbo))
 385		return PTR_ERR(nvbo);
 386
 387	nvbo->bo.base.size = size;
 388	dma_resv_init(&nvbo->bo.base._resv);
 389	drm_vma_node_reset(&nvbo->bo.base.vma_node);
 390
 391	/* This must be called before ttm_bo_init_reserved(). Subsequent
 392	 * bo_move() callbacks might already iterate the GEMs GPUVA list.
 393	 */
 394	drm_gem_gpuva_init(&nvbo->bo.base);
 395
 396	ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj);
 397	if (ret)
 398		return ret;
 399
 400	*pnvbo = nvbo;
 401	return 0;
 402}
 403
 404void
 405nouveau_bo_unpin_del(struct nouveau_bo **pnvbo)
 406{
 407	struct nouveau_bo *nvbo = *pnvbo;
 408
 409	if (!nvbo)
 410		return;
 411
 412	nouveau_bo_unmap(nvbo);
 413	nouveau_bo_unpin(nvbo);
 414	nouveau_bo_fini(nvbo);
 415
 416	*pnvbo = NULL;
 417}
 418
 419int
 420nouveau_bo_new_pin(struct nouveau_cli *cli, u32 domain, u32 size, struct nouveau_bo **pnvbo)
 421{
 422	struct nouveau_bo *nvbo;
 423	int ret;
 424
 425	ret = nouveau_bo_new(cli, size, 0, domain, 0, 0, NULL, NULL, &nvbo);
 426	if (ret)
 427		return ret;
 428
 429	ret = nouveau_bo_pin(nvbo, domain, false);
 430	if (ret) {
 431		nouveau_bo_fini(nvbo);
 432		return ret;
 433	}
 434
 435	*pnvbo = nvbo;
 436	return 0;
 437}
 438
 439int
 440nouveau_bo_new_map(struct nouveau_cli *cli, u32 domain, u32 size, struct nouveau_bo **pnvbo)
 441{
 442	struct nouveau_bo *nvbo;
 443	int ret;
 444
 445	ret = nouveau_bo_new_pin(cli, domain, size, &nvbo);
 446	if (ret)
 447		return ret;
 448
 449	ret = nouveau_bo_map(nvbo);
 450	if (ret) {
 451		nouveau_bo_unpin_del(&nvbo);
 452		return ret;
 453	}
 454
 455	*pnvbo = nvbo;
 456	return 0;
 457}
 458
 459int
 460nouveau_bo_new_map_gpu(struct nouveau_cli *cli, u32 domain, u32 size,
 461		       struct nouveau_bo **pnvbo, struct nouveau_vma **pvma)
 462{
 463	struct nouveau_vmm *vmm = nouveau_cli_vmm(cli);
 464	struct nouveau_bo *nvbo;
 465	int ret;
 466
 467	ret = nouveau_bo_new_map(cli, domain, size, &nvbo);
 468	if (ret)
 469		return ret;
 470
 471	ret = nouveau_vma_new(nvbo, vmm, pvma);
 472	if (ret) {
 473		nouveau_bo_unpin_del(&nvbo);
 474		return ret;
 475	}
 476
 477	*pnvbo = nvbo;
 478	return 0;
 479}
 480
 481static void
 482set_placement_range(struct nouveau_bo *nvbo, uint32_t domain)
 483{
 484	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 485	u64 vram_size = drm->client.device.info.ram_size;
 486	unsigned i, fpfn, lpfn;
 487
 488	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
 489	    nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) &&
 490	    nvbo->bo.base.size < vram_size / 4) {
 491		/*
 492		 * Make sure that the color and depth buffers are handled
 493		 * by independent memory controller units. Up to a 9x
 494		 * speed up when alpha-blending and depth-test are enabled
 495		 * at the same time.
 496		 */
 497		if (nvbo->zeta) {
 498			fpfn = (vram_size / 2) >> PAGE_SHIFT;
 499			lpfn = ~0;
 500		} else {
 501			fpfn = 0;
 502			lpfn = (vram_size / 2) >> PAGE_SHIFT;
 503		}
 504		for (i = 0; i < nvbo->placement.num_placement; ++i) {
 505			nvbo->placements[i].fpfn = fpfn;
 506			nvbo->placements[i].lpfn = lpfn;
 507		}
 508	}
 509}
 510
 511void
 512nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain,
 513			 uint32_t busy)
 514{
 515	unsigned int *n = &nvbo->placement.num_placement;
 516	struct ttm_place *pl = nvbo->placements;
 517
 518	domain |= busy;
 519
 520	*n = 0;
 521	if (domain & NOUVEAU_GEM_DOMAIN_VRAM) {
 522		pl[*n].mem_type = TTM_PL_VRAM;
 523		pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_VRAM ?
 524			TTM_PL_FLAG_FALLBACK : 0;
 525		(*n)++;
 526	}
 527	if (domain & NOUVEAU_GEM_DOMAIN_GART) {
 528		pl[*n].mem_type = TTM_PL_TT;
 529		pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_GART ?
 530			TTM_PL_FLAG_FALLBACK : 0;
 531		(*n)++;
 532	}
 533	if (domain & NOUVEAU_GEM_DOMAIN_CPU) {
 534		pl[*n].mem_type = TTM_PL_SYSTEM;
 535		pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_CPU ?
 536			TTM_PL_FLAG_FALLBACK : 0;
 537		(*n)++;
 538	}
 539
 540	nvbo->placement.placement = nvbo->placements;
 541	set_placement_range(nvbo, domain);
 542}
 543
 544int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
 545{
 546	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 547	struct ttm_buffer_object *bo = &nvbo->bo;
 548	bool force = false, evict = false;
 549	int ret = 0;
 550
 551	dma_resv_assert_held(bo->base.resv);
 552
 553	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
 554	    domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) {
 555		if (!nvbo->contig) {
 556			nvbo->contig = true;
 557			force = true;
 558			evict = true;
 559		}
 560	}
 561
 562	if (nvbo->bo.pin_count) {
 563		bool error = evict;
 564
 565		switch (bo->resource->mem_type) {
 566		case TTM_PL_VRAM:
 567			error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM);
 568			break;
 569		case TTM_PL_TT:
 570			error |= !(domain & NOUVEAU_GEM_DOMAIN_GART);
 571			break;
 572		default:
 573			break;
 574		}
 575
 576		if (error) {
 577			NV_ERROR(drm, "bo %p pinned elsewhere: "
 578				      "0x%08x vs 0x%08x\n", bo,
 579				 bo->resource->mem_type, domain);
 580			ret = -EBUSY;
 581		}
 582		ttm_bo_pin(&nvbo->bo);
 583		goto out;
 584	}
 585
 586	if (evict) {
 587		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0);
 588		ret = nouveau_bo_validate(nvbo, false, false);
 589		if (ret)
 590			goto out;
 591	}
 592
 593	nouveau_bo_placement_set(nvbo, domain, 0);
 594	ret = nouveau_bo_validate(nvbo, false, false);
 595	if (ret)
 596		goto out;
 597
 598	ttm_bo_pin(&nvbo->bo);
 599
 600	switch (bo->resource->mem_type) {
 601	case TTM_PL_VRAM:
 602		drm->gem.vram_available -= bo->base.size;
 603		break;
 604	case TTM_PL_TT:
 605		drm->gem.gart_available -= bo->base.size;
 606		break;
 607	default:
 608		break;
 609	}
 610
 611out:
 612	if (force && ret)
 613		nvbo->contig = false;
 614	return ret;
 615}
 616
 617void nouveau_bo_unpin_locked(struct nouveau_bo *nvbo)
 618{
 619	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 620	struct ttm_buffer_object *bo = &nvbo->bo;
 621
 622	dma_resv_assert_held(bo->base.resv);
 623
 624	ttm_bo_unpin(&nvbo->bo);
 625	if (!nvbo->bo.pin_count) {
 626		switch (bo->resource->mem_type) {
 627		case TTM_PL_VRAM:
 628			drm->gem.vram_available += bo->base.size;
 629			break;
 630		case TTM_PL_TT:
 631			drm->gem.gart_available += bo->base.size;
 632			break;
 633		default:
 634			break;
 635		}
 636	}
 637}
 638
 639int nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
 640{
 641	struct ttm_buffer_object *bo = &nvbo->bo;
 642	int ret;
 643
 644	ret = ttm_bo_reserve(bo, false, false, NULL);
 645	if (ret)
 646		return ret;
 647	ret = nouveau_bo_pin_locked(nvbo, domain, contig);
 648	ttm_bo_unreserve(bo);
 649
 650	return ret;
 651}
 652
 653int nouveau_bo_unpin(struct nouveau_bo *nvbo)
 654{
 655	struct ttm_buffer_object *bo = &nvbo->bo;
 656	int ret;
 657
 658	ret = ttm_bo_reserve(bo, false, false, NULL);
 659	if (ret)
 660		return ret;
 661	nouveau_bo_unpin_locked(nvbo);
 662	ttm_bo_unreserve(bo);
 663
 664	return 0;
 665}
 666
 667int
 668nouveau_bo_map(struct nouveau_bo *nvbo)
 669{
 670	int ret;
 671
 672	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
 673	if (ret)
 674		return ret;
 675
 676	ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap);
 677
 678	ttm_bo_unreserve(&nvbo->bo);
 679	return ret;
 680}
 681
 682void
 683nouveau_bo_unmap(struct nouveau_bo *nvbo)
 684{
 685	if (!nvbo)
 686		return;
 687
 688	ttm_bo_kunmap(&nvbo->kmap);
 689}
 690
 691void
 692nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
 693{
 694	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 695	struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm;
 696	int i, j;
 697
 698	if (!ttm_dma || !ttm_dma->dma_address)
 699		return;
 700	if (!ttm_dma->pages) {
 701		NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma);
 702		return;
 703	}
 704
 705	/* Don't waste time looping if the object is coherent */
 706	if (nvbo->force_coherent)
 707		return;
 708
 709	i = 0;
 710	while (i < ttm_dma->num_pages) {
 711		struct page *p = ttm_dma->pages[i];
 712		size_t num_pages = 1;
 713
 714		for (j = i + 1; j < ttm_dma->num_pages; ++j) {
 715			if (++p != ttm_dma->pages[j])
 716				break;
 717
 718			++num_pages;
 719		}
 720		dma_sync_single_for_device(drm->dev->dev,
 721					   ttm_dma->dma_address[i],
 722					   num_pages * PAGE_SIZE, DMA_TO_DEVICE);
 723		i += num_pages;
 724	}
 725}
 726
 727void
 728nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
 729{
 730	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 731	struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm;
 732	int i, j;
 733
 734	if (!ttm_dma || !ttm_dma->dma_address)
 735		return;
 736	if (!ttm_dma->pages) {
 737		NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma);
 738		return;
 739	}
 740
 741	/* Don't waste time looping if the object is coherent */
 742	if (nvbo->force_coherent)
 743		return;
 744
 745	i = 0;
 746	while (i < ttm_dma->num_pages) {
 747		struct page *p = ttm_dma->pages[i];
 748		size_t num_pages = 1;
 749
 750		for (j = i + 1; j < ttm_dma->num_pages; ++j) {
 751			if (++p != ttm_dma->pages[j])
 752				break;
 753
 754			++num_pages;
 755		}
 756
 757		dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i],
 758					num_pages * PAGE_SIZE, DMA_FROM_DEVICE);
 759		i += num_pages;
 760	}
 761}
 762
 763void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo)
 764{
 765	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 766	struct nouveau_bo *nvbo = nouveau_bo(bo);
 767
 768	mutex_lock(&drm->ttm.io_reserve_mutex);
 769	list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru);
 770	mutex_unlock(&drm->ttm.io_reserve_mutex);
 771}
 772
 773void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo)
 774{
 775	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 776	struct nouveau_bo *nvbo = nouveau_bo(bo);
 777
 778	mutex_lock(&drm->ttm.io_reserve_mutex);
 779	list_del_init(&nvbo->io_reserve_lru);
 780	mutex_unlock(&drm->ttm.io_reserve_mutex);
 781}
 782
 783int
 784nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
 785		    bool no_wait_gpu)
 786{
 787	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
 788	int ret;
 789
 790	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx);
 791	if (ret)
 792		return ret;
 793
 794	nouveau_bo_sync_for_device(nvbo);
 795
 796	return 0;
 797}
 798
 799void
 800nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
 801{
 802	bool is_iomem;
 803	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 804
 805	mem += index;
 806
 807	if (is_iomem)
 808		iowrite16_native(val, (void __force __iomem *)mem);
 809	else
 810		*mem = val;
 811}
 812
 813u32
 814nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
 815{
 816	bool is_iomem;
 817	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 818
 819	mem += index;
 820
 821	if (is_iomem)
 822		return ioread32_native((void __force __iomem *)mem);
 823	else
 824		return *mem;
 825}
 826
 827void
 828nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
 829{
 830	bool is_iomem;
 831	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 832
 833	mem += index;
 834
 835	if (is_iomem)
 836		iowrite32_native(val, (void __force __iomem *)mem);
 837	else
 838		*mem = val;
 839}
 840
 841static struct ttm_tt *
 842nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags)
 843{
 844#if IS_ENABLED(CONFIG_AGP)
 845	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 846
 847	if (drm->agp.bridge) {
 848		return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags);
 849	}
 850#endif
 851
 852	return nouveau_sgdma_create_ttm(bo, page_flags);
 853}
 854
 855static int
 856nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm,
 857		    struct ttm_resource *reg)
 858{
 859#if IS_ENABLED(CONFIG_AGP)
 860	struct nouveau_drm *drm = nouveau_bdev(bdev);
 861#endif
 862	if (!reg)
 863		return -EINVAL;
 864#if IS_ENABLED(CONFIG_AGP)
 865	if (drm->agp.bridge)
 866		return ttm_agp_bind(ttm, reg);
 867#endif
 868	return nouveau_sgdma_bind(bdev, ttm, reg);
 869}
 870
 871static void
 872nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm)
 873{
 874#if IS_ENABLED(CONFIG_AGP)
 875	struct nouveau_drm *drm = nouveau_bdev(bdev);
 876
 877	if (drm->agp.bridge) {
 878		ttm_agp_unbind(ttm);
 879		return;
 880	}
 881#endif
 882	nouveau_sgdma_unbind(bdev, ttm);
 883}
 884
 885static void
 886nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
 887{
 888	struct nouveau_bo *nvbo = nouveau_bo(bo);
 889
 890	switch (bo->resource->mem_type) {
 891	case TTM_PL_VRAM:
 892		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART,
 893					 NOUVEAU_GEM_DOMAIN_CPU);
 894		break;
 895	default:
 896		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0);
 897		break;
 898	}
 899
 900	*pl = nvbo->placement;
 901}
 902
 903static int
 904nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
 905		     struct ttm_resource *reg)
 906{
 907	struct nouveau_mem *old_mem = nouveau_mem(bo->resource);
 908	struct nouveau_mem *new_mem = nouveau_mem(reg);
 909	struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 910	int ret;
 911
 912	ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0,
 913			   old_mem->mem.size, &old_mem->vma[0]);
 914	if (ret)
 915		return ret;
 916
 917	ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0,
 918			   new_mem->mem.size, &old_mem->vma[1]);
 919	if (ret)
 920		goto done;
 921
 922	ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]);
 923	if (ret)
 924		goto done;
 925
 926	ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]);
 927done:
 928	if (ret) {
 929		nvif_vmm_put(vmm, &old_mem->vma[1]);
 930		nvif_vmm_put(vmm, &old_mem->vma[0]);
 931	}
 932	return ret;
 933}
 934
 935static int
 936nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict,
 937		     struct ttm_operation_ctx *ctx,
 938		     struct ttm_resource *new_reg)
 939{
 940	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 941	struct nouveau_channel *chan = drm->ttm.chan;
 942	struct nouveau_cli *cli = chan->cli;
 943	struct nouveau_fence *fence;
 944	int ret;
 945
 946	/* create temporary vmas for the transfer and attach them to the
 947	 * old nvkm_mem node, these will get cleaned up after ttm has
 948	 * destroyed the ttm_resource
 949	 */
 950	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
 951		ret = nouveau_bo_move_prep(drm, bo, new_reg);
 952		if (ret)
 953			return ret;
 954	}
 955
 956	if (drm_drv_uses_atomic_modeset(drm->dev))
 957		mutex_lock(&cli->mutex);
 958	else
 959		mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
 960
 961	ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible);
 962	if (ret)
 963		goto out_unlock;
 964
 965	ret = drm->ttm.move(chan, bo, bo->resource, new_reg);
 966	if (ret)
 967		goto out_unlock;
 968
 969	ret = nouveau_fence_new(&fence, chan);
 970	if (ret)
 971		goto out_unlock;
 972
 973	/* TODO: figure out a better solution here
 974	 *
 975	 * wait on the fence here explicitly as going through
 976	 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it.
 977	 *
 978	 * Without this the operation can timeout and we'll fallback to a
 979	 * software copy, which might take several minutes to finish.
 980	 */
 981	nouveau_fence_wait(fence, false, false);
 982	ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false,
 983					new_reg);
 984	nouveau_fence_unref(&fence);
 985
 986out_unlock:
 987	mutex_unlock(&cli->mutex);
 988	return ret;
 989}
 990
 991void
 992nouveau_bo_move_init(struct nouveau_drm *drm)
 993{
 994	static const struct _method_table {
 995		const char *name;
 996		int engine;
 997		s32 oclass;
 998		int (*exec)(struct nouveau_channel *,
 999			    struct ttm_buffer_object *,
1000			    struct ttm_resource *, struct ttm_resource *);
1001		int (*init)(struct nouveau_channel *, u32 handle);
1002	} _methods[] = {
1003		{  "COPY", 4, 0xcab5, nve0_bo_move_copy, nve0_bo_move_init },
1004		{  "COPY", 4, 0xc9b5, nve0_bo_move_copy, nve0_bo_move_init },
1005		{  "COPY", 4, 0xc8b5, nve0_bo_move_copy, nve0_bo_move_init },
1006		{  "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init },
1007		{  "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init },
1008		{  "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init },
1009		{  "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init },
1010		{  "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init },
1011		{  "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init },
1012		{  "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init },
1013		{  "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init },
1014		{  "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
1015		{  "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
1016		{  "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
1017		{  "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1018		{  "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
1019		{  "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1020		{  "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1021		{  "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1022		{ "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
1023		{ "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
1024		{  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
1025		{ "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
1026		{  "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
1027		{  "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
1028		{  "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1029		{},
1030	};
1031	const struct _method_table *mthd = _methods;
1032	const char *name = "CPU";
1033	int ret;
1034
1035	do {
1036		struct nouveau_channel *chan;
1037
1038		if (mthd->engine)
1039			chan = drm->cechan;
1040		else
1041			chan = drm->channel;
1042		if (chan == NULL)
1043			continue;
1044
1045		ret = nvif_object_ctor(&chan->user, "ttmBoMove",
1046				       mthd->oclass | (mthd->engine << 16),
1047				       mthd->oclass, NULL, 0,
1048				       &drm->ttm.copy);
1049		if (ret == 0) {
1050			ret = mthd->init(chan, drm->ttm.copy.handle);
1051			if (ret) {
1052				nvif_object_dtor(&drm->ttm.copy);
1053				continue;
1054			}
1055
1056			drm->ttm.move = mthd->exec;
1057			drm->ttm.chan = chan;
1058			name = mthd->name;
1059			break;
1060		}
1061	} while ((++mthd)->exec);
1062
1063	NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1064}
1065
1066static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo,
1067				 struct ttm_resource *new_reg)
1068{
1069	struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL;
1070	struct nouveau_bo *nvbo = nouveau_bo(bo);
1071	struct nouveau_vma *vma;
1072	long ret;
1073
1074	/* ttm can now (stupidly) pass the driver bos it didn't create... */
1075	if (bo->destroy != nouveau_bo_del_ttm)
1076		return;
1077
1078	nouveau_bo_del_io_reserve_lru(bo);
1079
1080	if (mem && new_reg->mem_type != TTM_PL_SYSTEM &&
1081	    mem->mem.page == nvbo->page) {
1082		list_for_each_entry(vma, &nvbo->vma_list, head) {
1083			nouveau_vma_map(vma, mem);
1084		}
1085		nouveau_uvmm_bo_map_all(nvbo, mem);
1086	} else {
1087		list_for_each_entry(vma, &nvbo->vma_list, head) {
1088			ret = dma_resv_wait_timeout(bo->base.resv,
1089						    DMA_RESV_USAGE_BOOKKEEP,
1090						    false, 15 * HZ);
1091			WARN_ON(ret <= 0);
1092			nouveau_vma_unmap(vma);
1093		}
1094		nouveau_uvmm_bo_unmap_all(nvbo);
1095	}
1096
1097	if (new_reg)
1098		nvbo->offset = (new_reg->start << PAGE_SHIFT);
1099
1100}
1101
1102static int
1103nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg,
1104		   struct nouveau_drm_tile **new_tile)
1105{
1106	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1107	struct drm_device *dev = drm->dev;
1108	struct nouveau_bo *nvbo = nouveau_bo(bo);
1109	u64 offset = new_reg->start << PAGE_SHIFT;
1110
1111	*new_tile = NULL;
1112	if (new_reg->mem_type != TTM_PL_VRAM)
1113		return 0;
1114
1115	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1116		*new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size,
1117					       nvbo->mode, nvbo->zeta);
1118	}
1119
1120	return 0;
1121}
1122
1123static void
1124nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1125		      struct nouveau_drm_tile *new_tile,
1126		      struct nouveau_drm_tile **old_tile)
1127{
1128	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1129	struct drm_device *dev = drm->dev;
1130	struct dma_fence *fence;
1131	int ret;
1132
1133	ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE,
1134				     &fence);
1135	if (ret)
1136		dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE,
1137				      false, MAX_SCHEDULE_TIMEOUT);
1138
1139	nv10_bo_put_tile_region(dev, *old_tile, fence);
1140	*old_tile = new_tile;
1141}
1142
1143static int
1144nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
1145		struct ttm_operation_ctx *ctx,
1146		struct ttm_resource *new_reg,
1147		struct ttm_place *hop)
1148{
1149	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1150	struct nouveau_bo *nvbo = nouveau_bo(bo);
1151	struct drm_gem_object *obj = &bo->base;
1152	struct ttm_resource *old_reg = bo->resource;
1153	struct nouveau_drm_tile *new_tile = NULL;
1154	int ret = 0;
1155
1156	if (new_reg->mem_type == TTM_PL_TT) {
1157		ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg);
1158		if (ret)
1159			return ret;
1160	}
1161
1162	drm_gpuvm_bo_gem_evict(obj, evict);
1163	nouveau_bo_move_ntfy(bo, new_reg);
1164	ret = ttm_bo_wait_ctx(bo, ctx);
1165	if (ret)
1166		goto out_ntfy;
1167
1168	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1169		ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
1170		if (ret)
1171			goto out_ntfy;
1172	}
1173
1174	/* Fake bo copy. */
1175	if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM &&
1176			 !bo->ttm)) {
1177		ttm_bo_move_null(bo, new_reg);
1178		goto out;
1179	}
1180
1181	if (old_reg->mem_type == TTM_PL_SYSTEM &&
1182	    new_reg->mem_type == TTM_PL_TT) {
1183		ttm_bo_move_null(bo, new_reg);
1184		goto out;
1185	}
1186
1187	if (old_reg->mem_type == TTM_PL_TT &&
1188	    new_reg->mem_type == TTM_PL_SYSTEM) {
1189		nouveau_ttm_tt_unbind(bo->bdev, bo->ttm);
1190		ttm_resource_free(bo, &bo->resource);
1191		ttm_bo_assign_mem(bo, new_reg);
1192		goto out;
1193	}
1194
1195	/* Hardware assisted copy. */
1196	if (drm->ttm.move) {
1197		if ((old_reg->mem_type == TTM_PL_SYSTEM &&
1198		     new_reg->mem_type == TTM_PL_VRAM) ||
1199		    (old_reg->mem_type == TTM_PL_VRAM &&
1200		     new_reg->mem_type == TTM_PL_SYSTEM)) {
1201			hop->fpfn = 0;
1202			hop->lpfn = 0;
1203			hop->mem_type = TTM_PL_TT;
1204			hop->flags = 0;
1205			return -EMULTIHOP;
1206		}
1207		ret = nouveau_bo_move_m2mf(bo, evict, ctx,
1208					   new_reg);
1209	} else
1210		ret = -ENODEV;
1211
1212	if (ret) {
1213		/* Fallback to software copy. */
1214		ret = ttm_bo_move_memcpy(bo, ctx, new_reg);
1215	}
1216
1217out:
1218	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1219		if (ret)
1220			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1221		else
1222			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1223	}
1224out_ntfy:
1225	if (ret) {
1226		nouveau_bo_move_ntfy(bo, bo->resource);
1227		drm_gpuvm_bo_gem_evict(obj, !evict);
1228	}
1229	return ret;
1230}
1231
1232static void
1233nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm,
1234			       struct ttm_resource *reg)
1235{
1236	struct nouveau_mem *mem = nouveau_mem(reg);
1237
1238	if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
1239		switch (reg->mem_type) {
1240		case TTM_PL_TT:
1241			if (mem->kind)
1242				nvif_object_unmap_handle(&mem->mem.object);
1243			break;
1244		case TTM_PL_VRAM:
1245			nvif_object_unmap_handle(&mem->mem.object);
1246			break;
1247		default:
1248			break;
1249		}
1250	}
1251}
1252
1253static int
1254nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg)
1255{
1256	struct nouveau_drm *drm = nouveau_bdev(bdev);
1257	struct nvkm_device *device = nvxx_device(drm);
1258	struct nouveau_mem *mem = nouveau_mem(reg);
1259	struct nvif_mmu *mmu = &drm->client.mmu;
1260	int ret;
1261
1262	mutex_lock(&drm->ttm.io_reserve_mutex);
1263retry:
1264	switch (reg->mem_type) {
1265	case TTM_PL_SYSTEM:
1266		/* System memory */
1267		ret = 0;
1268		goto out;
1269	case TTM_PL_TT:
1270#if IS_ENABLED(CONFIG_AGP)
1271		if (drm->agp.bridge) {
1272			reg->bus.offset = (reg->start << PAGE_SHIFT) +
1273				drm->agp.base;
1274			reg->bus.is_iomem = !drm->agp.cma;
1275			reg->bus.caching = ttm_write_combined;
1276		}
1277#endif
1278		if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 ||
1279		    !mem->kind) {
1280			/* untiled */
1281			ret = 0;
1282			break;
1283		}
1284		fallthrough;	/* tiled memory */
1285	case TTM_PL_VRAM:
1286		reg->bus.offset = (reg->start << PAGE_SHIFT) +
1287			device->func->resource_addr(device, NVKM_BAR1_FB);
1288		reg->bus.is_iomem = true;
1289
1290		/* Some BARs do not support being ioremapped WC */
1291		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
1292		    mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED)
1293			reg->bus.caching = ttm_uncached;
1294		else
1295			reg->bus.caching = ttm_write_combined;
1296
1297		if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
1298			union {
1299				struct nv50_mem_map_v0 nv50;
1300				struct gf100_mem_map_v0 gf100;
1301			} args;
1302			u64 handle, length;
1303			u32 argc = 0;
1304
1305			switch (mem->mem.object.oclass) {
1306			case NVIF_CLASS_MEM_NV50:
1307				args.nv50.version = 0;
1308				args.nv50.ro = 0;
1309				args.nv50.kind = mem->kind;
1310				args.nv50.comp = mem->comp;
1311				argc = sizeof(args.nv50);
1312				break;
1313			case NVIF_CLASS_MEM_GF100:
1314				args.gf100.version = 0;
1315				args.gf100.ro = 0;
1316				args.gf100.kind = mem->kind;
1317				argc = sizeof(args.gf100);
1318				break;
1319			default:
1320				WARN_ON(1);
1321				break;
1322			}
1323
1324			ret = nvif_object_map_handle(&mem->mem.object,
1325						     &args, argc,
1326						     &handle, &length);
1327			if (ret != 1) {
1328				if (WARN_ON(ret == 0))
1329					ret = -EINVAL;
1330				goto out;
1331			}
1332
1333			reg->bus.offset = handle;
1334		}
1335		ret = 0;
1336		break;
1337	default:
1338		ret = -EINVAL;
1339	}
1340
1341out:
1342	if (ret == -ENOSPC) {
1343		struct nouveau_bo *nvbo;
1344
1345		nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru,
1346						typeof(*nvbo),
1347						io_reserve_lru);
1348		if (nvbo) {
1349			list_del_init(&nvbo->io_reserve_lru);
1350			drm_vma_node_unmap(&nvbo->bo.base.vma_node,
1351					   bdev->dev_mapping);
1352			nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource);
1353			nvbo->bo.resource->bus.offset = 0;
1354			nvbo->bo.resource->bus.addr = NULL;
1355			goto retry;
1356		}
1357
1358	}
1359	mutex_unlock(&drm->ttm.io_reserve_mutex);
1360	return ret;
1361}
1362
1363static void
1364nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg)
1365{
1366	struct nouveau_drm *drm = nouveau_bdev(bdev);
1367
1368	mutex_lock(&drm->ttm.io_reserve_mutex);
1369	nouveau_ttm_io_mem_free_locked(drm, reg);
1370	mutex_unlock(&drm->ttm.io_reserve_mutex);
1371}
1372
1373vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1374{
1375	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1376	struct nouveau_bo *nvbo = nouveau_bo(bo);
1377	struct nvkm_device *device = nvxx_device(drm);
1378	u32 mappable = device->func->resource_size(device, NVKM_BAR1_FB) >> PAGE_SHIFT;
1379	int i, ret;
1380
1381	/* as long as the bo isn't in vram, and isn't tiled, we've got
1382	 * nothing to do here.
1383	 */
1384	if (bo->resource->mem_type != TTM_PL_VRAM) {
1385		if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1386		    !nvbo->kind)
1387			return 0;
1388
1389		if (bo->resource->mem_type != TTM_PL_SYSTEM)
1390			return 0;
1391
1392		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0);
1393
1394	} else {
1395		/* make sure bo is in mappable vram */
1396		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1397		    bo->resource->start + PFN_UP(bo->resource->size) < mappable)
1398			return 0;
1399
1400		for (i = 0; i < nvbo->placement.num_placement; ++i) {
1401			nvbo->placements[i].fpfn = 0;
1402			nvbo->placements[i].lpfn = mappable;
1403		}
1404
1405		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0);
1406	}
1407
1408	ret = nouveau_bo_validate(nvbo, false, false);
1409	if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS))
1410		return VM_FAULT_NOPAGE;
1411	else if (unlikely(ret))
1412		return VM_FAULT_SIGBUS;
1413
1414	ttm_bo_move_to_lru_tail_unlocked(bo);
1415	return 0;
1416}
1417
1418static int
1419nouveau_ttm_tt_populate(struct ttm_device *bdev,
1420			struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
1421{
1422	struct ttm_tt *ttm_dma = (void *)ttm;
1423	struct nouveau_drm *drm;
1424	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
1425
1426	if (ttm_tt_is_populated(ttm))
1427		return 0;
1428
1429	if (slave && ttm->sg) {
1430		drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address,
1431					       ttm->num_pages);
1432		return 0;
1433	}
1434
1435	drm = nouveau_bdev(bdev);
1436
1437	return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx);
1438}
1439
1440static void
1441nouveau_ttm_tt_unpopulate(struct ttm_device *bdev,
1442			  struct ttm_tt *ttm)
1443{
1444	struct nouveau_drm *drm;
1445	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
1446
1447	if (slave)
1448		return;
1449
1450	nouveau_ttm_tt_unbind(bdev, ttm);
1451
1452	drm = nouveau_bdev(bdev);
1453
1454	return ttm_pool_free(&drm->ttm.bdev.pool, ttm);
1455}
1456
1457static void
1458nouveau_ttm_tt_destroy(struct ttm_device *bdev,
1459		       struct ttm_tt *ttm)
1460{
1461#if IS_ENABLED(CONFIG_AGP)
1462	struct nouveau_drm *drm = nouveau_bdev(bdev);
1463	if (drm->agp.bridge) {
1464		ttm_agp_destroy(ttm);
1465		return;
1466	}
1467#endif
1468	nouveau_sgdma_destroy(bdev, ttm);
1469}
1470
1471void
1472nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1473{
1474	struct dma_resv *resv = nvbo->bo.base.resv;
1475
1476	if (!fence)
1477		return;
1478
1479	dma_resv_add_fence(resv, &fence->base, exclusive ?
1480			   DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ);
1481}
1482
1483static void
1484nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo)
1485{
1486	nouveau_bo_move_ntfy(bo, NULL);
1487}
1488
1489struct ttm_device_funcs nouveau_bo_driver = {
1490	.ttm_tt_create = &nouveau_ttm_tt_create,
1491	.ttm_tt_populate = &nouveau_ttm_tt_populate,
1492	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1493	.ttm_tt_destroy = &nouveau_ttm_tt_destroy,
1494	.eviction_valuable = ttm_bo_eviction_valuable,
1495	.evict_flags = nouveau_bo_evict_flags,
1496	.delete_mem_notify = nouveau_bo_delete_mem_notify,
1497	.move = nouveau_bo_move,
1498	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1499	.io_mem_free = &nouveau_ttm_io_mem_free,
1500};