pvr_vm.c

   1// SPDX-License-Identifier: GPL-2.0-only OR MIT
   2/* Copyright (c) 2023 Imagination Technologies Ltd. */
   3
   4#include "pvr_vm.h"
   5
   6#include "pvr_device.h"
   7#include "pvr_drv.h"
   8#include "pvr_gem.h"
   9#include "pvr_mmu.h"
  10#include "pvr_rogue_fwif.h"
  11#include "pvr_rogue_heap_config.h"
  12
  13#include <drm/drm_exec.h>
  14#include <drm/drm_gem.h>
  15#include <drm/drm_gpuvm.h>
  16#include <drm/drm_print.h>
  17
  18#include <linux/bug.h>
  19#include <linux/container_of.h>
  20#include <linux/err.h>
  21#include <linux/errno.h>
  22#include <linux/gfp_types.h>
  23#include <linux/kref.h>
  24#include <linux/mutex.h>
  25#include <linux/stddef.h>
  26
  27/**
  28 * DOC: Memory context
  29 *
  30 * This is the "top level" datatype in the VM code. It's exposed in the public
  31 * API as an opaque handle.
  32 */
  33
  34/**
  35 * struct pvr_vm_context - Context type used to represent a single VM.
  36 */
  37struct pvr_vm_context {
  38	/**
  39	 * @pvr_dev: The PowerVR device to which this context is bound.
  40	 * This binding is immutable for the life of the context.
  41	 */
  42	struct pvr_device *pvr_dev;
  43
  44	/** @mmu_ctx: The context for binding to physical memory. */
  45	struct pvr_mmu_context *mmu_ctx;
  46
  47	/** @gpuvm_mgr: GPUVM object associated with this context. */
  48	struct drm_gpuvm gpuvm_mgr;
  49
  50	/** @lock: Global lock on this VM. */
  51	struct mutex lock;
  52
  53	/**
  54	 * @fw_mem_ctx_obj: Firmware object representing firmware memory
  55	 * context.
  56	 */
  57	struct pvr_fw_object *fw_mem_ctx_obj;
  58
  59	/** @ref_count: Reference count of object. */
  60	struct kref ref_count;
  61
  62	/**
  63	 * @dummy_gem: GEM object to enable VM reservation. All private BOs
  64	 * should use the @dummy_gem.resv and not their own _resv field.
  65	 */
  66	struct drm_gem_object dummy_gem;
  67};
  68
  69static inline
  70struct pvr_vm_context *to_pvr_vm_context(struct drm_gpuvm *gpuvm)
  71{
  72	return container_of(gpuvm, struct pvr_vm_context, gpuvm_mgr);
  73}
  74
  75struct pvr_vm_context *pvr_vm_context_get(struct pvr_vm_context *vm_ctx)
  76{
  77	if (vm_ctx)
  78		kref_get(&vm_ctx->ref_count);
  79
  80	return vm_ctx;
  81}
  82
  83/**
  84 * pvr_vm_get_page_table_root_addr() - Get the DMA address of the root of the
  85 *                                     page table structure behind a VM context.
  86 * @vm_ctx: Target VM context.
  87 */
  88dma_addr_t pvr_vm_get_page_table_root_addr(struct pvr_vm_context *vm_ctx)
  89{
  90	return pvr_mmu_get_root_table_dma_addr(vm_ctx->mmu_ctx);
  91}
  92
  93/**
  94 * pvr_vm_get_dma_resv() - Expose the dma_resv owned by the VM context.
  95 * @vm_ctx: Target VM context.
  96 *
  97 * This is used to allow private BOs to share a dma_resv for faster fence
  98 * updates.
  99 *
 100 * Returns: The dma_resv pointer.
 101 */
 102struct dma_resv *pvr_vm_get_dma_resv(struct pvr_vm_context *vm_ctx)
 103{
 104	return vm_ctx->dummy_gem.resv;
 105}
 106
 107/**
 108 * DOC: Memory mappings
 109 */
 110
 111/**
 112 * struct pvr_vm_gpuva - Wrapper type representing a single VM mapping.
 113 */
 114struct pvr_vm_gpuva {
 115	/** @base: The wrapped drm_gpuva object. */
 116	struct drm_gpuva base;
 117};
 118
 119#define to_pvr_vm_gpuva(va) container_of_const(va, struct pvr_vm_gpuva, base)
 120
 121enum pvr_vm_bind_type {
 122	PVR_VM_BIND_TYPE_MAP,
 123	PVR_VM_BIND_TYPE_UNMAP,
 124};
 125
 126/**
 127 * struct pvr_vm_bind_op - Context of a map/unmap operation.
 128 */
 129struct pvr_vm_bind_op {
 130	/** @type: Map or unmap. */
 131	enum pvr_vm_bind_type type;
 132
 133	/** @pvr_obj: Object associated with mapping (map only). */
 134	struct pvr_gem_object *pvr_obj;
 135
 136	/**
 137	 * @vm_ctx: VM context where the mapping will be created or destroyed.
 138	 */
 139	struct pvr_vm_context *vm_ctx;
 140
 141	/** @mmu_op_ctx: MMU op context. */
 142	struct pvr_mmu_op_context *mmu_op_ctx;
 143
 144	/** @gpuvm_bo: Prealloced wrapped BO for attaching to the gpuvm. */
 145	struct drm_gpuvm_bo *gpuvm_bo;
 146
 147	/**
 148	 * @new_va: Prealloced VA mapping object (init in callback).
 149	 * Used when creating a mapping.
 150	 */
 151	struct pvr_vm_gpuva *new_va;
 152
 153	/**
 154	 * @prev_va: Prealloced VA mapping object (init in callback).
 155	 * Used when a mapping or unmapping operation overlaps an existing
 156	 * mapping and splits away the beginning into a new mapping.
 157	 */
 158	struct pvr_vm_gpuva *prev_va;
 159
 160	/**
 161	 * @next_va: Prealloced VA mapping object (init in callback).
 162	 * Used when a mapping or unmapping operation overlaps an existing
 163	 * mapping and splits away the end into a new mapping.
 164	 */
 165	struct pvr_vm_gpuva *next_va;
 166
 167	/** @offset: Offset into @pvr_obj to begin mapping from. */
 168	u64 offset;
 169
 170	/** @device_addr: Device-virtual address at the start of the mapping. */
 171	u64 device_addr;
 172
 173	/** @size: Size of the desired mapping. */
 174	u64 size;
 175};
 176
 177/**
 178 * pvr_vm_bind_op_exec() - Execute a single bind op.
 179 * @bind_op: Bind op context.
 180 *
 181 * Returns:
 182 *  * 0 on success,
 183 *  * Any error code returned by drm_gpuva_sm_map(), drm_gpuva_sm_unmap(), or
 184 *    a callback function.
 185 */
 186static int pvr_vm_bind_op_exec(struct pvr_vm_bind_op *bind_op)
 187{
 188	switch (bind_op->type) {
 189	case PVR_VM_BIND_TYPE_MAP: {
 190		const struct drm_gpuvm_map_req map_req = {
 191			.map.va.addr = bind_op->device_addr,
 192			.map.va.range = bind_op->size,
 193			.map.gem.obj = gem_from_pvr_gem(bind_op->pvr_obj),
 194			.map.gem.offset = bind_op->offset,
 195		};
 196
 197		return drm_gpuvm_sm_map(&bind_op->vm_ctx->gpuvm_mgr,
 198					bind_op, &map_req);
 199	}
 200
 201	case PVR_VM_BIND_TYPE_UNMAP:
 202		return drm_gpuvm_sm_unmap(&bind_op->vm_ctx->gpuvm_mgr,
 203					  bind_op, bind_op->device_addr,
 204					  bind_op->size);
 205	}
 206
 207	/*
 208	 * This shouldn't happen unless something went wrong
 209	 * in drm_sched.
 210	 */
 211	WARN_ON(1);
 212	return -EINVAL;
 213}
 214
 215static void pvr_vm_bind_op_fini(struct pvr_vm_bind_op *bind_op)
 216{
 217	drm_gpuvm_bo_put(bind_op->gpuvm_bo);
 218
 219	kfree(bind_op->new_va);
 220	kfree(bind_op->prev_va);
 221	kfree(bind_op->next_va);
 222
 223	if (bind_op->pvr_obj)
 224		pvr_gem_object_put(bind_op->pvr_obj);
 225
 226	if (bind_op->mmu_op_ctx)
 227		pvr_mmu_op_context_destroy(bind_op->mmu_op_ctx);
 228}
 229
 230static int
 231pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op,
 232			struct pvr_vm_context *vm_ctx,
 233			struct pvr_gem_object *pvr_obj, u64 offset,
 234			u64 device_addr, u64 size)
 235{
 236	struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);
 237	const bool is_user = vm_ctx != vm_ctx->pvr_dev->kernel_vm_ctx;
 238	const u64 pvr_obj_size = pvr_gem_object_size(pvr_obj);
 239	struct sg_table *sgt;
 240	u64 offset_plus_size;
 241	int err;
 242
 243	if (check_add_overflow(offset, size, &offset_plus_size))
 244		return -EINVAL;
 245
 246	if (is_user &&
 247	    !pvr_find_heap_containing(vm_ctx->pvr_dev, device_addr, size)) {
 248		return -EINVAL;
 249	}
 250
 251	if (!pvr_device_addr_and_size_are_valid(vm_ctx, device_addr, size) ||
 252	    offset & ~PAGE_MASK || size & ~PAGE_MASK ||
 253	    offset >= pvr_obj_size || offset_plus_size > pvr_obj_size)
 254		return -EINVAL;
 255
 256	bind_op->type = PVR_VM_BIND_TYPE_MAP;
 257
 258	dma_resv_lock(obj->resv, NULL);
 259	bind_op->gpuvm_bo = drm_gpuvm_bo_obtain(&vm_ctx->gpuvm_mgr, obj);
 260	dma_resv_unlock(obj->resv);
 261	if (IS_ERR(bind_op->gpuvm_bo))
 262		return PTR_ERR(bind_op->gpuvm_bo);
 263
 264	bind_op->new_va = kzalloc(sizeof(*bind_op->new_va), GFP_KERNEL);
 265	bind_op->prev_va = kzalloc(sizeof(*bind_op->prev_va), GFP_KERNEL);
 266	bind_op->next_va = kzalloc(sizeof(*bind_op->next_va), GFP_KERNEL);
 267	if (!bind_op->new_va || !bind_op->prev_va || !bind_op->next_va) {
 268		err = -ENOMEM;
 269		goto err_bind_op_fini;
 270	}
 271
 272	/* Pin pages so they're ready for use. */
 273	sgt = pvr_gem_object_get_pages_sgt(pvr_obj);
 274	err = PTR_ERR_OR_ZERO(sgt);
 275	if (err)
 276		goto err_bind_op_fini;
 277
 278	bind_op->mmu_op_ctx =
 279		pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, offset, size);
 280	err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx);
 281	if (err) {
 282		bind_op->mmu_op_ctx = NULL;
 283		goto err_bind_op_fini;
 284	}
 285
 286	bind_op->pvr_obj = pvr_obj;
 287	bind_op->vm_ctx = vm_ctx;
 288	bind_op->device_addr = device_addr;
 289	bind_op->size = size;
 290	bind_op->offset = offset;
 291
 292	return 0;
 293
 294err_bind_op_fini:
 295	pvr_vm_bind_op_fini(bind_op);
 296
 297	return err;
 298}
 299
 300static int
 301pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op,
 302			  struct pvr_vm_context *vm_ctx,
 303			  struct pvr_gem_object *pvr_obj,
 304			  u64 device_addr, u64 size)
 305{
 306	int err;
 307
 308	if (!pvr_device_addr_and_size_are_valid(vm_ctx, device_addr, size))
 309		return -EINVAL;
 310
 311	bind_op->type = PVR_VM_BIND_TYPE_UNMAP;
 312
 313	bind_op->prev_va = kzalloc(sizeof(*bind_op->prev_va), GFP_KERNEL);
 314	bind_op->next_va = kzalloc(sizeof(*bind_op->next_va), GFP_KERNEL);
 315	if (!bind_op->prev_va || !bind_op->next_va) {
 316		err = -ENOMEM;
 317		goto err_bind_op_fini;
 318	}
 319
 320	bind_op->mmu_op_ctx =
 321		pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, 0, 0);
 322	err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx);
 323	if (err) {
 324		bind_op->mmu_op_ctx = NULL;
 325		goto err_bind_op_fini;
 326	}
 327
 328	bind_op->pvr_obj = pvr_obj;
 329	bind_op->vm_ctx = vm_ctx;
 330	bind_op->device_addr = device_addr;
 331	bind_op->size = size;
 332
 333	return 0;
 334
 335err_bind_op_fini:
 336	pvr_vm_bind_op_fini(bind_op);
 337
 338	return err;
 339}
 340
 341/**
 342 * pvr_vm_gpuva_map() - Insert a mapping into a memory context.
 343 * @op: gpuva op containing the remap details.
 344 * @op_ctx: Operation context.
 345 *
 346 * Context: Called by drm_gpuvm_sm_map following a successful mapping while
 347 * @op_ctx.vm_ctx mutex is held.
 348 *
 349 * Return:
 350 *  * 0 on success, or
 351 *  * Any error returned by pvr_mmu_map().
 352 */
 353static int
 354pvr_vm_gpuva_map(struct drm_gpuva_op *op, void *op_ctx)
 355{
 356	struct pvr_gem_object *pvr_gem = gem_to_pvr_gem(op->map.gem.obj);
 357	struct pvr_vm_bind_op *ctx = op_ctx;
 358	int err;
 359
 360	if ((op->map.gem.offset | op->map.va.range) & ~PVR_DEVICE_PAGE_MASK)
 361		return -EINVAL;
 362
 363	err = pvr_mmu_map(ctx->mmu_op_ctx, op->map.va.range, pvr_gem->flags,
 364			  op->map.va.addr);
 365	if (err)
 366		return err;
 367
 368	drm_gpuva_map(&ctx->vm_ctx->gpuvm_mgr, &ctx->new_va->base, &op->map);
 369	drm_gpuva_link(&ctx->new_va->base, ctx->gpuvm_bo);
 370	ctx->new_va = NULL;
 371
 372	return 0;
 373}
 374
 375/**
 376 * pvr_vm_gpuva_unmap() - Remove a mapping from a memory context.
 377 * @op: gpuva op containing the unmap details.
 378 * @op_ctx: Operation context.
 379 *
 380 * Context: Called by drm_gpuvm_sm_unmap following a successful unmapping while
 381 * @op_ctx.vm_ctx mutex is held.
 382 *
 383 * Return:
 384 *  * 0 on success, or
 385 *  * Any error returned by pvr_mmu_unmap().
 386 */
 387static int
 388pvr_vm_gpuva_unmap(struct drm_gpuva_op *op, void *op_ctx)
 389{
 390	struct pvr_vm_bind_op *ctx = op_ctx;
 391
 392	int err = pvr_mmu_unmap(ctx->mmu_op_ctx, op->unmap.va->va.addr,
 393				op->unmap.va->va.range);
 394
 395	if (err)
 396		return err;
 397
 398	drm_gpuva_unmap(&op->unmap);
 399	drm_gpuva_unlink(op->unmap.va);
 400	kfree(to_pvr_vm_gpuva(op->unmap.va));
 401
 402	return 0;
 403}
 404
 405/**
 406 * pvr_vm_gpuva_remap() - Remap a mapping within a memory context.
 407 * @op: gpuva op containing the remap details.
 408 * @op_ctx: Operation context.
 409 *
 410 * Context: Called by either drm_gpuvm_sm_map or drm_gpuvm_sm_unmap when a
 411 * mapping or unmapping operation causes a region to be split. The
 412 * @op_ctx.vm_ctx mutex is held.
 413 *
 414 * Return:
 415 *  * 0 on success, or
 416 *  * Any error returned by pvr_vm_gpuva_unmap() or pvr_vm_gpuva_unmap().
 417 */
 418static int
 419pvr_vm_gpuva_remap(struct drm_gpuva_op *op, void *op_ctx)
 420{
 421	struct pvr_vm_bind_op *ctx = op_ctx;
 422	u64 va_start = 0, va_range = 0;
 423	int err;
 424
 425	drm_gpuva_op_remap_to_unmap_range(&op->remap, &va_start, &va_range);
 426	err = pvr_mmu_unmap(ctx->mmu_op_ctx, va_start, va_range);
 427	if (err)
 428		return err;
 429
 430	/* No actual remap required: the page table tree depth is fixed to 3,
 431	 * and we use 4k page table entries only for now.
 432	 */
 433	drm_gpuva_remap(&ctx->prev_va->base, &ctx->next_va->base, &op->remap);
 434
 435	if (op->remap.prev) {
 436		pvr_gem_object_get(gem_to_pvr_gem(ctx->prev_va->base.gem.obj));
 437		drm_gpuva_link(&ctx->prev_va->base, ctx->gpuvm_bo);
 438		ctx->prev_va = NULL;
 439	}
 440
 441	if (op->remap.next) {
 442		pvr_gem_object_get(gem_to_pvr_gem(ctx->next_va->base.gem.obj));
 443		drm_gpuva_link(&ctx->next_va->base, ctx->gpuvm_bo);
 444		ctx->next_va = NULL;
 445	}
 446
 447	drm_gpuva_unlink(op->remap.unmap->va);
 448	kfree(to_pvr_vm_gpuva(op->remap.unmap->va));
 449
 450	return 0;
 451}
 452
 453/*
 454 * Public API
 455 *
 456 * For an overview of these functions, see *DOC: Public API* in "pvr_vm.h".
 457 */
 458
 459/**
 460 * pvr_device_addr_is_valid() - Tests whether a device-virtual address
 461 *                              is valid.
 462 * @device_addr: Virtual device address to test.
 463 *
 464 * Return:
 465 *  * %true if @device_addr is within the valid range for a device page
 466 *    table and is aligned to the device page size, or
 467 *  * %false otherwise.
 468 */
 469bool
 470pvr_device_addr_is_valid(u64 device_addr)
 471{
 472	return (device_addr & ~PVR_PAGE_TABLE_ADDR_MASK) == 0 &&
 473	       (device_addr & ~PVR_DEVICE_PAGE_MASK) == 0;
 474}
 475
 476/**
 477 * pvr_device_addr_and_size_are_valid() - Tests whether a device-virtual
 478 * address and associated size are both valid.
 479 * @vm_ctx: Target VM context.
 480 * @device_addr: Virtual device address to test.
 481 * @size: Size of the range based at @device_addr to test.
 482 *
 483 * Calling pvr_device_addr_is_valid() twice (once on @size, and again on
 484 * @device_addr + @size) to verify a device-virtual address range initially
 485 * seems intuitive, but it produces a false-negative when the address range
 486 * is right at the end of device-virtual address space.
 487 *
 488 * This function catches that corner case, as well as checking that
 489 * @size is non-zero.
 490 *
 491 * Return:
 492 *  * %true if @device_addr is device page aligned; @size is device page
 493 *    aligned; the range specified by @device_addr and @size is within the
 494 *    bounds of the device-virtual address space, and @size is non-zero, or
 495 *  * %false otherwise.
 496 */
 497bool
 498pvr_device_addr_and_size_are_valid(struct pvr_vm_context *vm_ctx,
 499				   u64 device_addr, u64 size)
 500{
 501	return pvr_device_addr_is_valid(device_addr) &&
 502	       drm_gpuvm_range_valid(&vm_ctx->gpuvm_mgr, device_addr, size) &&
 503	       size != 0 && (size & ~PVR_DEVICE_PAGE_MASK) == 0 &&
 504	       (device_addr + size <= PVR_PAGE_TABLE_ADDR_SPACE_SIZE);
 505}
 506
 507static void pvr_gpuvm_free(struct drm_gpuvm *gpuvm)
 508{
 509	kfree(to_pvr_vm_context(gpuvm));
 510}
 511
 512static const struct drm_gpuvm_ops pvr_vm_gpuva_ops = {
 513	.vm_free = pvr_gpuvm_free,
 514	.sm_step_map = pvr_vm_gpuva_map,
 515	.sm_step_remap = pvr_vm_gpuva_remap,
 516	.sm_step_unmap = pvr_vm_gpuva_unmap,
 517};
 518
 519static void
 520fw_mem_context_init(void *cpu_ptr, void *priv)
 521{
 522	struct rogue_fwif_fwmemcontext *fw_mem_ctx = cpu_ptr;
 523	struct pvr_vm_context *vm_ctx = priv;
 524
 525	fw_mem_ctx->pc_dev_paddr = pvr_vm_get_page_table_root_addr(vm_ctx);
 526	fw_mem_ctx->page_cat_base_reg_set = ROGUE_FW_BIF_INVALID_PCSET;
 527}
 528
 529/**
 530 * pvr_vm_create_context() - Create a new VM context.
 531 * @pvr_dev: Target PowerVR device.
 532 * @is_userspace_context: %true if this context is for userspace. This will
 533 *                        create a firmware memory context for the VM context
 534 *                        and disable warnings when tearing down mappings.
 535 *
 536 * Return:
 537 *  * A handle to the newly-minted VM context on success,
 538 *  * -%EINVAL if the feature "virtual address space bits" on @pvr_dev is
 539 *    missing or has an unsupported value,
 540 *  * -%ENOMEM if allocation of the structure behind the opaque handle fails,
 541 *    or
 542 *  * Any error encountered while setting up internal structures.
 543 */
 544struct pvr_vm_context *
 545pvr_vm_create_context(struct pvr_device *pvr_dev, bool is_userspace_context)
 546{
 547	struct drm_device *drm_dev = from_pvr_device(pvr_dev);
 548
 549	struct pvr_vm_context *vm_ctx;
 550	u16 device_addr_bits;
 551
 552	int err;
 553
 554	err = PVR_FEATURE_VALUE(pvr_dev, virtual_address_space_bits,
 555				&device_addr_bits);
 556	if (err) {
 557		drm_err(drm_dev,
 558			"Failed to get device virtual address space bits\n");
 559		return ERR_PTR(err);
 560	}
 561
 562	if (device_addr_bits != PVR_PAGE_TABLE_ADDR_BITS) {
 563		drm_err(drm_dev,
 564			"Device has unsupported virtual address space size\n");
 565		return ERR_PTR(-EINVAL);
 566	}
 567
 568	vm_ctx = kzalloc(sizeof(*vm_ctx), GFP_KERNEL);
 569	if (!vm_ctx)
 570		return ERR_PTR(-ENOMEM);
 571
 572	vm_ctx->pvr_dev = pvr_dev;
 573
 574	vm_ctx->mmu_ctx = pvr_mmu_context_create(pvr_dev);
 575	err = PTR_ERR_OR_ZERO(vm_ctx->mmu_ctx);
 576	if (err)
 577		goto err_free;
 578
 579	if (is_userspace_context) {
 580		err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_fwmemcontext),
 581					   PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 582					   fw_mem_context_init, vm_ctx, &vm_ctx->fw_mem_ctx_obj);
 583
 584		if (err)
 585			goto err_page_table_destroy;
 586	}
 587
 588	drm_gem_private_object_init(&pvr_dev->base, &vm_ctx->dummy_gem, 0);
 589	drm_gpuvm_init(&vm_ctx->gpuvm_mgr,
 590		       is_userspace_context ? "PowerVR-user-VM" : "PowerVR-FW-VM",
 591		       0, &pvr_dev->base, &vm_ctx->dummy_gem,
 592		       0, 1ULL << device_addr_bits, 0, 0, &pvr_vm_gpuva_ops);
 593
 594	mutex_init(&vm_ctx->lock);
 595	kref_init(&vm_ctx->ref_count);
 596
 597	return vm_ctx;
 598
 599err_page_table_destroy:
 600	pvr_mmu_context_destroy(vm_ctx->mmu_ctx);
 601
 602err_free:
 603	kfree(vm_ctx);
 604
 605	return ERR_PTR(err);
 606}
 607
 608/**
 609 * pvr_vm_context_release() - Teardown a VM context.
 610 * @ref_count: Pointer to reference counter of the VM context.
 611 *
 612 * This function also ensures that no mappings are left dangling by calling
 613 * pvr_vm_unmap_all.
 614 */
 615static void
 616pvr_vm_context_release(struct kref *ref_count)
 617{
 618	struct pvr_vm_context *vm_ctx =
 619		container_of(ref_count, struct pvr_vm_context, ref_count);
 620
 621	if (vm_ctx->fw_mem_ctx_obj)
 622		pvr_fw_object_destroy(vm_ctx->fw_mem_ctx_obj);
 623
 624	pvr_vm_unmap_all(vm_ctx);
 625
 626	pvr_mmu_context_destroy(vm_ctx->mmu_ctx);
 627	drm_gem_private_object_fini(&vm_ctx->dummy_gem);
 628	mutex_destroy(&vm_ctx->lock);
 629
 630	drm_gpuvm_put(&vm_ctx->gpuvm_mgr);
 631}
 632
 633/**
 634 * pvr_vm_context_lookup() - Look up VM context from handle
 635 * @pvr_file: Pointer to pvr_file structure.
 636 * @handle: Object handle.
 637 *
 638 * Takes reference on VM context object. Call pvr_vm_context_put() to release.
 639 *
 640 * Returns:
 641 *  * The requested object on success, or
 642 *  * %NULL on failure (object does not exist in list, or is not a VM context)
 643 */
 644struct pvr_vm_context *
 645pvr_vm_context_lookup(struct pvr_file *pvr_file, u32 handle)
 646{
 647	struct pvr_vm_context *vm_ctx;
 648
 649	xa_lock(&pvr_file->vm_ctx_handles);
 650	vm_ctx = xa_load(&pvr_file->vm_ctx_handles, handle);
 651	pvr_vm_context_get(vm_ctx);
 652	xa_unlock(&pvr_file->vm_ctx_handles);
 653
 654	return vm_ctx;
 655}
 656
 657/**
 658 * pvr_vm_context_put() - Release a reference on a VM context
 659 * @vm_ctx: Target VM context.
 660 *
 661 * Returns:
 662 *  * %true if the VM context was destroyed, or
 663 *  * %false if there are any references still remaining.
 664 */
 665bool
 666pvr_vm_context_put(struct pvr_vm_context *vm_ctx)
 667{
 668	if (vm_ctx)
 669		return kref_put(&vm_ctx->ref_count, pvr_vm_context_release);
 670
 671	return true;
 672}
 673
 674/**
 675 * pvr_destroy_vm_contexts_for_file: Destroy any VM contexts associated with the
 676 * given file.
 677 * @pvr_file: Pointer to pvr_file structure.
 678 *
 679 * Removes all vm_contexts associated with @pvr_file from the device VM context
 680 * list and drops initial references. vm_contexts will then be destroyed once
 681 * all outstanding references are dropped.
 682 */
 683void pvr_destroy_vm_contexts_for_file(struct pvr_file *pvr_file)
 684{
 685	struct pvr_vm_context *vm_ctx;
 686	unsigned long handle;
 687
 688	xa_for_each(&pvr_file->vm_ctx_handles, handle, vm_ctx) {
 689		/* vm_ctx is not used here because that would create a race with xa_erase */
 690		pvr_vm_context_put(xa_erase(&pvr_file->vm_ctx_handles, handle));
 691	}
 692}
 693
 694static int
 695pvr_vm_lock_extra(struct drm_gpuvm_exec *vm_exec)
 696{
 697	struct pvr_vm_bind_op *bind_op = vm_exec->extra.priv;
 698	struct pvr_gem_object *pvr_obj = bind_op->pvr_obj;
 699
 700	/* Acquire lock on the GEM object being mapped/unmapped. */
 701	return drm_exec_lock_obj(&vm_exec->exec, gem_from_pvr_gem(pvr_obj));
 702}
 703
 704/**
 705 * pvr_vm_map() - Map a section of physical memory into a section of
 706 * device-virtual memory.
 707 * @vm_ctx: Target VM context.
 708 * @pvr_obj: Target PowerVR memory object.
 709 * @pvr_obj_offset: Offset into @pvr_obj to map from.
 710 * @device_addr: Virtual device address at the start of the requested mapping.
 711 * @size: Size of the requested mapping.
 712 *
 713 * No handle is returned to represent the mapping. Instead, callers should
 714 * remember @device_addr and use that as a handle.
 715 *
 716 * Return:
 717 *  * 0 on success,
 718 *  * -%EINVAL if @device_addr is not a valid page-aligned device-virtual
 719 *    address; the region specified by @pvr_obj_offset and @size does not fall
 720 *    entirely within @pvr_obj, or any part of the specified region of @pvr_obj
 721 *    is not device-virtual page-aligned,
 722 *  * Any error encountered while performing internal operations required to
 723 *    destroy the mapping (returned from pvr_vm_gpuva_map or
 724 *    pvr_vm_gpuva_remap).
 725 */
 726int
 727pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
 728	   u64 pvr_obj_offset, u64 device_addr, u64 size)
 729{
 730	struct pvr_vm_bind_op bind_op = {0};
 731	struct drm_gpuvm_exec vm_exec = {
 732		.vm = &vm_ctx->gpuvm_mgr,
 733		.flags = DRM_EXEC_INTERRUPTIBLE_WAIT |
 734			 DRM_EXEC_IGNORE_DUPLICATES,
 735		.extra = {
 736			.fn = pvr_vm_lock_extra,
 737			.priv = &bind_op,
 738		},
 739	};
 740
 741	int err = pvr_vm_bind_op_map_init(&bind_op, vm_ctx, pvr_obj,
 742					  pvr_obj_offset, device_addr,
 743					  size);
 744
 745	if (err)
 746		return err;
 747
 748	pvr_gem_object_get(pvr_obj);
 749
 750	err = drm_gpuvm_exec_lock(&vm_exec);
 751	if (err)
 752		goto err_cleanup;
 753
 754	err = pvr_vm_bind_op_exec(&bind_op);
 755
 756	drm_gpuvm_exec_unlock(&vm_exec);
 757
 758err_cleanup:
 759	pvr_vm_bind_op_fini(&bind_op);
 760
 761	return err;
 762}
 763
 764/**
 765 * pvr_vm_unmap_obj_locked() - Unmap an already mapped section of device-virtual
 766 * memory.
 767 * @vm_ctx: Target VM context.
 768 * @pvr_obj: Target PowerVR memory object.
 769 * @device_addr: Virtual device address at the start of the target mapping.
 770 * @size: Size of the target mapping.
 771 *
 772 * Return:
 773 *  * 0 on success,
 774 *  * -%EINVAL if @device_addr is not a valid page-aligned device-virtual
 775 *    address,
 776 *  * Any error encountered while performing internal operations required to
 777 *    destroy the mapping (returned from pvr_vm_gpuva_unmap or
 778 *    pvr_vm_gpuva_remap).
 779 *
 780 * The vm_ctx->lock must be held when calling this function.
 781 */
 782static int
 783pvr_vm_unmap_obj_locked(struct pvr_vm_context *vm_ctx,
 784			struct pvr_gem_object *pvr_obj,
 785			u64 device_addr, u64 size)
 786{
 787	struct pvr_vm_bind_op bind_op = {0};
 788	struct drm_gpuvm_exec vm_exec = {
 789		.vm = &vm_ctx->gpuvm_mgr,
 790		.flags = DRM_EXEC_INTERRUPTIBLE_WAIT |
 791			 DRM_EXEC_IGNORE_DUPLICATES,
 792		.extra = {
 793			.fn = pvr_vm_lock_extra,
 794			.priv = &bind_op,
 795		},
 796	};
 797
 798	int err = pvr_vm_bind_op_unmap_init(&bind_op, vm_ctx, pvr_obj,
 799					    device_addr, size);
 800	if (err)
 801		return err;
 802
 803	pvr_gem_object_get(pvr_obj);
 804
 805	err = drm_gpuvm_exec_lock(&vm_exec);
 806	if (err)
 807		goto err_cleanup;
 808
 809	err = pvr_vm_bind_op_exec(&bind_op);
 810
 811	drm_gpuvm_exec_unlock(&vm_exec);
 812
 813err_cleanup:
 814	pvr_vm_bind_op_fini(&bind_op);
 815
 816	return err;
 817}
 818
 819/**
 820 * pvr_vm_unmap_obj() - Unmap an already mapped section of device-virtual
 821 * memory.
 822 * @vm_ctx: Target VM context.
 823 * @pvr_obj: Target PowerVR memory object.
 824 * @device_addr: Virtual device address at the start of the target mapping.
 825 * @size: Size of the target mapping.
 826 *
 827 * Return:
 828 *  * 0 on success,
 829 *  * Any error encountered by pvr_vm_unmap_obj_locked.
 830 */
 831int
 832pvr_vm_unmap_obj(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,
 833		 u64 device_addr, u64 size)
 834{
 835	int err;
 836
 837	mutex_lock(&vm_ctx->lock);
 838	err = pvr_vm_unmap_obj_locked(vm_ctx, pvr_obj, device_addr, size);
 839	mutex_unlock(&vm_ctx->lock);
 840
 841	return err;
 842}
 843
 844/**
 845 * pvr_vm_unmap() - Unmap an already mapped section of device-virtual memory.
 846 * @vm_ctx: Target VM context.
 847 * @device_addr: Virtual device address at the start of the target mapping.
 848 * @size: Size of the target mapping.
 849 *
 850 * Return:
 851 *  * 0 on success,
 852 *  * Any error encountered by drm_gpuva_find,
 853 *  * Any error encountered by pvr_vm_unmap_obj_locked.
 854 */
 855int
 856pvr_vm_unmap(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size)
 857{
 858	struct pvr_gem_object *pvr_obj;
 859	struct drm_gpuva *va;
 860	int err;
 861
 862	mutex_lock(&vm_ctx->lock);
 863
 864	va = drm_gpuva_find(&vm_ctx->gpuvm_mgr, device_addr, size);
 865	if (va) {
 866		pvr_obj = gem_to_pvr_gem(va->gem.obj);
 867		err = pvr_vm_unmap_obj_locked(vm_ctx, pvr_obj,
 868					      va->va.addr, va->va.range);
 869	} else {
 870		err = -ENOENT;
 871	}
 872
 873	mutex_unlock(&vm_ctx->lock);
 874
 875	return err;
 876}
 877
 878/**
 879 * pvr_vm_unmap_all() - Unmap all mappings associated with a VM context.
 880 * @vm_ctx: Target VM context.
 881 *
 882 * This function ensures that no mappings are left dangling by unmapping them
 883 * all in order of ascending device-virtual address.
 884 */
 885void
 886pvr_vm_unmap_all(struct pvr_vm_context *vm_ctx)
 887{
 888	mutex_lock(&vm_ctx->lock);
 889
 890	for (;;) {
 891		struct pvr_gem_object *pvr_obj;
 892		struct drm_gpuva *va;
 893
 894		va = drm_gpuva_find_first(&vm_ctx->gpuvm_mgr,
 895					  vm_ctx->gpuvm_mgr.mm_start,
 896					  vm_ctx->gpuvm_mgr.mm_range);
 897		if (!va)
 898			break;
 899
 900		pvr_obj = gem_to_pvr_gem(va->gem.obj);
 901
 902		WARN_ON(pvr_vm_unmap_obj_locked(vm_ctx, pvr_obj,
 903						va->va.addr, va->va.range));
 904	}
 905
 906	mutex_unlock(&vm_ctx->lock);
 907}
 908
 909/* Static data areas are determined by firmware. */
 910static const struct drm_pvr_static_data_area static_data_areas[] = {
 911	{
 912		.area_usage = DRM_PVR_STATIC_DATA_AREA_FENCE,
 913		.location_heap_id = DRM_PVR_HEAP_GENERAL,
 914		.offset = 0,
 915		.size = 128,
 916	},
 917	{
 918		.area_usage = DRM_PVR_STATIC_DATA_AREA_YUV_CSC,
 919		.location_heap_id = DRM_PVR_HEAP_GENERAL,
 920		.offset = 128,
 921		.size = 1024,
 922	},
 923	{
 924		.area_usage = DRM_PVR_STATIC_DATA_AREA_VDM_SYNC,
 925		.location_heap_id = DRM_PVR_HEAP_PDS_CODE_DATA,
 926		.offset = 0,
 927		.size = 128,
 928	},
 929	{
 930		.area_usage = DRM_PVR_STATIC_DATA_AREA_EOT,
 931		.location_heap_id = DRM_PVR_HEAP_PDS_CODE_DATA,
 932		.offset = 128,
 933		.size = 128,
 934	},
 935	{
 936		.area_usage = DRM_PVR_STATIC_DATA_AREA_VDM_SYNC,
 937		.location_heap_id = DRM_PVR_HEAP_USC_CODE,
 938		.offset = 0,
 939		.size = 128,
 940	},
 941};
 942
 943#define GET_RESERVED_SIZE(last_offset, last_size) round_up((last_offset) + (last_size), PAGE_SIZE)
 944
 945/*
 946 * The values given to GET_RESERVED_SIZE() are taken from the last entry in the corresponding
 947 * static data area for each heap.
 948 */
 949static const struct drm_pvr_heap pvr_heaps[] = {
 950	[DRM_PVR_HEAP_GENERAL] = {
 951		.base = ROGUE_GENERAL_HEAP_BASE,
 952		.size = ROGUE_GENERAL_HEAP_SIZE,
 953		.flags = 0,
 954		.page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
 955	},
 956	[DRM_PVR_HEAP_PDS_CODE_DATA] = {
 957		.base = ROGUE_PDSCODEDATA_HEAP_BASE,
 958		.size = ROGUE_PDSCODEDATA_HEAP_SIZE,
 959		.flags = 0,
 960		.page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
 961	},
 962	[DRM_PVR_HEAP_USC_CODE] = {
 963		.base = ROGUE_USCCODE_HEAP_BASE,
 964		.size = ROGUE_USCCODE_HEAP_SIZE,
 965		.flags = 0,
 966		.page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
 967	},
 968	[DRM_PVR_HEAP_RGNHDR] = {
 969		.base = ROGUE_RGNHDR_HEAP_BASE,
 970		.size = ROGUE_RGNHDR_HEAP_SIZE,
 971		.flags = 0,
 972		.page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
 973	},
 974	[DRM_PVR_HEAP_VIS_TEST] = {
 975		.base = ROGUE_VISTEST_HEAP_BASE,
 976		.size = ROGUE_VISTEST_HEAP_SIZE,
 977		.flags = 0,
 978		.page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
 979	},
 980	[DRM_PVR_HEAP_TRANSFER_FRAG] = {
 981		.base = ROGUE_TRANSFER_FRAG_HEAP_BASE,
 982		.size = ROGUE_TRANSFER_FRAG_HEAP_SIZE,
 983		.flags = 0,
 984		.page_size_log2 = PVR_DEVICE_PAGE_SHIFT,
 985	},
 986};
 987
 988int
 989pvr_static_data_areas_get(const struct pvr_device *pvr_dev,
 990			  struct drm_pvr_ioctl_dev_query_args *args)
 991{
 992	struct drm_pvr_dev_query_static_data_areas query = {0};
 993	int err;
 994
 995	if (!args->pointer) {
 996		args->size = sizeof(struct drm_pvr_dev_query_static_data_areas);
 997		return 0;
 998	}
 999
1000	err = PVR_UOBJ_GET(query, args->size, args->pointer);
1001	if (err < 0)
1002		return err;
1003
1004	if (!query.static_data_areas.array) {
1005		query.static_data_areas.count = ARRAY_SIZE(static_data_areas);
1006		query.static_data_areas.stride = sizeof(struct drm_pvr_static_data_area);
1007		goto copy_out;
1008	}
1009
1010	if (query.static_data_areas.count > ARRAY_SIZE(static_data_areas))
1011		query.static_data_areas.count = ARRAY_SIZE(static_data_areas);
1012
1013	err = PVR_UOBJ_SET_ARRAY(&query.static_data_areas, static_data_areas);
1014	if (err < 0)
1015		return err;
1016
1017copy_out:
1018	err = PVR_UOBJ_SET(args->pointer, args->size, query);
1019	if (err < 0)
1020		return err;
1021
1022	args->size = sizeof(query);
1023	return 0;
1024}
1025
1026int
1027pvr_heap_info_get(const struct pvr_device *pvr_dev,
1028		  struct drm_pvr_ioctl_dev_query_args *args)
1029{
1030	struct drm_pvr_dev_query_heap_info query = {0};
1031	u64 dest;
1032	int err;
1033
1034	if (!args->pointer) {
1035		args->size = sizeof(struct drm_pvr_dev_query_heap_info);
1036		return 0;
1037	}
1038
1039	err = PVR_UOBJ_GET(query, args->size, args->pointer);
1040	if (err < 0)
1041		return err;
1042
1043	if (!query.heaps.array) {
1044		query.heaps.count = ARRAY_SIZE(pvr_heaps);
1045		query.heaps.stride = sizeof(struct drm_pvr_heap);
1046		goto copy_out;
1047	}
1048
1049	if (query.heaps.count > ARRAY_SIZE(pvr_heaps))
1050		query.heaps.count = ARRAY_SIZE(pvr_heaps);
1051
1052	/* Region header heap is only present if BRN63142 is present. */
1053	dest = query.heaps.array;
1054	for (size_t i = 0; i < query.heaps.count; i++) {
1055		struct drm_pvr_heap heap = pvr_heaps[i];
1056
1057		if (i == DRM_PVR_HEAP_RGNHDR && !PVR_HAS_QUIRK(pvr_dev, 63142))
1058			heap.size = 0;
1059
1060		err = PVR_UOBJ_SET(dest, query.heaps.stride, heap);
1061		if (err < 0)
1062			return err;
1063
1064		dest += query.heaps.stride;
1065	}
1066
1067copy_out:
1068	err = PVR_UOBJ_SET(args->pointer, args->size, query);
1069	if (err < 0)
1070		return err;
1071
1072	args->size = sizeof(query);
1073	return 0;
1074}
1075
1076/**
1077 * pvr_heap_contains_range() - Determine if a given heap contains the specified
1078 *                             device-virtual address range.
1079 * @pvr_heap: Target heap.
1080 * @start: Inclusive start of the target range.
1081 * @end: Inclusive end of the target range.
1082 *
1083 * It is an error to call this function with values of @start and @end that do
1084 * not satisfy the condition @start <= @end.
1085 */
1086static __always_inline bool
1087pvr_heap_contains_range(const struct drm_pvr_heap *pvr_heap, u64 start, u64 end)
1088{
1089	return pvr_heap->base <= start && end < pvr_heap->base + pvr_heap->size;
1090}
1091
1092/**
1093 * pvr_find_heap_containing() - Find a heap which contains the specified
1094 *                              device-virtual address range.
1095 * @pvr_dev: Target PowerVR device.
1096 * @start: Start of the target range.
1097 * @size: Size of the target range.
1098 *
1099 * Return:
1100 *  * A pointer to a constant instance of struct drm_pvr_heap representing the
1101 *    heap containing the entire range specified by @start and @size on
1102 *    success, or
1103 *  * %NULL if no such heap exists.
1104 */
1105const struct drm_pvr_heap *
1106pvr_find_heap_containing(struct pvr_device *pvr_dev, u64 start, u64 size)
1107{
1108	u64 end;
1109
1110	if (check_add_overflow(start, size - 1, &end))
1111		return NULL;
1112
1113	/*
1114	 * There are no guarantees about the order of address ranges in
1115	 * &pvr_heaps, so iterate over the entire array for a heap whose
1116	 * range completely encompasses the given range.
1117	 */
1118	for (u32 heap_id = 0; heap_id < ARRAY_SIZE(pvr_heaps); heap_id++) {
1119		/* Filter heaps that present only with an associated quirk */
1120		if (heap_id == DRM_PVR_HEAP_RGNHDR &&
1121		    !PVR_HAS_QUIRK(pvr_dev, 63142)) {
1122			continue;
1123		}
1124
1125		if (pvr_heap_contains_range(&pvr_heaps[heap_id], start, end))
1126			return &pvr_heaps[heap_id];
1127	}
1128
1129	return NULL;
1130}
1131
1132/**
1133 * pvr_vm_find_gem_object() - Look up a buffer object from a given
1134 *                            device-virtual address.
1135 * @vm_ctx: [IN] Target VM context.
1136 * @device_addr: [IN] Virtual device address at the start of the required
1137 *               object.
1138 * @mapped_offset_out: [OUT] Pointer to location to write offset of the start
1139 *                     of the mapped region within the buffer object. May be
1140 *                     %NULL if this information is not required.
1141 * @mapped_size_out: [OUT] Pointer to location to write size of the mapped
1142 *                   region. May be %NULL if this information is not required.
1143 *
1144 * If successful, a reference will be taken on the buffer object. The caller
1145 * must drop the reference with pvr_gem_object_put().
1146 *
1147 * Return:
1148 *  * The PowerVR buffer object mapped at @device_addr if one exists, or
1149 *  * %NULL otherwise.
1150 */
1151struct pvr_gem_object *
1152pvr_vm_find_gem_object(struct pvr_vm_context *vm_ctx, u64 device_addr,
1153		       u64 *mapped_offset_out, u64 *mapped_size_out)
1154{
1155	struct pvr_gem_object *pvr_obj;
1156	struct drm_gpuva *va;
1157
1158	mutex_lock(&vm_ctx->lock);
1159
1160	va = drm_gpuva_find_first(&vm_ctx->gpuvm_mgr, device_addr, 1);
1161	if (!va)
1162		goto err_unlock;
1163
1164	pvr_obj = gem_to_pvr_gem(va->gem.obj);
1165	pvr_gem_object_get(pvr_obj);
1166
1167	if (mapped_offset_out)
1168		*mapped_offset_out = va->gem.offset;
1169	if (mapped_size_out)
1170		*mapped_size_out = va->va.range;
1171
1172	mutex_unlock(&vm_ctx->lock);
1173
1174	return pvr_obj;
1175
1176err_unlock:
1177	mutex_unlock(&vm_ctx->lock);
1178
1179	return NULL;
1180}
1181
1182/**
1183 * pvr_vm_get_fw_mem_context: Get object representing firmware memory context
1184 * @vm_ctx: Target VM context.
1185 *
1186 * Returns:
1187 *  * FW object representing firmware memory context, or
1188 *  * %NULL if this VM context does not have a firmware memory context.
1189 */
1190struct pvr_fw_object *
1191pvr_vm_get_fw_mem_context(struct pvr_vm_context *vm_ctx)
1192{
1193	return vm_ctx->fw_mem_ctx_obj;
1194}