panthor_mmu.c

   1// SPDX-License-Identifier: GPL-2.0 or MIT
   2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
   3/* Copyright 2023 Collabora ltd. */
   4
   5#include <drm/drm_debugfs.h>
   6#include <drm/drm_drv.h>
   7#include <drm/drm_exec.h>
   8#include <drm/drm_gpuvm.h>
   9#include <drm/drm_managed.h>
  10#include <drm/drm_print.h>
  11#include <drm/gpu_scheduler.h>
  12#include <drm/panthor_drm.h>
  13
  14#include <linux/atomic.h>
  15#include <linux/bitfield.h>
  16#include <linux/delay.h>
  17#include <linux/dma-mapping.h>
  18#include <linux/interrupt.h>
  19#include <linux/io.h>
  20#include <linux/iopoll.h>
  21#include <linux/io-pgtable.h>
  22#include <linux/iommu.h>
  23#include <linux/kmemleak.h>
  24#include <linux/platform_device.h>
  25#include <linux/pm_runtime.h>
  26#include <linux/rwsem.h>
  27#include <linux/sched.h>
  28#include <linux/shmem_fs.h>
  29#include <linux/sizes.h>
  30
  31#include "panthor_device.h"
  32#include "panthor_gem.h"
  33#include "panthor_gpu.h"
  34#include "panthor_heap.h"
  35#include "panthor_mmu.h"
  36#include "panthor_regs.h"
  37#include "panthor_sched.h"
  38
  39#define MAX_AS_SLOTS			32
  40
  41struct panthor_vm;
  42
  43/**
  44 * struct panthor_as_slot - Address space slot
  45 */
  46struct panthor_as_slot {
  47	/** @vm: VM bound to this slot. NULL is no VM is bound. */
  48	struct panthor_vm *vm;
  49};
  50
  51/**
  52 * struct panthor_mmu - MMU related data
  53 */
  54struct panthor_mmu {
  55	/** @irq: The MMU irq. */
  56	struct panthor_irq irq;
  57
  58	/**
  59	 * @as: Address space related fields.
  60	 *
  61	 * The GPU has a limited number of address spaces (AS) slots, forcing
  62	 * us to re-assign them to re-assign slots on-demand.
  63	 */
  64	struct {
  65		/** @as.slots_lock: Lock protecting access to all other AS fields. */
  66		struct mutex slots_lock;
  67
  68		/** @as.alloc_mask: Bitmask encoding the allocated slots. */
  69		unsigned long alloc_mask;
  70
  71		/** @as.faulty_mask: Bitmask encoding the faulty slots. */
  72		unsigned long faulty_mask;
  73
  74		/** @as.slots: VMs currently bound to the AS slots. */
  75		struct panthor_as_slot slots[MAX_AS_SLOTS];
  76
  77		/**
  78		 * @as.lru_list: List of least recently used VMs.
  79		 *
  80		 * We use this list to pick a VM to evict when all slots are
  81		 * used.
  82		 *
  83		 * There should be no more active VMs than there are AS slots,
  84		 * so this LRU is just here to keep VMs bound until there's
  85		 * a need to release a slot, thus avoid unnecessary TLB/cache
  86		 * flushes.
  87		 */
  88		struct list_head lru_list;
  89	} as;
  90
  91	/** @vm: VMs management fields */
  92	struct {
  93		/** @vm.lock: Lock protecting access to list. */
  94		struct mutex lock;
  95
  96		/** @vm.list: List containing all VMs. */
  97		struct list_head list;
  98
  99		/** @vm.reset_in_progress: True if a reset is in progress. */
 100		bool reset_in_progress;
 101
 102		/** @vm.wq: Workqueue used for the VM_BIND queues. */
 103		struct workqueue_struct *wq;
 104	} vm;
 105};
 106
 107/**
 108 * struct panthor_vm_pool - VM pool object
 109 */
 110struct panthor_vm_pool {
 111	/** @xa: Array used for VM handle tracking. */
 112	struct xarray xa;
 113};
 114
 115/**
 116 * struct panthor_vma - GPU mapping object
 117 *
 118 * This is used to track GEM mappings in GPU space.
 119 */
 120struct panthor_vma {
 121	/** @base: Inherits from drm_gpuva. */
 122	struct drm_gpuva base;
 123
 124	/** @node: Used to implement deferred release of VMAs. */
 125	struct list_head node;
 126
 127	/**
 128	 * @flags: Combination of drm_panthor_vm_bind_op_flags.
 129	 *
 130	 * Only map related flags are accepted.
 131	 */
 132	u32 flags;
 133};
 134
 135/**
 136 * struct panthor_vm_op_ctx - VM operation context
 137 *
 138 * With VM operations potentially taking place in a dma-signaling path, we
 139 * need to make sure everything that might require resource allocation is
 140 * pre-allocated upfront. This is what this operation context is far.
 141 *
 142 * We also collect resources that have been freed, so we can release them
 143 * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
 144 * request.
 145 */
 146struct panthor_vm_op_ctx {
 147	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
 148	struct {
 149		/** @rsvd_page_tables.count: Number of pages reserved. */
 150		u32 count;
 151
 152		/** @rsvd_page_tables.ptr: Point to the first unused page in the @pages table. */
 153		u32 ptr;
 154
 155		/**
 156		 * @rsvd_page_tables.pages: Array of pages to be used for an MMU page table update.
 157		 *
 158		 * After an VM operation, there might be free pages left in this array.
 159		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
 160		 */
 161		void **pages;
 162	} rsvd_page_tables;
 163
 164	/**
 165	 * @preallocated_vmas: Pre-allocated VMAs to handle the remap case.
 166	 *
 167	 * Partial unmap requests or map requests overlapping existing mappings will
 168	 * trigger a remap call, which need to register up to three panthor_vma objects
 169	 * (one for the new mapping, and two for the previous and next mappings).
 170	 */
 171	struct panthor_vma *preallocated_vmas[3];
 172
 173	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
 174	u32 flags;
 175
 176	/** @va: Virtual range targeted by the VM operation. */
 177	struct {
 178		/** @va.addr: Start address. */
 179		u64 addr;
 180
 181		/** @va.range: Range size. */
 182		u64 range;
 183	} va;
 184
 185	/** @map: Fields specific to a map operation. */
 186	struct {
 187		/** @map.vm_bo: Buffer object to map. */
 188		struct drm_gpuvm_bo *vm_bo;
 189
 190		/** @map.bo_offset: Offset in the buffer object. */
 191		u64 bo_offset;
 192
 193		/**
 194		 * @map.sgt: sg-table pointing to pages backing the GEM object.
 195		 *
 196		 * This is gathered at job creation time, such that we don't have
 197		 * to allocate in ::run_job().
 198		 */
 199		struct sg_table *sgt;
 200
 201		/**
 202		 * @map.new_vma: The new VMA object that will be inserted to the VA tree.
 203		 */
 204		struct panthor_vma *new_vma;
 205	} map;
 206};
 207
 208/**
 209 * struct panthor_vm - VM object
 210 *
 211 * A VM is an object representing a GPU (or MCU) virtual address space.
 212 * It embeds the MMU page table for this address space, a tree containing
 213 * all the virtual mappings of GEM objects, and other things needed to manage
 214 * the VM.
 215 *
 216 * Except for the MCU VM, which is managed by the kernel, all other VMs are
 217 * created by userspace and mostly managed by userspace, using the
 218 * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.
 219 *
 220 * A portion of the virtual address space is reserved for kernel objects,
 221 * like heap chunks, and userspace gets to decide how much of the virtual
 222 * address space is left to the kernel (half of the virtual address space
 223 * by default).
 224 */
 225struct panthor_vm {
 226	/**
 227	 * @base: Inherit from drm_gpuvm.
 228	 *
 229	 * We delegate all the VA management to the common drm_gpuvm framework
 230	 * and only implement hooks to update the MMU page table.
 231	 */
 232	struct drm_gpuvm base;
 233
 234	/**
 235	 * @sched: Scheduler used for asynchronous VM_BIND request.
 236	 *
 237	 * We use a 1:1 scheduler here.
 238	 */
 239	struct drm_gpu_scheduler sched;
 240
 241	/**
 242	 * @entity: Scheduling entity representing the VM_BIND queue.
 243	 *
 244	 * There's currently one bind queue per VM. It doesn't make sense to
 245	 * allow more given the VM operations are serialized anyway.
 246	 */
 247	struct drm_sched_entity entity;
 248
 249	/** @ptdev: Device. */
 250	struct panthor_device *ptdev;
 251
 252	/** @memattr: Value to program to the AS_MEMATTR register. */
 253	u64 memattr;
 254
 255	/** @pgtbl_ops: Page table operations. */
 256	struct io_pgtable_ops *pgtbl_ops;
 257
 258	/** @root_page_table: Stores the root page table pointer. */
 259	void *root_page_table;
 260
 261	/**
 262	 * @op_lock: Lock used to serialize operations on a VM.
 263	 *
 264	 * The serialization of jobs queued to the VM_BIND queue is already
 265	 * taken care of by drm_sched, but we need to serialize synchronous
 266	 * and asynchronous VM_BIND request. This is what this lock is for.
 267	 */
 268	struct mutex op_lock;
 269
 270	/**
 271	 * @op_ctx: The context attached to the currently executing VM operation.
 272	 *
 273	 * NULL when no operation is in progress.
 274	 */
 275	struct panthor_vm_op_ctx *op_ctx;
 276
 277	/**
 278	 * @mm: Memory management object representing the auto-VA/kernel-VA.
 279	 *
 280	 * Used to auto-allocate VA space for kernel-managed objects (tiler
 281	 * heaps, ...).
 282	 *
 283	 * For the MCU VM, this is managing the VA range that's used to map
 284	 * all shared interfaces.
 285	 *
 286	 * For user VMs, the range is specified by userspace, and must not
 287	 * exceed half of the VA space addressable.
 288	 */
 289	struct drm_mm mm;
 290
 291	/** @mm_lock: Lock protecting the @mm field. */
 292	struct mutex mm_lock;
 293
 294	/** @kernel_auto_va: Automatic VA-range for kernel BOs. */
 295	struct {
 296		/** @kernel_auto_va.start: Start of the automatic VA-range for kernel BOs. */
 297		u64 start;
 298
 299		/** @kernel_auto_va.size: Size of the automatic VA-range for kernel BOs. */
 300		u64 end;
 301	} kernel_auto_va;
 302
 303	/** @as: Address space related fields. */
 304	struct {
 305		/**
 306		 * @as.id: ID of the address space this VM is bound to.
 307		 *
 308		 * A value of -1 means the VM is inactive/not bound.
 309		 */
 310		int id;
 311
 312		/** @as.active_cnt: Number of active users of this VM. */
 313		refcount_t active_cnt;
 314
 315		/**
 316		 * @as.lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.
 317		 *
 318		 * Active VMs should not be inserted in the LRU list.
 319		 */
 320		struct list_head lru_node;
 321	} as;
 322
 323	/**
 324	 * @heaps: Tiler heap related fields.
 325	 */
 326	struct {
 327		/**
 328		 * @heaps.pool: The heap pool attached to this VM.
 329		 *
 330		 * Will stay NULL until someone creates a heap context on this VM.
 331		 */
 332		struct panthor_heap_pool *pool;
 333
 334		/** @heaps.lock: Lock used to protect access to @pool. */
 335		struct mutex lock;
 336	} heaps;
 337
 338	/** @node: Used to insert the VM in the panthor_mmu::vm::list. */
 339	struct list_head node;
 340
 341	/** @for_mcu: True if this is the MCU VM. */
 342	bool for_mcu;
 343
 344	/**
 345	 * @destroyed: True if the VM was destroyed.
 346	 *
 347	 * No further bind requests should be queued to a destroyed VM.
 348	 */
 349	bool destroyed;
 350
 351	/**
 352	 * @unusable: True if the VM has turned unusable because something
 353	 * bad happened during an asynchronous request.
 354	 *
 355	 * We don't try to recover from such failures, because this implies
 356	 * informing userspace about the specific operation that failed, and
 357	 * hoping the userspace driver can replay things from there. This all
 358	 * sounds very complicated for little gain.
 359	 *
 360	 * Instead, we should just flag the VM as unusable, and fail any
 361	 * further request targeting this VM.
 362	 *
 363	 * We also provide a way to query a VM state, so userspace can destroy
 364	 * it and create a new one.
 365	 *
 366	 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST
 367	 * situation, where the logical device needs to be re-created.
 368	 */
 369	bool unusable;
 370
 371	/**
 372	 * @unhandled_fault: Unhandled fault happened.
 373	 *
 374	 * This should be reported to the scheduler, and the queue/group be
 375	 * flagged as faulty as a result.
 376	 */
 377	bool unhandled_fault;
 378};
 379
 380/**
 381 * struct panthor_vm_bind_job - VM bind job
 382 */
 383struct panthor_vm_bind_job {
 384	/** @base: Inherit from drm_sched_job. */
 385	struct drm_sched_job base;
 386
 387	/** @refcount: Reference count. */
 388	struct kref refcount;
 389
 390	/** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */
 391	struct work_struct cleanup_op_ctx_work;
 392
 393	/** @vm: VM targeted by the VM operation. */
 394	struct panthor_vm *vm;
 395
 396	/** @ctx: Operation context. */
 397	struct panthor_vm_op_ctx ctx;
 398};
 399
 400/*
 401 * @pt_cache: Cache used to allocate MMU page tables.
 402 *
 403 * The pre-allocation pattern forces us to over-allocate to plan for
 404 * the worst case scenario, and return the pages we didn't use.
 405 *
 406 * Having a kmem_cache allows us to speed allocations.
 407 */
 408static struct kmem_cache *pt_cache;
 409
 410/**
 411 * alloc_pt() - Custom page table allocator
 412 * @cookie: Cookie passed at page table allocation time.
 413 * @size: Size of the page table. This size should be fixed,
 414 * and determined at creation time based on the granule size.
 415 * @gfp: GFP flags.
 416 *
 417 * We want a custom allocator so we can use a cache for page table
 418 * allocations and amortize the cost of the over-reservation that's
 419 * done to allow asynchronous VM operations.
 420 *
 421 * Return: non-NULL on success, NULL if the allocation failed for any
 422 * reason.
 423 */
 424static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)
 425{
 426	struct panthor_vm *vm = cookie;
 427	void *page;
 428
 429	/* Allocation of the root page table happening during init. */
 430	if (unlikely(!vm->root_page_table)) {
 431		struct page *p;
 432
 433		drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
 434		p = alloc_pages_node(dev_to_node(vm->ptdev->base.dev),
 435				     gfp | __GFP_ZERO, get_order(size));
 436		page = p ? page_address(p) : NULL;
 437		vm->root_page_table = page;
 438		return page;
 439	}
 440
 441	/* We're not supposed to have anything bigger than 4k here, because we picked a
 442	 * 4k granule size at init time.
 443	 */
 444	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
 445		return NULL;
 446
 447	/* We must have some op_ctx attached to the VM and it must have at least one
 448	 * free page.
 449	 */
 450	if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
 451	    drm_WARN_ON(&vm->ptdev->base,
 452			vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
 453		return NULL;
 454
 455	page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
 456	memset(page, 0, SZ_4K);
 457
 458	/* Page table entries don't use virtual addresses, which trips out
 459	 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
 460	 * are mixed with other fields, and I fear kmemleak won't detect that
 461	 * either.
 462	 *
 463	 * Let's just ignore memory passed to the page-table driver for now.
 464	 */
 465	kmemleak_ignore(page);
 466	return page;
 467}
 468
 469/**
 470 * free_pt() - Custom page table free function
 471 * @cookie: Cookie passed at page table allocation time.
 472 * @data: Page table to free.
 473 * @size: Size of the page table. This size should be fixed,
 474 * and determined at creation time based on the granule size.
 475 */
 476static void free_pt(void *cookie, void *data, size_t size)
 477{
 478	struct panthor_vm *vm = cookie;
 479
 480	if (unlikely(vm->root_page_table == data)) {
 481		free_pages((unsigned long)data, get_order(size));
 482		vm->root_page_table = NULL;
 483		return;
 484	}
 485
 486	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
 487		return;
 488
 489	/* Return the page to the pt_cache. */
 490	kmem_cache_free(pt_cache, data);
 491}
 492
 493static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
 494{
 495	int ret;
 496	u32 val;
 497
 498	/* Wait for the MMU status to indicate there is no active command, in
 499	 * case one is pending.
 500	 */
 501	ret = gpu_read_relaxed_poll_timeout_atomic(ptdev, AS_STATUS(as_nr), val,
 502						   !(val & AS_STATUS_AS_ACTIVE),
 503						   10, 100000);
 504
 505	if (ret) {
 506		panthor_device_schedule_reset(ptdev);
 507		drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");
 508	}
 509
 510	return ret;
 511}
 512
 513static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
 514{
 515	int status;
 516
 517	/* write AS_COMMAND when MMU is ready to accept another command */
 518	status = wait_ready(ptdev, as_nr);
 519	if (!status) {
 520		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
 521		status = wait_ready(ptdev, as_nr);
 522	}
 523
 524	return status;
 525}
 526
 527static int lock_region(struct panthor_device *ptdev, u32 as_nr,
 528		       u64 region_start, u64 size)
 529{
 530	u8 region_width;
 531	u64 region;
 532	u64 region_end = region_start + size;
 533
 534	if (!size)
 535		return 0;
 536
 537	/*
 538	 * The locked region is a naturally aligned power of 2 block encoded as
 539	 * log2 minus(1).
 540	 * Calculate the desired start/end and look for the highest bit which
 541	 * differs. The smallest naturally aligned block must include this bit
 542	 * change, the desired region starts with this bit (and subsequent bits)
 543	 * zeroed and ends with the bit (and subsequent bits) set to one.
 544	 */
 545	region_width = max(fls64(region_start ^ (region_end - 1)),
 546			   const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;
 547
 548	/*
 549	 * Mask off the low bits of region_start (which would be ignored by
 550	 * the hardware anyway)
 551	 */
 552	region_start &= GENMASK_ULL(63, region_width);
 553
 554	region = region_width | region_start;
 555
 556	/* Lock the region that needs to be updated */
 557	gpu_write64(ptdev, AS_LOCKADDR(as_nr), region);
 558	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_LOCK);
 559}
 560
 561static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
 562				      u64 iova, u64 size, u32 op)
 563{
 564	const u32 l2_flush_op = CACHE_CLEAN | CACHE_INV;
 565	u32 lsc_flush_op;
 566	int ret;
 567
 568	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
 569
 570	switch (op) {
 571	case AS_COMMAND_FLUSH_MEM:
 572		lsc_flush_op = CACHE_CLEAN | CACHE_INV;
 573		break;
 574	case AS_COMMAND_FLUSH_PT:
 575		lsc_flush_op = 0;
 576		break;
 577	default:
 578		drm_WARN(&ptdev->base, 1, "Unexpected AS_COMMAND: %d", op);
 579		return -EINVAL;
 580	}
 581
 582	if (as_nr < 0)
 583		return 0;
 584
 585	/*
 586	 * If the AS number is greater than zero, then we can be sure
 587	 * the device is up and running, so we don't need to explicitly
 588	 * power it up
 589	 */
 590
 591	ret = lock_region(ptdev, as_nr, iova, size);
 592	if (ret)
 593		return ret;
 594
 595	ret = panthor_gpu_flush_caches(ptdev, l2_flush_op, lsc_flush_op, 0);
 596	if (ret)
 597		return ret;
 598
 599	/*
 600	 * Explicitly unlock the region as the AS is not unlocked automatically
 601	 * at the end of the GPU_CONTROL cache flush command, unlike
 602	 * AS_COMMAND_FLUSH_MEM or AS_COMMAND_FLUSH_PT.
 603	 */
 604	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UNLOCK);
 605}
 606
 607static int mmu_hw_do_operation(struct panthor_vm *vm,
 608			       u64 iova, u64 size, u32 op)
 609{
 610	struct panthor_device *ptdev = vm->ptdev;
 611	int ret;
 612
 613	mutex_lock(&ptdev->mmu->as.slots_lock);
 614	ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
 615	mutex_unlock(&ptdev->mmu->as.slots_lock);
 616
 617	return ret;
 618}
 619
 620static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
 621				 u64 transtab, u64 transcfg, u64 memattr)
 622{
 623	int ret;
 624
 625	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
 626	if (ret)
 627		return ret;
 628
 629	gpu_write64(ptdev, AS_TRANSTAB(as_nr), transtab);
 630	gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr);
 631	gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg);
 632
 633	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE);
 634}
 635
 636static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
 637{
 638	int ret;
 639
 640	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
 641	if (ret)
 642		return ret;
 643
 644	gpu_write64(ptdev, AS_TRANSTAB(as_nr), 0);
 645	gpu_write64(ptdev, AS_MEMATTR(as_nr), 0);
 646	gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
 647
 648	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE);
 649}
 650
 651static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
 652{
 653	/* Bits 16 to 31 mean REQ_COMPLETE. */
 654	return value & GENMASK(15, 0);
 655}
 656
 657static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
 658{
 659	return BIT(as);
 660}
 661
 662/**
 663 * panthor_vm_has_unhandled_faults() - Check if a VM has unhandled faults
 664 * @vm: VM to check.
 665 *
 666 * Return: true if the VM has unhandled faults, false otherwise.
 667 */
 668bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm)
 669{
 670	return vm->unhandled_fault;
 671}
 672
 673/**
 674 * panthor_vm_is_unusable() - Check if the VM is still usable
 675 * @vm: VM to check.
 676 *
 677 * Return: true if the VM is unusable, false otherwise.
 678 */
 679bool panthor_vm_is_unusable(struct panthor_vm *vm)
 680{
 681	return vm->unusable;
 682}
 683
 684static void panthor_vm_release_as_locked(struct panthor_vm *vm)
 685{
 686	struct panthor_device *ptdev = vm->ptdev;
 687
 688	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
 689
 690	if (drm_WARN_ON(&ptdev->base, vm->as.id < 0))
 691		return;
 692
 693	ptdev->mmu->as.slots[vm->as.id].vm = NULL;
 694	clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
 695	refcount_set(&vm->as.active_cnt, 0);
 696	list_del_init(&vm->as.lru_node);
 697	vm->as.id = -1;
 698}
 699
 700/**
 701 * panthor_vm_active() - Flag a VM as active
 702 * @vm: VM to flag as active.
 703 *
 704 * Assigns an address space to a VM so it can be used by the GPU/MCU.
 705 *
 706 * Return: 0 on success, a negative error code otherwise.
 707 */
 708int panthor_vm_active(struct panthor_vm *vm)
 709{
 710	struct panthor_device *ptdev = vm->ptdev;
 711	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
 712	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
 713	int ret = 0, as, cookie;
 714	u64 transtab, transcfg;
 715
 716	if (!drm_dev_enter(&ptdev->base, &cookie))
 717		return -ENODEV;
 718
 719	if (refcount_inc_not_zero(&vm->as.active_cnt))
 720		goto out_dev_exit;
 721
 722	mutex_lock(&ptdev->mmu->as.slots_lock);
 723
 724	if (refcount_inc_not_zero(&vm->as.active_cnt))
 725		goto out_unlock;
 726
 727	as = vm->as.id;
 728	if (as >= 0) {
 729		/* Unhandled pagefault on this AS, the MMU was disabled. We need to
 730		 * re-enable the MMU after clearing+unmasking the AS interrupts.
 731		 */
 732		if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as))
 733			goto out_enable_as;
 734
 735		goto out_make_active;
 736	}
 737
 738	/* Check for a free AS */
 739	if (vm->for_mcu) {
 740		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
 741		as = 0;
 742	} else {
 743		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
 744	}
 745
 746	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
 747		struct panthor_vm *lru_vm;
 748
 749		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
 750						  struct panthor_vm,
 751						  as.lru_node);
 752		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
 753			ret = -EBUSY;
 754			goto out_unlock;
 755		}
 756
 757		drm_WARN_ON(&ptdev->base, refcount_read(&lru_vm->as.active_cnt));
 758		as = lru_vm->as.id;
 759		panthor_vm_release_as_locked(lru_vm);
 760	}
 761
 762	/* Assign the free or reclaimed AS to the FD */
 763	vm->as.id = as;
 764	set_bit(as, &ptdev->mmu->as.alloc_mask);
 765	ptdev->mmu->as.slots[as].vm = vm;
 766
 767out_enable_as:
 768	transtab = cfg->arm_lpae_s1_cfg.ttbr;
 769	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
 770		   AS_TRANSCFG_PTW_RA |
 771		   AS_TRANSCFG_ADRMODE_AARCH64_4K |
 772		   AS_TRANSCFG_INA_BITS(55 - va_bits);
 773	if (ptdev->coherent)
 774		transcfg |= AS_TRANSCFG_PTW_SH_OS;
 775
 776	/* If the VM is re-activated, we clear the fault. */
 777	vm->unhandled_fault = false;
 778
 779	/* Unhandled pagefault on this AS, clear the fault and re-enable interrupts
 780	 * before enabling the AS.
 781	 */
 782	if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) {
 783		gpu_write(ptdev, MMU_INT_CLEAR, panthor_mmu_as_fault_mask(ptdev, as));
 784		ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as);
 785		ptdev->mmu->irq.mask |= panthor_mmu_as_fault_mask(ptdev, as);
 786		gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
 787	}
 788
 789	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
 790
 791out_make_active:
 792	if (!ret) {
 793		refcount_set(&vm->as.active_cnt, 1);
 794		list_del_init(&vm->as.lru_node);
 795	}
 796
 797out_unlock:
 798	mutex_unlock(&ptdev->mmu->as.slots_lock);
 799
 800out_dev_exit:
 801	drm_dev_exit(cookie);
 802	return ret;
 803}
 804
 805/**
 806 * panthor_vm_idle() - Flag a VM idle
 807 * @vm: VM to flag as idle.
 808 *
 809 * When we know the GPU is done with the VM (no more jobs to process),
 810 * we can relinquish the AS slot attached to this VM, if any.
 811 *
 812 * We don't release the slot immediately, but instead place the VM in
 813 * the LRU list, so it can be evicted if another VM needs an AS slot.
 814 * This way, VMs keep attached to the AS they were given until we run
 815 * out of free slot, limiting the number of MMU operations (TLB flush
 816 * and other AS updates).
 817 */
 818void panthor_vm_idle(struct panthor_vm *vm)
 819{
 820	struct panthor_device *ptdev = vm->ptdev;
 821
 822	if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock))
 823		return;
 824
 825	if (!drm_WARN_ON(&ptdev->base, vm->as.id == -1 || !list_empty(&vm->as.lru_node)))
 826		list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
 827
 828	refcount_set(&vm->as.active_cnt, 0);
 829	mutex_unlock(&ptdev->mmu->as.slots_lock);
 830}
 831
 832u32 panthor_vm_page_size(struct panthor_vm *vm)
 833{
 834	const struct io_pgtable *pgt = io_pgtable_ops_to_pgtable(vm->pgtbl_ops);
 835	u32 pg_shift = ffs(pgt->cfg.pgsize_bitmap) - 1;
 836
 837	return 1u << pg_shift;
 838}
 839
 840static void panthor_vm_stop(struct panthor_vm *vm)
 841{
 842	drm_sched_stop(&vm->sched, NULL);
 843}
 844
 845static void panthor_vm_start(struct panthor_vm *vm)
 846{
 847	drm_sched_start(&vm->sched, 0);
 848}
 849
 850/**
 851 * panthor_vm_as() - Get the AS slot attached to a VM
 852 * @vm: VM to get the AS slot of.
 853 *
 854 * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.
 855 */
 856int panthor_vm_as(struct panthor_vm *vm)
 857{
 858	return vm->as.id;
 859}
 860
 861static size_t get_pgsize(u64 addr, size_t size, size_t *count)
 862{
 863	/*
 864	 * io-pgtable only operates on multiple pages within a single table
 865	 * entry, so we need to split at boundaries of the table size, i.e.
 866	 * the next block size up. The distance from address A to the next
 867	 * boundary of block size B is logically B - A % B, but in unsigned
 868	 * two's complement where B is a power of two we get the equivalence
 869	 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)
 870	 */
 871	size_t blk_offset = -addr % SZ_2M;
 872
 873	if (blk_offset || size < SZ_2M) {
 874		*count = min_not_zero(blk_offset, size) / SZ_4K;
 875		return SZ_4K;
 876	}
 877	blk_offset = -addr % SZ_1G ?: SZ_1G;
 878	*count = min(blk_offset, size) / SZ_2M;
 879	return SZ_2M;
 880}
 881
 882static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
 883{
 884	struct panthor_device *ptdev = vm->ptdev;
 885	int ret = 0, cookie;
 886
 887	if (vm->as.id < 0)
 888		return 0;
 889
 890	/* If the device is unplugged, we just silently skip the flush. */
 891	if (!drm_dev_enter(&ptdev->base, &cookie))
 892		return 0;
 893
 894	ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
 895
 896	drm_dev_exit(cookie);
 897	return ret;
 898}
 899
 900static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
 901{
 902	struct panthor_device *ptdev = vm->ptdev;
 903	struct io_pgtable_ops *ops = vm->pgtbl_ops;
 904	u64 start_iova = iova;
 905	u64 offset = 0;
 906
 907	while (offset < size) {
 908		size_t unmapped_sz = 0, pgcount;
 909		size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
 910
 911		unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
 912
 913		if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
 914			drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
 915				iova + offset + unmapped_sz,
 916				iova + offset + pgsize * pgcount,
 917				iova, iova + size);
 918			panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
 919			return  -EINVAL;
 920		}
 921
 922		drm_dbg(&ptdev->base,
 923			"unmap: as=%d, iova=0x%llx, sz=%llu, va=0x%llx, pgcnt=%zu, pgsz=%zu",
 924			vm->as.id, start_iova, size, iova + offset,
 925			unmapped_sz / pgsize, pgsize);
 926
 927		offset += unmapped_sz;
 928	}
 929
 930	return panthor_vm_flush_range(vm, iova, size);
 931}
 932
 933static int
 934panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
 935		     struct sg_table *sgt, u64 offset, u64 size)
 936{
 937	struct panthor_device *ptdev = vm->ptdev;
 938	unsigned int count;
 939	struct scatterlist *sgl;
 940	struct io_pgtable_ops *ops = vm->pgtbl_ops;
 941	u64 start_iova = iova;
 942	u64 start_size = size;
 943	int ret;
 944
 945	if (!size)
 946		return 0;
 947
 948	for_each_sgtable_dma_sg(sgt, sgl, count) {
 949		dma_addr_t paddr = sg_dma_address(sgl);
 950		size_t len = sg_dma_len(sgl);
 951
 952		if (len <= offset) {
 953			offset -= len;
 954			continue;
 955		}
 956
 957		paddr += offset;
 958		len -= offset;
 959		len = min_t(size_t, len, size);
 960		size -= len;
 961
 962		while (len) {
 963			size_t pgcount, mapped = 0;
 964			size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
 965
 966			ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
 967					     GFP_KERNEL, &mapped);
 968
 969			drm_dbg(&ptdev->base,
 970				"map: as=%d, iova=0x%llx, sz=%llu, va=0x%llx, pa=%pad, pgcnt=%zu, pgsz=%zu",
 971				vm->as.id, start_iova, start_size, iova, &paddr,
 972				mapped / pgsize, pgsize);
 973
 974			iova += mapped;
 975			paddr += mapped;
 976			len -= mapped;
 977
 978			if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
 979				ret = -ENOMEM;
 980
 981			if (ret) {
 982				/* If something failed, unmap what we've already mapped before
 983				 * returning. The unmap call is not supposed to fail.
 984				 */
 985				drm_WARN_ON(&ptdev->base,
 986					    panthor_vm_unmap_pages(vm, start_iova,
 987								   iova - start_iova));
 988				return ret;
 989			}
 990		}
 991
 992		if (!size)
 993			break;
 994
 995		offset = 0;
 996	}
 997
 998	return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
 999}
1000
1001static int flags_to_prot(u32 flags)
1002{
1003	int prot = 0;
1004
1005	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)
1006		prot |= IOMMU_NOEXEC;
1007
1008	if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))
1009		prot |= IOMMU_CACHE;
1010
1011	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)
1012		prot |= IOMMU_READ;
1013	else
1014		prot |= IOMMU_READ | IOMMU_WRITE;
1015
1016	return prot;
1017}
1018
1019/**
1020 * panthor_vm_alloc_va() - Allocate a region in the auto-va space
1021 * @vm: VM to allocate a region on.
1022 * @va: start of the VA range. Can be PANTHOR_VM_KERNEL_AUTO_VA if the user
1023 * wants the VA to be automatically allocated from the auto-VA range.
1024 * @size: size of the VA range.
1025 * @va_node: drm_mm_node to initialize. Must be zero-initialized.
1026 *
1027 * Some GPU objects, like heap chunks, are fully managed by the kernel and
1028 * need to be mapped to the userspace VM, in the region reserved for kernel
1029 * objects.
1030 *
1031 * This function takes care of allocating a region in the kernel auto-VA space.
1032 *
1033 * Return: 0 on success, an error code otherwise.
1034 */
1035int
1036panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size,
1037		    struct drm_mm_node *va_node)
1038{
1039	ssize_t vm_pgsz = panthor_vm_page_size(vm);
1040	int ret;
1041
1042	if (!size || !IS_ALIGNED(size, vm_pgsz))
1043		return -EINVAL;
1044
1045	if (va != PANTHOR_VM_KERNEL_AUTO_VA && !IS_ALIGNED(va, vm_pgsz))
1046		return -EINVAL;
1047
1048	mutex_lock(&vm->mm_lock);
1049	if (va != PANTHOR_VM_KERNEL_AUTO_VA) {
1050		va_node->start = va;
1051		va_node->size = size;
1052		ret = drm_mm_reserve_node(&vm->mm, va_node);
1053	} else {
1054		ret = drm_mm_insert_node_in_range(&vm->mm, va_node, size,
1055						  size >= SZ_2M ? SZ_2M : SZ_4K,
1056						  0, vm->kernel_auto_va.start,
1057						  vm->kernel_auto_va.end,
1058						  DRM_MM_INSERT_BEST);
1059	}
1060	mutex_unlock(&vm->mm_lock);
1061
1062	return ret;
1063}
1064
1065/**
1066 * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()
1067 * @vm: VM to free the region on.
1068 * @va_node: Memory node representing the region to free.
1069 */
1070void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node)
1071{
1072	mutex_lock(&vm->mm_lock);
1073	drm_mm_remove_node(va_node);
1074	mutex_unlock(&vm->mm_lock);
1075}
1076
1077static void panthor_vm_bo_free(struct drm_gpuvm_bo *vm_bo)
1078{
1079	struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
1080
1081	if (!drm_gem_is_imported(&bo->base.base))
1082		drm_gem_shmem_unpin(&bo->base);
1083	kfree(vm_bo);
1084}
1085
1086static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1087				      struct panthor_vm *vm)
1088{
1089	u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
1090				 op_ctx->rsvd_page_tables.ptr;
1091
1092	if (remaining_pt_count) {
1093		kmem_cache_free_bulk(pt_cache, remaining_pt_count,
1094				     op_ctx->rsvd_page_tables.pages +
1095				     op_ctx->rsvd_page_tables.ptr);
1096	}
1097
1098	kfree(op_ctx->rsvd_page_tables.pages);
1099
1100	if (op_ctx->map.vm_bo)
1101		drm_gpuvm_bo_put_deferred(op_ctx->map.vm_bo);
1102
1103	for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++)
1104		kfree(op_ctx->preallocated_vmas[i]);
1105
1106	drm_gpuvm_bo_deferred_cleanup(&vm->base);
1107}
1108
1109static void
1110panthor_vm_op_ctx_return_vma(struct panthor_vm_op_ctx *op_ctx,
1111			     struct panthor_vma *vma)
1112{
1113	for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) {
1114		if (!op_ctx->preallocated_vmas[i]) {
1115			op_ctx->preallocated_vmas[i] = vma;
1116			return;
1117		}
1118	}
1119
1120	WARN_ON_ONCE(1);
1121}
1122
1123static struct panthor_vma *
1124panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx *op_ctx)
1125{
1126	for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) {
1127		struct panthor_vma *vma = op_ctx->preallocated_vmas[i];
1128
1129		if (vma) {
1130			op_ctx->preallocated_vmas[i] = NULL;
1131			return vma;
1132		}
1133	}
1134
1135	return NULL;
1136}
1137
1138static int
1139panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx *op_ctx)
1140{
1141	u32 vma_count;
1142
1143	switch (op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
1144	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
1145		/* One VMA for the new mapping, and two more VMAs for the remap case
1146		 * which might contain both a prev and next VA.
1147		 */
1148		vma_count = 3;
1149		break;
1150
1151	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
1152		/* Two VMAs can be needed for an unmap, as an unmap can happen
1153		 * in the middle of a drm_gpuva, requiring a remap with both
1154		 * prev & next VA. Or an unmap can span more than one drm_gpuva
1155		 * where the first and last ones are covered partially, requring
1156		 * a remap for the first with a prev VA and remap for the last
1157		 * with a next VA.
1158		 */
1159		vma_count = 2;
1160		break;
1161
1162	default:
1163		return 0;
1164	}
1165
1166	for (u32 i = 0; i < vma_count; i++) {
1167		struct panthor_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
1168
1169		if (!vma)
1170			return -ENOMEM;
1171
1172		op_ctx->preallocated_vmas[i] = vma;
1173	}
1174
1175	return 0;
1176}
1177
1178#define PANTHOR_VM_BIND_OP_MAP_FLAGS \
1179	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
1180	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
1181	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
1182	 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
1183
1184static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1185					 struct panthor_vm *vm,
1186					 struct panthor_gem_object *bo,
1187					 u64 offset,
1188					 u64 size, u64 va,
1189					 u32 flags)
1190{
1191	struct drm_gpuvm_bo *preallocated_vm_bo;
1192	struct sg_table *sgt = NULL;
1193	u64 pt_count;
1194	int ret;
1195
1196	if (!bo)
1197		return -EINVAL;
1198
1199	if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
1200	    (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
1201		return -EINVAL;
1202
1203	/* Make sure the VA and size are in-bounds. */
1204	if (size > bo->base.base.size || offset > bo->base.base.size - size)
1205		return -EINVAL;
1206
1207	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
1208	if (bo->exclusive_vm_root_gem &&
1209	    bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))
1210		return -EINVAL;
1211
1212	memset(op_ctx, 0, sizeof(*op_ctx));
1213	op_ctx->flags = flags;
1214	op_ctx->va.range = size;
1215	op_ctx->va.addr = va;
1216
1217	ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
1218	if (ret)
1219		goto err_cleanup;
1220
1221	if (!drm_gem_is_imported(&bo->base.base)) {
1222		/* Pre-reserve the BO pages, so the map operation doesn't have to
1223		 * allocate. This pin is dropped in panthor_vm_bo_free(), so
1224		 * once we have successfully called drm_gpuvm_bo_create(),
1225		 * GPUVM will take care of dropping the pin for us.
1226		 */
1227		ret = drm_gem_shmem_pin(&bo->base);
1228		if (ret)
1229			goto err_cleanup;
1230	}
1231
1232	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
1233	if (IS_ERR(sgt)) {
1234		if (!drm_gem_is_imported(&bo->base.base))
1235			drm_gem_shmem_unpin(&bo->base);
1236
1237		ret = PTR_ERR(sgt);
1238		goto err_cleanup;
1239	}
1240
1241	op_ctx->map.sgt = sgt;
1242
1243	preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base);
1244	if (!preallocated_vm_bo) {
1245		if (!drm_gem_is_imported(&bo->base.base))
1246			drm_gem_shmem_unpin(&bo->base);
1247
1248		ret = -ENOMEM;
1249		goto err_cleanup;
1250	}
1251
1252	op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo);
1253
1254	op_ctx->map.bo_offset = offset;
1255
1256	/* L1, L2 and L3 page tables.
1257	 * We could optimize L3 allocation by iterating over the sgt and merging
1258	 * 2M contiguous blocks, but it's simpler to over-provision and return
1259	 * the pages if they're not used.
1260	 */
1261	pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
1262		   ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
1263		   ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
1264
1265	op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
1266						 sizeof(*op_ctx->rsvd_page_tables.pages),
1267						 GFP_KERNEL);
1268	if (!op_ctx->rsvd_page_tables.pages) {
1269		ret = -ENOMEM;
1270		goto err_cleanup;
1271	}
1272
1273	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
1274				    op_ctx->rsvd_page_tables.pages);
1275	op_ctx->rsvd_page_tables.count = ret;
1276	if (ret != pt_count) {
1277		ret = -ENOMEM;
1278		goto err_cleanup;
1279	}
1280
1281	/* Insert BO into the extobj list last, when we know nothing can fail. */
1282	dma_resv_lock(panthor_vm_resv(vm), NULL);
1283	drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo);
1284	dma_resv_unlock(panthor_vm_resv(vm));
1285
1286	return 0;
1287
1288err_cleanup:
1289	panthor_vm_cleanup_op_ctx(op_ctx, vm);
1290	return ret;
1291}
1292
1293static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1294					   struct panthor_vm *vm,
1295					   u64 va, u64 size)
1296{
1297	u32 pt_count = 0;
1298	int ret;
1299
1300	memset(op_ctx, 0, sizeof(*op_ctx));
1301	op_ctx->va.range = size;
1302	op_ctx->va.addr = va;
1303	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;
1304
1305	/* Pre-allocate L3 page tables to account for the split-2M-block
1306	 * situation on unmap.
1307	 */
1308	if (va != ALIGN(va, SZ_2M))
1309		pt_count++;
1310
1311	if (va + size != ALIGN(va + size, SZ_2M) &&
1312	    ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))
1313		pt_count++;
1314
1315	ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
1316	if (ret)
1317		goto err_cleanup;
1318
1319	if (pt_count) {
1320		op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
1321							 sizeof(*op_ctx->rsvd_page_tables.pages),
1322							 GFP_KERNEL);
1323		if (!op_ctx->rsvd_page_tables.pages) {
1324			ret = -ENOMEM;
1325			goto err_cleanup;
1326		}
1327
1328		ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
1329					    op_ctx->rsvd_page_tables.pages);
1330		if (ret != pt_count) {
1331			ret = -ENOMEM;
1332			goto err_cleanup;
1333		}
1334		op_ctx->rsvd_page_tables.count = pt_count;
1335	}
1336
1337	return 0;
1338
1339err_cleanup:
1340	panthor_vm_cleanup_op_ctx(op_ctx, vm);
1341	return ret;
1342}
1343
1344static void panthor_vm_prepare_sync_only_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1345						struct panthor_vm *vm)
1346{
1347	memset(op_ctx, 0, sizeof(*op_ctx));
1348	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY;
1349}
1350
1351/**
1352 * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address
1353 * @vm: VM to look into.
1354 * @va: Virtual address to search for.
1355 * @bo_offset: Offset of the GEM object mapped at this virtual address.
1356 * Only valid on success.
1357 *
1358 * The object returned by this function might no longer be mapped when the
1359 * function returns. It's the caller responsibility to ensure there's no
1360 * concurrent map/unmap operations making the returned value invalid, or
1361 * make sure it doesn't matter if the object is no longer mapped.
1362 *
1363 * Return: A valid pointer on success, an ERR_PTR() otherwise.
1364 */
1365struct panthor_gem_object *
1366panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
1367{
1368	struct panthor_gem_object *bo = ERR_PTR(-ENOENT);
1369	struct drm_gpuva *gpuva;
1370	struct panthor_vma *vma;
1371
1372	/* Take the VM lock to prevent concurrent map/unmap operations. */
1373	mutex_lock(&vm->op_lock);
1374	gpuva = drm_gpuva_find_first(&vm->base, va, 1);
1375	vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;
1376	if (vma && vma->base.gem.obj) {
1377		drm_gem_object_get(vma->base.gem.obj);
1378		bo = to_panthor_bo(vma->base.gem.obj);
1379		*bo_offset = vma->base.gem.offset + (va - vma->base.va.addr);
1380	}
1381	mutex_unlock(&vm->op_lock);
1382
1383	return bo;
1384}
1385
1386#define PANTHOR_VM_MIN_KERNEL_VA_SIZE	SZ_256M
1387
1388static u64
1389panthor_vm_create_get_user_va_range(const struct drm_panthor_vm_create *args,
1390				    u64 full_va_range)
1391{
1392	u64 user_va_range;
1393
1394	/* Make sure we have a minimum amount of VA space for kernel objects. */
1395	if (full_va_range < PANTHOR_VM_MIN_KERNEL_VA_SIZE)
1396		return 0;
1397
1398	if (args->user_va_range) {
1399		/* Use the user provided value if != 0. */
1400		user_va_range = args->user_va_range;
1401	} else if (TASK_SIZE_OF(current) < full_va_range) {
1402		/* If the task VM size is smaller than the GPU VA range, pick this
1403		 * as our default user VA range, so userspace can CPU/GPU map buffers
1404		 * at the same address.
1405		 */
1406		user_va_range = TASK_SIZE_OF(current);
1407	} else {
1408		/* If the GPU VA range is smaller than the task VM size, we
1409		 * just have to live with the fact we won't be able to map
1410		 * all buffers at the same GPU/CPU address.
1411		 *
1412		 * If the GPU VA range is bigger than 4G (more than 32-bit of
1413		 * VA), we split the range in two, and assign half of it to
1414		 * the user and the other half to the kernel, if it's not, we
1415		 * keep the kernel VA space as small as possible.
1416		 */
1417		user_va_range = full_va_range > SZ_4G ?
1418				full_va_range / 2 :
1419				full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;
1420	}
1421
1422	if (full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE < user_va_range)
1423		user_va_range = full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;
1424
1425	return user_va_range;
1426}
1427
1428#define PANTHOR_VM_CREATE_FLAGS		0
1429
1430static int
1431panthor_vm_create_check_args(const struct panthor_device *ptdev,
1432			     const struct drm_panthor_vm_create *args,
1433			     u64 *kernel_va_start, u64 *kernel_va_range)
1434{
1435	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
1436	u64 full_va_range = 1ull << va_bits;
1437	u64 user_va_range;
1438
1439	if (args->flags & ~PANTHOR_VM_CREATE_FLAGS)
1440		return -EINVAL;
1441
1442	user_va_range = panthor_vm_create_get_user_va_range(args, full_va_range);
1443	if (!user_va_range || (args->user_va_range && args->user_va_range > user_va_range))
1444		return -EINVAL;
1445
1446	/* Pick a kernel VA range that's a power of two, to have a clear split. */
1447	*kernel_va_range = rounddown_pow_of_two(full_va_range - user_va_range);
1448	*kernel_va_start = full_va_range - *kernel_va_range;
1449	return 0;
1450}
1451
1452/*
1453 * Only 32 VMs per open file. If that becomes a limiting factor, we can
1454 * increase this number.
1455 */
1456#define PANTHOR_MAX_VMS_PER_FILE	32
1457
1458/**
1459 * panthor_vm_pool_create_vm() - Create a VM
1460 * @ptdev: The panthor device
1461 * @pool: The VM to create this VM on.
1462 * @args: VM creation args.
1463 *
1464 * Return: a positive VM ID on success, a negative error code otherwise.
1465 */
1466int panthor_vm_pool_create_vm(struct panthor_device *ptdev,
1467			      struct panthor_vm_pool *pool,
1468			      struct drm_panthor_vm_create *args)
1469{
1470	u64 kernel_va_start, kernel_va_range;
1471	struct panthor_vm *vm;
1472	int ret;
1473	u32 id;
1474
1475	ret = panthor_vm_create_check_args(ptdev, args, &kernel_va_start, &kernel_va_range);
1476	if (ret)
1477		return ret;
1478
1479	vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range,
1480			       kernel_va_start, kernel_va_range);
1481	if (IS_ERR(vm))
1482		return PTR_ERR(vm);
1483
1484	ret = xa_alloc(&pool->xa, &id, vm,
1485		       XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
1486
1487	if (ret) {
1488		panthor_vm_put(vm);
1489		return ret;
1490	}
1491
1492	args->user_va_range = kernel_va_start;
1493	return id;
1494}
1495
1496static void panthor_vm_destroy(struct panthor_vm *vm)
1497{
1498	if (!vm)
1499		return;
1500
1501	vm->destroyed = true;
1502
1503	/* Tell scheduler to stop all GPU work related to this VM */
1504	if (refcount_read(&vm->as.active_cnt) > 0)
1505		panthor_sched_prepare_for_vm_destruction(vm->ptdev);
1506
1507	mutex_lock(&vm->heaps.lock);
1508	panthor_heap_pool_destroy(vm->heaps.pool);
1509	vm->heaps.pool = NULL;
1510	mutex_unlock(&vm->heaps.lock);
1511
1512	drm_WARN_ON(&vm->ptdev->base,
1513		    panthor_vm_unmap_range(vm, vm->base.mm_start, vm->base.mm_range));
1514	panthor_vm_put(vm);
1515}
1516
1517/**
1518 * panthor_vm_pool_destroy_vm() - Destroy a VM.
1519 * @pool: VM pool.
1520 * @handle: VM handle.
1521 *
1522 * This function doesn't free the VM object or its resources, it just kills
1523 * all mappings, and makes sure nothing can be mapped after that point.
1524 *
1525 * If there was any active jobs at the time this function is called, these
1526 * jobs should experience page faults and be killed as a result.
1527 *
1528 * The VM resources are freed when the last reference on the VM object is
1529 * dropped.
1530 *
1531 * Return: %0 for success, negative errno value for failure
1532 */
1533int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)
1534{
1535	struct panthor_vm *vm;
1536
1537	vm = xa_erase(&pool->xa, handle);
1538
1539	panthor_vm_destroy(vm);
1540
1541	return vm ? 0 : -EINVAL;
1542}
1543
1544/**
1545 * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle
1546 * @pool: VM pool to check.
1547 * @handle: Handle of the VM to retrieve.
1548 *
1549 * Return: A valid pointer if the VM exists, NULL otherwise.
1550 */
1551struct panthor_vm *
1552panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)
1553{
1554	struct panthor_vm *vm;
1555
1556	xa_lock(&pool->xa);
1557	vm = panthor_vm_get(xa_load(&pool->xa, handle));
1558	xa_unlock(&pool->xa);
1559
1560	return vm;
1561}
1562
1563/**
1564 * panthor_vm_pool_destroy() - Destroy a VM pool.
1565 * @pfile: File.
1566 *
1567 * Destroy all VMs in the pool, and release the pool resources.
1568 *
1569 * Note that VMs can outlive the pool they were created from if other
1570 * objects hold a reference to there VMs.
1571 */
1572void panthor_vm_pool_destroy(struct panthor_file *pfile)
1573{
1574	struct panthor_vm *vm;
1575	unsigned long i;
1576
1577	if (!pfile->vms)
1578		return;
1579
1580	xa_for_each(&pfile->vms->xa, i, vm)
1581		panthor_vm_destroy(vm);
1582
1583	xa_destroy(&pfile->vms->xa);
1584	kfree(pfile->vms);
1585}
1586
1587/**
1588 * panthor_vm_pool_create() - Create a VM pool
1589 * @pfile: File.
1590 *
1591 * Return: 0 on success, a negative error code otherwise.
1592 */
1593int panthor_vm_pool_create(struct panthor_file *pfile)
1594{
1595	pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);
1596	if (!pfile->vms)
1597		return -ENOMEM;
1598
1599	xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
1600	return 0;
1601}
1602
1603/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
1604static void mmu_tlb_flush_all(void *cookie)
1605{
1606}
1607
1608static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)
1609{
1610}
1611
1612static const struct iommu_flush_ops mmu_tlb_ops = {
1613	.tlb_flush_all = mmu_tlb_flush_all,
1614	.tlb_flush_walk = mmu_tlb_flush_walk,
1615};
1616
1617static const char *access_type_name(struct panthor_device *ptdev,
1618				    u32 fault_status)
1619{
1620	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
1621	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
1622		return "ATOMIC";
1623	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
1624		return "READ";
1625	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
1626		return "WRITE";
1627	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
1628		return "EXECUTE";
1629	default:
1630		drm_WARN_ON(&ptdev->base, 1);
1631		return NULL;
1632	}
1633}
1634
1635static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
1636{
1637	bool has_unhandled_faults = false;
1638
1639	status = panthor_mmu_fault_mask(ptdev, status);
1640	while (status) {
1641		u32 as = ffs(status | (status >> 16)) - 1;
1642		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
1643		u32 new_int_mask;
1644		u64 addr;
1645		u32 fault_status;
1646		u32 exception_type;
1647		u32 access_type;
1648		u32 source_id;
1649
1650		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
1651		addr = gpu_read64(ptdev, AS_FAULTADDRESS(as));
1652
1653		/* decode the fault status */
1654		exception_type = fault_status & 0xFF;
1655		access_type = (fault_status >> 8) & 0x3;
1656		source_id = (fault_status >> 16);
1657
1658		mutex_lock(&ptdev->mmu->as.slots_lock);
1659
1660		ptdev->mmu->as.faulty_mask |= mask;
1661		new_int_mask =
1662			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
1663
1664		/* terminal fault, print info about the fault */
1665		drm_err(&ptdev->base,
1666			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
1667			"raw fault status: 0x%X\n"
1668			"decoded fault status: %s\n"
1669			"exception type 0x%X: %s\n"
1670			"access type 0x%X: %s\n"
1671			"source id 0x%X\n",
1672			as, addr,
1673			fault_status,
1674			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
1675			exception_type, panthor_exception_name(ptdev, exception_type),
1676			access_type, access_type_name(ptdev, fault_status),
1677			source_id);
1678
1679		/* We don't handle VM faults at the moment, so let's just clear the
1680		 * interrupt and let the writer/reader crash.
1681		 * Note that COMPLETED irqs are never cleared, but this is fine
1682		 * because they are always masked.
1683		 */
1684		gpu_write(ptdev, MMU_INT_CLEAR, mask);
1685
1686		/* Ignore MMU interrupts on this AS until it's been
1687		 * re-enabled.
1688		 */
1689		ptdev->mmu->irq.mask = new_int_mask;
1690
1691		if (ptdev->mmu->as.slots[as].vm)
1692			ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
1693
1694		/* Disable the MMU to kill jobs on this AS. */
1695		panthor_mmu_as_disable(ptdev, as);
1696		mutex_unlock(&ptdev->mmu->as.slots_lock);
1697
1698		status &= ~mask;
1699		has_unhandled_faults = true;
1700	}
1701
1702	if (has_unhandled_faults)
1703		panthor_sched_report_mmu_fault(ptdev);
1704}
1705PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
1706
1707/**
1708 * panthor_mmu_suspend() - Suspend the MMU logic
1709 * @ptdev: Device.
1710 *
1711 * All we do here is de-assign the AS slots on all active VMs, so things
1712 * get flushed to the main memory, and no further access to these VMs are
1713 * possible.
1714 *
1715 * We also suspend the MMU IRQ.
1716 */
1717void panthor_mmu_suspend(struct panthor_device *ptdev)
1718{
1719	mutex_lock(&ptdev->mmu->as.slots_lock);
1720	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
1721		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
1722
1723		if (vm) {
1724			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
1725			panthor_vm_release_as_locked(vm);
1726		}
1727	}
1728	mutex_unlock(&ptdev->mmu->as.slots_lock);
1729
1730	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
1731}
1732
1733/**
1734 * panthor_mmu_resume() - Resume the MMU logic
1735 * @ptdev: Device.
1736 *
1737 * Resume the IRQ.
1738 *
1739 * We don't re-enable previously active VMs. We assume other parts of the
1740 * driver will call panthor_vm_active() on the VMs they intend to use.
1741 */
1742void panthor_mmu_resume(struct panthor_device *ptdev)
1743{
1744	mutex_lock(&ptdev->mmu->as.slots_lock);
1745	ptdev->mmu->as.alloc_mask = 0;
1746	ptdev->mmu->as.faulty_mask = 0;
1747	mutex_unlock(&ptdev->mmu->as.slots_lock);
1748
1749	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
1750}
1751
1752/**
1753 * panthor_mmu_pre_reset() - Prepare for a reset
1754 * @ptdev: Device.
1755 *
1756 * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we
1757 * don't get asked to do a VM operation while the GPU is down.
1758 *
1759 * We don't cleanly shutdown the AS slots here, because the reset might
1760 * come from an AS_ACTIVE_BIT stuck situation.
1761 */
1762void panthor_mmu_pre_reset(struct panthor_device *ptdev)
1763{
1764	struct panthor_vm *vm;
1765
1766	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
1767
1768	mutex_lock(&ptdev->mmu->vm.lock);
1769	ptdev->mmu->vm.reset_in_progress = true;
1770	list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
1771		panthor_vm_stop(vm);
1772	mutex_unlock(&ptdev->mmu->vm.lock);
1773}
1774
1775/**
1776 * panthor_mmu_post_reset() - Restore things after a reset
1777 * @ptdev: Device.
1778 *
1779 * Put the MMU logic back in action after a reset. That implies resuming the
1780 * IRQ and re-enabling the VM_BIND queues.
1781 */
1782void panthor_mmu_post_reset(struct panthor_device *ptdev)
1783{
1784	struct panthor_vm *vm;
1785
1786	mutex_lock(&ptdev->mmu->as.slots_lock);
1787
1788	/* Now that the reset is effective, we can assume that none of the
1789	 * AS slots are setup, and clear the faulty flags too.
1790	 */
1791	ptdev->mmu->as.alloc_mask = 0;
1792	ptdev->mmu->as.faulty_mask = 0;
1793
1794	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
1795		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
1796
1797		if (vm)
1798			panthor_vm_release_as_locked(vm);
1799	}
1800
1801	mutex_unlock(&ptdev->mmu->as.slots_lock);
1802
1803	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
1804
1805	/* Restart the VM_BIND queues. */
1806	mutex_lock(&ptdev->mmu->vm.lock);
1807	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
1808		panthor_vm_start(vm);
1809	}
1810	ptdev->mmu->vm.reset_in_progress = false;
1811	mutex_unlock(&ptdev->mmu->vm.lock);
1812}
1813
1814static void panthor_vm_free(struct drm_gpuvm *gpuvm)
1815{
1816	struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
1817	struct panthor_device *ptdev = vm->ptdev;
1818
1819	mutex_lock(&vm->heaps.lock);
1820	if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
1821		panthor_heap_pool_destroy(vm->heaps.pool);
1822	mutex_unlock(&vm->heaps.lock);
1823	mutex_destroy(&vm->heaps.lock);
1824
1825	mutex_lock(&ptdev->mmu->vm.lock);
1826	list_del(&vm->node);
1827	/* Restore the scheduler state so we can call drm_sched_entity_destroy()
1828	 * and drm_sched_fini(). If get there, that means we have no job left
1829	 * and no new jobs can be queued, so we can start the scheduler without
1830	 * risking interfering with the reset.
1831	 */
1832	if (ptdev->mmu->vm.reset_in_progress)
1833		panthor_vm_start(vm);
1834	mutex_unlock(&ptdev->mmu->vm.lock);
1835
1836	drm_sched_entity_destroy(&vm->entity);
1837	drm_sched_fini(&vm->sched);
1838
1839	mutex_lock(&ptdev->mmu->as.slots_lock);
1840	if (vm->as.id >= 0) {
1841		int cookie;
1842
1843		if (drm_dev_enter(&ptdev->base, &cookie)) {
1844			panthor_mmu_as_disable(ptdev, vm->as.id);
1845			drm_dev_exit(cookie);
1846		}
1847
1848		ptdev->mmu->as.slots[vm->as.id].vm = NULL;
1849		clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
1850		list_del(&vm->as.lru_node);
1851	}
1852	mutex_unlock(&ptdev->mmu->as.slots_lock);
1853
1854	free_io_pgtable_ops(vm->pgtbl_ops);
1855
1856	drm_mm_takedown(&vm->mm);
1857	kfree(vm);
1858}
1859
1860/**
1861 * panthor_vm_put() - Release a reference on a VM
1862 * @vm: VM to release the reference on. Can be NULL.
1863 */
1864void panthor_vm_put(struct panthor_vm *vm)
1865{
1866	drm_gpuvm_put(vm ? &vm->base : NULL);
1867}
1868
1869/**
1870 * panthor_vm_get() - Get a VM reference
1871 * @vm: VM to get the reference on. Can be NULL.
1872 *
1873 * Return: @vm value.
1874 */
1875struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
1876{
1877	if (vm)
1878		drm_gpuvm_get(&vm->base);
1879
1880	return vm;
1881}
1882
1883/**
1884 * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM
1885 * @vm: VM to query the heap pool on.
1886 * @create: True if the heap pool should be created when it doesn't exist.
1887 *
1888 * Heap pools are per-VM. This function allows one to retrieve the heap pool
1889 * attached to a VM.
1890 *
1891 * If no heap pool exists yet, and @create is true, we create one.
1892 *
1893 * The returned panthor_heap_pool should be released with panthor_heap_pool_put().
1894 *
1895 * Return: A valid pointer on success, an ERR_PTR() otherwise.
1896 */
1897struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
1898{
1899	struct panthor_heap_pool *pool;
1900
1901	mutex_lock(&vm->heaps.lock);
1902	if (!vm->heaps.pool && create) {
1903		if (vm->destroyed)
1904			pool = ERR_PTR(-EINVAL);
1905		else
1906			pool = panthor_heap_pool_create(vm->ptdev, vm);
1907
1908		if (!IS_ERR(pool))
1909			vm->heaps.pool = panthor_heap_pool_get(pool);
1910	} else {
1911		pool = panthor_heap_pool_get(vm->heaps.pool);
1912		if (!pool)
1913			pool = ERR_PTR(-ENOENT);
1914	}
1915	mutex_unlock(&vm->heaps.lock);
1916
1917	return pool;
1918}
1919
1920/**
1921 * panthor_vm_heaps_sizes() - Calculate size of all heap chunks across all
1922 * heaps over all the heap pools in a VM
1923 * @pfile: File.
1924 * @stats: Memory stats to be updated.
1925 *
1926 * Calculate all heap chunk sizes in all heap pools bound to a VM. If the VM
1927 * is active, record the size as active as well.
1928 */
1929void panthor_vm_heaps_sizes(struct panthor_file *pfile, struct drm_memory_stats *stats)
1930{
1931	struct panthor_vm *vm;
1932	unsigned long i;
1933
1934	if (!pfile->vms)
1935		return;
1936
1937	xa_lock(&pfile->vms->xa);
1938	xa_for_each(&pfile->vms->xa, i, vm) {
1939		size_t size = panthor_heap_pool_size(vm->heaps.pool);
1940		stats->resident += size;
1941		if (vm->as.id >= 0)
1942			stats->active += size;
1943	}
1944	xa_unlock(&pfile->vms->xa);
1945}
1946
1947static u64 mair_to_memattr(u64 mair, bool coherent)
1948{
1949	u64 memattr = 0;
1950	u32 i;
1951
1952	for (i = 0; i < 8; i++) {
1953		u8 in_attr = mair >> (8 * i), out_attr;
1954		u8 outer = in_attr >> 4, inner = in_attr & 0xf;
1955
1956		/* For caching to be enabled, inner and outer caching policy
1957		 * have to be both write-back, if one of them is write-through
1958		 * or non-cacheable, we just choose non-cacheable. Device
1959		 * memory is also translated to non-cacheable.
1960		 */
1961		if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {
1962			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |
1963				   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |
1964				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);
1965		} else {
1966			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |
1967				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);
1968			/* Use SH_MIDGARD_INNER mode when device isn't coherent,
1969			 * so SH_IS, which is used when IOMMU_CACHE is set, maps
1970			 * to Mali's internal-shareable mode. As per the Mali
1971			 * Spec, inner and outer-shareable modes aren't allowed
1972			 * for WB memory when coherency is disabled.
1973			 * Use SH_CPU_INNER mode when coherency is enabled, so
1974			 * that SH_IS actually maps to the standard definition of
1975			 * inner-shareable.
1976			 */
1977			if (!coherent)
1978				out_attr |= AS_MEMATTR_AARCH64_SH_MIDGARD_INNER;
1979			else
1980				out_attr |= AS_MEMATTR_AARCH64_SH_CPU_INNER;
1981		}
1982
1983		memattr |= (u64)out_attr << (8 * i);
1984	}
1985
1986	return memattr;
1987}
1988
1989static void panthor_vma_link(struct panthor_vm *vm,
1990			     struct panthor_vma *vma,
1991			     struct drm_gpuvm_bo *vm_bo)
1992{
1993	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
1994
1995	mutex_lock(&bo->base.base.gpuva.lock);
1996	drm_gpuva_link(&vma->base, vm_bo);
1997	mutex_unlock(&bo->base.base.gpuva.lock);
1998}
1999
2000static void panthor_vma_unlink(struct panthor_vma *vma)
2001{
2002	drm_gpuva_unlink_defer(&vma->base);
2003	kfree(vma);
2004}
2005
2006static void panthor_vma_init(struct panthor_vma *vma, u32 flags)
2007{
2008	INIT_LIST_HEAD(&vma->node);
2009	vma->flags = flags;
2010}
2011
2012#define PANTHOR_VM_MAP_FLAGS \
2013	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
2014	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
2015	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
2016
2017static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
2018{
2019	struct panthor_vm *vm = priv;
2020	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
2021	struct panthor_vma *vma = panthor_vm_op_ctx_get_vma(op_ctx);
2022	int ret;
2023
2024	if (!vma)
2025		return -EINVAL;
2026
2027	panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
2028
2029	ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
2030				   op_ctx->map.sgt, op->map.gem.offset,
2031				   op->map.va.range);
2032	if (ret) {
2033		panthor_vm_op_ctx_return_vma(op_ctx, vma);
2034		return ret;
2035	}
2036
2037	drm_gpuva_map(&vm->base, &vma->base, &op->map);
2038	panthor_vma_link(vm, vma, op_ctx->map.vm_bo);
2039
2040	drm_gpuvm_bo_put_deferred(op_ctx->map.vm_bo);
2041	op_ctx->map.vm_bo = NULL;
2042
2043	return 0;
2044}
2045
2046static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
2047				       void *priv)
2048{
2049	struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);
2050	struct panthor_vm *vm = priv;
2051	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
2052	struct panthor_vma *prev_vma = NULL, *next_vma = NULL;
2053	u64 unmap_start, unmap_range;
2054	int ret;
2055
2056	drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
2057	ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
2058	if (ret)
2059		return ret;
2060
2061	if (op->remap.prev) {
2062		prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);
2063		panthor_vma_init(prev_vma, unmap_vma->flags);
2064	}
2065
2066	if (op->remap.next) {
2067		next_vma = panthor_vm_op_ctx_get_vma(op_ctx);
2068		panthor_vma_init(next_vma, unmap_vma->flags);
2069	}
2070
2071	drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL,
2072			next_vma ? &next_vma->base : NULL,
2073			&op->remap);
2074
2075	if (prev_vma) {
2076		/* panthor_vma_link() transfers the vm_bo ownership to
2077		 * the VMA object. Since the vm_bo we're passing is still
2078		 * owned by the old mapping which will be released when this
2079		 * mapping is destroyed, we need to grab a ref here.
2080		 */
2081		panthor_vma_link(vm, prev_vma, op->remap.unmap->va->vm_bo);
2082	}
2083
2084	if (next_vma) {
2085		panthor_vma_link(vm, next_vma, op->remap.unmap->va->vm_bo);
2086	}
2087
2088	panthor_vma_unlink(unmap_vma);
2089	return 0;
2090}
2091
2092static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
2093				       void *priv)
2094{
2095	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
2096	struct panthor_vm *vm = priv;
2097	int ret;
2098
2099	ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
2100				     unmap_vma->base.va.range);
2101	if (drm_WARN_ON(&vm->ptdev->base, ret))
2102		return ret;
2103
2104	drm_gpuva_unmap(&op->unmap);
2105	panthor_vma_unlink(unmap_vma);
2106	return 0;
2107}
2108
2109static const struct drm_gpuvm_ops panthor_gpuvm_ops = {
2110	.vm_free = panthor_vm_free,
2111	.vm_bo_free = panthor_vm_bo_free,
2112	.sm_step_map = panthor_gpuva_sm_step_map,
2113	.sm_step_remap = panthor_gpuva_sm_step_remap,
2114	.sm_step_unmap = panthor_gpuva_sm_step_unmap,
2115};
2116
2117/**
2118 * panthor_vm_resv() - Get the dma_resv object attached to a VM.
2119 * @vm: VM to get the dma_resv of.
2120 *
2121 * Return: A dma_resv object.
2122 */
2123struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
2124{
2125	return drm_gpuvm_resv(&vm->base);
2126}
2127
2128struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm)
2129{
2130	if (!vm)
2131		return NULL;
2132
2133	return vm->base.r_obj;
2134}
2135
2136static int
2137panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
2138		   bool flag_vm_unusable_on_failure)
2139{
2140	u32 op_type = op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK;
2141	int ret;
2142
2143	if (op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY)
2144		return 0;
2145
2146	mutex_lock(&vm->op_lock);
2147	vm->op_ctx = op;
2148	switch (op_type) {
2149	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: {
2150		const struct drm_gpuvm_map_req map_req = {
2151			.map.va.addr = op->va.addr,
2152			.map.va.range = op->va.range,
2153			.map.gem.obj = op->map.vm_bo->obj,
2154			.map.gem.offset = op->map.bo_offset,
2155		};
2156
2157		if (vm->unusable) {
2158			ret = -EINVAL;
2159			break;
2160		}
2161
2162		ret = drm_gpuvm_sm_map(&vm->base, vm, &map_req);
2163		break;
2164	}
2165
2166	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
2167		ret = drm_gpuvm_sm_unmap(&vm->base, vm, op->va.addr, op->va.range);
2168		break;
2169
2170	default:
2171		ret = -EINVAL;
2172		break;
2173	}
2174
2175	if (ret && flag_vm_unusable_on_failure)
2176		vm->unusable = true;
2177
2178	vm->op_ctx = NULL;
2179	mutex_unlock(&vm->op_lock);
2180
2181	return ret;
2182}
2183
2184static struct dma_fence *
2185panthor_vm_bind_run_job(struct drm_sched_job *sched_job)
2186{
2187	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
2188	bool cookie;
2189	int ret;
2190
2191	/* Not only we report an error whose result is propagated to the
2192	 * drm_sched finished fence, but we also flag the VM as unusable, because
2193	 * a failure in the async VM_BIND results in an inconsistent state. VM needs
2194	 * to be destroyed and recreated.
2195	 */
2196	cookie = dma_fence_begin_signalling();
2197	ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
2198	dma_fence_end_signalling(cookie);
2199
2200	return ret ? ERR_PTR(ret) : NULL;
2201}
2202
2203static void panthor_vm_bind_job_release(struct kref *kref)
2204{
2205	struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);
2206
2207	if (job->base.s_fence)
2208		drm_sched_job_cleanup(&job->base);
2209
2210	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
2211	panthor_vm_put(job->vm);
2212	kfree(job);
2213}
2214
2215/**
2216 * panthor_vm_bind_job_put() - Release a VM_BIND job reference
2217 * @sched_job: Job to release the reference on.
2218 */
2219void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)
2220{
2221	struct panthor_vm_bind_job *job =
2222		container_of(sched_job, struct panthor_vm_bind_job, base);
2223
2224	if (sched_job)
2225		kref_put(&job->refcount, panthor_vm_bind_job_release);
2226}
2227
2228static void
2229panthor_vm_bind_free_job(struct drm_sched_job *sched_job)
2230{
2231	struct panthor_vm_bind_job *job =
2232		container_of(sched_job, struct panthor_vm_bind_job, base);
2233
2234	drm_sched_job_cleanup(sched_job);
2235
2236	/* Do the heavy cleanups asynchronously, so we're out of the
2237	 * dma-signaling path and can acquire dma-resv locks safely.
2238	 */
2239	queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);
2240}
2241
2242static enum drm_gpu_sched_stat
2243panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)
2244{
2245	WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");
2246	return DRM_GPU_SCHED_STAT_RESET;
2247}
2248
2249static const struct drm_sched_backend_ops panthor_vm_bind_ops = {
2250	.run_job = panthor_vm_bind_run_job,
2251	.free_job = panthor_vm_bind_free_job,
2252	.timedout_job = panthor_vm_bind_timedout_job,
2253};
2254
2255/**
2256 * panthor_vm_create() - Create a VM
2257 * @ptdev: Device.
2258 * @for_mcu: True if this is the FW MCU VM.
2259 * @kernel_va_start: Start of the range reserved for kernel BO mapping.
2260 * @kernel_va_size: Size of the range reserved for kernel BO mapping.
2261 * @auto_kernel_va_start: Start of the auto-VA kernel range.
2262 * @auto_kernel_va_size: Size of the auto-VA kernel range.
2263 *
2264 * Return: A valid pointer on success, an ERR_PTR() otherwise.
2265 */
2266struct panthor_vm *
2267panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
2268		  u64 kernel_va_start, u64 kernel_va_size,
2269		  u64 auto_kernel_va_start, u64 auto_kernel_va_size)
2270{
2271	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
2272	u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
2273	u64 full_va_range = 1ull << va_bits;
2274	struct drm_gem_object *dummy_gem;
2275	struct drm_gpu_scheduler *sched;
2276	const struct drm_sched_init_args sched_args = {
2277		.ops = &panthor_vm_bind_ops,
2278		.submit_wq = ptdev->mmu->vm.wq,
2279		.num_rqs = 1,
2280		.credit_limit = 1,
2281		/* Bind operations are synchronous for now, no timeout needed. */
2282		.timeout = MAX_SCHEDULE_TIMEOUT,
2283		.name = "panthor-vm-bind",
2284		.dev = ptdev->base.dev,
2285	};
2286	struct io_pgtable_cfg pgtbl_cfg;
2287	u64 mair, min_va, va_range;
2288	struct panthor_vm *vm;
2289	int ret;
2290
2291	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2292	if (!vm)
2293		return ERR_PTR(-ENOMEM);
2294
2295	/* We allocate a dummy GEM for the VM. */
2296	dummy_gem = drm_gpuvm_resv_object_alloc(&ptdev->base);
2297	if (!dummy_gem) {
2298		ret = -ENOMEM;
2299		goto err_free_vm;
2300	}
2301
2302	mutex_init(&vm->heaps.lock);
2303	vm->for_mcu = for_mcu;
2304	vm->ptdev = ptdev;
2305	mutex_init(&vm->op_lock);
2306
2307	if (for_mcu) {
2308		/* CSF MCU is a cortex M7, and can only address 4G */
2309		min_va = 0;
2310		va_range = SZ_4G;
2311	} else {
2312		min_va = 0;
2313		va_range = full_va_range;
2314	}
2315
2316	mutex_init(&vm->mm_lock);
2317	drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size);
2318	vm->kernel_auto_va.start = auto_kernel_va_start;
2319	vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
2320
2321	INIT_LIST_HEAD(&vm->node);
2322	INIT_LIST_HEAD(&vm->as.lru_node);
2323	vm->as.id = -1;
2324	refcount_set(&vm->as.active_cnt, 0);
2325
2326	pgtbl_cfg = (struct io_pgtable_cfg) {
2327		.pgsize_bitmap	= SZ_4K | SZ_2M,
2328		.ias		= va_bits,
2329		.oas		= pa_bits,
2330		.coherent_walk	= ptdev->coherent,
2331		.tlb		= &mmu_tlb_ops,
2332		.iommu_dev	= ptdev->base.dev,
2333		.alloc		= alloc_pt,
2334		.free		= free_pt,
2335	};
2336
2337	vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
2338	if (!vm->pgtbl_ops) {
2339		ret = -EINVAL;
2340		goto err_mm_takedown;
2341	}
2342
2343	ret = drm_sched_init(&vm->sched, &sched_args);
2344	if (ret)
2345		goto err_free_io_pgtable;
2346
2347	sched = &vm->sched;
2348	ret = drm_sched_entity_init(&vm->entity, 0, &sched, 1, NULL);
2349	if (ret)
2350		goto err_sched_fini;
2351
2352	mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
2353	vm->memattr = mair_to_memattr(mair, ptdev->coherent);
2354
2355	mutex_lock(&ptdev->mmu->vm.lock);
2356	list_add_tail(&vm->node, &ptdev->mmu->vm.list);
2357
2358	/* If a reset is in progress, stop the scheduler. */
2359	if (ptdev->mmu->vm.reset_in_progress)
2360		panthor_vm_stop(vm);
2361	mutex_unlock(&ptdev->mmu->vm.lock);
2362
2363	/* We intentionally leave the reserved range to zero, because we want kernel VMAs
2364	 * to be handled the same way user VMAs are.
2365	 */
2366	drm_gpuvm_init(&vm->base, for_mcu ? "panthor-MCU-VM" : "panthor-GPU-VM",
2367		       DRM_GPUVM_RESV_PROTECTED | DRM_GPUVM_IMMEDIATE_MODE,
2368		       &ptdev->base, dummy_gem, min_va, va_range, 0, 0,
2369		       &panthor_gpuvm_ops);
2370	drm_gem_object_put(dummy_gem);
2371	return vm;
2372
2373err_sched_fini:
2374	drm_sched_fini(&vm->sched);
2375
2376err_free_io_pgtable:
2377	free_io_pgtable_ops(vm->pgtbl_ops);
2378
2379err_mm_takedown:
2380	drm_mm_takedown(&vm->mm);
2381	drm_gem_object_put(dummy_gem);
2382
2383err_free_vm:
2384	kfree(vm);
2385	return ERR_PTR(ret);
2386}
2387
2388static int
2389panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
2390			       struct panthor_vm *vm,
2391			       const struct drm_panthor_vm_bind_op *op,
2392			       struct panthor_vm_op_ctx *op_ctx)
2393{
2394	ssize_t vm_pgsz = panthor_vm_page_size(vm);
2395	struct drm_gem_object *gem;
2396	int ret;
2397
2398	/* Aligned on page size. */
2399	if (!IS_ALIGNED(op->va | op->size | op->bo_offset, vm_pgsz))
2400		return -EINVAL;
2401
2402	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
2403	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
2404		gem = drm_gem_object_lookup(file, op->bo_handle);
2405		ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
2406						    gem ? to_panthor_bo(gem) : NULL,
2407						    op->bo_offset,
2408						    op->size,
2409						    op->va,
2410						    op->flags);
2411		drm_gem_object_put(gem);
2412		return ret;
2413
2414	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
2415		if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
2416			return -EINVAL;
2417
2418		if (op->bo_handle || op->bo_offset)
2419			return -EINVAL;
2420
2421		return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
2422
2423	case DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY:
2424		if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
2425			return -EINVAL;
2426
2427		if (op->bo_handle || op->bo_offset)
2428			return -EINVAL;
2429
2430		if (op->va || op->size)
2431			return -EINVAL;
2432
2433		if (!op->syncs.count)
2434			return -EINVAL;
2435
2436		panthor_vm_prepare_sync_only_op_ctx(op_ctx, vm);
2437		return 0;
2438
2439	default:
2440		return -EINVAL;
2441	}
2442}
2443
2444static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)
2445{
2446	struct panthor_vm_bind_job *job =
2447		container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);
2448
2449	panthor_vm_bind_job_put(&job->base);
2450}
2451
2452/**
2453 * panthor_vm_bind_job_create() - Create a VM_BIND job
2454 * @file: File.
2455 * @vm: VM targeted by the VM_BIND job.
2456 * @op: VM operation data.
2457 *
2458 * Return: A valid pointer on success, an ERR_PTR() otherwise.
2459 */
2460struct drm_sched_job *
2461panthor_vm_bind_job_create(struct drm_file *file,
2462			   struct panthor_vm *vm,
2463			   const struct drm_panthor_vm_bind_op *op)
2464{
2465	struct panthor_vm_bind_job *job;
2466	int ret;
2467
2468	if (!vm)
2469		return ERR_PTR(-EINVAL);
2470
2471	if (vm->destroyed || vm->unusable)
2472		return ERR_PTR(-EINVAL);
2473
2474	job = kzalloc(sizeof(*job), GFP_KERNEL);
2475	if (!job)
2476		return ERR_PTR(-ENOMEM);
2477
2478	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
2479	if (ret) {
2480		kfree(job);
2481		return ERR_PTR(ret);
2482	}
2483
2484	INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);
2485	kref_init(&job->refcount);
2486	job->vm = panthor_vm_get(vm);
2487
2488	ret = drm_sched_job_init(&job->base, &vm->entity, 1, vm, file->client_id);
2489	if (ret)
2490		goto err_put_job;
2491
2492	return &job->base;
2493
2494err_put_job:
2495	panthor_vm_bind_job_put(&job->base);
2496	return ERR_PTR(ret);
2497}
2498
2499/**
2500 * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs
2501 * @exec: The locking/preparation context.
2502 * @sched_job: The job to prepare resvs on.
2503 *
2504 * Locks and prepare the VM resv.
2505 *
2506 * If this is a map operation, locks and prepares the GEM resv.
2507 *
2508 * Return: 0 on success, a negative error code otherwise.
2509 */
2510int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
2511				      struct drm_sched_job *sched_job)
2512{
2513	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
2514	int ret;
2515
2516	/* Acquire the VM lock an reserve a slot for this VM bind job. */
2517	ret = drm_gpuvm_prepare_vm(&job->vm->base, exec, 1);
2518	if (ret)
2519		return ret;
2520
2521	if (job->ctx.map.vm_bo) {
2522		/* Lock/prepare the GEM being mapped. */
2523		ret = drm_exec_prepare_obj(exec, job->ctx.map.vm_bo->obj, 1);
2524		if (ret)
2525			return ret;
2526	}
2527
2528	return 0;
2529}
2530
2531/**
2532 * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job
2533 * @exec: drm_exec context.
2534 * @sched_job: Job to update the resvs on.
2535 */
2536void panthor_vm_bind_job_update_resvs(struct drm_exec *exec,
2537				      struct drm_sched_job *sched_job)
2538{
2539	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
2540
2541	/* Explicit sync => we just register our job finished fence as bookkeep. */
2542	drm_gpuvm_resv_add_fence(&job->vm->base, exec,
2543				 &sched_job->s_fence->finished,
2544				 DMA_RESV_USAGE_BOOKKEEP,
2545				 DMA_RESV_USAGE_BOOKKEEP);
2546}
2547
2548void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec,
2549			     struct dma_fence *fence,
2550			     enum dma_resv_usage private_usage,
2551			     enum dma_resv_usage extobj_usage)
2552{
2553	drm_gpuvm_resv_add_fence(&vm->base, exec, fence, private_usage, extobj_usage);
2554}
2555
2556/**
2557 * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.
2558 * @file: File.
2559 * @vm: VM targeted by the VM operation.
2560 * @op: Data describing the VM operation.
2561 *
2562 * Return: 0 on success, a negative error code otherwise.
2563 */
2564int panthor_vm_bind_exec_sync_op(struct drm_file *file,
2565				 struct panthor_vm *vm,
2566				 struct drm_panthor_vm_bind_op *op)
2567{
2568	struct panthor_vm_op_ctx op_ctx;
2569	int ret;
2570
2571	/* No sync objects allowed on synchronous operations. */
2572	if (op->syncs.count)
2573		return -EINVAL;
2574
2575	if (!op->size)
2576		return 0;
2577
2578	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
2579	if (ret)
2580		return ret;
2581
2582	ret = panthor_vm_exec_op(vm, &op_ctx, false);
2583	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2584
2585	return ret;
2586}
2587
2588/**
2589 * panthor_vm_map_bo_range() - Map a GEM object range to a VM
2590 * @vm: VM to map the GEM to.
2591 * @bo: GEM object to map.
2592 * @offset: Offset in the GEM object.
2593 * @size: Size to map.
2594 * @va: Virtual address to map the object to.
2595 * @flags: Combination of drm_panthor_vm_bind_op_flags flags.
2596 * Only map-related flags are valid.
2597 *
2598 * Internal use only. For userspace requests, use
2599 * panthor_vm_bind_exec_sync_op() instead.
2600 *
2601 * Return: 0 on success, a negative error code otherwise.
2602 */
2603int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
2604			    u64 offset, u64 size, u64 va, u32 flags)
2605{
2606	struct panthor_vm_op_ctx op_ctx;
2607	int ret;
2608
2609	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
2610	if (ret)
2611		return ret;
2612
2613	ret = panthor_vm_exec_op(vm, &op_ctx, false);
2614	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2615
2616	return ret;
2617}
2618
2619/**
2620 * panthor_vm_unmap_range() - Unmap a portion of the VA space
2621 * @vm: VM to unmap the region from.
2622 * @va: Virtual address to unmap. Must be 4k aligned.
2623 * @size: Size of the region to unmap. Must be 4k aligned.
2624 *
2625 * Internal use only. For userspace requests, use
2626 * panthor_vm_bind_exec_sync_op() instead.
2627 *
2628 * Return: 0 on success, a negative error code otherwise.
2629 */
2630int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size)
2631{
2632	struct panthor_vm_op_ctx op_ctx;
2633	int ret;
2634
2635	ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
2636	if (ret)
2637		return ret;
2638
2639	ret = panthor_vm_exec_op(vm, &op_ctx, false);
2640	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2641
2642	return ret;
2643}
2644
2645/**
2646 * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.
2647 * @exec: Locking/preparation context.
2648 * @vm: VM targeted by the GPU job.
2649 * @slot_count: Number of slots to reserve.
2650 *
2651 * GPU jobs assume all BOs bound to the VM at the time the job is submitted
2652 * are available when the job is executed. In order to guarantee that, we
2653 * need to reserve a slot on all BOs mapped to a VM and update this slot with
2654 * the job fence after its submission.
2655 *
2656 * Return: 0 on success, a negative error code otherwise.
2657 */
2658int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm,
2659					u32 slot_count)
2660{
2661	int ret;
2662
2663	/* Acquire the VM lock and reserve a slot for this GPU job. */
2664	ret = drm_gpuvm_prepare_vm(&vm->base, exec, slot_count);
2665	if (ret)
2666		return ret;
2667
2668	return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
2669}
2670
2671/**
2672 * panthor_mmu_unplug() - Unplug the MMU logic
2673 * @ptdev: Device.
2674 *
2675 * No access to the MMU regs should be done after this function is called.
2676 * We suspend the IRQ and disable all VMs to guarantee that.
2677 */
2678void panthor_mmu_unplug(struct panthor_device *ptdev)
2679{
2680	if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev))
2681		panthor_mmu_irq_suspend(&ptdev->mmu->irq);
2682
2683	mutex_lock(&ptdev->mmu->as.slots_lock);
2684	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
2685		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
2686
2687		if (vm) {
2688			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
2689			panthor_vm_release_as_locked(vm);
2690		}
2691	}
2692	mutex_unlock(&ptdev->mmu->as.slots_lock);
2693}
2694
2695static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
2696{
2697	destroy_workqueue(res);
2698}
2699
2700/**
2701 * panthor_mmu_init() - Initialize the MMU logic.
2702 * @ptdev: Device.
2703 *
2704 * Return: 0 on success, a negative error code otherwise.
2705 */
2706int panthor_mmu_init(struct panthor_device *ptdev)
2707{
2708	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
2709	struct panthor_mmu *mmu;
2710	int ret, irq;
2711
2712	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
2713	if (!mmu)
2714		return -ENOMEM;
2715
2716	INIT_LIST_HEAD(&mmu->as.lru_list);
2717
2718	ret = drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);
2719	if (ret)
2720		return ret;
2721
2722	INIT_LIST_HEAD(&mmu->vm.list);
2723	ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
2724	if (ret)
2725		return ret;
2726
2727	ptdev->mmu = mmu;
2728
2729	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
2730	if (irq <= 0)
2731		return -ENODEV;
2732
2733	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
2734				      panthor_mmu_fault_mask(ptdev, ~0));
2735	if (ret)
2736		return ret;
2737
2738	mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
2739	if (!mmu->vm.wq)
2740		return -ENOMEM;
2741
2742	/* On 32-bit kernels, the VA space is limited by the io_pgtable_ops abstraction,
2743	 * which passes iova as an unsigned long. Patch the mmu_features to reflect this
2744	 * limitation.
2745	 */
2746	if (va_bits > BITS_PER_LONG) {
2747		ptdev->gpu_info.mmu_features &= ~GENMASK(7, 0);
2748		ptdev->gpu_info.mmu_features |= BITS_PER_LONG;
2749	}
2750
2751	return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
2752}
2753
2754#ifdef CONFIG_DEBUG_FS
2755static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
2756{
2757	int ret;
2758
2759	mutex_lock(&vm->op_lock);
2760	ret = drm_debugfs_gpuva_info(m, &vm->base);
2761	mutex_unlock(&vm->op_lock);
2762
2763	return ret;
2764}
2765
2766static int show_each_vm(struct seq_file *m, void *arg)
2767{
2768	struct drm_info_node *node = (struct drm_info_node *)m->private;
2769	struct drm_device *ddev = node->minor->dev;
2770	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
2771	int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;
2772	struct panthor_vm *vm;
2773	int ret = 0;
2774
2775	mutex_lock(&ptdev->mmu->vm.lock);
2776	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
2777		ret = show(vm, m);
2778		if (ret < 0)
2779			break;
2780
2781		seq_puts(m, "\n");
2782	}
2783	mutex_unlock(&ptdev->mmu->vm.lock);
2784
2785	return ret;
2786}
2787
2788static struct drm_info_list panthor_mmu_debugfs_list[] = {
2789	DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),
2790};
2791
2792/**
2793 * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
2794 * @minor: Minor.
2795 */
2796void panthor_mmu_debugfs_init(struct drm_minor *minor)
2797{
2798	drm_debugfs_create_files(panthor_mmu_debugfs_list,
2799				 ARRAY_SIZE(panthor_mmu_debugfs_list),
2800				 minor->debugfs_root, minor);
2801}
2802#endif /* CONFIG_DEBUG_FS */
2803
2804/**
2805 * panthor_mmu_pt_cache_init() - Initialize the page table cache.
2806 *
2807 * Return: 0 on success, a negative error code otherwise.
2808 */
2809int panthor_mmu_pt_cache_init(void)
2810{
2811	pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);
2812	if (!pt_cache)
2813		return -ENOMEM;
2814
2815	return 0;
2816}
2817
2818/**
2819 * panthor_mmu_pt_cache_fini() - Destroy the page table cache.
2820 */
2821void panthor_mmu_pt_cache_fini(void)
2822{
2823	kmem_cache_destroy(pt_cache);
2824}