xe_vm_types.h

  1/* SPDX-License-Identifier: MIT */
  2/*
  3 * Copyright © 2022 Intel Corporation
  4 */
  5
  6#ifndef _XE_VM_TYPES_H_
  7#define _XE_VM_TYPES_H_
  8
  9#include <drm/drm_gpusvm.h>
 10#include <drm/drm_gpuvm.h>
 11
 12#include <linux/dma-resv.h>
 13#include <linux/kref.h>
 14#include <linux/mmu_notifier.h>
 15#include <linux/scatterlist.h>
 16
 17#include "xe_device_types.h"
 18#include "xe_pt_types.h"
 19#include "xe_range_fence.h"
 20#include "xe_userptr.h"
 21
 22struct xe_bo;
 23struct xe_svm_range;
 24struct xe_sync_entry;
 25struct xe_user_fence;
 26struct xe_vm;
 27struct xe_vm_pgtable_update_op;
 28
 29#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 30#define TEST_VM_OPS_ERROR
 31#define FORCE_OP_ERROR	BIT(31)
 32
 33#define FORCE_OP_ERROR_LOCK	0
 34#define FORCE_OP_ERROR_PREPARE	1
 35#define FORCE_OP_ERROR_RUN	2
 36#define FORCE_OP_ERROR_COUNT	3
 37#endif
 38
 39#define XE_VMA_READ_ONLY	DRM_GPUVA_USERBITS
 40#define XE_VMA_DESTROYED	(DRM_GPUVA_USERBITS << 1)
 41#define XE_VMA_ATOMIC_PTE_BIT	(DRM_GPUVA_USERBITS << 2)
 42#define XE_VMA_PTE_4K		(DRM_GPUVA_USERBITS << 3)
 43#define XE_VMA_PTE_2M		(DRM_GPUVA_USERBITS << 4)
 44#define XE_VMA_PTE_1G		(DRM_GPUVA_USERBITS << 5)
 45#define XE_VMA_PTE_64K		(DRM_GPUVA_USERBITS << 6)
 46#define XE_VMA_PTE_COMPACT	(DRM_GPUVA_USERBITS << 7)
 47#define XE_VMA_DUMPABLE		(DRM_GPUVA_USERBITS << 8)
 48#define XE_VMA_SYSTEM_ALLOCATOR	(DRM_GPUVA_USERBITS << 9)
 49#define XE_VMA_MADV_AUTORESET	(DRM_GPUVA_USERBITS << 10)
 50
 51/**
 52 * struct xe_vma_mem_attr - memory attributes associated with vma
 53 */
 54struct xe_vma_mem_attr {
 55	/** @preferred_loc: preferred memory_location */
 56	struct {
 57		/** @preferred_loc.migration_policy: Pages migration policy */
 58		u32 migration_policy;
 59
 60		/**
 61		 * @preferred_loc.devmem_fd: used for determining pagemap_fd
 62		 * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and
 63		 * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or
 64		 * closest device memory respectively.
 65		 */
 66		u32 devmem_fd;
 67	} preferred_loc;
 68
 69	/**
 70	 * @atomic_access: The atomic access type for the vma
 71	 * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE,
 72	 * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible
 73	 * values. These are defined in uapi/drm/xe_drm.h.
 74	 */
 75	u32 atomic_access;
 76
 77	/**
 78	 * @default_pat_index: The pat index for VMA set during first bind by user.
 79	 */
 80	u16 default_pat_index;
 81
 82	/**
 83	 * @pat_index: The pat index to use when encoding the PTEs for this vma.
 84	 * same as default_pat_index unless overwritten by madvise.
 85	 */
 86	u16 pat_index;
 87};
 88
 89struct xe_vma {
 90	/** @gpuva: Base GPUVA object */
 91	struct drm_gpuva gpuva;
 92
 93	/**
 94	 * @combined_links: links into lists which are mutually exclusive.
 95	 * Locking: vm lock in write mode OR vm lock in read mode and the vm's
 96	 * resv.
 97	 */
 98	union {
 99		/** @rebind: link into VM if this VMA needs rebinding. */
100		struct list_head rebind;
101		/** @destroy: link to contested list when VM is being closed. */
102		struct list_head destroy;
103	} combined_links;
104
105	union {
106		/** @destroy_cb: callback to destroy VMA when unbind job is done */
107		struct dma_fence_cb destroy_cb;
108		/** @destroy_work: worker to destroy this BO */
109		struct work_struct destroy_work;
110	};
111
112	/**
113	 * @tile_invalidated: Tile mask of binding are invalidated for this VMA.
114	 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in
115	 * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and
116	 * the vm->resv. For stable reading, BO's resv or userptr
117	 * vm->svm.gpusvm.notifier_lock in read mode is required. Can be
118	 * opportunistically read with READ_ONCE outside of locks.
119	 */
120	u8 tile_invalidated;
121
122	/** @tile_mask: Tile mask of where to create binding for this VMA */
123	u8 tile_mask;
124
125	/**
126	 * @tile_present: Tile mask of binding are present for this VMA.
127	 * protected by vm->lock, vm->resv and for userptrs,
128	 * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading,
129	 * but if reading is done under the vm->lock only, it needs to be held
130	 * in write mode.
131	 */
132	u8 tile_present;
133
134	/** @tile_staged: bind is staged for this VMA */
135	u8 tile_staged;
136
137	/**
138	 * @skip_invalidation: Used in madvise to avoid invalidation
139	 * if mem attributes doesn't change
140	 */
141	bool skip_invalidation;
142
143	/**
144	 * @ufence: The user fence that was provided with MAP.
145	 * Needs to be signalled before UNMAP can be processed.
146	 */
147	struct xe_user_fence *ufence;
148
149	/**
150	 * @attr: The attributes of vma which determines the migration policy
151	 * and encoding of the PTEs for this vma.
152	 */
153	struct xe_vma_mem_attr attr;
154};
155
156/**
157 * struct xe_userptr_vma - A userptr vma subclass
158 * @vma: The vma.
159 * @userptr: Additional userptr information.
160 */
161struct xe_userptr_vma {
162	struct xe_vma vma;
163	struct xe_userptr userptr;
164};
165
166struct xe_device;
167
168struct xe_vm {
169	/** @gpuvm: base GPUVM used to track VMAs */
170	struct drm_gpuvm gpuvm;
171
172	/** @svm: Shared virtual memory state */
173	struct {
174		/** @svm.gpusvm: base GPUSVM used to track fault allocations */
175		struct drm_gpusvm gpusvm;
176		/**
177		 * @svm.garbage_collector: Garbage collector which is used unmap
178		 * SVM range's GPU bindings and destroy the ranges.
179		 */
180		struct {
181			/** @svm.garbage_collector.lock: Protect's range list */
182			spinlock_t lock;
183			/**
184			 * @svm.garbage_collector.range_list: List of SVM ranges
185			 * in the garbage collector.
186			 */
187			struct list_head range_list;
188			/**
189			 * @svm.garbage_collector.work: Worker which the
190			 * garbage collector runs on.
191			 */
192			struct work_struct work;
193		} garbage_collector;
194	} svm;
195
196	struct xe_device *xe;
197
198	/* exec queue used for (un)binding vma's */
199	struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE];
200
201	/** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
202	struct ttm_lru_bulk_move lru_bulk_move;
203
204	u64 size;
205
206	struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE];
207	struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL];
208
209	/**
210	 * @flags: flags for this VM, statically setup a creation time aside
211	 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely
212	 */
213#define XE_VM_FLAG_64K			BIT(0)
214#define XE_VM_FLAG_LR_MODE		BIT(1)
215#define XE_VM_FLAG_MIGRATION		BIT(2)
216#define XE_VM_FLAG_SCRATCH_PAGE		BIT(3)
217#define XE_VM_FLAG_FAULT_MODE		BIT(4)
218#define XE_VM_FLAG_BANNED		BIT(5)
219#define XE_VM_FLAG_TILE_ID(flags)	FIELD_GET(GENMASK(7, 6), flags)
220#define XE_VM_FLAG_SET_TILE_ID(tile)	FIELD_PREP(GENMASK(7, 6), (tile)->id)
221#define XE_VM_FLAG_GSC			BIT(8)
222	unsigned long flags;
223
224	/**
225	 * @lock: outer most lock, protects objects of anything attached to this
226	 * VM
227	 */
228	struct rw_semaphore lock;
229	/**
230	 * @snap_mutex: Mutex used to guard insertions and removals from gpuva,
231	 * so we can take a snapshot safely from devcoredump.
232	 */
233	struct mutex snap_mutex;
234
235	/**
236	 * @rebind_list: list of VMAs that need rebinding. Protected by the
237	 * vm->lock in write mode, OR (the vm->lock in read mode and the
238	 * vm resv).
239	 */
240	struct list_head rebind_list;
241
242	/**
243	 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling
244	 * from an irq context can be last put and the destroy needs to be able
245	 * to sleep.
246	 */
247	struct work_struct destroy_work;
248
249	/**
250	 * @rftree: range fence tree to track updates to page table structure.
251	 * Used to implement conflict tracking between independent bind engines.
252	 */
253	struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
254
255	const struct xe_pt_ops *pt_ops;
256
257	/** @userptr: user pointer state */
258	struct xe_userptr_vm userptr;
259
260	/** @preempt: preempt state */
261	struct {
262		/**
263		 * @min_run_period_ms: The minimum run period before preempting
264		 * an engine again
265		 */
266		unsigned int min_run_period_ms;
267		/** @exec_queues: list of exec queues attached to this VM */
268		struct list_head exec_queues;
269		/** @num_exec_queues: number exec queues attached to this VM */
270		int num_exec_queues;
271		/**
272		 * @rebind_deactivated: Whether rebind has been temporarily deactivated
273		 * due to no work available. Protected by the vm resv.
274		 */
275		bool rebind_deactivated;
276		/**
277		 * @rebind_work: worker to rebind invalidated userptrs / evicted
278		 * BOs
279		 */
280		struct work_struct rebind_work;
281		/**
282		 * @preempt.pm_activate_link: Link to list of rebind workers to be
283		 * kicked on resume.
284		 */
285		struct list_head pm_activate_link;
286	} preempt;
287
288	/** @um: unified memory state */
289	struct {
290		/** @asid: address space ID, unique to each VM */
291		u32 asid;
292		/**
293		 * @last_fault_vma: Last fault VMA, used for fast lookup when we
294		 * get a flood of faults to the same VMA
295		 */
296		struct xe_vma *last_fault_vma;
297	} usm;
298
299	/** @error_capture: allow to track errors */
300	struct {
301		/** @capture_once: capture only one error per VM */
302		bool capture_once;
303	} error_capture;
304
305	/**
306	 * @validation: Validation data only valid with the vm resv held.
307	 * Note: This is really task state of the task holding the vm resv,
308	 * and moving forward we should
309	 * come up with a better way of passing this down the call-
310	 * chain.
311	 */
312	struct {
313		/**
314		 * @validation.validating: The task that is currently making bos resident.
315		 * for this vm.
316		 * Protected by the VM's resv for writing. Opportunistic reading can be done
317		 * using READ_ONCE. Note: This is a workaround for the
318		 * TTM eviction_valuable() callback not being passed a struct
319		 * ttm_operation_context(). Future work might want to address this.
320		 */
321		struct task_struct *validating;
322		/**
323		 *  @validation.exec The drm_exec context used when locking the vm resv.
324		 *  Protected by the vm's resv.
325		 */
326		struct drm_exec *_exec;
327	} validation;
328
329	/**
330	 * @tlb_flush_seqno: Required TLB flush seqno for the next exec.
331	 * protected by the vm resv.
332	 */
333	u64 tlb_flush_seqno;
334	/** @batch_invalidate_tlb: Always invalidate TLB before batch start */
335	bool batch_invalidate_tlb;
336	/** @xef: Xe file handle for tracking this VM's drm client */
337	struct xe_file *xef;
338};
339
340/** struct xe_vma_op_map - VMA map operation */
341struct xe_vma_op_map {
342	/** @vma: VMA to map */
343	struct xe_vma *vma;
344	unsigned int vma_flags;
345	/** @immediate: Immediate bind */
346	bool immediate;
347	/** @read_only: Read only */
348	bool invalidate_on_bind;
349	/** @pat_index: The pat index to use for this operation. */
350	u16 pat_index;
351};
352
353/** struct xe_vma_op_remap - VMA remap operation */
354struct xe_vma_op_remap {
355	/** @prev: VMA preceding part of a split mapping */
356	struct xe_vma *prev;
357	/** @next: VMA subsequent part of a split mapping */
358	struct xe_vma *next;
359	/** @start: start of the VMA unmap */
360	u64 start;
361	/** @range: range of the VMA unmap */
362	u64 range;
363	/** @old_start: Original start of the VMA we unmap */
364	u64 old_start;
365	/** @old_range: Original range of the VMA we unmap */
366	u64 old_range;
367	/** @skip_prev: skip prev rebind */
368	bool skip_prev;
369	/** @skip_next: skip next rebind */
370	bool skip_next;
371	/** @unmap_done: unmap operation in done */
372	bool unmap_done;
373};
374
375/** struct xe_vma_op_prefetch - VMA prefetch operation */
376struct xe_vma_op_prefetch {
377	/** @region: memory region to prefetch to */
378	u32 region;
379};
380
381/** struct xe_vma_op_map_range - VMA map range operation */
382struct xe_vma_op_map_range {
383	/** @vma: VMA to map (system allocator VMA) */
384	struct xe_vma *vma;
385	/** @range: SVM range to map */
386	struct xe_svm_range *range;
387};
388
389/** struct xe_vma_op_unmap_range - VMA unmap range operation */
390struct xe_vma_op_unmap_range {
391	/** @range: SVM range to unmap */
392	struct xe_svm_range *range;
393};
394
395/** struct xe_vma_op_prefetch_range - VMA prefetch range operation */
396struct xe_vma_op_prefetch_range {
397	/** @range: xarray for SVM ranges data */
398	struct xarray range;
399	/** @ranges_count: number of svm ranges to map */
400	u32 ranges_count;
401	/**
402	 * @tile: Pointer to the tile structure containing memory to prefetch.
403	 *        NULL if prefetch requested region is smem
404	 */
405	struct xe_tile *tile;
406};
407
408/** enum xe_vma_op_flags - flags for VMA operation */
409enum xe_vma_op_flags {
410	/** @XE_VMA_OP_COMMITTED: VMA operation committed */
411	XE_VMA_OP_COMMITTED		= BIT(0),
412	/** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
413	XE_VMA_OP_PREV_COMMITTED	= BIT(1),
414	/** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
415	XE_VMA_OP_NEXT_COMMITTED	= BIT(2),
416};
417
418/** enum xe_vma_subop - VMA sub-operation */
419enum xe_vma_subop {
420	/** @XE_VMA_SUBOP_MAP_RANGE: Map range */
421	XE_VMA_SUBOP_MAP_RANGE,
422	/** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */
423	XE_VMA_SUBOP_UNMAP_RANGE,
424};
425
426/** struct xe_vma_op - VMA operation */
427struct xe_vma_op {
428	/** @base: GPUVA base operation */
429	struct drm_gpuva_op base;
430	/** @link: async operation link */
431	struct list_head link;
432	/** @flags: operation flags */
433	enum xe_vma_op_flags flags;
434	/** @subop: user defined sub-operation */
435	enum xe_vma_subop subop;
436	/** @tile_mask: Tile mask for operation */
437	u8 tile_mask;
438
439	union {
440		/** @map: VMA map operation specific data */
441		struct xe_vma_op_map map;
442		/** @remap: VMA remap operation specific data */
443		struct xe_vma_op_remap remap;
444		/** @prefetch: VMA prefetch operation specific data */
445		struct xe_vma_op_prefetch prefetch;
446		/** @map_range: VMA map range operation specific data */
447		struct xe_vma_op_map_range map_range;
448		/** @unmap_range: VMA unmap range operation specific data */
449		struct xe_vma_op_unmap_range unmap_range;
450		/** @prefetch_range: VMA prefetch range operation specific data */
451		struct xe_vma_op_prefetch_range prefetch_range;
452	};
453};
454
455/** struct xe_vma_ops - VMA operations */
456struct xe_vma_ops {
457	/** @list: list of VMA operations */
458	struct list_head list;
459	/** @vm: VM */
460	struct xe_vm *vm;
461	/** @q: exec queue for VMA operations */
462	struct xe_exec_queue *q;
463	/** @syncs: syncs these operation */
464	struct xe_sync_entry *syncs;
465	/** @num_syncs: number of syncs */
466	u32 num_syncs;
467	/** @pt_update_ops: page table update operations */
468	struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
469	/** @flag: signify the properties within xe_vma_ops*/
470#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0)
471#define XE_VMA_OPS_FLAG_MADVISE          BIT(1)
472#define XE_VMA_OPS_ARRAY_OF_BINDS	 BIT(2)
473#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT	 BIT(3)
474	u32 flags;
475#ifdef TEST_VM_OPS_ERROR
476	/** @inject_error: inject error to test error handling */
477	bool inject_error;
478#endif
479};
480
481#endif