xe_bo_evict.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2022 Intel Corporation
  4 */
  5
  6#include "xe_bo_evict.h"
  7
  8#include "xe_bo.h"
  9#include "xe_device.h"
 10#include "xe_ggtt.h"
 11#include "xe_tile.h"
 12
 13typedef int (*xe_pinned_fn)(struct xe_bo *bo);
 14
 15static int xe_bo_apply_to_pinned(struct xe_device *xe,
 16				 struct list_head *pinned_list,
 17				 struct list_head *new_list,
 18				 const xe_pinned_fn pinned_fn)
 19{
 20	LIST_HEAD(still_in_list);
 21	struct xe_bo *bo;
 22	int ret = 0;
 23
 24	spin_lock(&xe->pinned.lock);
 25	while (!ret) {
 26		bo = list_first_entry_or_null(pinned_list, typeof(*bo),
 27					      pinned_link);
 28		if (!bo)
 29			break;
 30		xe_bo_get(bo);
 31		list_move_tail(&bo->pinned_link, &still_in_list);
 32		spin_unlock(&xe->pinned.lock);
 33
 34		ret = pinned_fn(bo);
 35		if (ret && pinned_list != new_list) {
 36			spin_lock(&xe->pinned.lock);
 37			/*
 38			 * We might no longer be pinned, since PM notifier can
 39			 * call this. If the pinned link is now empty, keep it
 40			 * that way.
 41			 */
 42			if (!list_empty(&bo->pinned_link))
 43				list_move(&bo->pinned_link, pinned_list);
 44			spin_unlock(&xe->pinned.lock);
 45		}
 46		xe_bo_put(bo);
 47		spin_lock(&xe->pinned.lock);
 48	}
 49	list_splice_tail(&still_in_list, new_list);
 50	spin_unlock(&xe->pinned.lock);
 51
 52	return ret;
 53}
 54
 55/**
 56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all
 57 * pinned VRAM objects which need to be saved.
 58 * @xe: xe device
 59 *
 60 * Should be called from PM notifier when preparing for s3/s4.
 61 *
 62 * Return: 0 on success, negative error code on error.
 63 */
 64int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe)
 65{
 66	int ret;
 67
 68	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
 69				    &xe->pinned.early.kernel_bo_present,
 70				    xe_bo_notifier_prepare_pinned);
 71	if (!ret)
 72		ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
 73					    &xe->pinned.late.kernel_bo_present,
 74					    xe_bo_notifier_prepare_pinned);
 75
 76	if (!ret)
 77		ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
 78					    &xe->pinned.late.external,
 79					    xe_bo_notifier_prepare_pinned);
 80
 81	return ret;
 82}
 83
 84/**
 85 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all
 86 * pinned VRAM objects which have been restored.
 87 * @xe: xe device
 88 *
 89 * Should be called from PM notifier after exiting s3/s4 (either on success or
 90 * failure).
 91 */
 92void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe)
 93{
 94	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
 95				    &xe->pinned.early.kernel_bo_present,
 96				    xe_bo_notifier_unprepare_pinned);
 97
 98	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
 99				    &xe->pinned.late.kernel_bo_present,
100				    xe_bo_notifier_unprepare_pinned);
101
102	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
103				    &xe->pinned.late.external,
104				    xe_bo_notifier_unprepare_pinned);
105}
106
107/**
108 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM
109 * @xe: xe device
110 *
111 * Evict non-pinned user BOs (via GPU).
112 *
113 * Evict == move VRAM BOs to temporary (typically system) memory.
114 */
115int xe_bo_evict_all_user(struct xe_device *xe)
116{
117	struct ttm_device *bdev = &xe->ttm;
118	u32 mem_type;
119	int ret;
120
121	/* User memory */
122	for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) {
123		struct ttm_resource_manager *man =
124			ttm_manager_type(bdev, mem_type);
125
126		/*
127		 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS
128		 * state since this state lives inside graphics stolen memory which doesn't survive
129		 * hibernation.
130		 *
131		 * This can be further improved by only evicting objects that we know have actually
132		 * used a compression enabled PAT index.
133		 */
134		if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe)))
135			continue;
136
137		if (man) {
138			ret = ttm_resource_manager_evict_all(bdev, man);
139			if (ret)
140				return ret;
141		}
142	}
143
144	return 0;
145}
146
147/**
148 * xe_bo_evict_all - evict all BOs from VRAM
149 * @xe: xe device
150 *
151 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next
152 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU.
153 * All eviction magic done via TTM calls.
154 *
155 * Evict == move VRAM BOs to temporary (typically system) memory.
156 *
157 * This function should be called before the device goes into a suspend state
158 * where the VRAM loses power.
159 */
160int xe_bo_evict_all(struct xe_device *xe)
161{
162	struct xe_tile *tile;
163	u8 id;
164	int ret;
165
166	ret = xe_bo_evict_all_user(xe);
167	if (ret)
168		return ret;
169
170	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
171				    &xe->pinned.late.external, xe_bo_evict_pinned);
172
173	if (!ret)
174		ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
175					    &xe->pinned.late.evicted, xe_bo_evict_pinned);
176
177	/*
178	 * Wait for all user BO to be evicted as those evictions depend on the
179	 * memory moved below.
180	 */
181	for_each_tile(tile, xe, id)
182		xe_tile_migrate_wait(tile);
183
184	if (ret)
185		return ret;
186
187	return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
188				     &xe->pinned.early.evicted,
189				     xe_bo_evict_pinned);
190}
191
192static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo)
193{
194	int ret;
195
196	ret = xe_bo_restore_pinned(bo);
197	if (ret)
198		return ret;
199
200	if (bo->flags & XE_BO_FLAG_GGTT) {
201		struct xe_tile *tile;
202		u8 id;
203
204		for_each_tile(tile, xe_bo_device(bo), id) {
205			if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile)))
206				continue;
207
208			xe_ggtt_map_bo_unlocked(tile->mem.ggtt, bo);
209		}
210	}
211
212	return 0;
213}
214
215/**
216 * xe_bo_restore_early - restore early phase kernel BOs to VRAM
217 *
218 * @xe: xe device
219 *
220 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All
221 * moves done via TTM calls.
222 *
223 * This function should be called early, before trying to init the GT, on device
224 * resume.
225 */
226int xe_bo_restore_early(struct xe_device *xe)
227{
228	return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted,
229				     &xe->pinned.early.kernel_bo_present,
230				     xe_bo_restore_and_map_ggtt);
231}
232
233/**
234 * xe_bo_restore_late - restore pinned late phase BOs
235 *
236 * @xe: xe device
237 *
238 * Move pinned user and kernel BOs which can use blitter from temporary
239 * (typically system) memory to VRAM. All moves done via TTM calls.
240 *
241 * This function should be called late, after GT init, on device resume.
242 */
243int xe_bo_restore_late(struct xe_device *xe)
244{
245	struct xe_tile *tile;
246	int ret, id;
247
248	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted,
249				    &xe->pinned.late.kernel_bo_present,
250				    xe_bo_restore_and_map_ggtt);
251
252	for_each_tile(tile, xe, id)
253		xe_tile_migrate_wait(tile);
254
255	if (ret)
256		return ret;
257
258	if (!IS_DGFX(xe))
259		return 0;
260
261	/* Pinned user memory in VRAM should be validated on resume */
262	ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
263				    &xe->pinned.late.external,
264				    xe_bo_restore_pinned);
265
266	/* Wait for restore to complete */
267	for_each_tile(tile, xe, id)
268		xe_tile_migrate_wait(tile);
269
270	return ret;
271}
272
273static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe)
274{
275	struct xe_tile *tile;
276	unsigned int id;
277
278	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
279				    &xe->pinned.late.external,
280				    xe_bo_dma_unmap_pinned);
281	for_each_tile(tile, xe, id)
282		xe_tile_migrate_wait(tile);
283}
284
285/**
286 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed
287 * @xe: The xe device.
288 *
289 * On pci_device removal we need to drop all dma mappings and move
290 * the data of exported bos out to system. This includes SVM bos and
291 * exported dma-buf bos. This is done by evicting all bos, but
292 * the evict placement in xe_evict_flags() is chosen such that all
293 * bos except those mentioned are purged, and thus their memory
294 * is released.
295 *
296 * For pinned bos, we're unmapping dma.
297 */
298void xe_bo_pci_dev_remove_all(struct xe_device *xe)
299{
300	unsigned int mem_type;
301
302	/*
303	 * Move pagemap bos and exported dma-buf to system, and
304	 * purge everything else.
305	 */
306	for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) {
307		struct ttm_resource_manager *man =
308			ttm_manager_type(&xe->ttm, mem_type);
309
310		if (man) {
311			int ret = ttm_resource_manager_evict_all(&xe->ttm, man);
312
313			drm_WARN_ON(&xe->drm, ret);
314		}
315	}
316
317	xe_bo_pci_dev_remove_pinned(xe);
318}
319
320static void xe_bo_pinned_fini(void *arg)
321{
322	struct xe_device *xe = arg;
323
324	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
325				    &xe->pinned.late.kernel_bo_present,
326				    xe_bo_dma_unmap_pinned);
327	(void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
328				    &xe->pinned.early.kernel_bo_present,
329				    xe_bo_dma_unmap_pinned);
330}
331
332/**
333 * xe_bo_pinned_init() - Initialize pinned bo tracking
334 * @xe: The xe device.
335 *
336 * Initializes the lists and locks required for pinned bo
337 * tracking and registers a callback to dma-unmap
338 * any remaining pinned bos on pci device removal.
339 *
340 * Return: %0 on success, negative error code on error.
341 */
342int xe_bo_pinned_init(struct xe_device *xe)
343{
344	spin_lock_init(&xe->pinned.lock);
345	INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present);
346	INIT_LIST_HEAD(&xe->pinned.early.evicted);
347	INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present);
348	INIT_LIST_HEAD(&xe->pinned.late.evicted);
349	INIT_LIST_HEAD(&xe->pinned.late.external);
350
351	return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe);
352}