i915_gem_ttm_pm.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021 Intel Corporation
  4 */
  5
  6#include <drm/drm_print.h>
  7#include <drm/ttm/ttm_placement.h>
  8#include <drm/ttm/ttm_tt.h>
  9
 10#include "i915_drv.h"
 11#include "intel_memory_region.h"
 12#include "intel_region_ttm.h"
 13
 14#include "gem/i915_gem_region.h"
 15#include "gem/i915_gem_ttm.h"
 16#include "gem/i915_gem_ttm_move.h"
 17#include "gem/i915_gem_ttm_pm.h"
 18
 19/**
 20 * i915_ttm_backup_free - Free any backup attached to this object
 21 * @obj: The object whose backup is to be freed.
 22 */
 23void i915_ttm_backup_free(struct drm_i915_gem_object *obj)
 24{
 25	if (obj->ttm.backup) {
 26		i915_gem_object_put(obj->ttm.backup);
 27		obj->ttm.backup = NULL;
 28	}
 29}
 30
 31/**
 32 * struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore
 33 * @base: The i915_gem_apply_to_region we derive from.
 34 * @allow_gpu: Whether using the gpu blitter is allowed.
 35 * @backup_pinned: On backup, backup also pinned objects.
 36 */
 37struct i915_gem_ttm_pm_apply {
 38	struct i915_gem_apply_to_region base;
 39	bool allow_gpu : 1;
 40	bool backup_pinned : 1;
 41};
 42
 43static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
 44			   struct drm_i915_gem_object *obj)
 45{
 46	struct i915_gem_ttm_pm_apply *pm_apply =
 47		container_of(apply, typeof(*pm_apply), base);
 48	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
 49	struct ttm_buffer_object *backup_bo;
 50	struct drm_i915_private *i915 =
 51		container_of(bo->bdev, typeof(*i915), bdev);
 52	struct drm_i915_gem_object *backup;
 53	struct ttm_operation_ctx ctx = {};
 54	unsigned int flags;
 55	int err = 0;
 56
 57	if (!i915_ttm_cpu_maps_iomem(bo->resource) || obj->ttm.backup)
 58		return 0;
 59
 60	if (pm_apply->allow_gpu && i915_gem_object_evictable(obj))
 61		return ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx);
 62
 63	if (!pm_apply->backup_pinned ||
 64	    (pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_PM_EARLY)))
 65		return 0;
 66
 67	if (obj->flags & I915_BO_ALLOC_PM_VOLATILE)
 68		return 0;
 69
 70	/*
 71	 * It seems that we might have some framebuffers still pinned at this
 72	 * stage, but for such objects we might also need to deal with the CCS
 73	 * aux state. Make sure we force the save/restore of the CCS state,
 74	 * otherwise we might observe display corruption, when returning from
 75	 * suspend.
 76	 */
 77	flags = 0;
 78	if (i915_gem_object_needs_ccs_pages(obj)) {
 79		WARN_ON_ONCE(!i915_gem_object_is_framebuffer(obj));
 80		WARN_ON_ONCE(!pm_apply->allow_gpu);
 81
 82		flags = I915_BO_ALLOC_CCS_AUX;
 83	}
 84	backup = i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
 85					       obj->base.size, 0, flags);
 86	if (IS_ERR(backup))
 87		return PTR_ERR(backup);
 88
 89	err = i915_gem_object_lock(backup, apply->ww);
 90	if (err)
 91		goto out_no_lock;
 92
 93	backup_bo = i915_gem_to_ttm(backup);
 94	err = ttm_bo_populate(backup_bo, &ctx);
 95	if (err)
 96		goto out_no_populate;
 97
 98	err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false);
 99	if (err) {
100		drm_err(&i915->drm,
101			"Unable to copy from device to system memory, err:%pe\n",
102			ERR_PTR(err));
103		goto out_no_populate;
104	}
105	ttm_bo_wait_ctx(backup_bo, &ctx);
106
107	obj->ttm.backup = backup;
108	return 0;
109
110out_no_populate:
111	i915_gem_ww_unlock_single(backup);
112out_no_lock:
113	i915_gem_object_put(backup);
114
115	return err;
116}
117
118static int i915_ttm_recover(struct i915_gem_apply_to_region *apply,
119			    struct drm_i915_gem_object *obj)
120{
121	i915_ttm_backup_free(obj);
122	return 0;
123}
124
125/**
126 * i915_ttm_recover_region - Free the backup of all objects of a region
127 * @mr: The memory region
128 *
129 * Checks all objects of a region if there is backup attached and if so
130 * frees that backup. Typically this is called to recover after a partially
131 * performed backup.
132 */
133void i915_ttm_recover_region(struct intel_memory_region *mr)
134{
135	static const struct i915_gem_apply_to_region_ops recover_ops = {
136		.process_obj = i915_ttm_recover,
137	};
138	struct i915_gem_apply_to_region apply = {.ops = &recover_ops};
139	int ret;
140
141	ret = i915_gem_process_region(mr, &apply);
142	GEM_WARN_ON(ret);
143}
144
145/**
146 * i915_ttm_backup_region - Back up all objects of a region to smem.
147 * @mr: The memory region
148 * @flags: TTM backup flags
149 *
150 * Loops over all objects of a region and either evicts them if they are
151 * evictable or backs them up using a backup object if they are pinned.
152 *
153 * Return: Zero on success. Negative error code on error.
154 */
155int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags)
156{
157	static const struct i915_gem_apply_to_region_ops backup_ops = {
158		.process_obj = i915_ttm_backup,
159	};
160	struct i915_gem_ttm_pm_apply pm_apply = {
161		.base = {.ops = &backup_ops},
162		.allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
163		.backup_pinned = flags & I915_TTM_BACKUP_PINNED,
164	};
165
166	return i915_gem_process_region(mr, &pm_apply.base);
167}
168
169static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
170			    struct drm_i915_gem_object *obj)
171{
172	struct i915_gem_ttm_pm_apply *pm_apply =
173		container_of(apply, typeof(*pm_apply), base);
174	struct drm_i915_gem_object *backup = obj->ttm.backup;
175	struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(backup);
176	struct ttm_operation_ctx ctx = {};
177	int err;
178
179	if (!backup)
180		return 0;
181
182	if (!pm_apply->allow_gpu && !(obj->flags & I915_BO_ALLOC_PM_EARLY))
183		return 0;
184
185	err = i915_gem_object_lock(backup, apply->ww);
186	if (err)
187		return err;
188
189	/* Content may have been swapped. */
190	if (!backup_bo->resource)
191		err = ttm_bo_validate(backup_bo, i915_ttm_sys_placement(), &ctx);
192	if (!err)
193		err = ttm_bo_populate(backup_bo, &ctx);
194	if (!err) {
195		err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu,
196					    false);
197		GEM_WARN_ON(err);
198		ttm_bo_wait_ctx(backup_bo, &ctx);
199
200		obj->ttm.backup = NULL;
201		err = 0;
202	}
203
204	i915_gem_ww_unlock_single(backup);
205
206	if (!err)
207		i915_gem_object_put(backup);
208
209	return err;
210}
211
212/**
213 * i915_ttm_restore_region - Restore backed-up objects of a region from smem.
214 * @mr: The memory region
215 * @flags: TTM backup flags
216 *
217 * Loops over all objects of a region and if they are backed-up, restores
218 * them from smem.
219 *
220 * Return: Zero on success. Negative error code on error.
221 */
222int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags)
223{
224	static const struct i915_gem_apply_to_region_ops restore_ops = {
225		.process_obj = i915_ttm_restore,
226	};
227	struct i915_gem_ttm_pm_apply pm_apply = {
228		.base = {.ops = &restore_ops},
229		.allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
230	};
231
232	return i915_gem_process_region(mr, &pm_apply.base);
233}