i915_gem_pm.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include "gem/i915_gem_pm.h"
  7#include "gem/i915_gem_ttm_pm.h"
  8#include "gt/intel_gt.h"
  9#include "gt/intel_gt_pm.h"
 10#include "gt/intel_gt_requests.h"
 11
 12#include "i915_driver.h"
 13#include "i915_drv.h"
 14
 15#if IS_ENABLED(CONFIG_X86)
 16#include <asm/smp.h>
 17#else
 18#define wbinvd_on_all_cpus() \
 19	pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
 20#endif
 21
 22void i915_gem_suspend(struct drm_i915_private *i915)
 23{
 24	struct intel_gt *gt;
 25	unsigned int i;
 26
 27	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
 28
 29	intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0);
 30	/*
 31	 * On rare occasions, we've observed the fence completion triggers
 32	 * free_engines asynchronously via rcu_call. Ensure those are done.
 33	 * This path is only called on suspend, so it's an acceptable cost.
 34	 */
 35	rcu_barrier();
 36
 37	flush_workqueue(i915->wq);
 38
 39	/*
 40	 * We have to flush all the executing contexts to main memory so
 41	 * that they can saved in the hibernation image. To ensure the last
 42	 * context image is coherent, we have to switch away from it. That
 43	 * leaves the i915->kernel_context still active when
 44	 * we actually suspend, and its image in memory may not match the GPU
 45	 * state. Fortunately, the kernel_context is disposable and we do
 46	 * not rely on its state.
 47	 */
 48	for_each_gt(gt, i915, i)
 49		intel_gt_suspend_prepare(gt);
 50
 51	i915_gem_drain_freed_objects(i915);
 52}
 53
 54static int lmem_restore(struct drm_i915_private *i915, u32 flags)
 55{
 56	struct intel_memory_region *mr;
 57	int ret = 0, id;
 58
 59	for_each_memory_region(mr, i915, id) {
 60		if (mr->type == INTEL_MEMORY_LOCAL) {
 61			ret = i915_ttm_restore_region(mr, flags);
 62			if (ret)
 63				break;
 64		}
 65	}
 66
 67	return ret;
 68}
 69
 70static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
 71{
 72	struct intel_memory_region *mr;
 73	int ret = 0, id;
 74
 75	for_each_memory_region(mr, i915, id) {
 76		if (mr->type == INTEL_MEMORY_LOCAL) {
 77			ret = i915_ttm_backup_region(mr, flags);
 78			if (ret)
 79				break;
 80		}
 81	}
 82
 83	return ret;
 84}
 85
 86static void lmem_recover(struct drm_i915_private *i915)
 87{
 88	struct intel_memory_region *mr;
 89	int id;
 90
 91	for_each_memory_region(mr, i915, id)
 92		if (mr->type == INTEL_MEMORY_LOCAL)
 93			i915_ttm_recover_region(mr);
 94}
 95
 96int i915_gem_backup_suspend(struct drm_i915_private *i915)
 97{
 98	int ret;
 99
100	/* Opportunistically try to evict unpinned objects */
101	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
102	if (ret)
103		goto out_recover;
104
105	i915_gem_suspend(i915);
106
107	/*
108	 * More objects may have become unpinned as requests were
109	 * retired. Now try to evict again. The gt may be wedged here
110	 * in which case we automatically fall back to memcpy.
111	 * We allow also backing up pinned objects that have not been
112	 * marked for early recover, and that may contain, for example,
113	 * page-tables for the migrate context.
114	 */
115	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
116			   I915_TTM_BACKUP_PINNED);
117	if (ret)
118		goto out_recover;
119
120	/*
121	 * Remaining objects are backed up using memcpy once we've stopped
122	 * using the migrate context.
123	 */
124	ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
125	if (ret)
126		goto out_recover;
127
128	return 0;
129
130out_recover:
131	lmem_recover(i915);
132
133	return ret;
134}
135
136void i915_gem_suspend_late(struct drm_i915_private *i915)
137{
138	struct drm_i915_gem_object *obj;
139	struct list_head *phases[] = {
140		&i915->mm.shrink_list,
141		&i915->mm.purge_list,
142		NULL
143	}, **phase;
144	struct intel_gt *gt;
145	unsigned long flags;
146	unsigned int i;
147	bool flush = false;
148
149	/*
150	 * Neither the BIOS, ourselves or any other kernel
151	 * expects the system to be in execlists mode on startup,
152	 * so we need to reset the GPU back to legacy mode. And the only
153	 * known way to disable logical contexts is through a GPU reset.
154	 *
155	 * So in order to leave the system in a known default configuration,
156	 * always reset the GPU upon unload and suspend. Afterwards we then
157	 * clean up the GEM state tracking, flushing off the requests and
158	 * leaving the system in a known idle state.
159	 *
160	 * Note that is of the upmost importance that the GPU is idle and
161	 * all stray writes are flushed *before* we dismantle the backing
162	 * storage for the pinned objects.
163	 *
164	 * However, since we are uncertain that resetting the GPU on older
165	 * machines is a good idea, we don't - just in case it leaves the
166	 * machine in an unusable condition.
167	 */
168
169	/* Like i915_gem_suspend, flush tasks staged from fence triggers */
170	rcu_barrier();
171
172	for_each_gt(gt, i915, i)
173		intel_gt_suspend_late(gt);
174
175	spin_lock_irqsave(&i915->mm.obj_lock, flags);
176	for (phase = phases; *phase; phase++) {
177		list_for_each_entry(obj, *phase, mm.link) {
178			if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
179				flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
180			__start_cpu_write(obj); /* presume auto-hibernate */
181		}
182	}
183	spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
184	if (flush)
185		wbinvd_on_all_cpus();
186}
187
188int i915_gem_freeze(struct drm_i915_private *i915)
189{
190	/* Discard all purgeable objects, let userspace recover those as
191	 * required after resuming.
192	 */
193	i915_gem_shrink_all(i915);
194
195	return 0;
196}
197
198int i915_gem_freeze_late(struct drm_i915_private *i915)
199{
200	struct drm_i915_gem_object *obj;
201	intel_wakeref_t wakeref;
202
203	/*
204	 * Called just before we write the hibernation image.
205	 *
206	 * We need to update the domain tracking to reflect that the CPU
207	 * will be accessing all the pages to create and restore from the
208	 * hibernation, and so upon restoration those pages will be in the
209	 * CPU domain.
210	 *
211	 * To make sure the hibernation image contains the latest state,
212	 * we update that state just before writing out the image.
213	 *
214	 * To try and reduce the hibernation image, we manually shrink
215	 * the objects as well, see i915_gem_freeze()
216	 */
217
218	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
219		i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
220	i915_gem_drain_freed_objects(i915);
221
222	wbinvd_on_all_cpus();
223	list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
224		__start_cpu_write(obj);
225
226	return 0;
227}
228
229void i915_gem_resume(struct drm_i915_private *i915)
230{
231	struct intel_gt *gt;
232	int ret, i, j;
233
234	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
235
236	ret = lmem_restore(i915, 0);
237	GEM_WARN_ON(ret);
238
239	/*
240	 * As we didn't flush the kernel context before suspend, we cannot
241	 * guarantee that the context image is complete. So let's just reset
242	 * it and start again.
243	 */
244	for_each_gt(gt, i915, i)
245		if (intel_gt_resume(gt))
246			goto err_wedged;
247
248	ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
249	GEM_WARN_ON(ret);
250
251	return;
252
253err_wedged:
254	for_each_gt(gt, i915, j) {
255		if (!intel_gt_is_wedged(gt)) {
256			dev_err(i915->drm.dev,
257				"Failed to re-initialize GPU[%u], declaring it wedged!\n",
258				j);
259			intel_gt_set_wedged(gt);
260		}
261
262		if (j == i)
263			break;
264	}
265}