vlv_suspend.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2020 Intel Corporation
  4 */
  5
  6#include <linux/string_helpers.h>
  7#include <linux/kernel.h>
  8
  9#include <drm/drm_print.h>
 10
 11#include "gt/intel_gt_regs.h"
 12
 13#include "i915_drv.h"
 14#include "i915_reg.h"
 15#include "i915_trace.h"
 16#include "i915_utils.h"
 17#include "i915_wait_util.h"
 18#include "intel_clock_gating.h"
 19#include "intel_uncore_trace.h"
 20#include "vlv_suspend.h"
 21
 22struct vlv_s0ix_state {
 23	/* GAM */
 24	u32 wr_watermark;
 25	u32 gfx_prio_ctrl;
 26	u32 arb_mode;
 27	u32 gfx_pend_tlb0;
 28	u32 gfx_pend_tlb1;
 29	u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
 30	u32 media_max_req_count;
 31	u32 gfx_max_req_count;
 32	u32 render_hwsp;
 33	u32 ecochk;
 34	u32 bsd_hwsp;
 35	u32 blt_hwsp;
 36	u32 tlb_rd_addr;
 37
 38	/* MBC */
 39	u32 g3dctl;
 40	u32 gsckgctl;
 41	u32 mbctl;
 42
 43	/* GCP */
 44	u32 ucgctl1;
 45	u32 ucgctl3;
 46	u32 rcgctl1;
 47	u32 rcgctl2;
 48	u32 rstctl;
 49	u32 misccpctl;
 50
 51	/* GPM */
 52	u32 gfxpause;
 53	u32 rpdeuhwtc;
 54	u32 rpdeuc;
 55	u32 ecobus;
 56	u32 pwrdwnupctl;
 57	u32 rp_down_timeout;
 58	u32 rp_deucsw;
 59	u32 rcubmabdtmr;
 60	u32 rcedata;
 61	u32 spare2gh;
 62
 63	/* Display 1 CZ domain */
 64	u32 gt_imr;
 65	u32 gt_ier;
 66	u32 pm_imr;
 67	u32 pm_ier;
 68	u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
 69
 70	/* GT SA CZ domain */
 71	u32 tilectl;
 72	u32 gt_fifoctl;
 73	u32 gtlc_wake_ctrl;
 74	u32 gtlc_survive;
 75	u32 pmwgicz;
 76
 77	/* Display 2 CZ domain */
 78	u32 gu_ctl0;
 79	u32 gu_ctl1;
 80	u32 pcbr;
 81	u32 clock_gate_dis2;
 82};
 83
 84/*
 85 * Save all Gunit registers that may be lost after a D3 and a subsequent
 86 * S0i[R123] transition. The list of registers needing a save/restore is
 87 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
 88 * registers in the following way:
 89 * - Driver: saved/restored by the driver
 90 * - Punit : saved/restored by the Punit firmware
 91 * - No, w/o marking: no need to save/restore, since the register is R/O or
 92 *                    used internally by the HW in a way that doesn't depend
 93 *                    keeping the content across a suspend/resume.
 94 * - Debug : used for debugging
 95 *
 96 * We save/restore all registers marked with 'Driver', with the following
 97 * exceptions:
 98 * - Registers out of use, including also registers marked with 'Debug'.
 99 *   These have no effect on the driver's operation, so we don't save/restore
100 *   them to reduce the overhead.
101 * - Registers that are fully setup by an initialization function called from
102 *   the resume path. For example many clock gating and RPS/RC6 registers.
103 * - Registers that provide the right functionality with their reset defaults.
104 *
105 * TODO: Except for registers that based on the above 3 criteria can be safely
106 * ignored, we save/restore all others, practically treating the HW context as
107 * a black-box for the driver. Further investigation is needed to reduce the
108 * saved/restored registers even further, by following the same 3 criteria.
109 */
110static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915)
111{
112	struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
113	struct intel_uncore *uncore = &i915->uncore;
114	int i;
115
116	if (!s)
117		return;
118
119	/* GAM 0x4000-0x4770 */
120	s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK);
121	s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL);
122	s->arb_mode = intel_uncore_read(uncore, ARB_MODE);
123	s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0);
124	s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1);
125
126	for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
127		s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i));
128
129	s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT);
130	s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT);
131
132	s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7);
133	s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
134	s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7);
135	s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7);
136
137	s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR);
138
139	/* MBC 0x9024-0x91D0, 0x8500 */
140	s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL);
141	s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL);
142	s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL);
143
144	/* GCP 0x9400-0x9424, 0x8100-0x810C */
145	s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1);
146	s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3);
147	s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1);
148	s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2);
149	s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL);
150	s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL);
151
152	/* GPM 0xA000-0xAA84, 0x8000-0x80FC */
153	s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE);
154	s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC);
155	s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC);
156	s->ecobus = intel_uncore_read(uncore, ECOBUS);
157	s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL);
158	s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT);
159	s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW);
160	s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR);
161	s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA);
162	s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H);
163
164	/* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
165	s->gt_imr = intel_uncore_read(uncore, GTIMR);
166	s->gt_ier = intel_uncore_read(uncore, GTIER);
167	s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
168	s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
169
170	for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
171		s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i));
172
173	/* GT SA CZ domain, 0x100000-0x138124 */
174	s->tilectl = intel_uncore_read(uncore, TILECTL);
175	s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL);
176	s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
177	s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
178	s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ);
179
180	/* Gunit-Display CZ domain, 0x182028-0x1821CF */
181	s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0);
182	s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1);
183	s->pcbr = intel_uncore_read(uncore, VLV_PCBR);
184	s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2);
185
186	/*
187	 * Not saving any of:
188	 * DFT,		0x9800-0x9EC0
189	 * SARB,	0xB000-0xB1FC
190	 * GAC,		0x5208-0x524C, 0x14000-0x14C000
191	 * PCI CFG
192	 */
193}
194
195static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915)
196{
197	struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
198	struct intel_uncore *uncore = &i915->uncore;
199	int i;
200
201	if (!s)
202		return;
203
204	/* GAM 0x4000-0x4770 */
205	intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark);
206	intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
207	intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16));
208	intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
209	intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
210
211	for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
212		intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]);
213
214	intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
215	intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
216
217	intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp);
218	intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk);
219	intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp);
220	intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp);
221
222	intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
223
224	/* MBC 0x9024-0x91D0, 0x8500 */
225	intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl);
226	intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl);
227	intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl);
228
229	/* GCP 0x9400-0x9424, 0x8100-0x810C */
230	intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1);
231	intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3);
232	intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1);
233	intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2);
234	intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl);
235	intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl);
236
237	/* GPM 0xA000-0xAA84, 0x8000-0x80FC */
238	intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause);
239	intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc);
240	intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc);
241	intel_uncore_write(uncore, ECOBUS, s->ecobus);
242	intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl);
243	intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout);
244	intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw);
245	intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr);
246	intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata);
247	intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh);
248
249	/* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
250	intel_uncore_write(uncore, GTIMR, s->gt_imr);
251	intel_uncore_write(uncore, GTIER, s->gt_ier);
252	intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr);
253	intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier);
254
255	for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
256		intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
257
258	/* GT SA CZ domain, 0x100000-0x138124 */
259	intel_uncore_write(uncore, TILECTL, s->tilectl);
260	intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl);
261	/*
262	 * Preserve the GT allow wake and GFX force clock bit, they are not
263	 * be restored, as they are used to control the s0ix suspend/resume
264	 * sequence by the caller.
265	 */
266	intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, ~VLV_GTLC_ALLOWWAKEREQ,
267			 s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ);
268
269	intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, ~VLV_GFX_CLK_FORCE_ON_BIT,
270			 s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT);
271
272	intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz);
273
274	/* Gunit-Display CZ domain, 0x182028-0x1821CF */
275	intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0);
276	intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1);
277	intel_uncore_write(uncore, VLV_PCBR, s->pcbr);
278	intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
279}
280
281static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
282				  u32 mask, u32 val)
283{
284	i915_reg_t reg = VLV_GTLC_PW_STATUS;
285	u32 reg_value;
286	int ret;
287
288	/* The HW does not like us polling for PW_STATUS frequently, so
289	 * use the sleeping loop rather than risk the busy spin within
290	 * intel_wait_for_register().
291	 *
292	 * Transitioning between RC6 states should be at most 2ms (see
293	 * valleyview_enable_rps) so use a 3ms timeout.
294	 */
295	ret = wait_for(((reg_value =
296			 intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
297		       == val, 3);
298
299	/* just trace the final value */
300	trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
301
302	return ret;
303}
304
305static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on)
306{
307	struct intel_uncore *uncore = &i915->uncore;
308	int err;
309
310	intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, VLV_GFX_CLK_FORCE_ON_BIT,
311			 force_on ? VLV_GFX_CLK_FORCE_ON_BIT : 0);
312
313	if (!force_on)
314		return 0;
315
316	err = intel_wait_for_register(uncore,
317				      VLV_GTLC_SURVIVABILITY_REG,
318				      VLV_GFX_CLK_STATUS_BIT,
319				      VLV_GFX_CLK_STATUS_BIT,
320				      20);
321	if (err)
322		drm_err(&i915->drm,
323			"timeout waiting for GFX clock force-on (%08x)\n",
324			intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG));
325
326	return err;
327}
328
329static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow)
330{
331	struct intel_uncore *uncore = &i915->uncore;
332	u32 mask;
333	u32 val;
334	int err;
335
336	intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ,
337			 allow ? VLV_GTLC_ALLOWWAKEREQ : 0);
338	intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL);
339
340	mask = VLV_GTLC_ALLOWWAKEACK;
341	val = allow ? mask : 0;
342
343	err = vlv_wait_for_pw_status(i915, mask, val);
344	if (err)
345		drm_err(&i915->drm, "timeout disabling GT waking\n");
346
347	return err;
348}
349
350static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
351				  bool wait_for_on)
352{
353	u32 mask;
354	u32 val;
355
356	mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
357	val = wait_for_on ? mask : 0;
358
359	/*
360	 * RC6 transitioning can be delayed up to 2 msec (see
361	 * valleyview_enable_rps), use 3 msec for safety.
362	 *
363	 * This can fail to turn off the rc6 if the GPU is stuck after a failed
364	 * reset and we are trying to force the machine to sleep.
365	 */
366	if (vlv_wait_for_pw_status(dev_priv, mask, val))
367		drm_dbg(&dev_priv->drm,
368			"timeout waiting for GT wells to go %s\n",
369			str_on_off(wait_for_on));
370}
371
372static void vlv_check_no_gt_access(struct drm_i915_private *i915)
373{
374	struct intel_uncore *uncore = &i915->uncore;
375
376	if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
377		return;
378
379	drm_dbg(&i915->drm, "GT register access while GT waking disabled\n");
380	intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
381}
382
383int vlv_suspend_complete(struct drm_i915_private *dev_priv)
384{
385	u32 mask;
386	int err;
387
388	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
389		return 0;
390
391	/*
392	 * Bspec defines the following GT well on flags as debug only, so
393	 * don't treat them as hard failures.
394	 */
395	vlv_wait_for_gt_wells(dev_priv, false);
396
397	mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
398	drm_WARN_ON(&dev_priv->drm,
399		    (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask);
400
401	vlv_check_no_gt_access(dev_priv);
402
403	err = vlv_force_gfx_clock(dev_priv, true);
404	if (err)
405		goto err1;
406
407	err = vlv_allow_gt_wake(dev_priv, false);
408	if (err)
409		goto err2;
410
411	vlv_save_gunit_s0ix_state(dev_priv);
412
413	err = vlv_force_gfx_clock(dev_priv, false);
414	if (err)
415		goto err2;
416
417	return 0;
418
419err2:
420	/* For safety always re-enable waking and disable gfx clock forcing */
421	vlv_allow_gt_wake(dev_priv, true);
422err1:
423	vlv_force_gfx_clock(dev_priv, false);
424
425	return err;
426}
427
428int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
429{
430	int err;
431	int ret;
432
433	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
434		return 0;
435
436	/*
437	 * If any of the steps fail just try to continue, that's the best we
438	 * can do at this point. Return the first error code (which will also
439	 * leave RPM permanently disabled).
440	 */
441	ret = vlv_force_gfx_clock(dev_priv, true);
442
443	vlv_restore_gunit_s0ix_state(dev_priv);
444
445	err = vlv_allow_gt_wake(dev_priv, true);
446	if (!ret)
447		ret = err;
448
449	err = vlv_force_gfx_clock(dev_priv, false);
450	if (!ret)
451		ret = err;
452
453	vlv_check_no_gt_access(dev_priv);
454
455	if (rpm_resume)
456		intel_clock_gating_init(dev_priv);
457
458	return ret;
459}
460
461int vlv_suspend_init(struct drm_i915_private *i915)
462{
463	if (!IS_VALLEYVIEW(i915))
464		return 0;
465
466	/* we write all the values in the struct, so no need to zero it out */
467	i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
468				       GFP_KERNEL);
469	if (!i915->vlv_s0ix_state)
470		return -ENOMEM;
471
472	return 0;
473}
474
475void vlv_suspend_cleanup(struct drm_i915_private *i915)
476{
477	if (!i915->vlv_s0ix_state)
478		return;
479
480	kfree(i915->vlv_s0ix_state);
481	i915->vlv_s0ix_state = NULL;
482}