intel_clock_gating.c

  1/*
  2 * Copyright © 2012 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 * Authors:
 24 *    Eugeni Dodonov <eugeni.dodonov@intel.com>
 25 *
 26 */
 27
 28#include <drm/drm_print.h>
 29
 30#include "display/i9xx_plane_regs.h"
 31#include "display/intel_display.h"
 32#include "display/intel_display_core.h"
 33
 34#include "gt/intel_engine_regs.h"
 35#include "gt/intel_gt.h"
 36#include "gt/intel_gt_mcr.h"
 37#include "gt/intel_gt_regs.h"
 38
 39#include "i915_drv.h"
 40#include "i915_reg.h"
 41#include "intel_clock_gating.h"
 42#include "intel_mchbar_regs.h"
 43#include "vlv_iosf_sb.h"
 44
 45struct drm_i915_clock_gating_funcs {
 46	void (*init_clock_gating)(struct drm_i915_private *i915);
 47};
 48
 49static void gen9_init_clock_gating(struct drm_i915_private *i915)
 50{
 51	if (HAS_LLC(i915)) {
 52		/*
 53		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
 54		 * Display WA #0390: skl,kbl
 55		 *
 56		 * Must match Sampler, Pixel Back End, and Media. See
 57		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
 58		 */
 59		intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
 60	}
 61
 62	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
 63	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
 64
 65	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
 66	intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
 67
 68	/*
 69	 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
 70	 * Display WA #0859: skl,bxt,kbl,glk,cfl
 71	 */
 72	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
 73}
 74
 75static void bxt_init_clock_gating(struct drm_i915_private *i915)
 76{
 77	gen9_init_clock_gating(i915);
 78
 79	/* WaDisableSDEUnitClockGating:bxt */
 80	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 81
 82	/*
 83	 * FIXME:
 84	 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 85	 */
 86	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 87
 88	/*
 89	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
 90	 * to stay fully on.
 91	 */
 92	intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
 93			   intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
 94			   PWM1_GATING_DIS | PWM2_GATING_DIS);
 95
 96	/*
 97	 * Lower the display internal timeout.
 98	 * This is needed to avoid any hard hangs when DSI port PLL
 99	 * is off and a MMIO access is attempted by any privilege
100	 * application, using batch buffers or any other means.
101	 */
102	intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
103
104	/*
105	 * WaFbcTurnOffFbcWatermark:bxt
106	 * Display WA #0562: bxt
107	 */
108	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
109}
110
111static void glk_init_clock_gating(struct drm_i915_private *i915)
112{
113	gen9_init_clock_gating(i915);
114
115	/*
116	 * WaDisablePWMClockGating:glk
117	 * Backlight PWM may stop in the asserted state, causing backlight
118	 * to stay fully on.
119	 */
120	intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
121			   intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
122			   PWM1_GATING_DIS | PWM2_GATING_DIS);
123}
124
125static void ibx_init_clock_gating(struct drm_i915_private *i915)
126{
127	/*
128	 * On Ibex Peak and Cougar Point, we need to disable clock
129	 * gating for the panel power sequencer or it will fail to
130	 * start up when no ports are active.
131	 */
132	intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
133}
134
135static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
136{
137	struct intel_display *display = dev_priv->display;
138	enum pipe pipe;
139
140	for_each_pipe(display, pipe) {
141		intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(display, pipe),
142				 0, DISP_TRICKLE_FEED_DISABLE);
143
144		intel_uncore_rmw(&dev_priv->uncore, DSPSURF(display, pipe),
145				 0, 0);
146		intel_uncore_posting_read(&dev_priv->uncore,
147					  DSPSURF(display, pipe));
148	}
149}
150
151static void ilk_init_clock_gating(struct drm_i915_private *i915)
152{
153	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
154
155	/*
156	 * Required for FBC
157	 * WaFbcDisableDpfcClockGating:ilk
158	 */
159	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
160		   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
161		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
162
163	intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
164			   MARIUNIT_CLOCK_GATE_DISABLE |
165			   SVSMUNIT_CLOCK_GATE_DISABLE);
166	intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
167			   VFMUNIT_CLOCK_GATE_DISABLE);
168
169	/*
170	 * According to the spec the following bits should be set in
171	 * order to enable memory self-refresh
172	 * The bit 22/21 of 0x42004
173	 * The bit 5 of 0x42020
174	 * The bit 15 of 0x45000
175	 */
176	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
177			   (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
178			    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
179	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
180	intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
181			   (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
182			    DISP_FBC_WM_DIS));
183
184	/*
185	 * Based on the document from hardware guys the following bits
186	 * should be set unconditionally in order to enable FBC.
187	 * The bit 22 of 0x42000
188	 * The bit 22 of 0x42004
189	 * The bit 7,8,9 of 0x42020.
190	 */
191	if (IS_IRONLAKE_M(i915)) {
192		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
193		intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
194		intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
195	}
196
197	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
198
199	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
200
201	g4x_disable_trickle_feed(i915);
202
203	ibx_init_clock_gating(i915);
204}
205
206static void cpt_init_clock_gating(struct drm_i915_private *i915)
207{
208	struct intel_display *display = i915->display;
209	enum pipe pipe;
210	u32 val;
211
212	/*
213	 * On Ibex Peak and Cougar Point, we need to disable clock
214	 * gating for the panel power sequencer or it will fail to
215	 * start up when no ports are active.
216	 */
217	intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
218			   PCH_DPLUNIT_CLOCK_GATE_DISABLE |
219			   PCH_CPUNIT_CLOCK_GATE_DISABLE);
220	intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
221	/* The below fixes the weird display corruption, a few pixels shifted
222	 * downward, on (only) LVDS of some HP laptops with IVY.
223	 */
224	for_each_pipe(display, pipe) {
225		val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
226		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
227		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
228		if (display->vbt.fdi_rx_polarity_inverted)
229			val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
230		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
231		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
232		intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
233	}
234	/* WADP0ClockGatingDisable */
235	for_each_pipe(display, pipe) {
236		intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
237				   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
238	}
239}
240
241static void gen6_check_mch_setup(struct drm_i915_private *i915)
242{
243	u32 tmp;
244
245	tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
246	if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
247		drm_dbg_kms(&i915->drm,
248			    "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
249			    tmp);
250}
251
252static void gen6_init_clock_gating(struct drm_i915_private *i915)
253{
254	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
255
256	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
257
258	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
259
260	intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
261			   intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
262			   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
263			   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
264
265	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
266	 * gating disable must be set.  Failure to set it results in
267	 * flickering pixels due to Z write ordering failures after
268	 * some amount of runtime in the Mesa "fire" demo, and Unigine
269	 * Sanctuary and Tropics, and apparently anything else with
270	 * alpha test or pixel discard.
271	 *
272	 * According to the spec, bit 11 (RCCUNIT) must also be set,
273	 * but we didn't debug actual testcases to find it out.
274	 *
275	 * WaDisableRCCUnitClockGating:snb
276	 * WaDisableRCPBUnitClockGating:snb
277	 */
278	intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
279			   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
280			   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
281
282	/*
283	 * According to the spec the following bits should be
284	 * set in order to enable memory self-refresh and fbc:
285	 * The bit21 and bit22 of 0x42000
286	 * The bit21 and bit22 of 0x42004
287	 * The bit5 and bit7 of 0x42020
288	 * The bit14 of 0x70180
289	 * The bit14 of 0x71180
290	 *
291	 * WaFbcAsynchFlipDisableFbcQueue:snb
292	 */
293	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
294			   intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
295			   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
296	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
297			   intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
298			   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
299	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
300			   intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
301			   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
302			   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
303
304	g4x_disable_trickle_feed(i915);
305
306	cpt_init_clock_gating(i915);
307
308	gen6_check_mch_setup(i915);
309}
310
311static void lpt_init_clock_gating(struct drm_i915_private *i915)
312{
313	struct intel_display *display = i915->display;
314
315	/*
316	 * TODO: this bit should only be enabled when really needed, then
317	 * disabled when not needed anymore in order to save power.
318	 */
319	if (HAS_PCH_LPT_LP(display))
320		intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
321				 0, PCH_LP_PARTITION_LEVEL_DISABLE);
322
323	/* WADPOClockGatingDisable:hsw */
324	intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
325			 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
326}
327
328static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
329				   int general_prio_credits,
330				   int high_prio_credits)
331{
332	u32 misccpctl;
333	u32 val;
334
335	/* WaTempDisableDOPClkGating:bdw */
336	misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
337				     GEN7_DOP_CLOCK_GATE_ENABLE, 0);
338
339	val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
340	val &= ~L3_PRIO_CREDITS_MASK;
341	val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
342	val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
343	intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
344
345	/*
346	 * Wait at least 100 clocks before re-enabling clock gating.
347	 * See the definition of L3SQCREG1 in BSpec.
348	 */
349	intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
350	udelay(1);
351	intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
352}
353
354static void dg2_init_clock_gating(struct drm_i915_private *i915)
355{
356	/* Wa_22010954014:dg2 */
357	intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
358			 SGSI_SIDECLK_DIS);
359}
360
361static void cnp_init_clock_gating(struct drm_i915_private *i915)
362{
363	struct intel_display *display = i915->display;
364
365	if (!HAS_PCH_CNP(display))
366		return;
367
368	/* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
369	intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
370}
371
372static void cfl_init_clock_gating(struct drm_i915_private *i915)
373{
374	cnp_init_clock_gating(i915);
375	gen9_init_clock_gating(i915);
376
377	/* WAC6entrylatency:cfl */
378	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
379
380	/*
381	 * WaFbcTurnOffFbcWatermark:cfl
382	 * Display WA #0562: cfl
383	 */
384	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
385}
386
387static void kbl_init_clock_gating(struct drm_i915_private *i915)
388{
389	gen9_init_clock_gating(i915);
390
391	/* WAC6entrylatency:kbl */
392	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
393
394	/* WaDisableSDEUnitClockGating:kbl */
395	if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
396		intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
397				 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
398
399	/* WaDisableGamClockGating:kbl */
400	if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
401		intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
402				 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
403
404	/*
405	 * WaFbcTurnOffFbcWatermark:kbl
406	 * Display WA #0562: kbl
407	 */
408	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
409}
410
411static void skl_init_clock_gating(struct drm_i915_private *i915)
412{
413	gen9_init_clock_gating(i915);
414
415	/* WaDisableDopClockGating:skl */
416	intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
417			 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
418
419	/* WAC6entrylatency:skl */
420	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
421
422	/*
423	 * WaFbcTurnOffFbcWatermark:skl
424	 * Display WA #0562: skl
425	 */
426	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
427}
428
429static void bdw_init_clock_gating(struct drm_i915_private *i915)
430{
431	struct intel_display *display = i915->display;
432	enum pipe pipe;
433
434	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
435	intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
436
437	/* WaSwitchSolVfFArbitrationPriority:bdw */
438	intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
439
440	/* WaPsrDPAMaskVBlankInSRD:bdw */
441	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
442
443	for_each_pipe(display, pipe) {
444		/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
445		intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
446				 0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
447	}
448
449	/* WaVSRefCountFullforceMissDisable:bdw */
450	/* WaDSRefCountFullforceMissDisable:bdw */
451	intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
452			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
453
454	intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
455			   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
456
457	/* WaDisableSDEUnitClockGating:bdw */
458	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
459
460	/* WaProgramL3SqcReg1Default:bdw */
461	gen8_set_l3sqc_credits(i915, 30, 2);
462
463	/* WaKVMNotificationOnConfigChange:bdw */
464	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
465			 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
466
467	lpt_init_clock_gating(i915);
468
469	/* WaDisableDopClockGating:bdw
470	 *
471	 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
472	 * clock gating.
473	 */
474	intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
475}
476
477static void hsw_init_clock_gating(struct drm_i915_private *i915)
478{
479	struct intel_display *display = i915->display;
480	enum pipe pipe;
481
482	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
483	intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
484
485	/* WaPsrDPAMaskVBlankInSRD:hsw */
486	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
487
488	for_each_pipe(display, pipe) {
489		/* WaPsrDPRSUnmaskVBlankInSRD:hsw */
490		intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
491				 0, HSW_UNMASK_VBL_TO_REGS_IN_SRD);
492	}
493
494	/* This is required by WaCatErrorRejectionIssue:hsw */
495	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
496			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
497
498	/* WaSwitchSolVfFArbitrationPriority:hsw */
499	intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
500
501	lpt_init_clock_gating(i915);
502}
503
504static void ivb_init_clock_gating(struct drm_i915_private *i915)
505{
506	struct intel_display *display = i915->display;
507
508	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
509
510	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
511	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
512
513	/* WaDisableBackToBackFlipFix:ivb */
514	intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
515			   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
516			   CHICKEN3_DGMG_DONE_FIX_DISABLE);
517
518	if (INTEL_INFO(i915)->gt == 1)
519		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
520				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
521	else {
522		/* must write both registers */
523		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
524				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
525		intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
526				   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
527	}
528
529	/*
530	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
531	 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
532	 */
533	intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
534			   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
535
536	/* This is required by WaCatErrorRejectionIssue:ivb */
537	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
538			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
539
540	g4x_disable_trickle_feed(i915);
541
542	intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
543			 GEN6_MBC_SNPCR_MED);
544
545	if (!HAS_PCH_NOP(display))
546		cpt_init_clock_gating(i915);
547
548	gen6_check_mch_setup(i915);
549}
550
551static void vlv_init_clock_gating(struct drm_i915_private *i915)
552{
553	/* WaDisableBackToBackFlipFix:vlv */
554	intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
555			   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
556			   CHICKEN3_DGMG_DONE_FIX_DISABLE);
557
558	/* WaDisableDopClockGating:vlv */
559	intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
560			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
561
562	/* This is required by WaCatErrorRejectionIssue:vlv */
563	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
564			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
565
566	/*
567	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
568	 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
569	 */
570	intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
571			   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
572
573	/* WaDisableL3Bank2xClockGate:vlv
574	 * Disabling L3 clock gating- MMIO 940c[25] = 1
575	 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
576	intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
577
578	/*
579	 * WaDisableVLVClockGating_VBIIssue:vlv
580	 * Disable clock gating on th GCFG unit to prevent a delay
581	 * in the reporting of vblank events.
582	 */
583	intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
584}
585
586static void chv_init_clock_gating(struct drm_i915_private *i915)
587{
588	/* WaVSRefCountFullforceMissDisable:chv */
589	/* WaDSRefCountFullforceMissDisable:chv */
590	intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
591			 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
592
593	/* WaDisableSemaphoreAndSyncFlipWait:chv */
594	intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
595			   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
596
597	/* WaDisableCSUnitClockGating:chv */
598	intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
599
600	/* WaDisableSDEUnitClockGating:chv */
601	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
602
603	/*
604	 * WaProgramL3SqcReg1Default:chv
605	 * See gfxspecs/Related Documents/Performance Guide/
606	 * LSQC Setting Recommendations.
607	 */
608	gen8_set_l3sqc_credits(i915, 38, 2);
609}
610
611static void g4x_init_clock_gating(struct drm_i915_private *i915)
612{
613	u32 dspclk_gate;
614
615	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
616	intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
617			   GS_UNIT_CLOCK_GATE_DISABLE |
618			   CL_UNIT_CLOCK_GATE_DISABLE);
619	intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
620	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
621		OVRUNIT_CLOCK_GATE_DISABLE |
622		OVCUNIT_CLOCK_GATE_DISABLE;
623	if (IS_GM45(i915))
624		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
625	intel_uncore_write(&i915->uncore, DSPCLK_GATE_D, dspclk_gate);
626
627	g4x_disable_trickle_feed(i915);
628}
629
630static void i965gm_init_clock_gating(struct drm_i915_private *i915)
631{
632	struct intel_uncore *uncore = &i915->uncore;
633
634	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
635	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
636	intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
637	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
638	intel_uncore_write16(uncore, DEUC, 0);
639	intel_uncore_write(uncore,
640			   MI_ARB_STATE,
641			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
642}
643
644static void i965g_init_clock_gating(struct drm_i915_private *i915)
645{
646	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
647			   I965_RCC_CLOCK_GATE_DISABLE |
648			   I965_RCPB_CLOCK_GATE_DISABLE |
649			   I965_ISC_CLOCK_GATE_DISABLE |
650			   I965_FBC_CLOCK_GATE_DISABLE);
651	intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
652	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
653			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
654}
655
656static void gen3_init_clock_gating(struct drm_i915_private *i915)
657{
658	u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
659
660	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
661		DSTATE_DOT_CLOCK_GATING;
662	intel_uncore_write(&i915->uncore, D_STATE, dstate);
663
664	if (IS_PINEVIEW(i915))
665		intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
666				   _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
667
668	/* IIR "flip pending" means done if this bit is set */
669	intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
670			   _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
671
672	/* interrupts should cause a wake up from C3 */
673	intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
674
675	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
676	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
677			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
678
679	intel_uncore_write(&i915->uncore, MI_ARB_STATE,
680			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
681}
682
683static void i85x_init_clock_gating(struct drm_i915_private *i915)
684{
685	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
686
687	/* interrupts should cause a wake up from C3 */
688	intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
689			   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
690
691	intel_uncore_write(&i915->uncore, MEM_MODE,
692			   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
693
694	/*
695	 * Have FBC ignore 3D activity since we use software
696	 * render tracking, and otherwise a pure 3D workload
697	 * (even if it just renders a single frame and then does
698	 * absolutely nothing) would not allow FBC to recompress
699	 * until a 2D blit occurs.
700	 */
701	intel_uncore_write(&i915->uncore, SCPD0,
702			   _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
703}
704
705static void i830_init_clock_gating(struct drm_i915_private *i915)
706{
707	intel_uncore_write(&i915->uncore, MEM_MODE,
708			   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
709			   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
710}
711
712void intel_clock_gating_init(struct drm_i915_private *i915)
713{
714	i915->clock_gating_funcs->init_clock_gating(i915);
715}
716
717static void nop_init_clock_gating(struct drm_i915_private *i915)
718{
719	drm_dbg_kms(&i915->drm,
720		    "No clock gating settings or workarounds applied.\n");
721}
722
723#define CG_FUNCS(platform)						\
724static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
725	.init_clock_gating = platform##_init_clock_gating,		\
726}
727
728CG_FUNCS(dg2);
729CG_FUNCS(cfl);
730CG_FUNCS(skl);
731CG_FUNCS(kbl);
732CG_FUNCS(bxt);
733CG_FUNCS(glk);
734CG_FUNCS(bdw);
735CG_FUNCS(chv);
736CG_FUNCS(hsw);
737CG_FUNCS(ivb);
738CG_FUNCS(vlv);
739CG_FUNCS(gen6);
740CG_FUNCS(ilk);
741CG_FUNCS(g4x);
742CG_FUNCS(i965gm);
743CG_FUNCS(i965g);
744CG_FUNCS(gen3);
745CG_FUNCS(i85x);
746CG_FUNCS(i830);
747CG_FUNCS(nop);
748#undef CG_FUNCS
749
750/**
751 * intel_clock_gating_hooks_init - setup the clock gating hooks
752 * @i915: device private
753 *
754 * Setup the hooks that configure which clocks of a given platform can be
755 * gated and also apply various GT and display specific workarounds for these
756 * platforms. Note that some GT specific workarounds are applied separately
757 * when GPU contexts or batchbuffers start their execution.
758 */
759void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
760{
761	if (IS_DG2(i915))
762		i915->clock_gating_funcs = &dg2_clock_gating_funcs;
763	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
764		i915->clock_gating_funcs = &cfl_clock_gating_funcs;
765	else if (IS_SKYLAKE(i915))
766		i915->clock_gating_funcs = &skl_clock_gating_funcs;
767	else if (IS_KABYLAKE(i915))
768		i915->clock_gating_funcs = &kbl_clock_gating_funcs;
769	else if (IS_BROXTON(i915))
770		i915->clock_gating_funcs = &bxt_clock_gating_funcs;
771	else if (IS_GEMINILAKE(i915))
772		i915->clock_gating_funcs = &glk_clock_gating_funcs;
773	else if (IS_BROADWELL(i915))
774		i915->clock_gating_funcs = &bdw_clock_gating_funcs;
775	else if (IS_CHERRYVIEW(i915))
776		i915->clock_gating_funcs = &chv_clock_gating_funcs;
777	else if (IS_HASWELL(i915))
778		i915->clock_gating_funcs = &hsw_clock_gating_funcs;
779	else if (IS_IVYBRIDGE(i915))
780		i915->clock_gating_funcs = &ivb_clock_gating_funcs;
781	else if (IS_VALLEYVIEW(i915))
782		i915->clock_gating_funcs = &vlv_clock_gating_funcs;
783	else if (GRAPHICS_VER(i915) == 6)
784		i915->clock_gating_funcs = &gen6_clock_gating_funcs;
785	else if (GRAPHICS_VER(i915) == 5)
786		i915->clock_gating_funcs = &ilk_clock_gating_funcs;
787	else if (IS_G4X(i915))
788		i915->clock_gating_funcs = &g4x_clock_gating_funcs;
789	else if (IS_I965GM(i915))
790		i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
791	else if (IS_I965G(i915))
792		i915->clock_gating_funcs = &i965g_clock_gating_funcs;
793	else if (GRAPHICS_VER(i915) == 3)
794		i915->clock_gating_funcs = &gen3_clock_gating_funcs;
795	else if (IS_I85X(i915) || IS_I865G(i915))
796		i915->clock_gating_funcs = &i85x_clock_gating_funcs;
797	else if (GRAPHICS_VER(i915) == 2)
798		i915->clock_gating_funcs = &i830_clock_gating_funcs;
799	else
800		i915->clock_gating_funcs = &nop_clock_gating_funcs;
801}