i9xx_wm.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2023 Intel Corporation
   4 */
   5
   6#include <linux/iopoll.h>
   7
   8#include <drm/drm_print.h>
   9
  10#include "soc/intel_dram.h"
  11
  12#include "i915_drv.h"
  13#include "i915_reg.h"
  14#include "i9xx_wm.h"
  15#include "i9xx_wm_regs.h"
  16#include "intel_atomic.h"
  17#include "intel_bo.h"
  18#include "intel_de.h"
  19#include "intel_display.h"
  20#include "intel_display_regs.h"
  21#include "intel_display_trace.h"
  22#include "intel_fb.h"
  23#include "intel_mchbar_regs.h"
  24#include "intel_wm.h"
  25#include "skl_watermark.h"
  26#include "vlv_sideband.h"
  27
  28struct intel_watermark_params {
  29	u16 fifo_size;
  30	u16 max_wm;
  31	u8 default_wm;
  32	u8 guard_size;
  33	u8 cacheline_size;
  34};
  35
  36/* used in computing the new watermarks state */
  37struct intel_wm_config {
  38	unsigned int num_pipes_active;
  39	bool sprites_enabled;
  40	bool sprites_scaled;
  41};
  42
  43struct cxsr_latency {
  44	bool is_desktop : 1;
  45	bool is_ddr3 : 1;
  46	u16 fsb_freq;
  47	u16 mem_freq;
  48	u16 display_sr;
  49	u16 display_hpll_disable;
  50	u16 cursor_sr;
  51	u16 cursor_hpll_disable;
  52};
  53
  54static const struct cxsr_latency cxsr_latency_table[] = {
  55	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
  56	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
  57	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
  58	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
  59	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
  60
  61	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
  62	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
  63	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
  64	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
  65	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
  66
  67	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
  68	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
  69	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
  70	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
  71	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
  72
  73	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
  74	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
  75	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
  76	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
  77	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
  78
  79	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
  80	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
  81	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
  82	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
  83	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
  84
  85	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
  86	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
  87	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
  88	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
  89	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
  90};
  91
  92static const struct cxsr_latency *pnv_get_cxsr_latency(struct intel_display *display)
  93{
  94	const struct dram_info *dram_info = intel_dram_info(display->drm);
  95	bool is_ddr3 = dram_info->type == INTEL_DRAM_DDR3;
  96	int i;
  97
  98	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
  99		const struct cxsr_latency *latency = &cxsr_latency_table[i];
 100		bool is_desktop = !display->platform.mobile;
 101
 102		if (is_desktop == latency->is_desktop &&
 103		    is_ddr3 == latency->is_ddr3 &&
 104		    DIV_ROUND_CLOSEST(dram_info->fsb_freq, 1000) == latency->fsb_freq &&
 105		    DIV_ROUND_CLOSEST(dram_info->mem_freq, 1000) == latency->mem_freq)
 106			return latency;
 107	}
 108
 109	drm_dbg_kms(display->drm,
 110		    "Could not find CxSR latency for %s, FSB %u kHz, MEM %u kHz\n",
 111		    intel_dram_type_str(dram_info->type),
 112		    dram_info->fsb_freq, dram_info->mem_freq);
 113
 114	return NULL;
 115}
 116
 117static void chv_set_memory_dvfs(struct intel_display *display, bool enable)
 118{
 119	u32 val;
 120	int ret;
 121
 122	vlv_punit_get(display->drm);
 123
 124	val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2);
 125	if (enable)
 126		val &= ~FORCE_DDR_HIGH_FREQ;
 127	else
 128		val |= FORCE_DDR_HIGH_FREQ;
 129	val &= ~FORCE_DDR_LOW_FREQ;
 130	val |= FORCE_DDR_FREQ_REQ_ACK;
 131	vlv_punit_write(display->drm, PUNIT_REG_DDR_SETUP2, val);
 132
 133	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
 134			      (val & FORCE_DDR_FREQ_REQ_ACK) == 0,
 135			      500, 3000, false);
 136	if (ret)
 137		drm_err(display->drm,
 138			"timed out waiting for Punit DDR DVFS request\n");
 139
 140	vlv_punit_put(display->drm);
 141}
 142
 143static void chv_set_memory_pm5(struct intel_display *display, bool enable)
 144{
 145	u32 val;
 146
 147	vlv_punit_get(display->drm);
 148
 149	val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
 150	if (enable)
 151		val |= DSP_MAXFIFO_PM5_ENABLE;
 152	else
 153		val &= ~DSP_MAXFIFO_PM5_ENABLE;
 154	vlv_punit_write(display->drm, PUNIT_REG_DSPSSPM, val);
 155
 156	vlv_punit_put(display->drm);
 157}
 158
 159#define FW_WM(value, plane) \
 160	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 161
 162static bool _intel_set_memory_cxsr(struct intel_display *display, bool enable)
 163{
 164	bool was_enabled;
 165	u32 val;
 166
 167	if (display->platform.valleyview || display->platform.cherryview) {
 168		was_enabled = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 169		intel_de_write(display, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 170		intel_de_posting_read(display, FW_BLC_SELF_VLV);
 171	} else if (display->platform.g4x || display->platform.i965gm) {
 172		was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
 173		intel_de_write(display, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 174		intel_de_posting_read(display, FW_BLC_SELF);
 175	} else if (display->platform.pineview) {
 176		val = intel_de_read(display, DSPFW3(display));
 177		was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 178		if (enable)
 179			val |= PINEVIEW_SELF_REFRESH_EN;
 180		else
 181			val &= ~PINEVIEW_SELF_REFRESH_EN;
 182		intel_de_write(display, DSPFW3(display), val);
 183		intel_de_posting_read(display, DSPFW3(display));
 184	} else if (display->platform.i945g || display->platform.i945gm) {
 185		was_enabled = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
 186		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 187			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 188		intel_de_write(display, FW_BLC_SELF, val);
 189		intel_de_posting_read(display, FW_BLC_SELF);
 190	} else if (display->platform.i915gm) {
 191		/*
 192		 * FIXME can't find a bit like this for 915G, and
 193		 * yet it does have the related watermark in
 194		 * FW_BLC_SELF. What's going on?
 195		 */
 196		was_enabled = intel_de_read(display, INSTPM) & INSTPM_SELF_EN;
 197		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 198			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 199		intel_de_write(display, INSTPM, val);
 200		intel_de_posting_read(display, INSTPM);
 201	} else {
 202		return false;
 203	}
 204
 205	trace_intel_memory_cxsr(display, was_enabled, enable);
 206
 207	drm_dbg_kms(display->drm, "memory self-refresh is %s (was %s)\n",
 208		    str_enabled_disabled(enable),
 209		    str_enabled_disabled(was_enabled));
 210
 211	return was_enabled;
 212}
 213
 214/**
 215 * intel_set_memory_cxsr - Configure CxSR state
 216 * @display: display device
 217 * @enable: Allow vs. disallow CxSR
 218 *
 219 * Allow or disallow the system to enter a special CxSR
 220 * (C-state self refresh) state. What typically happens in CxSR mode
 221 * is that several display FIFOs may get combined into a single larger
 222 * FIFO for a particular plane (so called max FIFO mode) to allow the
 223 * system to defer memory fetches longer, and the memory will enter
 224 * self refresh.
 225 *
 226 * Note that enabling CxSR does not guarantee that the system enter
 227 * this special mode, nor does it guarantee that the system stays
 228 * in that mode once entered. So this just allows/disallows the system
 229 * to autonomously utilize the CxSR mode. Other factors such as core
 230 * C-states will affect when/if the system actually enters/exits the
 231 * CxSR mode.
 232 *
 233 * Note that on VLV/CHV this actually only controls the max FIFO mode,
 234 * and the system is free to enter/exit memory self refresh at any time
 235 * even when the use of CxSR has been disallowed.
 236 *
 237 * While the system is actually in the CxSR/max FIFO mode, some plane
 238 * control registers will not get latched on vblank. Thus in order to
 239 * guarantee the system will respond to changes in the plane registers
 240 * we must always disallow CxSR prior to making changes to those registers.
 241 * Unfortunately the system will re-evaluate the CxSR conditions at
 242 * frame start which happens after vblank start (which is when the plane
 243 * registers would get latched), so we can't proceed with the plane update
 244 * during the same frame where we disallowed CxSR.
 245 *
 246 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 247 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 248 * the hardware w.r.t. HPLL SR when writing to plane registers.
 249 * Disallowing just CxSR is sufficient.
 250 */
 251bool intel_set_memory_cxsr(struct intel_display *display, bool enable)
 252{
 253	bool ret;
 254
 255	mutex_lock(&display->wm.wm_mutex);
 256	ret = _intel_set_memory_cxsr(display, enable);
 257	if (display->platform.valleyview || display->platform.cherryview)
 258		display->wm.vlv.cxsr = enable;
 259	else if (display->platform.g4x)
 260		display->wm.g4x.cxsr = enable;
 261	mutex_unlock(&display->wm.wm_mutex);
 262
 263	return ret;
 264}
 265
 266/*
 267 * Latency for FIFO fetches is dependent on several factors:
 268 *   - memory configuration (speed, channels)
 269 *   - chipset
 270 *   - current MCH state
 271 * It can be fairly high in some situations, so here we assume a fairly
 272 * pessimal value.  It's a tradeoff between extra memory fetches (if we
 273 * set this value too high, the FIFO will fetch frequently to stay full)
 274 * and power consumption (set it too low to save power and we might see
 275 * FIFO underruns and display "flicker").
 276 *
 277 * A value of 5us seems to be a good balance; safe for very low end
 278 * platforms but not overly aggressive on lower latency configs.
 279 */
 280static const int pessimal_latency_ns = 5000;
 281
 282#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 283	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 284
 285static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 286{
 287	struct intel_display *display = to_intel_display(crtc_state);
 288	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 289	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 290	enum pipe pipe = crtc->pipe;
 291	int sprite0_start, sprite1_start;
 292	u32 dsparb, dsparb2, dsparb3;
 293
 294	switch (pipe) {
 295	case PIPE_A:
 296		dsparb = intel_de_read(display, DSPARB(display));
 297		dsparb2 = intel_de_read(display, DSPARB2);
 298		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 299		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 300		break;
 301	case PIPE_B:
 302		dsparb = intel_de_read(display, DSPARB(display));
 303		dsparb2 = intel_de_read(display, DSPARB2);
 304		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 305		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 306		break;
 307	case PIPE_C:
 308		dsparb2 = intel_de_read(display, DSPARB2);
 309		dsparb3 = intel_de_read(display, DSPARB3);
 310		sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 311		sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 312		break;
 313	default:
 314		MISSING_CASE(pipe);
 315		return;
 316	}
 317
 318	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 319	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 320	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 321	fifo_state->plane[PLANE_CURSOR] = 63;
 322}
 323
 324static int i9xx_get_fifo_size(struct intel_display *display,
 325			      enum i9xx_plane_id i9xx_plane)
 326{
 327	u32 dsparb = intel_de_read(display, DSPARB(display));
 328	int size;
 329
 330	size = dsparb & 0x7f;
 331	if (i9xx_plane == PLANE_B)
 332		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 333
 334	drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
 335		    dsparb, plane_name(i9xx_plane), size);
 336
 337	return size;
 338}
 339
 340static int i830_get_fifo_size(struct intel_display *display,
 341			      enum i9xx_plane_id i9xx_plane)
 342{
 343	u32 dsparb = intel_de_read(display, DSPARB(display));
 344	int size;
 345
 346	size = dsparb & 0x1ff;
 347	if (i9xx_plane == PLANE_B)
 348		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 349	size >>= 1; /* Convert to cachelines */
 350
 351	drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
 352		    dsparb, plane_name(i9xx_plane), size);
 353
 354	return size;
 355}
 356
 357static int i845_get_fifo_size(struct intel_display *display,
 358			      enum i9xx_plane_id i9xx_plane)
 359{
 360	u32 dsparb = intel_de_read(display, DSPARB(display));
 361	int size;
 362
 363	size = dsparb & 0x7f;
 364	size >>= 2; /* Convert to cachelines */
 365
 366	drm_dbg_kms(display->drm, "FIFO size - (0x%08x) %c: %d\n",
 367		    dsparb, plane_name(i9xx_plane), size);
 368
 369	return size;
 370}
 371
 372/* Pineview has different values for various configs */
 373static const struct intel_watermark_params pnv_display_wm = {
 374	.fifo_size = PINEVIEW_DISPLAY_FIFO,
 375	.max_wm = PINEVIEW_MAX_WM,
 376	.default_wm = PINEVIEW_DFT_WM,
 377	.guard_size = PINEVIEW_GUARD_WM,
 378	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 379};
 380
 381static const struct intel_watermark_params pnv_display_hplloff_wm = {
 382	.fifo_size = PINEVIEW_DISPLAY_FIFO,
 383	.max_wm = PINEVIEW_MAX_WM,
 384	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 385	.guard_size = PINEVIEW_GUARD_WM,
 386	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 387};
 388
 389static const struct intel_watermark_params pnv_cursor_wm = {
 390	.fifo_size = PINEVIEW_CURSOR_FIFO,
 391	.max_wm = PINEVIEW_CURSOR_MAX_WM,
 392	.default_wm = PINEVIEW_CURSOR_DFT_WM,
 393	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
 394	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 395};
 396
 397static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 398	.fifo_size = PINEVIEW_CURSOR_FIFO,
 399	.max_wm = PINEVIEW_CURSOR_MAX_WM,
 400	.default_wm = PINEVIEW_CURSOR_DFT_WM,
 401	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
 402	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 403};
 404
 405static const struct intel_watermark_params i965_cursor_wm_info = {
 406	.fifo_size = I965_CURSOR_FIFO,
 407	.max_wm = I965_CURSOR_MAX_WM,
 408	.default_wm = I965_CURSOR_DFT_WM,
 409	.guard_size = 2,
 410	.cacheline_size = I915_FIFO_LINE_SIZE,
 411};
 412
 413static const struct intel_watermark_params i945_wm_info = {
 414	.fifo_size = I945_FIFO_SIZE,
 415	.max_wm = I915_MAX_WM,
 416	.default_wm = 1,
 417	.guard_size = 2,
 418	.cacheline_size = I915_FIFO_LINE_SIZE,
 419};
 420
 421static const struct intel_watermark_params i915_wm_info = {
 422	.fifo_size = I915_FIFO_SIZE,
 423	.max_wm = I915_MAX_WM,
 424	.default_wm = 1,
 425	.guard_size = 2,
 426	.cacheline_size = I915_FIFO_LINE_SIZE,
 427};
 428
 429static const struct intel_watermark_params i830_a_wm_info = {
 430	.fifo_size = I855GM_FIFO_SIZE,
 431	.max_wm = I915_MAX_WM,
 432	.default_wm = 1,
 433	.guard_size = 2,
 434	.cacheline_size = I830_FIFO_LINE_SIZE,
 435};
 436
 437static const struct intel_watermark_params i830_bc_wm_info = {
 438	.fifo_size = I855GM_FIFO_SIZE,
 439	.max_wm = I915_MAX_WM / 2,
 440	.default_wm = 1,
 441	.guard_size = 2,
 442	.cacheline_size = I830_FIFO_LINE_SIZE,
 443};
 444
 445static const struct intel_watermark_params i845_wm_info = {
 446	.fifo_size = I830_FIFO_SIZE,
 447	.max_wm = I915_MAX_WM,
 448	.default_wm = 1,
 449	.guard_size = 2,
 450	.cacheline_size = I830_FIFO_LINE_SIZE,
 451};
 452
 453/**
 454 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 455 * @pixel_rate: Pipe pixel rate in kHz
 456 * @cpp: Plane bytes per pixel
 457 * @latency: Memory wakeup latency in 0.1us units
 458 *
 459 * Compute the watermark using the method 1 or "small buffer"
 460 * formula. The caller may additionally add extra cachelines
 461 * to account for TLB misses and clock crossings.
 462 *
 463 * This method is concerned with the short term drain rate
 464 * of the FIFO, ie. it does not account for blanking periods
 465 * which would effectively reduce the average drain rate across
 466 * a longer period. The name "small" refers to the fact the
 467 * FIFO is relatively small compared to the amount of data
 468 * fetched.
 469 *
 470 * The FIFO level vs. time graph might look something like:
 471 *
 472 *   |\   |\
 473 *   | \  | \
 474 * __---__---__ (- plane active, _ blanking)
 475 * -> time
 476 *
 477 * or perhaps like this:
 478 *
 479 *   |\|\  |\|\
 480 * __----__----__ (- plane active, _ blanking)
 481 * -> time
 482 *
 483 * Returns:
 484 * The watermark in bytes
 485 */
 486static unsigned int intel_wm_method1(unsigned int pixel_rate,
 487				     unsigned int cpp,
 488				     unsigned int latency)
 489{
 490	u64 ret;
 491
 492	ret = mul_u32_u32(pixel_rate, cpp * latency);
 493	ret = DIV_ROUND_UP_ULL(ret, 10000);
 494
 495	return ret;
 496}
 497
 498/**
 499 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 500 * @pixel_rate: Pipe pixel rate in kHz
 501 * @htotal: Pipe horizontal total
 502 * @width: Plane width in pixels
 503 * @cpp: Plane bytes per pixel
 504 * @latency: Memory wakeup latency in 0.1us units
 505 *
 506 * Compute the watermark using the method 2 or "large buffer"
 507 * formula. The caller may additionally add extra cachelines
 508 * to account for TLB misses and clock crossings.
 509 *
 510 * This method is concerned with the long term drain rate
 511 * of the FIFO, ie. it does account for blanking periods
 512 * which effectively reduce the average drain rate across
 513 * a longer period. The name "large" refers to the fact the
 514 * FIFO is relatively large compared to the amount of data
 515 * fetched.
 516 *
 517 * The FIFO level vs. time graph might look something like:
 518 *
 519 *    |\___       |\___
 520 *    |    \___   |    \___
 521 *    |        \  |        \
 522 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 523 * -> time
 524 *
 525 * Returns:
 526 * The watermark in bytes
 527 */
 528static unsigned int intel_wm_method2(unsigned int pixel_rate,
 529				     unsigned int htotal,
 530				     unsigned int width,
 531				     unsigned int cpp,
 532				     unsigned int latency)
 533{
 534	unsigned int ret;
 535
 536	/*
 537	 * FIXME remove once all users are computing
 538	 * watermarks in the correct place.
 539	 */
 540	if (WARN_ON_ONCE(htotal == 0))
 541		htotal = 1;
 542
 543	ret = (latency * pixel_rate) / (htotal * 10000);
 544	ret = (ret + 1) * width * cpp;
 545
 546	return ret;
 547}
 548
 549/**
 550 * intel_calculate_wm - calculate watermark level
 551 * @display: display device
 552 * @pixel_rate: pixel clock
 553 * @wm: chip FIFO params
 554 * @fifo_size: size of the FIFO buffer
 555 * @cpp: bytes per pixel
 556 * @latency_ns: memory latency for the platform
 557 *
 558 * Calculate the watermark level (the level at which the display plane will
 559 * start fetching from memory again).  Each chip has a different display
 560 * FIFO size and allocation, so the caller needs to figure that out and pass
 561 * in the correct intel_watermark_params structure.
 562 *
 563 * As the pixel clock runs, the FIFO will be drained at a rate that depends
 564 * on the pixel size.  When it reaches the watermark level, it'll start
 565 * fetching FIFO line sized based chunks from memory until the FIFO fills
 566 * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 567 * will occur, and a display engine hang could result.
 568 */
 569static unsigned int intel_calculate_wm(struct intel_display *display,
 570				       int pixel_rate,
 571				       const struct intel_watermark_params *wm,
 572				       int fifo_size, int cpp,
 573				       unsigned int latency_ns)
 574{
 575	int entries, wm_size;
 576
 577	/*
 578	 * Note: we need to make sure we don't overflow for various clock &
 579	 * latency values.
 580	 * clocks go from a few thousand to several hundred thousand.
 581	 * latency is usually a few thousand
 582	 */
 583	entries = intel_wm_method1(pixel_rate, cpp,
 584				   latency_ns / 100);
 585	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 586		wm->guard_size;
 587	drm_dbg_kms(display->drm, "FIFO entries required for mode: %d\n", entries);
 588
 589	wm_size = fifo_size - entries;
 590	drm_dbg_kms(display->drm, "FIFO watermark level: %d\n", wm_size);
 591
 592	/* Don't promote wm_size to unsigned... */
 593	if (wm_size > wm->max_wm)
 594		wm_size = wm->max_wm;
 595	if (wm_size <= 0)
 596		wm_size = wm->default_wm;
 597
 598	/*
 599	 * Bspec seems to indicate that the value shouldn't be lower than
 600	 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 601	 * Lets go for 8 which is the burst size since certain platforms
 602	 * already use a hardcoded 8 (which is what the spec says should be
 603	 * done).
 604	 */
 605	if (wm_size <= 8)
 606		wm_size = 8;
 607
 608	return wm_size;
 609}
 610
 611static bool is_disabling(int old, int new, int threshold)
 612{
 613	return old >= threshold && new < threshold;
 614}
 615
 616static bool is_enabling(int old, int new, int threshold)
 617{
 618	return old < threshold && new >= threshold;
 619}
 620
 621static bool intel_crtc_active(struct intel_crtc *crtc)
 622{
 623	/* Be paranoid as we can arrive here with only partial
 624	 * state retrieved from the hardware during setup.
 625	 *
 626	 * We can ditch the adjusted_mode.crtc_clock check as soon
 627	 * as Haswell has gained clock readout/fastboot support.
 628	 *
 629	 * We can ditch the crtc->primary->state->fb check as soon as we can
 630	 * properly reconstruct framebuffers.
 631	 *
 632	 * FIXME: The intel_crtc->active here should be switched to
 633	 * crtc->state->active once we have proper CRTC states wired up
 634	 * for atomic.
 635	 */
 636	return crtc->active && crtc->base.primary->state->fb &&
 637		crtc->config->hw.adjusted_mode.crtc_clock;
 638}
 639
 640static struct intel_crtc *single_enabled_crtc(struct intel_display *display)
 641{
 642	struct intel_crtc *crtc, *enabled = NULL;
 643
 644	for_each_intel_crtc(display->drm, crtc) {
 645		if (intel_crtc_active(crtc)) {
 646			if (enabled)
 647				return NULL;
 648			enabled = crtc;
 649		}
 650	}
 651
 652	return enabled;
 653}
 654
 655static void pnv_update_wm(struct intel_display *display)
 656{
 657	struct intel_crtc *crtc;
 658	const struct cxsr_latency *latency;
 659	u32 reg;
 660	unsigned int wm;
 661
 662	latency = pnv_get_cxsr_latency(display);
 663	if (!latency) {
 664		drm_dbg_kms(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
 665		intel_set_memory_cxsr(display, false);
 666		return;
 667	}
 668
 669	crtc = single_enabled_crtc(display);
 670	if (crtc) {
 671		const struct drm_framebuffer *fb =
 672			crtc->base.primary->state->fb;
 673		int pixel_rate = crtc->config->pixel_rate;
 674		int cpp = fb->format->cpp[0];
 675
 676		/* Display SR */
 677		wm = intel_calculate_wm(display, pixel_rate,
 678					&pnv_display_wm,
 679					pnv_display_wm.fifo_size,
 680					cpp, latency->display_sr);
 681		reg = intel_de_read(display, DSPFW1(display));
 682		reg &= ~DSPFW_SR_MASK;
 683		reg |= FW_WM(wm, SR);
 684		intel_de_write(display, DSPFW1(display), reg);
 685		drm_dbg_kms(display->drm, "DSPFW1 register is %x\n", reg);
 686
 687		/* cursor SR */
 688		wm = intel_calculate_wm(display, pixel_rate,
 689					&pnv_cursor_wm,
 690					pnv_display_wm.fifo_size,
 691					4, latency->cursor_sr);
 692		intel_de_rmw(display, DSPFW3(display),
 693			     DSPFW_CURSOR_SR_MASK, FW_WM(wm, CURSOR_SR));
 694
 695		/* Display HPLL off SR */
 696		wm = intel_calculate_wm(display, pixel_rate,
 697					&pnv_display_hplloff_wm,
 698					pnv_display_hplloff_wm.fifo_size,
 699					cpp, latency->display_hpll_disable);
 700		intel_de_rmw(display, DSPFW3(display),
 701			     DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
 702
 703		/* cursor HPLL off SR */
 704		wm = intel_calculate_wm(display, pixel_rate,
 705					&pnv_cursor_hplloff_wm,
 706					pnv_display_hplloff_wm.fifo_size,
 707					4, latency->cursor_hpll_disable);
 708		reg = intel_de_read(display, DSPFW3(display));
 709		reg &= ~DSPFW_HPLL_CURSOR_MASK;
 710		reg |= FW_WM(wm, HPLL_CURSOR);
 711		intel_de_write(display, DSPFW3(display), reg);
 712		drm_dbg_kms(display->drm, "DSPFW3 register is %x\n", reg);
 713
 714		intel_set_memory_cxsr(display, true);
 715	} else {
 716		intel_set_memory_cxsr(display, false);
 717	}
 718}
 719
 720static bool i9xx_wm_need_update(const struct intel_plane_state *old_plane_state,
 721				const struct intel_plane_state *new_plane_state)
 722{
 723	/* Update watermarks on tiling or size changes. */
 724	if (old_plane_state->uapi.visible != new_plane_state->uapi.visible)
 725		return true;
 726
 727	if (!old_plane_state->hw.fb || !new_plane_state->hw.fb)
 728		return false;
 729
 730	if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier ||
 731	    old_plane_state->hw.rotation != new_plane_state->hw.rotation ||
 732	    drm_rect_width(&old_plane_state->uapi.src) != drm_rect_width(&new_plane_state->uapi.src) ||
 733	    drm_rect_height(&old_plane_state->uapi.src) != drm_rect_height(&new_plane_state->uapi.src) ||
 734	    drm_rect_width(&old_plane_state->uapi.dst) != drm_rect_width(&new_plane_state->uapi.dst) ||
 735	    drm_rect_height(&old_plane_state->uapi.dst) != drm_rect_height(&new_plane_state->uapi.dst))
 736		return true;
 737
 738	return false;
 739}
 740
 741static void i9xx_wm_compute(struct intel_crtc_state *new_crtc_state,
 742			    const struct intel_plane_state *old_plane_state,
 743			    const struct intel_plane_state *new_plane_state)
 744{
 745	bool turn_off, turn_on, visible, was_visible, mode_changed;
 746
 747	mode_changed = intel_crtc_needs_modeset(new_crtc_state);
 748	was_visible = old_plane_state->uapi.visible;
 749	visible = new_plane_state->uapi.visible;
 750
 751	if (!was_visible && !visible)
 752		return;
 753
 754	turn_off = was_visible && (!visible || mode_changed);
 755	turn_on = visible && (!was_visible || mode_changed);
 756
 757	/* FIXME nuke when all wm code is atomic */
 758	if (turn_on) {
 759		new_crtc_state->update_wm_pre = true;
 760	} else if (turn_off) {
 761		new_crtc_state->update_wm_post = true;
 762	} else if (i9xx_wm_need_update(old_plane_state, new_plane_state)) {
 763		/* FIXME bollocks */
 764		new_crtc_state->update_wm_pre = true;
 765		new_crtc_state->update_wm_post = true;
 766	}
 767}
 768
 769static int i9xx_compute_watermarks(struct intel_atomic_state *state,
 770				   struct intel_crtc *crtc)
 771{
 772	struct intel_crtc_state *new_crtc_state =
 773		intel_atomic_get_new_crtc_state(state, crtc);
 774	const struct intel_plane_state *old_plane_state;
 775	const struct intel_plane_state *new_plane_state;
 776	struct intel_plane *plane;
 777	int i;
 778
 779	for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
 780					     new_plane_state, i) {
 781		if (plane->pipe != crtc->pipe)
 782			continue;
 783
 784		i9xx_wm_compute(new_crtc_state, old_plane_state, new_plane_state);
 785	}
 786
 787	return 0;
 788}
 789
 790/*
 791 * Documentation says:
 792 * "If the line size is small, the TLB fetches can get in the way of the
 793 *  data fetches, causing some lag in the pixel data return which is not
 794 *  accounted for in the above formulas. The following adjustment only
 795 *  needs to be applied if eight whole lines fit in the buffer at once.
 796 *  The WM is adjusted upwards by the difference between the FIFO size
 797 *  and the size of 8 whole lines. This adjustment is always performed
 798 *  in the actual pixel depth regardless of whether FBC is enabled or not."
 799 */
 800static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 801{
 802	int tlb_miss = fifo_size * 64 - width * cpp * 8;
 803
 804	return max(0, tlb_miss);
 805}
 806
 807static void g4x_write_wm_values(struct intel_display *display,
 808				const struct g4x_wm_values *wm)
 809{
 810	enum pipe pipe;
 811
 812	for_each_pipe(display, pipe)
 813		trace_g4x_wm(intel_crtc_for_pipe(display, pipe), wm);
 814
 815	intel_de_write(display, DSPFW1(display),
 816		       FW_WM(wm->sr.plane, SR) |
 817		       FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 818		       FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 819		       FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 820	intel_de_write(display, DSPFW2(display),
 821		       (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 822		       FW_WM(wm->sr.fbc, FBC_SR) |
 823		       FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 824		       FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 825		       FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 826		       FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 827	intel_de_write(display, DSPFW3(display),
 828		       (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 829		       FW_WM(wm->sr.cursor, CURSOR_SR) |
 830		       FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 831		       FW_WM(wm->hpll.plane, HPLL_SR));
 832
 833	intel_de_posting_read(display, DSPFW1(display));
 834}
 835
 836#define FW_WM_VLV(value, plane) \
 837	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 838
 839static void vlv_write_wm_values(struct intel_display *display,
 840				const struct vlv_wm_values *wm)
 841{
 842	enum pipe pipe;
 843
 844	for_each_pipe(display, pipe) {
 845		trace_vlv_wm(intel_crtc_for_pipe(display, pipe), wm);
 846
 847		intel_de_write(display, VLV_DDL(pipe),
 848			       (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 849			       (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 850			       (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 851			       (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 852	}
 853
 854	/*
 855	 * Zero the (unused) WM1 watermarks, and also clear all the
 856	 * high order bits so that there are no out of bounds values
 857	 * present in the registers during the reprogramming.
 858	 */
 859	intel_de_write(display, DSPHOWM, 0);
 860	intel_de_write(display, DSPHOWM1, 0);
 861	intel_de_write(display, DSPFW4, 0);
 862	intel_de_write(display, DSPFW5, 0);
 863	intel_de_write(display, DSPFW6, 0);
 864
 865	intel_de_write(display, DSPFW1(display),
 866		       FW_WM(wm->sr.plane, SR) |
 867		       FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 868		       FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 869		       FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 870	intel_de_write(display, DSPFW2(display),
 871		       FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 872		       FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 873		       FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 874	intel_de_write(display, DSPFW3(display),
 875		       FW_WM(wm->sr.cursor, CURSOR_SR));
 876
 877	if (display->platform.cherryview) {
 878		intel_de_write(display, DSPFW7_CHV,
 879			       FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 880			       FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 881		intel_de_write(display, DSPFW8_CHV,
 882			       FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
 883			       FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
 884		intel_de_write(display, DSPFW9_CHV,
 885			       FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
 886			       FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
 887		intel_de_write(display, DSPHOWM,
 888			       FW_WM(wm->sr.plane >> 9, SR_HI) |
 889			       FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
 890			       FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
 891			       FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
 892			       FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
 893			       FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
 894			       FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
 895			       FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
 896			       FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
 897			       FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 898	} else {
 899		intel_de_write(display, DSPFW7,
 900			       FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
 901			       FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
 902		intel_de_write(display, DSPHOWM,
 903			       FW_WM(wm->sr.plane >> 9, SR_HI) |
 904			       FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
 905			       FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
 906			       FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
 907			       FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
 908			       FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
 909			       FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
 910	}
 911
 912	intel_de_posting_read(display, DSPFW1(display));
 913}
 914
 915#undef FW_WM_VLV
 916
 917static void g4x_setup_wm_latency(struct intel_display *display)
 918{
 919	/* all latencies in usec */
 920	display->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
 921	display->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
 922	display->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
 923
 924	display->wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
 925}
 926
 927static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
 928{
 929	/*
 930	 * DSPCNTR[13] supposedly controls whether the
 931	 * primary plane can use the FIFO space otherwise
 932	 * reserved for the sprite plane. It's not 100% clear
 933	 * what the actual FIFO size is, but it looks like we
 934	 * can happily set both primary and sprite watermarks
 935	 * up to 127 cachelines. So that would seem to mean
 936	 * that either DSPCNTR[13] doesn't do anything, or that
 937	 * the total FIFO is >= 256 cachelines in size. Either
 938	 * way, we don't seem to have to worry about this
 939	 * repartitioning as the maximum watermark value the
 940	 * register can hold for each plane is lower than the
 941	 * minimum FIFO size.
 942	 */
 943	switch (plane_id) {
 944	case PLANE_CURSOR:
 945		return 63;
 946	case PLANE_PRIMARY:
 947		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
 948	case PLANE_SPRITE0:
 949		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
 950	default:
 951		MISSING_CASE(plane_id);
 952		return 0;
 953	}
 954}
 955
 956static int g4x_fbc_fifo_size(int level)
 957{
 958	switch (level) {
 959	case G4X_WM_LEVEL_SR:
 960		return 7;
 961	case G4X_WM_LEVEL_HPLL:
 962		return 15;
 963	default:
 964		MISSING_CASE(level);
 965		return 0;
 966	}
 967}
 968
 969static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 970			  const struct intel_plane_state *plane_state,
 971			  int level)
 972{
 973	struct intel_display *display = to_intel_display(plane_state);
 974	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 975	const struct drm_display_mode *pipe_mode =
 976		&crtc_state->hw.pipe_mode;
 977	unsigned int latency = display->wm.pri_latency[level] * 10;
 978	unsigned int pixel_rate, htotal, cpp, width, wm;
 979
 980	if (latency == 0)
 981		return USHRT_MAX;
 982
 983	if (!intel_wm_plane_visible(crtc_state, plane_state))
 984		return 0;
 985
 986	cpp = plane_state->hw.fb->format->cpp[0];
 987
 988	/*
 989	 * WaUse32BppForSRWM:ctg,elk
 990	 *
 991	 * The spec fails to list this restriction for the
 992	 * HPLL watermark, which seems a little strange.
 993	 * Let's use 32bpp for the HPLL watermark as well.
 994	 */
 995	if (plane->id == PLANE_PRIMARY &&
 996	    level != G4X_WM_LEVEL_NORMAL)
 997		cpp = max(cpp, 4u);
 998
 999	pixel_rate = crtc_state->pixel_rate;
1000	htotal = pipe_mode->crtc_htotal;
1001	width = drm_rect_width(&plane_state->uapi.src) >> 16;
1002
1003	if (plane->id == PLANE_CURSOR) {
1004		wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1005	} else if (plane->id == PLANE_PRIMARY &&
1006		   level == G4X_WM_LEVEL_NORMAL) {
1007		wm = intel_wm_method1(pixel_rate, cpp, latency);
1008	} else {
1009		unsigned int small, large;
1010
1011		small = intel_wm_method1(pixel_rate, cpp, latency);
1012		large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1013
1014		wm = min(small, large);
1015	}
1016
1017	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1018			      width, cpp);
1019
1020	wm = DIV_ROUND_UP(wm, 64) + 2;
1021
1022	return min_t(unsigned int, wm, USHRT_MAX);
1023}
1024
1025static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1026				 int level, enum plane_id plane_id, u16 value)
1027{
1028	struct intel_display *display = to_intel_display(crtc_state);
1029	bool dirty = false;
1030
1031	for (; level < display->wm.num_levels; level++) {
1032		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1033
1034		dirty |= raw->plane[plane_id] != value;
1035		raw->plane[plane_id] = value;
1036	}
1037
1038	return dirty;
1039}
1040
1041static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1042			       int level, u16 value)
1043{
1044	struct intel_display *display = to_intel_display(crtc_state);
1045	bool dirty = false;
1046
1047	/* NORMAL level doesn't have an FBC watermark */
1048	level = max(level, G4X_WM_LEVEL_SR);
1049
1050	for (; level < display->wm.num_levels; level++) {
1051		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1052
1053		dirty |= raw->fbc != value;
1054		raw->fbc = value;
1055	}
1056
1057	return dirty;
1058}
1059
1060static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1061			      const struct intel_plane_state *plane_state,
1062			      u32 pri_val);
1063
1064static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1065				     const struct intel_plane_state *plane_state)
1066{
1067	struct intel_display *display = to_intel_display(crtc_state);
1068	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1069	enum plane_id plane_id = plane->id;
1070	bool dirty = false;
1071	int level;
1072
1073	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1074		dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1075		if (plane_id == PLANE_PRIMARY)
1076			dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1077		goto out;
1078	}
1079
1080	for (level = 0; level < display->wm.num_levels; level++) {
1081		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1082		int wm, max_wm;
1083
1084		wm = g4x_compute_wm(crtc_state, plane_state, level);
1085		max_wm = g4x_plane_fifo_size(plane_id, level);
1086
1087		if (wm > max_wm)
1088			break;
1089
1090		dirty |= raw->plane[plane_id] != wm;
1091		raw->plane[plane_id] = wm;
1092
1093		if (plane_id != PLANE_PRIMARY ||
1094		    level == G4X_WM_LEVEL_NORMAL)
1095			continue;
1096
1097		wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1098					raw->plane[plane_id]);
1099		max_wm = g4x_fbc_fifo_size(level);
1100
1101		/*
1102		 * FBC wm is not mandatory as we
1103		 * can always just disable its use.
1104		 */
1105		if (wm > max_wm)
1106			wm = USHRT_MAX;
1107
1108		dirty |= raw->fbc != wm;
1109		raw->fbc = wm;
1110	}
1111
1112	/* mark watermarks as invalid */
1113	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1114
1115	if (plane_id == PLANE_PRIMARY)
1116		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1117
1118 out:
1119	if (dirty) {
1120		drm_dbg_kms(display->drm,
1121			    "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1122			    plane->base.name,
1123			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1124			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1125			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1126
1127		if (plane_id == PLANE_PRIMARY)
1128			drm_dbg_kms(display->drm,
1129				    "FBC watermarks: SR=%d, HPLL=%d\n",
1130				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1131				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1132	}
1133
1134	return dirty;
1135}
1136
1137static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1138				      enum plane_id plane_id, int level)
1139{
1140	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1141
1142	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1143}
1144
1145static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1146				     int level)
1147{
1148	struct intel_display *display = to_intel_display(crtc_state);
1149
1150	if (level >= display->wm.num_levels)
1151		return false;
1152
1153	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1154		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1155		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1156}
1157
1158/* mark all levels starting from 'level' as invalid */
1159static void g4x_invalidate_wms(struct intel_crtc *crtc,
1160			       struct g4x_wm_state *wm_state, int level)
1161{
1162	if (level <= G4X_WM_LEVEL_NORMAL) {
1163		enum plane_id plane_id;
1164
1165		for_each_plane_id_on_crtc(crtc, plane_id)
1166			wm_state->wm.plane[plane_id] = USHRT_MAX;
1167	}
1168
1169	if (level <= G4X_WM_LEVEL_SR) {
1170		wm_state->cxsr = false;
1171		wm_state->sr.cursor = USHRT_MAX;
1172		wm_state->sr.plane = USHRT_MAX;
1173		wm_state->sr.fbc = USHRT_MAX;
1174	}
1175
1176	if (level <= G4X_WM_LEVEL_HPLL) {
1177		wm_state->hpll_en = false;
1178		wm_state->hpll.cursor = USHRT_MAX;
1179		wm_state->hpll.plane = USHRT_MAX;
1180		wm_state->hpll.fbc = USHRT_MAX;
1181	}
1182}
1183
1184static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1185			       int level)
1186{
1187	if (level < G4X_WM_LEVEL_SR)
1188		return false;
1189
1190	if (level >= G4X_WM_LEVEL_SR &&
1191	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1192		return false;
1193
1194	if (level >= G4X_WM_LEVEL_HPLL &&
1195	    wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1196		return false;
1197
1198	return true;
1199}
1200
1201static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1202{
1203	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1204	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1205	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1206	const struct g4x_pipe_wm *raw;
1207	enum plane_id plane_id;
1208	int level;
1209
1210	level = G4X_WM_LEVEL_NORMAL;
1211	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1212		goto out;
1213
1214	raw = &crtc_state->wm.g4x.raw[level];
1215	for_each_plane_id_on_crtc(crtc, plane_id)
1216		wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1217
1218	level = G4X_WM_LEVEL_SR;
1219	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1220		goto out;
1221
1222	raw = &crtc_state->wm.g4x.raw[level];
1223	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1224	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1225	wm_state->sr.fbc = raw->fbc;
1226
1227	wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1228
1229	level = G4X_WM_LEVEL_HPLL;
1230	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1231		goto out;
1232
1233	raw = &crtc_state->wm.g4x.raw[level];
1234	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1235	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1236	wm_state->hpll.fbc = raw->fbc;
1237
1238	wm_state->hpll_en = wm_state->cxsr;
1239
1240	level++;
1241
1242 out:
1243	if (level == G4X_WM_LEVEL_NORMAL)
1244		return -EINVAL;
1245
1246	/* invalidate the higher levels */
1247	g4x_invalidate_wms(crtc, wm_state, level);
1248
1249	/*
1250	 * Determine if the FBC watermark(s) can be used. IF
1251	 * this isn't the case we prefer to disable the FBC
1252	 * watermark(s) rather than disable the SR/HPLL
1253	 * level(s) entirely. 'level-1' is the highest valid
1254	 * level here.
1255	 */
1256	wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1257
1258	return 0;
1259}
1260
1261static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1262			       struct intel_crtc *crtc)
1263{
1264	struct intel_crtc_state *crtc_state =
1265		intel_atomic_get_new_crtc_state(state, crtc);
1266	const struct intel_plane_state *old_plane_state;
1267	const struct intel_plane_state *new_plane_state;
1268	struct intel_plane *plane;
1269	unsigned int dirty = 0;
1270	int i;
1271
1272	for_each_oldnew_intel_plane_in_state(state, plane,
1273					     old_plane_state,
1274					     new_plane_state, i) {
1275		if (new_plane_state->hw.crtc != &crtc->base &&
1276		    old_plane_state->hw.crtc != &crtc->base)
1277			continue;
1278
1279		if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1280			dirty |= BIT(plane->id);
1281	}
1282
1283	if (!dirty)
1284		return 0;
1285
1286	return _g4x_compute_pipe_wm(crtc_state);
1287}
1288
1289static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1290				       struct intel_crtc *crtc)
1291{
1292	struct intel_display *display = to_intel_display(state);
1293	struct intel_crtc_state *new_crtc_state =
1294		intel_atomic_get_new_crtc_state(state, crtc);
1295	const struct intel_crtc_state *old_crtc_state =
1296		intel_atomic_get_old_crtc_state(state, crtc);
1297	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1298	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1299	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1300	enum plane_id plane_id;
1301
1302	if (!new_crtc_state->hw.active ||
1303	    intel_crtc_needs_modeset(new_crtc_state)) {
1304		*intermediate = *optimal;
1305
1306		intermediate->cxsr = false;
1307		intermediate->hpll_en = false;
1308		goto out;
1309	}
1310
1311	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1312		!new_crtc_state->disable_cxsr;
1313	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1314		!new_crtc_state->disable_cxsr;
1315	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1316
1317	for_each_plane_id_on_crtc(crtc, plane_id) {
1318		intermediate->wm.plane[plane_id] =
1319			max(optimal->wm.plane[plane_id],
1320			    active->wm.plane[plane_id]);
1321
1322		drm_WARN_ON(display->drm, intermediate->wm.plane[plane_id] >
1323			    g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1324	}
1325
1326	intermediate->sr.plane = max(optimal->sr.plane,
1327				     active->sr.plane);
1328	intermediate->sr.cursor = max(optimal->sr.cursor,
1329				      active->sr.cursor);
1330	intermediate->sr.fbc = max(optimal->sr.fbc,
1331				   active->sr.fbc);
1332
1333	intermediate->hpll.plane = max(optimal->hpll.plane,
1334				       active->hpll.plane);
1335	intermediate->hpll.cursor = max(optimal->hpll.cursor,
1336					active->hpll.cursor);
1337	intermediate->hpll.fbc = max(optimal->hpll.fbc,
1338				     active->hpll.fbc);
1339
1340	drm_WARN_ON(display->drm,
1341		    (intermediate->sr.plane >
1342		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1343		     intermediate->sr.cursor >
1344		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1345		    intermediate->cxsr);
1346	drm_WARN_ON(display->drm,
1347		    (intermediate->sr.plane >
1348		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1349		     intermediate->sr.cursor >
1350		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1351		    intermediate->hpll_en);
1352
1353	drm_WARN_ON(display->drm,
1354		    intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1355		    intermediate->fbc_en && intermediate->cxsr);
1356	drm_WARN_ON(display->drm,
1357		    intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1358		    intermediate->fbc_en && intermediate->hpll_en);
1359
1360out:
1361	/*
1362	 * If our intermediate WM are identical to the final WM, then we can
1363	 * omit the post-vblank programming; only update if it's different.
1364	 */
1365	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1366		new_crtc_state->wm.need_postvbl_update = true;
1367
1368	return 0;
1369}
1370
1371static int g4x_compute_watermarks(struct intel_atomic_state *state,
1372				  struct intel_crtc *crtc)
1373{
1374	int ret;
1375
1376	ret = g4x_compute_pipe_wm(state, crtc);
1377	if (ret)
1378		return ret;
1379
1380	ret = g4x_compute_intermediate_wm(state, crtc);
1381	if (ret)
1382		return ret;
1383
1384	return 0;
1385}
1386
1387static void g4x_merge_wm(struct intel_display *display,
1388			 struct g4x_wm_values *wm)
1389{
1390	struct intel_crtc *crtc;
1391	int num_active_pipes = 0;
1392
1393	wm->cxsr = true;
1394	wm->hpll_en = true;
1395	wm->fbc_en = true;
1396
1397	for_each_intel_crtc(display->drm, crtc) {
1398		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1399
1400		if (!crtc->active)
1401			continue;
1402
1403		if (!wm_state->cxsr)
1404			wm->cxsr = false;
1405		if (!wm_state->hpll_en)
1406			wm->hpll_en = false;
1407		if (!wm_state->fbc_en)
1408			wm->fbc_en = false;
1409
1410		num_active_pipes++;
1411	}
1412
1413	if (num_active_pipes != 1) {
1414		wm->cxsr = false;
1415		wm->hpll_en = false;
1416		wm->fbc_en = false;
1417	}
1418
1419	for_each_intel_crtc(display->drm, crtc) {
1420		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1421		enum pipe pipe = crtc->pipe;
1422
1423		wm->pipe[pipe] = wm_state->wm;
1424		if (crtc->active && wm->cxsr)
1425			wm->sr = wm_state->sr;
1426		if (crtc->active && wm->hpll_en)
1427			wm->hpll = wm_state->hpll;
1428	}
1429}
1430
1431static void g4x_program_watermarks(struct intel_display *display)
1432{
1433	struct g4x_wm_values *old_wm = &display->wm.g4x;
1434	struct g4x_wm_values new_wm = {};
1435
1436	g4x_merge_wm(display, &new_wm);
1437
1438	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1439		return;
1440
1441	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1442		_intel_set_memory_cxsr(display, false);
1443
1444	g4x_write_wm_values(display, &new_wm);
1445
1446	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1447		_intel_set_memory_cxsr(display, true);
1448
1449	*old_wm = new_wm;
1450}
1451
1452static void g4x_initial_watermarks(struct intel_atomic_state *state,
1453				   struct intel_crtc *crtc)
1454{
1455	struct intel_display *display = to_intel_display(crtc);
1456	const struct intel_crtc_state *crtc_state =
1457		intel_atomic_get_new_crtc_state(state, crtc);
1458
1459	mutex_lock(&display->wm.wm_mutex);
1460	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1461	g4x_program_watermarks(display);
1462	mutex_unlock(&display->wm.wm_mutex);
1463}
1464
1465static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1466				    struct intel_crtc *crtc)
1467{
1468	struct intel_display *display = to_intel_display(crtc);
1469	const struct intel_crtc_state *crtc_state =
1470		intel_atomic_get_new_crtc_state(state, crtc);
1471
1472	if (!crtc_state->wm.need_postvbl_update)
1473		return;
1474
1475	mutex_lock(&display->wm.wm_mutex);
1476	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1477	g4x_program_watermarks(display);
1478	mutex_unlock(&display->wm.wm_mutex);
1479}
1480
1481/* latency must be in 0.1us units. */
1482static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1483				   unsigned int htotal,
1484				   unsigned int width,
1485				   unsigned int cpp,
1486				   unsigned int latency)
1487{
1488	unsigned int ret;
1489
1490	ret = intel_wm_method2(pixel_rate, htotal,
1491			       width, cpp, latency);
1492	ret = DIV_ROUND_UP(ret, 64);
1493
1494	return ret;
1495}
1496
1497static void vlv_setup_wm_latency(struct intel_display *display)
1498{
1499	/* all latencies in usec */
1500	display->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1501
1502	display->wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1503
1504	if (display->platform.cherryview) {
1505		display->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1506		display->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1507
1508		display->wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1509	}
1510}
1511
1512static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1513				const struct intel_plane_state *plane_state,
1514				int level)
1515{
1516	struct intel_display *display = to_intel_display(plane_state);
1517	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1518	const struct drm_display_mode *pipe_mode =
1519		&crtc_state->hw.pipe_mode;
1520	unsigned int pixel_rate, htotal, cpp, width, wm;
1521
1522	if (display->wm.pri_latency[level] == 0)
1523		return USHRT_MAX;
1524
1525	if (!intel_wm_plane_visible(crtc_state, plane_state))
1526		return 0;
1527
1528	cpp = plane_state->hw.fb->format->cpp[0];
1529	pixel_rate = crtc_state->pixel_rate;
1530	htotal = pipe_mode->crtc_htotal;
1531	width = drm_rect_width(&plane_state->uapi.src) >> 16;
1532
1533	if (plane->id == PLANE_CURSOR) {
1534		/*
1535		 * FIXME the formula gives values that are
1536		 * too big for the cursor FIFO, and hence we
1537		 * would never be able to use cursors. For
1538		 * now just hardcode the watermark.
1539		 */
1540		wm = 63;
1541	} else {
1542		wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1543				    display->wm.pri_latency[level] * 10);
1544	}
1545
1546	return min_t(unsigned int, wm, USHRT_MAX);
1547}
1548
1549static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1550{
1551	return (active_planes & (BIT(PLANE_SPRITE0) |
1552				 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1553}
1554
1555static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1556{
1557	struct intel_display *display = to_intel_display(crtc_state);
1558	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1559	const struct g4x_pipe_wm *raw =
1560		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1561	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1562	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1563	int num_active_planes = hweight8(active_planes);
1564	const int fifo_size = 511;
1565	int fifo_extra, fifo_left = fifo_size;
1566	int sprite0_fifo_extra = 0;
1567	unsigned int total_rate;
1568	enum plane_id plane_id;
1569
1570	/*
1571	 * When enabling sprite0 after sprite1 has already been enabled
1572	 * we tend to get an underrun unless sprite0 already has some
1573	 * FIFO space allocated. Hence we always allocate at least one
1574	 * cacheline for sprite0 whenever sprite1 is enabled.
1575	 *
1576	 * All other plane enable sequences appear immune to this problem.
1577	 */
1578	if (vlv_need_sprite0_fifo_workaround(active_planes))
1579		sprite0_fifo_extra = 1;
1580
1581	total_rate = raw->plane[PLANE_PRIMARY] +
1582		raw->plane[PLANE_SPRITE0] +
1583		raw->plane[PLANE_SPRITE1] +
1584		sprite0_fifo_extra;
1585
1586	if (total_rate > fifo_size)
1587		return -EINVAL;
1588
1589	if (total_rate == 0)
1590		total_rate = 1;
1591
1592	for_each_plane_id_on_crtc(crtc, plane_id) {
1593		unsigned int rate;
1594
1595		if ((active_planes & BIT(plane_id)) == 0) {
1596			fifo_state->plane[plane_id] = 0;
1597			continue;
1598		}
1599
1600		rate = raw->plane[plane_id];
1601		fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1602		fifo_left -= fifo_state->plane[plane_id];
1603	}
1604
1605	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1606	fifo_left -= sprite0_fifo_extra;
1607
1608	fifo_state->plane[PLANE_CURSOR] = 63;
1609
1610	fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1611
1612	/* spread the remainder evenly */
1613	for_each_plane_id_on_crtc(crtc, plane_id) {
1614		int plane_extra;
1615
1616		if (fifo_left == 0)
1617			break;
1618
1619		if ((active_planes & BIT(plane_id)) == 0)
1620			continue;
1621
1622		plane_extra = min(fifo_extra, fifo_left);
1623		fifo_state->plane[plane_id] += plane_extra;
1624		fifo_left -= plane_extra;
1625	}
1626
1627	drm_WARN_ON(display->drm, active_planes != 0 && fifo_left != 0);
1628
1629	/* give it all to the first plane if none are active */
1630	if (active_planes == 0) {
1631		drm_WARN_ON(display->drm, fifo_left != fifo_size);
1632		fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1633	}
1634
1635	return 0;
1636}
1637
1638/* mark all levels starting from 'level' as invalid */
1639static void vlv_invalidate_wms(struct intel_crtc *crtc,
1640			       struct vlv_wm_state *wm_state, int level)
1641{
1642	struct intel_display *display = to_intel_display(crtc);
1643
1644	for (; level < display->wm.num_levels; level++) {
1645		enum plane_id plane_id;
1646
1647		for_each_plane_id_on_crtc(crtc, plane_id)
1648			wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1649
1650		wm_state->sr[level].cursor = USHRT_MAX;
1651		wm_state->sr[level].plane = USHRT_MAX;
1652	}
1653}
1654
1655static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1656{
1657	if (wm > fifo_size)
1658		return USHRT_MAX;
1659	else
1660		return fifo_size - wm;
1661}
1662
1663/*
1664 * Starting from 'level' set all higher
1665 * levels to 'value' in the "raw" watermarks.
1666 */
1667static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1668				 int level, enum plane_id plane_id, u16 value)
1669{
1670	struct intel_display *display = to_intel_display(crtc_state);
1671	bool dirty = false;
1672
1673	for (; level < display->wm.num_levels; level++) {
1674		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1675
1676		dirty |= raw->plane[plane_id] != value;
1677		raw->plane[plane_id] = value;
1678	}
1679
1680	return dirty;
1681}
1682
1683static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1684				     const struct intel_plane_state *plane_state)
1685{
1686	struct intel_display *display = to_intel_display(crtc_state);
1687	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1688	enum plane_id plane_id = plane->id;
1689	int level;
1690	bool dirty = false;
1691
1692	if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1693		dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1694		goto out;
1695	}
1696
1697	for (level = 0; level < display->wm.num_levels; level++) {
1698		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1699		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1700		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1701
1702		if (wm > max_wm)
1703			break;
1704
1705		dirty |= raw->plane[plane_id] != wm;
1706		raw->plane[plane_id] = wm;
1707	}
1708
1709	/* mark all higher levels as invalid */
1710	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1711
1712out:
1713	if (dirty)
1714		drm_dbg_kms(display->drm,
1715			    "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1716			    plane->base.name,
1717			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1718			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1719			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1720
1721	return dirty;
1722}
1723
1724static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1725				      enum plane_id plane_id, int level)
1726{
1727	const struct g4x_pipe_wm *raw =
1728		&crtc_state->wm.vlv.raw[level];
1729	const struct vlv_fifo_state *fifo_state =
1730		&crtc_state->wm.vlv.fifo_state;
1731
1732	return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1733}
1734
1735static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1736{
1737	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1738		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1739		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1740		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1741}
1742
1743static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1744{
1745	struct intel_display *display = to_intel_display(crtc_state);
1746	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1747	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1748	const struct vlv_fifo_state *fifo_state =
1749		&crtc_state->wm.vlv.fifo_state;
1750	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1751	int num_active_planes = hweight8(active_planes);
1752	enum plane_id plane_id;
1753	int level;
1754
1755	/* initially allow all levels */
1756	wm_state->num_levels = display->wm.num_levels;
1757	/*
1758	 * Note that enabling cxsr with no primary/sprite planes
1759	 * enabled can wedge the pipe. Hence we only allow cxsr
1760	 * with exactly one enabled primary/sprite plane.
1761	 */
1762	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1763
1764	for (level = 0; level < wm_state->num_levels; level++) {
1765		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1766		const int sr_fifo_size = INTEL_NUM_PIPES(display) * 512 - 1;
1767
1768		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1769			break;
1770
1771		for_each_plane_id_on_crtc(crtc, plane_id) {
1772			wm_state->wm[level].plane[plane_id] =
1773				vlv_invert_wm_value(raw->plane[plane_id],
1774						    fifo_state->plane[plane_id]);
1775		}
1776
1777		wm_state->sr[level].plane =
1778			vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1779						 raw->plane[PLANE_SPRITE0],
1780						 raw->plane[PLANE_SPRITE1]),
1781					    sr_fifo_size);
1782
1783		wm_state->sr[level].cursor =
1784			vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1785					    63);
1786	}
1787
1788	if (level == 0)
1789		return -EINVAL;
1790
1791	/* limit to only levels we can actually handle */
1792	wm_state->num_levels = level;
1793
1794	/* invalidate the higher levels */
1795	vlv_invalidate_wms(crtc, wm_state, level);
1796
1797	return 0;
1798}
1799
1800static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1801			       struct intel_crtc *crtc)
1802{
1803	struct intel_crtc_state *crtc_state =
1804		intel_atomic_get_new_crtc_state(state, crtc);
1805	const struct intel_plane_state *old_plane_state;
1806	const struct intel_plane_state *new_plane_state;
1807	struct intel_plane *plane;
1808	unsigned int dirty = 0;
1809	int i;
1810
1811	for_each_oldnew_intel_plane_in_state(state, plane,
1812					     old_plane_state,
1813					     new_plane_state, i) {
1814		if (new_plane_state->hw.crtc != &crtc->base &&
1815		    old_plane_state->hw.crtc != &crtc->base)
1816			continue;
1817
1818		if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1819			dirty |= BIT(plane->id);
1820	}
1821
1822	/*
1823	 * DSPARB registers may have been reset due to the
1824	 * power well being turned off. Make sure we restore
1825	 * them to a consistent state even if no primary/sprite
1826	 * planes are initially active. We also force a FIFO
1827	 * recomputation so that we are sure to sanitize the
1828	 * FIFO setting we took over from the BIOS even if there
1829	 * are no active planes on the crtc.
1830	 */
1831	if (intel_crtc_needs_modeset(crtc_state))
1832		dirty = ~0;
1833
1834	if (!dirty)
1835		return 0;
1836
1837	/* cursor changes don't warrant a FIFO recompute */
1838	if (dirty & ~BIT(PLANE_CURSOR)) {
1839		const struct intel_crtc_state *old_crtc_state =
1840			intel_atomic_get_old_crtc_state(state, crtc);
1841		const struct vlv_fifo_state *old_fifo_state =
1842			&old_crtc_state->wm.vlv.fifo_state;
1843		const struct vlv_fifo_state *new_fifo_state =
1844			&crtc_state->wm.vlv.fifo_state;
1845		int ret;
1846
1847		ret = vlv_compute_fifo(crtc_state);
1848		if (ret)
1849			return ret;
1850
1851		if (intel_crtc_needs_modeset(crtc_state) ||
1852		    memcmp(old_fifo_state, new_fifo_state,
1853			   sizeof(*new_fifo_state)) != 0)
1854			crtc_state->fifo_changed = true;
1855	}
1856
1857	return _vlv_compute_pipe_wm(crtc_state);
1858}
1859
1860#define VLV_FIFO(plane, value) \
1861	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1862
1863static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1864				   struct intel_crtc *crtc)
1865{
1866	struct intel_display *display = to_intel_display(crtc);
1867	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1868	struct intel_uncore *uncore = &dev_priv->uncore;
1869	const struct intel_crtc_state *crtc_state =
1870		intel_atomic_get_new_crtc_state(state, crtc);
1871	const struct vlv_fifo_state *fifo_state =
1872		&crtc_state->wm.vlv.fifo_state;
1873	int sprite0_start, sprite1_start, fifo_size;
1874	u32 dsparb, dsparb2, dsparb3;
1875
1876	if (!crtc_state->fifo_changed)
1877		return;
1878
1879	sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1880	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1881	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1882
1883	drm_WARN_ON(display->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1884	drm_WARN_ON(display->drm, fifo_size != 511);
1885
1886	trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1887
1888	/*
1889	 * uncore.lock serves a double purpose here. It allows us to
1890	 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1891	 * it protects the DSPARB registers from getting clobbered by
1892	 * parallel updates from multiple pipes.
1893	 *
1894	 * intel_pipe_update_start() has already disabled interrupts
1895	 * for us, so a plain spin_lock() is sufficient here.
1896	 */
1897	spin_lock(&uncore->lock);
1898
1899	switch (crtc->pipe) {
1900	case PIPE_A:
1901		dsparb = intel_de_read_fw(display, DSPARB(display));
1902		dsparb2 = intel_de_read_fw(display, DSPARB2);
1903
1904		dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1905			    VLV_FIFO(SPRITEB, 0xff));
1906		dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1907			   VLV_FIFO(SPRITEB, sprite1_start));
1908
1909		dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1910			     VLV_FIFO(SPRITEB_HI, 0x1));
1911		dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1912			   VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1913
1914		intel_de_write_fw(display, DSPARB(display), dsparb);
1915		intel_de_write_fw(display, DSPARB2, dsparb2);
1916		break;
1917	case PIPE_B:
1918		dsparb = intel_de_read_fw(display, DSPARB(display));
1919		dsparb2 = intel_de_read_fw(display, DSPARB2);
1920
1921		dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1922			    VLV_FIFO(SPRITED, 0xff));
1923		dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1924			   VLV_FIFO(SPRITED, sprite1_start));
1925
1926		dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1927			     VLV_FIFO(SPRITED_HI, 0xff));
1928		dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1929			   VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1930
1931		intel_de_write_fw(display, DSPARB(display), dsparb);
1932		intel_de_write_fw(display, DSPARB2, dsparb2);
1933		break;
1934	case PIPE_C:
1935		dsparb3 = intel_de_read_fw(display, DSPARB3);
1936		dsparb2 = intel_de_read_fw(display, DSPARB2);
1937
1938		dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1939			     VLV_FIFO(SPRITEF, 0xff));
1940		dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1941			    VLV_FIFO(SPRITEF, sprite1_start));
1942
1943		dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1944			     VLV_FIFO(SPRITEF_HI, 0xff));
1945		dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1946			   VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1947
1948		intel_de_write_fw(display, DSPARB3, dsparb3);
1949		intel_de_write_fw(display, DSPARB2, dsparb2);
1950		break;
1951	default:
1952		break;
1953	}
1954
1955	intel_de_read_fw(display, DSPARB(display));
1956
1957	spin_unlock(&uncore->lock);
1958}
1959
1960#undef VLV_FIFO
1961
1962static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1963				       struct intel_crtc *crtc)
1964{
1965	struct intel_crtc_state *new_crtc_state =
1966		intel_atomic_get_new_crtc_state(state, crtc);
1967	const struct intel_crtc_state *old_crtc_state =
1968		intel_atomic_get_old_crtc_state(state, crtc);
1969	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1970	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1971	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1972	int level;
1973
1974	if (!new_crtc_state->hw.active ||
1975	    intel_crtc_needs_modeset(new_crtc_state)) {
1976		*intermediate = *optimal;
1977
1978		intermediate->cxsr = false;
1979		goto out;
1980	}
1981
1982	intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1983	intermediate->cxsr = optimal->cxsr && active->cxsr &&
1984		!new_crtc_state->disable_cxsr;
1985
1986	for (level = 0; level < intermediate->num_levels; level++) {
1987		enum plane_id plane_id;
1988
1989		for_each_plane_id_on_crtc(crtc, plane_id) {
1990			intermediate->wm[level].plane[plane_id] =
1991				min(optimal->wm[level].plane[plane_id],
1992				    active->wm[level].plane[plane_id]);
1993		}
1994
1995		intermediate->sr[level].plane = min(optimal->sr[level].plane,
1996						    active->sr[level].plane);
1997		intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1998						     active->sr[level].cursor);
1999	}
2000
2001	vlv_invalidate_wms(crtc, intermediate, level);
2002
2003out:
2004	/*
2005	 * If our intermediate WM are identical to the final WM, then we can
2006	 * omit the post-vblank programming; only update if it's different.
2007	 */
2008	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2009		new_crtc_state->wm.need_postvbl_update = true;
2010
2011	return 0;
2012}
2013
2014static int vlv_compute_watermarks(struct intel_atomic_state *state,
2015				  struct intel_crtc *crtc)
2016{
2017	int ret;
2018
2019	ret = vlv_compute_pipe_wm(state, crtc);
2020	if (ret)
2021		return ret;
2022
2023	ret = vlv_compute_intermediate_wm(state, crtc);
2024	if (ret)
2025		return ret;
2026
2027	return 0;
2028}
2029
2030static void vlv_merge_wm(struct intel_display *display,
2031			 struct vlv_wm_values *wm)
2032{
2033	struct intel_crtc *crtc;
2034	int num_active_pipes = 0;
2035
2036	wm->level = display->wm.num_levels - 1;
2037	wm->cxsr = true;
2038
2039	for_each_intel_crtc(display->drm, crtc) {
2040		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2041
2042		if (!crtc->active)
2043			continue;
2044
2045		if (!wm_state->cxsr)
2046			wm->cxsr = false;
2047
2048		num_active_pipes++;
2049		wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2050	}
2051
2052	if (num_active_pipes != 1)
2053		wm->cxsr = false;
2054
2055	if (num_active_pipes > 1)
2056		wm->level = VLV_WM_LEVEL_PM2;
2057
2058	for_each_intel_crtc(display->drm, crtc) {
2059		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2060		enum pipe pipe = crtc->pipe;
2061
2062		wm->pipe[pipe] = wm_state->wm[wm->level];
2063		if (crtc->active && wm->cxsr)
2064			wm->sr = wm_state->sr[wm->level];
2065
2066		wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2067		wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2068		wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2069		wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2070	}
2071}
2072
2073static void vlv_program_watermarks(struct intel_display *display)
2074{
2075	struct vlv_wm_values *old_wm = &display->wm.vlv;
2076	struct vlv_wm_values new_wm = {};
2077
2078	vlv_merge_wm(display, &new_wm);
2079
2080	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2081		return;
2082
2083	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2084		chv_set_memory_dvfs(display, false);
2085
2086	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2087		chv_set_memory_pm5(display, false);
2088
2089	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2090		_intel_set_memory_cxsr(display, false);
2091
2092	vlv_write_wm_values(display, &new_wm);
2093
2094	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2095		_intel_set_memory_cxsr(display, true);
2096
2097	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2098		chv_set_memory_pm5(display, true);
2099
2100	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2101		chv_set_memory_dvfs(display, true);
2102
2103	*old_wm = new_wm;
2104}
2105
2106static void vlv_initial_watermarks(struct intel_atomic_state *state,
2107				   struct intel_crtc *crtc)
2108{
2109	struct intel_display *display = to_intel_display(crtc);
2110	const struct intel_crtc_state *crtc_state =
2111		intel_atomic_get_new_crtc_state(state, crtc);
2112
2113	mutex_lock(&display->wm.wm_mutex);
2114	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2115	vlv_program_watermarks(display);
2116	mutex_unlock(&display->wm.wm_mutex);
2117}
2118
2119static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2120				    struct intel_crtc *crtc)
2121{
2122	struct intel_display *display = to_intel_display(crtc);
2123	const struct intel_crtc_state *crtc_state =
2124		intel_atomic_get_new_crtc_state(state, crtc);
2125
2126	if (!crtc_state->wm.need_postvbl_update)
2127		return;
2128
2129	mutex_lock(&display->wm.wm_mutex);
2130	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2131	vlv_program_watermarks(display);
2132	mutex_unlock(&display->wm.wm_mutex);
2133}
2134
2135static void i965_update_wm(struct intel_display *display)
2136{
2137	struct intel_crtc *crtc;
2138	int srwm = 1;
2139	int cursor_sr = 16;
2140	bool cxsr_enabled;
2141
2142	/* Calc sr entries for one plane configs */
2143	crtc = single_enabled_crtc(display);
2144	if (crtc) {
2145		/* self-refresh has much higher latency */
2146		static const int sr_latency_ns = 12000;
2147		const struct drm_display_mode *pipe_mode =
2148			&crtc->config->hw.pipe_mode;
2149		const struct drm_framebuffer *fb =
2150			crtc->base.primary->state->fb;
2151		int pixel_rate = crtc->config->pixel_rate;
2152		int htotal = pipe_mode->crtc_htotal;
2153		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2154		int cpp = fb->format->cpp[0];
2155		int entries;
2156
2157		entries = intel_wm_method2(pixel_rate, htotal,
2158					   width, cpp, sr_latency_ns / 100);
2159		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2160		srwm = I965_FIFO_SIZE - entries;
2161		if (srwm < 0)
2162			srwm = 1;
2163		srwm &= 0x1ff;
2164		drm_dbg_kms(display->drm,
2165			    "self-refresh entries: %d, wm: %d\n",
2166			    entries, srwm);
2167
2168		entries = intel_wm_method2(pixel_rate, htotal,
2169					   crtc->base.cursor->state->crtc_w, 4,
2170					   sr_latency_ns / 100);
2171		entries = DIV_ROUND_UP(entries,
2172				       i965_cursor_wm_info.cacheline_size) +
2173			i965_cursor_wm_info.guard_size;
2174
2175		cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2176		if (cursor_sr > i965_cursor_wm_info.max_wm)
2177			cursor_sr = i965_cursor_wm_info.max_wm;
2178
2179		drm_dbg_kms(display->drm,
2180			    "self-refresh watermark: display plane %d "
2181			    "cursor %d\n", srwm, cursor_sr);
2182
2183		cxsr_enabled = true;
2184	} else {
2185		cxsr_enabled = false;
2186		/* Turn off self refresh if both pipes are enabled */
2187		intel_set_memory_cxsr(display, false);
2188	}
2189
2190	drm_dbg_kms(display->drm,
2191		    "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2192		    srwm);
2193
2194	/* 965 has limitations... */
2195	intel_de_write(display, DSPFW1(display),
2196		       FW_WM(srwm, SR) |
2197		       FW_WM(8, CURSORB) |
2198		       FW_WM(8, PLANEB) |
2199		       FW_WM(8, PLANEA));
2200	intel_de_write(display, DSPFW2(display),
2201		       FW_WM(8, CURSORA) |
2202		       FW_WM(8, PLANEC_OLD));
2203	/* update cursor SR watermark */
2204	intel_de_write(display, DSPFW3(display),
2205		       FW_WM(cursor_sr, CURSOR_SR));
2206
2207	if (cxsr_enabled)
2208		intel_set_memory_cxsr(display, true);
2209}
2210
2211#undef FW_WM
2212
2213static struct intel_crtc *intel_crtc_for_plane(struct intel_display *display,
2214					       enum i9xx_plane_id i9xx_plane)
2215{
2216	struct intel_plane *plane;
2217
2218	for_each_intel_plane(display->drm, plane) {
2219		if (plane->id == PLANE_PRIMARY &&
2220		    plane->i9xx_plane == i9xx_plane)
2221			return intel_crtc_for_pipe(display, plane->pipe);
2222	}
2223
2224	return NULL;
2225}
2226
2227static void i9xx_update_wm(struct intel_display *display)
2228{
2229	const struct intel_watermark_params *wm_info;
2230	u32 fwater_lo;
2231	u32 fwater_hi;
2232	int cwm, srwm = 1;
2233	int fifo_size;
2234	int planea_wm, planeb_wm;
2235	struct intel_crtc *crtc;
2236
2237	if (display->platform.i945gm)
2238		wm_info = &i945_wm_info;
2239	else if (DISPLAY_VER(display) != 2)
2240		wm_info = &i915_wm_info;
2241	else
2242		wm_info = &i830_a_wm_info;
2243
2244	if (DISPLAY_VER(display) == 2)
2245		fifo_size = i830_get_fifo_size(display, PLANE_A);
2246	else
2247		fifo_size = i9xx_get_fifo_size(display, PLANE_A);
2248	crtc = intel_crtc_for_plane(display, PLANE_A);
2249	if (intel_crtc_active(crtc)) {
2250		const struct drm_framebuffer *fb =
2251			crtc->base.primary->state->fb;
2252		int cpp;
2253
2254		if (DISPLAY_VER(display) == 2)
2255			cpp = 4;
2256		else
2257			cpp = fb->format->cpp[0];
2258
2259		planea_wm = intel_calculate_wm(display, crtc->config->pixel_rate,
2260					       wm_info, fifo_size, cpp,
2261					       pessimal_latency_ns);
2262	} else {
2263		planea_wm = fifo_size - wm_info->guard_size;
2264		if (planea_wm > (long)wm_info->max_wm)
2265			planea_wm = wm_info->max_wm;
2266	}
2267
2268	if (DISPLAY_VER(display) == 2)
2269		wm_info = &i830_bc_wm_info;
2270
2271	if (DISPLAY_VER(display) == 2)
2272		fifo_size = i830_get_fifo_size(display, PLANE_B);
2273	else
2274		fifo_size = i9xx_get_fifo_size(display, PLANE_B);
2275	crtc = intel_crtc_for_plane(display, PLANE_B);
2276	if (intel_crtc_active(crtc)) {
2277		const struct drm_framebuffer *fb =
2278			crtc->base.primary->state->fb;
2279		int cpp;
2280
2281		if (DISPLAY_VER(display) == 2)
2282			cpp = 4;
2283		else
2284			cpp = fb->format->cpp[0];
2285
2286		planeb_wm = intel_calculate_wm(display, crtc->config->pixel_rate,
2287					       wm_info, fifo_size, cpp,
2288					       pessimal_latency_ns);
2289	} else {
2290		planeb_wm = fifo_size - wm_info->guard_size;
2291		if (planeb_wm > (long)wm_info->max_wm)
2292			planeb_wm = wm_info->max_wm;
2293	}
2294
2295	drm_dbg_kms(display->drm,
2296		    "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2297
2298	crtc = single_enabled_crtc(display);
2299	if (display->platform.i915gm && crtc) {
2300		const struct drm_framebuffer *fb =
2301			crtc->base.primary->state->fb;
2302
2303		/* self-refresh seems busted with untiled */
2304		if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
2305			crtc = NULL;
2306	}
2307
2308	/*
2309	 * Overlay gets an aggressive default since video jitter is bad.
2310	 */
2311	cwm = 2;
2312
2313	/* Play safe and disable self-refresh before adjusting watermarks. */
2314	intel_set_memory_cxsr(display, false);
2315
2316	/* Calc sr entries for one plane configs */
2317	if (HAS_FW_BLC(display) && crtc) {
2318		/* self-refresh has much higher latency */
2319		static const int sr_latency_ns = 6000;
2320		const struct drm_display_mode *pipe_mode =
2321			&crtc->config->hw.pipe_mode;
2322		const struct drm_framebuffer *fb =
2323			crtc->base.primary->state->fb;
2324		int pixel_rate = crtc->config->pixel_rate;
2325		int htotal = pipe_mode->crtc_htotal;
2326		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2327		int cpp;
2328		int entries;
2329
2330		if (display->platform.i915gm || display->platform.i945gm)
2331			cpp = 4;
2332		else
2333			cpp = fb->format->cpp[0];
2334
2335		entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2336					   sr_latency_ns / 100);
2337		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2338		drm_dbg_kms(display->drm,
2339			    "self-refresh entries: %d\n", entries);
2340		srwm = wm_info->fifo_size - entries;
2341		if (srwm < 0)
2342			srwm = 1;
2343
2344		if (display->platform.i945g || display->platform.i945gm)
2345			intel_de_write(display, FW_BLC_SELF,
2346				       FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2347		else
2348			intel_de_write(display, FW_BLC_SELF, srwm & 0x3f);
2349	}
2350
2351	drm_dbg_kms(display->drm,
2352		    "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2353		     planea_wm, planeb_wm, cwm, srwm);
2354
2355	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2356	fwater_hi = (cwm & 0x1f);
2357
2358	/* Set request length to 8 cachelines per fetch */
2359	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2360	fwater_hi = fwater_hi | (1 << 8);
2361
2362	intel_de_write(display, FW_BLC, fwater_lo);
2363	intel_de_write(display, FW_BLC2, fwater_hi);
2364
2365	if (crtc)
2366		intel_set_memory_cxsr(display, true);
2367}
2368
2369static void i845_update_wm(struct intel_display *display)
2370{
2371	struct intel_crtc *crtc;
2372	u32 fwater_lo;
2373	int planea_wm;
2374
2375	crtc = single_enabled_crtc(display);
2376	if (crtc == NULL)
2377		return;
2378
2379	planea_wm = intel_calculate_wm(display, crtc->config->pixel_rate,
2380				       &i845_wm_info,
2381				       i845_get_fifo_size(display, PLANE_A),
2382				       4, pessimal_latency_ns);
2383	fwater_lo = intel_de_read(display, FW_BLC) & ~0xfff;
2384	fwater_lo |= (3<<8) | planea_wm;
2385
2386	drm_dbg_kms(display->drm,
2387		    "Setting FIFO watermarks - A: %d\n", planea_wm);
2388
2389	intel_de_write(display, FW_BLC, fwater_lo);
2390}
2391
2392/* latency must be in 0.1us units. */
2393static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2394				   unsigned int cpp,
2395				   unsigned int latency)
2396{
2397	unsigned int ret;
2398
2399	ret = intel_wm_method1(pixel_rate, cpp, latency);
2400	ret = DIV_ROUND_UP(ret, 64) + 2;
2401
2402	return ret;
2403}
2404
2405/* latency must be in 0.1us units. */
2406static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2407				   unsigned int htotal,
2408				   unsigned int width,
2409				   unsigned int cpp,
2410				   unsigned int latency)
2411{
2412	unsigned int ret;
2413
2414	ret = intel_wm_method2(pixel_rate, htotal,
2415			       width, cpp, latency);
2416	ret = DIV_ROUND_UP(ret, 64) + 2;
2417
2418	return ret;
2419}
2420
2421static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2422{
2423	/*
2424	 * Neither of these should be possible since this function shouldn't be
2425	 * called if the CRTC is off or the plane is invisible.  But let's be
2426	 * extra paranoid to avoid a potential divide-by-zero if we screw up
2427	 * elsewhere in the driver.
2428	 */
2429	if (WARN_ON(!cpp))
2430		return 0;
2431	if (WARN_ON(!horiz_pixels))
2432		return 0;
2433
2434	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2435}
2436
2437struct ilk_wm_maximums {
2438	u16 pri;
2439	u16 spr;
2440	u16 cur;
2441	u16 fbc;
2442};
2443
2444/*
2445 * For both WM_PIPE and WM_LP.
2446 * mem_value must be in 0.1us units.
2447 */
2448static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2449			      const struct intel_plane_state *plane_state,
2450			      u32 mem_value, bool is_lp)
2451{
2452	u32 method1, method2;
2453	int cpp;
2454
2455	if (mem_value == 0)
2456		return U32_MAX;
2457
2458	if (!intel_wm_plane_visible(crtc_state, plane_state))
2459		return 0;
2460
2461	cpp = plane_state->hw.fb->format->cpp[0];
2462
2463	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2464
2465	if (!is_lp)
2466		return method1;
2467
2468	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2469				 crtc_state->hw.pipe_mode.crtc_htotal,
2470				 drm_rect_width(&plane_state->uapi.src) >> 16,
2471				 cpp, mem_value);
2472
2473	return min(method1, method2);
2474}
2475
2476/*
2477 * For both WM_PIPE and WM_LP.
2478 * mem_value must be in 0.1us units.
2479 */
2480static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2481			      const struct intel_plane_state *plane_state,
2482			      u32 mem_value)
2483{
2484	u32 method1, method2;
2485	int cpp;
2486
2487	if (mem_value == 0)
2488		return U32_MAX;
2489
2490	if (!intel_wm_plane_visible(crtc_state, plane_state))
2491		return 0;
2492
2493	cpp = plane_state->hw.fb->format->cpp[0];
2494
2495	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2496	method2 = ilk_wm_method2(crtc_state->pixel_rate,
2497				 crtc_state->hw.pipe_mode.crtc_htotal,
2498				 drm_rect_width(&plane_state->uapi.src) >> 16,
2499				 cpp, mem_value);
2500	return min(method1, method2);
2501}
2502
2503/*
2504 * For both WM_PIPE and WM_LP.
2505 * mem_value must be in 0.1us units.
2506 */
2507static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2508			      const struct intel_plane_state *plane_state,
2509			      u32 mem_value)
2510{
2511	int cpp;
2512
2513	if (mem_value == 0)
2514		return U32_MAX;
2515
2516	if (!intel_wm_plane_visible(crtc_state, plane_state))
2517		return 0;
2518
2519	cpp = plane_state->hw.fb->format->cpp[0];
2520
2521	return ilk_wm_method2(crtc_state->pixel_rate,
2522			      crtc_state->hw.pipe_mode.crtc_htotal,
2523			      drm_rect_width(&plane_state->uapi.src) >> 16,
2524			      cpp, mem_value);
2525}
2526
2527/* Only for WM_LP. */
2528static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2529			      const struct intel_plane_state *plane_state,
2530			      u32 pri_val)
2531{
2532	int cpp;
2533
2534	if (!intel_wm_plane_visible(crtc_state, plane_state))
2535		return 0;
2536
2537	cpp = plane_state->hw.fb->format->cpp[0];
2538
2539	return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2540			  cpp);
2541}
2542
2543static unsigned int
2544ilk_display_fifo_size(struct intel_display *display)
2545{
2546	if (DISPLAY_VER(display) >= 8)
2547		return 3072;
2548	else if (DISPLAY_VER(display) >= 7)
2549		return 768;
2550	else
2551		return 512;
2552}
2553
2554static unsigned int
2555ilk_plane_wm_reg_max(struct intel_display *display,
2556		     int level, bool is_sprite)
2557{
2558	if (DISPLAY_VER(display) >= 8)
2559		/* BDW primary/sprite plane watermarks */
2560		return level == 0 ? 255 : 2047;
2561	else if (DISPLAY_VER(display) >= 7)
2562		/* IVB/HSW primary/sprite plane watermarks */
2563		return level == 0 ? 127 : 1023;
2564	else if (!is_sprite)
2565		/* ILK/SNB primary plane watermarks */
2566		return level == 0 ? 127 : 511;
2567	else
2568		/* ILK/SNB sprite plane watermarks */
2569		return level == 0 ? 63 : 255;
2570}
2571
2572static unsigned int
2573ilk_cursor_wm_reg_max(struct intel_display *display, int level)
2574{
2575	if (DISPLAY_VER(display) >= 7)
2576		return level == 0 ? 63 : 255;
2577	else
2578		return level == 0 ? 31 : 63;
2579}
2580
2581static unsigned int ilk_fbc_wm_reg_max(struct intel_display *display)
2582{
2583	if (DISPLAY_VER(display) >= 8)
2584		return 31;
2585	else
2586		return 15;
2587}
2588
2589/* Calculate the maximum primary/sprite plane watermark */
2590static unsigned int ilk_plane_wm_max(struct intel_display *display,
2591				     int level,
2592				     const struct intel_wm_config *config,
2593				     enum intel_ddb_partitioning ddb_partitioning,
2594				     bool is_sprite)
2595{
2596	unsigned int fifo_size = ilk_display_fifo_size(display);
2597
2598	/* if sprites aren't enabled, sprites get nothing */
2599	if (is_sprite && !config->sprites_enabled)
2600		return 0;
2601
2602	/* HSW allows LP1+ watermarks even with multiple pipes */
2603	if (level == 0 || config->num_pipes_active > 1) {
2604		fifo_size /= INTEL_NUM_PIPES(display);
2605
2606		/*
2607		 * For some reason the non self refresh
2608		 * FIFO size is only half of the self
2609		 * refresh FIFO size on ILK/SNB.
2610		 */
2611		if (DISPLAY_VER(display) < 7)
2612			fifo_size /= 2;
2613	}
2614
2615	if (config->sprites_enabled) {
2616		/* level 0 is always calculated with 1:1 split */
2617		if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2618			if (is_sprite)
2619				fifo_size *= 5;
2620			fifo_size /= 6;
2621		} else {
2622			fifo_size /= 2;
2623		}
2624	}
2625
2626	/* clamp to max that the registers can hold */
2627	return min(fifo_size, ilk_plane_wm_reg_max(display, level, is_sprite));
2628}
2629
2630/* Calculate the maximum cursor plane watermark */
2631static unsigned int ilk_cursor_wm_max(struct intel_display *display,
2632				      int level,
2633				      const struct intel_wm_config *config)
2634{
2635	/* HSW LP1+ watermarks w/ multiple pipes */
2636	if (level > 0 && config->num_pipes_active > 1)
2637		return 64;
2638
2639	/* otherwise just report max that registers can hold */
2640	return ilk_cursor_wm_reg_max(display, level);
2641}
2642
2643static void ilk_compute_wm_maximums(struct intel_display *display,
2644				    int level,
2645				    const struct intel_wm_config *config,
2646				    enum intel_ddb_partitioning ddb_partitioning,
2647				    struct ilk_wm_maximums *max)
2648{
2649	max->pri = ilk_plane_wm_max(display, level, config, ddb_partitioning, false);
2650	max->spr = ilk_plane_wm_max(display, level, config, ddb_partitioning, true);
2651	max->cur = ilk_cursor_wm_max(display, level, config);
2652	max->fbc = ilk_fbc_wm_reg_max(display);
2653}
2654
2655static void ilk_compute_wm_reg_maximums(struct intel_display *display,
2656					int level,
2657					struct ilk_wm_maximums *max)
2658{
2659	max->pri = ilk_plane_wm_reg_max(display, level, false);
2660	max->spr = ilk_plane_wm_reg_max(display, level, true);
2661	max->cur = ilk_cursor_wm_reg_max(display, level);
2662	max->fbc = ilk_fbc_wm_reg_max(display);
2663}
2664
2665static bool ilk_validate_wm_level(struct intel_display *display,
2666				  int level,
2667				  const struct ilk_wm_maximums *max,
2668				  struct intel_wm_level *result)
2669{
2670	bool ret;
2671
2672	/* already determined to be invalid? */
2673	if (!result->enable)
2674		return false;
2675
2676	result->enable = result->pri_val <= max->pri &&
2677			 result->spr_val <= max->spr &&
2678			 result->cur_val <= max->cur;
2679
2680	ret = result->enable;
2681
2682	/*
2683	 * HACK until we can pre-compute everything,
2684	 * and thus fail gracefully if LP0 watermarks
2685	 * are exceeded...
2686	 */
2687	if (level == 0 && !result->enable) {
2688		if (result->pri_val > max->pri)
2689			drm_dbg_kms(display->drm,
2690				    "Primary WM%d too large %u (max %u)\n",
2691				    level, result->pri_val, max->pri);
2692		if (result->spr_val > max->spr)
2693			drm_dbg_kms(display->drm,
2694				    "Sprite WM%d too large %u (max %u)\n",
2695				    level, result->spr_val, max->spr);
2696		if (result->cur_val > max->cur)
2697			drm_dbg_kms(display->drm,
2698				    "Cursor WM%d too large %u (max %u)\n",
2699				    level, result->cur_val, max->cur);
2700
2701		result->pri_val = min_t(u32, result->pri_val, max->pri);
2702		result->spr_val = min_t(u32, result->spr_val, max->spr);
2703		result->cur_val = min_t(u32, result->cur_val, max->cur);
2704		result->enable = true;
2705	}
2706
2707	return ret;
2708}
2709
2710static void ilk_compute_wm_level(struct intel_display *display,
2711				 const struct intel_crtc *crtc,
2712				 int level,
2713				 struct intel_crtc_state *crtc_state,
2714				 const struct intel_plane_state *pristate,
2715				 const struct intel_plane_state *sprstate,
2716				 const struct intel_plane_state *curstate,
2717				 struct intel_wm_level *result)
2718{
2719	u16 pri_latency = display->wm.pri_latency[level];
2720	u16 spr_latency = display->wm.spr_latency[level];
2721	u16 cur_latency = display->wm.cur_latency[level];
2722
2723	/* WM1+ latency values stored in 0.5us units */
2724	if (level > 0) {
2725		pri_latency *= 5;
2726		spr_latency *= 5;
2727		cur_latency *= 5;
2728	}
2729
2730	if (pristate) {
2731		result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2732						     pri_latency, level);
2733		result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2734	}
2735
2736	if (sprstate)
2737		result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2738
2739	if (curstate)
2740		result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2741
2742	result->enable = true;
2743}
2744
2745static void hsw_read_wm_latency(struct intel_display *display, u16 wm[])
2746{
2747	struct drm_i915_private *i915 = to_i915(display->drm);
2748	u64 sskpd;
2749
2750	display->wm.num_levels = 5;
2751
2752	sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2753
2754	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2755	if (wm[0] == 0)
2756		wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2757	wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2758	wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2759	wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2760	wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2761}
2762
2763static void snb_read_wm_latency(struct intel_display *display, u16 wm[])
2764{
2765	struct drm_i915_private *i915 = to_i915(display->drm);
2766	u32 sskpd;
2767
2768	display->wm.num_levels = 4;
2769
2770	sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2771
2772	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2773	wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2774	wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2775	wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2776}
2777
2778static void ilk_read_wm_latency(struct intel_display *display, u16 wm[])
2779{
2780	struct drm_i915_private *i915 = to_i915(display->drm);
2781	u32 mltr;
2782
2783	display->wm.num_levels = 3;
2784
2785	mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2786
2787	/* ILK primary LP0 latency is 700 ns */
2788	wm[0] = 7;
2789	wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2790	wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2791}
2792
2793static void intel_fixup_spr_wm_latency(struct intel_display *display, u16 wm[5])
2794{
2795	/* ILK sprite LP0 latency is 1300 ns */
2796	if (DISPLAY_VER(display) == 5)
2797		wm[0] = 13;
2798}
2799
2800static void intel_fixup_cur_wm_latency(struct intel_display *display, u16 wm[5])
2801{
2802	/* ILK cursor LP0 latency is 1300 ns */
2803	if (DISPLAY_VER(display) == 5)
2804		wm[0] = 13;
2805}
2806
2807static bool ilk_increase_wm_latency(struct intel_display *display, u16 wm[5], u16 min)
2808{
2809	int level;
2810
2811	if (wm[0] >= min)
2812		return false;
2813
2814	wm[0] = max(wm[0], min);
2815	for (level = 1; level < display->wm.num_levels; level++)
2816		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2817
2818	return true;
2819}
2820
2821static void snb_wm_latency_quirk(struct intel_display *display)
2822{
2823	bool changed;
2824
2825	/*
2826	 * The BIOS provided WM memory latency values are often
2827	 * inadequate for high resolution displays. Adjust them.
2828	 */
2829	changed = ilk_increase_wm_latency(display, display->wm.pri_latency, 12);
2830	changed |= ilk_increase_wm_latency(display, display->wm.spr_latency, 12);
2831	changed |= ilk_increase_wm_latency(display, display->wm.cur_latency, 12);
2832
2833	if (!changed)
2834		return;
2835
2836	drm_dbg_kms(display->drm,
2837		    "WM latency values increased to avoid potential underruns\n");
2838	intel_print_wm_latency(display, "Primary", display->wm.pri_latency);
2839	intel_print_wm_latency(display, "Sprite", display->wm.spr_latency);
2840	intel_print_wm_latency(display, "Cursor", display->wm.cur_latency);
2841}
2842
2843static void snb_wm_lp3_irq_quirk(struct intel_display *display)
2844{
2845	/*
2846	 * On some SNB machines (Thinkpad X220 Tablet at least)
2847	 * LP3 usage can cause vblank interrupts to be lost.
2848	 * The DEIIR bit will go high but it looks like the CPU
2849	 * never gets interrupted.
2850	 *
2851	 * It's not clear whether other interrupt source could
2852	 * be affected or if this is somehow limited to vblank
2853	 * interrupts only. To play it safe we disable LP3
2854	 * watermarks entirely.
2855	 */
2856	if (display->wm.pri_latency[3] == 0 &&
2857	    display->wm.spr_latency[3] == 0 &&
2858	    display->wm.cur_latency[3] == 0)
2859		return;
2860
2861	display->wm.pri_latency[3] = 0;
2862	display->wm.spr_latency[3] = 0;
2863	display->wm.cur_latency[3] = 0;
2864
2865	drm_dbg_kms(display->drm,
2866		    "LP3 watermarks disabled due to potential for lost interrupts\n");
2867	intel_print_wm_latency(display, "Primary", display->wm.pri_latency);
2868	intel_print_wm_latency(display, "Sprite", display->wm.spr_latency);
2869	intel_print_wm_latency(display, "Cursor", display->wm.cur_latency);
2870}
2871
2872static void ilk_setup_wm_latency(struct intel_display *display)
2873{
2874	if (display->platform.broadwell || display->platform.haswell)
2875		hsw_read_wm_latency(display, display->wm.pri_latency);
2876	else if (DISPLAY_VER(display) >= 6)
2877		snb_read_wm_latency(display, display->wm.pri_latency);
2878	else
2879		ilk_read_wm_latency(display, display->wm.pri_latency);
2880
2881	memcpy(display->wm.spr_latency, display->wm.pri_latency,
2882	       sizeof(display->wm.pri_latency));
2883	memcpy(display->wm.cur_latency, display->wm.pri_latency,
2884	       sizeof(display->wm.pri_latency));
2885
2886	intel_fixup_spr_wm_latency(display, display->wm.spr_latency);
2887	intel_fixup_cur_wm_latency(display, display->wm.cur_latency);
2888
2889	intel_print_wm_latency(display, "Primary", display->wm.pri_latency);
2890	intel_print_wm_latency(display, "Sprite", display->wm.spr_latency);
2891	intel_print_wm_latency(display, "Cursor", display->wm.cur_latency);
2892
2893	if (DISPLAY_VER(display) == 6) {
2894		snb_wm_latency_quirk(display);
2895		snb_wm_lp3_irq_quirk(display);
2896	}
2897}
2898
2899static bool ilk_validate_pipe_wm(struct intel_display *display,
2900				 struct intel_pipe_wm *pipe_wm)
2901{
2902	/* LP0 watermark maximums depend on this pipe alone */
2903	const struct intel_wm_config config = {
2904		.num_pipes_active = 1,
2905		.sprites_enabled = pipe_wm->sprites_enabled,
2906		.sprites_scaled = pipe_wm->sprites_scaled,
2907	};
2908	struct ilk_wm_maximums max;
2909
2910	/* LP0 watermarks always use 1/2 DDB partitioning */
2911	ilk_compute_wm_maximums(display, 0, &config, INTEL_DDB_PART_1_2, &max);
2912
2913	/* At least LP0 must be valid */
2914	if (!ilk_validate_wm_level(display, 0, &max, &pipe_wm->wm[0])) {
2915		drm_dbg_kms(display->drm, "LP0 watermark invalid\n");
2916		return false;
2917	}
2918
2919	return true;
2920}
2921
2922/* Compute new watermarks for the pipe */
2923static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2924			       struct intel_crtc *crtc)
2925{
2926	struct intel_display *display = to_intel_display(state);
2927	struct intel_crtc_state *crtc_state =
2928		intel_atomic_get_new_crtc_state(state, crtc);
2929	struct intel_pipe_wm *pipe_wm;
2930	struct intel_plane *plane;
2931	const struct intel_plane_state *plane_state;
2932	const struct intel_plane_state *pristate = NULL;
2933	const struct intel_plane_state *sprstate = NULL;
2934	const struct intel_plane_state *curstate = NULL;
2935	struct ilk_wm_maximums max;
2936	int level, usable_level;
2937
2938	pipe_wm = &crtc_state->wm.ilk.optimal;
2939
2940	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2941		if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2942			pristate = plane_state;
2943		else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2944			sprstate = plane_state;
2945		else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2946			curstate = plane_state;
2947	}
2948
2949	pipe_wm->pipe_enabled = crtc_state->hw.active;
2950	pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2951	pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2952
2953	usable_level = display->wm.num_levels - 1;
2954
2955	/* ILK/SNB: LP2+ watermarks only w/o sprites */
2956	if (DISPLAY_VER(display) < 7 && pipe_wm->sprites_enabled)
2957		usable_level = 1;
2958
2959	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2960	if (pipe_wm->sprites_scaled)
2961		usable_level = 0;
2962
2963	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2964	ilk_compute_wm_level(display, crtc, 0, crtc_state,
2965			     pristate, sprstate, curstate, &pipe_wm->wm[0]);
2966
2967	if (!ilk_validate_pipe_wm(display, pipe_wm))
2968		return -EINVAL;
2969
2970	ilk_compute_wm_reg_maximums(display, 1, &max);
2971
2972	for (level = 1; level <= usable_level; level++) {
2973		struct intel_wm_level *wm = &pipe_wm->wm[level];
2974
2975		ilk_compute_wm_level(display, crtc, level, crtc_state,
2976				     pristate, sprstate, curstate, wm);
2977
2978		/*
2979		 * Disable any watermark level that exceeds the
2980		 * register maximums since such watermarks are
2981		 * always invalid.
2982		 */
2983		if (!ilk_validate_wm_level(display, level, &max, wm)) {
2984			memset(wm, 0, sizeof(*wm));
2985			break;
2986		}
2987	}
2988
2989	return 0;
2990}
2991
2992/*
2993 * Build a set of 'intermediate' watermark values that satisfy both the old
2994 * state and the new state.  These can be programmed to the hardware
2995 * immediately.
2996 */
2997static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2998				       struct intel_crtc *crtc)
2999{
3000	struct intel_display *display = to_intel_display(crtc);
3001	struct intel_crtc_state *new_crtc_state =
3002		intel_atomic_get_new_crtc_state(state, crtc);
3003	const struct intel_crtc_state *old_crtc_state =
3004		intel_atomic_get_old_crtc_state(state, crtc);
3005	struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate;
3006	const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal;
3007	const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal;
3008	int level;
3009
3010	/*
3011	 * Start with the final, target watermarks, then combine with the
3012	 * currently active watermarks to get values that are safe both before
3013	 * and after the vblank.
3014	 */
3015	*intermediate = *optimal;
3016	if (!new_crtc_state->hw.active ||
3017	    intel_crtc_needs_modeset(new_crtc_state) ||
3018	    state->skip_intermediate_wm)
3019		return 0;
3020
3021	intermediate->pipe_enabled |= active->pipe_enabled;
3022	intermediate->sprites_enabled |= active->sprites_enabled;
3023	intermediate->sprites_scaled |= active->sprites_scaled;
3024
3025	for (level = 0; level < display->wm.num_levels; level++) {
3026		struct intel_wm_level *intermediate_wm = &intermediate->wm[level];
3027		const struct intel_wm_level *active_wm = &active->wm[level];
3028
3029		intermediate_wm->enable &= active_wm->enable;
3030		intermediate_wm->pri_val = max(intermediate_wm->pri_val,
3031					       active_wm->pri_val);
3032		intermediate_wm->spr_val = max(intermediate_wm->spr_val,
3033					       active_wm->spr_val);
3034		intermediate_wm->cur_val = max(intermediate_wm->cur_val,
3035					       active_wm->cur_val);
3036		intermediate_wm->fbc_val = max(intermediate_wm->fbc_val,
3037					       active_wm->fbc_val);
3038	}
3039
3040	/*
3041	 * We need to make sure that these merged watermark values are
3042	 * actually a valid configuration themselves.  If they're not,
3043	 * there's no safe way to transition from the old state to
3044	 * the new state, so we need to fail the atomic transaction.
3045	 */
3046	if (!ilk_validate_pipe_wm(display, intermediate))
3047		return -EINVAL;
3048
3049	/*
3050	 * If our intermediate WM are identical to the final WM, then we can
3051	 * omit the post-vblank programming; only update if it's different.
3052	 */
3053	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
3054		new_crtc_state->wm.need_postvbl_update = true;
3055
3056	return 0;
3057}
3058
3059static int ilk_compute_watermarks(struct intel_atomic_state *state,
3060				  struct intel_crtc *crtc)
3061{
3062	int ret;
3063
3064	ret = ilk_compute_pipe_wm(state, crtc);
3065	if (ret)
3066		return ret;
3067
3068	ret = ilk_compute_intermediate_wm(state, crtc);
3069	if (ret)
3070		return ret;
3071
3072	return 0;
3073}
3074
3075/*
3076 * Merge the watermarks from all active pipes for a specific level.
3077 */
3078static void ilk_merge_wm_level(struct intel_display *display,
3079			       int level,
3080			       struct intel_wm_level *ret_wm)
3081{
3082	const struct intel_crtc *crtc;
3083
3084	ret_wm->enable = true;
3085
3086	for_each_intel_crtc(display->drm, crtc) {
3087		const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3088		const struct intel_wm_level *wm = &active->wm[level];
3089
3090		if (!active->pipe_enabled)
3091			continue;
3092
3093		/*
3094		 * The watermark values may have been used in the past,
3095		 * so we must maintain them in the registers for some
3096		 * time even if the level is now disabled.
3097		 */
3098		if (!wm->enable)
3099			ret_wm->enable = false;
3100
3101		ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3102		ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3103		ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3104		ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3105	}
3106}
3107
3108/*
3109 * Merge all low power watermarks for all active pipes.
3110 */
3111static void ilk_wm_merge(struct intel_display *display,
3112			 const struct intel_wm_config *config,
3113			 const struct ilk_wm_maximums *max,
3114			 struct intel_pipe_wm *merged)
3115{
3116	int level, num_levels = display->wm.num_levels;
3117	int last_enabled_level = num_levels - 1;
3118
3119	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3120	if ((DISPLAY_VER(display) < 7 || display->platform.ivybridge) &&
3121	    config->num_pipes_active > 1)
3122		last_enabled_level = 0;
3123
3124	/* ILK: FBC WM must be disabled always */
3125	merged->fbc_wm_enabled = DISPLAY_VER(display) >= 6;
3126
3127	/* merge each WM1+ level */
3128	for (level = 1; level < num_levels; level++) {
3129		struct intel_wm_level *wm = &merged->wm[level];
3130
3131		ilk_merge_wm_level(display, level, wm);
3132
3133		if (level > last_enabled_level)
3134			wm->enable = false;
3135		else if (!ilk_validate_wm_level(display, level, max, wm))
3136			/* make sure all following levels get disabled */
3137			last_enabled_level = level - 1;
3138
3139		/*
3140		 * The spec says it is preferred to disable
3141		 * FBC WMs instead of disabling a WM level.
3142		 */
3143		if (wm->fbc_val > max->fbc) {
3144			if (wm->enable)
3145				merged->fbc_wm_enabled = false;
3146			wm->fbc_val = 0;
3147		}
3148	}
3149
3150	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3151	if (DISPLAY_VER(display) == 5 && HAS_FBC(display) &&
3152	    display->params.enable_fbc && !merged->fbc_wm_enabled) {
3153		for (level = 2; level < num_levels; level++) {
3154			struct intel_wm_level *wm = &merged->wm[level];
3155
3156			wm->enable = false;
3157		}
3158	}
3159}
3160
3161static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3162{
3163	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3164	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3165}
3166
3167/* The value we need to program into the WM_LPx latency field */
3168static unsigned int ilk_wm_lp_latency(struct intel_display *display,
3169				      int level)
3170{
3171	if (display->platform.haswell || display->platform.broadwell)
3172		return 2 * level;
3173	else
3174		return display->wm.pri_latency[level];
3175}
3176
3177static void ilk_compute_wm_results(struct intel_display *display,
3178				   const struct intel_pipe_wm *merged,
3179				   enum intel_ddb_partitioning partitioning,
3180				   struct ilk_wm_values *results)
3181{
3182	struct intel_crtc *crtc;
3183	int level, wm_lp;
3184
3185	results->enable_fbc_wm = merged->fbc_wm_enabled;
3186	results->partitioning = partitioning;
3187
3188	/* LP1+ register values */
3189	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3190		const struct intel_wm_level *r;
3191
3192		level = ilk_wm_lp_to_level(wm_lp, merged);
3193
3194		r = &merged->wm[level];
3195
3196		/*
3197		 * Maintain the watermark values even if the level is
3198		 * disabled. Doing otherwise could cause underruns.
3199		 */
3200		results->wm_lp[wm_lp - 1] =
3201			WM_LP_LATENCY(ilk_wm_lp_latency(display, level)) |
3202			WM_LP_PRIMARY(r->pri_val) |
3203			WM_LP_CURSOR(r->cur_val);
3204
3205		if (r->enable)
3206			results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3207
3208		if (DISPLAY_VER(display) >= 8)
3209			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3210		else
3211			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3212
3213		results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3214
3215		/*
3216		 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3217		 * level is disabled. Doing otherwise could cause underruns.
3218		 */
3219		if (DISPLAY_VER(display) < 7 && r->spr_val) {
3220			drm_WARN_ON(display->drm, wm_lp != 1);
3221			results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3222		}
3223	}
3224
3225	/* LP0 register values */
3226	for_each_intel_crtc(display->drm, crtc) {
3227		enum pipe pipe = crtc->pipe;
3228		const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3229		const struct intel_wm_level *r = &pipe_wm->wm[0];
3230
3231		if (drm_WARN_ON(display->drm, !r->enable))
3232			continue;
3233
3234		results->wm_pipe[pipe] =
3235			WM0_PIPE_PRIMARY(r->pri_val) |
3236			WM0_PIPE_SPRITE(r->spr_val) |
3237			WM0_PIPE_CURSOR(r->cur_val);
3238	}
3239}
3240
3241/*
3242 * Find the result with the highest level enabled. Check for enable_fbc_wm in
3243 * case both are at the same level. Prefer r1 in case they're the same.
3244 */
3245static struct intel_pipe_wm *
3246ilk_find_best_result(struct intel_display *display,
3247		     struct intel_pipe_wm *r1,
3248		     struct intel_pipe_wm *r2)
3249{
3250	int level, level1 = 0, level2 = 0;
3251
3252	for (level = 1; level < display->wm.num_levels; level++) {
3253		if (r1->wm[level].enable)
3254			level1 = level;
3255		if (r2->wm[level].enable)
3256			level2 = level;
3257	}
3258
3259	if (level1 == level2) {
3260		if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3261			return r2;
3262		else
3263			return r1;
3264	} else if (level1 > level2) {
3265		return r1;
3266	} else {
3267		return r2;
3268	}
3269}
3270
3271/* dirty bits used to track which watermarks need changes */
3272#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3273#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3274#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3275#define WM_DIRTY_FBC (1 << 24)
3276#define WM_DIRTY_DDB (1 << 25)
3277
3278static unsigned int ilk_compute_wm_dirty(struct intel_display *display,
3279					 const struct ilk_wm_values *old,
3280					 const struct ilk_wm_values *new)
3281{
3282	unsigned int dirty = 0;
3283	enum pipe pipe;
3284	int wm_lp;
3285
3286	for_each_pipe(display, pipe) {
3287		if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3288			dirty |= WM_DIRTY_PIPE(pipe);
3289			/* Must disable LP1+ watermarks too */
3290			dirty |= WM_DIRTY_LP_ALL;
3291		}
3292	}
3293
3294	if (old->enable_fbc_wm != new->enable_fbc_wm) {
3295		dirty |= WM_DIRTY_FBC;
3296		/* Must disable LP1+ watermarks too */
3297		dirty |= WM_DIRTY_LP_ALL;
3298	}
3299
3300	if (old->partitioning != new->partitioning) {
3301		dirty |= WM_DIRTY_DDB;
3302		/* Must disable LP1+ watermarks too */
3303		dirty |= WM_DIRTY_LP_ALL;
3304	}
3305
3306	/* LP1+ watermarks already deemed dirty, no need to continue */
3307	if (dirty & WM_DIRTY_LP_ALL)
3308		return dirty;
3309
3310	/* Find the lowest numbered LP1+ watermark in need of an update... */
3311	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3312		if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3313		    old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3314			break;
3315	}
3316
3317	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3318	for (; wm_lp <= 3; wm_lp++)
3319		dirty |= WM_DIRTY_LP(wm_lp);
3320
3321	return dirty;
3322}
3323
3324static bool _ilk_disable_lp_wm(struct intel_display *display,
3325			       unsigned int dirty)
3326{
3327	struct ilk_wm_values *previous = &display->wm.hw;
3328	bool changed = false;
3329
3330	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3331		previous->wm_lp[2] &= ~WM_LP_ENABLE;
3332		intel_de_write(display, WM3_LP_ILK, previous->wm_lp[2]);
3333		changed = true;
3334	}
3335	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3336		previous->wm_lp[1] &= ~WM_LP_ENABLE;
3337		intel_de_write(display, WM2_LP_ILK, previous->wm_lp[1]);
3338		changed = true;
3339	}
3340	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3341		previous->wm_lp[0] &= ~WM_LP_ENABLE;
3342		intel_de_write(display, WM1_LP_ILK, previous->wm_lp[0]);
3343		changed = true;
3344	}
3345
3346	/*
3347	 * Don't touch WM_LP_SPRITE_ENABLE here.
3348	 * Doing so could cause underruns.
3349	 */
3350
3351	return changed;
3352}
3353
3354/*
3355 * The spec says we shouldn't write when we don't need, because every write
3356 * causes WMs to be re-evaluated, expending some power.
3357 */
3358static void ilk_write_wm_values(struct intel_display *display,
3359				struct ilk_wm_values *results)
3360{
3361	struct ilk_wm_values *previous = &display->wm.hw;
3362	unsigned int dirty;
3363
3364	dirty = ilk_compute_wm_dirty(display, previous, results);
3365	if (!dirty)
3366		return;
3367
3368	_ilk_disable_lp_wm(display, dirty);
3369
3370	if (dirty & WM_DIRTY_PIPE(PIPE_A))
3371		intel_de_write(display, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3372	if (dirty & WM_DIRTY_PIPE(PIPE_B))
3373		intel_de_write(display, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3374	if (dirty & WM_DIRTY_PIPE(PIPE_C))
3375		intel_de_write(display, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3376
3377	if (dirty & WM_DIRTY_DDB) {
3378		if (display->platform.haswell || display->platform.broadwell)
3379			intel_de_rmw(display, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3380				     results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3381				     WM_MISC_DATA_PARTITION_5_6);
3382		else
3383			intel_de_rmw(display, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3384				     results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3385				     DISP_DATA_PARTITION_5_6);
3386	}
3387
3388	if (dirty & WM_DIRTY_FBC)
3389		intel_de_rmw(display, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3390			     results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3391
3392	if (dirty & WM_DIRTY_LP(1) &&
3393	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3394		intel_de_write(display, WM1S_LP_ILK, results->wm_lp_spr[0]);
3395
3396	if (DISPLAY_VER(display) >= 7) {
3397		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3398			intel_de_write(display, WM2S_LP_IVB, results->wm_lp_spr[1]);
3399		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3400			intel_de_write(display, WM3S_LP_IVB, results->wm_lp_spr[2]);
3401	}
3402
3403	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3404		intel_de_write(display, WM1_LP_ILK, results->wm_lp[0]);
3405	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3406		intel_de_write(display, WM2_LP_ILK, results->wm_lp[1]);
3407	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3408		intel_de_write(display, WM3_LP_ILK, results->wm_lp[2]);
3409
3410	display->wm.hw = *results;
3411}
3412
3413bool ilk_disable_cxsr(struct intel_display *display)
3414{
3415	return _ilk_disable_lp_wm(display, WM_DIRTY_LP_ALL);
3416}
3417
3418static void ilk_compute_wm_config(struct intel_display *display,
3419				  struct intel_wm_config *config)
3420{
3421	struct intel_crtc *crtc;
3422
3423	/* Compute the currently _active_ config */
3424	for_each_intel_crtc(display->drm, crtc) {
3425		const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3426
3427		if (!wm->pipe_enabled)
3428			continue;
3429
3430		config->sprites_enabled |= wm->sprites_enabled;
3431		config->sprites_scaled |= wm->sprites_scaled;
3432		config->num_pipes_active++;
3433	}
3434}
3435
3436static void ilk_program_watermarks(struct intel_display *display)
3437{
3438	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3439	struct ilk_wm_maximums max;
3440	struct intel_wm_config config = {};
3441	struct ilk_wm_values results = {};
3442	enum intel_ddb_partitioning partitioning;
3443
3444	ilk_compute_wm_config(display, &config);
3445
3446	ilk_compute_wm_maximums(display, 1, &config, INTEL_DDB_PART_1_2, &max);
3447	ilk_wm_merge(display, &config, &max, &lp_wm_1_2);
3448
3449	/* 5/6 split only in single pipe config on IVB+ */
3450	if (DISPLAY_VER(display) >= 7 &&
3451	    config.num_pipes_active == 1 && config.sprites_enabled) {
3452		ilk_compute_wm_maximums(display, 1, &config, INTEL_DDB_PART_5_6, &max);
3453		ilk_wm_merge(display, &config, &max, &lp_wm_5_6);
3454
3455		best_lp_wm = ilk_find_best_result(display, &lp_wm_1_2, &lp_wm_5_6);
3456	} else {
3457		best_lp_wm = &lp_wm_1_2;
3458	}
3459
3460	partitioning = (best_lp_wm == &lp_wm_1_2) ?
3461		       INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3462
3463	ilk_compute_wm_results(display, best_lp_wm, partitioning, &results);
3464
3465	ilk_write_wm_values(display, &results);
3466}
3467
3468static void ilk_initial_watermarks(struct intel_atomic_state *state,
3469				   struct intel_crtc *crtc)
3470{
3471	struct intel_display *display = to_intel_display(crtc);
3472	const struct intel_crtc_state *crtc_state =
3473		intel_atomic_get_new_crtc_state(state, crtc);
3474
3475	mutex_lock(&display->wm.wm_mutex);
3476	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3477	ilk_program_watermarks(display);
3478	mutex_unlock(&display->wm.wm_mutex);
3479}
3480
3481static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3482				    struct intel_crtc *crtc)
3483{
3484	struct intel_display *display = to_intel_display(crtc);
3485	const struct intel_crtc_state *crtc_state =
3486		intel_atomic_get_new_crtc_state(state, crtc);
3487
3488	if (!crtc_state->wm.need_postvbl_update)
3489		return;
3490
3491	mutex_lock(&display->wm.wm_mutex);
3492	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3493	ilk_program_watermarks(display);
3494	mutex_unlock(&display->wm.wm_mutex);
3495}
3496
3497static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3498{
3499	struct intel_display *display = to_intel_display(crtc);
3500	struct ilk_wm_values *hw = &display->wm.hw;
3501	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3502	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3503	enum pipe pipe = crtc->pipe;
3504
3505	hw->wm_pipe[pipe] = intel_de_read(display, WM0_PIPE_ILK(pipe));
3506
3507	memset(active, 0, sizeof(*active));
3508
3509	active->pipe_enabled = crtc->active;
3510
3511	if (active->pipe_enabled) {
3512		u32 tmp = hw->wm_pipe[pipe];
3513
3514		/*
3515		 * For active pipes LP0 watermark is marked as
3516		 * enabled, and LP1+ watermaks as disabled since
3517		 * we can't really reverse compute them in case
3518		 * multiple pipes are active.
3519		 */
3520		active->wm[0].enable = true;
3521		active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3522		active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3523		active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3524	} else {
3525		int level;
3526
3527		/*
3528		 * For inactive pipes, all watermark levels
3529		 * should be marked as enabled but zeroed,
3530		 * which is what we'd compute them to.
3531		 */
3532		for (level = 0; level < display->wm.num_levels; level++)
3533			active->wm[level].enable = true;
3534	}
3535
3536	crtc->wm.active.ilk = *active;
3537}
3538
3539static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3540{
3541	struct drm_plane *plane;
3542	struct intel_crtc *crtc;
3543
3544	for_each_intel_crtc(state->dev, crtc) {
3545		struct intel_crtc_state *crtc_state;
3546
3547		crtc_state = intel_atomic_get_crtc_state(state, crtc);
3548		if (IS_ERR(crtc_state))
3549			return PTR_ERR(crtc_state);
3550
3551		if (crtc_state->hw.active) {
3552			/*
3553			 * Preserve the inherited flag to avoid
3554			 * taking the full modeset path.
3555			 */
3556			crtc_state->inherited = true;
3557		}
3558	}
3559
3560	drm_for_each_plane(plane, state->dev) {
3561		struct drm_plane_state *plane_state;
3562
3563		plane_state = drm_atomic_get_plane_state(state, plane);
3564		if (IS_ERR(plane_state))
3565			return PTR_ERR(plane_state);
3566	}
3567
3568	return 0;
3569}
3570
3571/*
3572 * Calculate what we think the watermarks should be for the state we've read
3573 * out of the hardware and then immediately program those watermarks so that
3574 * we ensure the hardware settings match our internal state.
3575 *
3576 * We can calculate what we think WM's should be by creating a duplicate of the
3577 * current state (which was constructed during hardware readout) and running it
3578 * through the atomic check code to calculate new watermark values in the
3579 * state object.
3580 */
3581void ilk_wm_sanitize(struct intel_display *display)
3582{
3583	struct drm_atomic_state *state;
3584	struct intel_atomic_state *intel_state;
3585	struct intel_crtc *crtc;
3586	struct intel_crtc_state *crtc_state;
3587	struct drm_modeset_acquire_ctx ctx;
3588	int ret;
3589	int i;
3590
3591	/* Only supported on platforms that use atomic watermark design */
3592	if (!display->funcs.wm->optimize_watermarks)
3593		return;
3594
3595	if (drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 9))
3596		return;
3597
3598	state = drm_atomic_state_alloc(display->drm);
3599	if (drm_WARN_ON(display->drm, !state))
3600		return;
3601
3602	intel_state = to_intel_atomic_state(state);
3603
3604	drm_modeset_acquire_init(&ctx, 0);
3605
3606	state->acquire_ctx = &ctx;
3607	to_intel_atomic_state(state)->internal = true;
3608
3609retry:
3610	/*
3611	 * Hardware readout is the only time we don't want to calculate
3612	 * intermediate watermarks (since we don't trust the current
3613	 * watermarks).
3614	 */
3615	if (!HAS_GMCH(display))
3616		intel_state->skip_intermediate_wm = true;
3617
3618	ret = ilk_sanitize_watermarks_add_affected(state);
3619	if (ret)
3620		goto fail;
3621
3622	ret = intel_atomic_check(display->drm, state);
3623	if (ret)
3624		goto fail;
3625
3626	/* Write calculated watermark values back */
3627	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3628		crtc_state->wm.need_postvbl_update = true;
3629		intel_optimize_watermarks(intel_state, crtc);
3630
3631		to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3632	}
3633
3634fail:
3635	if (ret == -EDEADLK) {
3636		drm_atomic_state_clear(state);
3637		drm_modeset_backoff(&ctx);
3638		goto retry;
3639	}
3640
3641	/*
3642	 * If we fail here, it means that the hardware appears to be
3643	 * programmed in a way that shouldn't be possible, given our
3644	 * understanding of watermark requirements.  This might mean a
3645	 * mistake in the hardware readout code or a mistake in the
3646	 * watermark calculations for a given platform.  Raise a WARN
3647	 * so that this is noticeable.
3648	 *
3649	 * If this actually happens, we'll have to just leave the
3650	 * BIOS-programmed watermarks untouched and hope for the best.
3651	 */
3652	drm_WARN(display->drm, ret,
3653		 "Could not determine valid watermarks for inherited state\n");
3654
3655	drm_atomic_state_put(state);
3656
3657	drm_modeset_drop_locks(&ctx);
3658	drm_modeset_acquire_fini(&ctx);
3659}
3660
3661#define _FW_WM(value, plane) \
3662	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3663#define _FW_WM_VLV(value, plane) \
3664	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3665
3666static void g4x_read_wm_values(struct intel_display *display,
3667			       struct g4x_wm_values *wm)
3668{
3669	u32 tmp;
3670
3671	tmp = intel_de_read(display, DSPFW1(display));
3672	wm->sr.plane = _FW_WM(tmp, SR);
3673	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3674	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3675	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3676
3677	tmp = intel_de_read(display, DSPFW2(display));
3678	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3679	wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3680	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3681	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3682	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3683	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3684
3685	tmp = intel_de_read(display, DSPFW3(display));
3686	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3687	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3688	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3689	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3690}
3691
3692static void vlv_read_wm_values(struct intel_display *display,
3693			       struct vlv_wm_values *wm)
3694{
3695	enum pipe pipe;
3696	u32 tmp;
3697
3698	for_each_pipe(display, pipe) {
3699		tmp = intel_de_read(display, VLV_DDL(pipe));
3700
3701		wm->ddl[pipe].plane[PLANE_PRIMARY] =
3702			(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3703		wm->ddl[pipe].plane[PLANE_CURSOR] =
3704			(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3705		wm->ddl[pipe].plane[PLANE_SPRITE0] =
3706			(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3707		wm->ddl[pipe].plane[PLANE_SPRITE1] =
3708			(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3709	}
3710
3711	tmp = intel_de_read(display, DSPFW1(display));
3712	wm->sr.plane = _FW_WM(tmp, SR);
3713	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3714	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3715	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3716
3717	tmp = intel_de_read(display, DSPFW2(display));
3718	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3719	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3720	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3721
3722	tmp = intel_de_read(display, DSPFW3(display));
3723	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3724
3725	if (display->platform.cherryview) {
3726		tmp = intel_de_read(display, DSPFW7_CHV);
3727		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3728		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3729
3730		tmp = intel_de_read(display, DSPFW8_CHV);
3731		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3732		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3733
3734		tmp = intel_de_read(display, DSPFW9_CHV);
3735		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3736		wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3737
3738		tmp = intel_de_read(display, DSPHOWM);
3739		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3740		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3741		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3742		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3743		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3744		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3745		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3746		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3747		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3748		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3749	} else {
3750		tmp = intel_de_read(display, DSPFW7);
3751		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3752		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3753
3754		tmp = intel_de_read(display, DSPHOWM);
3755		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3756		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3757		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3758		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3759		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3760		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3761		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3762	}
3763}
3764
3765#undef _FW_WM
3766#undef _FW_WM_VLV
3767
3768static void g4x_wm_get_hw_state(struct intel_display *display)
3769{
3770	struct g4x_wm_values *wm = &display->wm.g4x;
3771	struct intel_crtc *crtc;
3772
3773	g4x_read_wm_values(display, wm);
3774
3775	wm->cxsr = intel_de_read(display, FW_BLC_SELF) & FW_BLC_SELF_EN;
3776
3777	for_each_intel_crtc(display->drm, crtc) {
3778		struct intel_crtc_state *crtc_state =
3779			to_intel_crtc_state(crtc->base.state);
3780		struct g4x_wm_state *active = &crtc->wm.active.g4x;
3781		struct g4x_pipe_wm *raw;
3782		enum pipe pipe = crtc->pipe;
3783		enum plane_id plane_id;
3784		int level, max_level;
3785
3786		active->cxsr = wm->cxsr;
3787		active->hpll_en = wm->hpll_en;
3788		active->fbc_en = wm->fbc_en;
3789
3790		active->sr = wm->sr;
3791		active->hpll = wm->hpll;
3792
3793		for_each_plane_id_on_crtc(crtc, plane_id) {
3794			active->wm.plane[plane_id] =
3795				wm->pipe[pipe].plane[plane_id];
3796		}
3797
3798		if (wm->cxsr && wm->hpll_en)
3799			max_level = G4X_WM_LEVEL_HPLL;
3800		else if (wm->cxsr)
3801			max_level = G4X_WM_LEVEL_SR;
3802		else
3803			max_level = G4X_WM_LEVEL_NORMAL;
3804
3805		level = G4X_WM_LEVEL_NORMAL;
3806		raw = &crtc_state->wm.g4x.raw[level];
3807		for_each_plane_id_on_crtc(crtc, plane_id)
3808			raw->plane[plane_id] = active->wm.plane[plane_id];
3809
3810		level = G4X_WM_LEVEL_SR;
3811		if (level > max_level)
3812			goto out;
3813
3814		raw = &crtc_state->wm.g4x.raw[level];
3815		raw->plane[PLANE_PRIMARY] = active->sr.plane;
3816		raw->plane[PLANE_CURSOR] = active->sr.cursor;
3817		raw->plane[PLANE_SPRITE0] = 0;
3818		raw->fbc = active->sr.fbc;
3819
3820		level = G4X_WM_LEVEL_HPLL;
3821		if (level > max_level)
3822			goto out;
3823
3824		raw = &crtc_state->wm.g4x.raw[level];
3825		raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3826		raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3827		raw->plane[PLANE_SPRITE0] = 0;
3828		raw->fbc = active->hpll.fbc;
3829
3830		level++;
3831	out:
3832		for_each_plane_id_on_crtc(crtc, plane_id)
3833			g4x_raw_plane_wm_set(crtc_state, level,
3834					     plane_id, USHRT_MAX);
3835		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3836
3837		g4x_invalidate_wms(crtc, active, level);
3838
3839		crtc_state->wm.g4x.optimal = *active;
3840		crtc_state->wm.g4x.intermediate = *active;
3841
3842		drm_dbg_kms(display->drm,
3843			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3844			    pipe_name(pipe),
3845			    wm->pipe[pipe].plane[PLANE_PRIMARY],
3846			    wm->pipe[pipe].plane[PLANE_CURSOR],
3847			    wm->pipe[pipe].plane[PLANE_SPRITE0]);
3848	}
3849
3850	drm_dbg_kms(display->drm,
3851		    "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3852		    wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3853	drm_dbg_kms(display->drm,
3854		    "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3855		    wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3856	drm_dbg_kms(display->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3857		    str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3858		    str_yes_no(wm->fbc_en));
3859}
3860
3861static void g4x_wm_sanitize(struct intel_display *display)
3862{
3863	struct intel_plane *plane;
3864	struct intel_crtc *crtc;
3865
3866	mutex_lock(&display->wm.wm_mutex);
3867
3868	for_each_intel_plane(display->drm, plane) {
3869		struct intel_crtc *crtc =
3870			intel_crtc_for_pipe(display, plane->pipe);
3871		struct intel_crtc_state *crtc_state =
3872			to_intel_crtc_state(crtc->base.state);
3873		struct intel_plane_state *plane_state =
3874			to_intel_plane_state(plane->base.state);
3875		enum plane_id plane_id = plane->id;
3876		int level;
3877
3878		if (plane_state->uapi.visible)
3879			continue;
3880
3881		for (level = 0; level < display->wm.num_levels; level++) {
3882			struct g4x_pipe_wm *raw =
3883				&crtc_state->wm.g4x.raw[level];
3884
3885			raw->plane[plane_id] = 0;
3886
3887			if (plane_id == PLANE_PRIMARY)
3888				raw->fbc = 0;
3889		}
3890	}
3891
3892	for_each_intel_crtc(display->drm, crtc) {
3893		struct intel_crtc_state *crtc_state =
3894			to_intel_crtc_state(crtc->base.state);
3895		int ret;
3896
3897		ret = _g4x_compute_pipe_wm(crtc_state);
3898		drm_WARN_ON(display->drm, ret);
3899
3900		crtc_state->wm.g4x.intermediate =
3901			crtc_state->wm.g4x.optimal;
3902		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3903	}
3904
3905	g4x_program_watermarks(display);
3906
3907	mutex_unlock(&display->wm.wm_mutex);
3908}
3909
3910static void vlv_wm_get_hw_state(struct intel_display *display)
3911{
3912	struct vlv_wm_values *wm = &display->wm.vlv;
3913	struct intel_crtc *crtc;
3914	u32 val;
3915	int ret;
3916
3917	vlv_read_wm_values(display, wm);
3918
3919	wm->cxsr = intel_de_read(display, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3920	wm->level = VLV_WM_LEVEL_PM2;
3921
3922	if (display->platform.cherryview) {
3923		vlv_punit_get(display->drm);
3924
3925		val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
3926		if (val & DSP_MAXFIFO_PM5_ENABLE)
3927			wm->level = VLV_WM_LEVEL_PM5;
3928
3929		/*
3930		 * If DDR DVFS is disabled in the BIOS, Punit
3931		 * will never ack the request. So if that happens
3932		 * assume we don't have to enable/disable DDR DVFS
3933		 * dynamically. To test that just set the REQ_ACK
3934		 * bit to poke the Punit, but don't change the
3935		 * HIGH/LOW bits so that we don't actually change
3936		 * the current state.
3937		 */
3938		val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2);
3939		val |= FORCE_DDR_FREQ_REQ_ACK;
3940		vlv_punit_write(display->drm, PUNIT_REG_DDR_SETUP2, val);
3941
3942		ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
3943				      (val & FORCE_DDR_FREQ_REQ_ACK) == 0,
3944				      500, 3000, false);
3945		if (ret) {
3946			drm_dbg_kms(display->drm,
3947				    "Punit not acking DDR DVFS request, "
3948				    "assuming DDR DVFS is disabled\n");
3949			display->wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3950		} else {
3951			val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2);
3952			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3953				wm->level = VLV_WM_LEVEL_DDR_DVFS;
3954		}
3955
3956		vlv_punit_put(display->drm);
3957	}
3958
3959	for_each_intel_crtc(display->drm, crtc) {
3960		struct intel_crtc_state *crtc_state =
3961			to_intel_crtc_state(crtc->base.state);
3962		struct vlv_wm_state *active = &crtc->wm.active.vlv;
3963		const struct vlv_fifo_state *fifo_state =
3964			&crtc_state->wm.vlv.fifo_state;
3965		enum pipe pipe = crtc->pipe;
3966		enum plane_id plane_id;
3967		int level;
3968
3969		vlv_get_fifo_size(crtc_state);
3970
3971		active->num_levels = wm->level + 1;
3972		active->cxsr = wm->cxsr;
3973
3974		for (level = 0; level < active->num_levels; level++) {
3975			struct g4x_pipe_wm *raw =
3976				&crtc_state->wm.vlv.raw[level];
3977
3978			active->sr[level].plane = wm->sr.plane;
3979			active->sr[level].cursor = wm->sr.cursor;
3980
3981			for_each_plane_id_on_crtc(crtc, plane_id) {
3982				active->wm[level].plane[plane_id] =
3983					wm->pipe[pipe].plane[plane_id];
3984
3985				raw->plane[plane_id] =
3986					vlv_invert_wm_value(active->wm[level].plane[plane_id],
3987							    fifo_state->plane[plane_id]);
3988			}
3989		}
3990
3991		for_each_plane_id_on_crtc(crtc, plane_id)
3992			vlv_raw_plane_wm_set(crtc_state, level,
3993					     plane_id, USHRT_MAX);
3994		vlv_invalidate_wms(crtc, active, level);
3995
3996		crtc_state->wm.vlv.optimal = *active;
3997		crtc_state->wm.vlv.intermediate = *active;
3998
3999		drm_dbg_kms(display->drm,
4000			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
4001			    pipe_name(pipe),
4002			    wm->pipe[pipe].plane[PLANE_PRIMARY],
4003			    wm->pipe[pipe].plane[PLANE_CURSOR],
4004			    wm->pipe[pipe].plane[PLANE_SPRITE0],
4005			    wm->pipe[pipe].plane[PLANE_SPRITE1]);
4006	}
4007
4008	drm_dbg_kms(display->drm,
4009		    "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4010		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4011}
4012
4013static void vlv_wm_sanitize(struct intel_display *display)
4014{
4015	struct intel_plane *plane;
4016	struct intel_crtc *crtc;
4017
4018	mutex_lock(&display->wm.wm_mutex);
4019
4020	for_each_intel_plane(display->drm, plane) {
4021		struct intel_crtc *crtc =
4022			intel_crtc_for_pipe(display, plane->pipe);
4023		struct intel_crtc_state *crtc_state =
4024			to_intel_crtc_state(crtc->base.state);
4025		struct intel_plane_state *plane_state =
4026			to_intel_plane_state(plane->base.state);
4027		enum plane_id plane_id = plane->id;
4028		int level;
4029
4030		if (plane_state->uapi.visible)
4031			continue;
4032
4033		for (level = 0; level < display->wm.num_levels; level++) {
4034			struct g4x_pipe_wm *raw =
4035				&crtc_state->wm.vlv.raw[level];
4036
4037			raw->plane[plane_id] = 0;
4038		}
4039	}
4040
4041	for_each_intel_crtc(display->drm, crtc) {
4042		struct intel_crtc_state *crtc_state =
4043			to_intel_crtc_state(crtc->base.state);
4044		int ret;
4045
4046		ret = _vlv_compute_pipe_wm(crtc_state);
4047		drm_WARN_ON(display->drm, ret);
4048
4049		crtc_state->wm.vlv.intermediate =
4050			crtc_state->wm.vlv.optimal;
4051		crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4052	}
4053
4054	vlv_program_watermarks(display);
4055
4056	mutex_unlock(&display->wm.wm_mutex);
4057}
4058
4059/*
4060 * FIXME should probably kill this and improve
4061 * the real watermark readout/sanitation instead
4062 */
4063static void ilk_init_lp_watermarks(struct intel_display *display)
4064{
4065	intel_de_rmw(display, WM3_LP_ILK, WM_LP_ENABLE, 0);
4066	intel_de_rmw(display, WM2_LP_ILK, WM_LP_ENABLE, 0);
4067	intel_de_rmw(display, WM1_LP_ILK, WM_LP_ENABLE, 0);
4068
4069	/*
4070	 * Don't touch WM_LP_SPRITE_ENABLE here.
4071	 * Doing so could cause underruns.
4072	 */
4073}
4074
4075static void ilk_wm_get_hw_state(struct intel_display *display)
4076{
4077	struct ilk_wm_values *hw = &display->wm.hw;
4078	struct intel_crtc *crtc;
4079
4080	ilk_init_lp_watermarks(display);
4081
4082	for_each_intel_crtc(display->drm, crtc)
4083		ilk_pipe_wm_get_hw_state(crtc);
4084
4085	hw->wm_lp[0] = intel_de_read(display, WM1_LP_ILK);
4086	hw->wm_lp[1] = intel_de_read(display, WM2_LP_ILK);
4087	hw->wm_lp[2] = intel_de_read(display, WM3_LP_ILK);
4088
4089	hw->wm_lp_spr[0] = intel_de_read(display, WM1S_LP_ILK);
4090	if (DISPLAY_VER(display) >= 7) {
4091		hw->wm_lp_spr[1] = intel_de_read(display, WM2S_LP_IVB);
4092		hw->wm_lp_spr[2] = intel_de_read(display, WM3S_LP_IVB);
4093	}
4094
4095	if (display->platform.haswell || display->platform.broadwell)
4096		hw->partitioning = (intel_de_read(display, WM_MISC) &
4097				    WM_MISC_DATA_PARTITION_5_6) ?
4098			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4099	else if (display->platform.ivybridge)
4100		hw->partitioning = (intel_de_read(display, DISP_ARB_CTL2) &
4101				    DISP_DATA_PARTITION_5_6) ?
4102			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4103
4104	hw->enable_fbc_wm =
4105		!(intel_de_read(display, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4106}
4107
4108static const struct intel_wm_funcs ilk_wm_funcs = {
4109	.compute_watermarks = ilk_compute_watermarks,
4110	.initial_watermarks = ilk_initial_watermarks,
4111	.optimize_watermarks = ilk_optimize_watermarks,
4112	.get_hw_state = ilk_wm_get_hw_state,
4113};
4114
4115static const struct intel_wm_funcs vlv_wm_funcs = {
4116	.compute_watermarks = vlv_compute_watermarks,
4117	.initial_watermarks = vlv_initial_watermarks,
4118	.optimize_watermarks = vlv_optimize_watermarks,
4119	.atomic_update_watermarks = vlv_atomic_update_fifo,
4120	.get_hw_state = vlv_wm_get_hw_state,
4121	.sanitize = vlv_wm_sanitize,
4122};
4123
4124static const struct intel_wm_funcs g4x_wm_funcs = {
4125	.compute_watermarks = g4x_compute_watermarks,
4126	.initial_watermarks = g4x_initial_watermarks,
4127	.optimize_watermarks = g4x_optimize_watermarks,
4128	.get_hw_state = g4x_wm_get_hw_state,
4129	.sanitize = g4x_wm_sanitize,
4130};
4131
4132static const struct intel_wm_funcs pnv_wm_funcs = {
4133	.compute_watermarks = i9xx_compute_watermarks,
4134	.update_wm = pnv_update_wm,
4135};
4136
4137static const struct intel_wm_funcs i965_wm_funcs = {
4138	.compute_watermarks = i9xx_compute_watermarks,
4139	.update_wm = i965_update_wm,
4140};
4141
4142static const struct intel_wm_funcs i9xx_wm_funcs = {
4143	.compute_watermarks = i9xx_compute_watermarks,
4144	.update_wm = i9xx_update_wm,
4145};
4146
4147static const struct intel_wm_funcs i845_wm_funcs = {
4148	.compute_watermarks = i9xx_compute_watermarks,
4149	.update_wm = i845_update_wm,
4150};
4151
4152static const struct intel_wm_funcs nop_funcs = {
4153};
4154
4155void i9xx_wm_init(struct intel_display *display)
4156{
4157	/* For FIFO watermark updates */
4158	if (HAS_PCH_SPLIT(display)) {
4159		ilk_setup_wm_latency(display);
4160		display->funcs.wm = &ilk_wm_funcs;
4161	} else if (display->platform.valleyview || display->platform.cherryview) {
4162		vlv_setup_wm_latency(display);
4163		display->funcs.wm = &vlv_wm_funcs;
4164	} else if (display->platform.g4x) {
4165		g4x_setup_wm_latency(display);
4166		display->funcs.wm = &g4x_wm_funcs;
4167	} else if (display->platform.pineview) {
4168		if (!pnv_get_cxsr_latency(display)) {
4169			drm_info(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
4170			/* Disable CxSR and never update its watermark again */
4171			intel_set_memory_cxsr(display, false);
4172			display->funcs.wm = &nop_funcs;
4173		} else {
4174			display->funcs.wm = &pnv_wm_funcs;
4175		}
4176	} else if (DISPLAY_VER(display) == 4) {
4177		display->funcs.wm = &i965_wm_funcs;
4178	} else if (DISPLAY_VER(display) == 3) {
4179		display->funcs.wm = &i9xx_wm_funcs;
4180	} else if (DISPLAY_VER(display) == 2) {
4181		if (INTEL_NUM_PIPES(display) == 1)
4182			display->funcs.wm = &i845_wm_funcs;
4183		else
4184			display->funcs.wm = &i9xx_wm_funcs;
4185	} else {
4186		drm_err(display->drm,
4187			"unexpected fall-through in %s\n", __func__);
4188		display->funcs.wm = &nop_funcs;
4189	}
4190}