intel_fbc.c

   1/*
   2 * Copyright © 2014 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21 * DEALINGS IN THE SOFTWARE.
  22 */
  23
  24/**
  25 * DOC: Frame Buffer Compression (FBC)
  26 *
  27 * FBC tries to save memory bandwidth (and so power consumption) by
  28 * compressing the amount of memory used by the display. It is total
  29 * transparent to user space and completely handled in the kernel.
  30 *
  31 * The benefits of FBC are mostly visible with solid backgrounds and
  32 * variation-less patterns. It comes from keeping the memory footprint small
  33 * and having fewer memory pages opened and accessed for refreshing the display.
  34 *
  35 * i915 is responsible to reserve stolen memory for FBC and configure its
  36 * offset on proper registers. The hardware takes care of all
  37 * compress/decompress. However there are many known cases where we have to
  38 * forcibly disable it to allow proper screen updates.
  39 */
  40
  41#include <linux/debugfs.h>
  42#include <linux/string_helpers.h>
  43
  44#include <drm/drm_blend.h>
  45#include <drm/drm_fourcc.h>
  46#include <drm/drm_print.h>
  47
  48#include "gem/i915_gem_stolen.h"
  49
  50#include "gt/intel_gt_types.h"
  51
  52#include "i915_drv.h"
  53#include "i915_vgpu.h"
  54#include "i915_vma.h"
  55#include "i9xx_plane_regs.h"
  56#include "intel_de.h"
  57#include "intel_display_device.h"
  58#include "intel_display_regs.h"
  59#include "intel_display_rpm.h"
  60#include "intel_display_trace.h"
  61#include "intel_display_types.h"
  62#include "intel_display_utils.h"
  63#include "intel_display_wa.h"
  64#include "intel_fbc.h"
  65#include "intel_fbc_regs.h"
  66#include "intel_frontbuffer.h"
  67
  68#define for_each_fbc_id(__display, __fbc_id) \
  69	for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
  70		for_each_if(DISPLAY_RUNTIME_INFO(__display)->fbc_mask & BIT(__fbc_id))
  71
  72#define for_each_intel_fbc(__display, __fbc, __fbc_id) \
  73	for_each_fbc_id((__display), (__fbc_id)) \
  74		for_each_if((__fbc) = (__display)->fbc[(__fbc_id)])
  75
  76struct intel_fbc_funcs {
  77	void (*activate)(struct intel_fbc *fbc);
  78	void (*deactivate)(struct intel_fbc *fbc);
  79	bool (*is_active)(struct intel_fbc *fbc);
  80	bool (*is_compressing)(struct intel_fbc *fbc);
  81	void (*nuke)(struct intel_fbc *fbc);
  82	void (*program_cfb)(struct intel_fbc *fbc);
  83	void (*set_false_color)(struct intel_fbc *fbc, bool enable);
  84};
  85
  86struct intel_fbc_state {
  87	struct intel_plane *plane;
  88	unsigned int cfb_stride;
  89	unsigned int cfb_size;
  90	unsigned int fence_y_offset;
  91	u16 override_cfb_stride;
  92	u16 interval;
  93	s8 fence_id;
  94	struct drm_rect dirty_rect;
  95};
  96
  97struct intel_fbc {
  98	struct intel_display *display;
  99	const struct intel_fbc_funcs *funcs;
 100
 101	/* This is always the outer lock when overlapping with stolen_lock */
 102	struct mutex lock;
 103	unsigned int busy_bits;
 104
 105	struct intel_stolen_node *compressed_fb;
 106	struct intel_stolen_node *compressed_llb;
 107
 108	enum intel_fbc_id id;
 109
 110	u8 limit;
 111
 112	bool false_color;
 113
 114	bool active;
 115	bool activated;
 116	bool flip_pending;
 117
 118	bool underrun_detected;
 119	struct work_struct underrun_work;
 120
 121	/*
 122	 * This structure contains everything that's relevant to program the
 123	 * hardware registers. When we want to figure out if we need to disable
 124	 * and re-enable FBC for a new configuration we just check if there's
 125	 * something different in the struct. The genx_fbc_activate functions
 126	 * are supposed to read from it in order to program the registers.
 127	 */
 128	struct intel_fbc_state state;
 129	const char *no_fbc_reason;
 130};
 131
 132/* plane stride in pixels */
 133static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
 134{
 135	const struct drm_framebuffer *fb = plane_state->hw.fb;
 136	unsigned int stride;
 137
 138	stride = plane_state->view.color_plane[0].mapping_stride;
 139	if (!drm_rotation_90_or_270(plane_state->hw.rotation))
 140		stride /= fb->format->cpp[0];
 141
 142	return stride;
 143}
 144
 145static unsigned int intel_fbc_cfb_cpp(const struct intel_plane_state *plane_state)
 146{
 147	const struct drm_framebuffer *fb = plane_state->hw.fb;
 148	unsigned int cpp = fb->format->cpp[0];
 149
 150	return max(cpp, 4);
 151}
 152
 153/* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
 154static unsigned int intel_fbc_plane_cfb_stride(const struct intel_plane_state *plane_state)
 155{
 156	unsigned int cpp = intel_fbc_cfb_cpp(plane_state);
 157
 158	return intel_fbc_plane_stride(plane_state) * cpp;
 159}
 160
 161/* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
 162static unsigned int skl_fbc_min_cfb_stride(struct intel_display *display,
 163					   unsigned int cpp, unsigned int width)
 164{
 165	unsigned int limit = 4; /* 1:4 compression limit is the worst case */
 166	unsigned int height = 4; /* FBC segment is 4 lines */
 167	unsigned int stride;
 168
 169	/* minimum segment stride we can use */
 170	stride = width * cpp * height / limit;
 171
 172	/*
 173	 * Wa_16011863758: icl+
 174	 * Avoid some hardware segment address miscalculation.
 175	 */
 176	if (DISPLAY_VER(display) >= 11)
 177		stride += 64;
 178
 179	/*
 180	 * At least some of the platforms require each 4 line segment to
 181	 * be 512 byte aligned. Just do it always for simplicity.
 182	 */
 183	stride = ALIGN(stride, 512);
 184
 185	/* convert back to single line equivalent with 1:1 compression limit */
 186	return stride * limit / height;
 187}
 188
 189/* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
 190static unsigned int _intel_fbc_cfb_stride(struct intel_display *display,
 191					  unsigned int cpp, unsigned int width,
 192					  unsigned int stride)
 193{
 194	/*
 195	 * At least some of the platforms require each 4 line segment to
 196	 * be 512 byte aligned. Aligning each line to 512 bytes guarantees
 197	 * that regardless of the compression limit we choose later.
 198	 */
 199	if (DISPLAY_VER(display) >= 9)
 200		return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(display, cpp, width));
 201	else
 202		return stride;
 203}
 204
 205static unsigned int intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 206{
 207	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 208	unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
 209	unsigned int width = drm_rect_width(&plane_state->uapi.src) >> 16;
 210	unsigned int cpp = intel_fbc_cfb_cpp(plane_state);
 211
 212	return _intel_fbc_cfb_stride(display, cpp, width, stride);
 213}
 214
 215/*
 216 * Maximum height the hardware will compress, on HSW+
 217 * additional lines (up to the actual plane height) will
 218 * remain uncompressed.
 219 */
 220static unsigned int intel_fbc_max_cfb_height(struct intel_display *display)
 221{
 222	if (DISPLAY_VER(display) >= 8)
 223		return 2560;
 224	else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
 225		return 2048;
 226	else
 227		return 1536;
 228}
 229
 230static unsigned int _intel_fbc_cfb_size(struct intel_display *display,
 231					unsigned int height, unsigned int stride)
 232{
 233	return min(height, intel_fbc_max_cfb_height(display)) * stride;
 234}
 235
 236static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_state)
 237{
 238	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 239	unsigned int height = drm_rect_height(&plane_state->uapi.src) >> 16;
 240
 241	return _intel_fbc_cfb_size(display, height, intel_fbc_cfb_stride(plane_state));
 242}
 243
 244static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_state)
 245{
 246	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 247	unsigned int stride_aligned = intel_fbc_cfb_stride(plane_state);
 248	unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
 249	const struct drm_framebuffer *fb = plane_state->hw.fb;
 250
 251	/*
 252	 * Override stride in 64 byte units per 4 line segment.
 253	 *
 254	 * Gen9 hw miscalculates cfb stride for linear as
 255	 * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
 256	 * we always need to use the override there.
 257	 *
 258	 * wa_14022269668 For bmg, always program the FBC_STRIDE before fbc enable
 259	 */
 260	if (stride != stride_aligned ||
 261	    (DISPLAY_VER(display) == 9 && fb->modifier == DRM_FORMAT_MOD_LINEAR) ||
 262	    display->platform.battlemage)
 263		return stride_aligned * 4 / 64;
 264
 265	return 0;
 266}
 267
 268static bool intel_fbc_has_fences(struct intel_display *display)
 269{
 270	struct drm_i915_private __maybe_unused *i915 = to_i915(display->drm);
 271
 272	return intel_gt_support_legacy_fencing(to_gt(i915));
 273}
 274
 275static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)
 276{
 277	struct intel_display *display = fbc->display;
 278	const struct intel_fbc_state *fbc_state = &fbc->state;
 279	unsigned int cfb_stride;
 280	u32 fbc_ctl;
 281
 282	cfb_stride = fbc_state->cfb_stride / fbc->limit;
 283
 284	/* FBC_CTL wants 32B or 64B units */
 285	if (DISPLAY_VER(display) == 2)
 286		cfb_stride = (cfb_stride / 32) - 1;
 287	else
 288		cfb_stride = (cfb_stride / 64) - 1;
 289
 290	fbc_ctl = FBC_CTL_PERIODIC |
 291		FBC_CTL_INTERVAL(fbc_state->interval) |
 292		FBC_CTL_STRIDE(cfb_stride);
 293
 294	if (display->platform.i945gm)
 295		fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
 296
 297	if (fbc_state->fence_id >= 0)
 298		fbc_ctl |= FBC_CTL_FENCENO(fbc_state->fence_id);
 299
 300	return fbc_ctl;
 301}
 302
 303static u32 i965_fbc_ctl2(struct intel_fbc *fbc)
 304{
 305	const struct intel_fbc_state *fbc_state = &fbc->state;
 306	u32 fbc_ctl2;
 307
 308	fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM |
 309		FBC_CTL_PLANE(fbc_state->plane->i9xx_plane);
 310
 311	if (fbc_state->fence_id >= 0)
 312		fbc_ctl2 |= FBC_CTL_CPU_FENCE_EN;
 313
 314	return fbc_ctl2;
 315}
 316
 317static void i8xx_fbc_deactivate(struct intel_fbc *fbc)
 318{
 319	struct intel_display *display = fbc->display;
 320	u32 fbc_ctl;
 321
 322	/* Disable compression */
 323	fbc_ctl = intel_de_read(display, FBC_CONTROL);
 324	if ((fbc_ctl & FBC_CTL_EN) == 0)
 325		return;
 326
 327	fbc_ctl &= ~FBC_CTL_EN;
 328	intel_de_write(display, FBC_CONTROL, fbc_ctl);
 329
 330	/* Wait for compressing bit to clear */
 331	if (intel_de_wait_for_clear_ms(display, FBC_STATUS,
 332				       FBC_STAT_COMPRESSING, 10)) {
 333		drm_dbg_kms(display->drm, "FBC idle timed out\n");
 334		return;
 335	}
 336}
 337
 338static void i8xx_fbc_activate(struct intel_fbc *fbc)
 339{
 340	struct intel_display *display = fbc->display;
 341	const struct intel_fbc_state *fbc_state = &fbc->state;
 342	int i;
 343
 344	/* Clear old tags */
 345	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
 346		intel_de_write(display, FBC_TAG(i), 0);
 347
 348	if (DISPLAY_VER(display) == 4) {
 349		intel_de_write(display, FBC_CONTROL2,
 350			       i965_fbc_ctl2(fbc));
 351		intel_de_write(display, FBC_FENCE_OFF,
 352			       fbc_state->fence_y_offset);
 353	}
 354
 355	intel_de_write(display, FBC_CONTROL,
 356		       FBC_CTL_EN | i8xx_fbc_ctl(fbc));
 357}
 358
 359static bool i8xx_fbc_is_active(struct intel_fbc *fbc)
 360{
 361	return intel_de_read(fbc->display, FBC_CONTROL) & FBC_CTL_EN;
 362}
 363
 364static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)
 365{
 366	return intel_de_read(fbc->display, FBC_STATUS) &
 367		(FBC_STAT_COMPRESSING | FBC_STAT_COMPRESSED);
 368}
 369
 370static void i8xx_fbc_nuke(struct intel_fbc *fbc)
 371{
 372	struct intel_display *display = fbc->display;
 373	struct intel_fbc_state *fbc_state = &fbc->state;
 374	enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 375
 376	intel_de_write_fw(display, DSPADDR(display, i9xx_plane),
 377			  intel_de_read_fw(display, DSPADDR(display, i9xx_plane)));
 378}
 379
 380static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)
 381{
 382	struct intel_display *display = fbc->display;
 383
 384	drm_WARN_ON(display->drm,
 385		    range_end_overflows_t(u64, i915_gem_stolen_area_address(display->drm),
 386					  i915_gem_stolen_node_offset(fbc->compressed_fb),
 387					  U32_MAX));
 388	drm_WARN_ON(display->drm,
 389		    range_end_overflows_t(u64, i915_gem_stolen_area_address(display->drm),
 390					  i915_gem_stolen_node_offset(fbc->compressed_llb),
 391					  U32_MAX));
 392	intel_de_write(display, FBC_CFB_BASE,
 393		       i915_gem_stolen_node_address(fbc->compressed_fb));
 394	intel_de_write(display, FBC_LL_BASE,
 395		       i915_gem_stolen_node_address(fbc->compressed_llb));
 396}
 397
 398static const struct intel_fbc_funcs i8xx_fbc_funcs = {
 399	.activate = i8xx_fbc_activate,
 400	.deactivate = i8xx_fbc_deactivate,
 401	.is_active = i8xx_fbc_is_active,
 402	.is_compressing = i8xx_fbc_is_compressing,
 403	.nuke = i8xx_fbc_nuke,
 404	.program_cfb = i8xx_fbc_program_cfb,
 405};
 406
 407static void i965_fbc_nuke(struct intel_fbc *fbc)
 408{
 409	struct intel_display *display = fbc->display;
 410	struct intel_fbc_state *fbc_state = &fbc->state;
 411	enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 412
 413	intel_de_write_fw(display, DSPSURF(display, i9xx_plane),
 414			  intel_de_read_fw(display, DSPSURF(display, i9xx_plane)));
 415}
 416
 417static const struct intel_fbc_funcs i965_fbc_funcs = {
 418	.activate = i8xx_fbc_activate,
 419	.deactivate = i8xx_fbc_deactivate,
 420	.is_active = i8xx_fbc_is_active,
 421	.is_compressing = i8xx_fbc_is_compressing,
 422	.nuke = i965_fbc_nuke,
 423	.program_cfb = i8xx_fbc_program_cfb,
 424};
 425
 426static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)
 427{
 428	switch (fbc->limit) {
 429	default:
 430		MISSING_CASE(fbc->limit);
 431		fallthrough;
 432	case 1:
 433		return DPFC_CTL_LIMIT_1X;
 434	case 2:
 435		return DPFC_CTL_LIMIT_2X;
 436	case 4:
 437		return DPFC_CTL_LIMIT_4X;
 438	}
 439}
 440
 441static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)
 442{
 443	struct intel_display *display = fbc->display;
 444	const struct intel_fbc_state *fbc_state = &fbc->state;
 445	u32 dpfc_ctl;
 446
 447	dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |
 448		DPFC_CTL_PLANE_G4X(fbc_state->plane->i9xx_plane);
 449
 450	if (display->platform.g4x)
 451		dpfc_ctl |= DPFC_CTL_SR_EN;
 452
 453	if (fbc_state->fence_id >= 0) {
 454		dpfc_ctl |= DPFC_CTL_FENCE_EN_G4X;
 455
 456		if (DISPLAY_VER(display) < 6)
 457			dpfc_ctl |= DPFC_CTL_FENCENO(fbc_state->fence_id);
 458	}
 459
 460	return dpfc_ctl;
 461}
 462
 463static void g4x_fbc_activate(struct intel_fbc *fbc)
 464{
 465	struct intel_display *display = fbc->display;
 466	const struct intel_fbc_state *fbc_state = &fbc->state;
 467
 468	intel_de_write(display, DPFC_FENCE_YOFF,
 469		       fbc_state->fence_y_offset);
 470
 471	intel_de_write(display, DPFC_CONTROL,
 472		       DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 473}
 474
 475static void g4x_fbc_deactivate(struct intel_fbc *fbc)
 476{
 477	struct intel_display *display = fbc->display;
 478	u32 dpfc_ctl;
 479
 480	/* Disable compression */
 481	dpfc_ctl = intel_de_read(display, DPFC_CONTROL);
 482	if (dpfc_ctl & DPFC_CTL_EN) {
 483		dpfc_ctl &= ~DPFC_CTL_EN;
 484		intel_de_write(display, DPFC_CONTROL, dpfc_ctl);
 485	}
 486}
 487
 488static bool g4x_fbc_is_active(struct intel_fbc *fbc)
 489{
 490	return intel_de_read(fbc->display, DPFC_CONTROL) & DPFC_CTL_EN;
 491}
 492
 493static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)
 494{
 495	return intel_de_read(fbc->display, DPFC_STATUS) & DPFC_COMP_SEG_MASK;
 496}
 497
 498static void g4x_fbc_program_cfb(struct intel_fbc *fbc)
 499{
 500	struct intel_display *display = fbc->display;
 501
 502	intel_de_write(display, DPFC_CB_BASE,
 503		       i915_gem_stolen_node_offset(fbc->compressed_fb));
 504}
 505
 506static const struct intel_fbc_funcs g4x_fbc_funcs = {
 507	.activate = g4x_fbc_activate,
 508	.deactivate = g4x_fbc_deactivate,
 509	.is_active = g4x_fbc_is_active,
 510	.is_compressing = g4x_fbc_is_compressing,
 511	.nuke = i965_fbc_nuke,
 512	.program_cfb = g4x_fbc_program_cfb,
 513};
 514
 515static void ilk_fbc_activate(struct intel_fbc *fbc)
 516{
 517	struct intel_display *display = fbc->display;
 518	struct intel_fbc_state *fbc_state = &fbc->state;
 519
 520	intel_de_write(display, ILK_DPFC_FENCE_YOFF(fbc->id),
 521		       fbc_state->fence_y_offset);
 522
 523	intel_de_write(display, ILK_DPFC_CONTROL(fbc->id),
 524		       DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 525}
 526
 527static void fbc_compressor_clkgate_disable_wa(struct intel_fbc *fbc,
 528					      bool disable)
 529{
 530	struct intel_display *display = fbc->display;
 531
 532	if (display->platform.dg2)
 533		intel_de_rmw(display, GEN9_CLKGATE_DIS_4, DG2_DPFC_GATING_DIS,
 534			     disable ? DG2_DPFC_GATING_DIS : 0);
 535	else if (DISPLAY_VER(display) >= 14)
 536		intel_de_rmw(display, MTL_PIPE_CLKGATE_DIS2(fbc->id),
 537			     MTL_DPFC_GATING_DIS,
 538			     disable ? MTL_DPFC_GATING_DIS : 0);
 539}
 540
 541static void ilk_fbc_deactivate(struct intel_fbc *fbc)
 542{
 543	struct intel_display *display = fbc->display;
 544	u32 dpfc_ctl;
 545
 546	if (HAS_FBC_DIRTY_RECT(display))
 547		intel_de_write(display, XE3_FBC_DIRTY_CTL(fbc->id), 0);
 548
 549	/* Disable compression */
 550	dpfc_ctl = intel_de_read(display, ILK_DPFC_CONTROL(fbc->id));
 551	if (dpfc_ctl & DPFC_CTL_EN) {
 552		dpfc_ctl &= ~DPFC_CTL_EN;
 553		intel_de_write(display, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 554	}
 555}
 556
 557static bool ilk_fbc_is_active(struct intel_fbc *fbc)
 558{
 559	return intel_de_read(fbc->display, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;
 560}
 561
 562static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)
 563{
 564	return intel_de_read(fbc->display, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;
 565}
 566
 567static void ilk_fbc_program_cfb(struct intel_fbc *fbc)
 568{
 569	struct intel_display *display = fbc->display;
 570
 571	intel_de_write(display, ILK_DPFC_CB_BASE(fbc->id),
 572		       i915_gem_stolen_node_offset(fbc->compressed_fb));
 573}
 574
 575static const struct intel_fbc_funcs ilk_fbc_funcs = {
 576	.activate = ilk_fbc_activate,
 577	.deactivate = ilk_fbc_deactivate,
 578	.is_active = ilk_fbc_is_active,
 579	.is_compressing = ilk_fbc_is_compressing,
 580	.nuke = i965_fbc_nuke,
 581	.program_cfb = ilk_fbc_program_cfb,
 582};
 583
 584static void snb_fbc_program_fence(struct intel_fbc *fbc)
 585{
 586	struct intel_display *display = fbc->display;
 587	const struct intel_fbc_state *fbc_state = &fbc->state;
 588	u32 ctl = 0;
 589
 590	if (fbc_state->fence_id >= 0)
 591		ctl = SNB_DPFC_FENCE_EN | SNB_DPFC_FENCENO(fbc_state->fence_id);
 592
 593	intel_de_write(display, SNB_DPFC_CTL_SA, ctl);
 594	intel_de_write(display, SNB_DPFC_CPU_FENCE_OFFSET, fbc_state->fence_y_offset);
 595}
 596
 597static void snb_fbc_activate(struct intel_fbc *fbc)
 598{
 599	snb_fbc_program_fence(fbc);
 600
 601	ilk_fbc_activate(fbc);
 602}
 603
 604static void snb_fbc_nuke(struct intel_fbc *fbc)
 605{
 606	struct intel_display *display = fbc->display;
 607
 608	intel_de_write(display, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);
 609	intel_de_posting_read(display, MSG_FBC_REND_STATE(fbc->id));
 610}
 611
 612static const struct intel_fbc_funcs snb_fbc_funcs = {
 613	.activate = snb_fbc_activate,
 614	.deactivate = ilk_fbc_deactivate,
 615	.is_active = ilk_fbc_is_active,
 616	.is_compressing = ilk_fbc_is_compressing,
 617	.nuke = snb_fbc_nuke,
 618	.program_cfb = ilk_fbc_program_cfb,
 619};
 620
 621static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)
 622{
 623	struct intel_display *display = fbc->display;
 624	const struct intel_fbc_state *fbc_state = &fbc->state;
 625	u32 val = 0;
 626
 627	if (fbc_state->override_cfb_stride)
 628		val |= FBC_STRIDE_OVERRIDE |
 629			FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 630
 631	intel_de_write(display, GLK_FBC_STRIDE(fbc->id), val);
 632}
 633
 634static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)
 635{
 636	struct intel_display *display = fbc->display;
 637	const struct intel_fbc_state *fbc_state = &fbc->state;
 638	u32 val = 0;
 639
 640	/* Display WA #0529: skl, kbl, bxt. */
 641	if (fbc_state->override_cfb_stride)
 642		val |= CHICKEN_FBC_STRIDE_OVERRIDE |
 643			CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 644
 645	intel_de_rmw(display, CHICKEN_MISC_4,
 646		     CHICKEN_FBC_STRIDE_OVERRIDE |
 647		     CHICKEN_FBC_STRIDE_MASK, val);
 648}
 649
 650static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)
 651{
 652	struct intel_display *display = fbc->display;
 653	const struct intel_fbc_state *fbc_state = &fbc->state;
 654	u32 dpfc_ctl;
 655
 656	dpfc_ctl = g4x_dpfc_ctl_limit(fbc);
 657
 658	if (display->platform.ivybridge)
 659		dpfc_ctl |= DPFC_CTL_PLANE_IVB(fbc_state->plane->i9xx_plane);
 660
 661	if (DISPLAY_VER(display) >= 20)
 662		dpfc_ctl |= DPFC_CTL_PLANE_BINDING(fbc_state->plane->id);
 663
 664	if (fbc_state->fence_id >= 0)
 665		dpfc_ctl |= DPFC_CTL_FENCE_EN_IVB;
 666
 667	if (fbc->false_color)
 668		dpfc_ctl |= DPFC_CTL_FALSE_COLOR;
 669
 670	return dpfc_ctl;
 671}
 672
 673static void ivb_fbc_activate(struct intel_fbc *fbc)
 674{
 675	struct intel_display *display = fbc->display;
 676	u32 dpfc_ctl;
 677
 678	if (DISPLAY_VER(display) >= 10)
 679		glk_fbc_program_cfb_stride(fbc);
 680	else if (DISPLAY_VER(display) == 9)
 681		skl_fbc_program_cfb_stride(fbc);
 682
 683	if (intel_fbc_has_fences(display))
 684		snb_fbc_program_fence(fbc);
 685
 686	/* wa_14019417088 Alternative WA*/
 687	dpfc_ctl = ivb_dpfc_ctl(fbc);
 688	if (DISPLAY_VER(display) >= 20)
 689		intel_de_write(display, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 690
 691	if (HAS_FBC_DIRTY_RECT(display))
 692		intel_de_write(display, XE3_FBC_DIRTY_CTL(fbc->id),
 693			       FBC_DIRTY_RECT_EN);
 694
 695	intel_de_write(display, ILK_DPFC_CONTROL(fbc->id),
 696		       DPFC_CTL_EN | dpfc_ctl);
 697}
 698
 699static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)
 700{
 701	return intel_de_read(fbc->display, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;
 702}
 703
 704static void ivb_fbc_set_false_color(struct intel_fbc *fbc,
 705				    bool enable)
 706{
 707	intel_de_rmw(fbc->display, ILK_DPFC_CONTROL(fbc->id),
 708		     DPFC_CTL_FALSE_COLOR, enable ? DPFC_CTL_FALSE_COLOR : 0);
 709}
 710
 711static const struct intel_fbc_funcs ivb_fbc_funcs = {
 712	.activate = ivb_fbc_activate,
 713	.deactivate = ilk_fbc_deactivate,
 714	.is_active = ilk_fbc_is_active,
 715	.is_compressing = ivb_fbc_is_compressing,
 716	.nuke = snb_fbc_nuke,
 717	.program_cfb = ilk_fbc_program_cfb,
 718	.set_false_color = ivb_fbc_set_false_color,
 719};
 720
 721static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)
 722{
 723	return fbc->funcs->is_active(fbc);
 724}
 725
 726static void intel_fbc_hw_activate(struct intel_fbc *fbc)
 727{
 728	trace_intel_fbc_activate(fbc->state.plane);
 729
 730	fbc->active = true;
 731	fbc->activated = true;
 732
 733	fbc->funcs->activate(fbc);
 734}
 735
 736static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)
 737{
 738	trace_intel_fbc_deactivate(fbc->state.plane);
 739
 740	fbc->active = false;
 741
 742	fbc->funcs->deactivate(fbc);
 743}
 744
 745static bool intel_fbc_is_compressing(struct intel_fbc *fbc)
 746{
 747	return fbc->funcs->is_compressing(fbc);
 748}
 749
 750static void intel_fbc_nuke(struct intel_fbc *fbc)
 751{
 752	struct intel_display *display = fbc->display;
 753
 754	lockdep_assert_held(&fbc->lock);
 755	drm_WARN_ON(display->drm, fbc->flip_pending);
 756
 757	trace_intel_fbc_nuke(fbc->state.plane);
 758
 759	fbc->funcs->nuke(fbc);
 760}
 761
 762static void intel_fbc_activate(struct intel_fbc *fbc)
 763{
 764	struct intel_display *display = fbc->display;
 765
 766	lockdep_assert_held(&fbc->lock);
 767
 768	/* only the fence can change for a flip nuke */
 769	if (fbc->active && !intel_fbc_has_fences(display))
 770		return;
 771	/*
 772	 * In case of FBC dirt rect, any updates to the FBC registers will
 773	 * trigger the nuke.
 774	 */
 775	drm_WARN_ON(display->drm, fbc->active && HAS_FBC_DIRTY_RECT(display));
 776
 777	intel_fbc_hw_activate(fbc);
 778	intel_fbc_nuke(fbc);
 779
 780	fbc->no_fbc_reason = NULL;
 781}
 782
 783static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)
 784{
 785	lockdep_assert_held(&fbc->lock);
 786
 787	if (fbc->active)
 788		intel_fbc_hw_deactivate(fbc);
 789
 790	fbc->no_fbc_reason = reason;
 791}
 792
 793static u64 intel_fbc_cfb_base_max(struct intel_display *display)
 794{
 795	if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
 796		return BIT_ULL(28);
 797	else
 798		return BIT_ULL(32);
 799}
 800
 801static u64 intel_fbc_stolen_end(struct intel_display *display)
 802{
 803	u64 end;
 804
 805	/* The FBC hardware for BDW/SKL doesn't have access to the stolen
 806	 * reserved range size, so it always assumes the maximum (8mb) is used.
 807	 * If we enable FBC using a CFB on that memory range we'll get FIFO
 808	 * underruns, even if that range is not reserved by the BIOS. */
 809	if (display->platform.broadwell ||
 810	    (DISPLAY_VER(display) == 9 && !display->platform.broxton))
 811		end = i915_gem_stolen_area_size(display->drm) - 8 * 1024 * 1024;
 812	else
 813		end = U64_MAX;
 814
 815	return min(end, intel_fbc_cfb_base_max(display));
 816}
 817
 818static int intel_fbc_min_limit(const struct intel_plane_state *plane_state)
 819{
 820	return plane_state->hw.fb->format->cpp[0] == 2 ? 2 : 1;
 821}
 822
 823static int intel_fbc_max_limit(struct intel_display *display)
 824{
 825	/* WaFbcOnly1to1Ratio:ctg */
 826	if (display->platform.g4x)
 827		return 1;
 828
 829	/*
 830	 * FBC2 can only do 1:1, 1:2, 1:4, we limit
 831	 * FBC1 to the same out of convenience.
 832	 */
 833	return 4;
 834}
 835
 836static int find_compression_limit(struct intel_fbc *fbc,
 837				  unsigned int size, int min_limit)
 838{
 839	struct intel_display *display = fbc->display;
 840	u64 end = intel_fbc_stolen_end(display);
 841	int ret, limit = min_limit;
 842
 843	size /= limit;
 844
 845	/* Try to over-allocate to reduce reallocations and fragmentation. */
 846	ret = i915_gem_stolen_insert_node_in_range(fbc->compressed_fb,
 847						   size <<= 1, 4096, 0, end);
 848	if (ret == 0)
 849		return limit;
 850
 851	for (; limit <= intel_fbc_max_limit(display); limit <<= 1) {
 852		ret = i915_gem_stolen_insert_node_in_range(fbc->compressed_fb,
 853							   size >>= 1, 4096, 0, end);
 854		if (ret == 0)
 855			return limit;
 856	}
 857
 858	return 0;
 859}
 860
 861static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
 862			       unsigned int size, int min_limit)
 863{
 864	struct intel_display *display = fbc->display;
 865	int ret;
 866
 867	drm_WARN_ON(display->drm,
 868		    i915_gem_stolen_node_allocated(fbc->compressed_fb));
 869	drm_WARN_ON(display->drm,
 870		    i915_gem_stolen_node_allocated(fbc->compressed_llb));
 871
 872	if (DISPLAY_VER(display) < 5 && !display->platform.g4x) {
 873		ret = i915_gem_stolen_insert_node(fbc->compressed_llb, 4096, 4096);
 874		if (ret)
 875			goto err;
 876	}
 877
 878	ret = find_compression_limit(fbc, size, min_limit);
 879	if (!ret)
 880		goto err_llb;
 881	else if (ret > min_limit)
 882		drm_info_once(display->drm,
 883			      "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
 884
 885	fbc->limit = ret;
 886
 887	drm_dbg_kms(display->drm,
 888		    "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
 889		    i915_gem_stolen_node_size(fbc->compressed_fb), fbc->limit);
 890	return 0;
 891
 892err_llb:
 893	if (i915_gem_stolen_node_allocated(fbc->compressed_llb))
 894		i915_gem_stolen_remove_node(fbc->compressed_llb);
 895err:
 896	if (i915_gem_stolen_initialized(display->drm))
 897		drm_info_once(display->drm,
 898			      "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
 899	return -ENOSPC;
 900}
 901
 902static void intel_fbc_program_cfb(struct intel_fbc *fbc)
 903{
 904	fbc->funcs->program_cfb(fbc);
 905}
 906
 907static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 908{
 909	struct intel_display *display = fbc->display;
 910
 911	if (display->platform.skylake || display->platform.broxton) {
 912		/*
 913		 * WaFbcHighMemBwCorruptionAvoidance:skl,bxt
 914		 * Display WA #0883: skl,bxt
 915		 */
 916		intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 917			     0, DPFC_DISABLE_DUMMY0);
 918	}
 919
 920	if (display->platform.skylake || display->platform.kabylake ||
 921	    display->platform.coffeelake || display->platform.cometlake) {
 922		/*
 923		 * WaFbcNukeOnHostModify:skl,kbl,cfl
 924		 * Display WA #0873: skl,kbl,cfl
 925		 */
 926		intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 927			     0, DPFC_NUKE_ON_ANY_MODIFICATION);
 928	}
 929
 930	/* Wa_1409120013:icl,jsl,tgl,dg1 */
 931	if (IS_DISPLAY_VER(display, 11, 12))
 932		intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 933			     0, DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 934	/*
 935	 * Wa_22014263786
 936	 * Fixes: Screen flicker with FBC and Package C state enabled
 937	 * Workaround: Forced SLB invalidation before start of new frame.
 938	 */
 939	if (intel_display_wa(display, 22014263786))
 940		intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 941			     0, DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
 942
 943	/* wa_18038517565 Disable DPFC clock gating before FBC enable */
 944	if (display->platform.dg2 || DISPLAY_VER(display) >= 14)
 945		fbc_compressor_clkgate_disable_wa(fbc, true);
 946}
 947
 948static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
 949{
 950	if (WARN_ON(intel_fbc_hw_is_active(fbc)))
 951		return;
 952
 953	if (i915_gem_stolen_node_allocated(fbc->compressed_llb))
 954		i915_gem_stolen_remove_node(fbc->compressed_llb);
 955	if (i915_gem_stolen_node_allocated(fbc->compressed_fb))
 956		i915_gem_stolen_remove_node(fbc->compressed_fb);
 957}
 958
 959void intel_fbc_cleanup(struct intel_display *display)
 960{
 961	struct intel_fbc *fbc;
 962	enum intel_fbc_id fbc_id;
 963
 964	for_each_intel_fbc(display, fbc, fbc_id) {
 965		mutex_lock(&fbc->lock);
 966		__intel_fbc_cleanup_cfb(fbc);
 967		mutex_unlock(&fbc->lock);
 968
 969		i915_gem_stolen_node_free(fbc->compressed_fb);
 970		i915_gem_stolen_node_free(fbc->compressed_llb);
 971
 972		kfree(fbc);
 973	}
 974}
 975
 976static bool i8xx_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 977{
 978	const struct drm_framebuffer *fb = plane_state->hw.fb;
 979	unsigned int stride = intel_fbc_plane_stride(plane_state) *
 980		fb->format->cpp[0];
 981
 982	return stride == 4096 || stride == 8192;
 983}
 984
 985static bool i965_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 986{
 987	const struct drm_framebuffer *fb = plane_state->hw.fb;
 988	unsigned int stride = intel_fbc_plane_stride(plane_state) *
 989		fb->format->cpp[0];
 990
 991	return stride >= 2048 && stride <= 16384;
 992}
 993
 994static bool g4x_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 995{
 996	return true;
 997}
 998
 999static bool skl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
1000{
1001	const struct drm_framebuffer *fb = plane_state->hw.fb;
1002	unsigned int stride = intel_fbc_plane_stride(plane_state) *
1003		fb->format->cpp[0];
1004
1005	/* Display WA #1105: skl,bxt,kbl,cfl,glk */
1006	if (fb->modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
1007		return false;
1008
1009	return true;
1010}
1011
1012static bool icl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
1013{
1014	return true;
1015}
1016
1017static bool stride_is_valid(const struct intel_plane_state *plane_state)
1018{
1019	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1020
1021	if (DISPLAY_VER(display) >= 11)
1022		return icl_fbc_stride_is_valid(plane_state);
1023	else if (DISPLAY_VER(display) >= 9)
1024		return skl_fbc_stride_is_valid(plane_state);
1025	else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
1026		return g4x_fbc_stride_is_valid(plane_state);
1027	else if (DISPLAY_VER(display) == 4)
1028		return i965_fbc_stride_is_valid(plane_state);
1029	else
1030		return i8xx_fbc_stride_is_valid(plane_state);
1031}
1032
1033static bool i8xx_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1034{
1035	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1036	const struct drm_framebuffer *fb = plane_state->hw.fb;
1037
1038	switch (fb->format->format) {
1039	case DRM_FORMAT_XRGB8888:
1040	case DRM_FORMAT_XBGR8888:
1041		return true;
1042	case DRM_FORMAT_XRGB1555:
1043	case DRM_FORMAT_RGB565:
1044		/* 16bpp not supported on gen2 */
1045		if (DISPLAY_VER(display) == 2)
1046			return false;
1047		return true;
1048	default:
1049		return false;
1050	}
1051}
1052
1053static bool g4x_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1054{
1055	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1056	const struct drm_framebuffer *fb = plane_state->hw.fb;
1057
1058	switch (fb->format->format) {
1059	case DRM_FORMAT_XRGB8888:
1060	case DRM_FORMAT_XBGR8888:
1061		return true;
1062	case DRM_FORMAT_RGB565:
1063		/* WaFbcOnly1to1Ratio:ctg */
1064		if (display->platform.g4x)
1065			return false;
1066		return true;
1067	default:
1068		return false;
1069	}
1070}
1071
1072static bool lnl_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1073{
1074	const struct drm_framebuffer *fb = plane_state->hw.fb;
1075
1076	switch (fb->format->format) {
1077	case DRM_FORMAT_XRGB8888:
1078	case DRM_FORMAT_XBGR8888:
1079	case DRM_FORMAT_ARGB8888:
1080	case DRM_FORMAT_ABGR8888:
1081	case DRM_FORMAT_RGB565:
1082		return true;
1083	default:
1084		return false;
1085	}
1086}
1087
1088static bool
1089xe3p_lpd_fbc_fp16_format_is_valid(const struct intel_plane_state *plane_state)
1090{
1091	const struct drm_framebuffer *fb = plane_state->hw.fb;
1092
1093	switch (fb->format->format) {
1094	case DRM_FORMAT_ARGB16161616F:
1095	case DRM_FORMAT_ABGR16161616F:
1096		return true;
1097	default:
1098		return false;
1099	}
1100}
1101
1102static bool xe3p_lpd_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1103{
1104	const struct drm_framebuffer *fb = plane_state->hw.fb;
1105
1106	if (lnl_fbc_pixel_format_is_valid(plane_state))
1107		return true;
1108
1109	if (xe3p_lpd_fbc_fp16_format_is_valid(plane_state))
1110		return true;
1111
1112	switch (fb->format->format) {
1113	case DRM_FORMAT_XRGB16161616:
1114	case DRM_FORMAT_XBGR16161616:
1115	case DRM_FORMAT_ARGB16161616:
1116	case DRM_FORMAT_ABGR16161616:
1117		return true;
1118	default:
1119		return false;
1120	}
1121}
1122
1123bool intel_fbc_need_pixel_normalizer(const struct intel_plane_state *plane_state)
1124{
1125	struct intel_display *display = to_intel_display(plane_state);
1126
1127	if (HAS_PIXEL_NORMALIZER(display) &&
1128	    xe3p_lpd_fbc_fp16_format_is_valid(plane_state))
1129		return true;
1130
1131	return false;
1132}
1133
1134static bool pixel_format_is_valid(const struct intel_plane_state *plane_state)
1135{
1136	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1137
1138	if (DISPLAY_VER(display) >= 35)
1139		return xe3p_lpd_fbc_pixel_format_is_valid(plane_state);
1140	else if (DISPLAY_VER(display) >= 20)
1141		return lnl_fbc_pixel_format_is_valid(plane_state);
1142	else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
1143		return g4x_fbc_pixel_format_is_valid(plane_state);
1144	else
1145		return i8xx_fbc_pixel_format_is_valid(plane_state);
1146}
1147
1148static bool i8xx_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1149{
1150	return plane_state->hw.rotation == DRM_MODE_ROTATE_0;
1151}
1152
1153static bool g4x_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1154{
1155	return true;
1156}
1157
1158static bool skl_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1159{
1160	const struct drm_framebuffer *fb = plane_state->hw.fb;
1161	unsigned int rotation = plane_state->hw.rotation;
1162
1163	if (fb->format->format == DRM_FORMAT_RGB565 &&
1164	    drm_rotation_90_or_270(rotation))
1165		return false;
1166
1167	return true;
1168}
1169
1170static bool rotation_is_valid(const struct intel_plane_state *plane_state)
1171{
1172	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1173
1174	if (DISPLAY_VER(display) >= 9)
1175		return skl_fbc_rotation_is_valid(plane_state);
1176	else if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
1177		return g4x_fbc_rotation_is_valid(plane_state);
1178	else
1179		return i8xx_fbc_rotation_is_valid(plane_state);
1180}
1181
1182static void intel_fbc_max_surface_size(struct intel_display *display,
1183				       unsigned int *w, unsigned int *h)
1184{
1185	if (DISPLAY_VER(display) >= 11) {
1186		*w = 8192;
1187		*h = 4096;
1188	} else if (DISPLAY_VER(display) >= 10) {
1189		*w = 5120;
1190		*h = 4096;
1191	} else if (DISPLAY_VER(display) >= 7) {
1192		*w = 4096;
1193		*h = 4096;
1194	} else if (DISPLAY_VER(display) >= 5 || display->platform.g4x) {
1195		*w = 4096;
1196		*h = 2048;
1197	} else {
1198		*w = 2048;
1199		*h = 1536;
1200	}
1201}
1202
1203/*
1204 * For some reason, the hardware tracking starts looking at whatever we
1205 * programmed as the display plane base address register. It does not look at
1206 * the X and Y offset registers. That's why we include the src x/y offsets
1207 * instead of just looking at the plane size.
1208 */
1209static bool intel_fbc_surface_size_ok(const struct intel_plane_state *plane_state)
1210{
1211	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1212	unsigned int effective_w, effective_h, max_w, max_h;
1213
1214	intel_fbc_max_surface_size(display, &max_w, &max_h);
1215
1216	effective_w = plane_state->view.color_plane[0].x +
1217		(drm_rect_width(&plane_state->uapi.src) >> 16);
1218	effective_h = plane_state->view.color_plane[0].y +
1219		(drm_rect_height(&plane_state->uapi.src) >> 16);
1220
1221	return effective_w <= max_w && effective_h <= max_h;
1222}
1223
1224static void intel_fbc_max_plane_size(struct intel_display *display,
1225				     unsigned int *w, unsigned int *h)
1226{
1227	if (DISPLAY_VER(display) >= 10) {
1228		*w = 5120;
1229		*h = 4096;
1230	} else if (DISPLAY_VER(display) >= 8 || display->platform.haswell) {
1231		*w = 4096;
1232		*h = 4096;
1233	} else if (DISPLAY_VER(display) >= 5 || display->platform.g4x) {
1234		*w = 4096;
1235		*h = 2048;
1236	} else {
1237		*w = 2048;
1238		*h = 1536;
1239	}
1240}
1241
1242static bool intel_fbc_plane_size_valid(const struct intel_plane_state *plane_state)
1243{
1244	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1245	unsigned int w, h, max_w, max_h;
1246
1247	intel_fbc_max_plane_size(display, &max_w, &max_h);
1248
1249	w = drm_rect_width(&plane_state->uapi.src) >> 16;
1250	h = drm_rect_height(&plane_state->uapi.src) >> 16;
1251
1252	return w <= max_w && h <= max_h;
1253}
1254
1255static bool i8xx_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1256{
1257	const struct drm_framebuffer *fb = plane_state->hw.fb;
1258
1259	return fb->modifier == I915_FORMAT_MOD_X_TILED;
1260}
1261
1262static bool skl_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1263{
1264	return true;
1265}
1266
1267static bool tiling_is_valid(const struct intel_plane_state *plane_state)
1268{
1269	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1270
1271	if (DISPLAY_VER(display) >= 9)
1272		return skl_fbc_tiling_valid(plane_state);
1273	else
1274		return i8xx_fbc_tiling_valid(plane_state);
1275}
1276
1277static void
1278intel_fbc_invalidate_dirty_rect(struct intel_fbc *fbc)
1279{
1280	lockdep_assert_held(&fbc->lock);
1281
1282	fbc->state.dirty_rect = DRM_RECT_INIT(0, 0, 0, 0);
1283}
1284
1285static void
1286intel_fbc_program_dirty_rect(struct intel_dsb *dsb, struct intel_fbc *fbc,
1287			     const struct drm_rect *fbc_dirty_rect)
1288{
1289	struct intel_display *display = fbc->display;
1290
1291	drm_WARN_ON(display->drm, fbc_dirty_rect->y2 == 0);
1292
1293	intel_de_write_dsb(display, dsb, XE3_FBC_DIRTY_RECT(fbc->id),
1294			   FBC_DIRTY_RECT_START_LINE(fbc_dirty_rect->y1) |
1295			   FBC_DIRTY_RECT_END_LINE(fbc_dirty_rect->y2 - 1));
1296}
1297
1298static void
1299intel_fbc_dirty_rect_update(struct intel_dsb *dsb, struct intel_fbc *fbc)
1300{
1301	const struct drm_rect *fbc_dirty_rect = &fbc->state.dirty_rect;
1302
1303	lockdep_assert_held(&fbc->lock);
1304
1305	if (!drm_rect_visible(fbc_dirty_rect))
1306		return;
1307
1308	intel_fbc_program_dirty_rect(dsb, fbc, fbc_dirty_rect);
1309}
1310
1311void
1312intel_fbc_dirty_rect_update_noarm(struct intel_dsb *dsb,
1313				  struct intel_plane *plane)
1314{
1315	struct intel_display *display = to_intel_display(plane);
1316	struct intel_fbc *fbc = plane->fbc;
1317
1318	if (!HAS_FBC_DIRTY_RECT(display))
1319		return;
1320
1321	mutex_lock(&fbc->lock);
1322
1323	if (fbc->state.plane == plane)
1324		intel_fbc_dirty_rect_update(dsb, fbc);
1325
1326	mutex_unlock(&fbc->lock);
1327}
1328
1329static void
1330intel_fbc_hw_intialize_dirty_rect(struct intel_fbc *fbc,
1331				  const struct intel_plane_state *plane_state)
1332{
1333	struct drm_rect src;
1334
1335	/*
1336	 * Initializing the FBC HW with the whole plane area as the dirty rect.
1337	 * This is to ensure that we have valid coords be written to the
1338	 * HW as dirty rect.
1339	 */
1340	drm_rect_fp_to_int(&src, &plane_state->uapi.src);
1341
1342	intel_fbc_program_dirty_rect(NULL, fbc, &src);
1343}
1344
1345static void intel_fbc_update_state(struct intel_atomic_state *state,
1346				   struct intel_crtc *crtc,
1347				   struct intel_plane *plane)
1348{
1349	struct intel_display *display = to_intel_display(state->base.dev);
1350	const struct intel_crtc_state *crtc_state =
1351		intel_atomic_get_new_crtc_state(state, crtc);
1352	const struct intel_plane_state *plane_state =
1353		intel_atomic_get_new_plane_state(state, plane);
1354	struct intel_fbc *fbc = plane->fbc;
1355	struct intel_fbc_state *fbc_state = &fbc->state;
1356
1357	WARN_ON(plane_state->no_fbc_reason);
1358	WARN_ON(fbc_state->plane && fbc_state->plane != plane);
1359
1360	fbc_state->plane = plane;
1361
1362	/* FBC1 compression interval: arbitrary choice of 1 second */
1363	fbc_state->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode);
1364
1365	fbc_state->fence_y_offset = intel_plane_fence_y_offset(plane_state);
1366
1367	drm_WARN_ON(display->drm, plane_state->flags & PLANE_HAS_FENCE &&
1368		    !intel_fbc_has_fences(display));
1369
1370	if (plane_state->flags & PLANE_HAS_FENCE)
1371		fbc_state->fence_id =  i915_vma_fence_id(plane_state->ggtt_vma);
1372	else
1373		fbc_state->fence_id = -1;
1374
1375	fbc_state->cfb_stride = intel_fbc_cfb_stride(plane_state);
1376	fbc_state->cfb_size = intel_fbc_cfb_size(plane_state);
1377	fbc_state->override_cfb_stride = intel_fbc_override_cfb_stride(plane_state);
1378}
1379
1380static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
1381{
1382	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1383
1384	/*
1385	 * The use of a CPU fence is one of two ways to detect writes by the
1386	 * CPU to the scanout and trigger updates to the FBC.
1387	 *
1388	 * The other method is by software tracking (see
1389	 * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
1390	 * the current compressed buffer and recompress it.
1391	 *
1392	 * Note that is possible for a tiled surface to be unmappable (and
1393	 * so have no fence associated with it) due to aperture constraints
1394	 * at the time of pinning.
1395	 */
1396	return DISPLAY_VER(display) >= 9 ||
1397		(plane_state->flags & PLANE_HAS_FENCE &&
1398		 i915_vma_fence_id(plane_state->ggtt_vma) != -1);
1399}
1400
1401static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
1402{
1403	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1404	struct intel_fbc *fbc = plane->fbc;
1405
1406	return intel_fbc_min_limit(plane_state) <= fbc->limit &&
1407		intel_fbc_cfb_size(plane_state) <= fbc->limit *
1408			i915_gem_stolen_node_size(fbc->compressed_fb);
1409}
1410
1411static bool intel_fbc_is_ok(const struct intel_plane_state *plane_state)
1412{
1413	return !plane_state->no_fbc_reason &&
1414		intel_fbc_is_fence_ok(plane_state) &&
1415		intel_fbc_is_cfb_ok(plane_state);
1416}
1417
1418static void
1419__intel_fbc_prepare_dirty_rect(const struct intel_plane_state *plane_state,
1420			       const struct intel_crtc_state *crtc_state)
1421{
1422	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1423	struct intel_fbc *fbc = plane->fbc;
1424	struct drm_rect *fbc_dirty_rect = &fbc->state.dirty_rect;
1425	int width = drm_rect_width(&plane_state->uapi.src) >> 16;
1426	const struct drm_rect *damage = &plane_state->damage;
1427	int y_offset = plane_state->view.color_plane[0].y;
1428
1429	lockdep_assert_held(&fbc->lock);
1430
1431	if (intel_crtc_needs_modeset(crtc_state) ||
1432	    !intel_fbc_is_ok(plane_state)) {
1433		intel_fbc_invalidate_dirty_rect(fbc);
1434		return;
1435	}
1436
1437	if (drm_rect_visible(damage))
1438		*fbc_dirty_rect = *damage;
1439	else
1440		/* dirty rect must cover at least one line */
1441		*fbc_dirty_rect = DRM_RECT_INIT(0, y_offset, width, 1);
1442}
1443
1444void
1445intel_fbc_prepare_dirty_rect(struct intel_atomic_state *state,
1446			     struct intel_crtc *crtc)
1447{
1448	struct intel_display *display = to_intel_display(state);
1449	const struct intel_crtc_state *crtc_state =
1450		intel_atomic_get_new_crtc_state(state, crtc);
1451	struct intel_plane_state *plane_state;
1452	struct intel_plane *plane;
1453	int i;
1454
1455	if (!HAS_FBC_DIRTY_RECT(display))
1456		return;
1457
1458	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1459		struct intel_fbc *fbc = plane->fbc;
1460
1461		if (!fbc || plane->pipe != crtc->pipe)
1462			continue;
1463
1464		mutex_lock(&fbc->lock);
1465
1466		if (fbc->state.plane == plane)
1467			__intel_fbc_prepare_dirty_rect(plane_state,
1468						       crtc_state);
1469
1470		mutex_unlock(&fbc->lock);
1471	}
1472}
1473
1474static int _intel_fbc_min_cdclk(const struct intel_crtc_state *crtc_state)
1475{
1476	struct intel_display *display = to_intel_display(crtc_state);
1477
1478	/* WaFbcExceedCdClockThreshold:hsw,bdw */
1479	if (display->platform.haswell || display->platform.broadwell)
1480		return DIV_ROUND_UP(crtc_state->pixel_rate * 100, 95);
1481
1482	/* no FBC specific limits to worry about */
1483	return 0;
1484}
1485
1486static int intel_fbc_check_plane(struct intel_atomic_state *state,
1487				 struct intel_plane *plane)
1488{
1489	struct intel_display *display = to_intel_display(state->base.dev);
1490	struct drm_i915_private *i915 = to_i915(display->drm);
1491	struct intel_plane_state *plane_state =
1492		intel_atomic_get_new_plane_state(state, plane);
1493	const struct drm_framebuffer *fb = plane_state->hw.fb;
1494	struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1495	const struct intel_crtc_state *crtc_state;
1496	struct intel_fbc *fbc = plane->fbc;
1497
1498	if (!fbc)
1499		return 0;
1500
1501	if (!i915_gem_stolen_initialized(display->drm)) {
1502		plane_state->no_fbc_reason = "stolen memory not initialised";
1503		return 0;
1504	}
1505
1506	if (intel_vgpu_active(i915)) {
1507		plane_state->no_fbc_reason = "VGPU active";
1508		return 0;
1509	}
1510
1511	if (!display->params.enable_fbc) {
1512		plane_state->no_fbc_reason = "disabled per module param or by default";
1513		return 0;
1514	}
1515
1516	if (!plane_state->uapi.visible) {
1517		plane_state->no_fbc_reason = "plane not visible";
1518		return 0;
1519	}
1520
1521	if (intel_display_wa(display, 16023588340)) {
1522		plane_state->no_fbc_reason = "Wa_16023588340";
1523		return 0;
1524	}
1525
1526	/* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1527	if (intel_display_vtd_active(display) &&
1528	    (display->platform.skylake || display->platform.broxton)) {
1529		plane_state->no_fbc_reason = "VT-d enabled";
1530		return 0;
1531	}
1532
1533	crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1534
1535	if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
1536		plane_state->no_fbc_reason = "interlaced mode not supported";
1537		return 0;
1538	}
1539
1540	if (crtc_state->double_wide) {
1541		plane_state->no_fbc_reason = "double wide pipe not supported";
1542		return 0;
1543	}
1544
1545	/*
1546	 * Display 12+ is not supporting FBC with PSR2.
1547	 * Recommendation is to keep this combination disabled
1548	 * Bspec: 50422 HSD: 14010260002
1549	 *
1550	 * TODO: Implement a logic to select between PSR2 selective fetch and
1551	 * FBC based on Bspec: 68881 in xe2lpd onwards.
1552	 *
1553	 * As we still see some strange underruns in those platforms while
1554	 * disabling PSR2, keep FBC disabled in case of selective update is on
1555	 * until the selection logic is implemented.
1556	 */
1557	if (DISPLAY_VER(display) >= 12 && crtc_state->has_sel_update) {
1558		plane_state->no_fbc_reason = "Selective update enabled";
1559		return 0;
1560	}
1561
1562	/* Wa_14016291713 */
1563	if ((IS_DISPLAY_VER(display, 12, 13) ||
1564	     IS_DISPLAY_VERx100_STEP(display, 1400, STEP_A0, STEP_C0)) &&
1565	    crtc_state->has_psr && !crtc_state->has_panel_replay) {
1566		plane_state->no_fbc_reason = "PSR1 enabled (Wa_14016291713)";
1567		return 0;
1568	}
1569
1570	if (!pixel_format_is_valid(plane_state)) {
1571		plane_state->no_fbc_reason = "pixel format not supported";
1572		return 0;
1573	}
1574
1575	if (!tiling_is_valid(plane_state)) {
1576		plane_state->no_fbc_reason = "tiling not supported";
1577		return 0;
1578	}
1579
1580	if (!rotation_is_valid(plane_state)) {
1581		plane_state->no_fbc_reason = "rotation not supported";
1582		return 0;
1583	}
1584
1585	if (!stride_is_valid(plane_state)) {
1586		plane_state->no_fbc_reason = "stride not supported";
1587		return 0;
1588	}
1589
1590	if (DISPLAY_VER(display) < 20 &&
1591	    plane_state->hw.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
1592	    fb->format->has_alpha) {
1593		plane_state->no_fbc_reason = "per-pixel alpha not supported";
1594		return 0;
1595	}
1596
1597	if (!intel_fbc_plane_size_valid(plane_state)) {
1598		plane_state->no_fbc_reason = "plane size too big";
1599		return 0;
1600	}
1601
1602	if (!intel_fbc_surface_size_ok(plane_state)) {
1603		plane_state->no_fbc_reason = "surface size too big";
1604		return 0;
1605	}
1606
1607	/*
1608	 * Work around a problem on GEN9+ HW, where enabling FBC on a plane
1609	 * having a Y offset that isn't divisible by 4 causes FIFO underrun
1610	 * and screen flicker.
1611	 */
1612	if (IS_DISPLAY_VER(display, 9, 12) &&
1613	    plane_state->view.color_plane[0].y & 3) {
1614		plane_state->no_fbc_reason = "plane start Y offset misaligned";
1615		return 0;
1616	}
1617
1618	/* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
1619	if (IS_DISPLAY_VER(display, 9, 12) &&
1620	    (plane_state->view.color_plane[0].y +
1621	     (drm_rect_height(&plane_state->uapi.src) >> 16)) & 3) {
1622		plane_state->no_fbc_reason = "plane end Y offset misaligned";
1623		return 0;
1624	}
1625
1626	if (_intel_fbc_min_cdclk(crtc_state) > display->cdclk.max_cdclk_freq) {
1627		plane_state->no_fbc_reason = "pixel rate too high";
1628		return 0;
1629	}
1630
1631	plane_state->no_fbc_reason = NULL;
1632
1633	return 0;
1634}
1635
1636int intel_fbc_min_cdclk(const struct intel_crtc_state *crtc_state)
1637{
1638	struct intel_display *display = to_intel_display(crtc_state);
1639	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1640	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1641	int min_cdclk;
1642
1643	if (!plane->fbc)
1644		return 0;
1645
1646	min_cdclk = _intel_fbc_min_cdclk(crtc_state);
1647
1648	/*
1649	 * Do not ask for more than the max CDCLK frequency,
1650	 * if that is not enough FBC will simply not be used.
1651	 */
1652	if (min_cdclk > display->cdclk.max_cdclk_freq)
1653		return 0;
1654
1655	return min_cdclk;
1656}
1657
1658static bool intel_fbc_can_flip_nuke(struct intel_atomic_state *state,
1659				    struct intel_crtc *crtc,
1660				    struct intel_plane *plane)
1661{
1662	const struct intel_crtc_state *new_crtc_state =
1663		intel_atomic_get_new_crtc_state(state, crtc);
1664	const struct intel_plane_state *old_plane_state =
1665		intel_atomic_get_old_plane_state(state, plane);
1666	const struct intel_plane_state *new_plane_state =
1667		intel_atomic_get_new_plane_state(state, plane);
1668	const struct drm_framebuffer *old_fb = old_plane_state->hw.fb;
1669	const struct drm_framebuffer *new_fb = new_plane_state->hw.fb;
1670
1671	if (intel_crtc_needs_modeset(new_crtc_state))
1672		return false;
1673
1674	if (!intel_fbc_is_ok(old_plane_state) ||
1675	    !intel_fbc_is_ok(new_plane_state))
1676		return false;
1677
1678	if (old_fb->format->format != new_fb->format->format)
1679		return false;
1680
1681	if (old_fb->modifier != new_fb->modifier)
1682		return false;
1683
1684	if (intel_fbc_plane_stride(old_plane_state) !=
1685	    intel_fbc_plane_stride(new_plane_state))
1686		return false;
1687
1688	if (intel_fbc_cfb_stride(old_plane_state) !=
1689	    intel_fbc_cfb_stride(new_plane_state))
1690		return false;
1691
1692	if (intel_fbc_cfb_size(old_plane_state) !=
1693	    intel_fbc_cfb_size(new_plane_state))
1694		return false;
1695
1696	if (intel_fbc_override_cfb_stride(old_plane_state) !=
1697	    intel_fbc_override_cfb_stride(new_plane_state))
1698		return false;
1699
1700	return true;
1701}
1702
1703static bool __intel_fbc_pre_update(struct intel_atomic_state *state,
1704				   struct intel_crtc *crtc,
1705				   struct intel_plane *plane)
1706{
1707	struct intel_display *display = to_intel_display(state->base.dev);
1708	struct intel_fbc *fbc = plane->fbc;
1709	bool need_vblank_wait = false;
1710
1711	lockdep_assert_held(&fbc->lock);
1712
1713	fbc->flip_pending = true;
1714
1715	if (intel_fbc_can_flip_nuke(state, crtc, plane))
1716		return need_vblank_wait;
1717
1718	intel_fbc_deactivate(fbc, "update pending");
1719
1720	/*
1721	 * Display WA #1198: glk+
1722	 * Need an extra vblank wait between FBC disable and most plane
1723	 * updates. Bspec says this is only needed for plane disable, but
1724	 * that is not true. Touching most plane registers will cause the
1725	 * corruption to appear. Also SKL/derivatives do not seem to be
1726	 * affected.
1727	 *
1728	 * TODO: could optimize this a bit by sampling the frame
1729	 * counter when we disable FBC (if it was already done earlier)
1730	 * and skipping the extra vblank wait before the plane update
1731	 * if at least one frame has already passed.
1732	 */
1733	if (fbc->activated && DISPLAY_VER(display) >= 10)
1734		need_vblank_wait = true;
1735	fbc->activated = false;
1736
1737	return need_vblank_wait;
1738}
1739
1740bool intel_fbc_pre_update(struct intel_atomic_state *state,
1741			  struct intel_crtc *crtc)
1742{
1743	const struct intel_plane_state __maybe_unused *plane_state;
1744	bool need_vblank_wait = false;
1745	struct intel_plane *plane;
1746	int i;
1747
1748	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1749		struct intel_fbc *fbc = plane->fbc;
1750
1751		if (!fbc || plane->pipe != crtc->pipe)
1752			continue;
1753
1754		mutex_lock(&fbc->lock);
1755
1756		if (fbc->state.plane == plane)
1757			need_vblank_wait |= __intel_fbc_pre_update(state, crtc, plane);
1758
1759		mutex_unlock(&fbc->lock);
1760	}
1761
1762	return need_vblank_wait;
1763}
1764
1765static void __intel_fbc_disable(struct intel_fbc *fbc)
1766{
1767	struct intel_display *display = fbc->display;
1768	struct intel_plane *plane = fbc->state.plane;
1769
1770	lockdep_assert_held(&fbc->lock);
1771	drm_WARN_ON(display->drm, fbc->active);
1772
1773	drm_dbg_kms(display->drm, "Disabling FBC on [PLANE:%d:%s]\n",
1774		    plane->base.base.id, plane->base.name);
1775
1776	intel_fbc_invalidate_dirty_rect(fbc);
1777
1778	__intel_fbc_cleanup_cfb(fbc);
1779
1780	/* wa_18038517565 Enable DPFC clock gating after FBC disable */
1781	if (display->platform.dg2 || DISPLAY_VER(display) >= 14)
1782		fbc_compressor_clkgate_disable_wa(fbc, false);
1783
1784	fbc->state.plane = NULL;
1785	fbc->flip_pending = false;
1786	fbc->busy_bits = 0;
1787}
1788
1789static void __intel_fbc_post_update(struct intel_fbc *fbc)
1790{
1791	lockdep_assert_held(&fbc->lock);
1792
1793	fbc->flip_pending = false;
1794	fbc->busy_bits = 0;
1795
1796	intel_fbc_activate(fbc);
1797}
1798
1799void intel_fbc_post_update(struct intel_atomic_state *state,
1800			   struct intel_crtc *crtc)
1801{
1802	const struct intel_plane_state __maybe_unused *plane_state;
1803	struct intel_plane *plane;
1804	int i;
1805
1806	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1807		struct intel_fbc *fbc = plane->fbc;
1808
1809		if (!fbc || plane->pipe != crtc->pipe)
1810			continue;
1811
1812		mutex_lock(&fbc->lock);
1813
1814		if (fbc->state.plane == plane)
1815			__intel_fbc_post_update(fbc);
1816
1817		mutex_unlock(&fbc->lock);
1818	}
1819}
1820
1821static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1822{
1823	if (fbc->state.plane)
1824		return fbc->state.plane->frontbuffer_bit;
1825	else
1826		return 0;
1827}
1828
1829static void __intel_fbc_invalidate(struct intel_fbc *fbc,
1830				   unsigned int frontbuffer_bits,
1831				   enum fb_op_origin origin)
1832{
1833	if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1834		return;
1835
1836	mutex_lock(&fbc->lock);
1837
1838	frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1839	if (!frontbuffer_bits)
1840		goto out;
1841
1842	fbc->busy_bits |= frontbuffer_bits;
1843	intel_fbc_deactivate(fbc, "frontbuffer write");
1844
1845out:
1846	mutex_unlock(&fbc->lock);
1847}
1848
1849void intel_fbc_invalidate(struct intel_display *display,
1850			  unsigned int frontbuffer_bits,
1851			  enum fb_op_origin origin)
1852{
1853	struct intel_fbc *fbc;
1854	enum intel_fbc_id fbc_id;
1855
1856	for_each_intel_fbc(display, fbc, fbc_id)
1857		__intel_fbc_invalidate(fbc, frontbuffer_bits, origin);
1858
1859}
1860
1861static void __intel_fbc_flush(struct intel_fbc *fbc,
1862			      unsigned int frontbuffer_bits,
1863			      enum fb_op_origin origin)
1864{
1865	mutex_lock(&fbc->lock);
1866
1867	frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1868	if (!frontbuffer_bits)
1869		goto out;
1870
1871	fbc->busy_bits &= ~frontbuffer_bits;
1872
1873	if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1874		goto out;
1875
1876	if (fbc->busy_bits || fbc->flip_pending)
1877		goto out;
1878
1879	if (fbc->active)
1880		intel_fbc_nuke(fbc);
1881	else
1882		intel_fbc_activate(fbc);
1883
1884out:
1885	mutex_unlock(&fbc->lock);
1886}
1887
1888void intel_fbc_flush(struct intel_display *display,
1889		     unsigned int frontbuffer_bits,
1890		     enum fb_op_origin origin)
1891{
1892	struct intel_fbc *fbc;
1893	enum intel_fbc_id fbc_id;
1894
1895	for_each_intel_fbc(display, fbc, fbc_id)
1896		__intel_fbc_flush(fbc, frontbuffer_bits, origin);
1897}
1898
1899int intel_fbc_atomic_check(struct intel_atomic_state *state)
1900{
1901	struct intel_plane_state __maybe_unused *plane_state;
1902	struct intel_plane *plane;
1903	int i;
1904
1905	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1906		int ret;
1907
1908		ret = intel_fbc_check_plane(state, plane);
1909		if (ret)
1910			return ret;
1911	}
1912
1913	return 0;
1914}
1915
1916static void __intel_fbc_enable(struct intel_atomic_state *state,
1917			       struct intel_crtc *crtc,
1918			       struct intel_plane *plane)
1919{
1920	struct intel_display *display = to_intel_display(state->base.dev);
1921	const struct intel_plane_state *plane_state =
1922		intel_atomic_get_new_plane_state(state, plane);
1923	struct intel_fbc *fbc = plane->fbc;
1924
1925	lockdep_assert_held(&fbc->lock);
1926
1927	if (fbc->state.plane) {
1928		if (fbc->state.plane != plane)
1929			return;
1930
1931		if (intel_fbc_is_ok(plane_state)) {
1932			intel_fbc_update_state(state, crtc, plane);
1933			return;
1934		}
1935
1936		__intel_fbc_disable(fbc);
1937	}
1938
1939	drm_WARN_ON(display->drm, fbc->active);
1940
1941	fbc->no_fbc_reason = plane_state->no_fbc_reason;
1942	if (fbc->no_fbc_reason)
1943		return;
1944
1945	if (!intel_fbc_is_fence_ok(plane_state)) {
1946		fbc->no_fbc_reason = "framebuffer not fenced";
1947		return;
1948	}
1949
1950	if (fbc->underrun_detected) {
1951		fbc->no_fbc_reason = "FIFO underrun";
1952		return;
1953	}
1954
1955	if (intel_fbc_alloc_cfb(fbc, intel_fbc_cfb_size(plane_state),
1956				intel_fbc_min_limit(plane_state))) {
1957		fbc->no_fbc_reason = "not enough stolen memory";
1958		return;
1959	}
1960
1961	drm_dbg_kms(display->drm, "Enabling FBC on [PLANE:%d:%s]\n",
1962		    plane->base.base.id, plane->base.name);
1963	fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1964
1965	intel_fbc_update_state(state, crtc, plane);
1966
1967	if (HAS_FBC_DIRTY_RECT(display))
1968		intel_fbc_hw_intialize_dirty_rect(fbc, plane_state);
1969
1970	intel_fbc_program_workarounds(fbc);
1971	intel_fbc_program_cfb(fbc);
1972}
1973
1974/**
1975 * intel_fbc_disable - disable FBC if it's associated with crtc
1976 * @crtc: the CRTC
1977 *
1978 * This function disables FBC if it's associated with the provided CRTC.
1979 */
1980void intel_fbc_disable(struct intel_crtc *crtc)
1981{
1982	struct intel_display *display = to_intel_display(crtc->base.dev);
1983	struct intel_plane *plane;
1984
1985	for_each_intel_plane(display->drm, plane) {
1986		struct intel_fbc *fbc = plane->fbc;
1987
1988		if (!fbc || plane->pipe != crtc->pipe)
1989			continue;
1990
1991		mutex_lock(&fbc->lock);
1992		if (fbc->state.plane == plane)
1993			__intel_fbc_disable(fbc);
1994		mutex_unlock(&fbc->lock);
1995	}
1996}
1997
1998void intel_fbc_update(struct intel_atomic_state *state,
1999		      struct intel_crtc *crtc)
2000{
2001	const struct intel_crtc_state *crtc_state =
2002		intel_atomic_get_new_crtc_state(state, crtc);
2003	const struct intel_plane_state *plane_state;
2004	struct intel_plane *plane;
2005	int i;
2006
2007	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2008		struct intel_fbc *fbc = plane->fbc;
2009
2010		if (!fbc || plane->pipe != crtc->pipe)
2011			continue;
2012
2013		mutex_lock(&fbc->lock);
2014
2015		if (intel_crtc_needs_fastset(crtc_state) &&
2016		    plane_state->no_fbc_reason) {
2017			if (fbc->state.plane == plane)
2018				__intel_fbc_disable(fbc);
2019		} else {
2020			__intel_fbc_enable(state, crtc, plane);
2021		}
2022
2023		mutex_unlock(&fbc->lock);
2024	}
2025}
2026
2027static void intel_fbc_underrun_work_fn(struct work_struct *work)
2028{
2029	struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);
2030	struct intel_display *display = fbc->display;
2031
2032	mutex_lock(&fbc->lock);
2033
2034	/* Maybe we were scheduled twice. */
2035	if (fbc->underrun_detected || !fbc->state.plane)
2036		goto out;
2037
2038	drm_dbg_kms(display->drm, "Disabling FBC due to FIFO underrun.\n");
2039	fbc->underrun_detected = true;
2040
2041	intel_fbc_deactivate(fbc, "FIFO underrun");
2042	if (!fbc->flip_pending)
2043		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(display, fbc->state.plane->pipe));
2044	__intel_fbc_disable(fbc);
2045out:
2046	mutex_unlock(&fbc->lock);
2047}
2048
2049static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)
2050{
2051	struct intel_display *display = fbc->display;
2052
2053	cancel_work_sync(&fbc->underrun_work);
2054
2055	mutex_lock(&fbc->lock);
2056
2057	if (fbc->underrun_detected) {
2058		drm_dbg_kms(display->drm,
2059			    "Re-allowing FBC after fifo underrun\n");
2060		fbc->no_fbc_reason = "FIFO underrun cleared";
2061	}
2062
2063	fbc->underrun_detected = false;
2064	mutex_unlock(&fbc->lock);
2065}
2066
2067/*
2068 * intel_fbc_reset_underrun - reset FBC fifo underrun status.
2069 * @display: display
2070 *
2071 * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
2072 * want to re-enable FBC after an underrun to increase test coverage.
2073 */
2074void intel_fbc_reset_underrun(struct intel_display *display)
2075{
2076	struct intel_fbc *fbc;
2077	enum intel_fbc_id fbc_id;
2078
2079	for_each_intel_fbc(display, fbc, fbc_id)
2080		__intel_fbc_reset_underrun(fbc);
2081}
2082
2083static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)
2084{
2085	struct intel_display *display = fbc->display;
2086
2087	/*
2088	 * There's no guarantee that underrun_detected won't be set to true
2089	 * right after this check and before the work is scheduled, but that's
2090	 * not a problem since we'll check it again under the work function
2091	 * while FBC is locked. This check here is just to prevent us from
2092	 * unnecessarily scheduling the work, and it relies on the fact that we
2093	 * never switch underrun_detect back to false after it's true.
2094	 */
2095	if (READ_ONCE(fbc->underrun_detected))
2096		return;
2097
2098	queue_work(display->wq.unordered, &fbc->underrun_work);
2099}
2100
2101/**
2102 * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
2103 * @display: display
2104 *
2105 * Without FBC, most underruns are harmless and don't really cause too many
2106 * problems, except for an annoying message on dmesg. With FBC, underruns can
2107 * become black screens or even worse, especially when paired with bad
2108 * watermarks. So in order for us to be on the safe side, completely disable FBC
2109 * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
2110 * already suggests that watermarks may be bad, so try to be as safe as
2111 * possible.
2112 *
2113 * This function is called from the IRQ handler.
2114 */
2115void intel_fbc_handle_fifo_underrun_irq(struct intel_display *display)
2116{
2117	struct intel_fbc *fbc;
2118	enum intel_fbc_id fbc_id;
2119
2120	for_each_intel_fbc(display, fbc, fbc_id)
2121		__intel_fbc_handle_fifo_underrun_irq(fbc);
2122}
2123
2124/*
2125 * The DDX driver changes its behavior depending on the value it reads from
2126 * i915.enable_fbc, so sanitize it by translating the default value into either
2127 * 0 or 1 in order to allow it to know what's going on.
2128 *
2129 * Notice that this is done at driver initialization and we still allow user
2130 * space to change the value during runtime without sanitizing it again. IGT
2131 * relies on being able to change i915.enable_fbc at runtime.
2132 */
2133static int intel_sanitize_fbc_option(struct intel_display *display)
2134{
2135	if (display->params.enable_fbc >= 0)
2136		return !!display->params.enable_fbc;
2137
2138	if (!HAS_FBC(display))
2139		return 0;
2140
2141	if (display->platform.broadwell || DISPLAY_VER(display) >= 9)
2142		return 1;
2143
2144	return 0;
2145}
2146
2147void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)
2148{
2149	plane->fbc = fbc;
2150}
2151
2152static struct intel_fbc *intel_fbc_create(struct intel_display *display,
2153					  enum intel_fbc_id fbc_id)
2154{
2155	struct intel_fbc *fbc;
2156
2157	fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);
2158	if (!fbc)
2159		return NULL;
2160
2161	fbc->compressed_fb = i915_gem_stolen_node_alloc(display->drm);
2162	if (!fbc->compressed_fb)
2163		goto err;
2164	fbc->compressed_llb = i915_gem_stolen_node_alloc(display->drm);
2165	if (!fbc->compressed_llb)
2166		goto err;
2167
2168	fbc->id = fbc_id;
2169	fbc->display = display;
2170	INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
2171	mutex_init(&fbc->lock);
2172
2173	if (DISPLAY_VER(display) >= 7)
2174		fbc->funcs = &ivb_fbc_funcs;
2175	else if (DISPLAY_VER(display) == 6)
2176		fbc->funcs = &snb_fbc_funcs;
2177	else if (DISPLAY_VER(display) == 5)
2178		fbc->funcs = &ilk_fbc_funcs;
2179	else if (display->platform.g4x)
2180		fbc->funcs = &g4x_fbc_funcs;
2181	else if (DISPLAY_VER(display) == 4)
2182		fbc->funcs = &i965_fbc_funcs;
2183	else
2184		fbc->funcs = &i8xx_fbc_funcs;
2185
2186	return fbc;
2187
2188err:
2189	i915_gem_stolen_node_free(fbc->compressed_llb);
2190	i915_gem_stolen_node_free(fbc->compressed_fb);
2191	kfree(fbc);
2192
2193	return NULL;
2194}
2195
2196/**
2197 * intel_fbc_init - Initialize FBC
2198 * @display: display
2199 *
2200 * This function might be called during PM init process.
2201 */
2202void intel_fbc_init(struct intel_display *display)
2203{
2204	enum intel_fbc_id fbc_id;
2205
2206	display->params.enable_fbc = intel_sanitize_fbc_option(display);
2207	drm_dbg_kms(display->drm, "Sanitized enable_fbc value: %d\n",
2208		    display->params.enable_fbc);
2209
2210	for_each_fbc_id(display, fbc_id)
2211		display->fbc[fbc_id] = intel_fbc_create(display, fbc_id);
2212}
2213
2214/**
2215 * intel_fbc_sanitize - Sanitize FBC
2216 * @display: display
2217 *
2218 * Make sure FBC is initially disabled since we have no
2219 * idea eg. into which parts of stolen it might be scribbling
2220 * into.
2221 */
2222void intel_fbc_sanitize(struct intel_display *display)
2223{
2224	struct intel_fbc *fbc;
2225	enum intel_fbc_id fbc_id;
2226
2227	for_each_intel_fbc(display, fbc, fbc_id) {
2228		if (intel_fbc_hw_is_active(fbc))
2229			intel_fbc_hw_deactivate(fbc);
2230	}
2231}
2232
2233static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
2234{
2235	struct intel_fbc *fbc = m->private;
2236	struct intel_display *display = fbc->display;
2237	struct intel_plane *plane;
2238	struct ref_tracker *wakeref;
2239
2240	drm_modeset_lock_all(display->drm);
2241
2242	wakeref = intel_display_rpm_get(display);
2243	mutex_lock(&fbc->lock);
2244
2245	if (fbc->active) {
2246		seq_puts(m, "FBC enabled\n");
2247		seq_printf(m, "Compressing: %s\n",
2248			   str_yes_no(intel_fbc_is_compressing(fbc)));
2249	} else {
2250		seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
2251	}
2252
2253	for_each_intel_plane(display->drm, plane) {
2254		const struct intel_plane_state *plane_state =
2255			to_intel_plane_state(plane->base.state);
2256
2257		if (plane->fbc != fbc)
2258			continue;
2259
2260		seq_printf(m, "%c [PLANE:%d:%s]: %s\n",
2261			   fbc->state.plane == plane ? '*' : ' ',
2262			   plane->base.base.id, plane->base.name,
2263			   plane_state->no_fbc_reason ?: "FBC possible");
2264	}
2265
2266	mutex_unlock(&fbc->lock);
2267	intel_display_rpm_put(display, wakeref);
2268
2269	drm_modeset_unlock_all(display->drm);
2270
2271	return 0;
2272}
2273
2274DEFINE_SHOW_ATTRIBUTE(intel_fbc_debugfs_status);
2275
2276static int intel_fbc_debugfs_false_color_get(void *data, u64 *val)
2277{
2278	struct intel_fbc *fbc = data;
2279
2280	*val = fbc->false_color;
2281
2282	return 0;
2283}
2284
2285static int intel_fbc_debugfs_false_color_set(void *data, u64 val)
2286{
2287	struct intel_fbc *fbc = data;
2288
2289	mutex_lock(&fbc->lock);
2290
2291	fbc->false_color = val;
2292
2293	if (fbc->active)
2294		fbc->funcs->set_false_color(fbc, fbc->false_color);
2295
2296	mutex_unlock(&fbc->lock);
2297
2298	return 0;
2299}
2300
2301DEFINE_DEBUGFS_ATTRIBUTE(intel_fbc_debugfs_false_color_fops,
2302			 intel_fbc_debugfs_false_color_get,
2303			 intel_fbc_debugfs_false_color_set,
2304			 "%llu\n");
2305
2306static void intel_fbc_debugfs_add(struct intel_fbc *fbc,
2307				  struct dentry *parent)
2308{
2309	debugfs_create_file("i915_fbc_status", 0444, parent,
2310			    fbc, &intel_fbc_debugfs_status_fops);
2311
2312	if (fbc->funcs->set_false_color)
2313		debugfs_create_file_unsafe("i915_fbc_false_color", 0644, parent,
2314					   fbc, &intel_fbc_debugfs_false_color_fops);
2315}
2316
2317void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc)
2318{
2319	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
2320
2321	if (plane->fbc)
2322		intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry);
2323}
2324
2325/* FIXME: remove this once igt is on board with per-crtc stuff */
2326void intel_fbc_debugfs_register(struct intel_display *display)
2327{
2328	struct intel_fbc *fbc;
2329
2330	fbc = display->fbc[INTEL_FBC_A];
2331	if (fbc)
2332		intel_fbc_debugfs_add(fbc, display->drm->debugfs_root);
2333}