intel_dmc.c

   1/*
   2 * Copyright © 2014 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/debugfs.h>
  26#include <linux/firmware.h>
  27#include <drm/drm_vblank.h>
  28
  29#include <drm/drm_file.h>
  30#include <drm/drm_print.h>
  31
  32#include "i915_reg.h"
  33#include "intel_crtc.h"
  34#include "intel_de.h"
  35#include "intel_display_power_well.h"
  36#include "intel_display_regs.h"
  37#include "intel_display_rpm.h"
  38#include "intel_display_types.h"
  39#include "intel_display_utils.h"
  40#include "intel_dmc.h"
  41#include "intel_dmc_regs.h"
  42#include "intel_flipq.h"
  43#include "intel_step.h"
  44
  45/**
  46 * DOC: DMC Firmware Support
  47 *
  48 * From gen9 onwards we have newly added DMC (Display microcontroller) in display
  49 * engine to save and restore the state of display engine when it enter into
  50 * low-power state and comes back to normal.
  51 */
  52
  53#define INTEL_DMC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"
  54
  55enum intel_dmc_id {
  56	DMC_FW_MAIN = 0,
  57	DMC_FW_PIPEA,
  58	DMC_FW_PIPEB,
  59	DMC_FW_PIPEC,
  60	DMC_FW_PIPED,
  61	DMC_FW_MAX
  62};
  63
  64struct intel_dmc {
  65	struct intel_display *display;
  66	struct work_struct work;
  67	const char *fw_path;
  68	u32 max_fw_size; /* bytes */
  69	u32 version;
  70	struct {
  71		u32 dc5_start;
  72		u32 count;
  73	} dc6_allowed;
  74	struct dmc_fw_info {
  75		u32 mmio_count;
  76		i915_reg_t mmioaddr[20];
  77		u32 mmiodata[20];
  78		u32 dmc_offset;
  79		u32 start_mmioaddr;
  80		u32 dmc_fw_size; /*dwords */
  81		u32 *payload;
  82		bool present;
  83	} dmc_info[DMC_FW_MAX];
  84};
  85
  86/* Note: This may be NULL. */
  87static struct intel_dmc *display_to_dmc(struct intel_display *display)
  88{
  89	return display->dmc.dmc;
  90}
  91
  92static const char *dmc_firmware_param(struct intel_display *display)
  93{
  94	const char *p = display->params.dmc_firmware_path;
  95
  96	return p && *p ? p : NULL;
  97}
  98
  99static bool dmc_firmware_param_disabled(struct intel_display *display)
 100{
 101	const char *p = dmc_firmware_param(display);
 102
 103	/* Magic path to indicate disabled */
 104	return p && !strcmp(p, "/dev/null");
 105}
 106
 107#define DMC_VERSION(major, minor)	((major) << 16 | (minor))
 108#define DMC_VERSION_MAJOR(version)	((version) >> 16)
 109#define DMC_VERSION_MINOR(version)	((version) & 0xffff)
 110
 111#define DMC_PATH(platform) \
 112	"i915/" __stringify(platform) "_dmc.bin"
 113
 114/*
 115 * New DMC additions should not use this. This is used solely to remain
 116 * compatible with systems that have not yet updated DMC blobs to use
 117 * unversioned file names.
 118 */
 119#define DMC_LEGACY_PATH(platform, major, minor) \
 120	"i915/"					\
 121	__stringify(platform) "_dmc_ver"	\
 122	__stringify(major) "_"			\
 123	__stringify(minor) ".bin"
 124
 125#define XE2LPD_DMC_MAX_FW_SIZE		0x8000
 126#define XELPDP_DMC_MAX_FW_SIZE		0x7000
 127#define DISPLAY_VER13_DMC_MAX_FW_SIZE	0x20000
 128#define DISPLAY_VER12_DMC_MAX_FW_SIZE	ICL_DMC_MAX_FW_SIZE
 129
 130#define XE3P_LPD_DMC_PATH		DMC_PATH(xe3p_lpd)
 131MODULE_FIRMWARE(XE3P_LPD_DMC_PATH);
 132
 133#define XE3LPD_3002_DMC_PATH		DMC_PATH(xe3lpd_3002)
 134MODULE_FIRMWARE(XE3LPD_3002_DMC_PATH);
 135
 136#define XE3LPD_DMC_PATH			DMC_PATH(xe3lpd)
 137MODULE_FIRMWARE(XE3LPD_DMC_PATH);
 138
 139#define XE2LPD_DMC_PATH			DMC_PATH(xe2lpd)
 140MODULE_FIRMWARE(XE2LPD_DMC_PATH);
 141
 142#define BMG_DMC_PATH			DMC_PATH(bmg)
 143MODULE_FIRMWARE(BMG_DMC_PATH);
 144
 145#define MTL_DMC_PATH			DMC_PATH(mtl)
 146MODULE_FIRMWARE(MTL_DMC_PATH);
 147
 148#define DG2_DMC_PATH			DMC_LEGACY_PATH(dg2, 2, 08)
 149MODULE_FIRMWARE(DG2_DMC_PATH);
 150
 151#define ADLP_DMC_PATH			DMC_PATH(adlp)
 152#define ADLP_DMC_FALLBACK_PATH		DMC_LEGACY_PATH(adlp, 2, 16)
 153MODULE_FIRMWARE(ADLP_DMC_PATH);
 154MODULE_FIRMWARE(ADLP_DMC_FALLBACK_PATH);
 155
 156#define ADLS_DMC_PATH			DMC_LEGACY_PATH(adls, 2, 01)
 157MODULE_FIRMWARE(ADLS_DMC_PATH);
 158
 159#define DG1_DMC_PATH			DMC_LEGACY_PATH(dg1, 2, 02)
 160MODULE_FIRMWARE(DG1_DMC_PATH);
 161
 162#define RKL_DMC_PATH			DMC_LEGACY_PATH(rkl, 2, 03)
 163MODULE_FIRMWARE(RKL_DMC_PATH);
 164
 165#define TGL_DMC_PATH			DMC_LEGACY_PATH(tgl, 2, 12)
 166MODULE_FIRMWARE(TGL_DMC_PATH);
 167
 168#define ICL_DMC_PATH			DMC_LEGACY_PATH(icl, 1, 09)
 169#define ICL_DMC_MAX_FW_SIZE		0x6000
 170MODULE_FIRMWARE(ICL_DMC_PATH);
 171
 172#define GLK_DMC_PATH			DMC_LEGACY_PATH(glk, 1, 04)
 173#define GLK_DMC_MAX_FW_SIZE		0x4000
 174MODULE_FIRMWARE(GLK_DMC_PATH);
 175
 176#define KBL_DMC_PATH			DMC_LEGACY_PATH(kbl, 1, 04)
 177#define KBL_DMC_MAX_FW_SIZE		BXT_DMC_MAX_FW_SIZE
 178MODULE_FIRMWARE(KBL_DMC_PATH);
 179
 180#define SKL_DMC_PATH			DMC_LEGACY_PATH(skl, 1, 27)
 181#define SKL_DMC_MAX_FW_SIZE		BXT_DMC_MAX_FW_SIZE
 182MODULE_FIRMWARE(SKL_DMC_PATH);
 183
 184#define BXT_DMC_PATH			DMC_LEGACY_PATH(bxt, 1, 07)
 185#define BXT_DMC_MAX_FW_SIZE		0x3000
 186MODULE_FIRMWARE(BXT_DMC_PATH);
 187
 188static const char *dmc_firmware_default(struct intel_display *display, u32 *size)
 189{
 190	const char *fw_path = NULL;
 191	u32 max_fw_size = 0;
 192
 193	if (DISPLAY_VERx100(display) == 3500) {
 194		fw_path = XE3P_LPD_DMC_PATH;
 195		max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
 196	} else if (DISPLAY_VERx100(display) == 3002) {
 197		fw_path = XE3LPD_3002_DMC_PATH;
 198		max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
 199	} else if (DISPLAY_VERx100(display) == 3000) {
 200		fw_path = XE3LPD_DMC_PATH;
 201		max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
 202	} else if (DISPLAY_VERx100(display) == 2000) {
 203		fw_path = XE2LPD_DMC_PATH;
 204		max_fw_size = XE2LPD_DMC_MAX_FW_SIZE;
 205	} else if (DISPLAY_VERx100(display) == 1401) {
 206		fw_path = BMG_DMC_PATH;
 207		max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
 208	} else if (DISPLAY_VERx100(display) == 1400) {
 209		fw_path = MTL_DMC_PATH;
 210		max_fw_size = XELPDP_DMC_MAX_FW_SIZE;
 211	} else if (display->platform.dg2) {
 212		fw_path = DG2_DMC_PATH;
 213		max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
 214	} else if (display->platform.alderlake_p) {
 215		fw_path = ADLP_DMC_PATH;
 216		max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE;
 217	} else if (display->platform.alderlake_s) {
 218		fw_path = ADLS_DMC_PATH;
 219		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
 220	} else if (display->platform.dg1) {
 221		fw_path = DG1_DMC_PATH;
 222		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
 223	} else if (display->platform.rocketlake) {
 224		fw_path = RKL_DMC_PATH;
 225		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
 226	} else if (display->platform.tigerlake) {
 227		fw_path = TGL_DMC_PATH;
 228		max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE;
 229	} else if (DISPLAY_VER(display) == 11) {
 230		fw_path = ICL_DMC_PATH;
 231		max_fw_size = ICL_DMC_MAX_FW_SIZE;
 232	} else if (display->platform.geminilake) {
 233		fw_path = GLK_DMC_PATH;
 234		max_fw_size = GLK_DMC_MAX_FW_SIZE;
 235	} else if (display->platform.kabylake ||
 236		   display->platform.coffeelake ||
 237		   display->platform.cometlake) {
 238		fw_path = KBL_DMC_PATH;
 239		max_fw_size = KBL_DMC_MAX_FW_SIZE;
 240	} else if (display->platform.skylake) {
 241		fw_path = SKL_DMC_PATH;
 242		max_fw_size = SKL_DMC_MAX_FW_SIZE;
 243	} else if (display->platform.broxton) {
 244		fw_path = BXT_DMC_PATH;
 245		max_fw_size = BXT_DMC_MAX_FW_SIZE;
 246	}
 247
 248	*size = max_fw_size;
 249
 250	return fw_path;
 251}
 252
 253#define DMC_DEFAULT_FW_OFFSET		0xFFFFFFFF
 254#define PACKAGE_MAX_FW_INFO_ENTRIES	20
 255#define PACKAGE_V2_MAX_FW_INFO_ENTRIES	32
 256#define DMC_V1_MAX_MMIO_COUNT		8
 257#define DMC_V3_MAX_MMIO_COUNT		20
 258#define DMC_V1_MMIO_START_RANGE		0x80000
 259
 260#define PIPE_TO_DMC_ID(pipe)		 (DMC_FW_PIPEA + ((pipe) - PIPE_A))
 261
 262struct intel_css_header {
 263	/* 0x09 for DMC */
 264	u32 module_type;
 265
 266	/* Includes the DMC specific header in dwords */
 267	u32 header_len;
 268
 269	/* always value would be 0x10000 */
 270	u32 header_ver;
 271
 272	/* Not used */
 273	u32 module_id;
 274
 275	/* Not used */
 276	u32 module_vendor;
 277
 278	/* in YYYYMMDD format */
 279	u32 date;
 280
 281	/* Size in dwords (CSS_Headerlen + PackageHeaderLen + dmc FWsLen)/4 */
 282	u32 size;
 283
 284	/* Not used */
 285	u32 key_size;
 286
 287	/* Not used */
 288	u32 modulus_size;
 289
 290	/* Not used */
 291	u32 exponent_size;
 292
 293	/* Not used */
 294	u32 reserved1[12];
 295
 296	/* Major Minor */
 297	u32 version;
 298
 299	/* Not used */
 300	u32 reserved2[8];
 301
 302	/* Not used */
 303	u32 kernel_header_info;
 304} __packed;
 305
 306struct intel_fw_info {
 307	u8 reserved1;
 308
 309	/* reserved on package_header version 1, must be 0 on version 2 */
 310	u8 dmc_id;
 311
 312	/* Stepping (A, B, C, ..., *). * is a wildcard */
 313	char stepping;
 314
 315	/* Sub-stepping (0, 1, ..., *). * is a wildcard */
 316	char substepping;
 317
 318	u32 offset;
 319	u32 reserved2;
 320} __packed;
 321
 322struct intel_package_header {
 323	/* DMC container header length in dwords */
 324	u8 header_len;
 325
 326	/* 0x01, 0x02 */
 327	u8 header_ver;
 328
 329	u8 reserved[10];
 330
 331	/* Number of valid entries in the FWInfo array below */
 332	u32 num_entries;
 333} __packed;
 334
 335struct intel_dmc_header_base {
 336	/* always value would be 0x40403E3E */
 337	u32 signature;
 338
 339	/* DMC binary header length */
 340	u8 header_len;
 341
 342	/* 0x01 */
 343	u8 header_ver;
 344
 345	/* Reserved */
 346	u16 dmcc_ver;
 347
 348	/* Major, Minor */
 349	u32 project;
 350
 351	/* Firmware program size (excluding header) in dwords */
 352	u32 fw_size;
 353
 354	/* Major Minor version */
 355	u32 fw_version;
 356} __packed;
 357
 358struct intel_dmc_header_v1 {
 359	struct intel_dmc_header_base base;
 360
 361	/* Number of valid MMIO cycles present. */
 362	u32 mmio_count;
 363
 364	/* MMIO address */
 365	u32 mmioaddr[DMC_V1_MAX_MMIO_COUNT];
 366
 367	/* MMIO data */
 368	u32 mmiodata[DMC_V1_MAX_MMIO_COUNT];
 369
 370	/* FW filename  */
 371	char dfile[32];
 372
 373	u32 reserved1[2];
 374} __packed;
 375
 376struct intel_dmc_header_v3 {
 377	struct intel_dmc_header_base base;
 378
 379	/* DMC RAM start MMIO address */
 380	u32 start_mmioaddr;
 381
 382	u32 reserved[9];
 383
 384	/* FW filename */
 385	char dfile[32];
 386
 387	/* Number of valid MMIO cycles present. */
 388	u32 mmio_count;
 389
 390	/* MMIO address */
 391	u32 mmioaddr[DMC_V3_MAX_MMIO_COUNT];
 392
 393	/* MMIO data */
 394	u32 mmiodata[DMC_V3_MAX_MMIO_COUNT];
 395} __packed;
 396
 397struct stepping_info {
 398	char stepping;
 399	char substepping;
 400};
 401
 402#define for_each_dmc_id(__dmc_id) \
 403	for ((__dmc_id) = DMC_FW_MAIN; (__dmc_id) < DMC_FW_MAX; (__dmc_id)++)
 404
 405static bool is_valid_dmc_id(enum intel_dmc_id dmc_id)
 406{
 407	return dmc_id >= DMC_FW_MAIN && dmc_id < DMC_FW_MAX;
 408}
 409
 410static bool has_dmc_id_fw(struct intel_display *display, enum intel_dmc_id dmc_id)
 411{
 412	struct intel_dmc *dmc = display_to_dmc(display);
 413
 414	return dmc && dmc->dmc_info[dmc_id].payload;
 415}
 416
 417bool intel_dmc_has_payload(struct intel_display *display)
 418{
 419	return has_dmc_id_fw(display, DMC_FW_MAIN);
 420}
 421
 422static const struct stepping_info *
 423intel_get_stepping_info(struct intel_display *display,
 424			struct stepping_info *si)
 425{
 426	const char *step_name = intel_step_name(INTEL_DISPLAY_STEP(display));
 427
 428	si->stepping = step_name[0];
 429	si->substepping = step_name[1];
 430	return si;
 431}
 432
 433static void gen9_set_dc_state_debugmask(struct intel_display *display)
 434{
 435	/* The below bit doesn't need to be cleared ever afterwards */
 436	intel_de_rmw(display, DC_STATE_DEBUG, 0,
 437		     DC_STATE_DEBUG_MASK_CORES | DC_STATE_DEBUG_MASK_MEMORY_UP);
 438	intel_de_posting_read(display, DC_STATE_DEBUG);
 439}
 440
 441static void disable_event_handler(struct intel_display *display,
 442				  i915_reg_t ctl_reg, i915_reg_t htp_reg)
 443{
 444	intel_de_write(display, ctl_reg,
 445		       REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
 446				      DMC_EVT_CTL_TYPE_EDGE_0_1) |
 447		       REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
 448				      DMC_EVENT_FALSE));
 449	intel_de_write(display, htp_reg, 0);
 450}
 451
 452static void disable_all_event_handlers(struct intel_display *display,
 453				       enum intel_dmc_id dmc_id)
 454{
 455	int handler;
 456
 457	/* TODO: disable the event handlers on pre-GEN12 platforms as well */
 458	if (DISPLAY_VER(display) < 12)
 459		return;
 460
 461	if (!has_dmc_id_fw(display, dmc_id))
 462		return;
 463
 464	for (handler = 0; handler < DMC_EVENT_HANDLER_COUNT_GEN12; handler++)
 465		disable_event_handler(display,
 466				      DMC_EVT_CTL(display, dmc_id, handler),
 467				      DMC_EVT_HTP(display, dmc_id, handler));
 468}
 469
 470static void adlp_pipedmc_clock_gating_wa(struct intel_display *display, bool enable)
 471{
 472	enum pipe pipe;
 473
 474	/*
 475	 * Wa_16015201720:adl-p,dg2
 476	 * The WA requires clock gating to be disabled all the time
 477	 * for pipe A and B.
 478	 * For pipe C and D clock gating needs to be disabled only
 479	 * during initializing the firmware.
 480	 */
 481	if (enable)
 482		for (pipe = PIPE_A; pipe <= PIPE_D; pipe++)
 483			intel_de_rmw(display, CLKGATE_DIS_PSL_EXT(pipe),
 484				     0, PIPEDMC_GATING_DIS);
 485	else
 486		for (pipe = PIPE_C; pipe <= PIPE_D; pipe++)
 487			intel_de_rmw(display, CLKGATE_DIS_PSL_EXT(pipe),
 488				     PIPEDMC_GATING_DIS, 0);
 489}
 490
 491static void mtl_pipedmc_clock_gating_wa(struct intel_display *display)
 492{
 493	/*
 494	 * Wa_16015201720
 495	 * The WA requires clock gating to be disabled all the time
 496	 * for pipe A and B.
 497	 */
 498	intel_de_rmw(display, GEN9_CLKGATE_DIS_0, 0,
 499		     MTL_PIPEDMC_GATING_DIS(PIPE_A) |
 500		     MTL_PIPEDMC_GATING_DIS(PIPE_B));
 501}
 502
 503static void pipedmc_clock_gating_wa(struct intel_display *display, bool enable)
 504{
 505	if (display->platform.meteorlake && enable)
 506		mtl_pipedmc_clock_gating_wa(display);
 507	else if (DISPLAY_VER(display) == 13)
 508		adlp_pipedmc_clock_gating_wa(display, enable);
 509}
 510
 511static u32 pipedmc_interrupt_mask(struct intel_display *display)
 512{
 513	/*
 514	 * FIXME PIPEDMC_ERROR not enabled for now due to LNL pipe B
 515	 * triggering it during the first DC state transition. Figure
 516	 * out what is going on...
 517	 */
 518	return PIPEDMC_FLIPQ_PROG_DONE |
 519		PIPEDMC_GTT_FAULT |
 520		PIPEDMC_ATS_FAULT;
 521}
 522
 523static u32 dmc_evt_ctl_disable(u32 dmc_evt_ctl)
 524{
 525	/*
 526	 * DMC_EVT_CTL_ENABLE cannot be cleared once set. Always
 527	 * configure it based on the original event definition to
 528	 * avoid mismatches in assert_dmc_loaded().
 529	 */
 530	return (dmc_evt_ctl & DMC_EVT_CTL_ENABLE) |
 531		REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
 532			       DMC_EVT_CTL_TYPE_EDGE_0_1) |
 533		REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
 534			       DMC_EVENT_FALSE);
 535}
 536
 537static bool is_dmc_evt_ctl_reg(struct intel_display *display,
 538			       enum intel_dmc_id dmc_id, i915_reg_t reg)
 539{
 540	u32 offset = i915_mmio_reg_offset(reg);
 541	u32 start = i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, 0));
 542	u32 end = i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
 543
 544	return offset >= start && offset < end;
 545}
 546
 547static bool is_dmc_evt_htp_reg(struct intel_display *display,
 548			       enum intel_dmc_id dmc_id, i915_reg_t reg)
 549{
 550	u32 offset = i915_mmio_reg_offset(reg);
 551	u32 start = i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, 0));
 552	u32 end = i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
 553
 554	return offset >= start && offset < end;
 555}
 556
 557static bool is_event_handler(struct intel_display *display,
 558			     enum intel_dmc_id dmc_id,
 559			     unsigned int event_id,
 560			     i915_reg_t reg, u32 data)
 561{
 562	return is_dmc_evt_ctl_reg(display, dmc_id, reg) &&
 563		REG_FIELD_GET(DMC_EVT_CTL_EVENT_ID_MASK, data) == event_id;
 564}
 565
 566static bool fixup_dmc_evt(struct intel_display *display,
 567			  enum intel_dmc_id dmc_id,
 568			  i915_reg_t reg_ctl, u32 *data_ctl,
 569			  i915_reg_t reg_htp, u32 *data_htp)
 570{
 571	if (!is_dmc_evt_ctl_reg(display, dmc_id, reg_ctl))
 572		return false;
 573
 574	if (!is_dmc_evt_htp_reg(display, dmc_id, reg_htp))
 575		return false;
 576
 577	/* make sure reg_ctl and reg_htp are for the same event */
 578	if (i915_mmio_reg_offset(reg_ctl) - i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, 0)) !=
 579	    i915_mmio_reg_offset(reg_htp) - i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, 0)))
 580		return false;
 581
 582	/*
 583	 * On ADL-S the HRR event handler is not restored after DC6.
 584	 * Clear it to zero from the beginning to avoid mismatches later.
 585	 */
 586	if (display->platform.alderlake_s && dmc_id == DMC_FW_MAIN &&
 587	    is_event_handler(display, dmc_id, MAINDMC_EVENT_VBLANK_A, reg_ctl, *data_ctl)) {
 588		*data_ctl = 0;
 589		*data_htp = 0;
 590		return true;
 591	}
 592
 593	/*
 594	 * TGL/ADL-S DMC firmware incorrectly uses the undelayed vblank
 595	 * event for the HRR handler, when it should be using the delayed
 596	 * vblank event instead. Fixed firmware was never released
 597	 * so the Windows driver just hacks around it by overriding
 598	 * the event ID. Do the same.
 599	 */
 600	if ((display->platform.tigerlake || display->platform.alderlake_s) &&
 601	    is_event_handler(display, dmc_id, MAINDMC_EVENT_VBLANK_A, reg_ctl, *data_ctl)) {
 602		*data_ctl &= ~DMC_EVT_CTL_EVENT_ID_MASK;
 603		*data_ctl |=  REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
 604					     MAINDMC_EVENT_VBLANK_DELAYED_A);
 605		return true;
 606	}
 607
 608	return false;
 609}
 610
 611static bool disable_dmc_evt(struct intel_display *display,
 612			    enum intel_dmc_id dmc_id,
 613			    i915_reg_t reg, u32 data)
 614{
 615	if (!is_dmc_evt_ctl_reg(display, dmc_id, reg))
 616		return false;
 617
 618	/* keep all pipe DMC events disabled by default */
 619	if (dmc_id != DMC_FW_MAIN)
 620		return true;
 621
 622	/* also disable the flip queue event on the main DMC on TGL */
 623	if (display->platform.tigerlake &&
 624	    is_event_handler(display, dmc_id, MAINDMC_EVENT_CLK_MSEC, reg, data))
 625		return true;
 626
 627	/* also disable the HRR event on the main DMC on TGL/ADLS */
 628	if ((display->platform.tigerlake || display->platform.alderlake_s) &&
 629	    is_event_handler(display, dmc_id, MAINDMC_EVENT_VBLANK_DELAYED_A, reg, data))
 630		return true;
 631
 632	return false;
 633}
 634
 635static u32 dmc_mmiodata(struct intel_display *display,
 636			struct intel_dmc *dmc,
 637			enum intel_dmc_id dmc_id, int i)
 638{
 639	if (disable_dmc_evt(display, dmc_id,
 640			    dmc->dmc_info[dmc_id].mmioaddr[i],
 641			    dmc->dmc_info[dmc_id].mmiodata[i]))
 642		return dmc_evt_ctl_disable(dmc->dmc_info[dmc_id].mmiodata[i]);
 643	else
 644		return dmc->dmc_info[dmc_id].mmiodata[i];
 645}
 646
 647static void dmc_load_mmio(struct intel_display *display, enum intel_dmc_id dmc_id)
 648{
 649	struct intel_dmc *dmc = display_to_dmc(display);
 650	int i;
 651
 652	for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
 653		intel_de_write(display, dmc->dmc_info[dmc_id].mmioaddr[i],
 654			       dmc_mmiodata(display, dmc, dmc_id, i));
 655	}
 656}
 657
 658static void dmc_load_program(struct intel_display *display, enum intel_dmc_id dmc_id)
 659{
 660	struct intel_dmc *dmc = display_to_dmc(display);
 661	int i;
 662
 663	disable_all_event_handlers(display, dmc_id);
 664
 665	preempt_disable();
 666
 667	for (i = 0; i < dmc->dmc_info[dmc_id].dmc_fw_size; i++) {
 668		intel_de_write_fw(display,
 669				  DMC_PROGRAM(dmc->dmc_info[dmc_id].start_mmioaddr, i),
 670				  dmc->dmc_info[dmc_id].payload[i]);
 671	}
 672
 673	preempt_enable();
 674
 675	dmc_load_mmio(display, dmc_id);
 676}
 677
 678static void assert_dmc_loaded(struct intel_display *display,
 679			      enum intel_dmc_id dmc_id)
 680{
 681	struct intel_dmc *dmc = display_to_dmc(display);
 682	u32 expected, found;
 683	int i;
 684
 685	if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(display, dmc_id))
 686		return;
 687
 688	found = intel_de_read(display, DMC_PROGRAM(dmc->dmc_info[dmc_id].start_mmioaddr, 0));
 689	expected = dmc->dmc_info[dmc_id].payload[0];
 690
 691	drm_WARN(display->drm, found != expected,
 692		 "DMC %d program storage start incorrect (expected 0x%x, current 0x%x)\n",
 693		 dmc_id, expected, found);
 694
 695	for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
 696		i915_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
 697
 698		found = intel_de_read(display, reg);
 699		expected = dmc_mmiodata(display, dmc, dmc_id, i);
 700
 701		drm_WARN(display->drm, found != expected,
 702			 "DMC %d mmio[%d]/0x%x incorrect (expected 0x%x, current 0x%x)\n",
 703			 dmc_id, i, i915_mmio_reg_offset(reg), expected, found);
 704	}
 705}
 706
 707void assert_main_dmc_loaded(struct intel_display *display)
 708{
 709	assert_dmc_loaded(display, DMC_FW_MAIN);
 710}
 711
 712static bool need_pipedmc_load_program(struct intel_display *display)
 713{
 714	/* On TGL/derivatives pipe DMC state is lost when PG1 is disabled */
 715	return DISPLAY_VER(display) == 12;
 716}
 717
 718static bool need_pipedmc_load_mmio(struct intel_display *display, enum pipe pipe)
 719{
 720	/*
 721	 * Xe3_LPD/Xe3p_LPD:
 722	 * - pipe A/B DMC doesn't need save/restore
 723	 * - pipe C/D DMC is in PG0, needs manual save/restore
 724	 */
 725	if (IS_DISPLAY_VER(display, 30, 35))
 726		return pipe >= PIPE_C;
 727
 728	/*
 729	 * FIXME LNL unclear, main DMC firmware has the pipe DMC A/B PG0
 730	 * save/restore, but so far unable to see the loss of pipe DMC state
 731	 * in action. Are we just failing to turn off PG0 due to some other
 732	 * SoC level stuff?
 733	 */
 734	if (DISPLAY_VER(display) == 20)
 735		return false;
 736
 737	/*
 738	 * FIXME BMG untested, main DMC firmware has the
 739	 * pipe DMC A/B PG0 save/restore...
 740	 */
 741	if (display->platform.battlemage)
 742		return false;
 743
 744	/*
 745	 * DG2:
 746	 * - Pipe DMCs presumably in PG0?
 747	 * - No DC6, and even DC9 doesn't seem to result
 748	 *   in loss of DMC state for whatever reason
 749	 */
 750	if (display->platform.dg2)
 751		return false;
 752
 753	/*
 754	 * ADL/MTL:
 755	 * - pipe A/B DMC is in PG0, saved/restored by the main DMC
 756	 * - pipe C/D DMC is in PG0, needs manual save/restore
 757	 */
 758	if (IS_DISPLAY_VER(display, 13, 14))
 759		return pipe >= PIPE_C;
 760
 761	return false;
 762}
 763
 764static bool can_enable_pipedmc(const struct intel_crtc_state *crtc_state)
 765{
 766	struct intel_display *display = to_intel_display(crtc_state);
 767
 768	/*
 769	 * On TGL/derivatives pipe DMC state is lost when PG1 is disabled.
 770	 * Do not even enable the pipe DMC when that can happen outside
 771	 * of driver control (PSR+DC5/6).
 772	 */
 773	if (DISPLAY_VER(display) == 12 && crtc_state->has_psr)
 774		return false;
 775
 776	return true;
 777}
 778
 779void intel_dmc_enable_pipe(const struct intel_crtc_state *crtc_state)
 780{
 781	struct intel_display *display = to_intel_display(crtc_state);
 782	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 783	enum pipe pipe = crtc->pipe;
 784	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
 785
 786	if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(display, dmc_id))
 787		return;
 788
 789	if (!can_enable_pipedmc(crtc_state)) {
 790		intel_dmc_disable_pipe(crtc_state);
 791		return;
 792	}
 793
 794	if (need_pipedmc_load_program(display))
 795		dmc_load_program(display, dmc_id);
 796	else if (need_pipedmc_load_mmio(display, pipe))
 797		dmc_load_mmio(display, dmc_id);
 798
 799	assert_dmc_loaded(display, dmc_id);
 800
 801	if (DISPLAY_VER(display) >= 20) {
 802		intel_flipq_reset(display, pipe);
 803
 804		intel_de_write(display, PIPEDMC_INTERRUPT(pipe), pipedmc_interrupt_mask(display));
 805		intel_de_write(display, PIPEDMC_INTERRUPT_MASK(pipe), ~pipedmc_interrupt_mask(display));
 806	}
 807
 808	if (DISPLAY_VER(display) >= 14)
 809		intel_de_rmw(display, MTL_PIPEDMC_CONTROL, 0, PIPEDMC_ENABLE_MTL(pipe));
 810	else
 811		intel_de_rmw(display, PIPEDMC_CONTROL(pipe), 0, PIPEDMC_ENABLE);
 812}
 813
 814void intel_dmc_disable_pipe(const struct intel_crtc_state *crtc_state)
 815{
 816	struct intel_display *display = to_intel_display(crtc_state);
 817	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 818	enum pipe pipe = crtc->pipe;
 819	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
 820
 821	if (!is_valid_dmc_id(dmc_id) || !has_dmc_id_fw(display, dmc_id))
 822		return;
 823
 824	if (DISPLAY_VER(display) >= 14)
 825		intel_de_rmw(display, MTL_PIPEDMC_CONTROL, PIPEDMC_ENABLE_MTL(pipe), 0);
 826	else
 827		intel_de_rmw(display, PIPEDMC_CONTROL(pipe), PIPEDMC_ENABLE, 0);
 828
 829	if (DISPLAY_VER(display) >= 20) {
 830		intel_de_write(display, PIPEDMC_INTERRUPT_MASK(pipe), ~0);
 831		intel_de_write(display, PIPEDMC_INTERRUPT(pipe), pipedmc_interrupt_mask(display));
 832
 833		intel_flipq_reset(display, pipe);
 834	}
 835}
 836
 837static void dmc_configure_event(struct intel_display *display,
 838				enum intel_dmc_id dmc_id,
 839				unsigned int event_id,
 840				bool enable)
 841{
 842	struct intel_dmc *dmc = display_to_dmc(display);
 843	int num_handlers = 0;
 844	int i;
 845
 846	for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
 847		i915_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
 848		u32 data = dmc->dmc_info[dmc_id].mmiodata[i];
 849
 850		if (!is_event_handler(display, dmc_id, event_id, reg, data))
 851			continue;
 852
 853		intel_de_write(display, reg, enable ? data : dmc_evt_ctl_disable(data));
 854		num_handlers++;
 855	}
 856
 857	drm_WARN_ONCE(display->drm, num_handlers != 1,
 858		      "DMC %d has %d handlers for event 0x%x\n",
 859		      dmc_id, num_handlers, event_id);
 860}
 861
 862/**
 863 * intel_dmc_block_pkgc() - block PKG C-state
 864 * @display: display instance
 865 * @pipe: pipe which register use to block
 866 * @block: block/unblock
 867 *
 868 * This interface is target for Wa_16025596647 usage. I.e. to set/clear
 869 * PIPEDMC_BLOCK_PKGC_SW_BLOCK_PKGC_ALWAYS bit in PIPEDMC_BLOCK_PKGC_SW register.
 870 */
 871void intel_dmc_block_pkgc(struct intel_display *display, enum pipe pipe,
 872			  bool block)
 873{
 874	intel_de_rmw(display, PIPEDMC_BLOCK_PKGC_SW(pipe),
 875		     PIPEDMC_BLOCK_PKGC_SW_BLOCK_PKGC_ALWAYS, block ?
 876		     PIPEDMC_BLOCK_PKGC_SW_BLOCK_PKGC_ALWAYS : 0);
 877}
 878
 879/**
 880 * intel_dmc_start_pkgc_exit_at_start_of_undelayed_vblank() - start of PKG
 881 * C-state exit
 882 * @display: display instance
 883 * @pipe: pipe which register use to block
 884 * @enable: enable/disable
 885 *
 886 * This interface is target for Wa_16025596647 usage. I.e. start the package C
 887 * exit at the start of the undelayed vblank
 888 */
 889void intel_dmc_start_pkgc_exit_at_start_of_undelayed_vblank(struct intel_display *display,
 890							    enum pipe pipe, bool enable)
 891{
 892	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(pipe);
 893
 894	dmc_configure_event(display, dmc_id, PIPEDMC_EVENT_VBLANK, enable);
 895}
 896
 897/**
 898 * intel_dmc_load_program() - write the firmware from memory to register.
 899 * @display: display instance
 900 *
 901 * DMC firmware is read from a .bin file and kept in internal memory one time.
 902 * Everytime display comes back from low power state this function is called to
 903 * copy the firmware from internal memory to registers.
 904 */
 905void intel_dmc_load_program(struct intel_display *display)
 906{
 907	struct i915_power_domains *power_domains = &display->power.domains;
 908	enum intel_dmc_id dmc_id;
 909
 910	if (!intel_dmc_has_payload(display))
 911		return;
 912
 913	assert_display_rpm_held(display);
 914
 915	pipedmc_clock_gating_wa(display, true);
 916
 917	for_each_dmc_id(dmc_id) {
 918		dmc_load_program(display, dmc_id);
 919		assert_dmc_loaded(display, dmc_id);
 920	}
 921
 922	if (DISPLAY_VER(display) >= 20)
 923		intel_de_write(display, DMC_FQ_W2_PTS_CFG_SEL,
 924			       PIPE_D_DMC_W2_PTS_CONFIG_SELECT(PIPE_D) |
 925			       PIPE_C_DMC_W2_PTS_CONFIG_SELECT(PIPE_C) |
 926			       PIPE_B_DMC_W2_PTS_CONFIG_SELECT(PIPE_B) |
 927			       PIPE_A_DMC_W2_PTS_CONFIG_SELECT(PIPE_A));
 928
 929	power_domains->dc_state = 0;
 930
 931	gen9_set_dc_state_debugmask(display);
 932
 933	pipedmc_clock_gating_wa(display, false);
 934}
 935
 936/**
 937 * intel_dmc_disable_program() - disable the firmware
 938 * @display: display instance
 939 *
 940 * Disable all event handlers in the firmware, making sure the firmware is
 941 * inactive after the display is uninitialized.
 942 */
 943void intel_dmc_disable_program(struct intel_display *display)
 944{
 945	enum intel_dmc_id dmc_id;
 946
 947	if (!intel_dmc_has_payload(display))
 948		return;
 949
 950	pipedmc_clock_gating_wa(display, true);
 951
 952	for_each_dmc_id(dmc_id)
 953		disable_all_event_handlers(display, dmc_id);
 954
 955	pipedmc_clock_gating_wa(display, false);
 956}
 957
 958static bool fw_info_matches_stepping(const struct intel_fw_info *fw_info,
 959				     const struct stepping_info *si)
 960{
 961	if ((fw_info->substepping == '*' && si->stepping == fw_info->stepping) ||
 962	    (si->stepping == fw_info->stepping && si->substepping == fw_info->substepping) ||
 963	    /*
 964	     * If we don't find a more specific one from above two checks, we
 965	     * then check for the generic one to be sure to work even with
 966	     * "broken firmware"
 967	     */
 968	    (si->stepping == '*' && si->substepping == fw_info->substepping) ||
 969	    (fw_info->stepping == '*' && fw_info->substepping == '*'))
 970		return true;
 971
 972	return false;
 973}
 974
 975/*
 976 * Search fw_info table for dmc_offset to find firmware binary: num_entries is
 977 * already sanitized.
 978 */
 979static void dmc_set_fw_offset(struct intel_dmc *dmc,
 980			      const struct intel_fw_info *fw_info,
 981			      unsigned int num_entries,
 982			      const struct stepping_info *si,
 983			      u8 package_ver)
 984{
 985	struct intel_display *display = dmc->display;
 986	enum intel_dmc_id dmc_id;
 987	unsigned int i;
 988
 989	for (i = 0; i < num_entries; i++) {
 990		dmc_id = package_ver <= 1 ? DMC_FW_MAIN : fw_info[i].dmc_id;
 991
 992		if (!is_valid_dmc_id(dmc_id)) {
 993			drm_dbg(display->drm, "Unsupported firmware id: %u\n", dmc_id);
 994			continue;
 995		}
 996
 997		/* More specific versions come first, so we don't even have to
 998		 * check for the stepping since we already found a previous FW
 999		 * for this id.
1000		 */
1001		if (dmc->dmc_info[dmc_id].present)
1002			continue;
1003
1004		if (fw_info_matches_stepping(&fw_info[i], si)) {
1005			dmc->dmc_info[dmc_id].present = true;
1006			dmc->dmc_info[dmc_id].dmc_offset = fw_info[i].offset;
1007		}
1008	}
1009}
1010
1011static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc,
1012				       const u32 *mmioaddr, u32 mmio_count,
1013				       int header_ver, enum intel_dmc_id dmc_id)
1014{
1015	struct intel_display *display = dmc->display;
1016	u32 start_range, end_range;
1017	int i;
1018
1019	if (header_ver == 1) {
1020		start_range = DMC_MMIO_START_RANGE;
1021		end_range = DMC_MMIO_END_RANGE;
1022	} else if (dmc_id == DMC_FW_MAIN) {
1023		start_range = TGL_MAIN_MMIO_START;
1024		end_range = TGL_MAIN_MMIO_END;
1025	} else if (DISPLAY_VER(display) >= 13) {
1026		start_range = ADLP_PIPE_MMIO_START;
1027		end_range = ADLP_PIPE_MMIO_END;
1028	} else if (DISPLAY_VER(display) >= 12) {
1029		start_range = TGL_PIPE_MMIO_START(dmc_id);
1030		end_range = TGL_PIPE_MMIO_END(dmc_id);
1031	} else {
1032		drm_warn(display->drm, "Unknown mmio range for sanity check");
1033		return false;
1034	}
1035
1036	for (i = 0; i < mmio_count; i++) {
1037		if (mmioaddr[i] < start_range || mmioaddr[i] > end_range)
1038			return false;
1039	}
1040
1041	return true;
1042}
1043
1044static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
1045			       const struct intel_dmc_header_base *dmc_header,
1046			       size_t rem_size, enum intel_dmc_id dmc_id)
1047{
1048	struct intel_display *display = dmc->display;
1049	struct dmc_fw_info *dmc_info = &dmc->dmc_info[dmc_id];
1050	unsigned int header_len_bytes, dmc_header_size, payload_size, i;
1051	const u32 *mmioaddr, *mmiodata;
1052	u32 mmio_count, mmio_count_max, start_mmioaddr;
1053	u8 *payload;
1054
1055	BUILD_BUG_ON(ARRAY_SIZE(dmc_info->mmioaddr) < DMC_V3_MAX_MMIO_COUNT ||
1056		     ARRAY_SIZE(dmc_info->mmioaddr) < DMC_V1_MAX_MMIO_COUNT);
1057
1058	/*
1059	 * Check if we can access common fields, we will checkc again below
1060	 * after we have read the version
1061	 */
1062	if (rem_size < sizeof(struct intel_dmc_header_base))
1063		goto error_truncated;
1064
1065	/* Cope with small differences between v1 and v3 */
1066	if (dmc_header->header_ver == 3) {
1067		const struct intel_dmc_header_v3 *v3 =
1068			(const struct intel_dmc_header_v3 *)dmc_header;
1069
1070		if (rem_size < sizeof(struct intel_dmc_header_v3))
1071			goto error_truncated;
1072
1073		mmioaddr = v3->mmioaddr;
1074		mmiodata = v3->mmiodata;
1075		mmio_count = v3->mmio_count;
1076		mmio_count_max = DMC_V3_MAX_MMIO_COUNT;
1077		/* header_len is in dwords */
1078		header_len_bytes = dmc_header->header_len * 4;
1079		start_mmioaddr = v3->start_mmioaddr;
1080		dmc_header_size = sizeof(*v3);
1081	} else if (dmc_header->header_ver == 1) {
1082		const struct intel_dmc_header_v1 *v1 =
1083			(const struct intel_dmc_header_v1 *)dmc_header;
1084
1085		if (rem_size < sizeof(struct intel_dmc_header_v1))
1086			goto error_truncated;
1087
1088		mmioaddr = v1->mmioaddr;
1089		mmiodata = v1->mmiodata;
1090		mmio_count = v1->mmio_count;
1091		mmio_count_max = DMC_V1_MAX_MMIO_COUNT;
1092		header_len_bytes = dmc_header->header_len;
1093		start_mmioaddr = DMC_V1_MMIO_START_RANGE;
1094		dmc_header_size = sizeof(*v1);
1095	} else {
1096		drm_err(display->drm, "Unknown DMC fw header version: %u\n",
1097			dmc_header->header_ver);
1098		return 0;
1099	}
1100
1101	if (header_len_bytes != dmc_header_size) {
1102		drm_err(display->drm, "DMC firmware has wrong dmc header length "
1103			"(%u bytes)\n", header_len_bytes);
1104		return 0;
1105	}
1106
1107	/* Cache the dmc header info. */
1108	if (mmio_count > mmio_count_max) {
1109		drm_err(display->drm, "DMC firmware has wrong mmio count %u\n", mmio_count);
1110		return 0;
1111	}
1112
1113	if (!dmc_mmio_addr_sanity_check(dmc, mmioaddr, mmio_count,
1114					dmc_header->header_ver, dmc_id)) {
1115		drm_err(display->drm, "DMC firmware has Wrong MMIO Addresses\n");
1116		return 0;
1117	}
1118
1119	drm_dbg_kms(display->drm, "DMC %d:\n", dmc_id);
1120	for (i = 0; i < mmio_count; i++) {
1121		dmc_info->mmioaddr[i] = _MMIO(mmioaddr[i]);
1122		dmc_info->mmiodata[i] = mmiodata[i];
1123	}
1124
1125	for (i = 0; i < mmio_count - 1; i++) {
1126		u32 orig_mmiodata[2] = {
1127			dmc_info->mmiodata[i],
1128			dmc_info->mmiodata[i+1],
1129		};
1130
1131		if (!fixup_dmc_evt(display, dmc_id,
1132				   dmc_info->mmioaddr[i], &dmc_info->mmiodata[i],
1133				   dmc_info->mmioaddr[i+1], &dmc_info->mmiodata[i+1]))
1134			continue;
1135
1136		drm_dbg_kms(display->drm,
1137			    " mmio[%d]: 0x%x = 0x%x->0x%x (EVT_CTL)\n",
1138			    i, i915_mmio_reg_offset(dmc_info->mmioaddr[i]),
1139			    orig_mmiodata[0], dmc_info->mmiodata[i]);
1140		drm_dbg_kms(display->drm,
1141			    " mmio[%d]: 0x%x = 0x%x->0x%x (EVT_HTP)\n",
1142			    i+1, i915_mmio_reg_offset(dmc_info->mmioaddr[i+1]),
1143			    orig_mmiodata[1], dmc_info->mmiodata[i+1]);
1144	}
1145
1146	for (i = 0; i < mmio_count; i++) {
1147		drm_dbg_kms(display->drm, " mmio[%d]: 0x%x = 0x%x%s%s\n",
1148			    i, i915_mmio_reg_offset(dmc_info->mmioaddr[i]), dmc_info->mmiodata[i],
1149			    is_dmc_evt_ctl_reg(display, dmc_id, dmc_info->mmioaddr[i]) ? " (EVT_CTL)" :
1150			    is_dmc_evt_htp_reg(display, dmc_id, dmc_info->mmioaddr[i]) ? " (EVT_HTP)" : "",
1151			    disable_dmc_evt(display, dmc_id, dmc_info->mmioaddr[i],
1152					    dmc_info->mmiodata[i]) ? " (disabling)" : "");
1153	}
1154	dmc_info->mmio_count = mmio_count;
1155	dmc_info->start_mmioaddr = start_mmioaddr;
1156
1157	rem_size -= header_len_bytes;
1158
1159	/* fw_size is in dwords, so multiplied by 4 to convert into bytes. */
1160	payload_size = dmc_header->fw_size * 4;
1161	if (rem_size < payload_size)
1162		goto error_truncated;
1163
1164	if (payload_size > dmc->max_fw_size) {
1165		drm_err(display->drm, "DMC FW too big (%u bytes)\n", payload_size);
1166		return 0;
1167	}
1168	dmc_info->dmc_fw_size = dmc_header->fw_size;
1169
1170	dmc_info->payload = kmalloc(payload_size, GFP_KERNEL);
1171	if (!dmc_info->payload)
1172		return 0;
1173
1174	payload = (u8 *)(dmc_header) + header_len_bytes;
1175	memcpy(dmc_info->payload, payload, payload_size);
1176
1177	return header_len_bytes + payload_size;
1178
1179error_truncated:
1180	drm_err(display->drm, "Truncated DMC firmware, refusing.\n");
1181	return 0;
1182}
1183
1184static u32
1185parse_dmc_fw_package(struct intel_dmc *dmc,
1186		     const struct intel_package_header *package_header,
1187		     const struct stepping_info *si,
1188		     size_t rem_size)
1189{
1190	struct intel_display *display = dmc->display;
1191	u32 package_size = sizeof(struct intel_package_header);
1192	u32 num_entries, max_entries;
1193	const struct intel_fw_info *fw_info;
1194
1195	if (rem_size < package_size)
1196		goto error_truncated;
1197
1198	if (package_header->header_ver == 1) {
1199		max_entries = PACKAGE_MAX_FW_INFO_ENTRIES;
1200	} else if (package_header->header_ver == 2) {
1201		max_entries = PACKAGE_V2_MAX_FW_INFO_ENTRIES;
1202	} else {
1203		drm_err(display->drm, "DMC firmware has unknown header version %u\n",
1204			package_header->header_ver);
1205		return 0;
1206	}
1207
1208	/*
1209	 * We should always have space for max_entries,
1210	 * even if not all are used
1211	 */
1212	package_size += max_entries * sizeof(struct intel_fw_info);
1213	if (rem_size < package_size)
1214		goto error_truncated;
1215
1216	if (package_header->header_len * 4 != package_size) {
1217		drm_err(display->drm, "DMC firmware has wrong package header length "
1218			"(%u bytes)\n", package_size);
1219		return 0;
1220	}
1221
1222	num_entries = package_header->num_entries;
1223	if (WARN_ON(num_entries > max_entries))
1224		num_entries = max_entries;
1225
1226	fw_info = (const struct intel_fw_info *)
1227		((u8 *)package_header + sizeof(*package_header));
1228	dmc_set_fw_offset(dmc, fw_info, num_entries, si,
1229			  package_header->header_ver);
1230
1231	/* dmc_offset is in dwords */
1232	return package_size;
1233
1234error_truncated:
1235	drm_err(display->drm, "Truncated DMC firmware, refusing.\n");
1236	return 0;
1237}
1238
1239/* Return number of bytes parsed or 0 on error */
1240static u32 parse_dmc_fw_css(struct intel_dmc *dmc,
1241			    struct intel_css_header *css_header,
1242			    size_t rem_size)
1243{
1244	struct intel_display *display = dmc->display;
1245
1246	if (rem_size < sizeof(struct intel_css_header)) {
1247		drm_err(display->drm, "Truncated DMC firmware, refusing.\n");
1248		return 0;
1249	}
1250
1251	if (sizeof(struct intel_css_header) !=
1252	    (css_header->header_len * 4)) {
1253		drm_err(display->drm, "DMC firmware has wrong CSS header length "
1254			"(%u bytes)\n",
1255			(css_header->header_len * 4));
1256		return 0;
1257	}
1258
1259	dmc->version = css_header->version;
1260
1261	return sizeof(struct intel_css_header);
1262}
1263
1264static int parse_dmc_fw(struct intel_dmc *dmc, const struct firmware *fw)
1265{
1266	struct intel_display *display = dmc->display;
1267	struct intel_css_header *css_header;
1268	struct intel_package_header *package_header;
1269	struct intel_dmc_header_base *dmc_header;
1270	struct stepping_info display_info = { '*', '*'};
1271	const struct stepping_info *si = intel_get_stepping_info(display, &display_info);
1272	enum intel_dmc_id dmc_id;
1273	u32 readcount = 0;
1274	u32 r, offset;
1275
1276	if (!fw)
1277		return -EINVAL;
1278
1279	/* Extract CSS Header information */
1280	css_header = (struct intel_css_header *)fw->data;
1281	r = parse_dmc_fw_css(dmc, css_header, fw->size);
1282	if (!r)
1283		return -EINVAL;
1284
1285	readcount += r;
1286
1287	/* Extract Package Header information */
1288	package_header = (struct intel_package_header *)&fw->data[readcount];
1289	r = parse_dmc_fw_package(dmc, package_header, si, fw->size - readcount);
1290	if (!r)
1291		return -EINVAL;
1292
1293	readcount += r;
1294
1295	for_each_dmc_id(dmc_id) {
1296		if (!dmc->dmc_info[dmc_id].present)
1297			continue;
1298
1299		offset = readcount + dmc->dmc_info[dmc_id].dmc_offset * 4;
1300		if (offset > fw->size) {
1301			drm_err(display->drm, "Reading beyond the fw_size\n");
1302			continue;
1303		}
1304
1305		dmc_header = (struct intel_dmc_header_base *)&fw->data[offset];
1306		parse_dmc_fw_header(dmc, dmc_header, fw->size - offset, dmc_id);
1307	}
1308
1309	if (!intel_dmc_has_payload(display)) {
1310		drm_err(display->drm, "DMC firmware main program not found\n");
1311		return -ENOENT;
1312	}
1313
1314	return 0;
1315}
1316
1317static void intel_dmc_runtime_pm_get(struct intel_display *display)
1318{
1319	drm_WARN_ON(display->drm, display->dmc.wakeref);
1320	display->dmc.wakeref = intel_display_power_get(display, POWER_DOMAIN_INIT);
1321}
1322
1323static void intel_dmc_runtime_pm_put(struct intel_display *display)
1324{
1325	intel_wakeref_t wakeref __maybe_unused =
1326		fetch_and_zero(&display->dmc.wakeref);
1327
1328	intel_display_power_put(display, POWER_DOMAIN_INIT, wakeref);
1329}
1330
1331static const char *dmc_fallback_path(struct intel_display *display)
1332{
1333	if (display->platform.alderlake_p)
1334		return ADLP_DMC_FALLBACK_PATH;
1335
1336	return NULL;
1337}
1338
1339static void dmc_load_work_fn(struct work_struct *work)
1340{
1341	struct intel_dmc *dmc = container_of(work, typeof(*dmc), work);
1342	struct intel_display *display = dmc->display;
1343	const struct firmware *fw = NULL;
1344	const char *fallback_path;
1345	int err;
1346
1347	err = request_firmware(&fw, dmc->fw_path, display->drm->dev);
1348
1349	if (err == -ENOENT && !dmc_firmware_param(display)) {
1350		fallback_path = dmc_fallback_path(display);
1351		if (fallback_path) {
1352			drm_dbg_kms(display->drm, "%s not found, falling back to %s\n",
1353				    dmc->fw_path, fallback_path);
1354			err = request_firmware(&fw, fallback_path, display->drm->dev);
1355			if (err == 0)
1356				dmc->fw_path = fallback_path;
1357		}
1358	}
1359
1360	if (err) {
1361		drm_notice(display->drm,
1362			   "Failed to load DMC firmware %s (%pe). Disabling runtime power management.\n",
1363			   dmc->fw_path, ERR_PTR(err));
1364		drm_notice(display->drm, "DMC firmware homepage: %s",
1365			   INTEL_DMC_FIRMWARE_URL);
1366		return;
1367	}
1368
1369	err = parse_dmc_fw(dmc, fw);
1370	if (err) {
1371		drm_notice(display->drm,
1372			   "Failed to parse DMC firmware %s (%pe). Disabling runtime power management.\n",
1373			   dmc->fw_path, ERR_PTR(err));
1374		goto out;
1375	}
1376
1377	intel_dmc_load_program(display);
1378	intel_dmc_runtime_pm_put(display);
1379
1380	drm_info(display->drm, "Finished loading DMC firmware %s (v%u.%u)\n",
1381		 dmc->fw_path, DMC_VERSION_MAJOR(dmc->version),
1382		 DMC_VERSION_MINOR(dmc->version));
1383
1384out:
1385	release_firmware(fw);
1386}
1387
1388/**
1389 * intel_dmc_init() - initialize the firmware loading.
1390 * @display: display instance
1391 *
1392 * This function is called at the time of loading the display driver to read
1393 * firmware from a .bin file and copied into a internal memory.
1394 */
1395void intel_dmc_init(struct intel_display *display)
1396{
1397	struct intel_dmc *dmc;
1398
1399	if (!HAS_DMC(display))
1400		return;
1401
1402	/*
1403	 * Obtain a runtime pm reference, until DMC is loaded, to avoid entering
1404	 * runtime-suspend.
1405	 *
1406	 * On error, we return with the rpm wakeref held to prevent runtime
1407	 * suspend as runtime suspend *requires* a working DMC for whatever
1408	 * reason.
1409	 */
1410	intel_dmc_runtime_pm_get(display);
1411
1412	dmc = kzalloc(sizeof(*dmc), GFP_KERNEL);
1413	if (!dmc)
1414		return;
1415
1416	dmc->display = display;
1417
1418	INIT_WORK(&dmc->work, dmc_load_work_fn);
1419
1420	dmc->fw_path = dmc_firmware_default(display, &dmc->max_fw_size);
1421
1422	if (dmc_firmware_param_disabled(display)) {
1423		drm_info(display->drm, "Disabling DMC firmware and runtime PM\n");
1424		goto out;
1425	}
1426
1427	if (dmc_firmware_param(display))
1428		dmc->fw_path = dmc_firmware_param(display);
1429
1430	if (!dmc->fw_path) {
1431		drm_dbg_kms(display->drm,
1432			    "No known DMC firmware for platform, disabling runtime PM\n");
1433		goto out;
1434	}
1435
1436	display->dmc.dmc = dmc;
1437
1438	drm_dbg_kms(display->drm, "Loading %s\n", dmc->fw_path);
1439	queue_work(display->wq.unordered, &dmc->work);
1440
1441	return;
1442
1443out:
1444	kfree(dmc);
1445}
1446
1447/**
1448 * intel_dmc_suspend() - prepare DMC firmware before system suspend
1449 * @display: display instance
1450 *
1451 * Prepare the DMC firmware before entering system suspend. This includes
1452 * flushing pending work items and releasing any resources acquired during
1453 * init.
1454 */
1455void intel_dmc_suspend(struct intel_display *display)
1456{
1457	struct intel_dmc *dmc = display_to_dmc(display);
1458
1459	if (!HAS_DMC(display))
1460		return;
1461
1462	if (dmc)
1463		flush_work(&dmc->work);
1464
1465	/* Drop the reference held in case DMC isn't loaded. */
1466	if (!intel_dmc_has_payload(display))
1467		intel_dmc_runtime_pm_put(display);
1468}
1469
1470void intel_dmc_wait_fw_load(struct intel_display *display)
1471{
1472	struct intel_dmc *dmc = display_to_dmc(display);
1473
1474	if (!HAS_DMC(display))
1475		return;
1476
1477	if (dmc)
1478		flush_work(&dmc->work);
1479}
1480
1481/**
1482 * intel_dmc_resume() - init DMC firmware during system resume
1483 * @display: display instance
1484 *
1485 * Reinitialize the DMC firmware during system resume, reacquiring any
1486 * resources released in intel_dmc_suspend().
1487 */
1488void intel_dmc_resume(struct intel_display *display)
1489{
1490	if (!HAS_DMC(display))
1491		return;
1492
1493	/*
1494	 * Reacquire the reference to keep RPM disabled in case DMC isn't
1495	 * loaded.
1496	 */
1497	if (!intel_dmc_has_payload(display))
1498		intel_dmc_runtime_pm_get(display);
1499}
1500
1501/**
1502 * intel_dmc_fini() - unload the DMC firmware.
1503 * @display: display instance
1504 *
1505 * Firmmware unloading includes freeing the internal memory and reset the
1506 * firmware loading status.
1507 */
1508void intel_dmc_fini(struct intel_display *display)
1509{
1510	struct intel_dmc *dmc = display_to_dmc(display);
1511	enum intel_dmc_id dmc_id;
1512
1513	if (!HAS_DMC(display))
1514		return;
1515
1516	intel_dmc_suspend(display);
1517	drm_WARN_ON(display->drm, display->dmc.wakeref);
1518
1519	if (dmc) {
1520		for_each_dmc_id(dmc_id)
1521			kfree(dmc->dmc_info[dmc_id].payload);
1522
1523		kfree(dmc);
1524		display->dmc.dmc = NULL;
1525	}
1526}
1527
1528struct intel_dmc_snapshot {
1529	bool initialized;
1530	bool loaded;
1531	u32 version;
1532};
1533
1534struct intel_dmc_snapshot *intel_dmc_snapshot_capture(struct intel_display *display)
1535{
1536	struct intel_dmc *dmc = display_to_dmc(display);
1537	struct intel_dmc_snapshot *snapshot;
1538
1539	if (!HAS_DMC(display))
1540		return NULL;
1541
1542	snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC);
1543	if (!snapshot)
1544		return NULL;
1545
1546	snapshot->initialized = dmc;
1547	snapshot->loaded = intel_dmc_has_payload(display);
1548	if (dmc)
1549		snapshot->version = dmc->version;
1550
1551	return snapshot;
1552}
1553
1554void intel_dmc_snapshot_print(const struct intel_dmc_snapshot *snapshot, struct drm_printer *p)
1555{
1556	if (!snapshot)
1557		return;
1558
1559	drm_printf(p, "DMC initialized: %s\n", str_yes_no(snapshot->initialized));
1560	drm_printf(p, "DMC loaded: %s\n", str_yes_no(snapshot->loaded));
1561	if (snapshot->initialized)
1562		drm_printf(p, "DMC fw version: %d.%d\n",
1563			   DMC_VERSION_MAJOR(snapshot->version),
1564			   DMC_VERSION_MINOR(snapshot->version));
1565}
1566
1567void intel_dmc_update_dc6_allowed_count(struct intel_display *display,
1568					bool start_tracking)
1569{
1570	struct intel_dmc *dmc = display_to_dmc(display);
1571	u32 dc5_cur_count;
1572
1573	if (DISPLAY_VER(dmc->display) < 14)
1574		return;
1575
1576	dc5_cur_count = intel_de_read(dmc->display, DG1_DMC_DEBUG_DC5_COUNT);
1577
1578	if (!start_tracking)
1579		dmc->dc6_allowed.count += dc5_cur_count - dmc->dc6_allowed.dc5_start;
1580
1581	dmc->dc6_allowed.dc5_start = dc5_cur_count;
1582}
1583
1584static bool intel_dmc_get_dc6_allowed_count(struct intel_display *display, u32 *count)
1585{
1586	struct i915_power_domains *power_domains = &display->power.domains;
1587	struct intel_dmc *dmc = display_to_dmc(display);
1588	bool dc6_enabled;
1589
1590	if (DISPLAY_VER(display) < 14)
1591		return false;
1592
1593	mutex_lock(&power_domains->lock);
1594	dc6_enabled = power_domains->dc_state & DC_STATE_EN_UPTO_DC6;
1595	if (dc6_enabled)
1596		intel_dmc_update_dc6_allowed_count(display, false);
1597
1598	*count = dmc->dc6_allowed.count;
1599	mutex_unlock(&power_domains->lock);
1600
1601	return true;
1602}
1603
1604static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
1605{
1606	struct intel_display *display = m->private;
1607	struct intel_dmc *dmc = display_to_dmc(display);
1608	struct ref_tracker *wakeref;
1609	i915_reg_t dc5_reg, dc6_reg = INVALID_MMIO_REG;
1610	u32 dc6_allowed_count;
1611
1612	if (!HAS_DMC(display))
1613		return -ENODEV;
1614
1615	wakeref = intel_display_rpm_get(display);
1616
1617	seq_printf(m, "DMC initialized: %s\n", str_yes_no(dmc));
1618	seq_printf(m, "fw loaded: %s\n",
1619		   str_yes_no(intel_dmc_has_payload(display)));
1620	seq_printf(m, "path: %s\n", dmc ? dmc->fw_path : "N/A");
1621	seq_printf(m, "Pipe A fw needed: %s\n",
1622		   str_yes_no(DISPLAY_VER(display) >= 12));
1623	seq_printf(m, "Pipe A fw loaded: %s\n",
1624		   str_yes_no(has_dmc_id_fw(display, DMC_FW_PIPEA)));
1625	seq_printf(m, "Pipe B fw needed: %s\n",
1626		   str_yes_no(display->platform.alderlake_p ||
1627			      DISPLAY_VER(display) >= 14));
1628	seq_printf(m, "Pipe B fw loaded: %s\n",
1629		   str_yes_no(has_dmc_id_fw(display, DMC_FW_PIPEB)));
1630
1631	if (!intel_dmc_has_payload(display))
1632		goto out;
1633
1634	seq_printf(m, "version: %d.%d\n", DMC_VERSION_MAJOR(dmc->version),
1635		   DMC_VERSION_MINOR(dmc->version));
1636
1637	if (DISPLAY_VER(display) >= 12) {
1638		i915_reg_t dc3co_reg;
1639
1640		if (display->platform.dgfx || DISPLAY_VER(display) >= 14) {
1641			dc3co_reg = DG1_DMC_DEBUG3;
1642			dc5_reg = DG1_DMC_DEBUG_DC5_COUNT;
1643		} else {
1644			dc3co_reg = TGL_DMC_DEBUG3;
1645			dc5_reg = TGL_DMC_DEBUG_DC5_COUNT;
1646			dc6_reg = TGL_DMC_DEBUG_DC6_COUNT;
1647		}
1648
1649		seq_printf(m, "DC3CO count: %d\n",
1650			   intel_de_read(display, dc3co_reg));
1651	} else {
1652		dc5_reg = display->platform.broxton ? BXT_DMC_DC3_DC5_COUNT :
1653			SKL_DMC_DC3_DC5_COUNT;
1654		if (!display->platform.geminilake && !display->platform.broxton)
1655			dc6_reg = SKL_DMC_DC5_DC6_COUNT;
1656	}
1657
1658	seq_printf(m, "DC3 -> DC5 count: %d\n", intel_de_read(display, dc5_reg));
1659
1660	if (intel_dmc_get_dc6_allowed_count(display, &dc6_allowed_count))
1661		seq_printf(m, "DC5 -> DC6 allowed count: %d\n",
1662			   dc6_allowed_count);
1663	else if (i915_mmio_reg_valid(dc6_reg))
1664		seq_printf(m, "DC5 -> DC6 count: %d\n",
1665			   intel_de_read(display, dc6_reg));
1666
1667	seq_printf(m, "program base: 0x%08x\n",
1668		   intel_de_read(display, DMC_PROGRAM(dmc->dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)));
1669
1670out:
1671	seq_printf(m, "ssp base: 0x%08x\n",
1672		   intel_de_read(display, DMC_SSP_BASE));
1673	seq_printf(m, "htp: 0x%08x\n", intel_de_read(display, DMC_HTP_SKL));
1674
1675	intel_display_rpm_put(display, wakeref);
1676
1677	return 0;
1678}
1679
1680DEFINE_SHOW_ATTRIBUTE(intel_dmc_debugfs_status);
1681
1682void intel_dmc_debugfs_register(struct intel_display *display)
1683{
1684	debugfs_create_file("i915_dmc_info", 0444, display->drm->debugfs_root,
1685			    display, &intel_dmc_debugfs_status_fops);
1686}
1687
1688void intel_pipedmc_irq_handler(struct intel_display *display, enum pipe pipe)
1689{
1690	struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
1691	u32 tmp = 0, int_vector;
1692
1693	if (DISPLAY_VER(display) >= 20) {
1694		tmp = intel_de_read(display, PIPEDMC_INTERRUPT(pipe));
1695		intel_de_write(display, PIPEDMC_INTERRUPT(pipe), tmp);
1696
1697		if (tmp & PIPEDMC_FLIPQ_PROG_DONE) {
1698			spin_lock(&display->drm->event_lock);
1699
1700			if (crtc->flipq_event) {
1701				/*
1702				 * Update vblank counter/timestamp in case it
1703				 * hasn't been done yet for this frame.
1704				 */
1705				drm_crtc_accurate_vblank_count(&crtc->base);
1706
1707				drm_crtc_send_vblank_event(&crtc->base, crtc->flipq_event);
1708				crtc->flipq_event = NULL;
1709			}
1710
1711			spin_unlock(&display->drm->event_lock);
1712		}
1713
1714		if (tmp & PIPEDMC_ATS_FAULT)
1715			drm_err_ratelimited(display->drm, "[CRTC:%d:%s] PIPEDMC ATS fault\n",
1716					    crtc->base.base.id, crtc->base.name);
1717		if (tmp & PIPEDMC_GTT_FAULT)
1718			drm_err_ratelimited(display->drm, "[CRTC:%d:%s] PIPEDMC GTT fault\n",
1719					    crtc->base.base.id, crtc->base.name);
1720		if (tmp & PIPEDMC_ERROR)
1721			drm_err(display->drm, "[CRTC:%d:%s] PIPEDMC error\n",
1722				crtc->base.base.id, crtc->base.name);
1723	}
1724
1725	int_vector = intel_de_read(display, PIPEDMC_STATUS(pipe)) & PIPEDMC_INT_VECTOR_MASK;
1726	if (tmp == 0 && int_vector != 0)
1727		drm_err(display->drm, "[CRTC:%d:%s] PIPEDMC interrupt vector 0x%x\n",
1728			crtc->base.base.id, crtc->base.name, int_vector);
1729}
1730
1731void intel_pipedmc_enable_event(struct intel_crtc *crtc,
1732				enum pipedmc_event_id event)
1733{
1734	struct intel_display *display = to_intel_display(crtc);
1735	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(crtc->pipe);
1736
1737	dmc_configure_event(display, dmc_id, event, true);
1738}
1739
1740void intel_pipedmc_disable_event(struct intel_crtc *crtc,
1741				 enum pipedmc_event_id event)
1742{
1743	struct intel_display *display = to_intel_display(crtc);
1744	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(crtc->pipe);
1745
1746	dmc_configure_event(display, dmc_id, event, false);
1747}
1748
1749u32 intel_pipedmc_start_mmioaddr(struct intel_crtc *crtc)
1750{
1751	struct intel_display *display = to_intel_display(crtc);
1752	struct intel_dmc *dmc = display_to_dmc(display);
1753	enum intel_dmc_id dmc_id = PIPE_TO_DMC_ID(crtc->pipe);
1754
1755	return dmc ? dmc->dmc_info[dmc_id].start_mmioaddr : 0;
1756}