core.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Intel Core SoC Power Management Controller Driver
   4 *
   5 * Copyright (c) 2016, Intel Corporation.
   6 * All Rights Reserved.
   7 *
   8 * Authors: Rajneesh Bhardwaj <rajneesh.bhardwaj@intel.com>
   9 *          Vishwanath Somayaji <vishwanath.somayaji@intel.com>
  10 */
  11
  12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14enum header_type {
  15	HEADER_STATUS,
  16	HEADER_VALUE,
  17};
  18
  19#include <linux/bitfield.h>
  20#include <linux/debugfs.h>
  21#include <linux/delay.h>
  22#include <linux/dmi.h>
  23#include <linux/err.h>
  24#include <linux/io.h>
  25#include <linux/module.h>
  26#include <linux/pci.h>
  27#include <linux/slab.h>
  28#include <linux/suspend.h>
  29#include <linux/units.h>
  30
  31#include <asm/cpuid/api.h>
  32#include <asm/cpu_device_id.h>
  33#include <asm/intel-family.h>
  34#include <asm/msr.h>
  35#include <asm/tsc.h>
  36
  37#include "core.h"
  38#include "ssram_telemetry.h"
  39#include "../pmt/telemetry.h"
  40
  41/* Maximum number of modes supported by platfoms that has low power mode capability */
  42const char *pmc_lpm_modes[] = {
  43	"S0i2.0",
  44	"S0i2.1",
  45	"S0i2.2",
  46	"S0i3.0",
  47	"S0i3.1",
  48	"S0i3.2",
  49	"S0i3.3",
  50	"S0i3.4",
  51	NULL
  52};
  53
  54/* PKGC MSRs are common across Intel Core SoCs */
  55const struct pmc_bit_map msr_map[] = {
  56	{"Package C2",                  MSR_PKG_C2_RESIDENCY},
  57	{"Package C3",                  MSR_PKG_C3_RESIDENCY},
  58	{"Package C6",                  MSR_PKG_C6_RESIDENCY},
  59	{"Package C7",                  MSR_PKG_C7_RESIDENCY},
  60	{"Package C8",                  MSR_PKG_C8_RESIDENCY},
  61	{"Package C9",                  MSR_PKG_C9_RESIDENCY},
  62	{"Package C10",                 MSR_PKG_C10_RESIDENCY},
  63	{}
  64};
  65
  66static inline u32 pmc_core_reg_read(struct pmc *pmc, int reg_offset)
  67{
  68	return readl(pmc->regbase + reg_offset);
  69}
  70
  71static inline void pmc_core_reg_write(struct pmc *pmc, int reg_offset,
  72				      u32 val)
  73{
  74	writel(val, pmc->regbase + reg_offset);
  75}
  76
  77static inline u64 pmc_core_adjust_slp_s0_step(struct pmc *pmc, u32 value)
  78{
  79	/*
  80	 * ADL PCH does not have the SLP_S0 counter and LPM Residency counters are
  81	 * used as a workaround which uses 30.5 usec tick. All other client
  82	 * programs have the legacy SLP_S0 residency counter that is using the 122
  83	 * usec tick.
  84	 */
  85	const int lpm_adj_x2 = pmc->map->lpm_res_counter_step_x2;
  86
  87	if (pmc->map == &adl_reg_map)
  88		return (u64)value * GET_X2_COUNTER((u64)lpm_adj_x2);
  89	else
  90		return (u64)value * pmc->map->slp_s0_res_counter_step;
  91}
  92
  93static int set_etr3(struct pmc_dev *pmcdev)
  94{
  95	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
  96	const struct pmc_reg_map *map = pmc->map;
  97	u32 reg;
  98
  99	if (!map->etr3_offset)
 100		return -EOPNOTSUPP;
 101
 102	guard(mutex)(&pmcdev->lock);
 103
 104	/* check if CF9 is locked */
 105	reg = pmc_core_reg_read(pmc, map->etr3_offset);
 106	if (reg & ETR3_CF9LOCK)
 107		return -EACCES;
 108
 109	/* write CF9 global reset bit */
 110	reg |= ETR3_CF9GR;
 111	pmc_core_reg_write(pmc, map->etr3_offset, reg);
 112
 113	reg = pmc_core_reg_read(pmc, map->etr3_offset);
 114	if (!(reg & ETR3_CF9GR))
 115		return -EIO;
 116
 117	return 0;
 118}
 119static umode_t etr3_is_visible(struct kobject *kobj,
 120				struct attribute *attr,
 121				int idx)
 122{
 123	struct device *dev = kobj_to_dev(kobj);
 124	struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 125	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 126	const struct pmc_reg_map *map = pmc->map;
 127	u32 reg;
 128
 129	scoped_guard(mutex, &pmcdev->lock)
 130		reg = pmc_core_reg_read(pmc, map->etr3_offset);
 131
 132	return reg & ETR3_CF9LOCK ? attr->mode & (SYSFS_PREALLOC | 0444) : attr->mode;
 133}
 134
 135static ssize_t etr3_show(struct device *dev,
 136				 struct device_attribute *attr, char *buf)
 137{
 138	struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 139	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 140	const struct pmc_reg_map *map = pmc->map;
 141	u32 reg;
 142
 143	if (!map->etr3_offset)
 144		return -EOPNOTSUPP;
 145
 146	scoped_guard(mutex, &pmcdev->lock) {
 147		reg = pmc_core_reg_read(pmc, map->etr3_offset);
 148		reg &= ETR3_CF9GR | ETR3_CF9LOCK;
 149	}
 150
 151	return sysfs_emit(buf, "0x%08x", reg);
 152}
 153
 154static ssize_t etr3_store(struct device *dev,
 155				  struct device_attribute *attr,
 156				  const char *buf, size_t len)
 157{
 158	struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 159	int err;
 160	u32 reg;
 161
 162	err = kstrtouint(buf, 16, &reg);
 163	if (err)
 164		return err;
 165
 166	/* allow only CF9 writes */
 167	if (reg != ETR3_CF9GR)
 168		return -EINVAL;
 169
 170	err = set_etr3(pmcdev);
 171	if (err)
 172		return err;
 173
 174	return len;
 175}
 176static DEVICE_ATTR_RW(etr3);
 177
 178static struct attribute *pmc_attrs[] = {
 179	&dev_attr_etr3.attr,
 180	NULL
 181};
 182
 183static const struct attribute_group pmc_attr_group = {
 184	.attrs = pmc_attrs,
 185	.is_visible = etr3_is_visible,
 186};
 187
 188static const struct attribute_group *pmc_dev_groups[] = {
 189	&pmc_attr_group,
 190	NULL
 191};
 192
 193static int pmc_core_dev_state_get(void *data, u64 *val)
 194{
 195	struct pmc *pmc = data;
 196	const struct pmc_reg_map *map = pmc->map;
 197	u32 value;
 198
 199	value = pmc_core_reg_read(pmc, map->slp_s0_offset);
 200	*val = pmc_core_adjust_slp_s0_step(pmc, value);
 201
 202	return 0;
 203}
 204
 205DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_dev_state, pmc_core_dev_state_get, NULL, "%llu\n");
 206
 207static int pmc_core_pson_residency_get(void *data, u64 *val)
 208{
 209	struct pmc *pmc = data;
 210	const struct pmc_reg_map *map = pmc->map;
 211	u32 value;
 212
 213	value = pmc_core_reg_read(pmc, map->pson_residency_offset);
 214	*val = (u64)value * map->pson_residency_counter_step;
 215
 216	return 0;
 217}
 218
 219DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_pson_residency, pmc_core_pson_residency_get, NULL, "%llu\n");
 220
 221static int pmc_core_check_read_lock_bit(struct pmc *pmc)
 222{
 223	u32 value;
 224
 225	value = pmc_core_reg_read(pmc, pmc->map->pm_cfg_offset);
 226	return value & BIT(pmc->map->pm_read_disable_bit);
 227}
 228
 229static void pmc_core_slps0_display(struct pmc *pmc, struct device *dev,
 230				   struct seq_file *s)
 231{
 232	const struct pmc_bit_map **maps = pmc->map->slps0_dbg_maps;
 233	const struct pmc_bit_map *map;
 234	int offset = pmc->map->slps0_dbg_offset;
 235	u32 data;
 236
 237	while (*maps) {
 238		map = *maps;
 239		data = pmc_core_reg_read(pmc, offset);
 240		offset += 4;
 241		while (map->name) {
 242			if (dev)
 243				dev_info(dev, "SLP_S0_DBG: %-32s\tState: %s\n",
 244					map->name,
 245					data & map->bit_mask ? "Yes" : "No");
 246			if (s)
 247				seq_printf(s, "SLP_S0_DBG: %-32s\tState: %s\n",
 248					   map->name,
 249					   data & map->bit_mask ? "Yes" : "No");
 250			++map;
 251		}
 252		++maps;
 253	}
 254}
 255
 256static unsigned int pmc_core_lpm_get_arr_size(const struct pmc_bit_map **maps)
 257{
 258	unsigned int idx;
 259
 260	for (idx = 0; maps[idx]; idx++)
 261		;/* Nothing */
 262
 263	return idx;
 264}
 265
 266static void pmc_core_lpm_display(struct pmc *pmc, struct device *dev,
 267				 struct seq_file *s, u32 offset, int pmc_index,
 268				 const char *str,
 269				 const struct pmc_bit_map **maps)
 270{
 271	unsigned int index, idx, len = 32, arr_size;
 272	u32 bit_mask, *lpm_regs;
 273
 274	arr_size = pmc_core_lpm_get_arr_size(maps);
 275	lpm_regs = kmalloc_array(arr_size, sizeof(*lpm_regs), GFP_KERNEL);
 276	if (!lpm_regs)
 277		return;
 278
 279	for (index = 0; index < arr_size; index++) {
 280		lpm_regs[index] = pmc_core_reg_read(pmc, offset);
 281		offset += 4;
 282	}
 283
 284	for (idx = 0; idx < arr_size; idx++) {
 285		if (dev)
 286			dev_info(dev, "\nPMC%d:LPM_%s_%d:\t0x%x\n", pmc_index, str, idx,
 287				lpm_regs[idx]);
 288		if (s)
 289			seq_printf(s, "\nPMC%d:LPM_%s_%d:\t0x%x\n", pmc_index, str, idx,
 290				   lpm_regs[idx]);
 291		for (index = 0; maps[idx][index].name && index < len; index++) {
 292			bit_mask = maps[idx][index].bit_mask;
 293			if (dev)
 294				dev_info(dev, "PMC%d:%-30s %-30d\n", pmc_index,
 295					maps[idx][index].name,
 296					lpm_regs[idx] & bit_mask ? 1 : 0);
 297			if (s)
 298				seq_printf(s, "PMC%d:%-30s %-30d\n", pmc_index,
 299					   maps[idx][index].name,
 300					   lpm_regs[idx] & bit_mask ? 1 : 0);
 301		}
 302	}
 303
 304	kfree(lpm_regs);
 305}
 306
 307static bool slps0_dbg_latch;
 308
 309static inline u8 pmc_core_reg_read_byte(struct pmc *pmc, int offset)
 310{
 311	return readb(pmc->regbase + offset);
 312}
 313
 314static void pmc_core_display_map(struct seq_file *s, int index, int idx, int ip,
 315				 int pmc_idx, u8 pf_reg, const struct pmc_bit_map **pf_map)
 316{
 317	seq_printf(s, "PMC%d:PCH IP: %-2d - %-32s\tState: %s\n",
 318		   pmc_idx, ip, pf_map[idx][index].name,
 319		   pf_map[idx][index].bit_mask & pf_reg ? "Off" : "On");
 320}
 321
 322static int pmc_core_ppfear_show(struct seq_file *s, void *unused)
 323{
 324	struct pmc_dev *pmcdev = s->private;
 325	unsigned int pmc_idx;
 326
 327	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
 328		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
 329		const struct pmc_bit_map **maps;
 330		u8 pf_regs[PPFEAR_MAX_NUM_ENTRIES];
 331		unsigned int index, iter, idx, ip = 0;
 332
 333		if (!pmc)
 334			continue;
 335
 336		maps = pmc->map->pfear_sts;
 337		iter = pmc->map->ppfear0_offset;
 338
 339		for (index = 0; index < pmc->map->ppfear_buckets &&
 340		     index < PPFEAR_MAX_NUM_ENTRIES; index++, iter++)
 341			pf_regs[index] = pmc_core_reg_read_byte(pmc, iter);
 342
 343		for (idx = 0; maps[idx]; idx++) {
 344			for (index = 0; maps[idx][index].name &&
 345			     index < pmc->map->ppfear_buckets * 8; ip++, index++)
 346				pmc_core_display_map(s, index, idx, ip, pmc_idx,
 347						     pf_regs[index / 8], maps);
 348		}
 349	}
 350
 351	return 0;
 352}
 353DEFINE_SHOW_ATTRIBUTE(pmc_core_ppfear);
 354
 355/* This function should return link status, 0 means ready */
 356static int pmc_core_mtpmc_link_status(struct pmc *pmc)
 357{
 358	u32 value;
 359
 360	value = pmc_core_reg_read(pmc, SPT_PMC_PM_STS_OFFSET);
 361	return value & BIT(SPT_PMC_MSG_FULL_STS_BIT);
 362}
 363
 364static int pmc_core_send_msg(struct pmc *pmc, u32 *addr_xram)
 365{
 366	u32 dest;
 367	int timeout;
 368
 369	for (timeout = NUM_RETRIES; timeout > 0; timeout--) {
 370		if (pmc_core_mtpmc_link_status(pmc) == 0)
 371			break;
 372		msleep(5);
 373	}
 374
 375	if (timeout <= 0 && pmc_core_mtpmc_link_status(pmc))
 376		return -EBUSY;
 377
 378	dest = (*addr_xram & MTPMC_MASK) | (1U << 1);
 379	pmc_core_reg_write(pmc, SPT_PMC_MTPMC_OFFSET, dest);
 380	return 0;
 381}
 382
 383static int pmc_core_mphy_pg_show(struct seq_file *s, void *unused)
 384{
 385	struct pmc_dev *pmcdev = s->private;
 386	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 387	const struct pmc_bit_map *map = pmc->map->mphy_sts;
 388	u32 mphy_core_reg_low, mphy_core_reg_high;
 389	u32 val_low, val_high;
 390	unsigned int index;
 391	int err = 0;
 392
 393	if (pmcdev->pmc_xram_read_bit) {
 394		seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS.");
 395		return 0;
 396	}
 397
 398	mphy_core_reg_low  = (SPT_PMC_MPHY_CORE_STS_0 << 16);
 399	mphy_core_reg_high = (SPT_PMC_MPHY_CORE_STS_1 << 16);
 400
 401	guard(mutex)(&pmcdev->lock);
 402
 403	err = pmc_core_send_msg(pmc, &mphy_core_reg_low);
 404	if (err)
 405		return err;
 406
 407	msleep(10);
 408	val_low = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET);
 409
 410	err = pmc_core_send_msg(pmc, &mphy_core_reg_high);
 411	if (err)
 412		return err;
 413
 414	msleep(10);
 415	val_high = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET);
 416
 417	for (index = 0; index < 8 && map[index].name; index++) {
 418		seq_printf(s, "%-32s\tState: %s\n",
 419			   map[index].name,
 420			   map[index].bit_mask & val_low ? "Not power gated" :
 421			   "Power gated");
 422	}
 423
 424	for (index = 8; map[index].name; index++) {
 425		seq_printf(s, "%-32s\tState: %s\n",
 426			   map[index].name,
 427			   map[index].bit_mask & val_high ? "Not power gated" :
 428			   "Power gated");
 429	}
 430
 431	return 0;
 432}
 433DEFINE_SHOW_ATTRIBUTE(pmc_core_mphy_pg);
 434
 435static int pmc_core_pll_show(struct seq_file *s, void *unused)
 436{
 437	struct pmc_dev *pmcdev = s->private;
 438	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 439	const struct pmc_bit_map *map = pmc->map->pll_sts;
 440	u32 mphy_common_reg, val;
 441	unsigned int index;
 442	int err = 0;
 443
 444	if (pmcdev->pmc_xram_read_bit) {
 445		seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS.");
 446		return 0;
 447	}
 448
 449	mphy_common_reg  = (SPT_PMC_MPHY_COM_STS_0 << 16);
 450	guard(mutex)(&pmcdev->lock);
 451
 452	err = pmc_core_send_msg(pmc, &mphy_common_reg);
 453	if (err)
 454		return err;
 455
 456	/* Observed PMC HW response latency for MTPMC-MFPMC is ~10 ms */
 457	msleep(10);
 458	val = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET);
 459
 460	for (index = 0; map[index].name ; index++) {
 461		seq_printf(s, "%-32s\tState: %s\n",
 462			   map[index].name,
 463			   map[index].bit_mask & val ? "Active" : "Idle");
 464	}
 465
 466	return 0;
 467}
 468DEFINE_SHOW_ATTRIBUTE(pmc_core_pll);
 469
 470int pmc_core_send_ltr_ignore(struct pmc_dev *pmcdev, u32 value, int ignore)
 471{
 472	struct pmc *pmc;
 473	const struct pmc_reg_map *map;
 474	u32 reg;
 475	unsigned int pmc_idx;
 476	int ltr_index;
 477
 478	ltr_index = value;
 479	/* For platforms with multiple pmcs, ltr index value given by user
 480	 * is based on the contiguous indexes from ltr_show output.
 481	 * pmc index and ltr index needs to be calculated from it.
 482	 */
 483	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs) && ltr_index >= 0; pmc_idx++) {
 484		pmc = pmcdev->pmcs[pmc_idx];
 485
 486		if (!pmc)
 487			continue;
 488
 489		map = pmc->map;
 490		if (ltr_index <= map->ltr_ignore_max)
 491			break;
 492
 493		/* Along with IP names, ltr_show map includes CURRENT_PLATFORM
 494		 * and AGGREGATED_SYSTEM values per PMC. Take these two index
 495		 * values into account in ltr_index calculation. Also, to start
 496		 * ltr index from zero for next pmc, subtract it by 1.
 497		 */
 498		ltr_index = ltr_index - (map->ltr_ignore_max + 2) - 1;
 499	}
 500
 501	if (pmc_idx >= ARRAY_SIZE(pmcdev->pmcs) || ltr_index < 0)
 502		return -EINVAL;
 503
 504	pr_debug("ltr_ignore for pmc%d: ltr_index:%d\n", pmc_idx, ltr_index);
 505
 506	guard(mutex)(&pmcdev->lock);
 507
 508	reg = pmc_core_reg_read(pmc, map->ltr_ignore_offset);
 509	if (ignore)
 510		reg |= BIT(ltr_index);
 511	else
 512		reg &= ~BIT(ltr_index);
 513	pmc_core_reg_write(pmc, map->ltr_ignore_offset, reg);
 514
 515	return 0;
 516}
 517
 518static ssize_t pmc_core_ltr_write(struct pmc_dev *pmcdev,
 519				  const char __user *userbuf,
 520				  size_t count, int ignore)
 521{
 522	u32 value;
 523	int err;
 524
 525	err = kstrtou32_from_user(userbuf, count, 10, &value);
 526	if (err)
 527		return err;
 528
 529	err = pmc_core_send_ltr_ignore(pmcdev, value, ignore);
 530
 531	return err ?: count;
 532}
 533
 534static ssize_t pmc_core_ltr_ignore_write(struct file *file,
 535					 const char __user *userbuf,
 536					 size_t count, loff_t *ppos)
 537{
 538	struct seq_file *s = file->private_data;
 539	struct pmc_dev *pmcdev = s->private;
 540
 541	return pmc_core_ltr_write(pmcdev, userbuf, count, 1);
 542}
 543
 544static int pmc_core_ltr_ignore_show(struct seq_file *s, void *unused)
 545{
 546	return 0;
 547}
 548DEFINE_SHOW_STORE_ATTRIBUTE(pmc_core_ltr_ignore);
 549
 550static ssize_t pmc_core_ltr_restore_write(struct file *file,
 551					  const char __user *userbuf,
 552					  size_t count, loff_t *ppos)
 553{
 554	struct seq_file *s = file->private_data;
 555	struct pmc_dev *pmcdev = s->private;
 556
 557	return pmc_core_ltr_write(pmcdev, userbuf, count, 0);
 558}
 559
 560static int pmc_core_ltr_restore_show(struct seq_file *s, void *unused)
 561{
 562	return 0;
 563}
 564DEFINE_SHOW_STORE_ATTRIBUTE(pmc_core_ltr_restore);
 565
 566static void pmc_core_slps0_dbg_latch(struct pmc_dev *pmcdev, bool reset)
 567{
 568	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 569	const struct pmc_reg_map *map = pmc->map;
 570	u32 fd;
 571
 572	guard(mutex)(&pmcdev->lock);
 573
 574	if (!reset && !slps0_dbg_latch)
 575		return;
 576
 577	fd = pmc_core_reg_read(pmc, map->slps0_dbg_offset);
 578	if (reset)
 579		fd &= ~CNP_PMC_LATCH_SLPS0_EVENTS;
 580	else
 581		fd |= CNP_PMC_LATCH_SLPS0_EVENTS;
 582	pmc_core_reg_write(pmc, map->slps0_dbg_offset, fd);
 583
 584	slps0_dbg_latch = false;
 585}
 586
 587static int pmc_core_slps0_dbg_show(struct seq_file *s, void *unused)
 588{
 589	struct pmc_dev *pmcdev = s->private;
 590
 591	pmc_core_slps0_dbg_latch(pmcdev, false);
 592	pmc_core_slps0_display(pmcdev->pmcs[PMC_IDX_MAIN], NULL, s);
 593	pmc_core_slps0_dbg_latch(pmcdev, true);
 594
 595	return 0;
 596}
 597DEFINE_SHOW_ATTRIBUTE(pmc_core_slps0_dbg);
 598
 599static u32 convert_ltr_scale(u32 val)
 600{
 601	/*
 602	 * As per PCIE specification supporting document
 603	 * ECN_LatencyTolnReporting_14Aug08.pdf the Latency
 604	 * Tolerance Reporting data payload is encoded in a
 605	 * 3 bit scale and 10 bit value fields. Values are
 606	 * multiplied by the indicated scale to yield an absolute time
 607	 * value, expressible in a range from 1 nanosecond to
 608	 * 2^25*(2^10-1) = 34,326,183,936 nanoseconds.
 609	 *
 610	 * scale encoding is as follows:
 611	 *
 612	 * ----------------------------------------------
 613	 * |scale factor	|	Multiplier (ns)	|
 614	 * ----------------------------------------------
 615	 * |	0		|	1		|
 616	 * |	1		|	32		|
 617	 * |	2		|	1024		|
 618	 * |	3		|	32768		|
 619	 * |	4		|	1048576		|
 620	 * |	5		|	33554432	|
 621	 * |	6		|	Invalid		|
 622	 * |	7		|	Invalid		|
 623	 * ----------------------------------------------
 624	 */
 625	if (val > 5) {
 626		pr_warn("Invalid LTR scale factor.\n");
 627		return 0;
 628	}
 629
 630	return 1U << (5 * val);
 631}
 632
 633static int pmc_core_ltr_show(struct seq_file *s, void *unused)
 634{
 635	struct pmc_dev *pmcdev = s->private;
 636	u64 decoded_snoop_ltr, decoded_non_snoop_ltr, val;
 637	u32 ltr_raw_data, scale;
 638	u16 snoop_ltr, nonsnoop_ltr;
 639	unsigned int pmc_idx, index, ltr_index = 0;
 640
 641	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
 642		struct pmc *pmc;
 643		const struct pmc_bit_map *map;
 644		u32 ltr_ign_reg;
 645
 646		pmc = pmcdev->pmcs[pmc_idx];
 647		if (!pmc)
 648			continue;
 649
 650		scoped_guard(mutex, &pmcdev->lock)
 651			ltr_ign_reg = pmc_core_reg_read(pmc, pmc->map->ltr_ignore_offset);
 652
 653		map = pmc->map->ltr_show_sts;
 654		for (index = 0; map[index].name; index++) {
 655			bool ltr_ign_data;
 656
 657			if (index > pmc->map->ltr_ignore_max)
 658				ltr_ign_data = false;
 659			else
 660				ltr_ign_data = ltr_ign_reg & BIT(index);
 661
 662			decoded_snoop_ltr = decoded_non_snoop_ltr = 0;
 663			ltr_raw_data = pmc_core_reg_read(pmc,
 664							 map[index].bit_mask);
 665			snoop_ltr = ltr_raw_data & ~MTPMC_MASK;
 666			nonsnoop_ltr = (ltr_raw_data >> 0x10) & ~MTPMC_MASK;
 667
 668			if (FIELD_GET(LTR_REQ_NONSNOOP, ltr_raw_data)) {
 669				scale = FIELD_GET(LTR_DECODED_SCALE, nonsnoop_ltr);
 670				val = FIELD_GET(LTR_DECODED_VAL, nonsnoop_ltr);
 671				decoded_non_snoop_ltr = val * convert_ltr_scale(scale);
 672			}
 673			if (FIELD_GET(LTR_REQ_SNOOP, ltr_raw_data)) {
 674				scale = FIELD_GET(LTR_DECODED_SCALE, snoop_ltr);
 675				val = FIELD_GET(LTR_DECODED_VAL, snoop_ltr);
 676				decoded_snoop_ltr = val * convert_ltr_scale(scale);
 677			}
 678
 679			seq_printf(s, "%d\tPMC%d:%-32s\tLTR: RAW: 0x%-16x\tNon-Snoop(ns): %-16llu\tSnoop(ns): %-16llu\tLTR_IGNORE: %d\n",
 680				   ltr_index, pmc_idx, map[index].name, ltr_raw_data,
 681				   decoded_non_snoop_ltr,
 682				   decoded_snoop_ltr, ltr_ign_data);
 683			ltr_index++;
 684		}
 685	}
 686	return 0;
 687}
 688DEFINE_SHOW_ATTRIBUTE(pmc_core_ltr);
 689
 690static int pmc_core_s0ix_blocker_show(struct seq_file *s, void *unused)
 691{
 692	struct pmc_dev *pmcdev = s->private;
 693	unsigned int pmc_idx;
 694
 695	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); pmc_idx++) {
 696		const struct pmc_bit_map **maps;
 697		unsigned int arr_size, r_idx;
 698		u32 offset, counter;
 699		struct pmc *pmc;
 700
 701		pmc = pmcdev->pmcs[pmc_idx];
 702		if (!pmc)
 703			continue;
 704		maps = pmc->map->s0ix_blocker_maps;
 705		offset = pmc->map->s0ix_blocker_offset;
 706		arr_size = pmc_core_lpm_get_arr_size(maps);
 707
 708		for (r_idx = 0; r_idx < arr_size; r_idx++) {
 709			const struct pmc_bit_map *map;
 710
 711			for (map = maps[r_idx]; map->name; map++) {
 712				if (!map->blk)
 713					continue;
 714				counter = pmc_core_reg_read(pmc, offset);
 715				seq_printf(s, "PMC%d:%-30s %-30d\n", pmc_idx,
 716					   map->name, counter);
 717				offset += map->blk * S0IX_BLK_SIZE;
 718			}
 719		}
 720	}
 721	return 0;
 722}
 723DEFINE_SHOW_ATTRIBUTE(pmc_core_s0ix_blocker);
 724
 725static void pmc_core_ltr_ignore_all(struct pmc_dev *pmcdev)
 726{
 727	unsigned int pmc_idx;
 728
 729	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); pmc_idx++) {
 730		struct pmc *pmc;
 731		u32 ltr_ign;
 732
 733		pmc = pmcdev->pmcs[pmc_idx];
 734		if (!pmc)
 735			continue;
 736
 737		guard(mutex)(&pmcdev->lock);
 738		pmc->ltr_ign = pmc_core_reg_read(pmc, pmc->map->ltr_ignore_offset);
 739
 740		/* ltr_ignore_max is the max index value for LTR ignore register */
 741		ltr_ign = pmc->ltr_ign | GENMASK(pmc->map->ltr_ignore_max, 0);
 742		pmc_core_reg_write(pmc, pmc->map->ltr_ignore_offset, ltr_ign);
 743	}
 744
 745	/*
 746	 * Ignoring ME during suspend is blocking platforms with ADL PCH to get to
 747	 * deeper S0ix substate.
 748	 */
 749	pmc_core_send_ltr_ignore(pmcdev, 6, 0);
 750}
 751
 752static void pmc_core_ltr_restore_all(struct pmc_dev *pmcdev)
 753{
 754	unsigned int pmc_idx;
 755
 756	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); pmc_idx++) {
 757		struct pmc *pmc;
 758
 759		pmc = pmcdev->pmcs[pmc_idx];
 760		if (!pmc)
 761			continue;
 762
 763		guard(mutex)(&pmcdev->lock);
 764		pmc_core_reg_write(pmc, pmc->map->ltr_ignore_offset, pmc->ltr_ign);
 765	}
 766}
 767
 768static inline u64 adjust_lpm_residency(struct pmc *pmc, u32 offset,
 769				       const int lpm_adj_x2)
 770{
 771	u64 lpm_res = pmc_core_reg_read(pmc, offset);
 772
 773	return GET_X2_COUNTER((u64)lpm_adj_x2 * lpm_res);
 774}
 775
 776static int pmc_core_substate_res_show(struct seq_file *s, void *unused)
 777{
 778	struct pmc_dev *pmcdev = s->private;
 779	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 780	const int lpm_adj_x2 = pmc->map->lpm_res_counter_step_x2;
 781	u32 offset = pmc->map->lpm_residency_offset;
 782	int mode;
 783
 784	seq_printf(s, "%-10s %-15s\n", "Substate", "Residency");
 785
 786	pmc_for_each_mode(mode, pmcdev) {
 787		seq_printf(s, "%-10s %-15llu\n", pmc_lpm_modes[mode],
 788			   adjust_lpm_residency(pmc, offset + (4 * mode), lpm_adj_x2));
 789	}
 790
 791	return 0;
 792}
 793DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_res);
 794
 795static int pmc_core_substate_sts_regs_show(struct seq_file *s, void *unused)
 796{
 797	struct pmc_dev *pmcdev = s->private;
 798	unsigned int pmc_idx;
 799
 800	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
 801		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
 802		const struct pmc_bit_map **maps;
 803		u32 offset;
 804
 805		if (!pmc)
 806			continue;
 807		maps = pmc->map->lpm_sts;
 808		offset = pmc->map->lpm_status_offset;
 809		pmc_core_lpm_display(pmc, NULL, s, offset, pmc_idx, "STATUS", maps);
 810	}
 811
 812	return 0;
 813}
 814DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_sts_regs);
 815
 816static int pmc_core_substate_l_sts_regs_show(struct seq_file *s, void *unused)
 817{
 818	struct pmc_dev *pmcdev = s->private;
 819	unsigned int pmc_idx;
 820
 821	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
 822		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
 823		const struct pmc_bit_map **maps;
 824		u32 offset;
 825
 826		if (!pmc)
 827			continue;
 828		maps = pmc->map->lpm_sts;
 829		offset = pmc->map->lpm_live_status_offset;
 830		pmc_core_lpm_display(pmc, NULL, s, offset, pmc_idx, "LIVE_STATUS", maps);
 831	}
 832
 833	return 0;
 834}
 835DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_l_sts_regs);
 836
 837static void pmc_core_substate_req_header_show(struct seq_file *s, int pmc_index,
 838					      enum header_type type)
 839{
 840	struct pmc_dev *pmcdev = s->private;
 841	int mode;
 842
 843	seq_printf(s, "%40s |", "Element");
 844	pmc_for_each_mode(mode, pmcdev)
 845		seq_printf(s, " %9s |", pmc_lpm_modes[mode]);
 846
 847	if (type == HEADER_STATUS) {
 848		seq_printf(s, " %9s |", "Status");
 849		seq_printf(s, " %11s |\n", "Live Status");
 850	} else {
 851		seq_printf(s, " %9s |\n", "Value");
 852	}
 853}
 854
 855static int pmc_core_substate_blk_req_show(struct seq_file *s, void *unused)
 856{
 857	struct pmc_dev *pmcdev = s->private;
 858	unsigned int pmc_idx;
 859
 860	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); pmc_idx++) {
 861		const struct pmc_bit_map **maps;
 862		unsigned int arr_size, r_idx;
 863		u32 offset, counter;
 864		u32 *lpm_req_regs;
 865		struct pmc *pmc;
 866
 867		pmc = pmcdev->pmcs[pmc_idx];
 868		if (!pmc || !pmc->lpm_req_regs)
 869			continue;
 870
 871		lpm_req_regs = pmc->lpm_req_regs;
 872		maps = pmc->map->s0ix_blocker_maps;
 873		offset = pmc->map->s0ix_blocker_offset;
 874		arr_size = pmc_core_lpm_get_arr_size(maps);
 875
 876		/* Display the header */
 877		pmc_core_substate_req_header_show(s, pmc_idx, HEADER_VALUE);
 878
 879		for (r_idx = 0; r_idx < arr_size; r_idx++) {
 880			const struct pmc_bit_map *map;
 881
 882			for (map = maps[r_idx]; map->name; map++) {
 883				int mode;
 884
 885				if (!map->blk)
 886					continue;
 887
 888				counter = pmc_core_reg_read(pmc, offset);
 889				seq_printf(s, "pmc%u: %34s |", pmc_idx, map->name);
 890				pmc_for_each_mode(mode, pmcdev) {
 891					bool required = *lpm_req_regs & BIT(mode);
 892
 893					seq_printf(s, " %9s |", required ? "Required" : " ");
 894				}
 895				seq_printf(s, " %9u |\n", counter);
 896				offset += map->blk * S0IX_BLK_SIZE;
 897				lpm_req_regs++;
 898			}
 899		}
 900	}
 901	return 0;
 902}
 903
 904static int pmc_core_substate_blk_req_open(struct inode *inode, struct file *file)
 905{
 906	return single_open(file, pmc_core_substate_blk_req_show, inode->i_private);
 907}
 908
 909const struct file_operations pmc_core_substate_blk_req_fops = {
 910	.owner		= THIS_MODULE,
 911	.open		= pmc_core_substate_blk_req_open,
 912	.read		= seq_read,
 913	.llseek		= seq_lseek,
 914	.release	= single_release,
 915};
 916
 917static int pmc_core_substate_req_regs_show(struct seq_file *s, void *unused)
 918{
 919	struct pmc_dev *pmcdev = s->private;
 920	u32 sts_offset;
 921	u32 sts_offset_live;
 922	u32 *lpm_req_regs;
 923	unsigned int mp, pmc_idx;
 924	int num_maps;
 925
 926	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
 927		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
 928		const struct pmc_bit_map **maps;
 929
 930		if (!pmc)
 931			continue;
 932
 933		maps = pmc->map->lpm_sts;
 934		num_maps = pmc->map->lpm_num_maps;
 935		sts_offset = pmc->map->lpm_status_offset;
 936		sts_offset_live = pmc->map->lpm_live_status_offset;
 937		lpm_req_regs = pmc->lpm_req_regs;
 938
 939		/*
 940		 * When there are multiple PMCs, though the PMC may exist, the
 941		 * requirement register discovery could have failed so check
 942		 * before accessing.
 943		 */
 944		if (!lpm_req_regs)
 945			continue;
 946
 947		/* Display the header */
 948		pmc_core_substate_req_header_show(s, pmc_idx, HEADER_STATUS);
 949
 950		/* Loop over maps */
 951		for (mp = 0; mp < num_maps; mp++) {
 952			u32 req_mask = 0;
 953			u32 lpm_status;
 954			u32 lpm_status_live;
 955			const struct pmc_bit_map *map;
 956			int mode, i, len = 32;
 957
 958			/*
 959			 * Capture the requirements and create a mask so that we only
 960			 * show an element if it's required for at least one of the
 961			 * enabled low power modes
 962			 */
 963			pmc_for_each_mode(mode, pmcdev)
 964				req_mask |= lpm_req_regs[mp + (mode * num_maps)];
 965
 966			/* Get the last latched status for this map */
 967			lpm_status = pmc_core_reg_read(pmc, sts_offset + (mp * 4));
 968
 969			/* Get the runtime status for this map */
 970			lpm_status_live = pmc_core_reg_read(pmc, sts_offset_live + (mp * 4));
 971
 972			/*  Loop over elements in this map */
 973			map = maps[mp];
 974			for (i = 0; map[i].name && i < len; i++) {
 975				u32 bit_mask = map[i].bit_mask;
 976
 977				if (!(bit_mask & req_mask)) {
 978					/*
 979					 * Not required for any enabled states
 980					 * so don't display
 981					 */
 982					continue;
 983				}
 984
 985				/* Display the element name in the first column */
 986				seq_printf(s, "pmc%d: %34s |", pmc_idx, map[i].name);
 987
 988				/* Loop over the enabled states and display if required */
 989				pmc_for_each_mode(mode, pmcdev) {
 990					bool required = lpm_req_regs[mp + (mode * num_maps)] &
 991							bit_mask;
 992					seq_printf(s, " %9s |", required ? "Required" : " ");
 993				}
 994
 995				/* In Status column, show the last captured state of this agent */
 996				seq_printf(s, " %9s |", lpm_status & bit_mask ? "Yes" : " ");
 997
 998				/* In Live status column, show the live state of this agent */
 999				seq_printf(s, " %11s |", lpm_status_live & bit_mask ? "Yes" : " ");
1000
1001				seq_puts(s, "\n");
1002			}
1003		}
1004	}
1005	return 0;
1006}
1007
1008static int pmc_core_substate_req_regs_open(struct inode *inode, struct file *file)
1009{
1010	return single_open(file, pmc_core_substate_req_regs_show, inode->i_private);
1011}
1012
1013const struct file_operations pmc_core_substate_req_regs_fops = {
1014	.owner		= THIS_MODULE,
1015	.open		= pmc_core_substate_req_regs_open,
1016	.read		= seq_read,
1017	.llseek		= seq_lseek,
1018	.release	= single_release,
1019};
1020
1021static unsigned int pmc_core_get_crystal_freq(void)
1022{
1023	unsigned int eax_denominator, ebx_numerator, ecx_hz, edx;
1024
1025	if (boot_cpu_data.cpuid_level < CPUID_LEAF_TSC)
1026		return 0;
1027
1028	eax_denominator = ebx_numerator = ecx_hz = edx = 0;
1029
1030	/* TSC/Crystal ratio, plus optionally Crystal Hz */
1031	cpuid(CPUID_LEAF_TSC, &eax_denominator, &ebx_numerator, &ecx_hz, &edx);
1032
1033	if (ebx_numerator == 0 || eax_denominator == 0)
1034		return 0;
1035
1036	return ecx_hz;
1037}
1038
1039static int pmc_core_die_c6_us_show(struct seq_file *s, void *unused)
1040{
1041	struct pmc_dev *pmcdev = s->private;
1042	u64 die_c6_res, count;
1043	int ret;
1044
1045	if (!pmcdev->crystal_freq) {
1046		dev_warn_once(&pmcdev->pdev->dev, "Crystal frequency unavailable\n");
1047		return -ENXIO;
1048	}
1049
1050	ret = pmt_telem_read(pmcdev->punit_ep, pmcdev->die_c6_offset,
1051			     &count, 1);
1052	if (ret)
1053		return ret;
1054
1055	die_c6_res = div64_u64(count * HZ_PER_MHZ, pmcdev->crystal_freq);
1056	seq_printf(s, "%llu\n", die_c6_res);
1057
1058	return 0;
1059}
1060DEFINE_SHOW_ATTRIBUTE(pmc_core_die_c6_us);
1061
1062static int pmc_core_lpm_latch_mode_show(struct seq_file *s, void *unused)
1063{
1064	struct pmc_dev *pmcdev = s->private;
1065	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1066	bool c10;
1067	u32 reg;
1068	int mode;
1069
1070	reg = pmc_core_reg_read(pmc, pmc->map->lpm_sts_latch_en_offset);
1071	if (reg & LPM_STS_LATCH_MODE) {
1072		seq_puts(s, "c10");
1073		c10 = false;
1074	} else {
1075		seq_puts(s, "[c10]");
1076		c10 = true;
1077	}
1078
1079	pmc_for_each_mode(mode, pmcdev) {
1080		if ((BIT(mode) & reg) && !c10)
1081			seq_printf(s, " [%s]", pmc_lpm_modes[mode]);
1082		else
1083			seq_printf(s, " %s", pmc_lpm_modes[mode]);
1084	}
1085
1086	seq_puts(s, " clear\n");
1087
1088	return 0;
1089}
1090
1091static ssize_t pmc_core_lpm_latch_mode_write(struct file *file,
1092					     const char __user *userbuf,
1093					     size_t count, loff_t *ppos)
1094{
1095	struct seq_file *s = file->private_data;
1096	struct pmc_dev *pmcdev = s->private;
1097	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1098	bool clear = false, c10 = false;
1099	unsigned char buf[8];
1100	int m, mode;
1101	u32 reg;
1102
1103	if (count > sizeof(buf) - 1)
1104		return -EINVAL;
1105	if (copy_from_user(buf, userbuf, count))
1106		return -EFAULT;
1107	buf[count] = '\0';
1108
1109	/*
1110	 * Allowed strings are:
1111	 *	Any enabled substate, e.g. 'S0i2.0'
1112	 *	'c10'
1113	 *	'clear'
1114	 */
1115	mode = sysfs_match_string(pmc_lpm_modes, buf);
1116
1117	/* Check string matches enabled mode */
1118	pmc_for_each_mode(m, pmcdev)
1119		if (mode == m)
1120			break;
1121
1122	if (mode != m || mode < 0) {
1123		if (sysfs_streq(buf, "clear"))
1124			clear = true;
1125		else if (sysfs_streq(buf, "c10"))
1126			c10 = true;
1127		else
1128			return -EINVAL;
1129	}
1130
1131	if (clear) {
1132		guard(mutex)(&pmcdev->lock);
1133
1134		reg = pmc_core_reg_read(pmc, pmc->map->etr3_offset);
1135		reg |= ETR3_CLEAR_LPM_EVENTS;
1136		pmc_core_reg_write(pmc, pmc->map->etr3_offset, reg);
1137
1138		return count;
1139	}
1140
1141	if (c10) {
1142		guard(mutex)(&pmcdev->lock);
1143
1144		reg = pmc_core_reg_read(pmc, pmc->map->lpm_sts_latch_en_offset);
1145		reg &= ~LPM_STS_LATCH_MODE;
1146		pmc_core_reg_write(pmc, pmc->map->lpm_sts_latch_en_offset, reg);
1147
1148		return count;
1149	}
1150
1151	/*
1152	 * For LPM mode latching we set the latch enable bit and selected mode
1153	 * and clear everything else.
1154	 */
1155	reg = LPM_STS_LATCH_MODE | BIT(mode);
1156	guard(mutex)(&pmcdev->lock);
1157	pmc_core_reg_write(pmc, pmc->map->lpm_sts_latch_en_offset, reg);
1158
1159	return count;
1160}
1161DEFINE_PMC_CORE_ATTR_WRITE(pmc_core_lpm_latch_mode);
1162
1163static int pmc_core_pkgc_show(struct seq_file *s, void *unused)
1164{
1165	struct pmc *pmc = s->private;
1166	const struct pmc_bit_map *map = pmc->map->msr_sts;
1167	u64 pcstate_count;
1168	unsigned int index;
1169
1170	for (index = 0; map[index].name ; index++) {
1171		if (rdmsrq_safe(map[index].bit_mask, &pcstate_count))
1172			continue;
1173
1174		pcstate_count *= 1000;
1175		do_div(pcstate_count, tsc_khz);
1176		seq_printf(s, "%-8s : %llu\n", map[index].name,
1177			   pcstate_count);
1178	}
1179
1180	return 0;
1181}
1182DEFINE_SHOW_ATTRIBUTE(pmc_core_pkgc);
1183
1184static bool pmc_core_pri_verify(u32 lpm_pri, u8 *mode_order)
1185{
1186	unsigned int i, j;
1187
1188	if (!lpm_pri)
1189		return false;
1190	/*
1191	 * Each byte contains the priority level for 2 modes (7:4 and 3:0).
1192	 * In a 32 bit register this allows for describing 8 modes. Store the
1193	 * levels and look for values out of range.
1194	 */
1195	for (i = 0; i < 8; i++) {
1196		int level = lpm_pri & GENMASK(3, 0);
1197
1198		if (level >= LPM_MAX_NUM_MODES)
1199			return false;
1200
1201		mode_order[i] = level;
1202		lpm_pri >>= 4;
1203	}
1204
1205	/* Check that we have unique values */
1206	for (i = 0; i < LPM_MAX_NUM_MODES - 1; i++)
1207		for (j = i + 1; j < LPM_MAX_NUM_MODES; j++)
1208			if (mode_order[i] == mode_order[j])
1209				return false;
1210
1211	return true;
1212}
1213
1214void pmc_core_get_low_power_modes(struct pmc_dev *pmcdev)
1215{
1216	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1217	u8 pri_order[LPM_MAX_NUM_MODES] = LPM_DEFAULT_PRI;
1218	u8 mode_order[LPM_MAX_NUM_MODES];
1219	u32 lpm_pri;
1220	u32 lpm_en;
1221	unsigned int i;
1222	int mode, p;
1223
1224	/* Use LPM Maps to indicate support for substates */
1225	if (!pmc->map->lpm_num_maps)
1226		return;
1227
1228	lpm_en = pmc_core_reg_read(pmc, pmc->map->lpm_en_offset);
1229	/* For MTL, BIT 31 is not an lpm mode but a enable bit.
1230	 * Lower byte is enough to cover the number of lpm modes for all
1231	 * platforms and hence mask the upper 3 bytes.
1232	 */
1233	pmcdev->num_lpm_modes = hweight32(lpm_en & 0xFF);
1234
1235	/* Read 32 bit LPM_PRI register */
1236	lpm_pri = pmc_core_reg_read(pmc, pmc->map->lpm_priority_offset);
1237
1238
1239	/*
1240	 * If lpm_pri value passes verification, then override the default
1241	 * modes here. Otherwise stick with the default.
1242	 */
1243	if (pmc_core_pri_verify(lpm_pri, mode_order))
1244		/* Get list of modes in priority order */
1245		for (mode = 0; mode < LPM_MAX_NUM_MODES; mode++)
1246			pri_order[mode_order[mode]] = mode;
1247	else
1248		dev_dbg(&pmcdev->pdev->dev,
1249			 "Assuming a default substate order for this platform\n");
1250
1251	/*
1252	 * Loop through all modes from lowest to highest priority,
1253	 * and capture all enabled modes in order
1254	 */
1255	i = 0;
1256	for (p = LPM_MAX_NUM_MODES - 1; p >= 0; p--) {
1257		int mode = pri_order[p];
1258
1259		if (!(BIT(mode) & lpm_en))
1260			continue;
1261
1262		pmcdev->lpm_en_modes[i++] = mode;
1263	}
1264}
1265
1266int get_primary_reg_base(struct pmc *pmc)
1267{
1268	u64 slp_s0_addr;
1269
1270	if (lpit_read_residency_count_address(&slp_s0_addr)) {
1271		pmc->base_addr = PMC_BASE_ADDR_DEFAULT;
1272
1273		if (page_is_ram(PHYS_PFN(pmc->base_addr)))
1274			return -ENODEV;
1275	} else {
1276		pmc->base_addr = slp_s0_addr - pmc->map->slp_s0_offset;
1277	}
1278
1279	pmc->regbase = ioremap(pmc->base_addr, pmc->map->regmap_length);
1280	if (!pmc->regbase)
1281		return -ENOMEM;
1282	return 0;
1283}
1284
1285static struct telem_endpoint *pmc_core_register_endpoint(struct pci_dev *pcidev, u32 *guids)
1286{
1287	struct telem_endpoint *ep;
1288	unsigned int i;
1289
1290	for (i = 0; guids[i]; i++) {
1291		ep = pmt_telem_find_and_register_endpoint(pcidev, guids[i], 0);
1292		if (!IS_ERR(ep))
1293			return ep;
1294	}
1295	return ERR_PTR(-ENODEV);
1296}
1297
1298void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, u32 *guids)
1299{
1300	struct telem_endpoint *ep;
1301	struct pci_dev *pcidev;
1302
1303	pcidev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(10, 0));
1304	if (!pcidev) {
1305		dev_err(&pmcdev->pdev->dev, "PUNIT PMT device not found.");
1306		return;
1307	}
1308
1309	ep = pmc_core_register_endpoint(pcidev, guids);
1310	pci_dev_put(pcidev);
1311	if (IS_ERR(ep)) {
1312		dev_err(&pmcdev->pdev->dev,
1313			"pmc_core: couldn't get DMU telem endpoint %ld",
1314			PTR_ERR(ep));
1315		return;
1316	}
1317
1318	pmcdev->punit_ep = ep;
1319	pmcdev->die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET;
1320}
1321
1322void pmc_core_set_device_d3(unsigned int device)
1323{
1324	struct pci_dev *pcidev;
1325
1326	pcidev = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1327	if (pcidev) {
1328		if (!device_trylock(&pcidev->dev)) {
1329			pci_dev_put(pcidev);
1330			return;
1331		}
1332		if (!pcidev->dev.driver) {
1333			dev_info(&pcidev->dev, "Setting to D3hot\n");
1334			pci_set_power_state(pcidev, PCI_D3hot);
1335		}
1336		device_unlock(&pcidev->dev);
1337		pci_dev_put(pcidev);
1338	}
1339}
1340
1341static bool pmc_core_is_pson_residency_enabled(struct pmc_dev *pmcdev)
1342{
1343	struct platform_device *pdev = pmcdev->pdev;
1344	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
1345	u8 val;
1346
1347	if (!adev)
1348		return false;
1349
1350	if (fwnode_property_read_u8(acpi_fwnode_handle(adev),
1351				    "intel-cec-pson-switching-enabled-in-s0",
1352				    &val))
1353		return false;
1354
1355	return val == 1;
1356}
1357
1358static void pmc_core_dbgfs_unregister(struct pmc_dev *pmcdev)
1359{
1360	debugfs_remove_recursive(pmcdev->dbgfs_dir);
1361}
1362
1363static void pmc_core_dbgfs_register(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
1364{
1365	struct pmc *primary_pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1366	struct dentry *dir;
1367
1368	dir = debugfs_create_dir("pmc_core", NULL);
1369	pmcdev->dbgfs_dir = dir;
1370
1371	debugfs_create_file("slp_s0_residency_usec", 0444, dir, primary_pmc,
1372			    &pmc_core_dev_state);
1373
1374	if (primary_pmc->map->pfear_sts)
1375		debugfs_create_file("pch_ip_power_gating_status", 0444, dir,
1376				    pmcdev, &pmc_core_ppfear_fops);
1377
1378	debugfs_create_file("ltr_ignore", 0644, dir, pmcdev,
1379			    &pmc_core_ltr_ignore_fops);
1380
1381	debugfs_create_file("ltr_restore", 0200, dir, pmcdev, &pmc_core_ltr_restore_fops);
1382
1383	debugfs_create_file("ltr_show", 0444, dir, pmcdev, &pmc_core_ltr_fops);
1384
1385	if (primary_pmc->map->s0ix_blocker_maps)
1386		debugfs_create_file("s0ix_blocker", 0444, dir, pmcdev, &pmc_core_s0ix_blocker_fops);
1387
1388	debugfs_create_file("package_cstate_show", 0444, dir, primary_pmc,
1389			    &pmc_core_pkgc_fops);
1390
1391	if (primary_pmc->map->pll_sts)
1392		debugfs_create_file("pll_status", 0444, dir, pmcdev,
1393				    &pmc_core_pll_fops);
1394
1395	if (primary_pmc->map->mphy_sts)
1396		debugfs_create_file("mphy_core_lanes_power_gating_status",
1397				    0444, dir, pmcdev,
1398				    &pmc_core_mphy_pg_fops);
1399
1400	if (primary_pmc->map->slps0_dbg_maps) {
1401		debugfs_create_file("slp_s0_debug_status", 0444,
1402				    dir, pmcdev,
1403				    &pmc_core_slps0_dbg_fops);
1404
1405		debugfs_create_bool("slp_s0_dbg_latch", 0644,
1406				    dir, &slps0_dbg_latch);
1407	}
1408
1409	if (primary_pmc->map->lpm_en_offset) {
1410		debugfs_create_file("substate_residencies", 0444,
1411				    pmcdev->dbgfs_dir, pmcdev,
1412				    &pmc_core_substate_res_fops);
1413	}
1414
1415	if (primary_pmc->map->lpm_status_offset) {
1416		debugfs_create_file("substate_status_registers", 0444,
1417				    pmcdev->dbgfs_dir, pmcdev,
1418				    &pmc_core_substate_sts_regs_fops);
1419		debugfs_create_file("substate_live_status_registers", 0444,
1420				    pmcdev->dbgfs_dir, pmcdev,
1421				    &pmc_core_substate_l_sts_regs_fops);
1422		debugfs_create_file("lpm_latch_mode", 0644,
1423				    pmcdev->dbgfs_dir, pmcdev,
1424				    &pmc_core_lpm_latch_mode_fops);
1425	}
1426
1427	if (primary_pmc->lpm_req_regs) {
1428		debugfs_create_file("substate_requirements", 0444,
1429				    pmcdev->dbgfs_dir, pmcdev,
1430				    pmc_dev_info->sub_req_show);
1431	}
1432
1433	if (primary_pmc->map->pson_residency_offset && pmc_core_is_pson_residency_enabled(pmcdev)) {
1434		debugfs_create_file("pson_residency_usec", 0444,
1435				    pmcdev->dbgfs_dir, primary_pmc, &pmc_core_pson_residency);
1436	}
1437
1438	if (pmcdev->punit_ep) {
1439		debugfs_create_file("die_c6_us_show", 0444,
1440				    pmcdev->dbgfs_dir, pmcdev,
1441				    &pmc_core_die_c6_us_fops);
1442	}
1443}
1444
1445/*
1446 * This function retrieves low power mode requirement data from PMC Low
1447 * Power Mode (LPM) table.
1448 *
1449 * In telemetry space, the LPM table contains a 4 byte header followed
1450 * by 8 consecutive mode blocks (one for each LPM mode). Each block
1451 * has a 4 byte header followed by a set of registers that describe the
1452 * IP state requirements for the given mode. The IP mapping is platform
1453 * specific but the same for each block, making for easy analysis.
1454 * Platforms only use a subset of the space to track the requirements
1455 * for their IPs. Callers provide the requirement registers they use as
1456 * a list of indices. Each requirement register is associated with an
1457 * IP map that's maintained by the caller.
1458 *
1459 * Header
1460 * +----+----------------------------+----------------------------+
1461 * |  0 |      REVISION              |      ENABLED MODES         |
1462 * +----+--------------+-------------+-------------+--------------+
1463 *
1464 * Low Power Mode 0 Block
1465 * +----+--------------+-------------+-------------+--------------+
1466 * |  1 |     SUB ID   |     SIZE    |   MAJOR     |   MINOR      |
1467 * +----+--------------+-------------+-------------+--------------+
1468 * |  2 |           LPM0 Requirements 0                           |
1469 * +----+---------------------------------------------------------+
1470 * |    |                  ...                                    |
1471 * +----+---------------------------------------------------------+
1472 * | 29 |           LPM0 Requirements 27                          |
1473 * +----+---------------------------------------------------------+
1474 *
1475 * ...
1476 *
1477 * Low Power Mode 7 Block
1478 * +----+--------------+-------------+-------------+--------------+
1479 * |    |     SUB ID   |     SIZE    |   MAJOR     |   MINOR      |
1480 * +----+--------------+-------------+-------------+--------------+
1481 * | 60 |           LPM7 Requirements 0                           |
1482 * +----+---------------------------------------------------------+
1483 * |    |                  ...                                    |
1484 * +----+---------------------------------------------------------+
1485 * | 87 |           LPM7 Requirements 27                          |
1486 * +----+---------------------------------------------------------+
1487 *
1488 */
1489int pmc_core_pmt_get_lpm_req(struct pmc_dev *pmcdev, struct pmc *pmc, struct telem_endpoint *ep)
1490{
1491	const u8 *lpm_indices;
1492	int num_maps, mode_offset = 0;
1493	int ret, mode;
1494	int lpm_size;
1495
1496	lpm_indices = pmc->map->lpm_reg_index;
1497	num_maps = pmc->map->lpm_num_maps;
1498	lpm_size = LPM_MAX_NUM_MODES * num_maps;
1499
1500	pmc->lpm_req_regs = devm_kzalloc(&pmcdev->pdev->dev,
1501					 lpm_size * sizeof(u32),
1502					 GFP_KERNEL);
1503	if (!pmc->lpm_req_regs)
1504		return -ENOMEM;
1505
1506	mode_offset = LPM_HEADER_OFFSET + LPM_MODE_OFFSET;
1507	pmc_for_each_mode(mode, pmcdev) {
1508		u32 *req_offset = pmc->lpm_req_regs + (mode * num_maps);
1509		int m;
1510
1511		for (m = 0; m < num_maps; m++) {
1512			u8 sample_id = lpm_indices[m] + mode_offset;
1513
1514			ret = pmt_telem_read32(ep, sample_id, req_offset, 1);
1515			if (ret) {
1516				dev_err(&pmcdev->pdev->dev,
1517					"couldn't read Low Power Mode requirements: %d\n", ret);
1518				return ret;
1519			}
1520			++req_offset;
1521		}
1522		mode_offset += LPM_REG_COUNT + LPM_MODE_OFFSET;
1523	}
1524	return ret;
1525}
1526
1527int pmc_core_pmt_get_blk_sub_req(struct pmc_dev *pmcdev, struct pmc *pmc,
1528				 struct telem_endpoint *ep)
1529{
1530	u32 num_blocker, sample_offset;
1531	unsigned int index;
1532	u32 *req_offset;
1533	int ret;
1534
1535	num_blocker = pmc->map->num_s0ix_blocker;
1536	sample_offset = pmc->map->blocker_req_offset;
1537
1538	pmc->lpm_req_regs = devm_kcalloc(&pmcdev->pdev->dev, num_blocker,
1539					 sizeof(u32), GFP_KERNEL);
1540	if (!pmc->lpm_req_regs)
1541		return -ENOMEM;
1542
1543	req_offset = pmc->lpm_req_regs;
1544	for (index = 0; index < num_blocker; index++, req_offset++) {
1545		ret = pmt_telem_read32(ep, index + sample_offset, req_offset, 1);
1546		if (ret) {
1547			dev_err(&pmcdev->pdev->dev,
1548				"couldn't read Low Power Mode requirements: %d\n", ret);
1549			return ret;
1550		}
1551	}
1552	return 0;
1553}
1554
1555static int pmc_core_get_telem_info(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
1556{
1557	struct pci_dev *pcidev __free(pci_dev_put) = NULL;
1558	struct telem_endpoint *ep;
1559	unsigned int pmc_idx;
1560	int ret;
1561
1562	pcidev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(20, pmc_dev_info->pci_func));
1563	if (!pcidev)
1564		return -ENODEV;
1565
1566	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
1567		struct pmc *pmc;
1568
1569		pmc = pmcdev->pmcs[pmc_idx];
1570		if (!pmc)
1571			continue;
1572
1573		if (!pmc->map->lpm_req_guid)
1574			return -ENXIO;
1575
1576		ep = pmt_telem_find_and_register_endpoint(pcidev, pmc->map->lpm_req_guid, 0);
1577		if (IS_ERR(ep)) {
1578			dev_dbg(&pmcdev->pdev->dev, "couldn't get telem endpoint %pe", ep);
1579			return -EPROBE_DEFER;
1580		}
1581
1582		ret = pmc_dev_info->sub_req(pmcdev, pmc, ep);
1583		pmt_telem_unregister_endpoint(ep);
1584		if (ret)
1585			return ret;
1586	}
1587
1588	return 0;
1589}
1590
1591static const struct pmc_reg_map *pmc_core_find_regmap(struct pmc_info *list, u16 devid)
1592{
1593	for (; list->map; ++list)
1594		if (devid == list->devid)
1595			return list->map;
1596
1597	return NULL;
1598}
1599
1600static int pmc_core_pmc_add(struct pmc_dev *pmcdev, unsigned int pmc_idx)
1601
1602{
1603	struct pmc_ssram_telemetry pmc_ssram_telemetry;
1604	const struct pmc_reg_map *map;
1605	struct pmc *pmc;
1606	int ret;
1607
1608	ret = pmc_ssram_telemetry_get_pmc_info(pmc_idx, &pmc_ssram_telemetry);
1609	if (ret)
1610		return ret;
1611
1612	map = pmc_core_find_regmap(pmcdev->regmap_list, pmc_ssram_telemetry.devid);
1613	if (!map)
1614		return -ENODEV;
1615
1616	pmc = pmcdev->pmcs[pmc_idx];
1617	/* Memory for primary PMC has been allocated */
1618	if (!pmc) {
1619		pmc = devm_kzalloc(&pmcdev->pdev->dev, sizeof(*pmc), GFP_KERNEL);
1620		if (!pmc)
1621			return -ENOMEM;
1622	}
1623
1624	pmc->map = map;
1625	pmc->base_addr = pmc_ssram_telemetry.base_addr;
1626	pmc->regbase = ioremap(pmc->base_addr, pmc->map->regmap_length);
1627
1628	if (!pmc->regbase) {
1629		devm_kfree(&pmcdev->pdev->dev, pmc);
1630		return -ENOMEM;
1631	}
1632
1633	pmcdev->pmcs[pmc_idx] = pmc;
1634
1635	return 0;
1636}
1637
1638static int pmc_core_ssram_get_reg_base(struct pmc_dev *pmcdev)
1639{
1640	int ret;
1641
1642	ret = pmc_core_pmc_add(pmcdev, PMC_IDX_MAIN);
1643	if (ret)
1644		return ret;
1645
1646	pmc_core_pmc_add(pmcdev, PMC_IDX_IOE);
1647	pmc_core_pmc_add(pmcdev, PMC_IDX_PCH);
1648
1649	return 0;
1650}
1651
1652/*
1653 * When supported, ssram init is used to achieve all available PMCs.
1654 * If ssram init fails, this function uses legacy method to at least get the
1655 * primary PMC.
1656 */
1657int generic_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
1658{
1659	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1660	bool ssram;
1661	int ret;
1662
1663	pmcdev->suspend = pmc_dev_info->suspend;
1664	pmcdev->resume = pmc_dev_info->resume;
1665
1666	ssram = pmc_dev_info->regmap_list != NULL;
1667	if (ssram) {
1668		pmcdev->regmap_list = pmc_dev_info->regmap_list;
1669		ret = pmc_core_ssram_get_reg_base(pmcdev);
1670		/*
1671		 * EAGAIN error code indicates Intel PMC SSRAM Telemetry driver
1672		 * has not finished probe and PMC info is not available yet. Try
1673		 * again later.
1674		 */
1675		if (ret == -EAGAIN)
1676			return -EPROBE_DEFER;
1677
1678		if (ret) {
1679			dev_warn(&pmcdev->pdev->dev,
1680				 "Failed to get PMC info from SSRAM, %d, using legacy init\n", ret);
1681			ssram = false;
1682		}
1683	}
1684
1685	if (!ssram) {
1686		pmc->map = pmc_dev_info->map;
1687		ret = get_primary_reg_base(pmc);
1688		if (ret)
1689			return ret;
1690	}
1691
1692	pmc_core_get_low_power_modes(pmcdev);
1693	if (pmc_dev_info->dmu_guids)
1694		pmc_core_punit_pmt_init(pmcdev, pmc_dev_info->dmu_guids);
1695
1696	if (ssram) {
1697		ret = pmc_core_get_telem_info(pmcdev, pmc_dev_info);
1698		if (ret)
1699			goto unmap_regbase;
1700	}
1701
1702	return 0;
1703
1704unmap_regbase:
1705	for (unsigned int pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
1706		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
1707
1708		if (pmc && pmc->regbase)
1709			iounmap(pmc->regbase);
1710	}
1711
1712	if (pmcdev->punit_ep)
1713		pmt_telem_unregister_endpoint(pmcdev->punit_ep);
1714
1715	return ret;
1716}
1717
1718static const struct x86_cpu_id intel_pmc_core_ids[] = {
1719	X86_MATCH_VFM(INTEL_SKYLAKE_L,		&spt_pmc_dev),
1720	X86_MATCH_VFM(INTEL_SKYLAKE,		&spt_pmc_dev),
1721	X86_MATCH_VFM(INTEL_KABYLAKE_L,		&spt_pmc_dev),
1722	X86_MATCH_VFM(INTEL_KABYLAKE,		&spt_pmc_dev),
1723	X86_MATCH_VFM(INTEL_CANNONLAKE_L,	&cnp_pmc_dev),
1724	X86_MATCH_VFM(INTEL_ICELAKE_L,		&icl_pmc_dev),
1725	X86_MATCH_VFM(INTEL_ICELAKE_NNPI,	&icl_pmc_dev),
1726	X86_MATCH_VFM(INTEL_COMETLAKE,		&cnp_pmc_dev),
1727	X86_MATCH_VFM(INTEL_COMETLAKE_L,	&cnp_pmc_dev),
1728	X86_MATCH_VFM(INTEL_TIGERLAKE_L,	&tgl_l_pmc_dev),
1729	X86_MATCH_VFM(INTEL_TIGERLAKE,		&tgl_pmc_dev),
1730	X86_MATCH_VFM(INTEL_ATOM_TREMONT,	&tgl_l_pmc_dev),
1731	X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,	&icl_pmc_dev),
1732	X86_MATCH_VFM(INTEL_ROCKETLAKE,		&tgl_pmc_dev),
1733	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&tgl_l_pmc_dev),
1734	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&tgl_l_pmc_dev),
1735	X86_MATCH_VFM(INTEL_ALDERLAKE,		&adl_pmc_dev),
1736	X86_MATCH_VFM(INTEL_RAPTORLAKE_P,	&tgl_l_pmc_dev),
1737	X86_MATCH_VFM(INTEL_RAPTORLAKE,		&adl_pmc_dev),
1738	X86_MATCH_VFM(INTEL_RAPTORLAKE_S,	&adl_pmc_dev),
1739	X86_MATCH_VFM(INTEL_BARTLETTLAKE,       &adl_pmc_dev),
1740	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&mtl_pmc_dev),
1741	X86_MATCH_VFM(INTEL_ARROWLAKE,		&arl_pmc_dev),
1742	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&arl_h_pmc_dev),
1743	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&arl_h_pmc_dev),
1744	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_pmc_dev),
1745	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&ptl_pmc_dev),
1746	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&wcl_pmc_dev),
1747	{}
1748};
1749
1750MODULE_DEVICE_TABLE(x86cpu, intel_pmc_core_ids);
1751
1752/*
1753 * This quirk can be used on those platforms where
1754 * the platform BIOS enforces 24Mhz crystal to shutdown
1755 * before PMC can assert SLP_S0#.
1756 */
1757static bool xtal_ignore;
1758static int quirk_xtal_ignore(const struct dmi_system_id *id)
1759{
1760	xtal_ignore = true;
1761	return 0;
1762}
1763
1764static void pmc_core_xtal_ignore(struct pmc *pmc)
1765{
1766	u32 value;
1767
1768	value = pmc_core_reg_read(pmc, pmc->map->pm_vric1_offset);
1769	/* 24MHz Crystal Shutdown Qualification Disable */
1770	value |= SPT_PMC_VRIC1_XTALSDQDIS;
1771	/* Low Voltage Mode Enable */
1772	value &= ~SPT_PMC_VRIC1_SLPS0LVEN;
1773	pmc_core_reg_write(pmc, pmc->map->pm_vric1_offset, value);
1774}
1775
1776static const struct dmi_system_id pmc_core_dmi_table[]  = {
1777	{
1778	.callback = quirk_xtal_ignore,
1779	.ident = "HP Elite x2 1013 G3",
1780	.matches = {
1781		DMI_MATCH(DMI_SYS_VENDOR, "HP"),
1782		DMI_MATCH(DMI_PRODUCT_NAME, "HP Elite x2 1013 G3"),
1783		},
1784	},
1785	{}
1786};
1787
1788static void pmc_core_do_dmi_quirks(struct pmc *pmc)
1789{
1790	dmi_check_system(pmc_core_dmi_table);
1791
1792	if (xtal_ignore)
1793		pmc_core_xtal_ignore(pmc);
1794}
1795
1796static void pmc_core_clean_structure(struct platform_device *pdev)
1797{
1798	struct pmc_dev *pmcdev = platform_get_drvdata(pdev);
1799	unsigned int pmc_idx;
1800
1801	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
1802		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
1803
1804		if (pmc && pmc->regbase)
1805			iounmap(pmc->regbase);
1806	}
1807
1808	if (pmcdev->punit_ep)
1809		pmt_telem_unregister_endpoint(pmcdev->punit_ep);
1810
1811	platform_set_drvdata(pdev, NULL);
1812}
1813
1814static int pmc_core_probe(struct platform_device *pdev)
1815{
1816	static bool device_initialized;
1817	struct pmc_dev *pmcdev;
1818	const struct x86_cpu_id *cpu_id;
1819	struct pmc_dev_info *pmc_dev_info;
1820	struct pmc *primary_pmc;
1821	int ret;
1822
1823	if (device_initialized)
1824		return -ENODEV;
1825
1826	pmcdev = devm_kzalloc(&pdev->dev, sizeof(*pmcdev), GFP_KERNEL);
1827	if (!pmcdev)
1828		return -ENOMEM;
1829
1830	pmcdev->crystal_freq = pmc_core_get_crystal_freq();
1831
1832	platform_set_drvdata(pdev, pmcdev);
1833	pmcdev->pdev = pdev;
1834
1835	cpu_id = x86_match_cpu(intel_pmc_core_ids);
1836	if (!cpu_id)
1837		return -ENODEV;
1838
1839	pmc_dev_info = (struct pmc_dev_info *)cpu_id->driver_data;
1840
1841	/* Primary PMC */
1842	primary_pmc = devm_kzalloc(&pdev->dev, sizeof(*primary_pmc), GFP_KERNEL);
1843	if (!primary_pmc)
1844		return -ENOMEM;
1845	pmcdev->pmcs[PMC_IDX_MAIN] = primary_pmc;
1846
1847	/* The last element in msr_map is empty */
1848	pmcdev->num_of_pkgc = ARRAY_SIZE(msr_map) - 1;
1849	pmcdev->pkgc_res_cnt = devm_kcalloc(&pdev->dev,
1850					    pmcdev->num_of_pkgc,
1851					    sizeof(*pmcdev->pkgc_res_cnt),
1852					    GFP_KERNEL);
1853	if (!pmcdev->pkgc_res_cnt)
1854		return -ENOMEM;
1855
1856	ret = devm_mutex_init(&pdev->dev, &pmcdev->lock);
1857	if (ret)
1858		return ret;
1859
1860	if (pmc_dev_info->init)
1861		ret = pmc_dev_info->init(pmcdev, pmc_dev_info);
1862	else
1863		ret = generic_core_init(pmcdev, pmc_dev_info);
1864
1865	if (ret) {
1866		platform_set_drvdata(pdev, NULL);
1867		return ret;
1868	}
1869
1870	pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit(primary_pmc);
1871	pmc_core_do_dmi_quirks(primary_pmc);
1872
1873	pmc_core_dbgfs_register(pmcdev, pmc_dev_info);
1874	pm_report_max_hw_sleep(FIELD_MAX(SLP_S0_RES_COUNTER_MASK) *
1875			       pmc_core_adjust_slp_s0_step(primary_pmc, 1));
1876
1877	device_initialized = true;
1878	dev_info(&pdev->dev, " initialized\n");
1879
1880	return 0;
1881}
1882
1883static void pmc_core_remove(struct platform_device *pdev)
1884{
1885	struct pmc_dev *pmcdev = platform_get_drvdata(pdev);
1886	pmc_core_dbgfs_unregister(pmcdev);
1887	pmc_core_clean_structure(pdev);
1888}
1889
1890static bool warn_on_s0ix_failures;
1891module_param(warn_on_s0ix_failures, bool, 0644);
1892MODULE_PARM_DESC(warn_on_s0ix_failures, "Check and warn for S0ix failures");
1893
1894static bool ltr_ignore_all_suspend = true;
1895module_param(ltr_ignore_all_suspend, bool, 0644);
1896MODULE_PARM_DESC(ltr_ignore_all_suspend, "Ignore all LTRs during suspend");
1897
1898static __maybe_unused int pmc_core_suspend(struct device *dev)
1899{
1900	struct pmc_dev *pmcdev = dev_get_drvdata(dev);
1901	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1902	unsigned int i;
1903
1904	if (pmcdev->suspend)
1905		pmcdev->suspend(pmcdev);
1906
1907	if (ltr_ignore_all_suspend)
1908		pmc_core_ltr_ignore_all(pmcdev);
1909
1910	/* Check if the syspend will actually use S0ix */
1911	if (pm_suspend_via_firmware())
1912		return 0;
1913
1914	/* Save PKGC residency for checking later */
1915	for (i = 0; i < pmcdev->num_of_pkgc; i++) {
1916		if (rdmsrq_safe(msr_map[i].bit_mask, &pmcdev->pkgc_res_cnt[i]))
1917			return -EIO;
1918	}
1919
1920	/* Save S0ix residency for checking later */
1921	if (pmc_core_dev_state_get(pmc, &pmcdev->s0ix_counter))
1922		return -EIO;
1923
1924	return 0;
1925}
1926
1927static inline bool pmc_core_is_deepest_pkgc_failed(struct pmc_dev *pmcdev)
1928{
1929	u32 deepest_pkgc_msr = msr_map[pmcdev->num_of_pkgc - 1].bit_mask;
1930	u64 deepest_pkgc_residency;
1931
1932	if (rdmsrq_safe(deepest_pkgc_msr, &deepest_pkgc_residency))
1933		return false;
1934
1935	if (deepest_pkgc_residency == pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1])
1936		return true;
1937
1938	return false;
1939}
1940
1941static inline bool pmc_core_is_s0ix_failed(struct pmc_dev *pmcdev)
1942{
1943	u64 s0ix_counter;
1944
1945	if (pmc_core_dev_state_get(pmcdev->pmcs[PMC_IDX_MAIN], &s0ix_counter))
1946		return false;
1947
1948	pm_report_hw_sleep_time((u32)(s0ix_counter - pmcdev->s0ix_counter));
1949
1950	if (s0ix_counter == pmcdev->s0ix_counter)
1951		return true;
1952
1953	return false;
1954}
1955
1956int pmc_core_resume_common(struct pmc_dev *pmcdev)
1957{
1958	struct device *dev = &pmcdev->pdev->dev;
1959	struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1960	const struct pmc_bit_map **maps = pmc->map->lpm_sts;
1961	int offset = pmc->map->lpm_status_offset;
1962	unsigned int pmc_idx, i;
1963
1964	/* Check if the syspend used S0ix */
1965	if (pm_suspend_via_firmware())
1966		return 0;
1967
1968	if (!pmc_core_is_s0ix_failed(pmcdev))
1969		return 0;
1970
1971	if (!warn_on_s0ix_failures)
1972		return 0;
1973
1974	if (pmc_core_is_deepest_pkgc_failed(pmcdev)) {
1975		/* S0ix failed because of deepest PKGC entry failure */
1976		dev_info(dev, "CPU did not enter %s!!! (%s cnt=0x%llx)\n",
1977			 msr_map[pmcdev->num_of_pkgc - 1].name,
1978			 msr_map[pmcdev->num_of_pkgc - 1].name,
1979			 pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1]);
1980
1981		for (i = 0; i < pmcdev->num_of_pkgc; i++) {
1982			u64 pc_cnt;
1983
1984			if (!rdmsrq_safe(msr_map[i].bit_mask, &pc_cnt)) {
1985				dev_info(dev, "Prev %s cnt = 0x%llx, Current %s cnt = 0x%llx\n",
1986					 msr_map[i].name, pmcdev->pkgc_res_cnt[i],
1987					 msr_map[i].name, pc_cnt);
1988			}
1989		}
1990		return 0;
1991	}
1992
1993	/* The real interesting case - S0ix failed - lets ask PMC why. */
1994	dev_warn(dev, "CPU did not enter SLP_S0!!! (S0ix cnt=%llu)\n",
1995		 pmcdev->s0ix_counter);
1996
1997	if (pmc->map->slps0_dbg_maps)
1998		pmc_core_slps0_display(pmc, dev, NULL);
1999
2000	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
2001		struct pmc *pmc = pmcdev->pmcs[pmc_idx];
2002
2003		if (!pmc)
2004			continue;
2005		if (pmc->map->lpm_sts)
2006			pmc_core_lpm_display(pmc, dev, NULL, offset, pmc_idx, "STATUS", maps);
2007	}
2008
2009	return 0;
2010}
2011
2012static __maybe_unused int pmc_core_resume(struct device *dev)
2013{
2014	struct pmc_dev *pmcdev = dev_get_drvdata(dev);
2015
2016	if (ltr_ignore_all_suspend)
2017		pmc_core_ltr_restore_all(pmcdev);
2018
2019	if (pmcdev->resume)
2020		return pmcdev->resume(pmcdev);
2021
2022	return pmc_core_resume_common(pmcdev);
2023}
2024
2025static const struct dev_pm_ops pmc_core_pm_ops = {
2026	SET_LATE_SYSTEM_SLEEP_PM_OPS(pmc_core_suspend, pmc_core_resume)
2027};
2028
2029static const struct acpi_device_id pmc_core_acpi_ids[] = {
2030	{"INT33A1", 0}, /* _HID for Intel Power Engine, _CID PNP0D80*/
2031	{ }
2032};
2033MODULE_DEVICE_TABLE(acpi, pmc_core_acpi_ids);
2034
2035static struct platform_driver pmc_core_driver = {
2036	.driver = {
2037		.name = "intel_pmc_core",
2038		.acpi_match_table = ACPI_PTR(pmc_core_acpi_ids),
2039		.pm = &pmc_core_pm_ops,
2040		.dev_groups = pmc_dev_groups,
2041	},
2042	.probe = pmc_core_probe,
2043	.remove = pmc_core_remove,
2044};
2045
2046module_platform_driver(pmc_core_driver);
2047
2048MODULE_IMPORT_NS("INTEL_PMT_TELEMETRY");
2049MODULE_LICENSE("GPL v2");
2050MODULE_DESCRIPTION("Intel PMC Core Driver");