cpr.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
   4 * Copyright (c) 2019, Linaro Limited
   5 */
   6
   7#include <linux/cleanup.h>
   8#include <linux/module.h>
   9#include <linux/err.h>
  10#include <linux/debugfs.h>
  11#include <linux/string.h>
  12#include <linux/kernel.h>
  13#include <linux/list.h>
  14#include <linux/init.h>
  15#include <linux/io.h>
  16#include <linux/bitops.h>
  17#include <linux/slab.h>
  18#include <linux/of.h>
  19#include <linux/platform_device.h>
  20#include <linux/pm_domain.h>
  21#include <linux/pm_opp.h>
  22#include <linux/interrupt.h>
  23#include <linux/regmap.h>
  24#include <linux/mfd/syscon.h>
  25#include <linux/regulator/consumer.h>
  26#include <linux/clk.h>
  27#include <linux/nvmem-consumer.h>
  28
  29/* Register Offsets for RB-CPR and Bit Definitions */
  30
  31/* RBCPR Version Register */
  32#define REG_RBCPR_VERSION		0
  33#define RBCPR_VER_2			0x02
  34#define FLAGS_IGNORE_1ST_IRQ_STATUS	BIT(0)
  35
  36/* RBCPR Gate Count and Target Registers */
  37#define REG_RBCPR_GCNT_TARGET(n)	(0x60 + 4 * (n))
  38
  39#define RBCPR_GCNT_TARGET_TARGET_SHIFT	0
  40#define RBCPR_GCNT_TARGET_TARGET_MASK	GENMASK(11, 0)
  41#define RBCPR_GCNT_TARGET_GCNT_SHIFT	12
  42#define RBCPR_GCNT_TARGET_GCNT_MASK	GENMASK(9, 0)
  43
  44/* RBCPR Timer Control */
  45#define REG_RBCPR_TIMER_INTERVAL	0x44
  46#define REG_RBIF_TIMER_ADJUST		0x4c
  47
  48#define RBIF_TIMER_ADJ_CONS_UP_MASK	GENMASK(3, 0)
  49#define RBIF_TIMER_ADJ_CONS_UP_SHIFT	0
  50#define RBIF_TIMER_ADJ_CONS_DOWN_MASK	GENMASK(3, 0)
  51#define RBIF_TIMER_ADJ_CONS_DOWN_SHIFT	4
  52#define RBIF_TIMER_ADJ_CLAMP_INT_MASK	GENMASK(7, 0)
  53#define RBIF_TIMER_ADJ_CLAMP_INT_SHIFT	8
  54
  55/* RBCPR Config Register */
  56#define REG_RBIF_LIMIT			0x48
  57#define RBIF_LIMIT_CEILING_MASK		GENMASK(5, 0)
  58#define RBIF_LIMIT_CEILING_SHIFT	6
  59#define RBIF_LIMIT_FLOOR_BITS		6
  60#define RBIF_LIMIT_FLOOR_MASK		GENMASK(5, 0)
  61
  62#define RBIF_LIMIT_CEILING_DEFAULT	RBIF_LIMIT_CEILING_MASK
  63#define RBIF_LIMIT_FLOOR_DEFAULT	0
  64
  65#define REG_RBIF_SW_VLEVEL		0x94
  66#define RBIF_SW_VLEVEL_DEFAULT		0x20
  67
  68#define REG_RBCPR_STEP_QUOT		0x80
  69#define RBCPR_STEP_QUOT_STEPQUOT_MASK	GENMASK(7, 0)
  70#define RBCPR_STEP_QUOT_IDLE_CLK_MASK	GENMASK(3, 0)
  71#define RBCPR_STEP_QUOT_IDLE_CLK_SHIFT	8
  72
  73/* RBCPR Control Register */
  74#define REG_RBCPR_CTL			0x90
  75
  76#define RBCPR_CTL_LOOP_EN			BIT(0)
  77#define RBCPR_CTL_TIMER_EN			BIT(3)
  78#define RBCPR_CTL_SW_AUTO_CONT_ACK_EN		BIT(5)
  79#define RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN	BIT(6)
  80#define RBCPR_CTL_COUNT_MODE			BIT(10)
  81#define RBCPR_CTL_UP_THRESHOLD_MASK	GENMASK(3, 0)
  82#define RBCPR_CTL_UP_THRESHOLD_SHIFT	24
  83#define RBCPR_CTL_DN_THRESHOLD_MASK	GENMASK(3, 0)
  84#define RBCPR_CTL_DN_THRESHOLD_SHIFT	28
  85
  86/* RBCPR Ack/Nack Response */
  87#define REG_RBIF_CONT_ACK_CMD		0x98
  88#define REG_RBIF_CONT_NACK_CMD		0x9c
  89
  90/* RBCPR Result status Register */
  91#define REG_RBCPR_RESULT_0		0xa0
  92
  93#define RBCPR_RESULT0_BUSY_SHIFT	19
  94#define RBCPR_RESULT0_BUSY_MASK		BIT(RBCPR_RESULT0_BUSY_SHIFT)
  95#define RBCPR_RESULT0_ERROR_LT0_SHIFT	18
  96#define RBCPR_RESULT0_ERROR_SHIFT	6
  97#define RBCPR_RESULT0_ERROR_MASK	GENMASK(11, 0)
  98#define RBCPR_RESULT0_ERROR_STEPS_SHIFT	2
  99#define RBCPR_RESULT0_ERROR_STEPS_MASK	GENMASK(3, 0)
 100#define RBCPR_RESULT0_STEP_UP_SHIFT	1
 101
 102/* RBCPR Interrupt Control Register */
 103#define REG_RBIF_IRQ_EN(n)		(0x100 + 4 * (n))
 104#define REG_RBIF_IRQ_CLEAR		0x110
 105#define REG_RBIF_IRQ_STATUS		0x114
 106
 107#define CPR_INT_DONE		BIT(0)
 108#define CPR_INT_MIN		BIT(1)
 109#define CPR_INT_DOWN		BIT(2)
 110#define CPR_INT_MID		BIT(3)
 111#define CPR_INT_UP		BIT(4)
 112#define CPR_INT_MAX		BIT(5)
 113#define CPR_INT_CLAMP		BIT(6)
 114#define CPR_INT_ALL	(CPR_INT_DONE | CPR_INT_MIN | CPR_INT_DOWN | \
 115			CPR_INT_MID | CPR_INT_UP | CPR_INT_MAX | CPR_INT_CLAMP)
 116#define CPR_INT_DEFAULT	(CPR_INT_UP | CPR_INT_DOWN)
 117
 118#define CPR_NUM_RING_OSC	8
 119
 120/* CPR eFuse parameters */
 121#define CPR_FUSE_TARGET_QUOT_BITS_MASK	GENMASK(11, 0)
 122
 123#define CPR_FUSE_MIN_QUOT_DIFF		50
 124
 125#define FUSE_REVISION_UNKNOWN		(-1)
 126
 127enum voltage_change_dir {
 128	NO_CHANGE,
 129	DOWN,
 130	UP,
 131};
 132
 133struct cpr_fuse {
 134	char *ring_osc;
 135	char *init_voltage;
 136	char *quotient;
 137	char *quotient_offset;
 138};
 139
 140struct fuse_corner_data {
 141	int ref_uV;
 142	int max_uV;
 143	int min_uV;
 144	int max_volt_scale;
 145	int max_quot_scale;
 146	/* fuse quot */
 147	int quot_offset;
 148	int quot_scale;
 149	int quot_adjust;
 150	/* fuse quot_offset */
 151	int quot_offset_scale;
 152	int quot_offset_adjust;
 153};
 154
 155struct cpr_fuses {
 156	int init_voltage_step;
 157	int init_voltage_width;
 158	struct fuse_corner_data *fuse_corner_data;
 159};
 160
 161struct corner_data {
 162	unsigned int fuse_corner;
 163	unsigned long freq;
 164};
 165
 166struct cpr_desc {
 167	unsigned int num_fuse_corners;
 168	int min_diff_quot;
 169	int *step_quot;
 170
 171	unsigned int		timer_delay_us;
 172	unsigned int		timer_cons_up;
 173	unsigned int		timer_cons_down;
 174	unsigned int		up_threshold;
 175	unsigned int		down_threshold;
 176	unsigned int		idle_clocks;
 177	unsigned int		gcnt_us;
 178	unsigned int		vdd_apc_step_up_limit;
 179	unsigned int		vdd_apc_step_down_limit;
 180	unsigned int		clamp_timer_interval;
 181
 182	struct cpr_fuses cpr_fuses;
 183	bool reduce_to_fuse_uV;
 184	bool reduce_to_corner_uV;
 185};
 186
 187struct acc_desc {
 188	unsigned int	enable_reg;
 189	u32		enable_mask;
 190
 191	struct reg_sequence	*config;
 192	struct reg_sequence	*settings;
 193	int			num_regs_per_fuse;
 194};
 195
 196struct cpr_acc_desc {
 197	const struct cpr_desc *cpr_desc;
 198	const struct acc_desc *acc_desc;
 199};
 200
 201struct fuse_corner {
 202	int min_uV;
 203	int max_uV;
 204	int uV;
 205	int quot;
 206	int step_quot;
 207	const struct reg_sequence *accs;
 208	int num_accs;
 209	unsigned long max_freq;
 210	u8 ring_osc_idx;
 211};
 212
 213struct corner {
 214	int min_uV;
 215	int max_uV;
 216	int uV;
 217	int last_uV;
 218	int quot_adjust;
 219	u32 save_ctl;
 220	u32 save_irq;
 221	unsigned long freq;
 222	struct fuse_corner *fuse_corner;
 223};
 224
 225struct cpr_drv {
 226	unsigned int		num_corners;
 227	unsigned int		ref_clk_khz;
 228
 229	struct generic_pm_domain pd;
 230	struct device		*dev;
 231	struct device		*attached_cpu_dev;
 232	struct mutex		lock;
 233	void __iomem		*base;
 234	struct corner		*corner;
 235	struct regulator	*vdd_apc;
 236	struct clk		*cpu_clk;
 237	struct regmap		*tcsr;
 238	bool			loop_disabled;
 239	u32			gcnt;
 240	unsigned long		flags;
 241
 242	struct fuse_corner	*fuse_corners;
 243	struct corner		*corners;
 244
 245	const struct cpr_desc *desc;
 246	const struct acc_desc *acc_desc;
 247	const struct cpr_fuse *cpr_fuses;
 248
 249	struct dentry *debugfs;
 250};
 251
 252static bool cpr_is_allowed(struct cpr_drv *drv)
 253{
 254	return !drv->loop_disabled;
 255}
 256
 257static void cpr_write(struct cpr_drv *drv, u32 offset, u32 value)
 258{
 259	writel_relaxed(value, drv->base + offset);
 260}
 261
 262static u32 cpr_read(struct cpr_drv *drv, u32 offset)
 263{
 264	return readl_relaxed(drv->base + offset);
 265}
 266
 267static void
 268cpr_masked_write(struct cpr_drv *drv, u32 offset, u32 mask, u32 value)
 269{
 270	u32 val;
 271
 272	val = readl_relaxed(drv->base + offset);
 273	val &= ~mask;
 274	val |= value & mask;
 275	writel_relaxed(val, drv->base + offset);
 276}
 277
 278static void cpr_irq_clr(struct cpr_drv *drv)
 279{
 280	cpr_write(drv, REG_RBIF_IRQ_CLEAR, CPR_INT_ALL);
 281}
 282
 283static void cpr_irq_clr_nack(struct cpr_drv *drv)
 284{
 285	cpr_irq_clr(drv);
 286	cpr_write(drv, REG_RBIF_CONT_NACK_CMD, 1);
 287}
 288
 289static void cpr_irq_clr_ack(struct cpr_drv *drv)
 290{
 291	cpr_irq_clr(drv);
 292	cpr_write(drv, REG_RBIF_CONT_ACK_CMD, 1);
 293}
 294
 295static void cpr_irq_set(struct cpr_drv *drv, u32 int_bits)
 296{
 297	cpr_write(drv, REG_RBIF_IRQ_EN(0), int_bits);
 298}
 299
 300static void cpr_ctl_modify(struct cpr_drv *drv, u32 mask, u32 value)
 301{
 302	cpr_masked_write(drv, REG_RBCPR_CTL, mask, value);
 303}
 304
 305static void cpr_ctl_enable(struct cpr_drv *drv, struct corner *corner)
 306{
 307	u32 val, mask;
 308	const struct cpr_desc *desc = drv->desc;
 309
 310	/* Program Consecutive Up & Down */
 311	val = desc->timer_cons_down << RBIF_TIMER_ADJ_CONS_DOWN_SHIFT;
 312	val |= desc->timer_cons_up << RBIF_TIMER_ADJ_CONS_UP_SHIFT;
 313	mask = RBIF_TIMER_ADJ_CONS_UP_MASK | RBIF_TIMER_ADJ_CONS_DOWN_MASK;
 314	cpr_masked_write(drv, REG_RBIF_TIMER_ADJUST, mask, val);
 315	cpr_masked_write(drv, REG_RBCPR_CTL,
 316			 RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN |
 317			 RBCPR_CTL_SW_AUTO_CONT_ACK_EN,
 318			 corner->save_ctl);
 319	cpr_irq_set(drv, corner->save_irq);
 320
 321	if (cpr_is_allowed(drv) && corner->max_uV > corner->min_uV)
 322		val = RBCPR_CTL_LOOP_EN;
 323	else
 324		val = 0;
 325	cpr_ctl_modify(drv, RBCPR_CTL_LOOP_EN, val);
 326}
 327
 328static void cpr_ctl_disable(struct cpr_drv *drv)
 329{
 330	cpr_irq_set(drv, 0);
 331	cpr_ctl_modify(drv, RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN |
 332		       RBCPR_CTL_SW_AUTO_CONT_ACK_EN, 0);
 333	cpr_masked_write(drv, REG_RBIF_TIMER_ADJUST,
 334			 RBIF_TIMER_ADJ_CONS_UP_MASK |
 335			 RBIF_TIMER_ADJ_CONS_DOWN_MASK, 0);
 336	cpr_irq_clr(drv);
 337	cpr_write(drv, REG_RBIF_CONT_ACK_CMD, 1);
 338	cpr_write(drv, REG_RBIF_CONT_NACK_CMD, 1);
 339	cpr_ctl_modify(drv, RBCPR_CTL_LOOP_EN, 0);
 340}
 341
 342static bool cpr_ctl_is_enabled(struct cpr_drv *drv)
 343{
 344	u32 reg_val;
 345
 346	reg_val = cpr_read(drv, REG_RBCPR_CTL);
 347	return reg_val & RBCPR_CTL_LOOP_EN;
 348}
 349
 350static bool cpr_ctl_is_busy(struct cpr_drv *drv)
 351{
 352	u32 reg_val;
 353
 354	reg_val = cpr_read(drv, REG_RBCPR_RESULT_0);
 355	return reg_val & RBCPR_RESULT0_BUSY_MASK;
 356}
 357
 358static void cpr_corner_save(struct cpr_drv *drv, struct corner *corner)
 359{
 360	corner->save_ctl = cpr_read(drv, REG_RBCPR_CTL);
 361	corner->save_irq = cpr_read(drv, REG_RBIF_IRQ_EN(0));
 362}
 363
 364static void cpr_corner_restore(struct cpr_drv *drv, struct corner *corner)
 365{
 366	u32 gcnt, ctl, irq, ro_sel, step_quot;
 367	struct fuse_corner *fuse = corner->fuse_corner;
 368	const struct cpr_desc *desc = drv->desc;
 369	int i;
 370
 371	ro_sel = fuse->ring_osc_idx;
 372	gcnt = drv->gcnt;
 373	gcnt |= fuse->quot - corner->quot_adjust;
 374
 375	/* Program the step quotient and idle clocks */
 376	step_quot = desc->idle_clocks << RBCPR_STEP_QUOT_IDLE_CLK_SHIFT;
 377	step_quot |= fuse->step_quot & RBCPR_STEP_QUOT_STEPQUOT_MASK;
 378	cpr_write(drv, REG_RBCPR_STEP_QUOT, step_quot);
 379
 380	/* Clear the target quotient value and gate count of all ROs */
 381	for (i = 0; i < CPR_NUM_RING_OSC; i++)
 382		cpr_write(drv, REG_RBCPR_GCNT_TARGET(i), 0);
 383
 384	cpr_write(drv, REG_RBCPR_GCNT_TARGET(ro_sel), gcnt);
 385	ctl = corner->save_ctl;
 386	cpr_write(drv, REG_RBCPR_CTL, ctl);
 387	irq = corner->save_irq;
 388	cpr_irq_set(drv, irq);
 389	dev_dbg(drv->dev, "gcnt = %#08x, ctl = %#08x, irq = %#08x\n", gcnt,
 390		ctl, irq);
 391}
 392
 393static void cpr_set_acc(struct regmap *tcsr, struct fuse_corner *f,
 394			struct fuse_corner *end)
 395{
 396	if (f == end)
 397		return;
 398
 399	if (f < end) {
 400		for (f += 1; f <= end; f++)
 401			regmap_multi_reg_write(tcsr, f->accs, f->num_accs);
 402	} else {
 403		for (f -= 1; f >= end; f--)
 404			regmap_multi_reg_write(tcsr, f->accs, f->num_accs);
 405	}
 406}
 407
 408static int cpr_pre_voltage(struct cpr_drv *drv,
 409			   struct fuse_corner *fuse_corner,
 410			   enum voltage_change_dir dir)
 411{
 412	struct fuse_corner *prev_fuse_corner = drv->corner->fuse_corner;
 413
 414	if (drv->tcsr && dir == DOWN)
 415		cpr_set_acc(drv->tcsr, prev_fuse_corner, fuse_corner);
 416
 417	return 0;
 418}
 419
 420static int cpr_post_voltage(struct cpr_drv *drv,
 421			    struct fuse_corner *fuse_corner,
 422			    enum voltage_change_dir dir)
 423{
 424	struct fuse_corner *prev_fuse_corner = drv->corner->fuse_corner;
 425
 426	if (drv->tcsr && dir == UP)
 427		cpr_set_acc(drv->tcsr, prev_fuse_corner, fuse_corner);
 428
 429	return 0;
 430}
 431
 432static int cpr_scale_voltage(struct cpr_drv *drv, struct corner *corner,
 433			     int new_uV, enum voltage_change_dir dir)
 434{
 435	int ret;
 436	struct fuse_corner *fuse_corner = corner->fuse_corner;
 437
 438	ret = cpr_pre_voltage(drv, fuse_corner, dir);
 439	if (ret)
 440		return ret;
 441
 442	ret = regulator_set_voltage(drv->vdd_apc, new_uV, new_uV);
 443	if (ret) {
 444		dev_err_ratelimited(drv->dev, "failed to set apc voltage %d\n",
 445				    new_uV);
 446		return ret;
 447	}
 448
 449	ret = cpr_post_voltage(drv, fuse_corner, dir);
 450	if (ret)
 451		return ret;
 452
 453	return 0;
 454}
 455
 456static unsigned int cpr_get_cur_perf_state(struct cpr_drv *drv)
 457{
 458	return drv->corner ? drv->corner - drv->corners + 1 : 0;
 459}
 460
 461static int cpr_scale(struct cpr_drv *drv, enum voltage_change_dir dir)
 462{
 463	u32 val, error_steps, reg_mask;
 464	int last_uV, new_uV, step_uV, ret;
 465	struct corner *corner;
 466	const struct cpr_desc *desc = drv->desc;
 467
 468	if (dir != UP && dir != DOWN)
 469		return 0;
 470
 471	step_uV = regulator_get_linear_step(drv->vdd_apc);
 472	if (!step_uV)
 473		return -EINVAL;
 474
 475	corner = drv->corner;
 476
 477	val = cpr_read(drv, REG_RBCPR_RESULT_0);
 478
 479	error_steps = val >> RBCPR_RESULT0_ERROR_STEPS_SHIFT;
 480	error_steps &= RBCPR_RESULT0_ERROR_STEPS_MASK;
 481	last_uV = corner->last_uV;
 482
 483	if (dir == UP) {
 484		if (desc->clamp_timer_interval &&
 485		    error_steps < desc->up_threshold) {
 486			/*
 487			 * Handle the case where another measurement started
 488			 * after the interrupt was triggered due to a core
 489			 * exiting from power collapse.
 490			 */
 491			error_steps = max(desc->up_threshold,
 492					  desc->vdd_apc_step_up_limit);
 493		}
 494
 495		if (last_uV >= corner->max_uV) {
 496			cpr_irq_clr_nack(drv);
 497
 498			/* Maximize the UP threshold */
 499			reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK;
 500			reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 501			val = reg_mask;
 502			cpr_ctl_modify(drv, reg_mask, val);
 503
 504			/* Disable UP interrupt */
 505			cpr_irq_set(drv, CPR_INT_DEFAULT & ~CPR_INT_UP);
 506
 507			return 0;
 508		}
 509
 510		if (error_steps > desc->vdd_apc_step_up_limit)
 511			error_steps = desc->vdd_apc_step_up_limit;
 512
 513		/* Calculate new voltage */
 514		new_uV = last_uV + error_steps * step_uV;
 515		new_uV = min(new_uV, corner->max_uV);
 516
 517		dev_dbg(drv->dev,
 518			"UP: -> new_uV: %d last_uV: %d perf state: %u\n",
 519			new_uV, last_uV, cpr_get_cur_perf_state(drv));
 520	} else {
 521		if (desc->clamp_timer_interval &&
 522		    error_steps < desc->down_threshold) {
 523			/*
 524			 * Handle the case where another measurement started
 525			 * after the interrupt was triggered due to a core
 526			 * exiting from power collapse.
 527			 */
 528			error_steps = max(desc->down_threshold,
 529					  desc->vdd_apc_step_down_limit);
 530		}
 531
 532		if (last_uV <= corner->min_uV) {
 533			cpr_irq_clr_nack(drv);
 534
 535			/* Enable auto nack down */
 536			reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 537			val = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 538
 539			cpr_ctl_modify(drv, reg_mask, val);
 540
 541			/* Disable DOWN interrupt */
 542			cpr_irq_set(drv, CPR_INT_DEFAULT & ~CPR_INT_DOWN);
 543
 544			return 0;
 545		}
 546
 547		if (error_steps > desc->vdd_apc_step_down_limit)
 548			error_steps = desc->vdd_apc_step_down_limit;
 549
 550		/* Calculate new voltage */
 551		new_uV = last_uV - error_steps * step_uV;
 552		new_uV = max(new_uV, corner->min_uV);
 553
 554		dev_dbg(drv->dev,
 555			"DOWN: -> new_uV: %d last_uV: %d perf state: %u\n",
 556			new_uV, last_uV, cpr_get_cur_perf_state(drv));
 557	}
 558
 559	ret = cpr_scale_voltage(drv, corner, new_uV, dir);
 560	if (ret) {
 561		cpr_irq_clr_nack(drv);
 562		return ret;
 563	}
 564	drv->corner->last_uV = new_uV;
 565
 566	if (dir == UP) {
 567		/* Disable auto nack down */
 568		reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 569		val = 0;
 570	} else {
 571		/* Restore default threshold for UP */
 572		reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK;
 573		reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 574		val = desc->up_threshold;
 575		val <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 576	}
 577
 578	cpr_ctl_modify(drv, reg_mask, val);
 579
 580	/* Re-enable default interrupts */
 581	cpr_irq_set(drv, CPR_INT_DEFAULT);
 582
 583	/* Ack */
 584	cpr_irq_clr_ack(drv);
 585
 586	return 0;
 587}
 588
 589static irqreturn_t cpr_irq_handler(int irq, void *dev)
 590{
 591	struct cpr_drv *drv = dev;
 592	const struct cpr_desc *desc = drv->desc;
 593	irqreturn_t ret = IRQ_HANDLED;
 594	u32 val;
 595
 596	mutex_lock(&drv->lock);
 597
 598	val = cpr_read(drv, REG_RBIF_IRQ_STATUS);
 599	if (drv->flags & FLAGS_IGNORE_1ST_IRQ_STATUS)
 600		val = cpr_read(drv, REG_RBIF_IRQ_STATUS);
 601
 602	dev_dbg(drv->dev, "IRQ_STATUS = %#02x\n", val);
 603
 604	if (!cpr_ctl_is_enabled(drv)) {
 605		dev_dbg(drv->dev, "CPR is disabled\n");
 606		ret = IRQ_NONE;
 607	} else if (cpr_ctl_is_busy(drv) && !desc->clamp_timer_interval) {
 608		dev_dbg(drv->dev, "CPR measurement is not ready\n");
 609	} else if (!cpr_is_allowed(drv)) {
 610		val = cpr_read(drv, REG_RBCPR_CTL);
 611		dev_err_ratelimited(drv->dev,
 612				    "Interrupt broken? RBCPR_CTL = %#02x\n",
 613				    val);
 614		ret = IRQ_NONE;
 615	} else {
 616		/*
 617		 * Following sequence of handling is as per each IRQ's
 618		 * priority
 619		 */
 620		if (val & CPR_INT_UP) {
 621			cpr_scale(drv, UP);
 622		} else if (val & CPR_INT_DOWN) {
 623			cpr_scale(drv, DOWN);
 624		} else if (val & CPR_INT_MIN) {
 625			cpr_irq_clr_nack(drv);
 626		} else if (val & CPR_INT_MAX) {
 627			cpr_irq_clr_nack(drv);
 628		} else if (val & CPR_INT_MID) {
 629			/* RBCPR_CTL_SW_AUTO_CONT_ACK_EN is enabled */
 630			dev_dbg(drv->dev, "IRQ occurred for Mid Flag\n");
 631		} else {
 632			dev_dbg(drv->dev,
 633				"IRQ occurred for unknown flag (%#08x)\n", val);
 634		}
 635
 636		/* Save register values for the corner */
 637		cpr_corner_save(drv, drv->corner);
 638	}
 639
 640	mutex_unlock(&drv->lock);
 641
 642	return ret;
 643}
 644
 645static int cpr_enable(struct cpr_drv *drv)
 646{
 647	int ret;
 648
 649	ret = regulator_enable(drv->vdd_apc);
 650	if (ret)
 651		return ret;
 652
 653	mutex_lock(&drv->lock);
 654
 655	if (cpr_is_allowed(drv) && drv->corner) {
 656		cpr_irq_clr(drv);
 657		cpr_corner_restore(drv, drv->corner);
 658		cpr_ctl_enable(drv, drv->corner);
 659	}
 660
 661	mutex_unlock(&drv->lock);
 662
 663	return 0;
 664}
 665
 666static int cpr_disable(struct cpr_drv *drv)
 667{
 668	mutex_lock(&drv->lock);
 669
 670	if (cpr_is_allowed(drv)) {
 671		cpr_ctl_disable(drv);
 672		cpr_irq_clr(drv);
 673	}
 674
 675	mutex_unlock(&drv->lock);
 676
 677	return regulator_disable(drv->vdd_apc);
 678}
 679
 680static int cpr_config(struct cpr_drv *drv)
 681{
 682	int i;
 683	u32 val, gcnt;
 684	struct corner *corner;
 685	const struct cpr_desc *desc = drv->desc;
 686
 687	/* Disable interrupt and CPR */
 688	cpr_write(drv, REG_RBIF_IRQ_EN(0), 0);
 689	cpr_write(drv, REG_RBCPR_CTL, 0);
 690
 691	/* Program the default HW ceiling, floor and vlevel */
 692	val = (RBIF_LIMIT_CEILING_DEFAULT & RBIF_LIMIT_CEILING_MASK)
 693		<< RBIF_LIMIT_CEILING_SHIFT;
 694	val |= RBIF_LIMIT_FLOOR_DEFAULT & RBIF_LIMIT_FLOOR_MASK;
 695	cpr_write(drv, REG_RBIF_LIMIT, val);
 696	cpr_write(drv, REG_RBIF_SW_VLEVEL, RBIF_SW_VLEVEL_DEFAULT);
 697
 698	/*
 699	 * Clear the target quotient value and gate count of all
 700	 * ring oscillators
 701	 */
 702	for (i = 0; i < CPR_NUM_RING_OSC; i++)
 703		cpr_write(drv, REG_RBCPR_GCNT_TARGET(i), 0);
 704
 705	/* Init and save gcnt */
 706	gcnt = (drv->ref_clk_khz * desc->gcnt_us) / 1000;
 707	gcnt = gcnt & RBCPR_GCNT_TARGET_GCNT_MASK;
 708	gcnt <<= RBCPR_GCNT_TARGET_GCNT_SHIFT;
 709	drv->gcnt = gcnt;
 710
 711	/* Program the delay count for the timer */
 712	val = (drv->ref_clk_khz * desc->timer_delay_us) / 1000;
 713	cpr_write(drv, REG_RBCPR_TIMER_INTERVAL, val);
 714	dev_dbg(drv->dev, "Timer count: %#0x (for %d us)\n", val,
 715		desc->timer_delay_us);
 716
 717	/* Program Consecutive Up & Down */
 718	val = desc->timer_cons_down << RBIF_TIMER_ADJ_CONS_DOWN_SHIFT;
 719	val |= desc->timer_cons_up << RBIF_TIMER_ADJ_CONS_UP_SHIFT;
 720	val |= desc->clamp_timer_interval << RBIF_TIMER_ADJ_CLAMP_INT_SHIFT;
 721	cpr_write(drv, REG_RBIF_TIMER_ADJUST, val);
 722
 723	/* Program the control register */
 724	val = desc->up_threshold << RBCPR_CTL_UP_THRESHOLD_SHIFT;
 725	val |= desc->down_threshold << RBCPR_CTL_DN_THRESHOLD_SHIFT;
 726	val |= RBCPR_CTL_TIMER_EN | RBCPR_CTL_COUNT_MODE;
 727	val |= RBCPR_CTL_SW_AUTO_CONT_ACK_EN;
 728	cpr_write(drv, REG_RBCPR_CTL, val);
 729
 730	for (i = 0; i < drv->num_corners; i++) {
 731		corner = &drv->corners[i];
 732		corner->save_ctl = val;
 733		corner->save_irq = CPR_INT_DEFAULT;
 734	}
 735
 736	cpr_irq_set(drv, CPR_INT_DEFAULT);
 737
 738	val = cpr_read(drv, REG_RBCPR_VERSION);
 739	if (val <= RBCPR_VER_2)
 740		drv->flags |= FLAGS_IGNORE_1ST_IRQ_STATUS;
 741
 742	return 0;
 743}
 744
 745static int cpr_set_performance_state(struct generic_pm_domain *domain,
 746				     unsigned int state)
 747{
 748	struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
 749	struct corner *corner, *end;
 750	enum voltage_change_dir dir;
 751	int ret, new_uV;
 752
 753	guard(mutex)(&drv->lock);
 754
 755	dev_dbg(drv->dev, "%s: setting perf state: %u (prev state: %u)\n",
 756		__func__, state, cpr_get_cur_perf_state(drv));
 757
 758	/*
 759	 * Determine new corner we're going to.
 760	 * Remove one since lowest performance state is 1.
 761	 */
 762	corner = drv->corners + state - 1;
 763	end = &drv->corners[drv->num_corners - 1];
 764	if (corner > end || corner < drv->corners)
 765		return -EINVAL;
 766
 767	/* Determine direction */
 768	if (drv->corner > corner)
 769		dir = DOWN;
 770	else if (drv->corner < corner)
 771		dir = UP;
 772	else
 773		dir = NO_CHANGE;
 774
 775	if (cpr_is_allowed(drv))
 776		new_uV = corner->last_uV;
 777	else
 778		new_uV = corner->uV;
 779
 780	if (cpr_is_allowed(drv))
 781		cpr_ctl_disable(drv);
 782
 783	ret = cpr_scale_voltage(drv, corner, new_uV, dir);
 784	if (ret)
 785		return ret;
 786
 787	if (cpr_is_allowed(drv)) {
 788		cpr_irq_clr(drv);
 789		if (drv->corner != corner)
 790			cpr_corner_restore(drv, corner);
 791		cpr_ctl_enable(drv, corner);
 792	}
 793
 794	drv->corner = corner;
 795
 796	return 0;
 797}
 798
 799static int
 800cpr_populate_ring_osc_idx(struct cpr_drv *drv)
 801{
 802	struct fuse_corner *fuse = drv->fuse_corners;
 803	struct fuse_corner *end = fuse + drv->desc->num_fuse_corners;
 804	const struct cpr_fuse *fuses = drv->cpr_fuses;
 805	u32 data;
 806	int ret;
 807
 808	for (; fuse < end; fuse++, fuses++) {
 809		ret = nvmem_cell_read_variable_le_u32(drv->dev, fuses->ring_osc, &data);
 810		if (ret)
 811			return ret;
 812		fuse->ring_osc_idx = data;
 813	}
 814
 815	return 0;
 816}
 817
 818static int cpr_read_fuse_uV(const struct cpr_desc *desc,
 819			    const struct fuse_corner_data *fdata,
 820			    const char *init_v_efuse,
 821			    int step_volt,
 822			    struct cpr_drv *drv)
 823{
 824	int step_size_uV, steps, uV;
 825	u32 bits = 0;
 826	int ret;
 827
 828	ret = nvmem_cell_read_variable_le_u32(drv->dev, init_v_efuse, &bits);
 829	if (ret)
 830		return ret;
 831
 832	steps = bits & ~BIT(desc->cpr_fuses.init_voltage_width - 1);
 833	/* Not two's complement.. instead highest bit is sign bit */
 834	if (bits & BIT(desc->cpr_fuses.init_voltage_width - 1))
 835		steps = -steps;
 836
 837	step_size_uV = desc->cpr_fuses.init_voltage_step;
 838
 839	uV = fdata->ref_uV + steps * step_size_uV;
 840	return DIV_ROUND_UP(uV, step_volt) * step_volt;
 841}
 842
 843static int cpr_fuse_corner_init(struct cpr_drv *drv)
 844{
 845	const struct cpr_desc *desc = drv->desc;
 846	const struct cpr_fuse *fuses = drv->cpr_fuses;
 847	const struct acc_desc *acc_desc = drv->acc_desc;
 848	int i;
 849	unsigned int step_volt;
 850	struct fuse_corner_data *fdata;
 851	struct fuse_corner *fuse, *end;
 852	int uV;
 853	const struct reg_sequence *accs;
 854	int ret;
 855
 856	accs = acc_desc->settings;
 857
 858	step_volt = regulator_get_linear_step(drv->vdd_apc);
 859	if (!step_volt)
 860		return -EINVAL;
 861
 862	/* Populate fuse_corner members */
 863	fuse = drv->fuse_corners;
 864	end = &fuse[desc->num_fuse_corners - 1];
 865	fdata = desc->cpr_fuses.fuse_corner_data;
 866
 867	for (i = 0; fuse <= end; fuse++, fuses++, i++, fdata++) {
 868		/*
 869		 * Update SoC voltages: platforms might choose a different
 870		 * regulators than the one used to characterize the algorithms
 871		 * (ie, init_voltage_step).
 872		 */
 873		fdata->min_uV = roundup(fdata->min_uV, step_volt);
 874		fdata->max_uV = roundup(fdata->max_uV, step_volt);
 875
 876		/* Populate uV */
 877		uV = cpr_read_fuse_uV(desc, fdata, fuses->init_voltage,
 878				      step_volt, drv);
 879		if (uV < 0)
 880			return uV;
 881
 882		fuse->min_uV = fdata->min_uV;
 883		fuse->max_uV = fdata->max_uV;
 884		fuse->uV = clamp(uV, fuse->min_uV, fuse->max_uV);
 885
 886		if (fuse == end) {
 887			/*
 888			 * Allow the highest fuse corner's PVS voltage to
 889			 * define the ceiling voltage for that corner in order
 890			 * to support SoC's in which variable ceiling values
 891			 * are required.
 892			 */
 893			end->max_uV = max(end->max_uV, end->uV);
 894		}
 895
 896		/* Populate target quotient by scaling */
 897		ret = nvmem_cell_read_variable_le_u32(drv->dev, fuses->quotient, &fuse->quot);
 898		if (ret)
 899			return ret;
 900
 901		fuse->quot *= fdata->quot_scale;
 902		fuse->quot += fdata->quot_offset;
 903		fuse->quot += fdata->quot_adjust;
 904		fuse->step_quot = desc->step_quot[fuse->ring_osc_idx];
 905
 906		/* Populate acc settings */
 907		fuse->accs = accs;
 908		fuse->num_accs = acc_desc->num_regs_per_fuse;
 909		accs += acc_desc->num_regs_per_fuse;
 910	}
 911
 912	/*
 913	 * Restrict all fuse corner PVS voltages based upon per corner
 914	 * ceiling and floor voltages.
 915	 */
 916	for (fuse = drv->fuse_corners, i = 0; fuse <= end; fuse++, i++) {
 917		if (fuse->uV > fuse->max_uV)
 918			fuse->uV = fuse->max_uV;
 919		else if (fuse->uV < fuse->min_uV)
 920			fuse->uV = fuse->min_uV;
 921
 922		ret = regulator_is_supported_voltage(drv->vdd_apc,
 923						     fuse->min_uV,
 924						     fuse->min_uV);
 925		if (!ret) {
 926			dev_err(drv->dev,
 927				"min uV: %d (fuse corner: %d) not supported by regulator\n",
 928				fuse->min_uV, i);
 929			return -EINVAL;
 930		}
 931
 932		ret = regulator_is_supported_voltage(drv->vdd_apc,
 933						     fuse->max_uV,
 934						     fuse->max_uV);
 935		if (!ret) {
 936			dev_err(drv->dev,
 937				"max uV: %d (fuse corner: %d) not supported by regulator\n",
 938				fuse->max_uV, i);
 939			return -EINVAL;
 940		}
 941
 942		dev_dbg(drv->dev,
 943			"fuse corner %d: [%d %d %d] RO%hhu quot %d squot %d\n",
 944			i, fuse->min_uV, fuse->uV, fuse->max_uV,
 945			fuse->ring_osc_idx, fuse->quot, fuse->step_quot);
 946	}
 947
 948	return 0;
 949}
 950
 951static int cpr_calculate_scaling(const char *quot_offset,
 952				 struct cpr_drv *drv,
 953				 const struct fuse_corner_data *fdata,
 954				 const struct corner *corner)
 955{
 956	u32 quot_diff = 0;
 957	unsigned long freq_diff;
 958	int scaling;
 959	const struct fuse_corner *fuse, *prev_fuse;
 960	int ret;
 961
 962	fuse = corner->fuse_corner;
 963	prev_fuse = fuse - 1;
 964
 965	if (quot_offset) {
 966		ret = nvmem_cell_read_variable_le_u32(drv->dev, quot_offset, &quot_diff);
 967		if (ret)
 968			return ret;
 969
 970		quot_diff *= fdata->quot_offset_scale;
 971		quot_diff += fdata->quot_offset_adjust;
 972	} else {
 973		quot_diff = fuse->quot - prev_fuse->quot;
 974	}
 975
 976	freq_diff = fuse->max_freq - prev_fuse->max_freq;
 977	freq_diff /= 1000000; /* Convert to MHz */
 978	scaling = 1000 * quot_diff / freq_diff;
 979	return min(scaling, fdata->max_quot_scale);
 980}
 981
 982static int cpr_interpolate(const struct corner *corner, int step_volt,
 983			   const struct fuse_corner_data *fdata)
 984{
 985	unsigned long f_high, f_low, f_diff;
 986	int uV_high, uV_low, uV;
 987	u64 temp, temp_limit;
 988	const struct fuse_corner *fuse, *prev_fuse;
 989
 990	fuse = corner->fuse_corner;
 991	prev_fuse = fuse - 1;
 992
 993	f_high = fuse->max_freq;
 994	f_low = prev_fuse->max_freq;
 995	uV_high = fuse->uV;
 996	uV_low = prev_fuse->uV;
 997	f_diff = fuse->max_freq - corner->freq;
 998
 999	/*
1000	 * Don't interpolate in the wrong direction. This could happen
1001	 * if the adjusted fuse voltage overlaps with the previous fuse's
1002	 * adjusted voltage.
1003	 */
1004	if (f_high <= f_low || uV_high <= uV_low || f_high <= corner->freq)
1005		return corner->uV;
1006
1007	temp = f_diff * (uV_high - uV_low);
1008	temp = div64_ul(temp, f_high - f_low);
1009
1010	/*
1011	 * max_volt_scale has units of uV/MHz while freq values
1012	 * have units of Hz.  Divide by 1000000 to convert to.
1013	 */
1014	temp_limit = f_diff * fdata->max_volt_scale;
1015	do_div(temp_limit, 1000000);
1016
1017	uV = uV_high - min(temp, temp_limit);
1018	return roundup(uV, step_volt);
1019}
1020
1021static unsigned int cpr_get_fuse_corner(struct dev_pm_opp *opp)
1022{
1023	struct device_node *np;
1024	unsigned int fuse_corner = 0;
1025
1026	np = dev_pm_opp_get_of_node(opp);
1027	if (of_property_read_u32(np, "qcom,opp-fuse-level", &fuse_corner))
1028		pr_err("%s: missing 'qcom,opp-fuse-level' property\n",
1029		       __func__);
1030
1031	of_node_put(np);
1032
1033	return fuse_corner;
1034}
1035
1036static unsigned long cpr_get_opp_hz_for_req(struct dev_pm_opp *ref,
1037					    struct device *cpu_dev)
1038{
1039	struct device_node *ref_np __free(device_node) = NULL;
1040	struct device_node *desc_np __free(device_node) =
1041		dev_pm_opp_of_get_opp_desc_node(cpu_dev);
1042
1043	if (!desc_np)
1044		return 0;
1045
1046	ref_np = dev_pm_opp_get_of_node(ref);
1047	if (!ref_np)
1048		return 0;
1049
1050	for_each_available_child_of_node_scoped(desc_np, child_np) {
1051		struct device_node *child_req_np __free(device_node) =
1052			of_parse_phandle(child_np, "required-opps", 0);
1053
1054		if (child_req_np == ref_np) {
1055			u64 rate = 0;
1056
1057			of_property_read_u64(child_np, "opp-hz", &rate);
1058			return (unsigned long) rate;
1059		}
1060	}
1061
1062	return 0;
1063}
1064
1065static int cpr_corner_init(struct cpr_drv *drv)
1066{
1067	const struct cpr_desc *desc = drv->desc;
1068	const struct cpr_fuse *fuses = drv->cpr_fuses;
1069	int i, level, scaling = 0;
1070	unsigned int fnum, fc;
1071	const char *quot_offset;
1072	struct fuse_corner *fuse, *prev_fuse;
1073	struct corner *corner, *end;
1074	struct corner_data *cdata;
1075	const struct fuse_corner_data *fdata;
1076	bool apply_scaling;
1077	unsigned long freq_diff, freq_diff_mhz;
1078	unsigned long freq;
1079	int step_volt = regulator_get_linear_step(drv->vdd_apc);
1080	struct dev_pm_opp *opp;
1081
1082	if (!step_volt)
1083		return -EINVAL;
1084
1085	corner = drv->corners;
1086	end = &corner[drv->num_corners - 1];
1087
1088	cdata = devm_kcalloc(drv->dev, drv->num_corners,
1089			     sizeof(struct corner_data),
1090			     GFP_KERNEL);
1091	if (!cdata)
1092		return -ENOMEM;
1093
1094	/*
1095	 * Store maximum frequency for each fuse corner based on the frequency
1096	 * plan
1097	 */
1098	for (level = 1; level <= drv->num_corners; level++) {
1099		opp = dev_pm_opp_find_level_exact(&drv->pd.dev, level);
1100		if (IS_ERR(opp))
1101			return -EINVAL;
1102		fc = cpr_get_fuse_corner(opp);
1103		if (!fc) {
1104			dev_pm_opp_put(opp);
1105			return -EINVAL;
1106		}
1107		fnum = fc - 1;
1108		freq = cpr_get_opp_hz_for_req(opp, drv->attached_cpu_dev);
1109		if (!freq) {
1110			dev_pm_opp_put(opp);
1111			return -EINVAL;
1112		}
1113		cdata[level - 1].fuse_corner = fnum;
1114		cdata[level - 1].freq = freq;
1115
1116		fuse = &drv->fuse_corners[fnum];
1117		dev_dbg(drv->dev, "freq: %lu level: %u fuse level: %u\n",
1118			freq, dev_pm_opp_get_level(opp) - 1, fnum);
1119		if (freq > fuse->max_freq)
1120			fuse->max_freq = freq;
1121		dev_pm_opp_put(opp);
1122	}
1123
1124	/*
1125	 * Get the quotient adjustment scaling factor, according to:
1126	 *
1127	 * scaling = min(1000 * (QUOT(corner_N) - QUOT(corner_N-1))
1128	 *		/ (freq(corner_N) - freq(corner_N-1)), max_factor)
1129	 *
1130	 * QUOT(corner_N):	quotient read from fuse for fuse corner N
1131	 * QUOT(corner_N-1):	quotient read from fuse for fuse corner (N - 1)
1132	 * freq(corner_N):	max frequency in MHz supported by fuse corner N
1133	 * freq(corner_N-1):	max frequency in MHz supported by fuse corner
1134	 *			 (N - 1)
1135	 *
1136	 * Then walk through the corners mapped to each fuse corner
1137	 * and calculate the quotient adjustment for each one using the
1138	 * following formula:
1139	 *
1140	 * quot_adjust = (freq_max - freq_corner) * scaling / 1000
1141	 *
1142	 * freq_max: max frequency in MHz supported by the fuse corner
1143	 * freq_corner: frequency in MHz corresponding to the corner
1144	 * scaling: calculated from above equation
1145	 *
1146	 *
1147	 *     +                           +
1148	 *     |                         v |
1149	 *   q |           f c           o |           f c
1150	 *   u |         c               l |         c
1151	 *   o |       f                 t |       f
1152	 *   t |     c                   a |     c
1153	 *     | c f                     g | c f
1154	 *     |                         e |
1155	 *     +---------------            +----------------
1156	 *       0 1 2 3 4 5 6               0 1 2 3 4 5 6
1157	 *          corner                      corner
1158	 *
1159	 *    c = corner
1160	 *    f = fuse corner
1161	 *
1162	 */
1163	for (apply_scaling = false, i = 0; corner <= end; corner++, i++) {
1164		fnum = cdata[i].fuse_corner;
1165		fdata = &desc->cpr_fuses.fuse_corner_data[fnum];
1166		quot_offset = fuses[fnum].quotient_offset;
1167		fuse = &drv->fuse_corners[fnum];
1168		if (fnum)
1169			prev_fuse = &drv->fuse_corners[fnum - 1];
1170		else
1171			prev_fuse = NULL;
1172
1173		corner->fuse_corner = fuse;
1174		corner->freq = cdata[i].freq;
1175		corner->uV = fuse->uV;
1176
1177		if (prev_fuse && cdata[i - 1].freq == prev_fuse->max_freq) {
1178			scaling = cpr_calculate_scaling(quot_offset, drv,
1179							fdata, corner);
1180			if (scaling < 0)
1181				return scaling;
1182
1183			apply_scaling = true;
1184		} else if (corner->freq == fuse->max_freq) {
1185			/* This is a fuse corner; don't scale anything */
1186			apply_scaling = false;
1187		}
1188
1189		if (apply_scaling) {
1190			freq_diff = fuse->max_freq - corner->freq;
1191			freq_diff_mhz = freq_diff / 1000000;
1192			corner->quot_adjust = scaling * freq_diff_mhz / 1000;
1193
1194			corner->uV = cpr_interpolate(corner, step_volt, fdata);
1195		}
1196
1197		corner->max_uV = fuse->max_uV;
1198		corner->min_uV = fuse->min_uV;
1199		corner->uV = clamp(corner->uV, corner->min_uV, corner->max_uV);
1200		corner->last_uV = corner->uV;
1201
1202		/* Reduce the ceiling voltage if needed */
1203		if (desc->reduce_to_corner_uV && corner->uV < corner->max_uV)
1204			corner->max_uV = corner->uV;
1205		else if (desc->reduce_to_fuse_uV && fuse->uV < corner->max_uV)
1206			corner->max_uV = max(corner->min_uV, fuse->uV);
1207
1208		dev_dbg(drv->dev, "corner %d: [%d %d %d] quot %d\n", i,
1209			corner->min_uV, corner->uV, corner->max_uV,
1210			fuse->quot - corner->quot_adjust);
1211	}
1212
1213	return 0;
1214}
1215
1216static const struct cpr_fuse *cpr_get_fuses(struct cpr_drv *drv)
1217{
1218	const struct cpr_desc *desc = drv->desc;
1219	struct cpr_fuse *fuses;
1220	int i;
1221
1222	fuses = devm_kcalloc(drv->dev, desc->num_fuse_corners,
1223			     sizeof(struct cpr_fuse),
1224			     GFP_KERNEL);
1225	if (!fuses)
1226		return ERR_PTR(-ENOMEM);
1227
1228	for (i = 0; i < desc->num_fuse_corners; i++) {
1229		char tbuf[32];
1230
1231		snprintf(tbuf, 32, "cpr_ring_osc%d", i + 1);
1232		fuses[i].ring_osc = devm_kstrdup(drv->dev, tbuf, GFP_KERNEL);
1233		if (!fuses[i].ring_osc)
1234			return ERR_PTR(-ENOMEM);
1235
1236		snprintf(tbuf, 32, "cpr_init_voltage%d", i + 1);
1237		fuses[i].init_voltage = devm_kstrdup(drv->dev, tbuf,
1238						     GFP_KERNEL);
1239		if (!fuses[i].init_voltage)
1240			return ERR_PTR(-ENOMEM);
1241
1242		snprintf(tbuf, 32, "cpr_quotient%d", i + 1);
1243		fuses[i].quotient = devm_kstrdup(drv->dev, tbuf, GFP_KERNEL);
1244		if (!fuses[i].quotient)
1245			return ERR_PTR(-ENOMEM);
1246
1247		snprintf(tbuf, 32, "cpr_quotient_offset%d", i + 1);
1248		fuses[i].quotient_offset = devm_kstrdup(drv->dev, tbuf,
1249							GFP_KERNEL);
1250		if (!fuses[i].quotient_offset)
1251			return ERR_PTR(-ENOMEM);
1252	}
1253
1254	return fuses;
1255}
1256
1257static void cpr_set_loop_allowed(struct cpr_drv *drv)
1258{
1259	drv->loop_disabled = false;
1260}
1261
1262static int cpr_init_parameters(struct cpr_drv *drv)
1263{
1264	const struct cpr_desc *desc = drv->desc;
1265	struct clk *clk;
1266
1267	clk = clk_get(drv->dev, "ref");
1268	if (IS_ERR(clk))
1269		return PTR_ERR(clk);
1270
1271	drv->ref_clk_khz = clk_get_rate(clk) / 1000;
1272	clk_put(clk);
1273
1274	if (desc->timer_cons_up > RBIF_TIMER_ADJ_CONS_UP_MASK ||
1275	    desc->timer_cons_down > RBIF_TIMER_ADJ_CONS_DOWN_MASK ||
1276	    desc->up_threshold > RBCPR_CTL_UP_THRESHOLD_MASK ||
1277	    desc->down_threshold > RBCPR_CTL_DN_THRESHOLD_MASK ||
1278	    desc->idle_clocks > RBCPR_STEP_QUOT_IDLE_CLK_MASK ||
1279	    desc->clamp_timer_interval > RBIF_TIMER_ADJ_CLAMP_INT_MASK)
1280		return -EINVAL;
1281
1282	dev_dbg(drv->dev, "up threshold = %u, down threshold = %u\n",
1283		desc->up_threshold, desc->down_threshold);
1284
1285	return 0;
1286}
1287
1288static int cpr_find_initial_corner(struct cpr_drv *drv)
1289{
1290	unsigned long rate;
1291	const struct corner *end;
1292	struct corner *iter;
1293	unsigned int i = 0;
1294
1295	if (!drv->cpu_clk) {
1296		dev_err(drv->dev, "cannot get rate from NULL clk\n");
1297		return -EINVAL;
1298	}
1299
1300	end = &drv->corners[drv->num_corners - 1];
1301	rate = clk_get_rate(drv->cpu_clk);
1302
1303	/*
1304	 * Some bootloaders set a CPU clock frequency that is not defined
1305	 * in the OPP table. When running at an unlisted frequency,
1306	 * cpufreq_online() will change to the OPP which has the lowest
1307	 * frequency, at or above the unlisted frequency.
1308	 * Since cpufreq_online() always "rounds up" in the case of an
1309	 * unlisted frequency, this function always "rounds down" in case
1310	 * of an unlisted frequency. That way, when cpufreq_online()
1311	 * triggers the first ever call to cpr_set_performance_state(),
1312	 * it will correctly determine the direction as UP.
1313	 */
1314	for (iter = drv->corners; iter <= end; iter++) {
1315		if (iter->freq > rate)
1316			break;
1317		i++;
1318		if (iter->freq == rate) {
1319			drv->corner = iter;
1320			break;
1321		}
1322		if (iter->freq < rate)
1323			drv->corner = iter;
1324	}
1325
1326	if (!drv->corner) {
1327		dev_err(drv->dev, "boot up corner not found\n");
1328		return -EINVAL;
1329	}
1330
1331	dev_dbg(drv->dev, "boot up perf state: %u\n", i);
1332
1333	return 0;
1334}
1335
1336static const struct cpr_desc qcs404_cpr_desc = {
1337	.num_fuse_corners = 3,
1338	.min_diff_quot = CPR_FUSE_MIN_QUOT_DIFF,
1339	.step_quot = (int []){ 25, 25, 25, },
1340	.timer_delay_us = 5000,
1341	.timer_cons_up = 0,
1342	.timer_cons_down = 2,
1343	.up_threshold = 1,
1344	.down_threshold = 3,
1345	.idle_clocks = 15,
1346	.gcnt_us = 1,
1347	.vdd_apc_step_up_limit = 1,
1348	.vdd_apc_step_down_limit = 1,
1349	.cpr_fuses = {
1350		.init_voltage_step = 8000,
1351		.init_voltage_width = 6,
1352		.fuse_corner_data = (struct fuse_corner_data[]){
1353			/* fuse corner 0 */
1354			{
1355				.ref_uV = 1224000,
1356				.max_uV = 1224000,
1357				.min_uV = 1048000,
1358				.max_volt_scale = 0,
1359				.max_quot_scale = 0,
1360				.quot_offset = 0,
1361				.quot_scale = 1,
1362				.quot_adjust = 0,
1363				.quot_offset_scale = 5,
1364				.quot_offset_adjust = 0,
1365			},
1366			/* fuse corner 1 */
1367			{
1368				.ref_uV = 1288000,
1369				.max_uV = 1288000,
1370				.min_uV = 1048000,
1371				.max_volt_scale = 2000,
1372				.max_quot_scale = 1400,
1373				.quot_offset = 0,
1374				.quot_scale = 1,
1375				.quot_adjust = -20,
1376				.quot_offset_scale = 5,
1377				.quot_offset_adjust = 0,
1378			},
1379			/* fuse corner 2 */
1380			{
1381				.ref_uV = 1352000,
1382				.max_uV = 1384000,
1383				.min_uV = 1088000,
1384				.max_volt_scale = 2000,
1385				.max_quot_scale = 1400,
1386				.quot_offset = 0,
1387				.quot_scale = 1,
1388				.quot_adjust = 0,
1389				.quot_offset_scale = 5,
1390				.quot_offset_adjust = 0,
1391			},
1392		},
1393	},
1394};
1395
1396static const struct acc_desc qcs404_acc_desc = {
1397	.settings = (struct reg_sequence[]){
1398		{ 0xb120, 0x1041040 },
1399		{ 0xb124, 0x41 },
1400		{ 0xb120, 0x0 },
1401		{ 0xb124, 0x0 },
1402		{ 0xb120, 0x0 },
1403		{ 0xb124, 0x0 },
1404	},
1405	.config = (struct reg_sequence[]){
1406		{ 0xb138, 0xff },
1407		{ 0xb130, 0x5555 },
1408	},
1409	.num_regs_per_fuse = 2,
1410};
1411
1412static const struct cpr_acc_desc qcs404_cpr_acc_desc = {
1413	.cpr_desc = &qcs404_cpr_desc,
1414	.acc_desc = &qcs404_acc_desc,
1415};
1416
1417static int cpr_power_off(struct generic_pm_domain *domain)
1418{
1419	struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1420
1421	return cpr_disable(drv);
1422}
1423
1424static int cpr_power_on(struct generic_pm_domain *domain)
1425{
1426	struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1427
1428	return cpr_enable(drv);
1429}
1430
1431static int cpr_pd_attach_dev(struct generic_pm_domain *domain,
1432			     struct device *dev)
1433{
1434	struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1435	const struct acc_desc *acc_desc = drv->acc_desc;
1436	int ret;
1437
1438	guard(mutex)(&drv->lock);
1439
1440	dev_dbg(drv->dev, "attach callback for: %s\n", dev_name(dev));
1441
1442	/*
1443	 * This driver only supports scaling voltage for a CPU cluster
1444	 * where all CPUs in the cluster share a single regulator.
1445	 * Therefore, save the struct device pointer only for the first
1446	 * CPU device that gets attached. There is no need to do any
1447	 * additional initialization when further CPUs get attached.
1448	 */
1449	if (drv->attached_cpu_dev)
1450		return 0;
1451
1452	/*
1453	 * cpr_scale_voltage() requires the direction (if we are changing
1454	 * to a higher or lower OPP). The first time
1455	 * cpr_set_performance_state() is called, there is no previous
1456	 * performance state defined. Therefore, we call
1457	 * cpr_find_initial_corner() that gets the CPU clock frequency
1458	 * set by the bootloader, so that we can determine the direction
1459	 * the first time cpr_set_performance_state() is called.
1460	 */
1461	drv->cpu_clk = devm_clk_get(dev, NULL);
1462	if (IS_ERR(drv->cpu_clk))
1463		return dev_err_probe(drv->dev, PTR_ERR(drv->cpu_clk),
1464				     "could not get cpu clk\n");
1465
1466	drv->attached_cpu_dev = dev;
1467
1468	dev_dbg(drv->dev, "using cpu clk from: %s\n",
1469		dev_name(drv->attached_cpu_dev));
1470
1471	/*
1472	 * Everything related to (virtual) corners has to be initialized
1473	 * here, when attaching to the power domain, since we need to know
1474	 * the maximum frequency for each fuse corner, and this is only
1475	 * available after the cpufreq driver has attached to us.
1476	 * The reason for this is that we need to know the highest
1477	 * frequency associated with each fuse corner.
1478	 */
1479	ret = dev_pm_opp_get_opp_count(&drv->pd.dev);
1480	if (ret < 0) {
1481		dev_err(drv->dev, "could not get OPP count\n");
1482		return ret;
1483	}
1484	drv->num_corners = ret;
1485
1486	if (drv->num_corners < 2) {
1487		dev_err(drv->dev, "need at least 2 OPPs to use CPR\n");
1488		return -EINVAL;
1489	}
1490
1491	drv->corners = devm_kcalloc(drv->dev, drv->num_corners,
1492				    sizeof(*drv->corners),
1493				    GFP_KERNEL);
1494	if (!drv->corners)
1495		return -ENOMEM;
1496
1497	ret = cpr_corner_init(drv);
1498	if (ret)
1499		return ret;
1500
1501	cpr_set_loop_allowed(drv);
1502
1503	ret = cpr_init_parameters(drv);
1504	if (ret)
1505		return ret;
1506
1507	/* Configure CPR HW but keep it disabled */
1508	ret = cpr_config(drv);
1509	if (ret)
1510		return ret;
1511
1512	ret = cpr_find_initial_corner(drv);
1513	if (ret)
1514		return ret;
1515
1516	if (acc_desc->config)
1517		regmap_multi_reg_write(drv->tcsr, acc_desc->config,
1518				       acc_desc->num_regs_per_fuse);
1519
1520	/* Enable ACC if required */
1521	if (acc_desc->enable_mask)
1522		regmap_update_bits(drv->tcsr, acc_desc->enable_reg,
1523				   acc_desc->enable_mask,
1524				   acc_desc->enable_mask);
1525
1526	dev_info(drv->dev, "driver initialized with %u OPPs\n",
1527		 drv->num_corners);
1528
1529	return 0;
1530}
1531
1532static int cpr_debug_info_show(struct seq_file *s, void *unused)
1533{
1534	u32 gcnt, ro_sel, ctl, irq_status, reg, error_steps;
1535	u32 step_dn, step_up, error, error_lt0, busy;
1536	struct cpr_drv *drv = s->private;
1537	struct fuse_corner *fuse_corner;
1538	struct corner *corner;
1539
1540	corner = drv->corner;
1541	fuse_corner = corner->fuse_corner;
1542
1543	seq_printf(s, "corner, current_volt = %d uV\n",
1544		       corner->last_uV);
1545
1546	ro_sel = fuse_corner->ring_osc_idx;
1547	gcnt = cpr_read(drv, REG_RBCPR_GCNT_TARGET(ro_sel));
1548	seq_printf(s, "rbcpr_gcnt_target (%u) = %#02X\n", ro_sel, gcnt);
1549
1550	ctl = cpr_read(drv, REG_RBCPR_CTL);
1551	seq_printf(s, "rbcpr_ctl = %#02X\n", ctl);
1552
1553	irq_status = cpr_read(drv, REG_RBIF_IRQ_STATUS);
1554	seq_printf(s, "rbcpr_irq_status = %#02X\n", irq_status);
1555
1556	reg = cpr_read(drv, REG_RBCPR_RESULT_0);
1557	seq_printf(s, "rbcpr_result_0 = %#02X\n", reg);
1558
1559	step_dn = reg & 0x01;
1560	step_up = (reg >> RBCPR_RESULT0_STEP_UP_SHIFT) & 0x01;
1561	seq_printf(s, "  [step_dn = %u", step_dn);
1562
1563	seq_printf(s, ", step_up = %u", step_up);
1564
1565	error_steps = (reg >> RBCPR_RESULT0_ERROR_STEPS_SHIFT)
1566				& RBCPR_RESULT0_ERROR_STEPS_MASK;
1567	seq_printf(s, ", error_steps = %u", error_steps);
1568
1569	error = (reg >> RBCPR_RESULT0_ERROR_SHIFT) & RBCPR_RESULT0_ERROR_MASK;
1570	seq_printf(s, ", error = %u", error);
1571
1572	error_lt0 = (reg >> RBCPR_RESULT0_ERROR_LT0_SHIFT) & 0x01;
1573	seq_printf(s, ", error_lt_0 = %u", error_lt0);
1574
1575	busy = (reg >> RBCPR_RESULT0_BUSY_SHIFT) & 0x01;
1576	seq_printf(s, ", busy = %u]\n", busy);
1577
1578	return 0;
1579}
1580DEFINE_SHOW_ATTRIBUTE(cpr_debug_info);
1581
1582static void cpr_debugfs_init(struct cpr_drv *drv)
1583{
1584	drv->debugfs = debugfs_create_dir("qcom_cpr", NULL);
1585
1586	debugfs_create_file("debug_info", 0444, drv->debugfs,
1587			    drv, &cpr_debug_info_fops);
1588}
1589
1590static int cpr_probe(struct platform_device *pdev)
1591{
1592	struct device *dev = &pdev->dev;
1593	struct cpr_drv *drv;
1594	int irq, ret;
1595	const struct cpr_acc_desc *data;
1596	struct device_node *np;
1597	u32 cpr_rev = FUSE_REVISION_UNKNOWN;
1598
1599	data = of_device_get_match_data(dev);
1600	if (!data || !data->cpr_desc || !data->acc_desc)
1601		return -EINVAL;
1602
1603	drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
1604	if (!drv)
1605		return -ENOMEM;
1606	drv->dev = dev;
1607	drv->desc = data->cpr_desc;
1608	drv->acc_desc = data->acc_desc;
1609
1610	drv->fuse_corners = devm_kcalloc(dev, drv->desc->num_fuse_corners,
1611					 sizeof(*drv->fuse_corners),
1612					 GFP_KERNEL);
1613	if (!drv->fuse_corners)
1614		return -ENOMEM;
1615
1616	np = of_parse_phandle(dev->of_node, "acc-syscon", 0);
1617	if (!np)
1618		return -ENODEV;
1619
1620	drv->tcsr = syscon_node_to_regmap(np);
1621	of_node_put(np);
1622	if (IS_ERR(drv->tcsr))
1623		return PTR_ERR(drv->tcsr);
1624
1625	drv->base = devm_platform_ioremap_resource(pdev, 0);
1626	if (IS_ERR(drv->base))
1627		return PTR_ERR(drv->base);
1628
1629	irq = platform_get_irq(pdev, 0);
1630	if (irq < 0)
1631		return -EINVAL;
1632
1633	drv->vdd_apc = devm_regulator_get(dev, "vdd-apc");
1634	if (IS_ERR(drv->vdd_apc))
1635		return PTR_ERR(drv->vdd_apc);
1636
1637	/*
1638	 * Initialize fuse corners, since it simply depends
1639	 * on data in efuses.
1640	 * Everything related to (virtual) corners has to be
1641	 * initialized after attaching to the power domain,
1642	 * since it depends on the CPU's OPP table.
1643	 */
1644	ret = nvmem_cell_read_variable_le_u32(dev, "cpr_fuse_revision", &cpr_rev);
1645	if (ret)
1646		return ret;
1647
1648	drv->cpr_fuses = cpr_get_fuses(drv);
1649	if (IS_ERR(drv->cpr_fuses))
1650		return PTR_ERR(drv->cpr_fuses);
1651
1652	ret = cpr_populate_ring_osc_idx(drv);
1653	if (ret)
1654		return ret;
1655
1656	ret = cpr_fuse_corner_init(drv);
1657	if (ret)
1658		return ret;
1659
1660	mutex_init(&drv->lock);
1661
1662	ret = devm_request_threaded_irq(dev, irq, NULL,
1663					cpr_irq_handler,
1664					IRQF_ONESHOT | IRQF_TRIGGER_RISING,
1665					"cpr", drv);
1666	if (ret)
1667		return ret;
1668
1669	drv->pd.name = devm_kstrdup_const(dev, dev->of_node->full_name,
1670					  GFP_KERNEL);
1671	if (!drv->pd.name)
1672		return -EINVAL;
1673
1674	drv->pd.power_off = cpr_power_off;
1675	drv->pd.power_on = cpr_power_on;
1676	drv->pd.set_performance_state = cpr_set_performance_state;
1677	drv->pd.attach_dev = cpr_pd_attach_dev;
1678
1679	ret = pm_genpd_init(&drv->pd, NULL, true);
1680	if (ret)
1681		return ret;
1682
1683	ret = of_genpd_add_provider_simple(dev->of_node, &drv->pd);
1684	if (ret)
1685		goto err_remove_genpd;
1686
1687	platform_set_drvdata(pdev, drv);
1688	cpr_debugfs_init(drv);
1689
1690	return 0;
1691
1692err_remove_genpd:
1693	pm_genpd_remove(&drv->pd);
1694	return ret;
1695}
1696
1697static void cpr_remove(struct platform_device *pdev)
1698{
1699	struct cpr_drv *drv = platform_get_drvdata(pdev);
1700
1701	if (cpr_is_allowed(drv)) {
1702		cpr_ctl_disable(drv);
1703		cpr_irq_set(drv, 0);
1704	}
1705
1706	of_genpd_del_provider(pdev->dev.of_node);
1707	pm_genpd_remove(&drv->pd);
1708
1709	debugfs_remove_recursive(drv->debugfs);
1710}
1711
1712static const struct of_device_id cpr_match_table[] = {
1713	{ .compatible = "qcom,qcs404-cpr", .data = &qcs404_cpr_acc_desc },
1714	{ }
1715};
1716MODULE_DEVICE_TABLE(of, cpr_match_table);
1717
1718static struct platform_driver cpr_driver = {
1719	.probe		= cpr_probe,
1720	.remove		= cpr_remove,
1721	.driver		= {
1722		.name	= "qcom-cpr",
1723		.of_match_table = cpr_match_table,
1724	},
1725};
1726module_platform_driver(cpr_driver);
1727
1728MODULE_DESCRIPTION("Core Power Reduction (CPR) driver");
1729MODULE_LICENSE("GPL v2");