rpmh-rsc.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
   4 * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved.
   5 */
   6
   7#define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
   8
   9#include <linux/atomic.h>
  10#include <linux/cpu_pm.h>
  11#include <linux/delay.h>
  12#include <linux/interrupt.h>
  13#include <linux/io.h>
  14#include <linux/iopoll.h>
  15#include <linux/kernel.h>
  16#include <linux/ktime.h>
  17#include <linux/list.h>
  18#include <linux/module.h>
  19#include <linux/notifier.h>
  20#include <linux/of.h>
  21#include <linux/of_irq.h>
  22#include <linux/of_platform.h>
  23#include <linux/platform_device.h>
  24#include <linux/pm_domain.h>
  25#include <linux/pm_runtime.h>
  26#include <linux/slab.h>
  27#include <linux/spinlock.h>
  28#include <linux/wait.h>
  29
  30#include <clocksource/arm_arch_timer.h>
  31#include <soc/qcom/cmd-db.h>
  32#include <soc/qcom/tcs.h>
  33#include <dt-bindings/soc/qcom,rpmh-rsc.h>
  34
  35#include "rpmh-internal.h"
  36
  37#define CREATE_TRACE_POINTS
  38#include "trace-rpmh.h"
  39
  40
  41#define RSC_DRV_ID			0
  42
  43#define MAJOR_VER_MASK			0xFF
  44#define MAJOR_VER_SHIFT			16
  45#define MINOR_VER_MASK			0xFF
  46#define MINOR_VER_SHIFT			8
  47
  48enum {
  49	RSC_DRV_TCS_OFFSET,
  50	RSC_DRV_CMD_OFFSET,
  51	DRV_SOLVER_CONFIG,
  52	DRV_PRNT_CHLD_CONFIG,
  53	RSC_DRV_IRQ_ENABLE,
  54	RSC_DRV_IRQ_STATUS,
  55	RSC_DRV_IRQ_CLEAR,
  56	RSC_DRV_CMD_WAIT_FOR_CMPL,
  57	RSC_DRV_CONTROL,
  58	RSC_DRV_STATUS,
  59	RSC_DRV_CMD_ENABLE,
  60	RSC_DRV_CMD_MSGID,
  61	RSC_DRV_CMD_ADDR,
  62	RSC_DRV_CMD_DATA,
  63	RSC_DRV_CMD_STATUS,
  64	RSC_DRV_CMD_RESP_DATA,
  65};
  66
  67/* DRV HW Solver Configuration Information Register */
  68#define DRV_HW_SOLVER_MASK		1
  69#define DRV_HW_SOLVER_SHIFT		24
  70
  71/* DRV TCS Configuration Information Register */
  72#define DRV_NUM_TCS_MASK		0x3F
  73#define DRV_NUM_TCS_SHIFT		6
  74#define DRV_NCPT_MASK			0x1F
  75#define DRV_NCPT_SHIFT			27
  76
  77/* Offsets for CONTROL TCS Registers */
  78#define RSC_DRV_CTL_TCS_DATA_HI		0x38
  79#define RSC_DRV_CTL_TCS_DATA_HI_MASK	0xFFFFFF
  80#define RSC_DRV_CTL_TCS_DATA_HI_VALID	BIT(31)
  81#define RSC_DRV_CTL_TCS_DATA_LO		0x40
  82#define RSC_DRV_CTL_TCS_DATA_LO_MASK	0xFFFFFFFF
  83#define RSC_DRV_CTL_TCS_DATA_SIZE	32
  84
  85#define TCS_AMC_MODE_ENABLE		BIT(16)
  86#define TCS_AMC_MODE_TRIGGER		BIT(24)
  87
  88/* TCS CMD register bit mask */
  89#define CMD_MSGID_LEN			8
  90#define CMD_MSGID_RESP_REQ		BIT(8)
  91#define CMD_MSGID_WRITE			BIT(16)
  92#define CMD_STATUS_ISSUED		BIT(8)
  93#define CMD_STATUS_COMPL		BIT(16)
  94
  95/*
  96 * Here's a high level overview of how all the registers in RPMH work
  97 * together:
  98 *
  99 * - The main rpmh-rsc address is the base of a register space that can
 100 *   be used to find overall configuration of the hardware
 101 *   (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
 102 *   space are all the TCS blocks. The offset of the TCS blocks is
 103 *   specified in the device tree by "qcom,tcs-offset" and used to
 104 *   compute tcs_base.
 105 * - TCS blocks come one after another. Type, count, and order are
 106 *   specified by the device tree as "qcom,tcs-config".
 107 * - Each TCS block has some registers, then space for up to 16 commands.
 108 *   Note that though address space is reserved for 16 commands, fewer
 109 *   might be present. See ncpt (num cmds per TCS).
 110 *
 111 * Here's a picture:
 112 *
 113 *  +---------------------------------------------------+
 114 *  |RSC                                                |
 115 *  | ctrl                                              |
 116 *  |                                                   |
 117 *  | Drvs:                                             |
 118 *  | +-----------------------------------------------+ |
 119 *  | |DRV0                                           | |
 120 *  | | ctrl/config                                   | |
 121 *  | | IRQ                                           | |
 122 *  | |                                               | |
 123 *  | | TCSes:                                        | |
 124 *  | | +------------------------------------------+  | |
 125 *  | | |TCS0  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
 126 *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
 127 *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
 128 *  | | +------------------------------------------+  | |
 129 *  | | +------------------------------------------+  | |
 130 *  | | |TCS1  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
 131 *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
 132 *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
 133 *  | | +------------------------------------------+  | |
 134 *  | | +------------------------------------------+  | |
 135 *  | | |TCS2  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
 136 *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
 137 *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
 138 *  | | +------------------------------------------+  | |
 139 *  | |                    ......                     | |
 140 *  | +-----------------------------------------------+ |
 141 *  | +-----------------------------------------------+ |
 142 *  | |DRV1                                           | |
 143 *  | | (same as DRV0)                                | |
 144 *  | +-----------------------------------------------+ |
 145 *  |                      ......                       |
 146 *  +---------------------------------------------------+
 147 */
 148
 149#define USECS_TO_CYCLES(time_usecs)			\
 150	xloops_to_cycles((time_usecs) * 0x10C7UL)
 151
 152static inline unsigned long xloops_to_cycles(u64 xloops)
 153{
 154	return (xloops * loops_per_jiffy * HZ) >> 32;
 155}
 156
 157static u32 rpmh_rsc_reg_offset_ver_2_7[] = {
 158	[RSC_DRV_TCS_OFFSET]		= 672,
 159	[RSC_DRV_CMD_OFFSET]		= 20,
 160	[DRV_SOLVER_CONFIG]		= 0x04,
 161	[DRV_PRNT_CHLD_CONFIG]		= 0x0C,
 162	[RSC_DRV_IRQ_ENABLE]		= 0x00,
 163	[RSC_DRV_IRQ_STATUS]		= 0x04,
 164	[RSC_DRV_IRQ_CLEAR]		= 0x08,
 165	[RSC_DRV_CMD_WAIT_FOR_CMPL]	= 0x10,
 166	[RSC_DRV_CONTROL]		= 0x14,
 167	[RSC_DRV_STATUS]		= 0x18,
 168	[RSC_DRV_CMD_ENABLE]		= 0x1C,
 169	[RSC_DRV_CMD_MSGID]		= 0x30,
 170	[RSC_DRV_CMD_ADDR]		= 0x34,
 171	[RSC_DRV_CMD_DATA]		= 0x38,
 172	[RSC_DRV_CMD_STATUS]		= 0x3C,
 173	[RSC_DRV_CMD_RESP_DATA]		= 0x40,
 174};
 175
 176static u32 rpmh_rsc_reg_offset_ver_3_0[] = {
 177	[RSC_DRV_TCS_OFFSET]		= 672,
 178	[RSC_DRV_CMD_OFFSET]		= 24,
 179	[DRV_SOLVER_CONFIG]		= 0x04,
 180	[DRV_PRNT_CHLD_CONFIG]		= 0x0C,
 181	[RSC_DRV_IRQ_ENABLE]		= 0x00,
 182	[RSC_DRV_IRQ_STATUS]		= 0x04,
 183	[RSC_DRV_IRQ_CLEAR]		= 0x08,
 184	[RSC_DRV_CMD_WAIT_FOR_CMPL]	= 0x20,
 185	[RSC_DRV_CONTROL]		= 0x24,
 186	[RSC_DRV_STATUS]		= 0x28,
 187	[RSC_DRV_CMD_ENABLE]		= 0x2C,
 188	[RSC_DRV_CMD_MSGID]		= 0x34,
 189	[RSC_DRV_CMD_ADDR]		= 0x38,
 190	[RSC_DRV_CMD_DATA]		= 0x3C,
 191	[RSC_DRV_CMD_STATUS]		= 0x40,
 192	[RSC_DRV_CMD_RESP_DATA]		= 0x44,
 193};
 194
 195static inline void __iomem *
 196tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
 197{
 198	return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg;
 199}
 200
 201static inline void __iomem *
 202tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
 203{
 204	return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id;
 205}
 206
 207static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
 208			int cmd_id)
 209{
 210	return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
 211}
 212
 213static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
 214{
 215	return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
 216}
 217
 218static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
 219			  int cmd_id, u32 data)
 220{
 221	writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
 222}
 223
 224static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
 225			  u32 data)
 226{
 227	writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
 228}
 229
 230static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
 231			       u32 data)
 232{
 233	int i;
 234
 235	writel(data, tcs_reg_addr(drv, reg, tcs_id));
 236
 237	/*
 238	 * Wait until we read back the same value.  Use a counter rather than
 239	 * ktime for timeout since this may be called after timekeeping stops.
 240	 */
 241	for (i = 0; i < USEC_PER_SEC; i++) {
 242		if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)
 243			return;
 244		udelay(1);
 245	}
 246	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
 247	       data, tcs_id, reg);
 248}
 249
 250/**
 251 * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
 252 * @drv:  The RSC controller.
 253 * @type: SLEEP_TCS or WAKE_TCS
 254 *
 255 * This will clear the "slots" variable of the given tcs_group and also
 256 * tell the hardware to forget about all entries.
 257 *
 258 * The caller must ensure that no other RPMH actions are happening when this
 259 * function is called, since otherwise the device may immediately become
 260 * used again even before this function exits.
 261 */
 262static void tcs_invalidate(struct rsc_drv *drv, int type)
 263{
 264	int m;
 265	struct tcs_group *tcs = &drv->tcs[type];
 266
 267	/* Caller ensures nobody else is running so no lock */
 268	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))
 269		return;
 270
 271	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++)
 272		write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0);
 273
 274	bitmap_zero(tcs->slots, MAX_TCS_SLOTS);
 275}
 276
 277/**
 278 * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
 279 * @drv: The RSC controller.
 280 *
 281 * The caller must ensure that no other RPMH actions are happening when this
 282 * function is called, since otherwise the device may immediately become
 283 * used again even before this function exits.
 284 */
 285void rpmh_rsc_invalidate(struct rsc_drv *drv)
 286{
 287	tcs_invalidate(drv, SLEEP_TCS);
 288	tcs_invalidate(drv, WAKE_TCS);
 289}
 290
 291/**
 292 * get_tcs_for_msg() - Get the tcs_group used to send the given message.
 293 * @drv: The RSC controller.
 294 * @msg: The message we want to send.
 295 *
 296 * This is normally pretty straightforward except if we are trying to send
 297 * an ACTIVE_ONLY message but don't have any active_only TCSes.
 298 *
 299 * Return: A pointer to a tcs_group or an ERR_PTR.
 300 */
 301static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
 302					 const struct tcs_request *msg)
 303{
 304	int type;
 305	struct tcs_group *tcs;
 306
 307	switch (msg->state) {
 308	case RPMH_ACTIVE_ONLY_STATE:
 309		type = ACTIVE_TCS;
 310		break;
 311	case RPMH_WAKE_ONLY_STATE:
 312		type = WAKE_TCS;
 313		break;
 314	case RPMH_SLEEP_STATE:
 315		type = SLEEP_TCS;
 316		break;
 317	default:
 318		return ERR_PTR(-EINVAL);
 319	}
 320
 321	/*
 322	 * If we are making an active request on a RSC that does not have a
 323	 * dedicated TCS for active state use, then re-purpose a wake TCS to
 324	 * send active votes. This is safe because we ensure any active-only
 325	 * transfers have finished before we use it (maybe by running from
 326	 * the last CPU in PM code).
 327	 */
 328	tcs = &drv->tcs[type];
 329	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
 330		tcs = &drv->tcs[WAKE_TCS];
 331
 332	return tcs;
 333}
 334
 335/**
 336 * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
 337 * @drv:    The RSC controller.
 338 * @tcs_id: The global ID of this TCS.
 339 *
 340 * For ACTIVE_ONLY transfers we want to call back into the client when the
 341 * transfer finishes. To do this we need the "request" that the client
 342 * originally provided us. This function grabs the request that we stashed
 343 * when we started the transfer.
 344 *
 345 * This only makes sense for ACTIVE_ONLY transfers since those are the only
 346 * ones we track sending (the only ones we enable interrupts for and the only
 347 * ones we call back to the client for).
 348 *
 349 * Return: The stashed request.
 350 */
 351static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
 352						  int tcs_id)
 353{
 354	struct tcs_group *tcs;
 355	int i;
 356
 357	for (i = 0; i < TCS_TYPE_NR; i++) {
 358		tcs = &drv->tcs[i];
 359		if (tcs->mask & BIT(tcs_id))
 360			return tcs->req[tcs_id - tcs->offset];
 361	}
 362
 363	return NULL;
 364}
 365
 366/**
 367 * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
 368 * @drv:     The controller.
 369 * @tcs_id:  The global ID of this TCS.
 370 * @trigger: If true then untrigger/retrigger. If false then just untrigger.
 371 *
 372 * In the normal case we only ever call with "trigger=true" to start a
 373 * transfer. That will un-trigger/disable the TCS from the last transfer
 374 * then trigger/enable for this transfer.
 375 *
 376 * If we borrowed a wake TCS for an active-only transfer we'll also call
 377 * this function with "trigger=false" to just do the un-trigger/disable
 378 * before using the TCS for wake purposes again.
 379 *
 380 * Note that the AP is only in charge of triggering active-only transfers.
 381 * The AP never triggers sleep/wake values using this function.
 382 */
 383static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
 384{
 385	u32 enable;
 386	u32 reg = drv->regs[RSC_DRV_CONTROL];
 387
 388	/*
 389	 * HW req: Clear the DRV_CONTROL and enable TCS again
 390	 * While clearing ensure that the AMC mode trigger is cleared
 391	 * and then the mode enable is cleared.
 392	 */
 393	enable = read_tcs_reg(drv, reg, tcs_id);
 394	enable &= ~TCS_AMC_MODE_TRIGGER;
 395	write_tcs_reg_sync(drv, reg, tcs_id, enable);
 396	enable &= ~TCS_AMC_MODE_ENABLE;
 397	write_tcs_reg_sync(drv, reg, tcs_id, enable);
 398
 399	if (trigger) {
 400		/* Enable the AMC mode on the TCS and then trigger the TCS */
 401		enable = TCS_AMC_MODE_ENABLE;
 402		write_tcs_reg_sync(drv, reg, tcs_id, enable);
 403		enable |= TCS_AMC_MODE_TRIGGER;
 404		write_tcs_reg(drv, reg, tcs_id, enable);
 405	}
 406}
 407
 408/**
 409 * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
 410 * @drv:     The controller.
 411 * @tcs_id:  The global ID of this TCS.
 412 * @enable:  If true then enable; if false then disable
 413 *
 414 * We only ever call this when we borrow a wake TCS for an active-only
 415 * transfer. For active-only TCSes interrupts are always left enabled.
 416 */
 417static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
 418{
 419	u32 data;
 420	u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE];
 421
 422	data = readl_relaxed(drv->tcs_base + reg);
 423	if (enable)
 424		data |= BIT(tcs_id);
 425	else
 426		data &= ~BIT(tcs_id);
 427	writel_relaxed(data, drv->tcs_base + reg);
 428}
 429
 430/**
 431 * tcs_tx_done() - TX Done interrupt handler.
 432 * @irq: The IRQ number (ignored).
 433 * @p:   Pointer to "struct rsc_drv".
 434 *
 435 * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
 436 * IRQ for) when a transfer is done.
 437 *
 438 * Return: IRQ_HANDLED
 439 */
 440static irqreturn_t tcs_tx_done(int irq, void *p)
 441{
 442	struct rsc_drv *drv = p;
 443	int i;
 444	unsigned long irq_status;
 445	const struct tcs_request *req;
 446
 447	irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);
 448
 449	for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) {
 450		req = get_req_from_tcs(drv, i);
 451		if (WARN_ON(!req))
 452			goto skip;
 453
 454		trace_rpmh_tx_done(drv, i, req);
 455
 456		/* Clear AMC trigger & enable modes and
 457		 * disable interrupt for this TCS
 458		 */
 459		__tcs_set_trigger(drv, i, false);
 460skip:
 461		/* Reclaim the TCS */
 462		write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0);
 463		writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]);
 464		spin_lock(&drv->lock);
 465		clear_bit(i, drv->tcs_in_use);
 466		/*
 467		 * Disable interrupt for WAKE TCS to avoid being
 468		 * spammed with interrupts coming when the solver
 469		 * sends its wake votes.
 470		 */
 471		if (!drv->tcs[ACTIVE_TCS].num_tcs)
 472			enable_tcs_irq(drv, i, false);
 473		spin_unlock(&drv->lock);
 474		wake_up(&drv->tcs_wait);
 475		if (req)
 476			rpmh_tx_done(req);
 477	}
 478
 479	return IRQ_HANDLED;
 480}
 481
 482/**
 483 * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
 484 * @drv:    The controller.
 485 * @tcs_id: The global ID of this TCS.
 486 * @cmd_id: The index within the TCS to start writing.
 487 * @msg:    The message we want to send, which will contain several addr/data
 488 *          pairs to program (but few enough that they all fit in one TCS).
 489 *
 490 * This is used for all types of transfers (active, sleep, and wake).
 491 */
 492static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
 493			       const struct tcs_request *msg)
 494{
 495	u32 msgid;
 496	u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE;
 497	u32 cmd_enable = 0;
 498	struct tcs_cmd *cmd;
 499	int i, j;
 500
 501	/* Convert all commands to RR when the request has wait_for_compl set */
 502	cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
 503
 504	for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
 505		cmd = &msg->cmds[i];
 506		cmd_enable |= BIT(j);
 507		msgid = cmd_msgid;
 508		/*
 509		 * Additionally, if the cmd->wait is set, make the command
 510		 * response reqd even if the overall request was fire-n-forget.
 511		 */
 512		msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;
 513
 514		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid);
 515		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr);
 516		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data);
 517		trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd);
 518	}
 519
 520	cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);
 521	write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable);
 522}
 523
 524/**
 525 * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
 526 * @drv: The controller.
 527 * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
 528 * @msg: The message we want to send, which will contain several addr/data
 529 *       pairs to program (but few enough that they all fit in one TCS).
 530 *
 531 * This will walk through the TCSes in the group and check if any of them
 532 * appear to be sending to addresses referenced in the message. If it finds
 533 * one it'll return -EBUSY.
 534 *
 535 * Only for use for active-only transfers.
 536 *
 537 * Must be called with the drv->lock held since that protects tcs_in_use.
 538 *
 539 * Return: 0 if nothing in flight or -EBUSY if we should try again later.
 540 *         The caller must re-enable interrupts between tries since that's
 541 *         the only way tcs_in_use will ever be updated and the only way
 542 *         RSC_DRV_CMD_ENABLE will ever be cleared.
 543 */
 544static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
 545				  const struct tcs_request *msg)
 546{
 547	unsigned long curr_enabled;
 548	u32 addr;
 549	int j, k;
 550	int i = tcs->offset;
 551
 552	for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {
 553		curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i);
 554
 555		for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
 556			addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j);
 557			for (k = 0; k < msg->num_cmds; k++) {
 558				if (cmd_db_match_resource_addr(msg->cmds[k].addr, addr))
 559					return -EBUSY;
 560			}
 561		}
 562	}
 563
 564	return 0;
 565}
 566
 567/**
 568 * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
 569 * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
 570 *       we borrowed it because there are zero active-only ones).
 571 *
 572 * Must be called with the drv->lock held since that protects tcs_in_use.
 573 *
 574 * Return: The first tcs that's free or -EBUSY if all in use.
 575 */
 576static int find_free_tcs(struct tcs_group *tcs)
 577{
 578	const struct rsc_drv *drv = tcs->drv;
 579	unsigned long i;
 580	unsigned long max = tcs->offset + tcs->num_tcs;
 581
 582	i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);
 583	if (i >= max)
 584		return -EBUSY;
 585
 586	return i;
 587}
 588
 589/**
 590 * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
 591 * @drv: The controller.
 592 * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
 593 * @msg: The data to be sent.
 594 *
 595 * Claims a tcs in the given tcs_group while making sure that no existing cmd
 596 * is in flight that would conflict with the one in @msg.
 597 *
 598 * Context: Must be called with the drv->lock held since that protects
 599 * tcs_in_use.
 600 *
 601 * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
 602 * or the tcs_group is full.
 603 */
 604static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
 605			     const struct tcs_request *msg)
 606{
 607	int ret;
 608
 609	/*
 610	 * The h/w does not like if we send a request to the same address,
 611	 * when one is already in-flight or being processed.
 612	 */
 613	ret = check_for_req_inflight(drv, tcs, msg);
 614	if (ret)
 615		return ret;
 616
 617	return find_free_tcs(tcs);
 618}
 619
 620/**
 621 * rpmh_rsc_send_data() - Write / trigger active-only message.
 622 * @drv: The controller.
 623 * @msg: The data to be sent.
 624 *
 625 * NOTES:
 626 * - This is only used for "ACTIVE_ONLY" since the limitations of this
 627 *   function don't make sense for sleep/wake cases.
 628 * - To do the transfer, we will grab a whole TCS for ourselves--we don't
 629 *   try to share. If there are none available we'll wait indefinitely
 630 *   for a free one.
 631 * - This function will not wait for the commands to be finished, only for
 632 *   data to be programmed into the RPMh. See rpmh_tx_done() which will
 633 *   be called when the transfer is fully complete.
 634 * - This function must be called with interrupts enabled. If the hardware
 635 *   is busy doing someone else's transfer we need that transfer to fully
 636 *   finish so that we can have the hardware, and to fully finish it needs
 637 *   the interrupt handler to run. If the interrupts is set to run on the
 638 *   active CPU this can never happen if interrupts are disabled.
 639 *
 640 * Return: 0 on success, -EINVAL on error.
 641 */
 642int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
 643{
 644	struct tcs_group *tcs;
 645	int tcs_id;
 646
 647	might_sleep();
 648
 649	tcs = get_tcs_for_msg(drv, msg);
 650	if (IS_ERR(tcs))
 651		return PTR_ERR(tcs);
 652
 653	spin_lock_irq(&drv->lock);
 654
 655	/* Wait forever for a free tcs. It better be there eventually! */
 656	wait_event_lock_irq(drv->tcs_wait,
 657			    (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
 658			    drv->lock);
 659
 660	tcs->req[tcs_id - tcs->offset] = msg;
 661	set_bit(tcs_id, drv->tcs_in_use);
 662	if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {
 663		/*
 664		 * Clear previously programmed WAKE commands in selected
 665		 * repurposed TCS to avoid triggering them. tcs->slots will be
 666		 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
 667		 */
 668		write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0);
 669		enable_tcs_irq(drv, tcs_id, true);
 670	}
 671	spin_unlock_irq(&drv->lock);
 672
 673	/*
 674	 * These two can be done after the lock is released because:
 675	 * - We marked "tcs_in_use" under lock.
 676	 * - Once "tcs_in_use" has been marked nobody else could be writing
 677	 *   to these registers until the interrupt goes off.
 678	 * - The interrupt can't go off until we trigger w/ the last line
 679	 *   of __tcs_set_trigger() below.
 680	 */
 681	__tcs_buffer_write(drv, tcs_id, 0, msg);
 682	__tcs_set_trigger(drv, tcs_id, true);
 683
 684	return 0;
 685}
 686
 687/**
 688 * find_slots() - Find a place to write the given message.
 689 * @tcs:    The tcs group to search.
 690 * @msg:    The message we want to find room for.
 691 * @tcs_id: If we return 0 from the function, we return the global ID of the
 692 *          TCS to write to here.
 693 * @cmd_id: If we return 0 from the function, we return the index of
 694 *          the command array of the returned TCS where the client should
 695 *          start writing the message.
 696 *
 697 * Only for use on sleep/wake TCSes since those are the only ones we maintain
 698 * tcs->slots for.
 699 *
 700 * Return: -ENOMEM if there was no room, else 0.
 701 */
 702static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
 703		      int *tcs_id, int *cmd_id)
 704{
 705	int slot, offset;
 706	int i = 0;
 707
 708	/* Do over, until we can fit the full payload in a single TCS */
 709	do {
 710		slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,
 711						  i, msg->num_cmds, 0);
 712		if (slot >= tcs->num_tcs * tcs->ncpt)
 713			return -ENOMEM;
 714		i += tcs->ncpt;
 715	} while (slot + msg->num_cmds - 1 >= i);
 716
 717	bitmap_set(tcs->slots, slot, msg->num_cmds);
 718
 719	offset = slot / tcs->ncpt;
 720	*tcs_id = offset + tcs->offset;
 721	*cmd_id = slot % tcs->ncpt;
 722
 723	return 0;
 724}
 725
 726/**
 727 * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
 728 * @drv: The controller.
 729 * @msg: The data to be written to the controller.
 730 *
 731 * This should only be called for sleep/wake state, never active-only
 732 * state.
 733 *
 734 * The caller must ensure that no other RPMH actions are happening and the
 735 * controller is idle when this function is called since it runs lockless.
 736 *
 737 * Return: 0 if no error; else -error.
 738 */
 739int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
 740{
 741	struct tcs_group *tcs;
 742	int tcs_id = 0, cmd_id = 0;
 743	int ret;
 744
 745	tcs = get_tcs_for_msg(drv, msg);
 746	if (IS_ERR(tcs))
 747		return PTR_ERR(tcs);
 748
 749	/* find the TCS id and the command in the TCS to write to */
 750	ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
 751	if (!ret)
 752		__tcs_buffer_write(drv, tcs_id, cmd_id, msg);
 753
 754	return ret;
 755}
 756
 757/**
 758 * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
 759 * @drv: The controller
 760 *
 761 * Checks if any of the AMCs are busy in handling ACTIVE sets.
 762 * This is called from the last cpu powering down before flushing
 763 * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
 764 * power collapse, so deny from the last cpu's pm notification.
 765 *
 766 * Context: Must be called with the drv->lock held.
 767 *
 768 * Return:
 769 * * False		- AMCs are idle
 770 * * True		- AMCs are busy
 771 */
 772static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
 773{
 774	unsigned long set;
 775	const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
 776	unsigned long max;
 777
 778	/*
 779	 * If we made an active request on a RSC that does not have a
 780	 * dedicated TCS for active state use, then re-purposed wake TCSes
 781	 * should be checked for not busy, because we used wake TCSes for
 782	 * active requests in this case.
 783	 */
 784	if (!tcs->num_tcs)
 785		tcs = &drv->tcs[WAKE_TCS];
 786
 787	max = tcs->offset + tcs->num_tcs;
 788	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);
 789
 790	return set < max;
 791}
 792
 793/**
 794 * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS.
 795 * @drv: The controller
 796 *
 797 * Writes maximum wakeup cycles when called from suspend.
 798 * Writes earliest hrtimer wakeup when called from idle.
 799 */
 800void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv)
 801{
 802	ktime_t now, wakeup;
 803	u64 wakeup_us, wakeup_cycles = ~0;
 804	u32 lo, hi;
 805
 806	if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call)
 807		return;
 808
 809	/* Set highest time when system (timekeeping) is suspended */
 810	if (system_state == SYSTEM_SUSPEND)
 811		goto exit;
 812
 813	/* Find the earliest hrtimer wakeup from online cpus */
 814	wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev);
 815
 816	/* Find the relative wakeup in kernel time scale */
 817	now = ktime_get();
 818	wakeup = ktime_sub(wakeup, now);
 819	wakeup_us = ktime_to_us(wakeup);
 820
 821	/* Convert the wakeup to arch timer scale */
 822	wakeup_cycles = USECS_TO_CYCLES(wakeup_us);
 823	wakeup_cycles += arch_timer_read_counter();
 824
 825exit:
 826	lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK;
 827	hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE;
 828	hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK;
 829	hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID;
 830
 831	writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO);
 832	writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI);
 833}
 834
 835/**
 836 * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
 837 * @nfb:    Pointer to the notifier block in struct rsc_drv.
 838 * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
 839 * @v:      Unused
 840 *
 841 * This function is given to cpu_pm_register_notifier so we can be informed
 842 * about when CPUs go down. When all CPUs go down we know no more active
 843 * transfers will be started so we write sleep/wake sets. This function gets
 844 * called from cpuidle code paths and also at system suspend time.
 845 *
 846 * If its last CPU going down and AMCs are not busy then writes cached sleep
 847 * and wake messages to TCSes. The firmware then takes care of triggering
 848 * them when entering deepest low power modes.
 849 *
 850 * Return: See cpu_pm_register_notifier()
 851 */
 852static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
 853				    unsigned long action, void *v)
 854{
 855	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
 856	int ret = NOTIFY_OK;
 857	int cpus_in_pm;
 858
 859	switch (action) {
 860	case CPU_PM_ENTER:
 861		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
 862		/*
 863		 * NOTE: comments for num_online_cpus() point out that it's
 864		 * only a snapshot so we need to be careful. It should be OK
 865		 * for us to use, though.  It's important for us not to miss
 866		 * if we're the last CPU going down so it would only be a
 867		 * problem if a CPU went offline right after we did the check
 868		 * AND that CPU was not idle AND that CPU was the last non-idle
 869		 * CPU. That can't happen. CPUs would have to come out of idle
 870		 * before the CPU could go offline.
 871		 */
 872		if (cpus_in_pm < num_online_cpus())
 873			return NOTIFY_OK;
 874		break;
 875	case CPU_PM_ENTER_FAILED:
 876	case CPU_PM_EXIT:
 877		atomic_dec(&drv->cpus_in_pm);
 878		return NOTIFY_OK;
 879	default:
 880		return NOTIFY_DONE;
 881	}
 882
 883	/*
 884	 * It's likely we're on the last CPU. Grab the drv->lock and write
 885	 * out the sleep/wake commands to RPMH hardware. Grabbing the lock
 886	 * means that if we race with another CPU coming up we are still
 887	 * guaranteed to be safe. If another CPU came up just after we checked
 888	 * and has grabbed the lock or started an active transfer then we'll
 889	 * notice we're busy and abort. If another CPU comes up after we start
 890	 * flushing it will be blocked from starting an active transfer until
 891	 * we're done flushing. If another CPU starts an active transfer after
 892	 * we release the lock we're still OK because we're no longer the last
 893	 * CPU.
 894	 */
 895	if (spin_trylock(&drv->lock)) {
 896		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
 897			ret = NOTIFY_BAD;
 898		spin_unlock(&drv->lock);
 899	} else {
 900		/* Another CPU must be up */
 901		return NOTIFY_OK;
 902	}
 903
 904	if (ret == NOTIFY_BAD) {
 905		/* Double-check if we're here because someone else is up */
 906		if (cpus_in_pm < num_online_cpus())
 907			ret = NOTIFY_OK;
 908		else
 909			/* We won't be called w/ CPU_PM_ENTER_FAILED */
 910			atomic_dec(&drv->cpus_in_pm);
 911	}
 912
 913	return ret;
 914}
 915
 916/**
 917 * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy.
 918 * @nfb:    Pointer to the genpd notifier block in struct rsc_drv.
 919 * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON.
 920 * @v:      Unused
 921 *
 922 * This function is given to dev_pm_genpd_add_notifier() so we can be informed
 923 * about when cluster-pd is going down. When cluster go down we know no more active
 924 * transfers will be started so we write sleep/wake sets. This function gets
 925 * called from cpuidle code paths and also at system suspend time.
 926 *
 927 * If AMCs are not busy then writes cached sleep and wake messages to TCSes.
 928 * The firmware then takes care of triggering them when entering deepest low power modes.
 929 *
 930 * Return:
 931 * * NOTIFY_OK          - success
 932 * * NOTIFY_BAD         - failure
 933 */
 934static int rpmh_rsc_pd_callback(struct notifier_block *nfb,
 935				unsigned long action, void *v)
 936{
 937	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb);
 938
 939	/* We don't need to lock as genpd on/off are serialized */
 940	if ((action == GENPD_NOTIFY_PRE_OFF) &&
 941	    (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)))
 942		return NOTIFY_BAD;
 943
 944	return NOTIFY_OK;
 945}
 946
 947static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev)
 948{
 949	int ret;
 950
 951	pm_runtime_enable(dev);
 952	drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;
 953	ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb);
 954	if (ret)
 955		pm_runtime_disable(dev);
 956
 957	return ret;
 958}
 959
 960static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv)
 961{
 962	struct tcs_type_config {
 963		u32 type;
 964		u32 n;
 965	} tcs_cfg[TCS_TYPE_NR] = { { 0 } };
 966	struct device_node *dn = pdev->dev.of_node;
 967	u32 config, max_tcs, ncpt, offset;
 968	int i, ret, n, st = 0;
 969	struct tcs_group *tcs;
 970
 971	ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);
 972	if (ret)
 973		return ret;
 974	drv->tcs_base = drv->base + offset;
 975
 976	config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]);
 977
 978	max_tcs = config;
 979	max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);
 980	max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);
 981
 982	ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);
 983	ncpt = ncpt >> DRV_NCPT_SHIFT;
 984
 985	n = of_property_count_u32_elems(dn, "qcom,tcs-config");
 986	if (n != 2 * TCS_TYPE_NR)
 987		return -EINVAL;
 988
 989	for (i = 0; i < TCS_TYPE_NR; i++) {
 990		ret = of_property_read_u32_index(dn, "qcom,tcs-config",
 991						 i * 2, &tcs_cfg[i].type);
 992		if (ret)
 993			return ret;
 994		if (tcs_cfg[i].type >= TCS_TYPE_NR)
 995			return -EINVAL;
 996
 997		ret = of_property_read_u32_index(dn, "qcom,tcs-config",
 998						 i * 2 + 1, &tcs_cfg[i].n);
 999		if (ret)
1000			return ret;
1001		if (tcs_cfg[i].n > MAX_TCS_PER_TYPE)
1002			return -EINVAL;
1003	}
1004
1005	for (i = 0; i < TCS_TYPE_NR; i++) {
1006		tcs = &drv->tcs[tcs_cfg[i].type];
1007		if (tcs->drv)
1008			return -EINVAL;
1009		tcs->drv = drv;
1010		tcs->type = tcs_cfg[i].type;
1011		tcs->num_tcs = tcs_cfg[i].n;
1012		tcs->ncpt = ncpt;
1013
1014		if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
1015			continue;
1016
1017		if (st + tcs->num_tcs > max_tcs ||
1018		    st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))
1019			return -EINVAL;
1020
1021		tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
1022		tcs->offset = st;
1023		st += tcs->num_tcs;
1024	}
1025
1026	drv->num_tcs = st;
1027
1028	return 0;
1029}
1030
1031static int rpmh_rsc_probe(struct platform_device *pdev)
1032{
1033	struct device_node *dn = pdev->dev.of_node;
1034	struct rsc_drv *drv;
1035	char drv_id[10] = {0};
1036	int ret, irq;
1037	u32 solver_config;
1038	u32 rsc_id;
1039
1040	/*
1041	 * Even though RPMh doesn't directly use cmd-db, all of its children
1042	 * do. To avoid adding this check to our children we'll do it now.
1043	 */
1044	ret = cmd_db_ready();
1045	if (ret)
1046		return dev_err_probe(&pdev->dev, ret,
1047				     "Command DB not available\n");
1048
1049	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);
1050	if (!drv)
1051		return -ENOMEM;
1052
1053	ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);
1054	if (ret)
1055		return ret;
1056
1057	drv->name = of_get_property(dn, "label", NULL);
1058	if (!drv->name)
1059		drv->name = dev_name(&pdev->dev);
1060
1061	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
1062	drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id);
1063	if (IS_ERR(drv->base))
1064		return PTR_ERR(drv->base);
1065
1066	rsc_id = readl_relaxed(drv->base + RSC_DRV_ID);
1067	drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT);
1068	drv->ver.major >>= MAJOR_VER_SHIFT;
1069	drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);
1070	drv->ver.minor >>= MINOR_VER_SHIFT;
1071
1072	if (drv->ver.major >= 3)
1073		drv->regs = rpmh_rsc_reg_offset_ver_3_0;
1074	else
1075		drv->regs = rpmh_rsc_reg_offset_ver_2_7;
1076
1077	ret = rpmh_probe_tcs_config(pdev, drv);
1078	if (ret)
1079		return ret;
1080
1081	spin_lock_init(&drv->lock);
1082	init_waitqueue_head(&drv->tcs_wait);
1083	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);
1084
1085	irq = platform_get_irq(pdev, drv->id);
1086	if (irq < 0)
1087		return irq;
1088
1089	ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,
1090			       IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND,
1091			       drv->name, drv);
1092	if (ret)
1093		return ret;
1094
1095	/*
1096	 * CPU PM/genpd notification are not required for controllers that support
1097	 * 'HW solver' mode where they can be in autonomous mode executing low
1098	 * power mode to power down.
1099	 */
1100	solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]);
1101	solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
1102	solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
1103	if (!solver_config) {
1104		if (pdev->dev.pm_domain) {
1105			ret = rpmh_rsc_pd_attach(drv, &pdev->dev);
1106			if (ret)
1107				return ret;
1108		} else {
1109			drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
1110			cpu_pm_register_notifier(&drv->rsc_pm);
1111		}
1112	}
1113
1114	/* Enable the active TCS to send requests immediately */
1115	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
1116		       drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
1117
1118	spin_lock_init(&drv->client.cache_lock);
1119	INIT_LIST_HEAD(&drv->client.cache);
1120	INIT_LIST_HEAD(&drv->client.batch_cache);
1121
1122	dev_set_drvdata(&pdev->dev, drv);
1123	drv->dev = &pdev->dev;
1124
1125	ret = devm_of_platform_populate(&pdev->dev);
1126	if (ret && pdev->dev.pm_domain) {
1127		dev_pm_genpd_remove_notifier(&pdev->dev);
1128		pm_runtime_disable(&pdev->dev);
1129	}
1130
1131	return ret;
1132}
1133
1134static const struct of_device_id rpmh_drv_match[] = {
1135	{ .compatible = "qcom,rpmh-rsc", },
1136	{ }
1137};
1138MODULE_DEVICE_TABLE(of, rpmh_drv_match);
1139
1140static struct platform_driver rpmh_driver = {
1141	.probe = rpmh_rsc_probe,
1142	.driver = {
1143		  .name = "rpmh",
1144		  .of_match_table = rpmh_drv_match,
1145		  .suppress_bind_attrs = true,
1146	},
1147};
1148
1149static int __init rpmh_driver_init(void)
1150{
1151	return platform_driver_register(&rpmh_driver);
1152}
1153core_initcall(rpmh_driver_init);
1154
1155MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
1156MODULE_LICENSE("GPL v2");