imx_dsp_rproc.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright 2021 NXP */
   3
   4#include <dt-bindings/firmware/imx/rsrc.h>
   5#include <linux/arm-smccc.h>
   6#include <linux/clk.h>
   7#include <linux/err.h>
   8#include <linux/firmware.h>
   9#include <linux/firmware/imx/sci.h>
  10#include <linux/interrupt.h>
  11#include <linux/kernel.h>
  12#include <linux/mailbox_client.h>
  13#include <linux/mfd/syscon.h>
  14#include <linux/module.h>
  15#include <linux/of.h>
  16#include <linux/of_reserved_mem.h>
  17#include <linux/platform_device.h>
  18#include <linux/pm_domain.h>
  19#include <linux/pm_runtime.h>
  20#include <linux/regmap.h>
  21#include <linux/remoteproc.h>
  22#include <linux/reset.h>
  23#include <linux/slab.h>
  24
  25#include "imx_rproc.h"
  26#include "remoteproc_elf_helpers.h"
  27#include "remoteproc_internal.h"
  28
  29#define DSP_RPROC_CLK_MAX			5
  30
  31/*
  32 * Module parameters
  33 */
  34static unsigned int no_mailboxes;
  35module_param_named(no_mailboxes, no_mailboxes, int, 0644);
  36MODULE_PARM_DESC(no_mailboxes,
  37		 "There is no mailbox between cores, so ignore remote proc reply after start, default is 0 (off).");
  38
  39/* Flag indicating that the remote is up and running */
  40#define REMOTE_IS_READY				BIT(0)
  41/* Flag indicating that the host should wait for a firmware-ready response */
  42#define WAIT_FW_READY				BIT(1)
  43#define REMOTE_READY_WAIT_MAX_RETRIES		500
  44
  45/*
  46 * This flag is set in the DSP resource table's features field to indicate
  47 * that the firmware requires the host NOT to wait for a FW_READY response.
  48 */
  49#define FEATURE_DONT_WAIT_FW_READY		BIT(0)
  50
  51/* att flags */
  52/* DSP own area */
  53#define ATT_OWN					BIT(31)
  54/* DSP instruction area */
  55#define ATT_IRAM				BIT(30)
  56
  57/* Definitions for i.MX8MP */
  58/* DAP registers */
  59#define IMX8M_DAP_DEBUG				0x28800000
  60#define IMX8M_DAP_DEBUG_SIZE			(64 * 1024)
  61#define IMX8M_DAP_PWRCTL			(0x4000 + 0x3020)
  62#define IMX8M_PWRCTL_CORERESET			BIT(16)
  63
  64/* DSP audio mix registers */
  65#define IMX8M_AudioDSP_REG0			0x100
  66#define IMX8M_AudioDSP_REG1			0x104
  67#define IMX8M_AudioDSP_REG2			0x108
  68#define IMX8M_AudioDSP_REG3			0x10c
  69
  70#define IMX8M_AudioDSP_REG2_RUNSTALL		BIT(5)
  71#define IMX8M_AudioDSP_REG2_PWAITMODE		BIT(1)
  72
  73/* Definitions for i.MX8ULP */
  74#define IMX8ULP_SIM_LPAV_REG_SYSCTRL0		0x8
  75#define IMX8ULP_SYSCTRL0_DSP_DBG_RST		BIT(25)
  76#define IMX8ULP_SYSCTRL0_DSP_PLAT_CLK_EN	BIT(19)
  77#define IMX8ULP_SYSCTRL0_DSP_PBCLK_EN		BIT(18)
  78#define IMX8ULP_SYSCTRL0_DSP_CLK_EN		BIT(17)
  79#define IMX8ULP_SYSCTRL0_DSP_RST		BIT(16)
  80#define IMX8ULP_SYSCTRL0_DSP_OCD_HALT		BIT(14)
  81#define IMX8ULP_SYSCTRL0_DSP_STALL		BIT(13)
  82
  83#define IMX8ULP_SIP_HIFI_XRDC			0xc200000e
  84
  85#define FW_RSC_NXP_S_MAGIC			((uint32_t)'n' << 24 |	\
  86						 (uint32_t)'x' << 16 |	\
  87						 (uint32_t)'p' << 8 |	\
  88						 (uint32_t)'s')
  89/*
  90 * enum - Predefined Mailbox Messages
  91 *
  92 * @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor
  93 *
  94 * @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a
  95 * suspend request
  96 *
  97 * @RP_MBOX_RESUME_SYSTEM: system resume request for the remote processor
  98 *
  99 * @RP_MBOX_RESUME_ACK: successful response from remote processor for a
 100 * resume request
 101 */
 102enum imx_dsp_rp_mbox_messages {
 103	RP_MBOX_SUSPEND_SYSTEM			= 0xFF11,
 104	RP_MBOX_SUSPEND_ACK			= 0xFF12,
 105	RP_MBOX_RESUME_SYSTEM			= 0xFF13,
 106	RP_MBOX_RESUME_ACK			= 0xFF14,
 107};
 108
 109/**
 110 * struct imx_dsp_rproc - DSP remote processor state
 111 * @regmap: regmap handler
 112 * @run_stall: reset control handle used for Run/Stall operation
 113 * @rproc: rproc handler
 114 * @dsp_dcfg: device configuration pointer
 115 * @clks: clocks needed by this device
 116 * @cl: mailbox client to request the mailbox channel
 117 * @cl_rxdb: mailbox client to request the mailbox channel for doorbell
 118 * @tx_ch: mailbox tx channel handle
 119 * @rx_ch: mailbox rx channel handle
 120 * @rxdb_ch: mailbox rx doorbell channel handle
 121 * @pd_list: power domain list
 122 * @ipc_handle: System Control Unit ipc handle
 123 * @rproc_work: work for processing virtio interrupts
 124 * @pm_comp: completion primitive to sync for suspend response
 125 * @flags: control flags
 126 */
 127struct imx_dsp_rproc {
 128	struct regmap				*regmap;
 129	struct reset_control			*run_stall;
 130	struct rproc				*rproc;
 131	const struct imx_dsp_rproc_dcfg		*dsp_dcfg;
 132	struct clk_bulk_data			clks[DSP_RPROC_CLK_MAX];
 133	struct mbox_client			cl;
 134	struct mbox_client			cl_rxdb;
 135	struct mbox_chan			*tx_ch;
 136	struct mbox_chan			*rx_ch;
 137	struct mbox_chan			*rxdb_ch;
 138	struct dev_pm_domain_list		*pd_list;
 139	struct imx_sc_ipc			*ipc_handle;
 140	struct work_struct			rproc_work;
 141	struct completion			pm_comp;
 142	u32					flags;
 143};
 144
 145/**
 146 * struct imx_dsp_rproc_dcfg - DSP remote processor configuration
 147 * @dcfg: imx_rproc_dcfg handler
 148 * @reset: reset callback function
 149 */
 150struct imx_dsp_rproc_dcfg {
 151	const struct imx_rproc_dcfg		*dcfg;
 152	int (*reset)(struct imx_dsp_rproc *priv);
 153};
 154
 155/**
 156 * struct fw_rsc_imx_dsp - i.MX DSP specific info
 157 *
 158 * @len: length of the resource entry
 159 * @magic_num: 32-bit magic number
 160 * @version: version of data structure
 161 * @features: feature flags supported by the i.MX DSP firmware
 162 *
 163 * This represents a DSP-specific resource in the firmware's
 164 * resource table, providing information on supported features.
 165 */
 166struct fw_rsc_imx_dsp {
 167	uint32_t len;
 168	uint32_t magic_num;
 169	uint32_t version;
 170	uint32_t features;
 171} __packed;
 172
 173static const struct imx_rproc_att imx_dsp_rproc_att_imx8qm[] = {
 174	/* dev addr , sys addr  , size	    , flags */
 175	{ 0x596e8000, 0x556e8000, 0x00008000, ATT_OWN },
 176	{ 0x596f0000, 0x556f0000, 0x00008000, ATT_OWN },
 177	{ 0x596f8000, 0x556f8000, 0x00000800, ATT_OWN | ATT_IRAM},
 178	{ 0x55700000, 0x55700000, 0x00070000, ATT_OWN },
 179	/* DDR (Data) */
 180	{ 0x80000000, 0x80000000, 0x60000000, 0},
 181};
 182
 183static const struct imx_rproc_att imx_dsp_rproc_att_imx8qxp[] = {
 184	/* dev addr , sys addr  , size	    , flags */
 185	{ 0x596e8000, 0x596e8000, 0x00008000, ATT_OWN },
 186	{ 0x596f0000, 0x596f0000, 0x00008000, ATT_OWN },
 187	{ 0x596f8000, 0x596f8000, 0x00000800, ATT_OWN | ATT_IRAM},
 188	{ 0x59700000, 0x59700000, 0x00070000, ATT_OWN },
 189	/* DDR (Data) */
 190	{ 0x80000000, 0x80000000, 0x60000000, 0},
 191};
 192
 193static const struct imx_rproc_att imx_dsp_rproc_att_imx8mp[] = {
 194	/* dev addr , sys addr  , size	    , flags */
 195	{ 0x3b6e8000, 0x3b6e8000, 0x00008000, ATT_OWN },
 196	{ 0x3b6f0000, 0x3b6f0000, 0x00008000, ATT_OWN },
 197	{ 0x3b6f8000, 0x3b6f8000, 0x00000800, ATT_OWN | ATT_IRAM},
 198	{ 0x3b700000, 0x3b700000, 0x00040000, ATT_OWN },
 199	/* DDR (Data) */
 200	{ 0x40000000, 0x40000000, 0x80000000, 0},
 201};
 202
 203static const struct imx_rproc_att imx_dsp_rproc_att_imx8ulp[] = {
 204	/* dev addr , sys addr  , size	    , flags */
 205	{ 0x21170000, 0x21170000, 0x00010000, ATT_OWN | ATT_IRAM},
 206	{ 0x21180000, 0x21180000, 0x00010000, ATT_OWN },
 207	/* DDR (Data) */
 208	{ 0x0c000000, 0x80000000, 0x10000000, 0},
 209	{ 0x30000000, 0x90000000, 0x10000000, 0},
 210};
 211
 212/* Initialize the mailboxes between cores, if exists */
 213static int (*imx_dsp_rproc_mbox_init)(struct imx_dsp_rproc *priv);
 214
 215/* Reset function for DSP on i.MX8MP */
 216static int imx8mp_dsp_reset(struct imx_dsp_rproc *priv)
 217{
 218	void __iomem *dap = ioremap_wc(IMX8M_DAP_DEBUG, IMX8M_DAP_DEBUG_SIZE);
 219	int pwrctl;
 220
 221	/* Put DSP into reset and stall */
 222	pwrctl = readl(dap + IMX8M_DAP_PWRCTL);
 223	pwrctl |= IMX8M_PWRCTL_CORERESET;
 224	writel(pwrctl, dap + IMX8M_DAP_PWRCTL);
 225
 226	/* Keep reset asserted for 10 cycles */
 227	usleep_range(1, 2);
 228
 229	reset_control_assert(priv->run_stall);
 230
 231	/* Take the DSP out of reset and keep stalled for FW loading */
 232	pwrctl = readl(dap + IMX8M_DAP_PWRCTL);
 233	pwrctl &= ~IMX8M_PWRCTL_CORERESET;
 234	writel(pwrctl, dap + IMX8M_DAP_PWRCTL);
 235
 236	iounmap(dap);
 237	return 0;
 238}
 239
 240/* Reset function for DSP on i.MX8ULP */
 241static int imx8ulp_dsp_reset(struct imx_dsp_rproc *priv)
 242{
 243	struct arm_smccc_res res;
 244
 245	/* Put DSP into reset and stall */
 246	regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
 247			   IMX8ULP_SYSCTRL0_DSP_RST, IMX8ULP_SYSCTRL0_DSP_RST);
 248	regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
 249			   IMX8ULP_SYSCTRL0_DSP_STALL,
 250			   IMX8ULP_SYSCTRL0_DSP_STALL);
 251
 252	/* Configure resources of DSP through TFA */
 253	arm_smccc_smc(IMX8ULP_SIP_HIFI_XRDC, 0, 0, 0, 0, 0, 0, 0, &res);
 254
 255	/* Take the DSP out of reset and keep stalled for FW loading */
 256	regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
 257			   IMX8ULP_SYSCTRL0_DSP_RST, 0);
 258	regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
 259			   IMX8ULP_SYSCTRL0_DSP_DBG_RST, 0);
 260
 261	return 0;
 262}
 263
 264static int imx_dsp_rproc_ready(struct rproc *rproc)
 265{
 266	struct imx_dsp_rproc *priv = rproc->priv;
 267	int i;
 268
 269	if (!priv->rxdb_ch)
 270		return 0;
 271
 272	for (i = 0; i < REMOTE_READY_WAIT_MAX_RETRIES; i++) {
 273		if (priv->flags & REMOTE_IS_READY)
 274			return 0;
 275		usleep_range(100, 200);
 276	}
 277
 278	return -ETIMEDOUT;
 279}
 280
 281/**
 282 * imx_dsp_rproc_handle_rsc() - Handle DSP-specific resource table entries
 283 * @rproc: remote processor instance
 284 * @rsc_type: resource type identifier
 285 * @rsc: pointer to the resource entry
 286 * @offset: offset of the resource entry
 287 * @avail: available space in the resource table
 288 *
 289 * Parse the DSP-specific resource entry and update flags accordingly.
 290 * If the WAIT_FW_READY feature is set, the host must wait for the firmware
 291 * to signal readiness before proceeding with execution.
 292 *
 293 * Return: RSC_HANDLED if processed successfully, RSC_IGNORED otherwise.
 294 */
 295static int imx_dsp_rproc_handle_rsc(struct rproc *rproc, u32 rsc_type,
 296				    void *rsc, int offset, int avail)
 297{
 298	struct imx_dsp_rproc *priv = rproc->priv;
 299	struct fw_rsc_imx_dsp *imx_dsp_rsc = rsc;
 300	struct device *dev = rproc->dev.parent;
 301
 302	if (!imx_dsp_rsc) {
 303		dev_dbg(dev, "Invalid fw_rsc_imx_dsp.\n");
 304		return RSC_IGNORED;
 305	}
 306
 307	/* Make sure resource isn't truncated */
 308	if (sizeof(struct fw_rsc_imx_dsp) > avail ||
 309	    sizeof(struct fw_rsc_imx_dsp) != imx_dsp_rsc->len) {
 310		dev_dbg(dev, "Resource fw_rsc_imx_dsp is truncated.\n");
 311		return RSC_IGNORED;
 312	}
 313
 314	/*
 315	 * If FW_RSC_NXP_S_MAGIC number is not found then
 316	 * wait for fw_ready reply (default work flow)
 317	 */
 318	if (imx_dsp_rsc->magic_num != FW_RSC_NXP_S_MAGIC) {
 319		dev_dbg(dev, "Invalid resource table magic number.\n");
 320		return RSC_IGNORED;
 321	}
 322
 323	/*
 324	 * For now, in struct fw_rsc_imx_dsp, version 0,
 325	 * only FEATURE_DONT_WAIT_FW_READY is valid.
 326	 *
 327	 * When adding new features, please upgrade version.
 328	 */
 329	if (imx_dsp_rsc->version > 0) {
 330		dev_warn(dev, "Unexpected fw_rsc_imx_dsp version %d.\n",
 331			 imx_dsp_rsc->version);
 332		return RSC_IGNORED;
 333	}
 334
 335	if (imx_dsp_rsc->features & FEATURE_DONT_WAIT_FW_READY)
 336		priv->flags &= ~WAIT_FW_READY;
 337
 338	return RSC_HANDLED;
 339}
 340
 341static int imx_dsp_rproc_mmio_start(struct rproc *rproc)
 342{
 343	struct imx_dsp_rproc *priv = rproc->priv;
 344	const struct imx_rproc_dcfg *dcfg = priv->dsp_dcfg->dcfg;
 345
 346	return regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, dcfg->src_start);
 347}
 348
 349static int imx_dsp_rproc_reset_ctrl_start(struct rproc *rproc)
 350{
 351	struct imx_dsp_rproc *priv = rproc->priv;
 352
 353	return reset_control_deassert(priv->run_stall);
 354}
 355
 356static int imx_dsp_rproc_scu_api_start(struct rproc *rproc)
 357{
 358	struct imx_dsp_rproc *priv = rproc->priv;
 359
 360	return imx_sc_pm_cpu_start(priv->ipc_handle, IMX_SC_R_DSP, true, rproc->bootaddr);
 361}
 362
 363/*
 364 * Start function for rproc_ops
 365 *
 366 * There is a handshake for start procedure: when DSP starts, it
 367 * will send a doorbell message to this driver, then the
 368 * REMOTE_IS_READY flags is set, then driver will kick
 369 * a message to DSP.
 370 */
 371static int imx_dsp_rproc_start(struct rproc *rproc)
 372{
 373	struct imx_dsp_rproc *priv = rproc->priv;
 374	const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
 375	const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
 376	struct device *dev = rproc->dev.parent;
 377	int ret;
 378
 379	if (!dcfg->ops || !dcfg->ops->start)
 380		return -EOPNOTSUPP;
 381
 382	ret = dcfg->ops->start(rproc);
 383	if (ret) {
 384		dev_err(dev, "Failed to enable remote core!\n");
 385		return ret;
 386	}
 387
 388	if (priv->flags & WAIT_FW_READY)
 389		return imx_dsp_rproc_ready(rproc);
 390
 391	return 0;
 392}
 393
 394static int imx_dsp_rproc_mmio_stop(struct rproc *rproc)
 395{
 396	struct imx_dsp_rproc *priv = rproc->priv;
 397	const struct imx_rproc_dcfg *dcfg = priv->dsp_dcfg->dcfg;
 398
 399	return regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, dcfg->src_stop);
 400}
 401
 402static int imx_dsp_rproc_reset_ctrl_stop(struct rproc *rproc)
 403{
 404	struct imx_dsp_rproc *priv = rproc->priv;
 405
 406	return reset_control_assert(priv->run_stall);
 407}
 408
 409static int imx_dsp_rproc_scu_api_stop(struct rproc *rproc)
 410{
 411	struct imx_dsp_rproc *priv = rproc->priv;
 412
 413	return imx_sc_pm_cpu_start(priv->ipc_handle, IMX_SC_R_DSP, false, rproc->bootaddr);
 414}
 415
 416/*
 417 * Stop function for rproc_ops
 418 * It clears the REMOTE_IS_READY flags
 419 */
 420static int imx_dsp_rproc_stop(struct rproc *rproc)
 421{
 422	struct imx_dsp_rproc *priv = rproc->priv;
 423	const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
 424	const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
 425	struct device *dev = rproc->dev.parent;
 426	int ret = 0;
 427
 428	if (rproc->state == RPROC_CRASHED) {
 429		priv->flags &= ~REMOTE_IS_READY;
 430		return 0;
 431	}
 432
 433	if (!dcfg->ops || !dcfg->ops->stop)
 434		return -EOPNOTSUPP;
 435
 436	ret = dcfg->ops->stop(rproc);
 437	if (ret) {
 438		dev_err(dev, "Failed to stop remote core\n");
 439		return ret;
 440	}
 441
 442	priv->flags &= ~REMOTE_IS_READY;
 443
 444	return 0;
 445}
 446
 447/**
 448 * imx_dsp_rproc_sys_to_da() - internal memory translation helper
 449 * @priv: private data pointer
 450 * @sys: system address (DDR address)
 451 * @len: length of the memory buffer
 452 * @da: device address to translate
 453 *
 454 * Convert system address (DDR address) to device address (DSP)
 455 * for there may be memory remap for device.
 456 */
 457static int imx_dsp_rproc_sys_to_da(struct imx_dsp_rproc *priv, u64 sys,
 458				   size_t len, u64 *da)
 459{
 460	const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
 461	const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
 462	int i;
 463
 464	/* Parse address translation table */
 465	for (i = 0; i < dcfg->att_size; i++) {
 466		const struct imx_rproc_att *att = &dcfg->att[i];
 467
 468		if (sys >= att->sa && sys + len <= att->sa + att->size) {
 469			unsigned int offset = sys - att->sa;
 470
 471			*da = att->da + offset;
 472			return 0;
 473		}
 474	}
 475
 476	return -ENOENT;
 477}
 478
 479/* Main virtqueue message work function
 480 *
 481 * This function is executed upon scheduling of the i.MX DSP remoteproc
 482 * driver's workqueue. The workqueue is scheduled by the mailbox rx
 483 * handler.
 484 *
 485 * This work function processes both the Tx and Rx virtqueue indices on
 486 * every invocation. The rproc_vq_interrupt function can detect if there
 487 * are new unprocessed messages or not (returns IRQ_NONE vs IRQ_HANDLED),
 488 * but there is no need to check for these return values. The index 0
 489 * triggering will process all pending Rx buffers, and the index 1 triggering
 490 * will process all newly available Tx buffers and will wakeup any potentially
 491 * blocked senders.
 492 *
 493 * NOTE:
 494 *    The current logic is based on an inherent design assumption of supporting
 495 *    only 2 vrings, but this can be changed if needed.
 496 */
 497static void imx_dsp_rproc_vq_work(struct work_struct *work)
 498{
 499	struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc,
 500						  rproc_work);
 501	struct rproc *rproc = priv->rproc;
 502
 503	mutex_lock(&rproc->lock);
 504
 505	if (rproc->state != RPROC_RUNNING)
 506		goto unlock_mutex;
 507
 508	rproc_vq_interrupt(priv->rproc, 0);
 509	rproc_vq_interrupt(priv->rproc, 1);
 510
 511unlock_mutex:
 512	mutex_unlock(&rproc->lock);
 513}
 514
 515/**
 516 * imx_dsp_rproc_rx_tx_callback() - inbound mailbox message handler
 517 * @cl: mailbox client pointer used for requesting the mailbox channel
 518 * @data: mailbox payload
 519 *
 520 * This handler is invoked by mailbox driver whenever a mailbox
 521 * message is received. Usually, the SUSPEND and RESUME related messages
 522 * are handled in this function, other messages are handled by remoteproc core
 523 */
 524static void imx_dsp_rproc_rx_tx_callback(struct mbox_client *cl, void *data)
 525{
 526	struct rproc *rproc = dev_get_drvdata(cl->dev);
 527	struct imx_dsp_rproc *priv = rproc->priv;
 528	struct device *dev = rproc->dev.parent;
 529	u32 message = (u32)(*(u32 *)data);
 530
 531	dev_dbg(dev, "mbox msg: 0x%x\n", message);
 532
 533	switch (message) {
 534	case RP_MBOX_SUSPEND_ACK:
 535		complete(&priv->pm_comp);
 536		break;
 537	case RP_MBOX_RESUME_ACK:
 538		complete(&priv->pm_comp);
 539		break;
 540	default:
 541		schedule_work(&priv->rproc_work);
 542		break;
 543	}
 544}
 545
 546/**
 547 * imx_dsp_rproc_rxdb_callback() - inbound mailbox message handler
 548 * @cl: mailbox client pointer used for requesting the mailbox channel
 549 * @data: mailbox payload
 550 *
 551 * For doorbell, there is no message specified, just set REMOTE_IS_READY
 552 * flag.
 553 */
 554static void imx_dsp_rproc_rxdb_callback(struct mbox_client *cl, void *data)
 555{
 556	struct rproc *rproc = dev_get_drvdata(cl->dev);
 557	struct imx_dsp_rproc *priv = rproc->priv;
 558
 559	/* Remote is ready after firmware is loaded and running */
 560	priv->flags |= REMOTE_IS_READY;
 561}
 562
 563/**
 564 * imx_dsp_rproc_mbox_alloc() - request mailbox channels
 565 * @priv: private data pointer
 566 *
 567 * Request three mailbox channels (tx, rx, rxdb).
 568 */
 569static int imx_dsp_rproc_mbox_alloc(struct imx_dsp_rproc *priv)
 570{
 571	struct device *dev = priv->rproc->dev.parent;
 572	struct mbox_client *cl;
 573	int ret;
 574
 575	if (!of_property_present(dev->of_node, "mbox-names"))
 576		return 0;
 577
 578	cl = &priv->cl;
 579	cl->dev = dev;
 580	cl->tx_block = true;
 581	cl->tx_tout = 100;
 582	cl->knows_txdone = false;
 583	cl->rx_callback = imx_dsp_rproc_rx_tx_callback;
 584
 585	/* Channel for sending message */
 586	priv->tx_ch = mbox_request_channel_byname(cl, "tx");
 587	if (IS_ERR(priv->tx_ch)) {
 588		ret = PTR_ERR(priv->tx_ch);
 589		dev_dbg(cl->dev, "failed to request tx mailbox channel: %d\n",
 590			ret);
 591		return ret;
 592	}
 593
 594	/* Channel for receiving message */
 595	priv->rx_ch = mbox_request_channel_byname(cl, "rx");
 596	if (IS_ERR(priv->rx_ch)) {
 597		ret = PTR_ERR(priv->rx_ch);
 598		dev_dbg(cl->dev, "failed to request rx mailbox channel: %d\n",
 599			ret);
 600		goto free_channel_tx;
 601	}
 602
 603	cl = &priv->cl_rxdb;
 604	cl->dev = dev;
 605	cl->rx_callback = imx_dsp_rproc_rxdb_callback;
 606
 607	/*
 608	 * RX door bell is used to receive the ready signal from remote
 609	 * after firmware loaded.
 610	 */
 611	priv->rxdb_ch = mbox_request_channel_byname(cl, "rxdb");
 612	if (IS_ERR(priv->rxdb_ch)) {
 613		ret = PTR_ERR(priv->rxdb_ch);
 614		dev_dbg(cl->dev, "failed to request mbox chan rxdb, ret %d\n",
 615			ret);
 616		goto free_channel_rx;
 617	}
 618
 619	return 0;
 620
 621free_channel_rx:
 622	mbox_free_channel(priv->rx_ch);
 623free_channel_tx:
 624	mbox_free_channel(priv->tx_ch);
 625	return ret;
 626}
 627
 628/*
 629 * imx_dsp_rproc_mbox_no_alloc()
 630 *
 631 * Empty function for no mailbox between cores
 632 *
 633 * Always return 0
 634 */
 635static int imx_dsp_rproc_mbox_no_alloc(struct imx_dsp_rproc *priv)
 636{
 637	return 0;
 638}
 639
 640static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv)
 641{
 642	mbox_free_channel(priv->tx_ch);
 643	mbox_free_channel(priv->rx_ch);
 644	mbox_free_channel(priv->rxdb_ch);
 645}
 646
 647static int imx_dsp_rproc_mem_alloc(struct rproc *rproc,
 648				   struct rproc_mem_entry *mem)
 649{
 650	struct device *dev = rproc->dev.parent;
 651	void *va;
 652
 653	va = ioremap_wc(mem->dma, mem->len);
 654	if (!va) {
 655		dev_err(dev, "Unable to map memory region: %pa+%zx\n",
 656			&mem->dma, mem->len);
 657		return -ENOMEM;
 658	}
 659
 660	mem->va = va;
 661
 662	return 0;
 663}
 664
 665static int imx_dsp_rproc_mem_release(struct rproc *rproc,
 666				     struct rproc_mem_entry *mem)
 667{
 668	iounmap(mem->va);
 669
 670	return 0;
 671}
 672
 673/**
 674 * imx_dsp_rproc_add_carveout() - request mailbox channels
 675 * @priv: private data pointer
 676 *
 677 * This function registers specified memory entry in @rproc carveouts list
 678 * The carveouts can help to mapping the memory address for DSP.
 679 */
 680static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 681{
 682	const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
 683	const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
 684	struct rproc *rproc = priv->rproc;
 685	struct device *dev = rproc->dev.parent;
 686	struct device_node *np = dev->of_node;
 687	struct rproc_mem_entry *mem;
 688	int a, i = 0;
 689	u64 da;
 690
 691	/* Remap required addresses */
 692	for (a = 0; a < dcfg->att_size; a++) {
 693		const struct imx_rproc_att *att = &dcfg->att[a];
 694
 695		if (!(att->flags & ATT_OWN))
 696			continue;
 697
 698		if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da))
 699			return -EINVAL;
 700
 701		/* Register memory region */
 702		mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)att->sa,
 703					   att->size, da, imx_dsp_rproc_mem_alloc,
 704					   imx_dsp_rproc_mem_release, "dsp_mem");
 705
 706		if (mem)
 707			rproc_coredump_add_segment(rproc, da, att->size);
 708		else
 709			return -ENOMEM;
 710
 711		rproc_add_carveout(rproc, mem);
 712	}
 713
 714	while (1) {
 715		int err;
 716		struct resource res;
 717
 718		err = of_reserved_mem_region_to_resource(np, i++, &res);
 719		if (err)
 720			return 0;
 721
 722		/*
 723		 * Ignore the first memory region which will be used vdev buffer.
 724		 * No need to do extra handlings, rproc_add_virtio_dev will handle it.
 725		 */
 726		if (strstarts(res.name, "vdev0buffer"))
 727			continue;
 728
 729		if (imx_dsp_rproc_sys_to_da(priv, res.start, resource_size(&res), &da))
 730			return -EINVAL;
 731
 732		/* Register memory region */
 733		mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)res.start,
 734					   resource_size(&res), da,
 735					    imx_dsp_rproc_mem_alloc,
 736					    imx_dsp_rproc_mem_release,
 737					   "%.*s", strchrnul(res.name, '@') - res.name, res.name);
 738		if (!mem)
 739			return -ENOMEM;
 740
 741		rproc_coredump_add_segment(rproc, da, resource_size(&res));
 742		rproc_add_carveout(rproc, mem);
 743	}
 744}
 745
 746/* Prepare function for rproc_ops */
 747static int imx_dsp_rproc_prepare(struct rproc *rproc)
 748{
 749	struct imx_dsp_rproc *priv = rproc->priv;
 750	struct device *dev = rproc->dev.parent;
 751	int ret;
 752
 753	ret = imx_dsp_rproc_add_carveout(priv);
 754	if (ret) {
 755		dev_err(dev, "failed on imx_dsp_rproc_add_carveout\n");
 756		return ret;
 757	}
 758
 759	pm_runtime_get_sync(dev);
 760
 761	return 0;
 762}
 763
 764/* Unprepare function for rproc_ops */
 765static int imx_dsp_rproc_unprepare(struct rproc *rproc)
 766{
 767	pm_runtime_put_sync(rproc->dev.parent);
 768
 769	return 0;
 770}
 771
 772/* Kick function for rproc_ops */
 773static void imx_dsp_rproc_kick(struct rproc *rproc, int vqid)
 774{
 775	struct imx_dsp_rproc *priv = rproc->priv;
 776	struct device *dev = rproc->dev.parent;
 777	int err;
 778	__u32 mmsg;
 779
 780	if (!priv->tx_ch) {
 781		dev_err(dev, "No initialized mbox tx channel\n");
 782		return;
 783	}
 784
 785	/*
 786	 * Send the index of the triggered virtqueue as the mu payload.
 787	 * Let remote processor know which virtqueue is used.
 788	 */
 789	mmsg = vqid;
 790
 791	err = mbox_send_message(priv->tx_ch, (void *)&mmsg);
 792	if (err < 0)
 793		dev_err(dev, "%s: failed (%d, err:%d)\n", __func__, vqid, err);
 794}
 795
 796/*
 797 * Custom memory copy implementation for i.MX DSP Cores
 798 *
 799 * The IRAM is part of the HiFi DSP.
 800 * According to hw specs only 32-bits writes are allowed.
 801 */
 802static int imx_dsp_rproc_memcpy(void *dst, const void *src, size_t size)
 803{
 804	void __iomem *dest = (void __iomem *)dst;
 805	const u8 *src_byte = src;
 806	const u32 *source = src;
 807	u32 affected_mask;
 808	int i, q, r;
 809	u32 tmp;
 810
 811	/* destination must be 32bit aligned */
 812	if (!IS_ALIGNED((uintptr_t)dest, 4))
 813		return -EINVAL;
 814
 815	q = size / 4;
 816	r = size % 4;
 817
 818	/* copy data in units of 32 bits at a time */
 819	for (i = 0; i < q; i++)
 820		writel(source[i], dest + i * 4);
 821
 822	if (r) {
 823		affected_mask = GENMASK(8 * r, 0);
 824
 825		/*
 826		 * first read the 32bit data of dest, then change affected
 827		 * bytes, and write back to dest.
 828		 * For unaffected bytes, it should not be changed
 829		 */
 830		tmp = readl(dest + q * 4);
 831		tmp &= ~affected_mask;
 832
 833		/* avoid reading after end of source */
 834		for (i = 0; i < r; i++)
 835			tmp |= (src_byte[q * 4 + i] << (8 * i));
 836
 837		writel(tmp, dest + q * 4);
 838	}
 839
 840	return 0;
 841}
 842
 843/*
 844 * Custom memset implementation for i.MX DSP Cores
 845 *
 846 * The IRAM is part of the HiFi DSP.
 847 * According to hw specs only 32-bits writes are allowed.
 848 */
 849static int imx_dsp_rproc_memset(void *addr, u8 value, size_t size)
 850{
 851	void __iomem *tmp_dst = (void __iomem *)addr;
 852	u32 tmp_val = value;
 853	u32 affected_mask;
 854	int q, r;
 855	u32 tmp;
 856
 857	/* destination must be 32bit aligned */
 858	if (!IS_ALIGNED((uintptr_t)addr, 4))
 859		return -EINVAL;
 860
 861	tmp_val |= tmp_val << 8;
 862	tmp_val |= tmp_val << 16;
 863
 864	q = size / 4;
 865	r = size % 4;
 866
 867	while (q--)
 868		writel(tmp_val, tmp_dst++);
 869
 870	if (r) {
 871		affected_mask = GENMASK(8 * r, 0);
 872
 873		/*
 874		 * first read the 32bit data of addr, then change affected
 875		 * bytes, and write back to addr.
 876		 * For unaffected bytes, it should not be changed
 877		 */
 878		tmp = readl(tmp_dst);
 879		tmp &= ~affected_mask;
 880
 881		tmp |= (tmp_val & affected_mask);
 882		writel(tmp, tmp_dst);
 883	}
 884
 885	return 0;
 886}
 887
 888/*
 889 * imx_dsp_rproc_elf_load_segments() - load firmware segments to memory
 890 * @rproc: remote processor which will be booted using these fw segments
 891 * @fw: the ELF firmware image
 892 *
 893 * This function loads the firmware segments to memory, where the remote
 894 * processor expects them.
 895 *
 896 * Return: 0 on success and an appropriate error code otherwise
 897 */
 898static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
 899{
 900	struct device *dev = &rproc->dev;
 901	const void *ehdr, *phdr;
 902	int i, ret = 0;
 903	u16 phnum;
 904	const u8 *elf_data = fw->data;
 905	u8 class = fw_elf_get_class(fw);
 906	u32 elf_phdr_get_size = elf_size_of_phdr(class);
 907
 908	ehdr = elf_data;
 909	phnum = elf_hdr_get_e_phnum(class, ehdr);
 910	phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr);
 911
 912	/* go through the available ELF segments */
 913	for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) {
 914		u64 da = elf_phdr_get_p_paddr(class, phdr);
 915		u64 memsz = elf_phdr_get_p_memsz(class, phdr);
 916		u64 filesz = elf_phdr_get_p_filesz(class, phdr);
 917		u64 offset = elf_phdr_get_p_offset(class, phdr);
 918		u32 type = elf_phdr_get_p_type(class, phdr);
 919		void *ptr;
 920
 921		if (type != PT_LOAD || !memsz)
 922			continue;
 923
 924		dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n",
 925			type, da, memsz, filesz);
 926
 927		if (filesz > memsz) {
 928			dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n",
 929				filesz, memsz);
 930			ret = -EINVAL;
 931			break;
 932		}
 933
 934		if (offset + filesz > fw->size) {
 935			dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n",
 936				offset + filesz, fw->size);
 937			ret = -EINVAL;
 938			break;
 939		}
 940
 941		if (!rproc_u64_fit_in_size_t(memsz)) {
 942			dev_err(dev, "size (%llx) does not fit in size_t type\n",
 943				memsz);
 944			ret = -EOVERFLOW;
 945			break;
 946		}
 947
 948		/* grab the kernel address for this device address */
 949		ptr = rproc_da_to_va(rproc, da, memsz, NULL);
 950		if (!ptr) {
 951			dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da,
 952				memsz);
 953			ret = -EINVAL;
 954			break;
 955		}
 956
 957		/* put the segment where the remote processor expects it */
 958		if (filesz) {
 959			ret = imx_dsp_rproc_memcpy(ptr, elf_data + offset, filesz);
 960			if (ret) {
 961				dev_err(dev, "memory copy failed for da 0x%llx memsz 0x%llx\n",
 962					da, memsz);
 963				break;
 964			}
 965		}
 966
 967		/* zero out remaining memory for this segment */
 968		if (memsz > filesz) {
 969			ret = imx_dsp_rproc_memset(ptr + filesz, 0, memsz - filesz);
 970			if (ret) {
 971				dev_err(dev, "memset failed for da 0x%llx memsz 0x%llx\n",
 972					da, memsz);
 973				break;
 974			}
 975		}
 976	}
 977
 978	return ret;
 979}
 980
 981static int imx_dsp_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 982{
 983	if (rproc_elf_load_rsc_table(rproc, fw))
 984		dev_warn(&rproc->dev, "no resource table found for this firmware\n");
 985
 986	return 0;
 987}
 988
 989static int imx_dsp_rproc_load(struct rproc *rproc, const struct firmware *fw)
 990{
 991	struct imx_dsp_rproc *priv = rproc->priv;
 992	const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
 993	struct rproc_mem_entry *carveout;
 994	int ret;
 995
 996	/* Reset DSP if needed */
 997	if (dsp_dcfg->reset)
 998		dsp_dcfg->reset(priv);
 999	/*
1000	 * Clear buffers after pm rumtime for internal ocram is not
1001	 * accessible if power and clock are not enabled.
1002	 */
1003	if (rproc->state == RPROC_OFFLINE) {
1004		list_for_each_entry(carveout, &rproc->carveouts, node) {
1005			if (carveout->va)
1006				memset(carveout->va, 0, carveout->len);
1007		}
1008	}
1009
1010	ret = imx_dsp_rproc_elf_load_segments(rproc, fw);
1011	if (ret)
1012		return ret;
1013
1014	return 0;
1015}
1016
1017static const struct rproc_ops imx_dsp_rproc_ops = {
1018	.prepare	= imx_dsp_rproc_prepare,
1019	.unprepare	= imx_dsp_rproc_unprepare,
1020	.start		= imx_dsp_rproc_start,
1021	.stop		= imx_dsp_rproc_stop,
1022	.kick		= imx_dsp_rproc_kick,
1023	.load		= imx_dsp_rproc_load,
1024	.parse_fw	= imx_dsp_rproc_parse_fw,
1025	.handle_rsc	= imx_dsp_rproc_handle_rsc,
1026	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
1027	.sanity_check	= rproc_elf_sanity_check,
1028	.get_boot_addr	= rproc_elf_get_boot_addr,
1029};
1030
1031/**
1032 * imx_dsp_attach_pm_domains() - attach the power domains
1033 * @priv: private data pointer
1034 *
1035 * On i.MX8QM and i.MX8QXP there is multiple power domains
1036 * required, so need to link them.
1037 */
1038static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv)
1039{
1040	struct device *dev = priv->rproc->dev.parent;
1041
1042	/* A single PM domain is already attached. */
1043	if (dev->pm_domain)
1044		return 0;
1045
1046	return devm_pm_domain_attach_list(dev, NULL, &priv->pd_list);
1047}
1048
1049static int imx_dsp_rproc_mmio_detect_mode(struct rproc *rproc)
1050{
1051	struct imx_dsp_rproc *priv = rproc->priv;
1052	struct device *dev = rproc->dev.parent;
1053	struct regmap *regmap;
1054
1055	regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,dsp-ctrl");
1056	if (IS_ERR(regmap)) {
1057		dev_err(dev, "failed to find syscon\n");
1058		return PTR_ERR(regmap);
1059	}
1060
1061	priv->regmap = regmap;
1062
1063	return 0;
1064}
1065
1066static int imx_dsp_rproc_reset_ctrl_detect_mode(struct rproc *rproc)
1067{
1068	struct imx_dsp_rproc *priv = rproc->priv;
1069	struct device *dev = rproc->dev.parent;
1070
1071	priv->run_stall = devm_reset_control_get_exclusive(dev, "runstall");
1072	if (IS_ERR(priv->run_stall)) {
1073		dev_err(dev, "Failed to get DSP runstall reset control\n");
1074		return PTR_ERR(priv->run_stall);
1075	}
1076
1077	return 0;
1078}
1079
1080static int imx_dsp_rproc_scu_api_detect_mode(struct rproc *rproc)
1081{
1082	struct imx_dsp_rproc *priv = rproc->priv;
1083
1084	return imx_scu_get_handle(&priv->ipc_handle);
1085}
1086
1087/**
1088 * imx_dsp_rproc_detect_mode() - detect DSP control mode
1089 * @priv: private data pointer
1090 *
1091 * Different platform has different control method for DSP, which depends
1092 * on how the DSP is integrated in platform.
1093 *
1094 * For i.MX8QXP and i.MX8QM, DSP should be started and stopped by System
1095 * Control Unit.
1096 * For i.MX8MP and i.MX8ULP, DSP should be started and stopped by system
1097 * integration module.
1098 */
1099static int imx_dsp_rproc_detect_mode(struct imx_dsp_rproc *priv)
1100{
1101	const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
1102	const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
1103
1104	if (dcfg->ops && dcfg->ops->detect_mode)
1105		return dcfg->ops->detect_mode(priv->rproc);
1106
1107	return -EOPNOTSUPP;
1108}
1109
1110static const char *imx_dsp_clks_names[DSP_RPROC_CLK_MAX] = {
1111	/* DSP clocks */
1112	"core", "ocram", "debug", "ipg", "mu",
1113};
1114
1115static int imx_dsp_rproc_clk_get(struct imx_dsp_rproc *priv)
1116{
1117	struct device *dev = priv->rproc->dev.parent;
1118	struct clk_bulk_data *clks = priv->clks;
1119	int i;
1120
1121	for (i = 0; i < DSP_RPROC_CLK_MAX; i++)
1122		clks[i].id = imx_dsp_clks_names[i];
1123
1124	return devm_clk_bulk_get_optional(dev, DSP_RPROC_CLK_MAX, clks);
1125}
1126
1127static int imx_dsp_rproc_probe(struct platform_device *pdev)
1128{
1129	const struct imx_dsp_rproc_dcfg *dsp_dcfg;
1130	struct device *dev = &pdev->dev;
1131	struct imx_dsp_rproc *priv;
1132	struct rproc *rproc;
1133	const char *fw_name;
1134	int ret;
1135
1136	dsp_dcfg = of_device_get_match_data(dev);
1137	if (!dsp_dcfg)
1138		return -ENODEV;
1139
1140	ret = rproc_of_parse_firmware(dev, 0, &fw_name);
1141	if (ret)
1142		return dev_err_probe(dev, ret, "failed to parse firmware-name property\n");
1143
1144	rproc = devm_rproc_alloc(dev, "imx-dsp-rproc", &imx_dsp_rproc_ops,
1145				 fw_name, sizeof(*priv));
1146	if (!rproc)
1147		return -ENOMEM;
1148
1149	priv = rproc->priv;
1150	priv->rproc = rproc;
1151	priv->dsp_dcfg = dsp_dcfg;
1152	/* By default, host waits for fw_ready reply */
1153	priv->flags |= WAIT_FW_READY;
1154
1155	if (no_mailboxes)
1156		imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_no_alloc;
1157	else
1158		imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_alloc;
1159
1160	dev_set_drvdata(dev, rproc);
1161
1162	INIT_WORK(&priv->rproc_work, imx_dsp_rproc_vq_work);
1163
1164	ret = imx_dsp_rproc_detect_mode(priv);
1165	if (ret)
1166		return dev_err_probe(dev, ret, "failed on imx_dsp_rproc_detect_mode\n");
1167
1168	/* There are multiple power domains required by DSP on some platform */
1169	ret = imx_dsp_attach_pm_domains(priv);
1170	if (ret < 0)
1171		return dev_err_probe(dev, ret, "failed on imx_dsp_attach_pm_domains\n");
1172
1173	/* Get clocks */
1174	ret = imx_dsp_rproc_clk_get(priv);
1175	if (ret)
1176		return dev_err_probe(dev, ret, "failed on imx_dsp_rproc_clk_get\n");
1177
1178	init_completion(&priv->pm_comp);
1179	rproc->auto_boot = false;
1180	ret = devm_rproc_add(dev, rproc);
1181	if (ret)
1182		return dev_err_probe(dev, ret, "rproc_add failed\n");
1183
1184	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_XTENSA);
1185
1186	return devm_pm_runtime_enable(dev);
1187}
1188
1189/* pm runtime functions */
1190static int imx_dsp_runtime_resume(struct device *dev)
1191{
1192	struct rproc *rproc = dev_get_drvdata(dev);
1193	struct imx_dsp_rproc *priv = rproc->priv;
1194	int ret;
1195
1196	/*
1197	 * There is power domain attached with mailbox, if setup mailbox
1198	 * in probe(), then the power of mailbox is always enabled,
1199	 * the power can't be saved.
1200	 * So move setup of mailbox to runtime resume.
1201	 */
1202	ret = imx_dsp_rproc_mbox_init(priv);
1203	if (ret) {
1204		dev_err(dev, "failed on imx_dsp_rproc_mbox_init\n");
1205		return ret;
1206	}
1207
1208	ret = clk_bulk_prepare_enable(DSP_RPROC_CLK_MAX, priv->clks);
1209	if (ret) {
1210		dev_err(dev, "failed on clk_bulk_prepare_enable\n");
1211		return ret;
1212	}
1213
1214	return 0;
1215}
1216
1217static int imx_dsp_runtime_suspend(struct device *dev)
1218{
1219	struct rproc *rproc = dev_get_drvdata(dev);
1220	struct imx_dsp_rproc *priv = rproc->priv;
1221
1222	clk_bulk_disable_unprepare(DSP_RPROC_CLK_MAX, priv->clks);
1223
1224	imx_dsp_rproc_free_mbox(priv);
1225
1226	return 0;
1227}
1228
1229static void imx_dsp_load_firmware(const struct firmware *fw, void *context)
1230{
1231	struct rproc *rproc = context;
1232	int ret;
1233
1234	/*
1235	 * Same flow as start procedure.
1236	 * Load the ELF segments to memory firstly.
1237	 */
1238	ret = rproc_load_segments(rproc, fw);
1239	if (ret)
1240		goto out;
1241
1242	/* Start the remote processor */
1243	ret = rproc->ops->start(rproc);
1244	if (ret)
1245		goto out;
1246
1247	rproc->ops->kick(rproc, 0);
1248
1249out:
1250	release_firmware(fw);
1251}
1252
1253static int imx_dsp_suspend(struct device *dev)
1254{
1255	struct rproc *rproc = dev_get_drvdata(dev);
1256	struct imx_dsp_rproc *priv = rproc->priv;
1257	__u32 mmsg = RP_MBOX_SUSPEND_SYSTEM;
1258	int ret;
1259
1260	if (rproc->state != RPROC_RUNNING)
1261		goto out;
1262
1263	/*
1264	 * No channel available for sending messages;
1265	 * indicates no mailboxes present, so trigger PM runtime suspend
1266	 */
1267	if (!priv->tx_ch) {
1268		dev_dbg(dev, "No initialized mbox tx channel, suspend directly.\n");
1269		goto out;
1270	}
1271
1272	reinit_completion(&priv->pm_comp);
1273
1274	/* Tell DSP that suspend is happening */
1275	ret = mbox_send_message(priv->tx_ch, (void *)&mmsg);
1276	if (ret < 0) {
1277		dev_err(dev, "PM mbox_send_message failed: %d\n", ret);
1278		return ret;
1279	}
1280
1281	/*
1282	 * DSP need to save the context at suspend.
1283	 * Here waiting the response for DSP, then power can be disabled.
1284	 */
1285	if (!wait_for_completion_timeout(&priv->pm_comp, msecs_to_jiffies(100)))
1286		return -EBUSY;
1287
1288out:
1289	/*
1290	 * The power of DSP is disabled in suspend, so force pm runtime
1291	 * to be suspend, then we can reenable the power and clocks at
1292	 * resume stage.
1293	 */
1294	return pm_runtime_force_suspend(dev);
1295}
1296
1297static int imx_dsp_resume(struct device *dev)
1298{
1299	struct rproc *rproc = dev_get_drvdata(dev);
1300	int ret = 0;
1301
1302	ret = pm_runtime_force_resume(dev);
1303	if (ret)
1304		return ret;
1305
1306	if (rproc->state != RPROC_RUNNING)
1307		return 0;
1308
1309	/*
1310	 * The power of DSP is disabled at suspend, the memory of dsp
1311	 * is reset, the image segments are lost. So need to reload
1312	 * firmware and restart the DSP if it is in running state.
1313	 */
1314	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,
1315				      rproc->firmware, dev, GFP_KERNEL,
1316				      rproc, imx_dsp_load_firmware);
1317	if (ret < 0) {
1318		dev_err(dev, "load firmware failed: %d\n", ret);
1319		goto err;
1320	}
1321
1322	return 0;
1323
1324err:
1325	pm_runtime_force_suspend(dev);
1326
1327	return ret;
1328}
1329
1330static const struct dev_pm_ops imx_dsp_rproc_pm_ops = {
1331	SYSTEM_SLEEP_PM_OPS(imx_dsp_suspend, imx_dsp_resume)
1332	RUNTIME_PM_OPS(imx_dsp_runtime_suspend, imx_dsp_runtime_resume, NULL)
1333};
1334
1335static const struct imx_rproc_plat_ops imx_dsp_rproc_ops_mmio = {
1336	.start		= imx_dsp_rproc_mmio_start,
1337	.stop		= imx_dsp_rproc_mmio_stop,
1338	.detect_mode	= imx_dsp_rproc_mmio_detect_mode,
1339};
1340
1341static const struct imx_rproc_plat_ops imx_dsp_rproc_ops_reset_ctrl = {
1342	.start		= imx_dsp_rproc_reset_ctrl_start,
1343	.stop		= imx_dsp_rproc_reset_ctrl_stop,
1344	.detect_mode	= imx_dsp_rproc_reset_ctrl_detect_mode,
1345};
1346
1347static const struct imx_rproc_plat_ops imx_dsp_rproc_ops_scu_api = {
1348	.start		= imx_dsp_rproc_scu_api_start,
1349	.stop		= imx_dsp_rproc_scu_api_stop,
1350	.detect_mode	= imx_dsp_rproc_scu_api_detect_mode,
1351};
1352
1353/* Specific configuration for i.MX8MP */
1354static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8mp = {
1355	.att		= imx_dsp_rproc_att_imx8mp,
1356	.att_size	= ARRAY_SIZE(imx_dsp_rproc_att_imx8mp),
1357	.ops		= &imx_dsp_rproc_ops_reset_ctrl,
1358};
1359
1360static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8mp = {
1361	.dcfg		= &dsp_rproc_cfg_imx8mp,
1362	.reset          = imx8mp_dsp_reset,
1363};
1364
1365/* Specific configuration for i.MX8ULP */
1366static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8ulp = {
1367	.src_reg	= IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
1368	.src_mask	= IMX8ULP_SYSCTRL0_DSP_STALL,
1369	.src_start	= 0,
1370	.src_stop	= IMX8ULP_SYSCTRL0_DSP_STALL,
1371	.att		= imx_dsp_rproc_att_imx8ulp,
1372	.att_size	= ARRAY_SIZE(imx_dsp_rproc_att_imx8ulp),
1373	.ops		= &imx_dsp_rproc_ops_mmio,
1374};
1375
1376static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8ulp = {
1377	.dcfg		= &dsp_rproc_cfg_imx8ulp,
1378	.reset          = imx8ulp_dsp_reset,
1379};
1380
1381/* Specific configuration for i.MX8QXP */
1382static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qxp = {
1383	.att		= imx_dsp_rproc_att_imx8qxp,
1384	.att_size	= ARRAY_SIZE(imx_dsp_rproc_att_imx8qxp),
1385	.ops		= &imx_dsp_rproc_ops_scu_api,
1386};
1387
1388static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qxp = {
1389	.dcfg		= &dsp_rproc_cfg_imx8qxp,
1390};
1391
1392/* Specific configuration for i.MX8QM */
1393static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qm = {
1394	.att		= imx_dsp_rproc_att_imx8qm,
1395	.att_size	= ARRAY_SIZE(imx_dsp_rproc_att_imx8qm),
1396	.ops		= &imx_dsp_rproc_ops_scu_api,
1397};
1398
1399static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qm = {
1400	.dcfg		= &dsp_rproc_cfg_imx8qm,
1401};
1402
1403static const struct of_device_id imx_dsp_rproc_of_match[] = {
1404	{ .compatible = "fsl,imx8qxp-hifi4", .data = &imx_dsp_rproc_cfg_imx8qxp },
1405	{ .compatible = "fsl,imx8qm-hifi4",  .data = &imx_dsp_rproc_cfg_imx8qm },
1406	{ .compatible = "fsl,imx8mp-hifi4",  .data = &imx_dsp_rproc_cfg_imx8mp },
1407	{ .compatible = "fsl,imx8ulp-hifi4", .data = &imx_dsp_rproc_cfg_imx8ulp },
1408	{},
1409};
1410MODULE_DEVICE_TABLE(of, imx_dsp_rproc_of_match);
1411
1412static struct platform_driver imx_dsp_rproc_driver = {
1413	.probe = imx_dsp_rproc_probe,
1414	.driver = {
1415		.name = "imx-dsp-rproc",
1416		.of_match_table = imx_dsp_rproc_of_match,
1417		.pm = pm_ptr(&imx_dsp_rproc_pm_ops),
1418	},
1419};
1420module_platform_driver(imx_dsp_rproc_driver);
1421
1422MODULE_LICENSE("GPL v2");
1423MODULE_DESCRIPTION("i.MX HiFi Core Remote Processor Control Driver");
1424MODULE_AUTHOR("Shengjiu Wang <shengjiu.wang@nxp.com>");