tegra-hsp.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2016-2025, NVIDIA CORPORATION.  All rights reserved.
   4 */
   5
   6#include <linux/delay.h>
   7#include <linux/interrupt.h>
   8#include <linux/io.h>
   9#include <linux/mailbox_controller.h>
  10#include <linux/of.h>
  11#include <linux/platform_device.h>
  12#include <linux/pm.h>
  13#include <linux/slab.h>
  14
  15#include <soc/tegra/fuse.h>
  16
  17#include <dt-bindings/mailbox/tegra186-hsp.h>
  18
  19#include "mailbox.h"
  20
  21#define HSP_INT_IE(x)		(0x100 + ((x) * 4))
  22#define HSP_INT_IV		0x300
  23#define HSP_INT_IR		0x304
  24
  25#define HSP_INT_EMPTY_SHIFT	0
  26#define HSP_INT_EMPTY_MASK	0xff
  27#define HSP_INT_FULL_SHIFT	8
  28#define HSP_INT_FULL_MASK	0xff
  29
  30#define HSP_INT_DIMENSIONING	0x380
  31
  32#define HSP_DB_TRIGGER	0x0
  33#define HSP_DB_ENABLE	0x4
  34#define HSP_DB_RAW	0x8
  35#define HSP_DB_PENDING	0xc
  36
  37#define HSP_SM_SHRD_MBOX	0x0
  38#define HSP_SM_SHRD_MBOX_FULL	BIT(31)
  39#define HSP_SM_SHRD_MBOX_FULL_INT_IE	0x04
  40#define HSP_SM_SHRD_MBOX_EMPTY_INT_IE	0x08
  41
  42#define HSP_SHRD_MBOX_TYPE1_TAG		0x40
  43#define HSP_SHRD_MBOX_TYPE1_DATA0	0x48
  44#define HSP_SHRD_MBOX_TYPE1_DATA1	0x4c
  45#define HSP_SHRD_MBOX_TYPE1_DATA2	0x50
  46#define HSP_SHRD_MBOX_TYPE1_DATA3	0x54
  47
  48#define HSP_DB_CCPLEX		1
  49#define HSP_DB_BPMP		3
  50#define HSP_DB_MAX		7
  51
  52#define HSP_MBOX_TYPE_MASK	0xff
  53
  54struct tegra_hsp_channel;
  55struct tegra_hsp;
  56
  57struct tegra_hsp_channel {
  58	struct tegra_hsp *hsp;
  59	struct mbox_chan *chan;
  60	void __iomem *regs;
  61};
  62
  63struct tegra_hsp_doorbell {
  64	struct tegra_hsp_channel channel;
  65	struct list_head list;
  66	const char *name;
  67	unsigned int master;
  68	unsigned int index;
  69};
  70
  71struct tegra_hsp_sm_ops {
  72	void (*send)(struct tegra_hsp_channel *channel, void *data);
  73	void (*recv)(struct tegra_hsp_channel *channel);
  74};
  75
  76struct tegra_hsp_mailbox {
  77	struct tegra_hsp_channel channel;
  78	const struct tegra_hsp_sm_ops *ops;
  79	unsigned int index;
  80	bool producer;
  81};
  82
  83struct tegra_hsp_db_map {
  84	const char *name;
  85	unsigned int master;
  86	unsigned int index;
  87};
  88
  89struct tegra_hsp_soc {
  90	const struct tegra_hsp_db_map *map;
  91	bool has_per_mb_ie;
  92	bool has_128_bit_mb;
  93	unsigned int reg_stride;
  94
  95	/* Shifts for dimensioning register. */
  96	unsigned int si_shift;
  97	unsigned int db_shift;
  98	unsigned int as_shift;
  99	unsigned int ss_shift;
 100	unsigned int sm_shift;
 101
 102	/* Masks for dimensioning register. */
 103	unsigned int si_mask;
 104	unsigned int db_mask;
 105	unsigned int as_mask;
 106	unsigned int ss_mask;
 107	unsigned int sm_mask;
 108};
 109
 110struct tegra_hsp {
 111	struct device *dev;
 112	const struct tegra_hsp_soc *soc;
 113	struct mbox_controller mbox_db;
 114	struct mbox_controller mbox_sm;
 115	void __iomem *regs;
 116	unsigned int doorbell_irq;
 117	unsigned int *shared_irqs;
 118	unsigned int shared_irq;
 119	unsigned int num_sm;
 120	unsigned int num_as;
 121	unsigned int num_ss;
 122	unsigned int num_db;
 123	unsigned int num_si;
 124
 125	spinlock_t lock;
 126	struct lock_class_key lock_key;
 127
 128	struct list_head doorbells;
 129	struct tegra_hsp_mailbox *mailboxes;
 130
 131	unsigned long mask;
 132};
 133
 134static inline u32 tegra_hsp_readl(struct tegra_hsp *hsp, unsigned int offset)
 135{
 136	return readl(hsp->regs + offset);
 137}
 138
 139static inline void tegra_hsp_writel(struct tegra_hsp *hsp, u32 value,
 140				    unsigned int offset)
 141{
 142	writel(value, hsp->regs + offset);
 143}
 144
 145static inline u32 tegra_hsp_channel_readl(struct tegra_hsp_channel *channel,
 146					  unsigned int offset)
 147{
 148	return readl(channel->regs + offset);
 149}
 150
 151static inline void tegra_hsp_channel_writel(struct tegra_hsp_channel *channel,
 152					    u32 value, unsigned int offset)
 153{
 154	writel(value, channel->regs + offset);
 155}
 156
 157static bool tegra_hsp_doorbell_can_ring(struct tegra_hsp_doorbell *db)
 158{
 159	u32 value;
 160
 161	value = tegra_hsp_channel_readl(&db->channel, HSP_DB_ENABLE);
 162
 163	return (value & BIT(TEGRA_HSP_DB_MASTER_CCPLEX)) != 0;
 164}
 165
 166static struct tegra_hsp_doorbell *
 167__tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master)
 168{
 169	struct tegra_hsp_doorbell *entry;
 170
 171	list_for_each_entry(entry, &hsp->doorbells, list)
 172		if (entry->master == master)
 173			return entry;
 174
 175	return NULL;
 176}
 177
 178static struct tegra_hsp_doorbell *
 179tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master)
 180{
 181	struct tegra_hsp_doorbell *db;
 182	unsigned long flags;
 183
 184	spin_lock_irqsave(&hsp->lock, flags);
 185	db = __tegra_hsp_doorbell_get(hsp, master);
 186	spin_unlock_irqrestore(&hsp->lock, flags);
 187
 188	return db;
 189}
 190
 191static irqreturn_t tegra_hsp_doorbell_irq(int irq, void *data)
 192{
 193	struct tegra_hsp *hsp = data;
 194	struct tegra_hsp_doorbell *db;
 195	unsigned long master, value;
 196
 197	db = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX);
 198	if (!db)
 199		return IRQ_NONE;
 200
 201	value = tegra_hsp_channel_readl(&db->channel, HSP_DB_PENDING);
 202	tegra_hsp_channel_writel(&db->channel, value, HSP_DB_PENDING);
 203
 204	spin_lock(&hsp->lock);
 205
 206	for_each_set_bit(master, &value, hsp->mbox_db.num_chans) {
 207		struct tegra_hsp_doorbell *db;
 208
 209		db = __tegra_hsp_doorbell_get(hsp, master);
 210		/*
 211		 * Depending on the bootloader chain, the CCPLEX doorbell will
 212		 * have some doorbells enabled, which means that requesting an
 213		 * interrupt will immediately fire.
 214		 *
 215		 * In that case, db->channel.chan will still be NULL here and
 216		 * cause a crash if not properly guarded.
 217		 *
 218		 * It remains to be seen if ignoring the doorbell in that case
 219		 * is the correct solution.
 220		 */
 221		if (db && db->channel.chan)
 222			mbox_chan_received_data(db->channel.chan, NULL);
 223	}
 224
 225	spin_unlock(&hsp->lock);
 226
 227	return IRQ_HANDLED;
 228}
 229
 230static irqreturn_t tegra_hsp_shared_irq(int irq, void *data)
 231{
 232	struct tegra_hsp *hsp = data;
 233	unsigned long bit, mask;
 234	u32 status;
 235
 236	status = tegra_hsp_readl(hsp, HSP_INT_IR) & hsp->mask;
 237
 238	/* process EMPTY interrupts first */
 239	mask = (status >> HSP_INT_EMPTY_SHIFT) & HSP_INT_EMPTY_MASK;
 240
 241	for_each_set_bit(bit, &mask, hsp->num_sm) {
 242		struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit];
 243
 244		if (mb->producer) {
 245			/*
 246			 * Disable EMPTY interrupts until data is sent with
 247			 * the next message. These interrupts are level-
 248			 * triggered, so if we kept them enabled they would
 249			 * constantly trigger until we next write data into
 250			 * the message.
 251			 */
 252			spin_lock(&hsp->lock);
 253
 254			hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index);
 255			tegra_hsp_writel(hsp, hsp->mask,
 256					 HSP_INT_IE(hsp->shared_irq));
 257
 258			spin_unlock(&hsp->lock);
 259
 260			mbox_chan_txdone(mb->channel.chan, 0);
 261		}
 262	}
 263
 264	/* process FULL interrupts */
 265	mask = (status >> HSP_INT_FULL_SHIFT) & HSP_INT_FULL_MASK;
 266
 267	for_each_set_bit(bit, &mask, hsp->num_sm) {
 268		struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit];
 269
 270		if (!mb->producer)
 271			mb->ops->recv(&mb->channel);
 272	}
 273
 274	return IRQ_HANDLED;
 275}
 276
 277static struct tegra_hsp_channel *
 278tegra_hsp_doorbell_create(struct tegra_hsp *hsp, const char *name,
 279			  unsigned int master, unsigned int index)
 280{
 281	struct tegra_hsp_doorbell *db;
 282	unsigned int offset;
 283	unsigned long flags;
 284
 285	db = devm_kzalloc(hsp->dev, sizeof(*db), GFP_KERNEL);
 286	if (!db)
 287		return ERR_PTR(-ENOMEM);
 288
 289	offset = (1 + (hsp->num_sm / 2) + hsp->num_ss + hsp->num_as) * SZ_64K;
 290	offset += index * hsp->soc->reg_stride;
 291
 292	db->channel.regs = hsp->regs + offset;
 293	db->channel.hsp = hsp;
 294
 295	db->name = devm_kstrdup_const(hsp->dev, name, GFP_KERNEL);
 296	db->master = master;
 297	db->index = index;
 298
 299	spin_lock_irqsave(&hsp->lock, flags);
 300	list_add_tail(&db->list, &hsp->doorbells);
 301	spin_unlock_irqrestore(&hsp->lock, flags);
 302
 303	return &db->channel;
 304}
 305
 306static int tegra_hsp_doorbell_send_data(struct mbox_chan *chan, void *data)
 307{
 308	struct tegra_hsp_doorbell *db = chan->con_priv;
 309
 310	tegra_hsp_channel_writel(&db->channel, 1, HSP_DB_TRIGGER);
 311
 312	return 0;
 313}
 314
 315static int tegra_hsp_doorbell_startup(struct mbox_chan *chan)
 316{
 317	struct tegra_hsp_doorbell *db = chan->con_priv;
 318	struct tegra_hsp *hsp = db->channel.hsp;
 319	struct tegra_hsp_doorbell *ccplex;
 320	unsigned long flags;
 321	u32 value;
 322
 323	if (db->master >= chan->mbox->num_chans) {
 324		dev_err(chan->mbox->dev,
 325			"invalid master ID %u for HSP channel\n",
 326			db->master);
 327		return -EINVAL;
 328	}
 329
 330	ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX);
 331	if (!ccplex)
 332		return -ENODEV;
 333
 334	/*
 335	 * On simulation platforms the BPMP hasn't had a chance yet to mark
 336	 * the doorbell as ringable by the CCPLEX, so we want to skip extra
 337	 * checks here.
 338	 */
 339	if (tegra_is_silicon() && !tegra_hsp_doorbell_can_ring(db))
 340		return -ENODEV;
 341
 342	spin_lock_irqsave(&hsp->lock, flags);
 343
 344	value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE);
 345	value |= BIT(db->master);
 346	tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE);
 347
 348	spin_unlock_irqrestore(&hsp->lock, flags);
 349
 350	return 0;
 351}
 352
 353static void tegra_hsp_doorbell_shutdown(struct mbox_chan *chan)
 354{
 355	struct tegra_hsp_doorbell *db = chan->con_priv;
 356	struct tegra_hsp *hsp = db->channel.hsp;
 357	struct tegra_hsp_doorbell *ccplex;
 358	unsigned long flags;
 359	u32 value;
 360
 361	ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX);
 362	if (!ccplex)
 363		return;
 364
 365	spin_lock_irqsave(&hsp->lock, flags);
 366
 367	value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE);
 368	value &= ~BIT(db->master);
 369	tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE);
 370
 371	spin_unlock_irqrestore(&hsp->lock, flags);
 372}
 373
 374static const struct mbox_chan_ops tegra_hsp_db_ops = {
 375	.send_data = tegra_hsp_doorbell_send_data,
 376	.startup = tegra_hsp_doorbell_startup,
 377	.shutdown = tegra_hsp_doorbell_shutdown,
 378};
 379
 380static void tegra_hsp_sm_send32(struct tegra_hsp_channel *channel, void *data)
 381{
 382	u32 value;
 383
 384	/* copy data and mark mailbox full */
 385	value = (u32)(unsigned long)data;
 386	value |= HSP_SM_SHRD_MBOX_FULL;
 387
 388	tegra_hsp_channel_writel(channel, value, HSP_SM_SHRD_MBOX);
 389}
 390
 391static void tegra_hsp_sm_recv32(struct tegra_hsp_channel *channel)
 392{
 393	u32 value;
 394	void *msg;
 395
 396	value = tegra_hsp_channel_readl(channel, HSP_SM_SHRD_MBOX);
 397	value &= ~HSP_SM_SHRD_MBOX_FULL;
 398	msg = (void *)(unsigned long)value;
 399
 400	/*
 401	 * Need to clear all bits here since some producers, such as TCU, depend
 402	 * on fields in the register getting cleared by the consumer.
 403	 *
 404	 * The mailbox API doesn't give the consumers a way of doing that
 405	 * explicitly, so we have to make sure we cover all possible cases.
 406	 */
 407	tegra_hsp_channel_writel(channel, 0x0, HSP_SM_SHRD_MBOX);
 408
 409	mbox_chan_received_data(channel->chan, msg);
 410}
 411
 412static const struct tegra_hsp_sm_ops tegra_hsp_sm_32bit_ops = {
 413	.send = tegra_hsp_sm_send32,
 414	.recv = tegra_hsp_sm_recv32,
 415};
 416
 417static void tegra_hsp_sm_send128(struct tegra_hsp_channel *channel, void *data)
 418{
 419	u32 value[4];
 420
 421	memcpy(value, data, sizeof(value));
 422
 423	/* Copy data */
 424	tegra_hsp_channel_writel(channel, value[0], HSP_SHRD_MBOX_TYPE1_DATA0);
 425	tegra_hsp_channel_writel(channel, value[1], HSP_SHRD_MBOX_TYPE1_DATA1);
 426	tegra_hsp_channel_writel(channel, value[2], HSP_SHRD_MBOX_TYPE1_DATA2);
 427	tegra_hsp_channel_writel(channel, value[3], HSP_SHRD_MBOX_TYPE1_DATA3);
 428
 429	/* Update tag to mark mailbox full */
 430	tegra_hsp_channel_writel(channel, HSP_SM_SHRD_MBOX_FULL,
 431				 HSP_SHRD_MBOX_TYPE1_TAG);
 432}
 433
 434static void tegra_hsp_sm_recv128(struct tegra_hsp_channel *channel)
 435{
 436	u32 value[4];
 437	void *msg;
 438
 439	value[0] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA0);
 440	value[1] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA1);
 441	value[2] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA2);
 442	value[3] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA3);
 443
 444	msg = (void *)(unsigned long)value;
 445
 446	/*
 447	 * Clear data registers and tag.
 448	 */
 449	tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA0);
 450	tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA1);
 451	tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA2);
 452	tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA3);
 453	tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_TAG);
 454
 455	mbox_chan_received_data(channel->chan, msg);
 456}
 457
 458static const struct tegra_hsp_sm_ops tegra_hsp_sm_128bit_ops = {
 459	.send = tegra_hsp_sm_send128,
 460	.recv = tegra_hsp_sm_recv128,
 461};
 462
 463static int tegra_hsp_mailbox_send_data(struct mbox_chan *chan, void *data)
 464{
 465	struct tegra_hsp_mailbox *mb = chan->con_priv;
 466	struct tegra_hsp *hsp = mb->channel.hsp;
 467	unsigned long flags;
 468
 469	if (WARN_ON(!mb->producer))
 470		return -EPERM;
 471
 472	mb->ops->send(&mb->channel, data);
 473
 474	/* enable EMPTY interrupt for the shared mailbox */
 475	spin_lock_irqsave(&hsp->lock, flags);
 476
 477	hsp->mask |= BIT(HSP_INT_EMPTY_SHIFT + mb->index);
 478	tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq));
 479
 480	spin_unlock_irqrestore(&hsp->lock, flags);
 481
 482	return 0;
 483}
 484
 485static int tegra_hsp_mailbox_flush(struct mbox_chan *chan,
 486				   unsigned long timeout)
 487{
 488	struct tegra_hsp_mailbox *mb = chan->con_priv;
 489	struct tegra_hsp_channel *ch = &mb->channel;
 490	u32 value;
 491
 492	timeout = jiffies + msecs_to_jiffies(timeout);
 493
 494	while (time_before(jiffies, timeout)) {
 495		value = tegra_hsp_channel_readl(ch, HSP_SM_SHRD_MBOX);
 496		if ((value & HSP_SM_SHRD_MBOX_FULL) == 0) {
 497			mbox_chan_txdone(chan, 0);
 498
 499			/* Wait until channel is empty */
 500			if (chan->active_req != NULL)
 501				continue;
 502
 503			return 0;
 504		}
 505
 506		udelay(1);
 507	}
 508
 509	return -ETIME;
 510}
 511
 512static int tegra_hsp_mailbox_startup(struct mbox_chan *chan)
 513{
 514	struct tegra_hsp_mailbox *mb = chan->con_priv;
 515	struct tegra_hsp_channel *ch = &mb->channel;
 516	struct tegra_hsp *hsp = mb->channel.hsp;
 517	unsigned long flags;
 518
 519	chan->txdone_method = TXDONE_BY_IRQ;
 520
 521	/*
 522	 * Shared mailboxes start out as consumers by default. FULL and EMPTY
 523	 * interrupts are coalesced at the same shared interrupt.
 524	 *
 525	 * Keep EMPTY interrupts disabled at startup and only enable them when
 526	 * the mailbox is actually full. This is required because the FULL and
 527	 * EMPTY interrupts are level-triggered, so keeping EMPTY interrupts
 528	 * enabled all the time would cause an interrupt storm while mailboxes
 529	 * are idle.
 530	 */
 531
 532	spin_lock_irqsave(&hsp->lock, flags);
 533
 534	if (mb->producer)
 535		hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index);
 536	else
 537		hsp->mask |= BIT(HSP_INT_FULL_SHIFT + mb->index);
 538
 539	tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq));
 540
 541	spin_unlock_irqrestore(&hsp->lock, flags);
 542
 543	if (hsp->soc->has_per_mb_ie) {
 544		if (mb->producer)
 545			tegra_hsp_channel_writel(ch, 0x0,
 546						 HSP_SM_SHRD_MBOX_EMPTY_INT_IE);
 547		else
 548			tegra_hsp_channel_writel(ch, 0x1,
 549						 HSP_SM_SHRD_MBOX_FULL_INT_IE);
 550	}
 551
 552	return 0;
 553}
 554
 555static void tegra_hsp_mailbox_shutdown(struct mbox_chan *chan)
 556{
 557	struct tegra_hsp_mailbox *mb = chan->con_priv;
 558	struct tegra_hsp_channel *ch = &mb->channel;
 559	struct tegra_hsp *hsp = mb->channel.hsp;
 560	unsigned long flags;
 561
 562	if (hsp->soc->has_per_mb_ie) {
 563		if (mb->producer)
 564			tegra_hsp_channel_writel(ch, 0x0,
 565						 HSP_SM_SHRD_MBOX_EMPTY_INT_IE);
 566		else
 567			tegra_hsp_channel_writel(ch, 0x0,
 568						 HSP_SM_SHRD_MBOX_FULL_INT_IE);
 569	}
 570
 571	spin_lock_irqsave(&hsp->lock, flags);
 572
 573	if (mb->producer)
 574		hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index);
 575	else
 576		hsp->mask &= ~BIT(HSP_INT_FULL_SHIFT + mb->index);
 577
 578	tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq));
 579
 580	spin_unlock_irqrestore(&hsp->lock, flags);
 581}
 582
 583static const struct mbox_chan_ops tegra_hsp_sm_ops = {
 584	.send_data = tegra_hsp_mailbox_send_data,
 585	.flush = tegra_hsp_mailbox_flush,
 586	.startup = tegra_hsp_mailbox_startup,
 587	.shutdown = tegra_hsp_mailbox_shutdown,
 588};
 589
 590static struct mbox_chan *tegra_hsp_db_xlate(struct mbox_controller *mbox,
 591					    const struct of_phandle_args *args)
 592{
 593	struct tegra_hsp *hsp = container_of(mbox, struct tegra_hsp, mbox_db);
 594	unsigned int type = args->args[0], master = args->args[1];
 595	struct tegra_hsp_channel *channel = ERR_PTR(-ENODEV);
 596	struct tegra_hsp_doorbell *db;
 597	struct mbox_chan *chan;
 598	unsigned long flags;
 599	unsigned int i;
 600
 601	if (type != TEGRA_HSP_MBOX_TYPE_DB || !hsp->doorbell_irq)
 602		return ERR_PTR(-ENODEV);
 603
 604	db = tegra_hsp_doorbell_get(hsp, master);
 605	if (db)
 606		channel = &db->channel;
 607
 608	if (IS_ERR(channel))
 609		return ERR_CAST(channel);
 610
 611	spin_lock_irqsave(&hsp->lock, flags);
 612
 613	for (i = 0; i < mbox->num_chans; i++) {
 614		chan = &mbox->chans[i];
 615		if (!chan->con_priv) {
 616			channel->chan = chan;
 617			chan->con_priv = db;
 618			break;
 619		}
 620
 621		chan = NULL;
 622	}
 623
 624	spin_unlock_irqrestore(&hsp->lock, flags);
 625
 626	return chan ?: ERR_PTR(-EBUSY);
 627}
 628
 629static struct mbox_chan *tegra_hsp_sm_xlate(struct mbox_controller *mbox,
 630					    const struct of_phandle_args *args)
 631{
 632	struct tegra_hsp *hsp = container_of(mbox, struct tegra_hsp, mbox_sm);
 633	unsigned int type = args->args[0], index;
 634	struct tegra_hsp_mailbox *mb;
 635
 636	index = args->args[1] & TEGRA_HSP_SM_MASK;
 637
 638	if ((type & HSP_MBOX_TYPE_MASK) != TEGRA_HSP_MBOX_TYPE_SM ||
 639	    !hsp->shared_irqs || index >= hsp->num_sm)
 640		return ERR_PTR(-ENODEV);
 641
 642	mb = &hsp->mailboxes[index];
 643
 644	if (type & TEGRA_HSP_MBOX_TYPE_SM_128BIT) {
 645		if (!hsp->soc->has_128_bit_mb)
 646			return ERR_PTR(-ENODEV);
 647
 648		mb->ops = &tegra_hsp_sm_128bit_ops;
 649	} else {
 650		mb->ops = &tegra_hsp_sm_32bit_ops;
 651	}
 652
 653	if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0)
 654		mb->producer = false;
 655	else
 656		mb->producer = true;
 657
 658	return mb->channel.chan;
 659}
 660
 661static int tegra_hsp_add_doorbells(struct tegra_hsp *hsp)
 662{
 663	const struct tegra_hsp_db_map *map = hsp->soc->map;
 664	struct tegra_hsp_channel *channel;
 665
 666	while (map->name) {
 667		channel = tegra_hsp_doorbell_create(hsp, map->name,
 668						    map->master, map->index);
 669		if (IS_ERR(channel))
 670			return PTR_ERR(channel);
 671
 672		map++;
 673	}
 674
 675	return 0;
 676}
 677
 678static int tegra_hsp_add_mailboxes(struct tegra_hsp *hsp, struct device *dev)
 679{
 680	int i;
 681
 682	hsp->mailboxes = devm_kcalloc(dev, hsp->num_sm, sizeof(*hsp->mailboxes),
 683				      GFP_KERNEL);
 684	if (!hsp->mailboxes)
 685		return -ENOMEM;
 686
 687	for (i = 0; i < hsp->num_sm; i++) {
 688		struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i];
 689
 690		mb->index = i;
 691
 692		mb->channel.hsp = hsp;
 693		mb->channel.regs = hsp->regs + SZ_64K + i * SZ_32K;
 694		mb->channel.chan = &hsp->mbox_sm.chans[i];
 695		mb->channel.chan->con_priv = mb;
 696	}
 697
 698	return 0;
 699}
 700
 701static int tegra_hsp_request_shared_irq(struct tegra_hsp *hsp)
 702{
 703	unsigned int i, irq = 0;
 704	int err;
 705
 706	for (i = 0; i < hsp->num_si; i++) {
 707		irq = hsp->shared_irqs[i];
 708		if (irq <= 0)
 709			continue;
 710
 711		err = devm_request_irq(hsp->dev, irq, tegra_hsp_shared_irq, 0,
 712				       dev_name(hsp->dev), hsp);
 713		if (err < 0) {
 714			dev_err(hsp->dev, "failed to request interrupt: %d\n",
 715				err);
 716			continue;
 717		}
 718
 719		hsp->shared_irq = i;
 720
 721		/* disable all interrupts */
 722		tegra_hsp_writel(hsp, 0, HSP_INT_IE(hsp->shared_irq));
 723
 724		dev_dbg(hsp->dev, "interrupt requested: %u\n", irq);
 725
 726		break;
 727	}
 728
 729	if (i == hsp->num_si) {
 730		dev_err(hsp->dev, "failed to find available interrupt\n");
 731		return -ENOENT;
 732	}
 733
 734	return 0;
 735}
 736
 737static int tegra_hsp_probe(struct platform_device *pdev)
 738{
 739	struct tegra_hsp *hsp;
 740	unsigned int i;
 741	u32 value;
 742	int err;
 743
 744	hsp = devm_kzalloc(&pdev->dev, sizeof(*hsp), GFP_KERNEL);
 745	if (!hsp)
 746		return -ENOMEM;
 747
 748	hsp->dev = &pdev->dev;
 749	hsp->soc = of_device_get_match_data(&pdev->dev);
 750	INIT_LIST_HEAD(&hsp->doorbells);
 751	spin_lock_init(&hsp->lock);
 752
 753	hsp->regs = devm_platform_ioremap_resource(pdev, 0);
 754	if (IS_ERR(hsp->regs))
 755		return PTR_ERR(hsp->regs);
 756
 757	value = tegra_hsp_readl(hsp, HSP_INT_DIMENSIONING);
 758	hsp->num_sm = (value >> hsp->soc->sm_shift) & hsp->soc->sm_mask;
 759	hsp->num_ss = (value >> hsp->soc->ss_shift) & hsp->soc->ss_mask;
 760	hsp->num_as = (value >> hsp->soc->as_shift) & hsp->soc->as_mask;
 761	hsp->num_db = (value >> hsp->soc->db_shift) & hsp->soc->db_mask;
 762	hsp->num_si = (value >> hsp->soc->si_shift) & hsp->soc->si_mask;
 763
 764	err = platform_get_irq_byname_optional(pdev, "doorbell");
 765	if (err >= 0)
 766		hsp->doorbell_irq = err;
 767
 768	if (hsp->num_si > 0) {
 769		unsigned int count = 0;
 770
 771		hsp->shared_irqs = devm_kcalloc(&pdev->dev, hsp->num_si,
 772						sizeof(*hsp->shared_irqs),
 773						GFP_KERNEL);
 774		if (!hsp->shared_irqs)
 775			return -ENOMEM;
 776
 777		for (i = 0; i < hsp->num_si; i++) {
 778			char *name;
 779
 780			name = kasprintf(GFP_KERNEL, "shared%u", i);
 781			if (!name)
 782				return -ENOMEM;
 783
 784			err = platform_get_irq_byname_optional(pdev, name);
 785			if (err >= 0) {
 786				hsp->shared_irqs[i] = err;
 787				count++;
 788			}
 789
 790			kfree(name);
 791		}
 792
 793		if (count == 0) {
 794			devm_kfree(&pdev->dev, hsp->shared_irqs);
 795			hsp->shared_irqs = NULL;
 796		}
 797	}
 798
 799	/* setup the doorbell controller */
 800	hsp->mbox_db.of_xlate = tegra_hsp_db_xlate;
 801	hsp->mbox_db.num_chans = 32;
 802	hsp->mbox_db.dev = &pdev->dev;
 803	hsp->mbox_db.ops = &tegra_hsp_db_ops;
 804
 805	hsp->mbox_db.chans = devm_kcalloc(&pdev->dev, hsp->mbox_db.num_chans,
 806					  sizeof(*hsp->mbox_db.chans),
 807					  GFP_KERNEL);
 808	if (!hsp->mbox_db.chans)
 809		return -ENOMEM;
 810
 811	if (hsp->doorbell_irq) {
 812		err = tegra_hsp_add_doorbells(hsp);
 813		if (err < 0) {
 814			dev_err(&pdev->dev, "failed to add doorbells: %d\n",
 815			        err);
 816			return err;
 817		}
 818	}
 819
 820	err = devm_mbox_controller_register(&pdev->dev, &hsp->mbox_db);
 821	if (err < 0) {
 822		dev_err(&pdev->dev, "failed to register doorbell mailbox: %d\n",
 823			err);
 824		return err;
 825	}
 826
 827	/* setup the shared mailbox controller */
 828	hsp->mbox_sm.of_xlate = tegra_hsp_sm_xlate;
 829	hsp->mbox_sm.num_chans = hsp->num_sm;
 830	hsp->mbox_sm.dev = &pdev->dev;
 831	hsp->mbox_sm.ops = &tegra_hsp_sm_ops;
 832
 833	hsp->mbox_sm.chans = devm_kcalloc(&pdev->dev, hsp->mbox_sm.num_chans,
 834					  sizeof(*hsp->mbox_sm.chans),
 835					  GFP_KERNEL);
 836	if (!hsp->mbox_sm.chans)
 837		return -ENOMEM;
 838
 839	if (hsp->shared_irqs) {
 840		err = tegra_hsp_add_mailboxes(hsp, &pdev->dev);
 841		if (err < 0) {
 842			dev_err(&pdev->dev, "failed to add mailboxes: %d\n",
 843			        err);
 844			return err;
 845		}
 846	}
 847
 848	err = devm_mbox_controller_register(&pdev->dev, &hsp->mbox_sm);
 849	if (err < 0) {
 850		dev_err(&pdev->dev, "failed to register shared mailbox: %d\n",
 851			err);
 852		return err;
 853	}
 854
 855	platform_set_drvdata(pdev, hsp);
 856
 857	if (hsp->doorbell_irq) {
 858		err = devm_request_irq(&pdev->dev, hsp->doorbell_irq,
 859				       tegra_hsp_doorbell_irq, IRQF_NO_SUSPEND,
 860				       dev_name(&pdev->dev), hsp);
 861		if (err < 0) {
 862			dev_err(&pdev->dev,
 863			        "failed to request doorbell IRQ#%u: %d\n",
 864				hsp->doorbell_irq, err);
 865			return err;
 866		}
 867	}
 868
 869	if (hsp->shared_irqs) {
 870		err = tegra_hsp_request_shared_irq(hsp);
 871		if (err < 0)
 872			return err;
 873	}
 874
 875	lockdep_register_key(&hsp->lock_key);
 876	lockdep_set_class(&hsp->lock, &hsp->lock_key);
 877
 878	return 0;
 879}
 880
 881static void tegra_hsp_remove(struct platform_device *pdev)
 882{
 883	struct tegra_hsp *hsp = platform_get_drvdata(pdev);
 884
 885	lockdep_unregister_key(&hsp->lock_key);
 886}
 887
 888static int __maybe_unused tegra_hsp_resume(struct device *dev)
 889{
 890	struct tegra_hsp *hsp = dev_get_drvdata(dev);
 891	unsigned int i;
 892	struct tegra_hsp_doorbell *db;
 893
 894	list_for_each_entry(db, &hsp->doorbells, list) {
 895		if (db->channel.chan)
 896			tegra_hsp_doorbell_startup(db->channel.chan);
 897	}
 898
 899	if (hsp->mailboxes) {
 900		for (i = 0; i < hsp->num_sm; i++) {
 901			struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i];
 902
 903			if (mb->channel.chan->cl)
 904				tegra_hsp_mailbox_startup(mb->channel.chan);
 905		}
 906	}
 907
 908	return 0;
 909}
 910
 911static const struct dev_pm_ops tegra_hsp_pm_ops = {
 912	.resume_noirq = tegra_hsp_resume,
 913};
 914
 915static const struct tegra_hsp_db_map tegra186_hsp_db_map[] = {
 916	{ "ccplex", TEGRA_HSP_DB_MASTER_CCPLEX, HSP_DB_CCPLEX, },
 917	{ "bpmp",   TEGRA_HSP_DB_MASTER_BPMP,   HSP_DB_BPMP,   },
 918	{ /* sentinel */ }
 919};
 920
 921static const struct tegra_hsp_soc tegra186_hsp_soc = {
 922	.map = tegra186_hsp_db_map,
 923	.has_per_mb_ie = false,
 924	.has_128_bit_mb = false,
 925	.reg_stride = 0x100,
 926	.si_shift = 16,
 927	.db_shift = 12,
 928	.as_shift = 8,
 929	.ss_shift = 4,
 930	.sm_shift = 0,
 931	.si_mask = 0xf,
 932	.db_mask = 0xf,
 933	.as_mask = 0xf,
 934	.ss_mask = 0xf,
 935	.sm_mask = 0xf,
 936};
 937
 938static const struct tegra_hsp_soc tegra194_hsp_soc = {
 939	.map = tegra186_hsp_db_map,
 940	.has_per_mb_ie = true,
 941	.has_128_bit_mb = false,
 942	.reg_stride = 0x100,
 943	.si_shift = 16,
 944	.db_shift = 12,
 945	.as_shift = 8,
 946	.ss_shift = 4,
 947	.sm_shift = 0,
 948	.si_mask = 0xf,
 949	.db_mask = 0xf,
 950	.as_mask = 0xf,
 951	.ss_mask = 0xf,
 952	.sm_mask = 0xf,
 953};
 954
 955static const struct tegra_hsp_soc tegra234_hsp_soc = {
 956	.map = tegra186_hsp_db_map,
 957	.has_per_mb_ie = false,
 958	.has_128_bit_mb = true,
 959	.reg_stride = 0x100,
 960	.si_shift = 16,
 961	.db_shift = 12,
 962	.as_shift = 8,
 963	.ss_shift = 4,
 964	.sm_shift = 0,
 965	.si_mask = 0xf,
 966	.db_mask = 0xf,
 967	.as_mask = 0xf,
 968	.ss_mask = 0xf,
 969	.sm_mask = 0xf,
 970};
 971
 972static const struct tegra_hsp_soc tegra264_hsp_soc = {
 973	.map = tegra186_hsp_db_map,
 974	.has_per_mb_ie = false,
 975	.has_128_bit_mb = true,
 976	.reg_stride = 0x1000,
 977	.si_shift = 17,
 978	.db_shift = 12,
 979	.as_shift = 8,
 980	.ss_shift = 4,
 981	.sm_shift = 0,
 982	.si_mask = 0x1f,
 983	.db_mask = 0x1f,
 984	.as_mask = 0xf,
 985	.ss_mask = 0xf,
 986	.sm_mask = 0xf,
 987};
 988
 989static const struct of_device_id tegra_hsp_match[] = {
 990	{ .compatible = "nvidia,tegra186-hsp", .data = &tegra186_hsp_soc },
 991	{ .compatible = "nvidia,tegra194-hsp", .data = &tegra194_hsp_soc },
 992	{ .compatible = "nvidia,tegra234-hsp", .data = &tegra234_hsp_soc },
 993	{ .compatible = "nvidia,tegra264-hsp", .data = &tegra264_hsp_soc },
 994	{ }
 995};
 996
 997static struct platform_driver tegra_hsp_driver = {
 998	.driver = {
 999		.name = "tegra-hsp",
1000		.of_match_table = tegra_hsp_match,
1001		.pm = &tegra_hsp_pm_ops,
1002	},
1003	.probe = tegra_hsp_probe,
1004	.remove = tegra_hsp_remove,
1005};
1006
1007static int __init tegra_hsp_init(void)
1008{
1009	return platform_driver_register(&tegra_hsp_driver);
1010}
1011core_initcall(tegra_hsp_init);