arm-smmu-v3.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * IOMMU API for ARM architected SMMUv3 implementations.
   4 *
   5 * Copyright (C) 2015 ARM Limited
   6 *
   7 * Author: Will Deacon <will.deacon@arm.com>
   8 *
   9 * This driver is powered by bad coffee and bombay mix.
  10 */
  11
  12#include <linux/acpi.h>
  13#include <linux/acpi_iort.h>
  14#include <linux/bitops.h>
  15#include <linux/crash_dump.h>
  16#include <linux/delay.h>
  17#include <linux/err.h>
  18#include <linux/interrupt.h>
  19#include <linux/io-pgtable.h>
  20#include <linux/iopoll.h>
  21#include <linux/module.h>
  22#include <linux/msi.h>
  23#include <linux/of.h>
  24#include <linux/of_address.h>
  25#include <linux/of_platform.h>
  26#include <linux/pci.h>
  27#include <linux/pci-ats.h>
  28#include <linux/platform_device.h>
  29#include <linux/string_choices.h>
  30#include <kunit/visibility.h>
  31#include <uapi/linux/iommufd.h>
  32
  33#include "arm-smmu-v3.h"
  34#include "../../dma-iommu.h"
  35
  36static bool disable_msipolling;
  37module_param(disable_msipolling, bool, 0444);
  38MODULE_PARM_DESC(disable_msipolling,
  39	"Disable MSI-based polling for CMD_SYNC completion.");
  40
  41static const struct iommu_ops arm_smmu_ops;
  42static struct iommu_dirty_ops arm_smmu_dirty_ops;
  43
  44enum arm_smmu_msi_index {
  45	EVTQ_MSI_INDEX,
  46	GERROR_MSI_INDEX,
  47	PRIQ_MSI_INDEX,
  48	ARM_SMMU_MAX_MSIS,
  49};
  50
  51#define NUM_ENTRY_QWORDS 8
  52static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64));
  53static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64));
  54
  55static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
  56	[EVTQ_MSI_INDEX] = {
  57		ARM_SMMU_EVTQ_IRQ_CFG0,
  58		ARM_SMMU_EVTQ_IRQ_CFG1,
  59		ARM_SMMU_EVTQ_IRQ_CFG2,
  60	},
  61	[GERROR_MSI_INDEX] = {
  62		ARM_SMMU_GERROR_IRQ_CFG0,
  63		ARM_SMMU_GERROR_IRQ_CFG1,
  64		ARM_SMMU_GERROR_IRQ_CFG2,
  65	},
  66	[PRIQ_MSI_INDEX] = {
  67		ARM_SMMU_PRIQ_IRQ_CFG0,
  68		ARM_SMMU_PRIQ_IRQ_CFG1,
  69		ARM_SMMU_PRIQ_IRQ_CFG2,
  70	},
  71};
  72
  73struct arm_smmu_option_prop {
  74	u32 opt;
  75	const char *prop;
  76};
  77
  78DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
  79DEFINE_MUTEX(arm_smmu_asid_lock);
  80
  81static struct arm_smmu_option_prop arm_smmu_options[] = {
  82	{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
  83	{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
  84	{ 0, NULL},
  85};
  86
  87static const char * const event_str[] = {
  88	[EVT_ID_BAD_STREAMID_CONFIG] = "C_BAD_STREAMID",
  89	[EVT_ID_STE_FETCH_FAULT] = "F_STE_FETCH",
  90	[EVT_ID_BAD_STE_CONFIG] = "C_BAD_STE",
  91	[EVT_ID_STREAM_DISABLED_FAULT] = "F_STREAM_DISABLED",
  92	[EVT_ID_BAD_SUBSTREAMID_CONFIG] = "C_BAD_SUBSTREAMID",
  93	[EVT_ID_CD_FETCH_FAULT] = "F_CD_FETCH",
  94	[EVT_ID_BAD_CD_CONFIG] = "C_BAD_CD",
  95	[EVT_ID_TRANSLATION_FAULT] = "F_TRANSLATION",
  96	[EVT_ID_ADDR_SIZE_FAULT] = "F_ADDR_SIZE",
  97	[EVT_ID_ACCESS_FAULT] = "F_ACCESS",
  98	[EVT_ID_PERMISSION_FAULT] = "F_PERMISSION",
  99	[EVT_ID_VMS_FETCH_FAULT] = "F_VMS_FETCH",
 100};
 101
 102static const char * const event_class_str[] = {
 103	[0] = "CD fetch",
 104	[1] = "Stage 1 translation table fetch",
 105	[2] = "Input address caused fault",
 106	[3] = "Reserved",
 107};
 108
 109static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
 110
 111static void parse_driver_options(struct arm_smmu_device *smmu)
 112{
 113	int i = 0;
 114
 115	do {
 116		if (of_property_read_bool(smmu->dev->of_node,
 117						arm_smmu_options[i].prop)) {
 118			smmu->options |= arm_smmu_options[i].opt;
 119			dev_notice(smmu->dev, "option %s\n",
 120				arm_smmu_options[i].prop);
 121		}
 122	} while (arm_smmu_options[++i].opt);
 123}
 124
 125/* Low-level queue manipulation functions */
 126static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
 127{
 128	u32 space, prod, cons;
 129
 130	prod = Q_IDX(q, q->prod);
 131	cons = Q_IDX(q, q->cons);
 132
 133	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
 134		space = (1 << q->max_n_shift) - (prod - cons);
 135	else
 136		space = cons - prod;
 137
 138	return space >= n;
 139}
 140
 141static bool queue_full(struct arm_smmu_ll_queue *q)
 142{
 143	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
 144	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
 145}
 146
 147static bool queue_empty(struct arm_smmu_ll_queue *q)
 148{
 149	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
 150	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
 151}
 152
 153static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
 154{
 155	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
 156		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
 157	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
 158		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
 159}
 160
 161static void queue_sync_cons_out(struct arm_smmu_queue *q)
 162{
 163	/*
 164	 * Ensure that all CPU accesses (reads and writes) to the queue
 165	 * are complete before we update the cons pointer.
 166	 */
 167	__iomb();
 168	writel_relaxed(q->llq.cons, q->cons_reg);
 169}
 170
 171static void queue_inc_cons(struct arm_smmu_ll_queue *q)
 172{
 173	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
 174	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
 175}
 176
 177static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
 178{
 179	struct arm_smmu_ll_queue *llq = &q->llq;
 180
 181	if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
 182		return;
 183
 184	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
 185		      Q_IDX(llq, llq->cons);
 186	queue_sync_cons_out(q);
 187}
 188
 189static int queue_sync_prod_in(struct arm_smmu_queue *q)
 190{
 191	u32 prod;
 192	int ret = 0;
 193
 194	/*
 195	 * We can't use the _relaxed() variant here, as we must prevent
 196	 * speculative reads of the queue before we have determined that
 197	 * prod has indeed moved.
 198	 */
 199	prod = readl(q->prod_reg);
 200
 201	if (Q_OVF(prod) != Q_OVF(q->llq.prod))
 202		ret = -EOVERFLOW;
 203
 204	q->llq.prod = prod;
 205	return ret;
 206}
 207
 208static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
 209{
 210	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
 211	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
 212}
 213
 214static void queue_poll_init(struct arm_smmu_device *smmu,
 215			    struct arm_smmu_queue_poll *qp)
 216{
 217	qp->delay = 1;
 218	qp->spin_cnt = 0;
 219	qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
 220	qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
 221}
 222
 223static int queue_poll(struct arm_smmu_queue_poll *qp)
 224{
 225	if (ktime_compare(ktime_get(), qp->timeout) > 0)
 226		return -ETIMEDOUT;
 227
 228	if (qp->wfe) {
 229		wfe();
 230	} else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
 231		cpu_relax();
 232	} else {
 233		udelay(qp->delay);
 234		qp->delay *= 2;
 235		qp->spin_cnt = 0;
 236	}
 237
 238	return 0;
 239}
 240
 241static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
 242{
 243	int i;
 244
 245	for (i = 0; i < n_dwords; ++i)
 246		*dst++ = cpu_to_le64(*src++);
 247}
 248
 249static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
 250{
 251	int i;
 252
 253	for (i = 0; i < n_dwords; ++i)
 254		*dst++ = le64_to_cpu(*src++);
 255}
 256
 257static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
 258{
 259	if (queue_empty(&q->llq))
 260		return -EAGAIN;
 261
 262	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
 263	queue_inc_cons(&q->llq);
 264	queue_sync_cons_out(q);
 265	return 0;
 266}
 267
 268/* High-level queue accessors */
 269static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
 270{
 271	memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
 272	cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
 273
 274	switch (ent->opcode) {
 275	case CMDQ_OP_TLBI_EL2_ALL:
 276	case CMDQ_OP_TLBI_NSNH_ALL:
 277		break;
 278	case CMDQ_OP_PREFETCH_CFG:
 279		cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
 280		break;
 281	case CMDQ_OP_CFGI_CD:
 282		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
 283		fallthrough;
 284	case CMDQ_OP_CFGI_STE:
 285		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
 286		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
 287		break;
 288	case CMDQ_OP_CFGI_CD_ALL:
 289		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
 290		break;
 291	case CMDQ_OP_CFGI_ALL:
 292		/* Cover the entire SID range */
 293		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
 294		break;
 295	case CMDQ_OP_TLBI_NH_VA:
 296		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 297		fallthrough;
 298	case CMDQ_OP_TLBI_EL2_VA:
 299		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
 300		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
 301		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 302		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
 303		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
 304		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
 305		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
 306		break;
 307	case CMDQ_OP_TLBI_S2_IPA:
 308		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
 309		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
 310		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 311		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
 312		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
 313		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
 314		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
 315		break;
 316	case CMDQ_OP_TLBI_NH_ASID:
 317		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 318		fallthrough;
 319	case CMDQ_OP_TLBI_NH_ALL:
 320	case CMDQ_OP_TLBI_S12_VMALL:
 321		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 322		break;
 323	case CMDQ_OP_TLBI_EL2_ASID:
 324		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 325		break;
 326	case CMDQ_OP_ATC_INV:
 327		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
 328		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
 329		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
 330		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
 331		cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
 332		cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
 333		break;
 334	case CMDQ_OP_PRI_RESP:
 335		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
 336		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
 337		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
 338		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
 339		switch (ent->pri.resp) {
 340		case PRI_RESP_DENY:
 341		case PRI_RESP_FAIL:
 342		case PRI_RESP_SUCC:
 343			break;
 344		default:
 345			return -EINVAL;
 346		}
 347		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
 348		break;
 349	case CMDQ_OP_RESUME:
 350		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
 351		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
 352		cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
 353		break;
 354	case CMDQ_OP_CMD_SYNC:
 355		if (ent->sync.msiaddr) {
 356			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
 357			cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
 358		} else {
 359			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
 360		}
 361		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
 362		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
 363		break;
 364	default:
 365		return -ENOENT;
 366	}
 367
 368	return 0;
 369}
 370
 371static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
 372					       struct arm_smmu_cmdq_ent *ent)
 373{
 374	struct arm_smmu_cmdq *cmdq = NULL;
 375
 376	if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
 377		cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
 378
 379	return cmdq ?: &smmu->cmdq;
 380}
 381
 382static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
 383					     struct arm_smmu_cmdq *cmdq)
 384{
 385	if (cmdq == &smmu->cmdq)
 386		return false;
 387
 388	return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
 389}
 390
 391static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
 392					 struct arm_smmu_cmdq *cmdq, u32 prod)
 393{
 394	struct arm_smmu_queue *q = &cmdq->q;
 395	struct arm_smmu_cmdq_ent ent = {
 396		.opcode = CMDQ_OP_CMD_SYNC,
 397	};
 398
 399	/*
 400	 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
 401	 * payload, so the write will zero the entire command on that platform.
 402	 */
 403	if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
 404		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
 405				   q->ent_dwords * 8;
 406	}
 407
 408	arm_smmu_cmdq_build_cmd(cmd, &ent);
 409	if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
 410		u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
 411}
 412
 413void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
 414			      struct arm_smmu_cmdq *cmdq)
 415{
 416	static const char * const cerror_str[] = {
 417		[CMDQ_ERR_CERROR_NONE_IDX]	= "No error",
 418		[CMDQ_ERR_CERROR_ILL_IDX]	= "Illegal command",
 419		[CMDQ_ERR_CERROR_ABT_IDX]	= "Abort on command fetch",
 420		[CMDQ_ERR_CERROR_ATC_INV_IDX]	= "ATC invalidate timeout",
 421	};
 422	struct arm_smmu_queue *q = &cmdq->q;
 423
 424	int i;
 425	u64 cmd[CMDQ_ENT_DWORDS];
 426	u32 cons = readl_relaxed(q->cons_reg);
 427	u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
 428	struct arm_smmu_cmdq_ent cmd_sync = {
 429		.opcode = CMDQ_OP_CMD_SYNC,
 430	};
 431
 432	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
 433		idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
 434
 435	switch (idx) {
 436	case CMDQ_ERR_CERROR_ABT_IDX:
 437		dev_err(smmu->dev, "retrying command fetch\n");
 438		return;
 439	case CMDQ_ERR_CERROR_NONE_IDX:
 440		return;
 441	case CMDQ_ERR_CERROR_ATC_INV_IDX:
 442		/*
 443		 * ATC Invalidation Completion timeout. CONS is still pointing
 444		 * at the CMD_SYNC. Attempt to complete other pending commands
 445		 * by repeating the CMD_SYNC, though we might well end up back
 446		 * here since the ATC invalidation may still be pending.
 447		 */
 448		return;
 449	case CMDQ_ERR_CERROR_ILL_IDX:
 450	default:
 451		break;
 452	}
 453
 454	/*
 455	 * We may have concurrent producers, so we need to be careful
 456	 * not to touch any of the shadow cmdq state.
 457	 */
 458	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
 459	dev_err(smmu->dev, "skipping command in error state:\n");
 460	for (i = 0; i < ARRAY_SIZE(cmd); ++i)
 461		dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
 462
 463	/* Convert the erroneous command into a CMD_SYNC */
 464	arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
 465	if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
 466		u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
 467
 468	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
 469}
 470
 471static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
 472{
 473	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq);
 474}
 475
 476/*
 477 * Command queue locking.
 478 * This is a form of bastardised rwlock with the following major changes:
 479 *
 480 * - The only LOCK routines are exclusive_trylock() and shared_lock().
 481 *   Neither have barrier semantics, and instead provide only a control
 482 *   dependency.
 483 *
 484 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
 485 *   fails if the caller appears to be the last lock holder (yes, this is
 486 *   racy). All successful UNLOCK routines have RELEASE semantics.
 487 */
 488static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
 489{
 490	/*
 491	 * When held in exclusive state, the lock counter is set to INT_MIN
 492	 * so these increments won't hurt as the value will remain negative.
 493	 * The increment will also signal the exclusive locker that there are
 494	 * shared waiters.
 495	 */
 496	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
 497		return;
 498
 499	/*
 500	 * Someone else is holding the lock in exclusive state, so wait
 501	 * for them to finish. Since we already incremented the lock counter,
 502	 * no exclusive lock can be acquired until we finish. We don't need
 503	 * the return value since we only care that the exclusive lock is
 504	 * released (i.e. the lock counter is non-negative).
 505	 * Once the exclusive locker releases the lock, the sign bit will
 506	 * be cleared and our increment will make the lock counter positive,
 507	 * allowing us to proceed.
 508	 */
 509	atomic_cond_read_relaxed(&cmdq->lock, VAL > 0);
 510}
 511
 512static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
 513{
 514	(void)atomic_dec_return_release(&cmdq->lock);
 515}
 516
 517static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
 518{
 519	if (atomic_read(&cmdq->lock) == 1)
 520		return false;
 521
 522	arm_smmu_cmdq_shared_unlock(cmdq);
 523	return true;
 524}
 525
 526#define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)		\
 527({									\
 528	bool __ret;							\
 529	local_irq_save(flags);						\
 530	__ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);	\
 531	if (!__ret)							\
 532		local_irq_restore(flags);				\
 533	__ret;								\
 534})
 535
 536/*
 537 * Only clear the sign bit when releasing the exclusive lock this will
 538 * allow any shared_lock() waiters to proceed without the possibility
 539 * of entering the exclusive lock in a tight loop.
 540 */
 541#define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)		\
 542({									\
 543	atomic_fetch_andnot_release(INT_MIN, &cmdq->lock);		\
 544	local_irq_restore(flags);					\
 545})
 546
 547
 548/*
 549 * Command queue insertion.
 550 * This is made fiddly by our attempts to achieve some sort of scalability
 551 * since there is one queue shared amongst all of the CPUs in the system.  If
 552 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
 553 * then you'll *love* this monstrosity.
 554 *
 555 * The basic idea is to split the queue up into ranges of commands that are
 556 * owned by a given CPU; the owner may not have written all of the commands
 557 * itself, but is responsible for advancing the hardware prod pointer when
 558 * the time comes. The algorithm is roughly:
 559 *
 560 * 	1. Allocate some space in the queue. At this point we also discover
 561 *	   whether the head of the queue is currently owned by another CPU,
 562 *	   or whether we are the owner.
 563 *
 564 *	2. Write our commands into our allocated slots in the queue.
 565 *
 566 *	3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
 567 *
 568 *	4. If we are an owner:
 569 *		a. Wait for the previous owner to finish.
 570 *		b. Mark the queue head as unowned, which tells us the range
 571 *		   that we are responsible for publishing.
 572 *		c. Wait for all commands in our owned range to become valid.
 573 *		d. Advance the hardware prod pointer.
 574 *		e. Tell the next owner we've finished.
 575 *
 576 *	5. If we are inserting a CMD_SYNC (we may or may not have been an
 577 *	   owner), then we need to stick around until it has completed:
 578 *		a. If we have MSIs, the SMMU can write back into the CMD_SYNC
 579 *		   to clear the first 4 bytes.
 580 *		b. Otherwise, we spin waiting for the hardware cons pointer to
 581 *		   advance past our command.
 582 *
 583 * The devil is in the details, particularly the use of locking for handling
 584 * SYNC completion and freeing up space in the queue before we think that it is
 585 * full.
 586 */
 587static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
 588					       u32 sprod, u32 eprod, bool set)
 589{
 590	u32 swidx, sbidx, ewidx, ebidx;
 591	struct arm_smmu_ll_queue llq = {
 592		.max_n_shift	= cmdq->q.llq.max_n_shift,
 593		.prod		= sprod,
 594	};
 595
 596	ewidx = BIT_WORD(Q_IDX(&llq, eprod));
 597	ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
 598
 599	while (llq.prod != eprod) {
 600		unsigned long mask;
 601		atomic_long_t *ptr;
 602		u32 limit = BITS_PER_LONG;
 603
 604		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
 605		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
 606
 607		ptr = &cmdq->valid_map[swidx];
 608
 609		if ((swidx == ewidx) && (sbidx < ebidx))
 610			limit = ebidx;
 611
 612		mask = GENMASK(limit - 1, sbidx);
 613
 614		/*
 615		 * The valid bit is the inverse of the wrap bit. This means
 616		 * that a zero-initialised queue is invalid and, after marking
 617		 * all entries as valid, they become invalid again when we
 618		 * wrap.
 619		 */
 620		if (set) {
 621			atomic_long_xor(mask, ptr);
 622		} else { /* Poll */
 623			unsigned long valid;
 624
 625			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
 626			atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
 627		}
 628
 629		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
 630	}
 631}
 632
 633/* Mark all entries in the range [sprod, eprod) as valid */
 634static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
 635					u32 sprod, u32 eprod)
 636{
 637	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
 638}
 639
 640/* Wait for all entries in the range [sprod, eprod) to become valid */
 641static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
 642					 u32 sprod, u32 eprod)
 643{
 644	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
 645}
 646
 647/* Wait for the command queue to become non-full */
 648static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
 649					     struct arm_smmu_cmdq *cmdq,
 650					     struct arm_smmu_ll_queue *llq)
 651{
 652	unsigned long flags;
 653	struct arm_smmu_queue_poll qp;
 654	int ret = 0;
 655
 656	/*
 657	 * Try to update our copy of cons by grabbing exclusive cmdq access. If
 658	 * that fails, spin until somebody else updates it for us.
 659	 */
 660	if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
 661		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
 662		arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
 663		llq->val = READ_ONCE(cmdq->q.llq.val);
 664		return 0;
 665	}
 666
 667	queue_poll_init(smmu, &qp);
 668	do {
 669		llq->val = READ_ONCE(cmdq->q.llq.val);
 670		if (!queue_full(llq))
 671			break;
 672
 673		ret = queue_poll(&qp);
 674	} while (!ret);
 675
 676	return ret;
 677}
 678
 679/*
 680 * Wait until the SMMU signals a CMD_SYNC completion MSI.
 681 * Must be called with the cmdq lock held in some capacity.
 682 */
 683static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
 684					  struct arm_smmu_cmdq *cmdq,
 685					  struct arm_smmu_ll_queue *llq)
 686{
 687	int ret = 0;
 688	struct arm_smmu_queue_poll qp;
 689	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
 690
 691	queue_poll_init(smmu, &qp);
 692
 693	/*
 694	 * The MSI won't generate an event, since it's being written back
 695	 * into the command queue.
 696	 */
 697	qp.wfe = false;
 698	smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
 699	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
 700	return ret;
 701}
 702
 703/*
 704 * Wait until the SMMU cons index passes llq->prod.
 705 * Must be called with the cmdq lock held in some capacity.
 706 */
 707static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
 708					       struct arm_smmu_cmdq *cmdq,
 709					       struct arm_smmu_ll_queue *llq)
 710{
 711	struct arm_smmu_queue_poll qp;
 712	u32 prod = llq->prod;
 713	int ret = 0;
 714
 715	queue_poll_init(smmu, &qp);
 716	llq->val = READ_ONCE(cmdq->q.llq.val);
 717	do {
 718		if (queue_consumed(llq, prod))
 719			break;
 720
 721		ret = queue_poll(&qp);
 722
 723		/*
 724		 * This needs to be a readl() so that our subsequent call
 725		 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
 726		 *
 727		 * Specifically, we need to ensure that we observe all
 728		 * shared_lock()s by other CMD_SYNCs that share our owner,
 729		 * so that a failing call to tryunlock() means that we're
 730		 * the last one out and therefore we can safely advance
 731		 * cmdq->q.llq.cons. Roughly speaking:
 732		 *
 733		 * CPU 0		CPU1			CPU2 (us)
 734		 *
 735		 * if (sync)
 736		 * 	shared_lock();
 737		 *
 738		 * dma_wmb();
 739		 * set_valid_map();
 740		 *
 741		 * 			if (owner) {
 742		 *				poll_valid_map();
 743		 *				<control dependency>
 744		 *				writel(prod_reg);
 745		 *
 746		 *						readl(cons_reg);
 747		 *						tryunlock();
 748		 *
 749		 * Requires us to see CPU 0's shared_lock() acquisition.
 750		 */
 751		llq->cons = readl(cmdq->q.cons_reg);
 752	} while (!ret);
 753
 754	return ret;
 755}
 756
 757static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
 758					 struct arm_smmu_cmdq *cmdq,
 759					 struct arm_smmu_ll_queue *llq)
 760{
 761	if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
 762	    !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
 763		return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
 764
 765	return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
 766}
 767
 768static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
 769					u32 prod, int n)
 770{
 771	int i;
 772	struct arm_smmu_ll_queue llq = {
 773		.max_n_shift	= cmdq->q.llq.max_n_shift,
 774		.prod		= prod,
 775	};
 776
 777	for (i = 0; i < n; ++i) {
 778		u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
 779
 780		prod = queue_inc_prod_n(&llq, i);
 781		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
 782	}
 783}
 784
 785/*
 786 * This is the actual insertion function, and provides the following
 787 * ordering guarantees to callers:
 788 *
 789 * - There is a dma_wmb() before publishing any commands to the queue.
 790 *   This can be relied upon to order prior writes to data structures
 791 *   in memory (such as a CD or an STE) before the command.
 792 *
 793 * - On completion of a CMD_SYNC, there is a control dependency.
 794 *   This can be relied upon to order subsequent writes to memory (e.g.
 795 *   freeing an IOVA) after completion of the CMD_SYNC.
 796 *
 797 * - Command insertion is totally ordered, so if two CPUs each race to
 798 *   insert their own list of commands then all of the commands from one
 799 *   CPU will appear before any of the commands from the other CPU.
 800 */
 801int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
 802				struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
 803				bool sync)
 804{
 805	u64 cmd_sync[CMDQ_ENT_DWORDS];
 806	u32 prod;
 807	unsigned long flags;
 808	bool owner;
 809	struct arm_smmu_ll_queue llq, head;
 810	int ret = 0;
 811
 812	llq.max_n_shift = cmdq->q.llq.max_n_shift;
 813
 814	/* 1. Allocate some space in the queue */
 815	local_irq_save(flags);
 816	llq.val = READ_ONCE(cmdq->q.llq.val);
 817	do {
 818		u64 old;
 819
 820		while (!queue_has_space(&llq, n + sync)) {
 821			local_irq_restore(flags);
 822			if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq))
 823				dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
 824			local_irq_save(flags);
 825		}
 826
 827		head.cons = llq.cons;
 828		head.prod = queue_inc_prod_n(&llq, n + sync) |
 829					     CMDQ_PROD_OWNED_FLAG;
 830
 831		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
 832		if (old == llq.val)
 833			break;
 834
 835		llq.val = old;
 836	} while (1);
 837	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
 838	head.prod &= ~CMDQ_PROD_OWNED_FLAG;
 839	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
 840
 841	/*
 842	 * 2. Write our commands into the queue
 843	 * Dependency ordering from the cmpxchg() loop above.
 844	 */
 845	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
 846	if (sync) {
 847		prod = queue_inc_prod_n(&llq, n);
 848		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
 849		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
 850
 851		/*
 852		 * In order to determine completion of our CMD_SYNC, we must
 853		 * ensure that the queue can't wrap twice without us noticing.
 854		 * We achieve that by taking the cmdq lock as shared before
 855		 * marking our slot as valid.
 856		 */
 857		arm_smmu_cmdq_shared_lock(cmdq);
 858	}
 859
 860	/* 3. Mark our slots as valid, ensuring commands are visible first */
 861	dma_wmb();
 862	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
 863
 864	/* 4. If we are the owner, take control of the SMMU hardware */
 865	if (owner) {
 866		/* a. Wait for previous owner to finish */
 867		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
 868
 869		/* b. Stop gathering work by clearing the owned flag */
 870		prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
 871						   &cmdq->q.llq.atomic.prod);
 872		prod &= ~CMDQ_PROD_OWNED_FLAG;
 873
 874		/*
 875		 * c. Wait for any gathered work to be written to the queue.
 876		 * Note that we read our own entries so that we have the control
 877		 * dependency required by (d).
 878		 */
 879		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
 880
 881		/*
 882		 * d. Advance the hardware prod pointer
 883		 * Control dependency ordering from the entries becoming valid.
 884		 */
 885		writel_relaxed(prod, cmdq->q.prod_reg);
 886
 887		/*
 888		 * e. Tell the next owner we're done
 889		 * Make sure we've updated the hardware first, so that we don't
 890		 * race to update prod and potentially move it backwards.
 891		 */
 892		atomic_set_release(&cmdq->owner_prod, prod);
 893	}
 894
 895	/* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
 896	if (sync) {
 897		llq.prod = queue_inc_prod_n(&llq, n);
 898		ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq);
 899		if (ret) {
 900			dev_err_ratelimited(smmu->dev,
 901					    "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
 902					    llq.prod,
 903					    readl_relaxed(cmdq->q.prod_reg),
 904					    readl_relaxed(cmdq->q.cons_reg));
 905		}
 906
 907		/*
 908		 * Try to unlock the cmdq lock. This will fail if we're the last
 909		 * reader, in which case we can safely update cmdq->q.llq.cons
 910		 */
 911		if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
 912			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
 913			arm_smmu_cmdq_shared_unlock(cmdq);
 914		}
 915	}
 916
 917	local_irq_restore(flags);
 918	return ret;
 919}
 920
 921static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
 922				     struct arm_smmu_cmdq_ent *ent,
 923				     bool sync)
 924{
 925	u64 cmd[CMDQ_ENT_DWORDS];
 926
 927	if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
 928		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
 929			 ent->opcode);
 930		return -EINVAL;
 931	}
 932
 933	return arm_smmu_cmdq_issue_cmdlist(
 934		smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
 935}
 936
 937static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
 938				   struct arm_smmu_cmdq_ent *ent)
 939{
 940	return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
 941}
 942
 943static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
 944					     struct arm_smmu_cmdq_ent *ent)
 945{
 946	return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
 947}
 948
 949static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
 950				     struct arm_smmu_cmdq_batch *cmds,
 951				     struct arm_smmu_cmdq_ent *ent)
 952{
 953	cmds->num = 0;
 954	cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
 955}
 956
 957static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
 958				    struct arm_smmu_cmdq_batch *cmds,
 959				    struct arm_smmu_cmdq_ent *cmd)
 960{
 961	bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
 962	bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
 963			  (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
 964	int index;
 965
 966	if (force_sync || unsupported_cmd) {
 967		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
 968					    cmds->num, true);
 969		arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
 970	}
 971
 972	if (cmds->num == CMDQ_BATCH_ENTRIES) {
 973		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
 974					    cmds->num, false);
 975		arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
 976	}
 977
 978	index = cmds->num * CMDQ_ENT_DWORDS;
 979	if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
 980		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
 981			 cmd->opcode);
 982		return;
 983	}
 984
 985	cmds->num++;
 986}
 987
 988static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
 989				      struct arm_smmu_cmdq_batch *cmds)
 990{
 991	return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
 992					   cmds->num, true);
 993}
 994
 995static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
 996				   struct iommu_page_response *resp)
 997{
 998	struct arm_smmu_cmdq_ent cmd = {0};
 999	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
1000	int sid = master->streams[0].id;
1001
1002	if (WARN_ON(!master->stall_enabled))
1003		return;
1004
1005	cmd.opcode		= CMDQ_OP_RESUME;
1006	cmd.resume.sid		= sid;
1007	cmd.resume.stag		= resp->grpid;
1008	switch (resp->code) {
1009	case IOMMU_PAGE_RESP_INVALID:
1010	case IOMMU_PAGE_RESP_FAILURE:
1011		cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
1012		break;
1013	case IOMMU_PAGE_RESP_SUCCESS:
1014		cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
1015		break;
1016	default:
1017		break;
1018	}
1019
1020	arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
1021	/*
1022	 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
1023	 * RESUME consumption guarantees that the stalled transaction will be
1024	 * terminated... at some point in the future. PRI_RESP is fire and
1025	 * forget.
1026	 */
1027}
1028
1029/* Context descriptor manipulation functions */
1030void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
1031{
1032	struct arm_smmu_cmdq_ent cmd = {
1033		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?
1034			CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
1035		.tlbi.asid = asid,
1036	};
1037
1038	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1039}
1040
1041/*
1042 * Based on the value of ent report which bits of the STE the HW will access. It
1043 * would be nice if this was complete according to the spec, but minimally it
1044 * has to capture the bits this driver uses.
1045 */
1046VISIBLE_IF_KUNIT
1047void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
1048{
1049	unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0]));
1050
1051	used_bits[0] = cpu_to_le64(STRTAB_STE_0_V);
1052	if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V)))
1053		return;
1054
1055	used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG);
1056
1057	/* S1 translates */
1058	if (cfg & BIT(0)) {
1059		used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT |
1060					    STRTAB_STE_0_S1CTXPTR_MASK |
1061					    STRTAB_STE_0_S1CDMAX);
1062		used_bits[1] |=
1063			cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
1064				    STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
1065				    STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
1066				    STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
1067		used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
1068
1069		/*
1070		 * See 13.5 Summary of attribute/permission configuration fields
1071		 * for the SHCFG behavior.
1072		 */
1073		if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) ==
1074		    STRTAB_STE_1_S1DSS_BYPASS)
1075			used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1076	}
1077
1078	/* S2 translates */
1079	if (cfg & BIT(1)) {
1080		used_bits[1] |=
1081			cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
1082				    STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
1083		used_bits[2] |=
1084			cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
1085				    STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
1086				    STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2S |
1087				    STRTAB_STE_2_S2R);
1088		used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK);
1089	}
1090
1091	if (cfg == STRTAB_STE_0_CFG_BYPASS)
1092		used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1093}
1094EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
1095
1096VISIBLE_IF_KUNIT
1097void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
1098				  __le64 *safe_bits)
1099{
1100	const __le64 eats_s1chk =
1101		FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
1102	const __le64 eats_trans =
1103		FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
1104
1105	/*
1106	 * When an STE changes EATS_TRANS, the sequencing code in the attach
1107	 * logic already will have the PCI cap for ATS disabled. Thus at this
1108	 * moment we can expect that the device will not generate ATS queries
1109	 * and so we don't care about the sequencing of EATS. The purpose of
1110	 * EATS_TRANS is to protect the system from hostile untrusted devices
1111	 * that issue ATS when the PCI config space is disabled. However, if
1112	 * EATS_TRANS is being changed, then we must have already trusted the
1113	 * device as the EATS_TRANS security block is being disabled.
1114	 *
1115	 *  Note: now the EATS_TRANS update is moved to the first entry_set().
1116	 *  Changing S2S and EATS might transiently result in S2S=1 and EATS=1
1117	 *  which is a bad STE (see "5.2 Stream Table Entry"). In such a case,
1118	 *  we can't do a hitless update. Also, it should not be added to the
1119	 *  safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be
1120	 *  effectively an errant 0b00 configuration.
1121	 */
1122	if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) &&
1123	    !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S)))
1124		safe_bits[1] |= cpu_to_le64(eats_trans);
1125
1126	/*
1127	 * MEV does not meaningfully impact the operation of the HW, it only
1128	 * changes how many fault events are generated, thus we can relax it
1129	 * when computing the ordering. The spec notes the device can act like
1130	 * MEV=1 anyhow:
1131	 *
1132	 *  Note: Software must expect, and be able to deal with, coalesced
1133	 *  fault records even when MEV == 0.
1134	 */
1135	safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
1136}
1137EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe);
1138
1139/*
1140 * Figure out if we can do a hitless update of entry to become target. Returns a
1141 * bit mask where 1 indicates that qword needs to be set disruptively.
1142 * unused_update is an intermediate value of entry that has unused bits set to
1143 * their new values.
1144 */
1145static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer,
1146				    const __le64 *entry, const __le64 *target,
1147				    __le64 *unused_update)
1148{
1149	__le64 target_used[NUM_ENTRY_QWORDS] = {};
1150	__le64 cur_used[NUM_ENTRY_QWORDS] = {};
1151	__le64 safe[NUM_ENTRY_QWORDS] = {};
1152	u8 used_qword_diff = 0;
1153	unsigned int i;
1154
1155	writer->ops->get_used(entry, cur_used);
1156	writer->ops->get_used(target, target_used);
1157	if (writer->ops->get_update_safe)
1158		writer->ops->get_update_safe(entry, target, safe);
1159
1160	for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
1161		/*
1162		 * Safe is only used for bits that are used by both entries,
1163		 * otherwise it is sequenced according to the unused entry.
1164		 */
1165		safe[i] &= target_used[i] & cur_used[i];
1166
1167		/*
1168		 * Check that masks are up to date, the make functions are not
1169		 * allowed to set a bit to 1 if the used function doesn't say it
1170		 * is used.
1171		 */
1172		WARN_ON_ONCE(target[i] & ~target_used[i]);
1173
1174		/* Bits can change because they are not currently being used */
1175		cur_used[i] &= ~safe[i];
1176		unused_update[i] = (entry[i] & cur_used[i]) |
1177				   (target[i] & ~cur_used[i]);
1178		/*
1179		 * Each bit indicates that a used bit in a qword needs to be
1180		 * changed after unused_update is applied.
1181		 */
1182		if ((unused_update[i] & target_used[i]) != target[i])
1183			used_qword_diff |= 1 << i;
1184	}
1185	return used_qword_diff;
1186}
1187
1188static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
1189		      const __le64 *target, unsigned int start,
1190		      unsigned int len)
1191{
1192	bool changed = false;
1193	unsigned int i;
1194
1195	for (i = start; len != 0; len--, i++) {
1196		if (entry[i] != target[i]) {
1197			WRITE_ONCE(entry[i], target[i]);
1198			changed = true;
1199		}
1200	}
1201
1202	if (changed)
1203		writer->ops->sync(writer);
1204}
1205
1206/*
1207 * Update the STE/CD to the target configuration. The transition from the
1208 * current entry to the target entry takes place over multiple steps that
1209 * attempts to make the transition hitless if possible. This function takes care
1210 * not to create a situation where the HW can perceive a corrupted entry. HW is
1211 * only required to have a 64 bit atomicity with stores from the CPU, while
1212 * entries are many 64 bit values big.
1213 *
1214 * The difference between the current value and the target value is analyzed to
1215 * determine which of three updates are required - disruptive, hitless or no
1216 * change.
1217 *
1218 * In the most general disruptive case we can make any update in three steps:
1219 *  - Disrupting the entry (V=0)
1220 *  - Fill now unused qwords, execpt qword 0 which contains V
1221 *  - Make qword 0 have the final value and valid (V=1) with a single 64
1222 *    bit store
1223 *
1224 * However this disrupts the HW while it is happening. There are several
1225 * interesting cases where a STE/CD can be updated without disturbing the HW
1226 * because only a small number of bits are changing (S1DSS, CONFIG, etc) or
1227 * because the used bits don't intersect. We can detect this by calculating how
1228 * many 64 bit values need update after adjusting the unused bits and skip the
1229 * V=0 process. This relies on the IGNORED behavior described in the
1230 * specification.
1231 */
1232VISIBLE_IF_KUNIT
1233void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
1234			  const __le64 *target)
1235{
1236	__le64 unused_update[NUM_ENTRY_QWORDS];
1237	u8 used_qword_diff;
1238
1239	used_qword_diff =
1240		arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
1241	if (hweight8(used_qword_diff) == 1) {
1242		/*
1243		 * Only one qword needs its used bits to be changed. This is a
1244		 * hitless update, update all bits the current STE/CD is
1245		 * ignoring to their new values, then update a single "critical
1246		 * qword" to change the STE/CD and finally 0 out any bits that
1247		 * are now unused in the target configuration.
1248		 */
1249		unsigned int critical_qword_index = ffs(used_qword_diff) - 1;
1250
1251		/*
1252		 * Skip writing unused bits in the critical qword since we'll be
1253		 * writing it in the next step anyways. This can save a sync
1254		 * when the only change is in that qword.
1255		 */
1256		unused_update[critical_qword_index] =
1257			entry[critical_qword_index];
1258		entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS);
1259		entry_set(writer, entry, target, critical_qword_index, 1);
1260		entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1261	} else if (used_qword_diff) {
1262		/*
1263		 * At least two qwords need their inuse bits to be changed. This
1264		 * requires a breaking update, zero the V bit, write all qwords
1265		 * but 0, then set qword 0
1266		 */
1267		unused_update[0] = 0;
1268		entry_set(writer, entry, unused_update, 0, 1);
1269		entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1);
1270		entry_set(writer, entry, target, 0, 1);
1271	} else {
1272		/*
1273		 * No inuse bit changed, though safe bits may have changed.
1274		 */
1275		entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1276	}
1277}
1278EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
1279
1280static void arm_smmu_sync_cd(struct arm_smmu_master *master,
1281			     int ssid, bool leaf)
1282{
1283	size_t i;
1284	struct arm_smmu_cmdq_batch cmds;
1285	struct arm_smmu_device *smmu = master->smmu;
1286	struct arm_smmu_cmdq_ent cmd = {
1287		.opcode	= CMDQ_OP_CFGI_CD,
1288		.cfgi	= {
1289			.ssid	= ssid,
1290			.leaf	= leaf,
1291		},
1292	};
1293
1294	arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
1295	for (i = 0; i < master->num_streams; i++) {
1296		cmd.cfgi.sid = master->streams[i].id;
1297		arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
1298	}
1299
1300	arm_smmu_cmdq_batch_submit(smmu, &cmds);
1301}
1302
1303static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
1304				      dma_addr_t l2ptr_dma)
1305{
1306	u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
1307
1308	/* The HW has 64 bit atomicity with stores to the L2 CD table */
1309	WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1310}
1311
1312static dma_addr_t arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 *src)
1313{
1314	return le64_to_cpu(src->l2ptr) & CTXDESC_L1_DESC_L2PTR_MASK;
1315}
1316
1317struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
1318					u32 ssid)
1319{
1320	struct arm_smmu_cdtab_l2 *l2;
1321	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1322
1323	if (!arm_smmu_cdtab_allocated(cd_table))
1324		return NULL;
1325
1326	if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1327		return &cd_table->linear.table[ssid];
1328
1329	l2 = cd_table->l2.l2ptrs[arm_smmu_cdtab_l1_idx(ssid)];
1330	if (!l2)
1331		return NULL;
1332	return &l2->cds[arm_smmu_cdtab_l2_idx(ssid)];
1333}
1334
1335static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
1336						 u32 ssid)
1337{
1338	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1339	struct arm_smmu_device *smmu = master->smmu;
1340
1341	might_sleep();
1342	iommu_group_mutex_assert(master->dev);
1343
1344	if (!arm_smmu_cdtab_allocated(cd_table)) {
1345		if (arm_smmu_alloc_cd_tables(master))
1346			return NULL;
1347	}
1348
1349	if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) {
1350		unsigned int idx = arm_smmu_cdtab_l1_idx(ssid);
1351		struct arm_smmu_cdtab_l2 **l2ptr = &cd_table->l2.l2ptrs[idx];
1352
1353		if (!*l2ptr) {
1354			dma_addr_t l2ptr_dma;
1355
1356			*l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr),
1357						    &l2ptr_dma, GFP_KERNEL);
1358			if (!*l2ptr)
1359				return NULL;
1360
1361			arm_smmu_write_cd_l1_desc(&cd_table->l2.l1tab[idx],
1362						  l2ptr_dma);
1363			/* An invalid L1CD can be cached */
1364			arm_smmu_sync_cd(master, ssid, false);
1365		}
1366	}
1367	return arm_smmu_get_cd_ptr(master, ssid);
1368}
1369
1370struct arm_smmu_cd_writer {
1371	struct arm_smmu_entry_writer writer;
1372	unsigned int ssid;
1373};
1374
1375VISIBLE_IF_KUNIT
1376void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits)
1377{
1378	used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V);
1379	if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V)))
1380		return;
1381	memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd));
1382
1383	/*
1384	 * If EPD0 is set by the make function it means
1385	 * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED
1386	 */
1387	if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) {
1388		used_bits[0] &= ~cpu_to_le64(
1389			CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 |
1390			CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 |
1391			CTXDESC_CD_0_TCR_SH0);
1392		used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK);
1393	}
1394}
1395EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used);
1396
1397static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1398{
1399	struct arm_smmu_cd_writer *cd_writer =
1400		container_of(writer, struct arm_smmu_cd_writer, writer);
1401
1402	arm_smmu_sync_cd(writer->master, cd_writer->ssid, true);
1403}
1404
1405static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = {
1406	.sync = arm_smmu_cd_writer_sync_entry,
1407	.get_used = arm_smmu_get_cd_used,
1408};
1409
1410void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
1411			     struct arm_smmu_cd *cdptr,
1412			     const struct arm_smmu_cd *target)
1413{
1414	bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1415	bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1416	struct arm_smmu_cd_writer cd_writer = {
1417		.writer = {
1418			.ops = &arm_smmu_cd_writer_ops,
1419			.master = master,
1420		},
1421		.ssid = ssid,
1422	};
1423
1424	if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) {
1425		if (cur_valid)
1426			master->cd_table.used_ssids--;
1427		else
1428			master->cd_table.used_ssids++;
1429	}
1430
1431	arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
1432}
1433
1434void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
1435			 struct arm_smmu_master *master,
1436			 struct arm_smmu_domain *smmu_domain)
1437{
1438	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
1439	const struct io_pgtable_cfg *pgtbl_cfg =
1440		&io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1441	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr =
1442		&pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1443
1444	memset(target, 0, sizeof(*target));
1445
1446	target->data[0] = cpu_to_le64(
1447		FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
1448		FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
1449		FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
1450		FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
1451		FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
1452#ifdef __BIG_ENDIAN
1453		CTXDESC_CD_0_ENDI |
1454#endif
1455		CTXDESC_CD_0_TCR_EPD1 |
1456		CTXDESC_CD_0_V |
1457		FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
1458		CTXDESC_CD_0_AA64 |
1459		(master->stall_enabled ? CTXDESC_CD_0_S : 0) |
1460		CTXDESC_CD_0_R |
1461		CTXDESC_CD_0_A |
1462		CTXDESC_CD_0_ASET |
1463		FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid)
1464		);
1465
1466	/* To enable dirty flag update, set both Access flag and dirty state update */
1467	if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD)
1468		target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA |
1469					       CTXDESC_CD_0_TCR_HD);
1470
1471	target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr &
1472				      CTXDESC_CD_1_TTB0_MASK);
1473	target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair);
1474}
1475EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd);
1476
1477void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
1478{
1479	struct arm_smmu_cd target = {};
1480	struct arm_smmu_cd *cdptr;
1481
1482	if (!arm_smmu_cdtab_allocated(&master->cd_table))
1483		return;
1484	cdptr = arm_smmu_get_cd_ptr(master, ssid);
1485	if (WARN_ON(!cdptr))
1486		return;
1487	arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
1488}
1489
1490static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
1491{
1492	int ret;
1493	size_t l1size;
1494	size_t max_contexts;
1495	struct arm_smmu_device *smmu = master->smmu;
1496	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1497
1498	cd_table->s1cdmax = master->ssid_bits;
1499	max_contexts = 1 << cd_table->s1cdmax;
1500
1501	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1502	    max_contexts <= CTXDESC_L2_ENTRIES) {
1503		cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1504		cd_table->linear.num_ents = max_contexts;
1505
1506		l1size = max_contexts * sizeof(struct arm_smmu_cd);
1507		cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
1508							    &cd_table->cdtab_dma,
1509							    GFP_KERNEL);
1510		if (!cd_table->linear.table)
1511			return -ENOMEM;
1512	} else {
1513		cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1514		cd_table->l2.num_l1_ents =
1515			DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
1516
1517		cd_table->l2.l2ptrs = kcalloc(cd_table->l2.num_l1_ents,
1518					     sizeof(*cd_table->l2.l2ptrs),
1519					     GFP_KERNEL);
1520		if (!cd_table->l2.l2ptrs)
1521			return -ENOMEM;
1522
1523		l1size = cd_table->l2.num_l1_ents * sizeof(struct arm_smmu_cdtab_l1);
1524		cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size,
1525							&cd_table->cdtab_dma,
1526							GFP_KERNEL);
1527		if (!cd_table->l2.l1tab) {
1528			ret = -ENOMEM;
1529			goto err_free_l2ptrs;
1530		}
1531	}
1532	return 0;
1533
1534err_free_l2ptrs:
1535	kfree(cd_table->l2.l2ptrs);
1536	cd_table->l2.l2ptrs = NULL;
1537	return ret;
1538}
1539
1540static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
1541{
1542	int i;
1543	struct arm_smmu_device *smmu = master->smmu;
1544	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1545
1546	if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) {
1547		for (i = 0; i < cd_table->l2.num_l1_ents; i++) {
1548			if (!cd_table->l2.l2ptrs[i])
1549				continue;
1550
1551			dma_free_coherent(smmu->dev,
1552					  sizeof(*cd_table->l2.l2ptrs[i]),
1553					  cd_table->l2.l2ptrs[i],
1554					  arm_smmu_cd_l1_get_desc(&cd_table->l2.l1tab[i]));
1555		}
1556		kfree(cd_table->l2.l2ptrs);
1557
1558		dma_free_coherent(smmu->dev,
1559				  cd_table->l2.num_l1_ents *
1560					  sizeof(struct arm_smmu_cdtab_l1),
1561				  cd_table->l2.l1tab, cd_table->cdtab_dma);
1562	} else {
1563		dma_free_coherent(smmu->dev,
1564				  cd_table->linear.num_ents *
1565					  sizeof(struct arm_smmu_cd),
1566				  cd_table->linear.table, cd_table->cdtab_dma);
1567	}
1568}
1569
1570/* Stream table manipulation functions */
1571static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
1572					  dma_addr_t l2ptr_dma)
1573{
1574	u64 val = 0;
1575
1576	val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
1577	val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1578
1579	/* The HW has 64 bit atomicity with stores to the L2 STE table */
1580	WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1581}
1582
1583struct arm_smmu_ste_writer {
1584	struct arm_smmu_entry_writer writer;
1585	u32 sid;
1586};
1587
1588static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1589{
1590	struct arm_smmu_ste_writer *ste_writer =
1591		container_of(writer, struct arm_smmu_ste_writer, writer);
1592	struct arm_smmu_cmdq_ent cmd = {
1593		.opcode	= CMDQ_OP_CFGI_STE,
1594		.cfgi	= {
1595			.sid	= ste_writer->sid,
1596			.leaf	= true,
1597		},
1598	};
1599
1600	arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
1601}
1602
1603static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
1604	.sync = arm_smmu_ste_writer_sync_entry,
1605	.get_used = arm_smmu_get_ste_used,
1606	.get_update_safe = arm_smmu_get_ste_update_safe,
1607};
1608
1609static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
1610			       struct arm_smmu_ste *ste,
1611			       const struct arm_smmu_ste *target)
1612{
1613	struct arm_smmu_device *smmu = master->smmu;
1614	struct arm_smmu_ste_writer ste_writer = {
1615		.writer = {
1616			.ops = &arm_smmu_ste_writer_ops,
1617			.master = master,
1618		},
1619		.sid = sid,
1620	};
1621
1622	arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
1623
1624	/* It's likely that we'll want to use the new STE soon */
1625	if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
1626		struct arm_smmu_cmdq_ent
1627			prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
1628					 .prefetch = {
1629						 .sid = sid,
1630					 } };
1631
1632		arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1633	}
1634}
1635
1636void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
1637{
1638	memset(target, 0, sizeof(*target));
1639	target->data[0] = cpu_to_le64(
1640		STRTAB_STE_0_V |
1641		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT));
1642}
1643EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste);
1644
1645VISIBLE_IF_KUNIT
1646void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
1647			      struct arm_smmu_ste *target)
1648{
1649	memset(target, 0, sizeof(*target));
1650	target->data[0] = cpu_to_le64(
1651		STRTAB_STE_0_V |
1652		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS));
1653
1654	if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1655		target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1656							 STRTAB_STE_1_SHCFG_INCOMING));
1657}
1658EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste);
1659
1660VISIBLE_IF_KUNIT
1661void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
1662			       struct arm_smmu_master *master, bool ats_enabled,
1663			       unsigned int s1dss)
1664{
1665	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1666	struct arm_smmu_device *smmu = master->smmu;
1667
1668	memset(target, 0, sizeof(*target));
1669	target->data[0] = cpu_to_le64(
1670		STRTAB_STE_0_V |
1671		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1672		FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) |
1673		(cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1674		FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax));
1675
1676	target->data[1] = cpu_to_le64(
1677		FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) |
1678		FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1679		FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1680		FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1681		((smmu->features & ARM_SMMU_FEAT_STALLS &&
1682		  !master->stall_enabled) ?
1683			 STRTAB_STE_1_S1STALLD :
1684			 0) |
1685		FIELD_PREP(STRTAB_STE_1_EATS,
1686			   ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1687
1688	if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
1689	    s1dss == STRTAB_STE_1_S1DSS_BYPASS)
1690		target->data[1] |= cpu_to_le64(FIELD_PREP(
1691			STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
1692
1693	if (smmu->features & ARM_SMMU_FEAT_E2H) {
1694		/*
1695		 * To support BTM the streamworld needs to match the
1696		 * configuration of the CPU so that the ASID broadcasts are
1697		 * properly matched. This means either S/NS-EL2-E2H (hypervisor)
1698		 * or NS-EL1 (guest). Since an SVA domain can be installed in a
1699		 * PASID this should always use a BTM compatible configuration
1700		 * if the HW supports it.
1701		 */
1702		target->data[1] |= cpu_to_le64(
1703			FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2));
1704	} else {
1705		target->data[1] |= cpu_to_le64(
1706			FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1707
1708		/*
1709		 * VMID 0 is reserved for stage-2 bypass EL1 STEs, see
1710		 * arm_smmu_domain_alloc_id()
1711		 */
1712		target->data[2] =
1713			cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0));
1714	}
1715}
1716EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
1717
1718void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
1719				 struct arm_smmu_master *master,
1720				 struct arm_smmu_domain *smmu_domain,
1721				 bool ats_enabled)
1722{
1723	struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;
1724	const struct io_pgtable_cfg *pgtbl_cfg =
1725		&io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1726	typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr =
1727		&pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1728	u64 vtcr_val;
1729	struct arm_smmu_device *smmu = master->smmu;
1730
1731	memset(target, 0, sizeof(*target));
1732	target->data[0] = cpu_to_le64(
1733		STRTAB_STE_0_V |
1734		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS));
1735
1736	target->data[1] = cpu_to_le64(
1737		FIELD_PREP(STRTAB_STE_1_EATS,
1738			   ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1739
1740	if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
1741		target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB);
1742	if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1743		target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1744							  STRTAB_STE_1_SHCFG_INCOMING));
1745
1746	vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
1747		   FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
1748		   FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
1749		   FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
1750		   FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
1751		   FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
1752		   FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
1753	target->data[2] = cpu_to_le64(
1754		FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1755		FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) |
1756		STRTAB_STE_2_S2AA64 |
1757#ifdef __BIG_ENDIAN
1758		STRTAB_STE_2_S2ENDI |
1759#endif
1760		STRTAB_STE_2_S2PTW |
1761		(master->stall_enabled ? STRTAB_STE_2_S2S : 0) |
1762		STRTAB_STE_2_S2R);
1763
1764	target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr &
1765				      STRTAB_STE_3_S2TTB_MASK);
1766}
1767EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste);
1768
1769/*
1770 * This can safely directly manipulate the STE memory without a sync sequence
1771 * because the STE table has not been installed in the SMMU yet.
1772 */
1773static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab,
1774				       unsigned int nent)
1775{
1776	unsigned int i;
1777
1778	for (i = 0; i < nent; ++i) {
1779		arm_smmu_make_abort_ste(strtab);
1780		strtab++;
1781	}
1782}
1783
1784static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1785{
1786	dma_addr_t l2ptr_dma;
1787	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1788	struct arm_smmu_strtab_l2 **l2table;
1789
1790	l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)];
1791	if (*l2table)
1792		return 0;
1793
1794	*l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table),
1795				       &l2ptr_dma, GFP_KERNEL);
1796	if (!*l2table) {
1797		dev_err(smmu->dev,
1798			"failed to allocate l2 stream table for SID %u\n",
1799			sid);
1800		return -ENOMEM;
1801	}
1802
1803	arm_smmu_init_initial_stes((*l2table)->stes,
1804				   ARRAY_SIZE((*l2table)->stes));
1805	arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)],
1806				      l2ptr_dma);
1807	return 0;
1808}
1809
1810static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs)
1811{
1812	struct arm_smmu_stream *stream_rhs =
1813		rb_entry(rhs, struct arm_smmu_stream, node);
1814	const u32 *sid_lhs = lhs;
1815
1816	if (*sid_lhs < stream_rhs->id)
1817		return -1;
1818	if (*sid_lhs > stream_rhs->id)
1819		return 1;
1820	return 0;
1821}
1822
1823static int arm_smmu_streams_cmp_node(struct rb_node *lhs,
1824				     const struct rb_node *rhs)
1825{
1826	return arm_smmu_streams_cmp_key(
1827		&rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
1828}
1829
1830static struct arm_smmu_master *
1831arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1832{
1833	struct rb_node *node;
1834
1835	lockdep_assert_held(&smmu->streams_mutex);
1836
1837	node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
1838	if (!node)
1839		return NULL;
1840	return rb_entry(node, struct arm_smmu_stream, node)->master;
1841}
1842
1843/* IRQ and event handlers */
1844static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw,
1845				  struct arm_smmu_event *event)
1846{
1847	struct arm_smmu_master *master;
1848
1849	event->id = FIELD_GET(EVTQ_0_ID, raw[0]);
1850	event->sid = FIELD_GET(EVTQ_0_SID, raw[0]);
1851	event->ssv = FIELD_GET(EVTQ_0_SSV, raw[0]);
1852	event->ssid = event->ssv ? FIELD_GET(EVTQ_0_SSID, raw[0]) : IOMMU_NO_PASID;
1853	event->privileged = FIELD_GET(EVTQ_1_PnU, raw[1]);
1854	event->instruction = FIELD_GET(EVTQ_1_InD, raw[1]);
1855	event->s2 = FIELD_GET(EVTQ_1_S2, raw[1]);
1856	event->read = FIELD_GET(EVTQ_1_RnW, raw[1]);
1857	event->stag = FIELD_GET(EVTQ_1_STAG, raw[1]);
1858	event->stall = FIELD_GET(EVTQ_1_STALL, raw[1]);
1859	event->class = FIELD_GET(EVTQ_1_CLASS, raw[1]);
1860	event->iova = FIELD_GET(EVTQ_2_ADDR, raw[2]);
1861	event->ipa = raw[3] & EVTQ_3_IPA;
1862	event->fetch_addr = raw[3] & EVTQ_3_FETCH_ADDR;
1863	event->ttrnw = FIELD_GET(EVTQ_1_TT_READ, raw[1]);
1864	event->class_tt = false;
1865	event->dev = NULL;
1866
1867	if (event->id == EVT_ID_PERMISSION_FAULT)
1868		event->class_tt = (event->class == EVTQ_1_CLASS_TT);
1869
1870	mutex_lock(&smmu->streams_mutex);
1871	master = arm_smmu_find_master(smmu, event->sid);
1872	if (master)
1873		event->dev = get_device(master->dev);
1874	mutex_unlock(&smmu->streams_mutex);
1875}
1876
1877static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,
1878				 struct arm_smmu_event *event)
1879{
1880	int ret = 0;
1881	u32 perm = 0;
1882	struct arm_smmu_master *master;
1883	struct iopf_fault fault_evt = { };
1884	struct iommu_fault *flt = &fault_evt.fault;
1885
1886	switch (event->id) {
1887	case EVT_ID_BAD_STE_CONFIG:
1888	case EVT_ID_STREAM_DISABLED_FAULT:
1889	case EVT_ID_BAD_SUBSTREAMID_CONFIG:
1890	case EVT_ID_BAD_CD_CONFIG:
1891	case EVT_ID_TRANSLATION_FAULT:
1892	case EVT_ID_ADDR_SIZE_FAULT:
1893	case EVT_ID_ACCESS_FAULT:
1894	case EVT_ID_PERMISSION_FAULT:
1895		break;
1896	default:
1897		return -EOPNOTSUPP;
1898	}
1899
1900	if (event->stall) {
1901		if (event->read)
1902			perm |= IOMMU_FAULT_PERM_READ;
1903		else
1904			perm |= IOMMU_FAULT_PERM_WRITE;
1905
1906		if (event->instruction)
1907			perm |= IOMMU_FAULT_PERM_EXEC;
1908
1909		if (event->privileged)
1910			perm |= IOMMU_FAULT_PERM_PRIV;
1911
1912		flt->type = IOMMU_FAULT_PAGE_REQ;
1913		flt->prm = (struct iommu_fault_page_request){
1914			.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1915			.grpid = event->stag,
1916			.perm = perm,
1917			.addr = event->iova,
1918		};
1919
1920		if (event->ssv) {
1921			flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1922			flt->prm.pasid = event->ssid;
1923		}
1924	}
1925
1926	mutex_lock(&smmu->streams_mutex);
1927	master = arm_smmu_find_master(smmu, event->sid);
1928	if (!master) {
1929		ret = -EINVAL;
1930		goto out_unlock;
1931	}
1932
1933	if (event->stall)
1934		ret = iommu_report_device_fault(master->dev, &fault_evt);
1935	else if (master->vmaster && !event->s2)
1936		ret = arm_vmaster_report_event(master->vmaster, evt);
1937	else
1938		ret = -EOPNOTSUPP; /* Unhandled events should be pinned */
1939out_unlock:
1940	mutex_unlock(&smmu->streams_mutex);
1941	return ret;
1942}
1943
1944static void arm_smmu_dump_raw_event(struct arm_smmu_device *smmu, u64 *raw,
1945				    struct arm_smmu_event *event)
1946{
1947	int i;
1948
1949	dev_err(smmu->dev, "event 0x%02x received:\n", event->id);
1950
1951	for (i = 0; i < EVTQ_ENT_DWORDS; ++i)
1952		dev_err(smmu->dev, "\t0x%016llx\n", raw[i]);
1953}
1954
1955#define ARM_SMMU_EVT_KNOWN(e)	((e)->id < ARRAY_SIZE(event_str) && event_str[(e)->id])
1956#define ARM_SMMU_LOG_EVT_STR(e) ARM_SMMU_EVT_KNOWN(e) ? event_str[(e)->id] : "UNKNOWN"
1957#define ARM_SMMU_LOG_CLIENT(e)	(e)->dev ? dev_name((e)->dev) : "(unassigned sid)"
1958
1959static void arm_smmu_dump_event(struct arm_smmu_device *smmu, u64 *raw,
1960				struct arm_smmu_event *evt,
1961				struct ratelimit_state *rs)
1962{
1963	if (!__ratelimit(rs))
1964		return;
1965
1966	arm_smmu_dump_raw_event(smmu, raw, evt);
1967
1968	switch (evt->id) {
1969	case EVT_ID_TRANSLATION_FAULT:
1970	case EVT_ID_ADDR_SIZE_FAULT:
1971	case EVT_ID_ACCESS_FAULT:
1972	case EVT_ID_PERMISSION_FAULT:
1973		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x iova: %#llx ipa: %#llx",
1974			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1975			evt->sid, evt->ssid, evt->iova, evt->ipa);
1976
1977		dev_err(smmu->dev, "%s %s %s %s \"%s\"%s%s stag: %#x",
1978			evt->privileged ? "priv" : "unpriv",
1979			evt->instruction ? "inst" : "data",
1980			str_read_write(evt->read),
1981			evt->s2 ? "s2" : "s1", event_class_str[evt->class],
1982			evt->class_tt ? (evt->ttrnw ? " ttd_read" : " ttd_write") : "",
1983			evt->stall ? " stall" : "", evt->stag);
1984
1985		break;
1986
1987	case EVT_ID_STE_FETCH_FAULT:
1988	case EVT_ID_CD_FETCH_FAULT:
1989	case EVT_ID_VMS_FETCH_FAULT:
1990		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x fetch_addr: %#llx",
1991			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1992			evt->sid, evt->ssid, evt->fetch_addr);
1993
1994		break;
1995
1996	default:
1997		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x",
1998			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1999			evt->sid, evt->ssid);
2000	}
2001}
2002
2003static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
2004{
2005	u64 evt[EVTQ_ENT_DWORDS];
2006	struct arm_smmu_event event = {0};
2007	struct arm_smmu_device *smmu = dev;
2008	struct arm_smmu_queue *q = &smmu->evtq.q;
2009	struct arm_smmu_ll_queue *llq = &q->llq;
2010	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
2011				      DEFAULT_RATELIMIT_BURST);
2012
2013	do {
2014		while (!queue_remove_raw(q, evt)) {
2015			arm_smmu_decode_event(smmu, evt, &event);
2016			if (arm_smmu_handle_event(smmu, evt, &event))
2017				arm_smmu_dump_event(smmu, evt, &event, &rs);
2018
2019			put_device(event.dev);
2020			cond_resched();
2021		}
2022
2023		/*
2024		 * Not much we can do on overflow, so scream and pretend we're
2025		 * trying harder.
2026		 */
2027		if (queue_sync_prod_in(q) == -EOVERFLOW)
2028			dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
2029	} while (!queue_empty(llq));
2030
2031	/* Sync our overflow flag, as we believe we're up to speed */
2032	queue_sync_cons_ovf(q);
2033	return IRQ_HANDLED;
2034}
2035
2036static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
2037{
2038	u32 sid, ssid;
2039	u16 grpid;
2040	bool ssv, last;
2041
2042	sid = FIELD_GET(PRIQ_0_SID, evt[0]);
2043	ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
2044	ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
2045	last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
2046	grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
2047
2048	dev_info(smmu->dev, "unexpected PRI request received:\n");
2049	dev_info(smmu->dev,
2050		 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
2051		 sid, ssid, grpid, last ? "L" : "",
2052		 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
2053		 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
2054		 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
2055		 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
2056		 evt[1] & PRIQ_1_ADDR_MASK);
2057
2058	if (last) {
2059		struct arm_smmu_cmdq_ent cmd = {
2060			.opcode			= CMDQ_OP_PRI_RESP,
2061			.substream_valid	= ssv,
2062			.pri			= {
2063				.sid	= sid,
2064				.ssid	= ssid,
2065				.grpid	= grpid,
2066				.resp	= PRI_RESP_DENY,
2067			},
2068		};
2069
2070		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2071	}
2072}
2073
2074static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
2075{
2076	struct arm_smmu_device *smmu = dev;
2077	struct arm_smmu_queue *q = &smmu->priq.q;
2078	struct arm_smmu_ll_queue *llq = &q->llq;
2079	u64 evt[PRIQ_ENT_DWORDS];
2080
2081	do {
2082		while (!queue_remove_raw(q, evt))
2083			arm_smmu_handle_ppr(smmu, evt);
2084
2085		if (queue_sync_prod_in(q) == -EOVERFLOW)
2086			dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
2087	} while (!queue_empty(llq));
2088
2089	/* Sync our overflow flag, as we believe we're up to speed */
2090	queue_sync_cons_ovf(q);
2091	return IRQ_HANDLED;
2092}
2093
2094static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2095
2096static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
2097{
2098	u32 gerror, gerrorn, active;
2099	struct arm_smmu_device *smmu = dev;
2100
2101	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
2102	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
2103
2104	active = gerror ^ gerrorn;
2105	if (!(active & GERROR_ERR_MASK))
2106		return IRQ_NONE; /* No errors pending */
2107
2108	dev_warn(smmu->dev,
2109		 "unexpected global error reported (0x%08x), this could be serious\n",
2110		 active);
2111
2112	if (active & GERROR_SFM_ERR) {
2113		dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
2114		arm_smmu_device_disable(smmu);
2115	}
2116
2117	if (active & GERROR_MSI_GERROR_ABT_ERR)
2118		dev_warn(smmu->dev, "GERROR MSI write aborted\n");
2119
2120	if (active & GERROR_MSI_PRIQ_ABT_ERR)
2121		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
2122
2123	if (active & GERROR_MSI_EVTQ_ABT_ERR)
2124		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
2125
2126	if (active & GERROR_MSI_CMDQ_ABT_ERR)
2127		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
2128
2129	if (active & GERROR_PRIQ_ABT_ERR)
2130		dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
2131
2132	if (active & GERROR_EVTQ_ABT_ERR)
2133		dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
2134
2135	if (active & GERROR_CMDQ_ERR)
2136		arm_smmu_cmdq_skip_err(smmu);
2137
2138	writel(gerror, smmu->base + ARM_SMMU_GERRORN);
2139	return IRQ_HANDLED;
2140}
2141
2142static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
2143{
2144	struct arm_smmu_device *smmu = dev;
2145
2146	arm_smmu_evtq_thread(irq, dev);
2147	if (smmu->features & ARM_SMMU_FEAT_PRI)
2148		arm_smmu_priq_thread(irq, dev);
2149
2150	return IRQ_HANDLED;
2151}
2152
2153static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
2154{
2155	arm_smmu_gerror_handler(irq, dev);
2156	return IRQ_WAKE_THREAD;
2157}
2158
2159static void
2160arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
2161			struct arm_smmu_cmdq_ent *cmd)
2162{
2163	size_t log2_span;
2164	size_t span_mask;
2165	/* ATC invalidates are always on 4096-bytes pages */
2166	size_t inval_grain_shift = 12;
2167	unsigned long page_start, page_end;
2168
2169	/*
2170	 * ATS and PASID:
2171	 *
2172	 * If substream_valid is clear, the PCIe TLP is sent without a PASID
2173	 * prefix. In that case all ATC entries within the address range are
2174	 * invalidated, including those that were requested with a PASID! There
2175	 * is no way to invalidate only entries without PASID.
2176	 *
2177	 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
2178	 * traffic), translation requests without PASID create ATC entries
2179	 * without PASID, which must be invalidated with substream_valid clear.
2180	 * This has the unpleasant side-effect of invalidating all PASID-tagged
2181	 * ATC entries within the address range.
2182	 */
2183	*cmd = (struct arm_smmu_cmdq_ent) {
2184		.opcode			= CMDQ_OP_ATC_INV,
2185		.substream_valid	= (ssid != IOMMU_NO_PASID),
2186		.atc.ssid		= ssid,
2187	};
2188
2189	if (!size) {
2190		cmd->atc.size = ATC_INV_SIZE_ALL;
2191		return;
2192	}
2193
2194	page_start	= iova >> inval_grain_shift;
2195	page_end	= (iova + size - 1) >> inval_grain_shift;
2196
2197	/*
2198	 * In an ATS Invalidate Request, the address must be aligned on the
2199	 * range size, which must be a power of two number of page sizes. We
2200	 * thus have to choose between grossly over-invalidating the region, or
2201	 * splitting the invalidation into multiple commands. For simplicity
2202	 * we'll go with the first solution, but should refine it in the future
2203	 * if multiple commands are shown to be more efficient.
2204	 *
2205	 * Find the smallest power of two that covers the range. The most
2206	 * significant differing bit between the start and end addresses,
2207	 * fls(start ^ end), indicates the required span. For example:
2208	 *
2209	 * We want to invalidate pages [8; 11]. This is already the ideal range:
2210	 *		x = 0b1000 ^ 0b1011 = 0b11
2211	 *		span = 1 << fls(x) = 4
2212	 *
2213	 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2214	 *		x = 0b0111 ^ 0b1010 = 0b1101
2215	 *		span = 1 << fls(x) = 16
2216	 */
2217	log2_span	= fls_long(page_start ^ page_end);
2218	span_mask	= (1ULL << log2_span) - 1;
2219
2220	page_start	&= ~span_mask;
2221
2222	cmd->atc.addr	= page_start << inval_grain_shift;
2223	cmd->atc.size	= log2_span;
2224}
2225
2226static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
2227				   ioasid_t ssid)
2228{
2229	int i;
2230	struct arm_smmu_cmdq_ent cmd;
2231	struct arm_smmu_cmdq_batch cmds;
2232
2233	arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
2234
2235	arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
2236	for (i = 0; i < master->num_streams; i++) {
2237		cmd.atc.sid = master->streams[i].id;
2238		arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
2239	}
2240
2241	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
2242}
2243
2244int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2245			    unsigned long iova, size_t size)
2246{
2247	struct arm_smmu_master_domain *master_domain;
2248	int i;
2249	unsigned long flags;
2250	struct arm_smmu_cmdq_ent cmd = {
2251		.opcode = CMDQ_OP_ATC_INV,
2252	};
2253	struct arm_smmu_cmdq_batch cmds;
2254
2255	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2256		return 0;
2257
2258	/*
2259	 * Ensure that we've completed prior invalidation of the main TLBs
2260	 * before we read 'nr_ats_masters' in case of a concurrent call to
2261	 * arm_smmu_enable_ats():
2262	 *
2263	 *	// unmap()			// arm_smmu_enable_ats()
2264	 *	TLBI+SYNC			atomic_inc(&nr_ats_masters);
2265	 *	smp_mb();			[...]
2266	 *	atomic_read(&nr_ats_masters);	pci_enable_ats() // writel()
2267	 *
2268	 * Ensures that we always see the incremented 'nr_ats_masters' count if
2269	 * ATS was enabled at the PCI device before completion of the TLBI.
2270	 */
2271	smp_mb();
2272	if (!atomic_read(&smmu_domain->nr_ats_masters))
2273		return 0;
2274
2275	arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
2276
2277	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2278	list_for_each_entry(master_domain, &smmu_domain->devices,
2279			    devices_elm) {
2280		struct arm_smmu_master *master = master_domain->master;
2281
2282		if (!master->ats_enabled)
2283			continue;
2284
2285		if (master_domain->nested_ats_flush) {
2286			/*
2287			 * If a S2 used as a nesting parent is changed we have
2288			 * no option but to completely flush the ATC.
2289			 */
2290			arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
2291		} else {
2292			arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size,
2293						&cmd);
2294		}
2295
2296		for (i = 0; i < master->num_streams; i++) {
2297			cmd.atc.sid = master->streams[i].id;
2298			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
2299		}
2300	}
2301	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2302
2303	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
2304}
2305
2306/* IO_PGTABLE API */
2307static void arm_smmu_tlb_inv_context(void *cookie)
2308{
2309	struct arm_smmu_domain *smmu_domain = cookie;
2310	struct arm_smmu_device *smmu = smmu_domain->smmu;
2311	struct arm_smmu_cmdq_ent cmd;
2312
2313	/*
2314	 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2315	 * PTEs previously cleared by unmaps on the current CPU not yet visible
2316	 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2317	 * insertion to guarantee those are observed before the TLBI. Do be
2318	 * careful, 007.
2319	 */
2320	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2321		arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
2322	} else {
2323		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
2324		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
2325		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
2326	}
2327	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
2328}
2329
2330static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
2331				     unsigned long iova, size_t size,
2332				     size_t granule,
2333				     struct arm_smmu_domain *smmu_domain)
2334{
2335	struct arm_smmu_device *smmu = smmu_domain->smmu;
2336	unsigned long end = iova + size, num_pages = 0, tg = 0;
2337	size_t inv_range = granule;
2338	struct arm_smmu_cmdq_batch cmds;
2339
2340	if (!size)
2341		return;
2342
2343	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2344		/* Get the leaf page size */
2345		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
2346
2347		num_pages = size >> tg;
2348
2349		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
2350		cmd->tlbi.tg = (tg - 10) / 2;
2351
2352		/*
2353		 * Determine what level the granule is at. For non-leaf, both
2354		 * io-pgtable and SVA pass a nominal last-level granule because
2355		 * they don't know what level(s) actually apply, so ignore that
2356		 * and leave TTL=0. However for various errata reasons we still
2357		 * want to use a range command, so avoid the SVA corner case
2358		 * where both scale and num could be 0 as well.
2359		 */
2360		if (cmd->tlbi.leaf)
2361			cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
2362		else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
2363			num_pages++;
2364	}
2365
2366	arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
2367
2368	while (iova < end) {
2369		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2370			/*
2371			 * On each iteration of the loop, the range is 5 bits
2372			 * worth of the aligned size remaining.
2373			 * The range in pages is:
2374			 *
2375			 * range = (num_pages & (0x1f << __ffs(num_pages)))
2376			 */
2377			unsigned long scale, num;
2378
2379			/* Determine the power of 2 multiple number of pages */
2380			scale = __ffs(num_pages);
2381			cmd->tlbi.scale = scale;
2382
2383			/* Determine how many chunks of 2^scale size we have */
2384			num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
2385			cmd->tlbi.num = num - 1;
2386
2387			/* range is num * 2^scale * pgsize */
2388			inv_range = num << (scale + tg);
2389
2390			/* Clear out the lower order bits for the next iteration */
2391			num_pages -= num << scale;
2392		}
2393
2394		cmd->tlbi.addr = iova;
2395		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
2396		iova += inv_range;
2397	}
2398	arm_smmu_cmdq_batch_submit(smmu, &cmds);
2399}
2400
2401static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
2402					  size_t granule, bool leaf,
2403					  struct arm_smmu_domain *smmu_domain)
2404{
2405	struct arm_smmu_cmdq_ent cmd = {
2406		.tlbi = {
2407			.leaf	= leaf,
2408		},
2409	};
2410
2411	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2412		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2413				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
2414		cmd.tlbi.asid	= smmu_domain->cd.asid;
2415	} else {
2416		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
2417		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
2418	}
2419	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2420
2421	if (smmu_domain->nest_parent) {
2422		/*
2423		 * When the S2 domain changes all the nested S1 ASIDs have to be
2424		 * flushed too.
2425		 */
2426		cmd.opcode = CMDQ_OP_TLBI_NH_ALL;
2427		arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd);
2428	}
2429
2430	/*
2431	 * Unfortunately, this can't be leaf-only since we may have
2432	 * zapped an entire table.
2433	 */
2434	arm_smmu_atc_inv_domain(smmu_domain, iova, size);
2435}
2436
2437void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
2438				 size_t granule, bool leaf,
2439				 struct arm_smmu_domain *smmu_domain)
2440{
2441	struct arm_smmu_cmdq_ent cmd = {
2442		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2443			  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
2444		.tlbi = {
2445			.asid	= asid,
2446			.leaf	= leaf,
2447		},
2448	};
2449
2450	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2451}
2452
2453static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2454					 unsigned long iova, size_t granule,
2455					 void *cookie)
2456{
2457	struct arm_smmu_domain *smmu_domain = cookie;
2458	struct iommu_domain *domain = &smmu_domain->domain;
2459
2460	iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2461}
2462
2463static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2464				  size_t granule, void *cookie)
2465{
2466	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2467}
2468
2469static const struct iommu_flush_ops arm_smmu_flush_ops = {
2470	.tlb_flush_all	= arm_smmu_tlb_inv_context,
2471	.tlb_flush_walk = arm_smmu_tlb_inv_walk,
2472	.tlb_add_page	= arm_smmu_tlb_inv_page_nosync,
2473};
2474
2475static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu)
2476{
2477	u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY);
2478
2479	return (smmu->features & features) == features;
2480}
2481
2482/* IOMMU API */
2483static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2484{
2485	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2486
2487	switch (cap) {
2488	case IOMMU_CAP_CACHE_COHERENCY:
2489		/* Assume that a coherent TCU implies coherent TBUs */
2490		return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2491	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2492		return arm_smmu_master_canwbs(master);
2493	case IOMMU_CAP_NOEXEC:
2494	case IOMMU_CAP_DEFERRED_FLUSH:
2495		return true;
2496	case IOMMU_CAP_DIRTY_TRACKING:
2497		return arm_smmu_dbm_capable(master->smmu);
2498	default:
2499		return false;
2500	}
2501}
2502
2503static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
2504{
2505	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2506	struct arm_smmu_master_domain *master_domain;
2507	unsigned long flags;
2508	bool ret = true;
2509
2510	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2511	list_for_each_entry(master_domain, &smmu_domain->devices,
2512			    devices_elm) {
2513		if (!arm_smmu_master_canwbs(master_domain->master)) {
2514			ret = false;
2515			break;
2516		}
2517	}
2518	smmu_domain->enforce_cache_coherency = ret;
2519	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2520	return ret;
2521}
2522
2523struct arm_smmu_domain *arm_smmu_domain_alloc(void)
2524{
2525	struct arm_smmu_domain *smmu_domain;
2526
2527	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2528	if (!smmu_domain)
2529		return ERR_PTR(-ENOMEM);
2530
2531	INIT_LIST_HEAD(&smmu_domain->devices);
2532	spin_lock_init(&smmu_domain->devices_lock);
2533
2534	return smmu_domain;
2535}
2536
2537static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
2538{
2539	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2540	struct arm_smmu_device *smmu = smmu_domain->smmu;
2541
2542	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2543
2544	/* Free the ASID or VMID */
2545	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2546		/* Prevent SVA from touching the CD while we're freeing it */
2547		mutex_lock(&arm_smmu_asid_lock);
2548		xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
2549		mutex_unlock(&arm_smmu_asid_lock);
2550	} else {
2551		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2552		if (cfg->vmid)
2553			ida_free(&smmu->vmid_map, cfg->vmid);
2554	}
2555
2556	kfree(smmu_domain);
2557}
2558
2559static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
2560				       struct arm_smmu_domain *smmu_domain)
2561{
2562	int ret;
2563	u32 asid = 0;
2564	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
2565
2566	/* Prevent SVA from modifying the ASID until it is written to the CD */
2567	mutex_lock(&arm_smmu_asid_lock);
2568	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
2569		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2570	cd->asid	= (u16)asid;
2571	mutex_unlock(&arm_smmu_asid_lock);
2572	return ret;
2573}
2574
2575static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu,
2576				       struct arm_smmu_domain *smmu_domain)
2577{
2578	int vmid;
2579	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2580
2581	/* Reserve VMID 0 for stage-2 bypass STEs */
2582	vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2583			       GFP_KERNEL);
2584	if (vmid < 0)
2585		return vmid;
2586
2587	cfg->vmid	= (u16)vmid;
2588	return 0;
2589}
2590
2591static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
2592				    struct arm_smmu_device *smmu, u32 flags)
2593{
2594	int ret;
2595	enum io_pgtable_fmt fmt;
2596	struct io_pgtable_cfg pgtbl_cfg;
2597	struct io_pgtable_ops *pgtbl_ops;
2598	int (*finalise_stage_fn)(struct arm_smmu_device *smmu,
2599				 struct arm_smmu_domain *smmu_domain);
2600	bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2601
2602	pgtbl_cfg = (struct io_pgtable_cfg) {
2603		.pgsize_bitmap	= smmu->pgsize_bitmap,
2604		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENCY,
2605		.tlb		= &arm_smmu_flush_ops,
2606		.iommu_dev	= smmu->dev,
2607	};
2608
2609	switch (smmu_domain->stage) {
2610	case ARM_SMMU_DOMAIN_S1: {
2611		unsigned long ias = (smmu->features &
2612				     ARM_SMMU_FEAT_VAX) ? 52 : 48;
2613
2614		pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
2615		pgtbl_cfg.oas = smmu->ias;
2616		if (enable_dirty)
2617			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
2618		fmt = ARM_64_LPAE_S1;
2619		finalise_stage_fn = arm_smmu_domain_finalise_s1;
2620		break;
2621	}
2622	case ARM_SMMU_DOMAIN_S2:
2623		if (enable_dirty)
2624			return -EOPNOTSUPP;
2625		pgtbl_cfg.ias = smmu->ias;
2626		pgtbl_cfg.oas = smmu->oas;
2627		fmt = ARM_64_LPAE_S2;
2628		finalise_stage_fn = arm_smmu_domain_finalise_s2;
2629		if ((smmu->features & ARM_SMMU_FEAT_S2FWB) &&
2630		    (flags & IOMMU_HWPT_ALLOC_NEST_PARENT))
2631			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_S2FWB;
2632		break;
2633	default:
2634		return -EINVAL;
2635	}
2636
2637	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2638	if (!pgtbl_ops)
2639		return -ENOMEM;
2640
2641	smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2642	smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2643	smmu_domain->domain.geometry.force_aperture = true;
2644	if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2645		smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops;
2646
2647	ret = finalise_stage_fn(smmu, smmu_domain);
2648	if (ret < 0) {
2649		free_io_pgtable_ops(pgtbl_ops);
2650		return ret;
2651	}
2652
2653	smmu_domain->pgtbl_ops = pgtbl_ops;
2654	smmu_domain->smmu = smmu;
2655	return 0;
2656}
2657
2658static struct arm_smmu_ste *
2659arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2660{
2661	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2662
2663	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2664		/* Two-level walk */
2665		return &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]
2666				->stes[arm_smmu_strtab_l2_idx(sid)];
2667	} else {
2668		/* Simple linear lookup */
2669		return &cfg->linear.table[sid];
2670	}
2671}
2672
2673void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
2674				  const struct arm_smmu_ste *target)
2675{
2676	int i, j;
2677	struct arm_smmu_device *smmu = master->smmu;
2678
2679	master->cd_table.in_ste =
2680		FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
2681		STRTAB_STE_0_CFG_S1_TRANS;
2682	master->ste_ats_enabled =
2683		FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
2684		STRTAB_STE_1_EATS_TRANS;
2685
2686	for (i = 0; i < master->num_streams; ++i) {
2687		u32 sid = master->streams[i].id;
2688		struct arm_smmu_ste *step =
2689			arm_smmu_get_step_for_sid(smmu, sid);
2690
2691		/* Bridged PCI devices may end up with duplicated IDs */
2692		for (j = 0; j < i; j++)
2693			if (master->streams[j].id == sid)
2694				break;
2695		if (j < i)
2696			continue;
2697
2698		arm_smmu_write_ste(master, sid, step, target);
2699	}
2700}
2701
2702static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2703{
2704	struct device *dev = master->dev;
2705	struct arm_smmu_device *smmu = master->smmu;
2706	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2707
2708	if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2709		return false;
2710
2711	if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2712		return false;
2713
2714	return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2715}
2716
2717static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2718{
2719	size_t stu;
2720	struct pci_dev *pdev;
2721	struct arm_smmu_device *smmu = master->smmu;
2722
2723	/* Smallest Translation Unit: log2 of the smallest supported granule */
2724	stu = __ffs(smmu->pgsize_bitmap);
2725	pdev = to_pci_dev(master->dev);
2726
2727	/*
2728	 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
2729	 */
2730	arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2731	if (pci_enable_ats(pdev, stu))
2732		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2733}
2734
2735static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2736{
2737	int ret;
2738	int features;
2739	int num_pasids;
2740	struct pci_dev *pdev;
2741
2742	if (!dev_is_pci(master->dev))
2743		return -ENODEV;
2744
2745	pdev = to_pci_dev(master->dev);
2746
2747	features = pci_pasid_features(pdev);
2748	if (features < 0)
2749		return features;
2750
2751	num_pasids = pci_max_pasids(pdev);
2752	if (num_pasids <= 0)
2753		return num_pasids;
2754
2755	ret = pci_enable_pasid(pdev, features);
2756	if (ret) {
2757		dev_err(&pdev->dev, "Failed to enable PASID\n");
2758		return ret;
2759	}
2760
2761	master->ssid_bits = min_t(u8, ilog2(num_pasids),
2762				  master->smmu->ssid_bits);
2763	return 0;
2764}
2765
2766static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2767{
2768	struct pci_dev *pdev;
2769
2770	if (!dev_is_pci(master->dev))
2771		return;
2772
2773	pdev = to_pci_dev(master->dev);
2774
2775	if (!pdev->pasid_enabled)
2776		return;
2777
2778	master->ssid_bits = 0;
2779	pci_disable_pasid(pdev);
2780}
2781
2782static struct arm_smmu_master_domain *
2783arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
2784			    struct iommu_domain *domain,
2785			    struct arm_smmu_master *master,
2786			    ioasid_t ssid, bool nested_ats_flush)
2787{
2788	struct arm_smmu_master_domain *master_domain;
2789
2790	lockdep_assert_held(&smmu_domain->devices_lock);
2791
2792	list_for_each_entry(master_domain, &smmu_domain->devices,
2793			    devices_elm) {
2794		if (master_domain->master == master &&
2795		    master_domain->domain == domain &&
2796		    master_domain->ssid == ssid &&
2797		    master_domain->nested_ats_flush == nested_ats_flush)
2798			return master_domain;
2799	}
2800	return NULL;
2801}
2802
2803/*
2804 * If the domain uses the smmu_domain->devices list return the arm_smmu_domain
2805 * structure, otherwise NULL. These domains track attached devices so they can
2806 * issue invalidations.
2807 */
2808static struct arm_smmu_domain *
2809to_smmu_domain_devices(struct iommu_domain *domain)
2810{
2811	/* The domain can be NULL only when processing the first attach */
2812	if (!domain)
2813		return NULL;
2814	if ((domain->type & __IOMMU_DOMAIN_PAGING) ||
2815	    domain->type == IOMMU_DOMAIN_SVA)
2816		return to_smmu_domain(domain);
2817	if (domain->type == IOMMU_DOMAIN_NESTED)
2818		return to_smmu_nested_domain(domain)->vsmmu->s2_parent;
2819	return NULL;
2820}
2821
2822static int arm_smmu_enable_iopf(struct arm_smmu_master *master,
2823				struct arm_smmu_master_domain *master_domain)
2824{
2825	int ret;
2826
2827	iommu_group_mutex_assert(master->dev);
2828
2829	if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA))
2830		return -EOPNOTSUPP;
2831
2832	/*
2833	 * Drivers for devices supporting PRI or stall require iopf others have
2834	 * device-specific fault handlers and don't need IOPF, so this is not a
2835	 * failure.
2836	 */
2837	if (!master->stall_enabled)
2838		return 0;
2839
2840	/* We're not keeping track of SIDs in fault events */
2841	if (master->num_streams != 1)
2842		return -EOPNOTSUPP;
2843
2844	if (master->iopf_refcount) {
2845		master->iopf_refcount++;
2846		master_domain->using_iopf = true;
2847		return 0;
2848	}
2849
2850	ret = iopf_queue_add_device(master->smmu->evtq.iopf, master->dev);
2851	if (ret)
2852		return ret;
2853	master->iopf_refcount = 1;
2854	master_domain->using_iopf = true;
2855	return 0;
2856}
2857
2858static void arm_smmu_disable_iopf(struct arm_smmu_master *master,
2859				  struct arm_smmu_master_domain *master_domain)
2860{
2861	iommu_group_mutex_assert(master->dev);
2862
2863	if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA))
2864		return;
2865
2866	if (!master_domain || !master_domain->using_iopf)
2867		return;
2868
2869	master->iopf_refcount--;
2870	if (master->iopf_refcount == 0)
2871		iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev);
2872}
2873
2874static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
2875					  struct iommu_domain *domain,
2876					  ioasid_t ssid)
2877{
2878	struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
2879	struct arm_smmu_master_domain *master_domain;
2880	bool nested_ats_flush = false;
2881	unsigned long flags;
2882
2883	if (!smmu_domain)
2884		return;
2885
2886	if (domain->type == IOMMU_DOMAIN_NESTED)
2887		nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats;
2888
2889	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2890	master_domain = arm_smmu_find_master_domain(smmu_domain, domain, master,
2891						    ssid, nested_ats_flush);
2892	if (master_domain) {
2893		list_del(&master_domain->devices_elm);
2894		if (master->ats_enabled)
2895			atomic_dec(&smmu_domain->nr_ats_masters);
2896	}
2897	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2898
2899	arm_smmu_disable_iopf(master, master_domain);
2900	kfree(master_domain);
2901}
2902
2903/*
2904 * Start the sequence to attach a domain to a master. The sequence contains three
2905 * steps:
2906 *  arm_smmu_attach_prepare()
2907 *  arm_smmu_install_ste_for_dev()
2908 *  arm_smmu_attach_commit()
2909 *
2910 * If prepare succeeds then the sequence must be completed. The STE installed
2911 * must set the STE.EATS field according to state.ats_enabled.
2912 *
2913 * If the device supports ATS then this determines if EATS should be enabled
2914 * in the STE, and starts sequencing EATS disable if required.
2915 *
2916 * The change of the EATS in the STE and the PCI ATS config space is managed by
2917 * this sequence to be in the right order so that if PCI ATS is enabled then
2918 * STE.ETAS is enabled.
2919 *
2920 * new_domain can be a non-paging domain. In this case ATS will not be enabled,
2921 * and invalidations won't be tracked.
2922 */
2923int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
2924			    struct iommu_domain *new_domain)
2925{
2926	struct arm_smmu_master *master = state->master;
2927	struct arm_smmu_master_domain *master_domain;
2928	struct arm_smmu_domain *smmu_domain =
2929		to_smmu_domain_devices(new_domain);
2930	unsigned long flags;
2931	int ret;
2932
2933	/*
2934	 * arm_smmu_share_asid() must not see two domains pointing to the same
2935	 * arm_smmu_master_domain contents otherwise it could randomly write one
2936	 * or the other to the CD.
2937	 */
2938	lockdep_assert_held(&arm_smmu_asid_lock);
2939
2940	if (smmu_domain || state->cd_needs_ats) {
2941		/*
2942		 * The SMMU does not support enabling ATS with bypass/abort.
2943		 * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS
2944		 * Translation Requests and Translated transactions are denied
2945		 * as though ATS is disabled for the stream (STE.EATS == 0b00),
2946		 * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events
2947		 * (IHI0070Ea 5.2 Stream Table Entry).
2948		 *
2949		 * However, if we have installed a CD table and are using S1DSS
2950		 * then ATS will work in S1DSS bypass. See "13.6.4 Full ATS
2951		 * skipping stage 1".
2952		 *
2953		 * Disable ATS if we are going to create a normal 0b100 bypass
2954		 * STE.
2955		 */
2956		state->ats_enabled = !state->disable_ats &&
2957				     arm_smmu_ats_supported(master);
2958	}
2959
2960	if (smmu_domain) {
2961		if (new_domain->type == IOMMU_DOMAIN_NESTED) {
2962			ret = arm_smmu_attach_prepare_vmaster(
2963				state, to_smmu_nested_domain(new_domain));
2964			if (ret)
2965				return ret;
2966		}
2967
2968		master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
2969		if (!master_domain) {
2970			ret = -ENOMEM;
2971			goto err_free_vmaster;
2972		}
2973		master_domain->domain = new_domain;
2974		master_domain->master = master;
2975		master_domain->ssid = state->ssid;
2976		if (new_domain->type == IOMMU_DOMAIN_NESTED)
2977			master_domain->nested_ats_flush =
2978				to_smmu_nested_domain(new_domain)->enable_ats;
2979
2980		if (new_domain->iopf_handler) {
2981			ret = arm_smmu_enable_iopf(master, master_domain);
2982			if (ret)
2983				goto err_free_master_domain;
2984		}
2985
2986		/*
2987		 * During prepare we want the current smmu_domain and new
2988		 * smmu_domain to be in the devices list before we change any
2989		 * HW. This ensures that both domains will send ATS
2990		 * invalidations to the master until we are done.
2991		 *
2992		 * It is tempting to make this list only track masters that are
2993		 * using ATS, but arm_smmu_share_asid() also uses this to change
2994		 * the ASID of a domain, unrelated to ATS.
2995		 *
2996		 * Notice if we are re-attaching the same domain then the list
2997		 * will have two identical entries and commit will remove only
2998		 * one of them.
2999		 */
3000		spin_lock_irqsave(&smmu_domain->devices_lock, flags);
3001		if (smmu_domain->enforce_cache_coherency &&
3002		    !arm_smmu_master_canwbs(master)) {
3003			spin_unlock_irqrestore(&smmu_domain->devices_lock,
3004					       flags);
3005			ret = -EINVAL;
3006			goto err_iopf;
3007		}
3008
3009		if (state->ats_enabled)
3010			atomic_inc(&smmu_domain->nr_ats_masters);
3011		list_add(&master_domain->devices_elm, &smmu_domain->devices);
3012		spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
3013	}
3014
3015	if (!state->ats_enabled && master->ats_enabled) {
3016		pci_disable_ats(to_pci_dev(master->dev));
3017		/*
3018		 * This is probably overkill, but the config write for disabling
3019		 * ATS should complete before the STE is configured to generate
3020		 * UR to avoid AER noise.
3021		 */
3022		wmb();
3023	}
3024	return 0;
3025
3026err_iopf:
3027	arm_smmu_disable_iopf(master, master_domain);
3028err_free_master_domain:
3029	kfree(master_domain);
3030err_free_vmaster:
3031	kfree(state->vmaster);
3032	return ret;
3033}
3034
3035/*
3036 * Commit is done after the STE/CD are configured with the EATS setting. It
3037 * completes synchronizing the PCI device's ATC and finishes manipulating the
3038 * smmu_domain->devices list.
3039 */
3040void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
3041{
3042	struct arm_smmu_master *master = state->master;
3043
3044	lockdep_assert_held(&arm_smmu_asid_lock);
3045
3046	arm_smmu_attach_commit_vmaster(state);
3047
3048	if (state->ats_enabled && !master->ats_enabled) {
3049		arm_smmu_enable_ats(master);
3050	} else if (state->ats_enabled && master->ats_enabled) {
3051		/*
3052		 * The translation has changed, flush the ATC. At this point the
3053		 * SMMU is translating for the new domain and both the old&new
3054		 * domain will issue invalidations.
3055		 */
3056		arm_smmu_atc_inv_master(master, state->ssid);
3057	} else if (!state->ats_enabled && master->ats_enabled) {
3058		/* ATS is being switched off, invalidate the entire ATC */
3059		arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
3060	}
3061
3062	arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
3063	master->ats_enabled = state->ats_enabled;
3064}
3065
3066static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev,
3067			       struct iommu_domain *old_domain)
3068{
3069	int ret = 0;
3070	struct arm_smmu_ste target;
3071	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3072	struct arm_smmu_device *smmu;
3073	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3074	struct arm_smmu_attach_state state = {
3075		.old_domain = old_domain,
3076		.ssid = IOMMU_NO_PASID,
3077	};
3078	struct arm_smmu_master *master;
3079	struct arm_smmu_cd *cdptr;
3080
3081	if (!fwspec)
3082		return -ENOENT;
3083
3084	state.master = master = dev_iommu_priv_get(dev);
3085	smmu = master->smmu;
3086
3087	if (smmu_domain->smmu != smmu)
3088		return -EINVAL;
3089
3090	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
3091		cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
3092		if (!cdptr)
3093			return -ENOMEM;
3094	} else if (arm_smmu_ssids_in_use(&master->cd_table))
3095		return -EBUSY;
3096
3097	/*
3098	 * Prevent arm_smmu_share_asid() from trying to change the ASID
3099	 * of either the old or new domain while we are working on it.
3100	 * This allows the STE and the smmu_domain->devices list to
3101	 * be inconsistent during this routine.
3102	 */
3103	mutex_lock(&arm_smmu_asid_lock);
3104
3105	ret = arm_smmu_attach_prepare(&state, domain);
3106	if (ret) {
3107		mutex_unlock(&arm_smmu_asid_lock);
3108		return ret;
3109	}
3110
3111	switch (smmu_domain->stage) {
3112	case ARM_SMMU_DOMAIN_S1: {
3113		struct arm_smmu_cd target_cd;
3114
3115		arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
3116		arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
3117					&target_cd);
3118		arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled,
3119					  STRTAB_STE_1_S1DSS_SSID0);
3120		arm_smmu_install_ste_for_dev(master, &target);
3121		break;
3122	}
3123	case ARM_SMMU_DOMAIN_S2:
3124		arm_smmu_make_s2_domain_ste(&target, master, smmu_domain,
3125					    state.ats_enabled);
3126		arm_smmu_install_ste_for_dev(master, &target);
3127		arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3128		break;
3129	}
3130
3131	arm_smmu_attach_commit(&state);
3132	mutex_unlock(&arm_smmu_asid_lock);
3133	return 0;
3134}
3135
3136static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
3137				     struct device *dev, ioasid_t id,
3138				     struct iommu_domain *old)
3139{
3140	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3141	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3142	struct arm_smmu_device *smmu = master->smmu;
3143	struct arm_smmu_cd target_cd;
3144
3145	if (smmu_domain->smmu != smmu)
3146		return -EINVAL;
3147
3148	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
3149		return -EINVAL;
3150
3151	/*
3152	 * We can read cd.asid outside the lock because arm_smmu_set_pasid()
3153	 * will fix it
3154	 */
3155	arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
3156	return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
3157				  &target_cd, old);
3158}
3159
3160static void arm_smmu_update_ste(struct arm_smmu_master *master,
3161				struct iommu_domain *sid_domain,
3162				bool ats_enabled)
3163{
3164	unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
3165	struct arm_smmu_ste ste;
3166
3167	if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled)
3168		return;
3169
3170	if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
3171		s1dss = STRTAB_STE_1_S1DSS_BYPASS;
3172	else
3173		WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
3174
3175	/*
3176	 * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
3177	 * using s1dss if necessary. If the cd_table is already installed then
3178	 * the S1DSS is correct and this will just update the EATS. Otherwise it
3179	 * installs the entire thing. This will be hitless.
3180	 */
3181	arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss);
3182	arm_smmu_install_ste_for_dev(master, &ste);
3183}
3184
3185int arm_smmu_set_pasid(struct arm_smmu_master *master,
3186		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
3187		       struct arm_smmu_cd *cd, struct iommu_domain *old)
3188{
3189	struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
3190	struct arm_smmu_attach_state state = {
3191		.master = master,
3192		.ssid = pasid,
3193		.old_domain = old,
3194	};
3195	struct arm_smmu_cd *cdptr;
3196	int ret;
3197
3198	/* The core code validates pasid */
3199
3200	if (smmu_domain->smmu != master->smmu)
3201		return -EINVAL;
3202
3203	if (!master->cd_table.in_ste &&
3204	    sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
3205	    sid_domain->type != IOMMU_DOMAIN_BLOCKED)
3206		return -EINVAL;
3207
3208	cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
3209	if (!cdptr)
3210		return -ENOMEM;
3211
3212	mutex_lock(&arm_smmu_asid_lock);
3213	ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
3214	if (ret)
3215		goto out_unlock;
3216
3217	/*
3218	 * We don't want to obtain to the asid_lock too early, so fix up the
3219	 * caller set ASID under the lock in case it changed.
3220	 */
3221	cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID);
3222	cd->data[0] |= cpu_to_le64(
3223		FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid));
3224
3225	arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
3226	arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
3227
3228	arm_smmu_attach_commit(&state);
3229
3230out_unlock:
3231	mutex_unlock(&arm_smmu_asid_lock);
3232	return ret;
3233}
3234
3235static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
3236					   struct device *dev, ioasid_t pasid,
3237					   struct iommu_domain *old_domain)
3238{
3239	struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain);
3240	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3241
3242	mutex_lock(&arm_smmu_asid_lock);
3243	arm_smmu_clear_cd(master, pasid);
3244	if (master->ats_enabled)
3245		arm_smmu_atc_inv_master(master, pasid);
3246	arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
3247	mutex_unlock(&arm_smmu_asid_lock);
3248
3249	/*
3250	 * When the last user of the CD table goes away downgrade the STE back
3251	 * to a non-cd_table one, by re-attaching its sid_domain.
3252	 */
3253	if (!arm_smmu_ssids_in_use(&master->cd_table)) {
3254		struct iommu_domain *sid_domain =
3255			iommu_get_domain_for_dev(master->dev);
3256
3257		if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
3258		    sid_domain->type == IOMMU_DOMAIN_BLOCKED)
3259			sid_domain->ops->attach_dev(sid_domain, dev,
3260						    sid_domain);
3261	}
3262	return 0;
3263}
3264
3265static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
3266				    struct iommu_domain *old_domain,
3267				    struct device *dev,
3268				    struct arm_smmu_ste *ste,
3269				    unsigned int s1dss)
3270{
3271	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3272	struct arm_smmu_attach_state state = {
3273		.master = master,
3274		.old_domain = old_domain,
3275		.ssid = IOMMU_NO_PASID,
3276	};
3277
3278	/*
3279	 * Do not allow any ASID to be changed while are working on the STE,
3280	 * otherwise we could miss invalidations.
3281	 */
3282	mutex_lock(&arm_smmu_asid_lock);
3283
3284	/*
3285	 * If the CD table is not in use we can use the provided STE, otherwise
3286	 * we use a cdtable STE with the provided S1DSS.
3287	 */
3288	if (arm_smmu_ssids_in_use(&master->cd_table)) {
3289		/*
3290		 * If a CD table has to be present then we need to run with ATS
3291		 * on because we have to assume a PASID is using ATS. For
3292		 * IDENTITY this will setup things so that S1DSS=bypass which
3293		 * follows the explanation in "13.6.4 Full ATS skipping stage 1"
3294		 * and allows for ATS on the RID to work.
3295		 */
3296		state.cd_needs_ats = true;
3297		arm_smmu_attach_prepare(&state, domain);
3298		arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss);
3299	} else {
3300		arm_smmu_attach_prepare(&state, domain);
3301	}
3302	arm_smmu_install_ste_for_dev(master, ste);
3303	arm_smmu_attach_commit(&state);
3304	mutex_unlock(&arm_smmu_asid_lock);
3305
3306	/*
3307	 * This has to be done after removing the master from the
3308	 * arm_smmu_domain->devices to avoid races updating the same context
3309	 * descriptor from arm_smmu_share_asid().
3310	 */
3311	arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3312}
3313
3314static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
3315					struct device *dev,
3316					struct iommu_domain *old_domain)
3317{
3318	struct arm_smmu_ste ste;
3319	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3320
3321	arm_smmu_master_clear_vmaster(master);
3322	arm_smmu_make_bypass_ste(master->smmu, &ste);
3323	arm_smmu_attach_dev_ste(domain, old_domain, dev, &ste,
3324				STRTAB_STE_1_S1DSS_BYPASS);
3325	return 0;
3326}
3327
3328static const struct iommu_domain_ops arm_smmu_identity_ops = {
3329	.attach_dev = arm_smmu_attach_dev_identity,
3330};
3331
3332static struct iommu_domain arm_smmu_identity_domain = {
3333	.type = IOMMU_DOMAIN_IDENTITY,
3334	.ops = &arm_smmu_identity_ops,
3335};
3336
3337static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
3338				       struct device *dev,
3339				       struct iommu_domain *old_domain)
3340{
3341	struct arm_smmu_ste ste;
3342	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3343
3344	arm_smmu_master_clear_vmaster(master);
3345	arm_smmu_make_abort_ste(&ste);
3346	arm_smmu_attach_dev_ste(domain, old_domain, dev, &ste,
3347				STRTAB_STE_1_S1DSS_TERMINATE);
3348	return 0;
3349}
3350
3351static const struct iommu_domain_ops arm_smmu_blocked_ops = {
3352	.attach_dev = arm_smmu_attach_dev_blocked,
3353	.set_dev_pasid = arm_smmu_blocking_set_dev_pasid,
3354};
3355
3356static struct iommu_domain arm_smmu_blocked_domain = {
3357	.type = IOMMU_DOMAIN_BLOCKED,
3358	.ops = &arm_smmu_blocked_ops,
3359};
3360
3361static struct iommu_domain *
3362arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags,
3363				   const struct iommu_user_data *user_data)
3364{
3365	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3366	struct arm_smmu_device *smmu = master->smmu;
3367	const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
3368				 IOMMU_HWPT_ALLOC_PASID |
3369				 IOMMU_HWPT_ALLOC_NEST_PARENT;
3370	struct arm_smmu_domain *smmu_domain;
3371	int ret;
3372
3373	if (flags & ~PAGING_FLAGS)
3374		return ERR_PTR(-EOPNOTSUPP);
3375	if (user_data)
3376		return ERR_PTR(-EOPNOTSUPP);
3377
3378	smmu_domain = arm_smmu_domain_alloc();
3379	if (IS_ERR(smmu_domain))
3380		return ERR_CAST(smmu_domain);
3381
3382	switch (flags) {
3383	case 0:
3384		/* Prefer S1 if available */
3385		if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
3386			smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3387		else
3388			smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3389		break;
3390	case IOMMU_HWPT_ALLOC_NEST_PARENT:
3391		if (!(smmu->features & ARM_SMMU_FEAT_NESTING)) {
3392			ret = -EOPNOTSUPP;
3393			goto err_free;
3394		}
3395		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3396		smmu_domain->nest_parent = true;
3397		break;
3398	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
3399	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_PASID:
3400	case IOMMU_HWPT_ALLOC_PASID:
3401		if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) {
3402			ret = -EOPNOTSUPP;
3403			goto err_free;
3404		}
3405		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3406		break;
3407	default:
3408		ret = -EOPNOTSUPP;
3409		goto err_free;
3410	}
3411
3412	smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
3413	smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
3414	ret = arm_smmu_domain_finalise(smmu_domain, smmu, flags);
3415	if (ret)
3416		goto err_free;
3417	return &smmu_domain->domain;
3418
3419err_free:
3420	kfree(smmu_domain);
3421	return ERR_PTR(ret);
3422}
3423
3424static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
3425			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
3426			      int prot, gfp_t gfp, size_t *mapped)
3427{
3428	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3429
3430	if (!ops)
3431		return -ENODEV;
3432
3433	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
3434}
3435
3436static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
3437				   size_t pgsize, size_t pgcount,
3438				   struct iommu_iotlb_gather *gather)
3439{
3440	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3441	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3442
3443	if (!ops)
3444		return 0;
3445
3446	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
3447}
3448
3449static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
3450{
3451	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3452
3453	if (smmu_domain->smmu)
3454		arm_smmu_tlb_inv_context(smmu_domain);
3455}
3456
3457static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
3458				struct iommu_iotlb_gather *gather)
3459{
3460	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3461
3462	if (!gather->pgsize)
3463		return;
3464
3465	arm_smmu_tlb_inv_range_domain(gather->start,
3466				      gather->end - gather->start + 1,
3467				      gather->pgsize, true, smmu_domain);
3468}
3469
3470static phys_addr_t
3471arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
3472{
3473	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3474
3475	if (!ops)
3476		return 0;
3477
3478	return ops->iova_to_phys(ops, iova);
3479}
3480
3481static struct platform_driver arm_smmu_driver;
3482
3483static
3484struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
3485{
3486	struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
3487
3488	put_device(dev);
3489	return dev ? dev_get_drvdata(dev) : NULL;
3490}
3491
3492static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
3493{
3494	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3495		return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents;
3496	return sid < smmu->strtab_cfg.linear.num_ents;
3497}
3498
3499static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
3500{
3501	/* Check the SIDs are in range of the SMMU and our stream table */
3502	if (!arm_smmu_sid_in_range(smmu, sid))
3503		return -ERANGE;
3504
3505	/* Ensure l2 strtab is initialised */
3506	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3507		return arm_smmu_init_l2_strtab(smmu, sid);
3508
3509	return 0;
3510}
3511
3512static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
3513				  struct arm_smmu_master *master)
3514{
3515	int i;
3516	int ret = 0;
3517	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3518
3519	master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
3520				  GFP_KERNEL);
3521	if (!master->streams)
3522		return -ENOMEM;
3523	master->num_streams = fwspec->num_ids;
3524
3525	mutex_lock(&smmu->streams_mutex);
3526	for (i = 0; i < fwspec->num_ids; i++) {
3527		struct arm_smmu_stream *new_stream = &master->streams[i];
3528		struct rb_node *existing;
3529		u32 sid = fwspec->ids[i];
3530
3531		new_stream->id = sid;
3532		new_stream->master = master;
3533
3534		ret = arm_smmu_init_sid_strtab(smmu, sid);
3535		if (ret)
3536			break;
3537
3538		/* Insert into SID tree */
3539		existing = rb_find_add(&new_stream->node, &smmu->streams,
3540				       arm_smmu_streams_cmp_node);
3541		if (existing) {
3542			struct arm_smmu_master *existing_master =
3543				rb_entry(existing, struct arm_smmu_stream, node)
3544					->master;
3545
3546			/* Bridged PCI devices may end up with duplicated IDs */
3547			if (existing_master == master)
3548				continue;
3549
3550			dev_warn(master->dev,
3551				 "Aliasing StreamID 0x%x (from %s) unsupported, expect DMA to be broken\n",
3552				 sid, dev_name(existing_master->dev));
3553			ret = -ENODEV;
3554			break;
3555		}
3556	}
3557
3558	if (ret) {
3559		for (i--; i >= 0; i--)
3560			rb_erase(&master->streams[i].node, &smmu->streams);
3561		kfree(master->streams);
3562	}
3563	mutex_unlock(&smmu->streams_mutex);
3564
3565	return ret;
3566}
3567
3568static void arm_smmu_remove_master(struct arm_smmu_master *master)
3569{
3570	int i;
3571	struct arm_smmu_device *smmu = master->smmu;
3572	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3573
3574	if (!smmu || !master->streams)
3575		return;
3576
3577	mutex_lock(&smmu->streams_mutex);
3578	for (i = 0; i < fwspec->num_ids; i++)
3579		rb_erase(&master->streams[i].node, &smmu->streams);
3580	mutex_unlock(&smmu->streams_mutex);
3581
3582	kfree(master->streams);
3583}
3584
3585static struct iommu_device *arm_smmu_probe_device(struct device *dev)
3586{
3587	int ret;
3588	struct arm_smmu_device *smmu;
3589	struct arm_smmu_master *master;
3590	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3591
3592	if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
3593		return ERR_PTR(-EBUSY);
3594
3595	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
3596	if (!smmu)
3597		return ERR_PTR(-ENODEV);
3598
3599	master = kzalloc(sizeof(*master), GFP_KERNEL);
3600	if (!master)
3601		return ERR_PTR(-ENOMEM);
3602
3603	master->dev = dev;
3604	master->smmu = smmu;
3605	dev_iommu_priv_set(dev, master);
3606
3607	ret = arm_smmu_insert_master(smmu, master);
3608	if (ret)
3609		goto err_free_master;
3610
3611	device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
3612	master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
3613
3614	/*
3615	 * Note that PASID must be enabled before, and disabled after ATS:
3616	 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
3617	 *
3618	 *   Behavior is undefined if this bit is Set and the value of the PASID
3619	 *   Enable, Execute Requested Enable, or Privileged Mode Requested bits
3620	 *   are changed.
3621	 */
3622	arm_smmu_enable_pasid(master);
3623
3624	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
3625		master->ssid_bits = min_t(u8, master->ssid_bits,
3626					  CTXDESC_LINEAR_CDMAX);
3627
3628	if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
3629	     device_property_read_bool(dev, "dma-can-stall")) ||
3630	    smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
3631		master->stall_enabled = true;
3632
3633	if (dev_is_pci(dev)) {
3634		unsigned int stu = __ffs(smmu->pgsize_bitmap);
3635
3636		pci_prepare_ats(to_pci_dev(dev), stu);
3637	}
3638
3639	return &smmu->iommu;
3640
3641err_free_master:
3642	kfree(master);
3643	return ERR_PTR(ret);
3644}
3645
3646static void arm_smmu_release_device(struct device *dev)
3647{
3648	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3649
3650	WARN_ON(master->iopf_refcount);
3651
3652	arm_smmu_disable_pasid(master);
3653	arm_smmu_remove_master(master);
3654	if (arm_smmu_cdtab_allocated(&master->cd_table))
3655		arm_smmu_free_cd_tables(master);
3656	kfree(master);
3657}
3658
3659static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain,
3660					 unsigned long iova, size_t size,
3661					 unsigned long flags,
3662					 struct iommu_dirty_bitmap *dirty)
3663{
3664	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3665	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3666
3667	return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
3668}
3669
3670static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain,
3671				       bool enabled)
3672{
3673	/*
3674	 * Always enabled and the dirty bitmap is cleared prior to
3675	 * set_dirty_tracking().
3676	 */
3677	return 0;
3678}
3679
3680static struct iommu_group *arm_smmu_device_group(struct device *dev)
3681{
3682	struct iommu_group *group;
3683
3684	/*
3685	 * We don't support devices sharing stream IDs other than PCI RID
3686	 * aliases, since the necessary ID-to-device lookup becomes rather
3687	 * impractical given a potential sparse 32-bit stream ID space.
3688	 */
3689	if (dev_is_pci(dev))
3690		group = pci_device_group(dev);
3691	else
3692		group = generic_device_group(dev);
3693
3694	return group;
3695}
3696
3697static int arm_smmu_of_xlate(struct device *dev,
3698			     const struct of_phandle_args *args)
3699{
3700	return iommu_fwspec_add_ids(dev, args->args, 1);
3701}
3702
3703static void arm_smmu_get_resv_regions(struct device *dev,
3704				      struct list_head *head)
3705{
3706	struct iommu_resv_region *region;
3707	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
3708
3709	region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
3710					 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
3711	if (!region)
3712		return;
3713
3714	list_add_tail(&region->list, head);
3715
3716	iommu_dma_get_resv_regions(dev, head);
3717}
3718
3719/*
3720 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
3721 * PCIe link and save the data to memory by DMA. The hardware is restricted to
3722 * use identity mapping only.
3723 */
3724#define IS_HISI_PTT_DEVICE(pdev)	((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
3725					 (pdev)->device == 0xa12e)
3726
3727static int arm_smmu_def_domain_type(struct device *dev)
3728{
3729	if (dev_is_pci(dev)) {
3730		struct pci_dev *pdev = to_pci_dev(dev);
3731
3732		if (IS_HISI_PTT_DEVICE(pdev))
3733			return IOMMU_DOMAIN_IDENTITY;
3734	}
3735
3736	return 0;
3737}
3738
3739static const struct iommu_ops arm_smmu_ops = {
3740	.identity_domain	= &arm_smmu_identity_domain,
3741	.blocked_domain		= &arm_smmu_blocked_domain,
3742	.release_domain		= &arm_smmu_blocked_domain,
3743	.capable		= arm_smmu_capable,
3744	.hw_info		= arm_smmu_hw_info,
3745	.domain_alloc_sva       = arm_smmu_sva_domain_alloc,
3746	.domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags,
3747	.probe_device		= arm_smmu_probe_device,
3748	.release_device		= arm_smmu_release_device,
3749	.device_group		= arm_smmu_device_group,
3750	.of_xlate		= arm_smmu_of_xlate,
3751	.get_resv_regions	= arm_smmu_get_resv_regions,
3752	.page_response		= arm_smmu_page_response,
3753	.def_domain_type	= arm_smmu_def_domain_type,
3754	.get_viommu_size	= arm_smmu_get_viommu_size,
3755	.viommu_init		= arm_vsmmu_init,
3756	.user_pasid_table	= 1,
3757	.owner			= THIS_MODULE,
3758	.default_domain_ops = &(const struct iommu_domain_ops) {
3759		.attach_dev		= arm_smmu_attach_dev,
3760		.enforce_cache_coherency = arm_smmu_enforce_cache_coherency,
3761		.set_dev_pasid		= arm_smmu_s1_set_dev_pasid,
3762		.map_pages		= arm_smmu_map_pages,
3763		.unmap_pages		= arm_smmu_unmap_pages,
3764		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
3765		.iotlb_sync		= arm_smmu_iotlb_sync,
3766		.iova_to_phys		= arm_smmu_iova_to_phys,
3767		.free			= arm_smmu_domain_free_paging,
3768	}
3769};
3770
3771static struct iommu_dirty_ops arm_smmu_dirty_ops = {
3772	.read_and_clear_dirty	= arm_smmu_read_and_clear_dirty,
3773	.set_dirty_tracking     = arm_smmu_set_dirty_tracking,
3774};
3775
3776/* Probing and initialisation functions */
3777int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3778			    struct arm_smmu_queue *q, void __iomem *page,
3779			    unsigned long prod_off, unsigned long cons_off,
3780			    size_t dwords, const char *name)
3781{
3782	size_t qsz;
3783
3784	do {
3785		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3786		q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3787					      GFP_KERNEL);
3788		if (q->base || qsz < PAGE_SIZE)
3789			break;
3790
3791		q->llq.max_n_shift--;
3792	} while (1);
3793
3794	if (!q->base) {
3795		dev_err(smmu->dev,
3796			"failed to allocate queue (0x%zx bytes) for %s\n",
3797			qsz, name);
3798		return -ENOMEM;
3799	}
3800
3801	if (!WARN_ON(q->base_dma & (qsz - 1))) {
3802		dev_info(smmu->dev, "allocated %u entries for %s\n",
3803			 1 << q->llq.max_n_shift, name);
3804	}
3805
3806	q->prod_reg	= page + prod_off;
3807	q->cons_reg	= page + cons_off;
3808	q->ent_dwords	= dwords;
3809
3810	q->q_base  = Q_BASE_RWA;
3811	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3812	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3813
3814	q->llq.prod = q->llq.cons = 0;
3815	return 0;
3816}
3817
3818int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
3819		       struct arm_smmu_cmdq *cmdq)
3820{
3821	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3822
3823	atomic_set(&cmdq->owner_prod, 0);
3824	atomic_set(&cmdq->lock, 0);
3825
3826	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
3827							      GFP_KERNEL);
3828	if (!cmdq->valid_map)
3829		return -ENOMEM;
3830
3831	return 0;
3832}
3833
3834static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
3835{
3836	int ret;
3837
3838	/* cmdq */
3839	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
3840				      ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
3841				      CMDQ_ENT_DWORDS, "cmdq");
3842	if (ret)
3843		return ret;
3844
3845	ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq);
3846	if (ret)
3847		return ret;
3848
3849	/* evtq */
3850	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
3851				      ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
3852				      EVTQ_ENT_DWORDS, "evtq");
3853	if (ret)
3854		return ret;
3855
3856	if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
3857	    (smmu->features & ARM_SMMU_FEAT_STALLS)) {
3858		smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
3859		if (!smmu->evtq.iopf)
3860			return -ENOMEM;
3861	}
3862
3863	/* priq */
3864	if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3865		return 0;
3866
3867	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
3868				       ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
3869				       PRIQ_ENT_DWORDS, "priq");
3870}
3871
3872static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3873{
3874	u32 l1size;
3875	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3876	unsigned int last_sid_idx =
3877		arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1);
3878
3879	/* Calculate the L1 size, capped to the SIDSIZE. */
3880	cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
3881	if (cfg->l2.num_l1_ents <= last_sid_idx)
3882		dev_warn(smmu->dev,
3883			 "2-level strtab only covers %u/%u bits of SID\n",
3884			 ilog2(cfg->l2.num_l1_ents * STRTAB_NUM_L2_STES),
3885			 smmu->sid_bits);
3886
3887	l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
3888	cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma,
3889					    GFP_KERNEL);
3890	if (!cfg->l2.l1tab) {
3891		dev_err(smmu->dev,
3892			"failed to allocate l1 stream table (%u bytes)\n",
3893			l1size);
3894		return -ENOMEM;
3895	}
3896
3897	cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents,
3898				      sizeof(*cfg->l2.l2ptrs), GFP_KERNEL);
3899	if (!cfg->l2.l2ptrs)
3900		return -ENOMEM;
3901
3902	return 0;
3903}
3904
3905static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3906{
3907	u32 size;
3908	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3909
3910	size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
3911	cfg->linear.table = dmam_alloc_coherent(smmu->dev, size,
3912						&cfg->linear.ste_dma,
3913						GFP_KERNEL);
3914	if (!cfg->linear.table) {
3915		dev_err(smmu->dev,
3916			"failed to allocate linear stream table (%u bytes)\n",
3917			size);
3918		return -ENOMEM;
3919	}
3920	cfg->linear.num_ents = 1 << smmu->sid_bits;
3921
3922	arm_smmu_init_initial_stes(cfg->linear.table, cfg->linear.num_ents);
3923	return 0;
3924}
3925
3926static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3927{
3928	int ret;
3929
3930	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3931		ret = arm_smmu_init_strtab_2lvl(smmu);
3932	else
3933		ret = arm_smmu_init_strtab_linear(smmu);
3934	if (ret)
3935		return ret;
3936
3937	ida_init(&smmu->vmid_map);
3938
3939	return 0;
3940}
3941
3942static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3943{
3944	int ret;
3945
3946	mutex_init(&smmu->streams_mutex);
3947	smmu->streams = RB_ROOT;
3948
3949	ret = arm_smmu_init_queues(smmu);
3950	if (ret)
3951		return ret;
3952
3953	ret = arm_smmu_init_strtab(smmu);
3954	if (ret)
3955		return ret;
3956
3957	if (smmu->impl_ops && smmu->impl_ops->init_structures)
3958		return smmu->impl_ops->init_structures(smmu);
3959
3960	return 0;
3961}
3962
3963static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3964				   unsigned int reg_off, unsigned int ack_off)
3965{
3966	u32 reg;
3967
3968	writel_relaxed(val, smmu->base + reg_off);
3969	return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3970					  1, ARM_SMMU_POLL_TIMEOUT_US);
3971}
3972
3973/* GBPA is "special" */
3974static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3975{
3976	int ret;
3977	u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3978
3979	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3980					 1, ARM_SMMU_POLL_TIMEOUT_US);
3981	if (ret)
3982		return ret;
3983
3984	reg &= ~clr;
3985	reg |= set;
3986	writel_relaxed(reg | GBPA_UPDATE, gbpa);
3987	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3988					 1, ARM_SMMU_POLL_TIMEOUT_US);
3989
3990	if (ret)
3991		dev_err(smmu->dev, "GBPA not responding to update\n");
3992	return ret;
3993}
3994
3995static void arm_smmu_free_msis(void *data)
3996{
3997	struct device *dev = data;
3998
3999	platform_device_msi_free_irqs_all(dev);
4000}
4001
4002static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
4003{
4004	phys_addr_t doorbell;
4005	struct device *dev = msi_desc_to_dev(desc);
4006	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
4007	phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
4008
4009	doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
4010	doorbell &= MSI_CFG0_ADDR_MASK;
4011
4012	writeq_relaxed(doorbell, smmu->base + cfg[0]);
4013	writel_relaxed(msg->data, smmu->base + cfg[1]);
4014	writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
4015}
4016
4017static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
4018{
4019	int ret, nvec = ARM_SMMU_MAX_MSIS;
4020	struct device *dev = smmu->dev;
4021
4022	/* Clear the MSI address regs */
4023	writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
4024	writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
4025
4026	if (smmu->features & ARM_SMMU_FEAT_PRI)
4027		writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
4028	else
4029		nvec--;
4030
4031	if (!(smmu->features & ARM_SMMU_FEAT_MSI))
4032		return;
4033
4034	if (!dev->msi.domain) {
4035		dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
4036		return;
4037	}
4038
4039	/* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
4040	ret = platform_device_msi_init_and_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
4041	if (ret) {
4042		dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
4043		return;
4044	}
4045
4046	smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
4047	smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
4048	smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
4049
4050	/* Add callback to free MSIs on teardown */
4051	devm_add_action_or_reset(dev, arm_smmu_free_msis, dev);
4052}
4053
4054static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
4055{
4056	int irq, ret;
4057
4058	arm_smmu_setup_msis(smmu);
4059
4060	/* Request interrupt lines */
4061	irq = smmu->evtq.q.irq;
4062	if (irq) {
4063		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
4064						arm_smmu_evtq_thread,
4065						IRQF_ONESHOT,
4066						"arm-smmu-v3-evtq", smmu);
4067		if (ret < 0)
4068			dev_warn(smmu->dev, "failed to enable evtq irq\n");
4069	} else {
4070		dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
4071	}
4072
4073	irq = smmu->gerr_irq;
4074	if (irq) {
4075		ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
4076				       0, "arm-smmu-v3-gerror", smmu);
4077		if (ret < 0)
4078			dev_warn(smmu->dev, "failed to enable gerror irq\n");
4079	} else {
4080		dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
4081	}
4082
4083	if (smmu->features & ARM_SMMU_FEAT_PRI) {
4084		irq = smmu->priq.q.irq;
4085		if (irq) {
4086			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
4087							arm_smmu_priq_thread,
4088							IRQF_ONESHOT,
4089							"arm-smmu-v3-priq",
4090							smmu);
4091			if (ret < 0)
4092				dev_warn(smmu->dev,
4093					 "failed to enable priq irq\n");
4094		} else {
4095			dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
4096		}
4097	}
4098}
4099
4100static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
4101{
4102	int ret, irq;
4103	u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
4104
4105	/* Disable IRQs first */
4106	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
4107				      ARM_SMMU_IRQ_CTRLACK);
4108	if (ret) {
4109		dev_err(smmu->dev, "failed to disable irqs\n");
4110		return ret;
4111	}
4112
4113	irq = smmu->combined_irq;
4114	if (irq) {
4115		/*
4116		 * Cavium ThunderX2 implementation doesn't support unique irq
4117		 * lines. Use a single irq line for all the SMMUv3 interrupts.
4118		 */
4119		ret = devm_request_threaded_irq(smmu->dev, irq,
4120					arm_smmu_combined_irq_handler,
4121					arm_smmu_combined_irq_thread,
4122					IRQF_ONESHOT,
4123					"arm-smmu-v3-combined-irq", smmu);
4124		if (ret < 0)
4125			dev_warn(smmu->dev, "failed to enable combined irq\n");
4126	} else
4127		arm_smmu_setup_unique_irqs(smmu);
4128
4129	if (smmu->features & ARM_SMMU_FEAT_PRI)
4130		irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
4131
4132	/* Enable interrupt generation on the SMMU */
4133	ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
4134				      ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
4135	if (ret)
4136		dev_warn(smmu->dev, "failed to enable irqs\n");
4137
4138	return 0;
4139}
4140
4141static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
4142{
4143	int ret;
4144
4145	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
4146	if (ret)
4147		dev_err(smmu->dev, "failed to clear cr0\n");
4148
4149	return ret;
4150}
4151
4152static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
4153{
4154	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
4155	dma_addr_t dma;
4156	u32 reg;
4157
4158	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
4159		reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4160				 STRTAB_BASE_CFG_FMT_2LVL) |
4161		      FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE,
4162				 ilog2(cfg->l2.num_l1_ents) + STRTAB_SPLIT) |
4163		      FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
4164		dma = cfg->l2.l1_dma;
4165	} else {
4166		reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4167				 STRTAB_BASE_CFG_FMT_LINEAR) |
4168		      FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
4169		dma = cfg->linear.ste_dma;
4170	}
4171	writeq_relaxed((dma & STRTAB_BASE_ADDR_MASK) | STRTAB_BASE_RA,
4172		       smmu->base + ARM_SMMU_STRTAB_BASE);
4173	writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
4174}
4175
4176static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
4177{
4178	int ret;
4179	u32 reg, enables;
4180	struct arm_smmu_cmdq_ent cmd;
4181
4182	/* Clear CR0 and sync (disables SMMU and queue processing) */
4183	reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
4184	if (reg & CR0_SMMUEN) {
4185		dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
4186		arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
4187	}
4188
4189	ret = arm_smmu_device_disable(smmu);
4190	if (ret)
4191		return ret;
4192
4193	/* CR1 (table and queue memory attributes) */
4194	reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
4195	      FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
4196	      FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
4197	      FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
4198	      FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
4199	      FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
4200	writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
4201
4202	/* CR2 (random crap) */
4203	reg = CR2_PTM | CR2_RECINVSID;
4204
4205	if (smmu->features & ARM_SMMU_FEAT_E2H)
4206		reg |= CR2_E2H;
4207
4208	writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
4209
4210	/* Stream table */
4211	arm_smmu_write_strtab(smmu);
4212
4213	/* Command queue */
4214	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
4215	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
4216	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
4217
4218	enables = CR0_CMDQEN;
4219	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4220				      ARM_SMMU_CR0ACK);
4221	if (ret) {
4222		dev_err(smmu->dev, "failed to enable command queue\n");
4223		return ret;
4224	}
4225
4226	/* Invalidate any cached configuration */
4227	cmd.opcode = CMDQ_OP_CFGI_ALL;
4228	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4229
4230	/* Invalidate any stale TLB entries */
4231	if (smmu->features & ARM_SMMU_FEAT_HYP) {
4232		cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
4233		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4234	}
4235
4236	cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
4237	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4238
4239	/* Event queue */
4240	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
4241	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
4242	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
4243
4244	enables |= CR0_EVTQEN;
4245	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4246				      ARM_SMMU_CR0ACK);
4247	if (ret) {
4248		dev_err(smmu->dev, "failed to enable event queue\n");
4249		return ret;
4250	}
4251
4252	/* PRI queue */
4253	if (smmu->features & ARM_SMMU_FEAT_PRI) {
4254		writeq_relaxed(smmu->priq.q.q_base,
4255			       smmu->base + ARM_SMMU_PRIQ_BASE);
4256		writel_relaxed(smmu->priq.q.llq.prod,
4257			       smmu->page1 + ARM_SMMU_PRIQ_PROD);
4258		writel_relaxed(smmu->priq.q.llq.cons,
4259			       smmu->page1 + ARM_SMMU_PRIQ_CONS);
4260
4261		enables |= CR0_PRIQEN;
4262		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4263					      ARM_SMMU_CR0ACK);
4264		if (ret) {
4265			dev_err(smmu->dev, "failed to enable PRI queue\n");
4266			return ret;
4267		}
4268	}
4269
4270	if (smmu->features & ARM_SMMU_FEAT_ATS) {
4271		enables |= CR0_ATSCHK;
4272		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4273					      ARM_SMMU_CR0ACK);
4274		if (ret) {
4275			dev_err(smmu->dev, "failed to enable ATS check\n");
4276			return ret;
4277		}
4278	}
4279
4280	ret = arm_smmu_setup_irqs(smmu);
4281	if (ret) {
4282		dev_err(smmu->dev, "failed to setup irqs\n");
4283		return ret;
4284	}
4285
4286	if (is_kdump_kernel())
4287		enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
4288
4289	/* Enable the SMMU interface */
4290	enables |= CR0_SMMUEN;
4291	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4292				      ARM_SMMU_CR0ACK);
4293	if (ret) {
4294		dev_err(smmu->dev, "failed to enable SMMU interface\n");
4295		return ret;
4296	}
4297
4298	if (smmu->impl_ops && smmu->impl_ops->device_reset) {
4299		ret = smmu->impl_ops->device_reset(smmu);
4300		if (ret) {
4301			dev_err(smmu->dev, "failed to reset impl\n");
4302			return ret;
4303		}
4304	}
4305
4306	return 0;
4307}
4308
4309#define IIDR_IMPLEMENTER_ARM		0x43b
4310#define IIDR_PRODUCTID_ARM_MMU_600	0x483
4311#define IIDR_PRODUCTID_ARM_MMU_700	0x487
4312
4313static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
4314{
4315	u32 reg;
4316	unsigned int implementer, productid, variant, revision;
4317
4318	reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
4319	implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
4320	productid = FIELD_GET(IIDR_PRODUCTID, reg);
4321	variant = FIELD_GET(IIDR_VARIANT, reg);
4322	revision = FIELD_GET(IIDR_REVISION, reg);
4323
4324	switch (implementer) {
4325	case IIDR_IMPLEMENTER_ARM:
4326		switch (productid) {
4327		case IIDR_PRODUCTID_ARM_MMU_600:
4328			/* Arm erratum 1076982 */
4329			if (variant == 0 && revision <= 2)
4330				smmu->features &= ~ARM_SMMU_FEAT_SEV;
4331			/* Arm erratum 1209401 */
4332			if (variant < 2)
4333				smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4334			break;
4335		case IIDR_PRODUCTID_ARM_MMU_700:
4336			/* Arm erratum 2812531 */
4337			smmu->features &= ~ARM_SMMU_FEAT_BTM;
4338			smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
4339			/* Arm errata 2268618, 2812531 */
4340			smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4341			break;
4342		}
4343		break;
4344	}
4345}
4346
4347static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
4348{
4349	u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
4350	u32 hw_features = 0;
4351
4352	switch (FIELD_GET(IDR0_HTTU, reg)) {
4353	case IDR0_HTTU_ACCESS_DIRTY:
4354		hw_features |= ARM_SMMU_FEAT_HD;
4355		fallthrough;
4356	case IDR0_HTTU_ACCESS:
4357		hw_features |= ARM_SMMU_FEAT_HA;
4358	}
4359
4360	if (smmu->dev->of_node)
4361		smmu->features |= hw_features;
4362	else if (hw_features != fw_features)
4363		/* ACPI IORT sets the HTTU bits */
4364		dev_warn(smmu->dev,
4365			 "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n",
4366			  hw_features, fw_features);
4367}
4368
4369static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
4370{
4371	u32 reg;
4372	bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
4373
4374	/* IDR0 */
4375	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
4376
4377	/* 2-level structures */
4378	if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
4379		smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
4380
4381	if (reg & IDR0_CD2L)
4382		smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
4383
4384	/*
4385	 * Translation table endianness.
4386	 * We currently require the same endianness as the CPU, but this
4387	 * could be changed later by adding a new IO_PGTABLE_QUIRK.
4388	 */
4389	switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
4390	case IDR0_TTENDIAN_MIXED:
4391		smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
4392		break;
4393#ifdef __BIG_ENDIAN
4394	case IDR0_TTENDIAN_BE:
4395		smmu->features |= ARM_SMMU_FEAT_TT_BE;
4396		break;
4397#else
4398	case IDR0_TTENDIAN_LE:
4399		smmu->features |= ARM_SMMU_FEAT_TT_LE;
4400		break;
4401#endif
4402	default:
4403		dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
4404		return -ENXIO;
4405	}
4406
4407	/* Boolean feature flags */
4408	if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
4409		smmu->features |= ARM_SMMU_FEAT_PRI;
4410
4411	if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
4412		smmu->features |= ARM_SMMU_FEAT_ATS;
4413
4414	if (reg & IDR0_SEV)
4415		smmu->features |= ARM_SMMU_FEAT_SEV;
4416
4417	if (reg & IDR0_MSI) {
4418		smmu->features |= ARM_SMMU_FEAT_MSI;
4419		if (coherent && !disable_msipolling)
4420			smmu->options |= ARM_SMMU_OPT_MSIPOLL;
4421	}
4422
4423	if (reg & IDR0_HYP) {
4424		smmu->features |= ARM_SMMU_FEAT_HYP;
4425		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
4426			smmu->features |= ARM_SMMU_FEAT_E2H;
4427	}
4428
4429	arm_smmu_get_httu(smmu, reg);
4430
4431	/*
4432	 * The coherency feature as set by FW is used in preference to the ID
4433	 * register, but warn on mismatch.
4434	 */
4435	if (!!(reg & IDR0_COHACC) != coherent)
4436		dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
4437			 str_true_false(coherent));
4438
4439	switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
4440	case IDR0_STALL_MODEL_FORCE:
4441		smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
4442		fallthrough;
4443	case IDR0_STALL_MODEL_STALL:
4444		smmu->features |= ARM_SMMU_FEAT_STALLS;
4445	}
4446
4447	if (reg & IDR0_S1P)
4448		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
4449
4450	if (reg & IDR0_S2P)
4451		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
4452
4453	if (!(reg & (IDR0_S1P | IDR0_S2P))) {
4454		dev_err(smmu->dev, "no translation support!\n");
4455		return -ENXIO;
4456	}
4457
4458	/* We only support the AArch64 table format at present */
4459	switch (FIELD_GET(IDR0_TTF, reg)) {
4460	case IDR0_TTF_AARCH32_64:
4461		smmu->ias = 40;
4462		fallthrough;
4463	case IDR0_TTF_AARCH64:
4464		break;
4465	default:
4466		dev_err(smmu->dev, "AArch64 table format not supported!\n");
4467		return -ENXIO;
4468	}
4469
4470	/* ASID/VMID sizes */
4471	smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
4472	smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
4473
4474	/* IDR1 */
4475	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
4476	if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
4477		dev_err(smmu->dev, "embedded implementation not supported\n");
4478		return -ENXIO;
4479	}
4480
4481	if (reg & IDR1_ATTR_TYPES_OVR)
4482		smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
4483
4484	/* Queue sizes, capped to ensure natural alignment */
4485	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
4486					     FIELD_GET(IDR1_CMDQS, reg));
4487	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
4488		/*
4489		 * We don't support splitting up batches, so one batch of
4490		 * commands plus an extra sync needs to fit inside the command
4491		 * queue. There's also no way we can handle the weird alignment
4492		 * restrictions on the base pointer for a unit-length queue.
4493		 */
4494		dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
4495			CMDQ_BATCH_ENTRIES);
4496		return -ENXIO;
4497	}
4498
4499	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
4500					     FIELD_GET(IDR1_EVTQS, reg));
4501	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
4502					     FIELD_GET(IDR1_PRIQS, reg));
4503
4504	/* SID/SSID sizes */
4505	smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
4506	smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
4507	smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
4508
4509	/*
4510	 * If the SMMU supports fewer bits than would fill a single L2 stream
4511	 * table, use a linear table instead.
4512	 */
4513	if (smmu->sid_bits <= STRTAB_SPLIT)
4514		smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
4515
4516	/* IDR3 */
4517	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
4518	if (FIELD_GET(IDR3_RIL, reg))
4519		smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
4520	if (FIELD_GET(IDR3_FWB, reg))
4521		smmu->features |= ARM_SMMU_FEAT_S2FWB;
4522
4523	if (FIELD_GET(IDR3_BBM, reg) == 2)
4524		smmu->features |= ARM_SMMU_FEAT_BBML2;
4525
4526	/* IDR5 */
4527	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
4528
4529	/* Maximum number of outstanding stalls */
4530	smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
4531
4532	/* Page sizes */
4533	if (reg & IDR5_GRAN64K)
4534		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
4535	if (reg & IDR5_GRAN16K)
4536		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
4537	if (reg & IDR5_GRAN4K)
4538		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
4539
4540	/* Input address size */
4541	if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
4542		smmu->features |= ARM_SMMU_FEAT_VAX;
4543
4544	/* Output address size */
4545	switch (FIELD_GET(IDR5_OAS, reg)) {
4546	case IDR5_OAS_32_BIT:
4547		smmu->oas = 32;
4548		break;
4549	case IDR5_OAS_36_BIT:
4550		smmu->oas = 36;
4551		break;
4552	case IDR5_OAS_40_BIT:
4553		smmu->oas = 40;
4554		break;
4555	case IDR5_OAS_42_BIT:
4556		smmu->oas = 42;
4557		break;
4558	case IDR5_OAS_44_BIT:
4559		smmu->oas = 44;
4560		break;
4561	case IDR5_OAS_52_BIT:
4562		smmu->oas = 52;
4563		smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
4564		break;
4565	default:
4566		dev_info(smmu->dev,
4567			"unknown output address size. Truncating to 48-bit\n");
4568		fallthrough;
4569	case IDR5_OAS_48_BIT:
4570		smmu->oas = 48;
4571	}
4572
4573	/* Set the DMA mask for our table walker */
4574	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
4575		dev_warn(smmu->dev,
4576			 "failed to set DMA mask for table walker\n");
4577
4578	smmu->ias = max(smmu->ias, smmu->oas);
4579
4580	if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
4581	    (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
4582		smmu->features |= ARM_SMMU_FEAT_NESTING;
4583
4584	arm_smmu_device_iidr_probe(smmu);
4585
4586	if (arm_smmu_sva_supported(smmu))
4587		smmu->features |= ARM_SMMU_FEAT_SVA;
4588
4589	dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
4590		 smmu->ias, smmu->oas, smmu->features);
4591	return 0;
4592}
4593
4594#ifdef CONFIG_ACPI
4595#ifdef CONFIG_TEGRA241_CMDQV
4596static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4597						struct arm_smmu_device *smmu)
4598{
4599	const char *uid = kasprintf(GFP_KERNEL, "%u", node->identifier);
4600	struct acpi_device *adev;
4601
4602	/* Look for an NVDA200C node whose _UID matches the SMMU node ID */
4603	adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
4604	if (adev) {
4605		/* Tegra241 CMDQV driver is responsible for put_device() */
4606		smmu->impl_dev = &adev->dev;
4607		smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
4608		dev_info(smmu->dev, "found companion CMDQV device: %s\n",
4609			 dev_name(smmu->impl_dev));
4610	}
4611	kfree(uid);
4612}
4613#else
4614static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4615						struct arm_smmu_device *smmu)
4616{
4617}
4618#endif
4619
4620static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
4621				      struct arm_smmu_device *smmu)
4622{
4623	struct acpi_iort_smmu_v3 *iort_smmu =
4624		(struct acpi_iort_smmu_v3 *)node->node_data;
4625
4626	switch (iort_smmu->model) {
4627	case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
4628		smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
4629		break;
4630	case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
4631		smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
4632		break;
4633	case ACPI_IORT_SMMU_V3_GENERIC:
4634		/*
4635		 * Tegra241 implementation stores its SMMU options and impl_dev
4636		 * in DSDT. Thus, go through the ACPI tables unconditionally.
4637		 */
4638		acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
4639		break;
4640	}
4641
4642	dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
4643	return 0;
4644}
4645
4646static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4647				      struct arm_smmu_device *smmu)
4648{
4649	struct acpi_iort_smmu_v3 *iort_smmu;
4650	struct device *dev = smmu->dev;
4651	struct acpi_iort_node *node;
4652
4653	node = *(struct acpi_iort_node **)dev_get_platdata(dev);
4654
4655	/* Retrieve SMMUv3 specific data */
4656	iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
4657
4658	if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
4659		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4660
4661	switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
4662	case IDR0_HTTU_ACCESS_DIRTY:
4663		smmu->features |= ARM_SMMU_FEAT_HD;
4664		fallthrough;
4665	case IDR0_HTTU_ACCESS:
4666		smmu->features |= ARM_SMMU_FEAT_HA;
4667	}
4668
4669	return acpi_smmu_iort_probe_model(node, smmu);
4670}
4671#else
4672static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4673					     struct arm_smmu_device *smmu)
4674{
4675	return -ENODEV;
4676}
4677#endif
4678
4679static int arm_smmu_device_dt_probe(struct platform_device *pdev,
4680				    struct arm_smmu_device *smmu)
4681{
4682	struct device *dev = &pdev->dev;
4683	u32 cells;
4684	int ret = -EINVAL;
4685
4686	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
4687		dev_err(dev, "missing #iommu-cells property\n");
4688	else if (cells != 1)
4689		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
4690	else
4691		ret = 0;
4692
4693	parse_driver_options(smmu);
4694
4695	if (of_dma_is_coherent(dev->of_node))
4696		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4697
4698	return ret;
4699}
4700
4701static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
4702{
4703	if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
4704		return SZ_64K;
4705	else
4706		return SZ_128K;
4707}
4708
4709static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
4710				      resource_size_t size)
4711{
4712	struct resource res = DEFINE_RES_MEM(start, size);
4713
4714	return devm_ioremap_resource(dev, &res);
4715}
4716
4717static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
4718{
4719	struct list_head rmr_list;
4720	struct iommu_resv_region *e;
4721
4722	INIT_LIST_HEAD(&rmr_list);
4723	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4724
4725	list_for_each_entry(e, &rmr_list, list) {
4726		struct iommu_iort_rmr_data *rmr;
4727		int ret, i;
4728
4729		rmr = container_of(e, struct iommu_iort_rmr_data, rr);
4730		for (i = 0; i < rmr->num_sids; i++) {
4731			ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
4732			if (ret) {
4733				dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
4734					rmr->sids[i]);
4735				continue;
4736			}
4737
4738			/*
4739			 * STE table is not programmed to HW, see
4740			 * arm_smmu_initial_bypass_stes()
4741			 */
4742			arm_smmu_make_bypass_ste(smmu,
4743				arm_smmu_get_step_for_sid(smmu, rmr->sids[i]));
4744		}
4745	}
4746
4747	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4748}
4749
4750static void arm_smmu_impl_remove(void *data)
4751{
4752	struct arm_smmu_device *smmu = data;
4753
4754	if (smmu->impl_ops && smmu->impl_ops->device_remove)
4755		smmu->impl_ops->device_remove(smmu);
4756}
4757
4758/*
4759 * Probe all the compiled in implementations. Each one checks to see if it
4760 * matches this HW and if so returns a devm_krealloc'd arm_smmu_device which
4761 * replaces the callers. Otherwise the original is returned or ERR_PTR.
4762 */
4763static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
4764{
4765	struct arm_smmu_device *new_smmu = ERR_PTR(-ENODEV);
4766	const struct arm_smmu_impl_ops *ops;
4767	int ret;
4768
4769	if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV))
4770		new_smmu = tegra241_cmdqv_probe(smmu);
4771
4772	if (new_smmu == ERR_PTR(-ENODEV))
4773		return smmu;
4774	if (IS_ERR(new_smmu))
4775		return new_smmu;
4776
4777	ops = new_smmu->impl_ops;
4778	if (ops) {
4779		/* get_viommu_size and vsmmu_init ops must be paired */
4780		if (WARN_ON(!ops->get_viommu_size != !ops->vsmmu_init)) {
4781			ret = -EINVAL;
4782			goto err_remove;
4783		}
4784	}
4785
4786	ret = devm_add_action_or_reset(new_smmu->dev, arm_smmu_impl_remove,
4787				       new_smmu);
4788	if (ret)
4789		return ERR_PTR(ret);
4790	return new_smmu;
4791
4792err_remove:
4793	arm_smmu_impl_remove(new_smmu);
4794	return ERR_PTR(ret);
4795}
4796
4797static int arm_smmu_device_probe(struct platform_device *pdev)
4798{
4799	int irq, ret;
4800	struct resource *res;
4801	resource_size_t ioaddr;
4802	struct arm_smmu_device *smmu;
4803	struct device *dev = &pdev->dev;
4804
4805	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
4806	if (!smmu)
4807		return -ENOMEM;
4808	smmu->dev = dev;
4809
4810	if (dev->of_node) {
4811		ret = arm_smmu_device_dt_probe(pdev, smmu);
4812	} else {
4813		ret = arm_smmu_device_acpi_probe(pdev, smmu);
4814	}
4815	if (ret)
4816		return ret;
4817
4818	smmu = arm_smmu_impl_probe(smmu);
4819	if (IS_ERR(smmu))
4820		return PTR_ERR(smmu);
4821
4822	/* Base address */
4823	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4824	if (!res)
4825		return -EINVAL;
4826	if (resource_size(res) < arm_smmu_resource_size(smmu)) {
4827		dev_err(dev, "MMIO region too small (%pr)\n", res);
4828		return -EINVAL;
4829	}
4830	ioaddr = res->start;
4831
4832	/*
4833	 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
4834	 * the PMCG registers which are reserved by the PMU driver.
4835	 */
4836	smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
4837	if (IS_ERR(smmu->base))
4838		return PTR_ERR(smmu->base);
4839
4840	if (arm_smmu_resource_size(smmu) > SZ_64K) {
4841		smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
4842					       ARM_SMMU_REG_SZ);
4843		if (IS_ERR(smmu->page1))
4844			return PTR_ERR(smmu->page1);
4845	} else {
4846		smmu->page1 = smmu->base;
4847	}
4848
4849	/* Interrupt lines */
4850
4851	irq = platform_get_irq_byname_optional(pdev, "combined");
4852	if (irq > 0)
4853		smmu->combined_irq = irq;
4854	else {
4855		irq = platform_get_irq_byname_optional(pdev, "eventq");
4856		if (irq > 0)
4857			smmu->evtq.q.irq = irq;
4858
4859		irq = platform_get_irq_byname_optional(pdev, "priq");
4860		if (irq > 0)
4861			smmu->priq.q.irq = irq;
4862
4863		irq = platform_get_irq_byname_optional(pdev, "gerror");
4864		if (irq > 0)
4865			smmu->gerr_irq = irq;
4866	}
4867	/* Probe the h/w */
4868	ret = arm_smmu_device_hw_probe(smmu);
4869	if (ret)
4870		return ret;
4871
4872	/* Initialise in-memory data structures */
4873	ret = arm_smmu_init_structures(smmu);
4874	if (ret)
4875		goto err_free_iopf;
4876
4877	/* Record our private device structure */
4878	platform_set_drvdata(pdev, smmu);
4879
4880	/* Check for RMRs and install bypass STEs if any */
4881	arm_smmu_rmr_install_bypass_ste(smmu);
4882
4883	/* Reset the device */
4884	ret = arm_smmu_device_reset(smmu);
4885	if (ret)
4886		goto err_disable;
4887
4888	/* And we're up. Go go go! */
4889	ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
4890				     "smmu3.%pa", &ioaddr);
4891	if (ret)
4892		goto err_disable;
4893
4894	ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
4895	if (ret) {
4896		dev_err(dev, "Failed to register iommu\n");
4897		goto err_free_sysfs;
4898	}
4899
4900	return 0;
4901
4902err_free_sysfs:
4903	iommu_device_sysfs_remove(&smmu->iommu);
4904err_disable:
4905	arm_smmu_device_disable(smmu);
4906err_free_iopf:
4907	iopf_queue_free(smmu->evtq.iopf);
4908	return ret;
4909}
4910
4911static void arm_smmu_device_remove(struct platform_device *pdev)
4912{
4913	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4914
4915	iommu_device_unregister(&smmu->iommu);
4916	iommu_device_sysfs_remove(&smmu->iommu);
4917	arm_smmu_device_disable(smmu);
4918	iopf_queue_free(smmu->evtq.iopf);
4919	ida_destroy(&smmu->vmid_map);
4920}
4921
4922static void arm_smmu_device_shutdown(struct platform_device *pdev)
4923{
4924	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4925
4926	arm_smmu_device_disable(smmu);
4927}
4928
4929static const struct of_device_id arm_smmu_of_match[] = {
4930	{ .compatible = "arm,smmu-v3", },
4931	{ },
4932};
4933MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
4934
4935static void arm_smmu_driver_unregister(struct platform_driver *drv)
4936{
4937	arm_smmu_sva_notifier_synchronize();
4938	platform_driver_unregister(drv);
4939}
4940
4941static struct platform_driver arm_smmu_driver = {
4942	.driver	= {
4943		.name			= "arm-smmu-v3",
4944		.of_match_table		= arm_smmu_of_match,
4945		.suppress_bind_attrs	= true,
4946	},
4947	.probe	= arm_smmu_device_probe,
4948	.remove = arm_smmu_device_remove,
4949	.shutdown = arm_smmu_device_shutdown,
4950};
4951module_driver(arm_smmu_driver, platform_driver_register,
4952	      arm_smmu_driver_unregister);
4953
4954MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4955MODULE_AUTHOR("Will Deacon <will@kernel.org>");
4956MODULE_ALIAS("platform:arm-smmu-v3");
4957MODULE_LICENSE("GPL v2");