apple-dart.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Apple DART (Device Address Resolution Table) IOMMU driver
   4 *
   5 * Copyright (C) 2021 The Asahi Linux Contributors
   6 *
   7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
   8 *  Copyright (C) 2013 ARM Limited
   9 *  Copyright (C) 2015 ARM Limited
  10 * and on exynos-iommu.c
  11 *  Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
  12 */
  13
  14#include <linux/atomic.h>
  15#include <linux/bitfield.h>
  16#include <linux/clk.h>
  17#include <linux/dev_printk.h>
  18#include <linux/dma-mapping.h>
  19#include <linux/err.h>
  20#include <linux/interrupt.h>
  21#include <linux/io-pgtable.h>
  22#include <linux/iommu.h>
  23#include <linux/iopoll.h>
  24#include <linux/module.h>
  25#include <linux/of.h>
  26#include <linux/of_address.h>
  27#include <linux/of_iommu.h>
  28#include <linux/of_platform.h>
  29#include <linux/pci.h>
  30#include <linux/platform_device.h>
  31#include <linux/slab.h>
  32#include <linux/swab.h>
  33#include <linux/types.h>
  34
  35#include "dma-iommu.h"
  36
  37#define DART_MAX_STREAMS 256
  38#define DART_MAX_TTBR 4
  39#define MAX_DARTS_PER_DEVICE 3
  40
  41/* Common registers */
  42
  43#define DART_PARAMS1 0x00
  44#define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24)
  45
  46#define DART_PARAMS2 0x04
  47#define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
  48
  49/* T8020/T6000 registers */
  50
  51#define DART_T8020_STREAM_COMMAND 0x20
  52#define DART_T8020_STREAM_COMMAND_BUSY BIT(2)
  53#define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20)
  54
  55#define DART_T8020_STREAM_SELECT 0x34
  56
  57#define DART_T8020_ERROR 0x40
  58#define DART_T8020_ERROR_STREAM GENMASK(27, 24)
  59#define DART_T8020_ERROR_CODE GENMASK(11, 0)
  60#define DART_T8020_ERROR_FLAG BIT(31)
  61
  62#define DART_T8020_ERROR_READ_FAULT BIT(4)
  63#define DART_T8020_ERROR_WRITE_FAULT BIT(3)
  64#define DART_T8020_ERROR_NO_PTE BIT(2)
  65#define DART_T8020_ERROR_NO_PMD BIT(1)
  66#define DART_T8020_ERROR_NO_TTBR BIT(0)
  67
  68#define DART_T8020_CONFIG 0x60
  69#define DART_T8020_CONFIG_LOCK BIT(15)
  70
  71#define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
  72
  73#define DART_T8020_ERROR_ADDR_HI 0x54
  74#define DART_T8020_ERROR_ADDR_LO 0x50
  75
  76#define DART_T8020_STREAMS_ENABLE 0xfc
  77
  78#define DART_T8020_TCR                  0x100
  79#define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
  80#define DART_T8020_TCR_BYPASS_DART      BIT(8)
  81#define DART_T8020_TCR_BYPASS_DAPF      BIT(12)
  82
  83#define DART_T8020_TTBR       0x200
  84#define DART_T8020_USB4_TTBR  0x400
  85#define DART_T8020_TTBR_VALID BIT(31)
  86#define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
  87#define DART_T8020_TTBR_SHIFT 12
  88
  89/* T8110 registers */
  90
  91#define DART_T8110_PARAMS3 0x08
  92#define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24)
  93#define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16)
  94#define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8)
  95#define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0)
  96
  97#define DART_T8110_PARAMS4 0x0c
  98#define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16)
  99#define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0)
 100
 101#define DART_T8110_TLB_CMD              0x80
 102#define DART_T8110_TLB_CMD_BUSY         BIT(31)
 103#define DART_T8110_TLB_CMD_OP           GENMASK(10, 8)
 104#define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0
 105#define DART_T8110_TLB_CMD_OP_FLUSH_SID 1
 106#define DART_T8110_TLB_CMD_STREAM       GENMASK(7, 0)
 107
 108#define DART_T8110_ERROR 0x100
 109#define DART_T8110_ERROR_STREAM GENMASK(27, 20)
 110#define DART_T8110_ERROR_CODE GENMASK(14, 0)
 111#define DART_T8110_ERROR_FLAG BIT(31)
 112
 113#define DART_T8110_ERROR_MASK 0x104
 114
 115#define DART_T8110_ERROR_READ_FAULT BIT(5)
 116#define DART_T8110_ERROR_WRITE_FAULT BIT(4)
 117#define DART_T8110_ERROR_NO_PTE BIT(3)
 118#define DART_T8110_ERROR_NO_PMD BIT(2)
 119#define DART_T8110_ERROR_NO_PGD BIT(1)
 120#define DART_T8110_ERROR_NO_TTBR BIT(0)
 121
 122#define DART_T8110_ERROR_ADDR_LO 0x170
 123#define DART_T8110_ERROR_ADDR_HI 0x174
 124
 125#define DART_T8110_ERROR_STREAMS 0x1c0
 126
 127#define DART_T8110_PROTECT 0x200
 128#define DART_T8110_UNPROTECT 0x204
 129#define DART_T8110_PROTECT_LOCK 0x208
 130#define DART_T8110_PROTECT_TTBR_TCR BIT(0)
 131
 132#define DART_T8110_ENABLE_STREAMS  0xc00
 133#define DART_T8110_DISABLE_STREAMS 0xc20
 134
 135#define DART_T8110_TCR                  0x1000
 136#define DART_T8110_TCR_REMAP            GENMASK(11, 8)
 137#define DART_T8110_TCR_REMAP_EN         BIT(7)
 138#define DART_T8110_TCR_FOUR_LEVEL       BIT(3)
 139#define DART_T8110_TCR_BYPASS_DAPF      BIT(2)
 140#define DART_T8110_TCR_BYPASS_DART      BIT(1)
 141#define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
 142
 143#define DART_T8110_TTBR       0x1400
 144#define DART_T8110_TTBR_VALID BIT(0)
 145#define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2
 146#define DART_T8110_TTBR_SHIFT 14
 147
 148#define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2))
 149
 150#define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \
 151				   (((dart)->hw->ttbr_count * (sid)) << 2) + \
 152				   ((idx) << 2))
 153
 154struct apple_dart_stream_map;
 155
 156enum dart_type {
 157	DART_T8020,
 158	DART_T6000,
 159	DART_T8110,
 160};
 161
 162struct apple_dart_hw {
 163	enum dart_type type;
 164	irqreturn_t (*irq_handler)(int irq, void *dev);
 165	int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
 166
 167	u32 oas;
 168	enum io_pgtable_fmt fmt;
 169
 170	int max_sid_count;
 171
 172	u32 lock;
 173	u32 lock_bit;
 174
 175	u32 error;
 176
 177	u32 enable_streams;
 178
 179	u32 tcr;
 180	u32 tcr_enabled;
 181	u32 tcr_disabled;
 182	u32 tcr_bypass;
 183	u32 tcr_4level;
 184
 185	u32 ttbr;
 186	u32 ttbr_valid;
 187	u32 ttbr_addr_field_shift;
 188	u32 ttbr_shift;
 189	int ttbr_count;
 190};
 191
 192/*
 193 * Private structure associated with each DART device.
 194 *
 195 * @dev: device struct
 196 * @hw: SoC-specific hardware data
 197 * @regs: mapped MMIO region
 198 * @irq: interrupt number, can be shared with other DARTs
 199 * @clks: clocks associated with this DART
 200 * @num_clks: number of @clks
 201 * @lock: lock for hardware operations involving this dart
 202 * @pgsize: pagesize supported by this DART
 203 * @supports_bypass: indicates if this DART supports bypass mode
 204 * @sid2group: maps stream ids to iommu_groups
 205 * @iommu: iommu core device
 206 */
 207struct apple_dart {
 208	struct device *dev;
 209	const struct apple_dart_hw *hw;
 210
 211	void __iomem *regs;
 212
 213	int irq;
 214	struct clk_bulk_data *clks;
 215	int num_clks;
 216
 217	spinlock_t lock;
 218
 219	u32 ias;
 220	u32 oas;
 221	u32 pgsize;
 222	u32 num_streams;
 223	u32 supports_bypass : 1;
 224	u32 four_level : 1;
 225
 226	struct iommu_group *sid2group[DART_MAX_STREAMS];
 227	struct iommu_device iommu;
 228
 229	u32 save_tcr[DART_MAX_STREAMS];
 230	u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
 231};
 232
 233/*
 234 * Convenience struct to identify streams.
 235 *
 236 * The normal variant is used inside apple_dart_master_cfg which isn't written
 237 * to concurrently.
 238 * The atomic variant is used inside apple_dart_domain where we have to guard
 239 * against races from potential parallel calls to attach/detach_device.
 240 * Note that even inside the atomic variant the apple_dart pointer is not
 241 * protected: This pointer is initialized once under the domain init mutex
 242 * and never changed again afterwards. Devices with different dart pointers
 243 * cannot be attached to the same domain.
 244 *
 245 * @dart dart pointer
 246 * @sid stream id bitmap
 247 */
 248struct apple_dart_stream_map {
 249	struct apple_dart *dart;
 250	DECLARE_BITMAP(sidmap, DART_MAX_STREAMS);
 251};
 252struct apple_dart_atomic_stream_map {
 253	struct apple_dart *dart;
 254	atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)];
 255};
 256
 257/*
 258 * This structure is attached to each iommu domain handled by a DART.
 259 *
 260 * @pgtbl_ops: pagetable ops allocated by io-pgtable
 261 * @finalized: true if the domain has been completely initialized
 262 * @init_lock: protects domain initialization
 263 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
 264 * @domain: core iommu domain pointer
 265 */
 266struct apple_dart_domain {
 267	struct io_pgtable_ops *pgtbl_ops;
 268
 269	bool finalized;
 270	struct mutex init_lock;
 271	struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 272
 273	struct iommu_domain domain;
 274};
 275
 276/*
 277 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
 278 * and contains a list of streams bound to this device.
 279 * So far the worst case seen is a single device with two streams
 280 * from different darts, such that this simple static array is enough.
 281 *
 282 * @streams: streams for this device
 283 */
 284struct apple_dart_master_cfg {
 285	/* Intersection of DART capabilitles */
 286	u32 supports_bypass : 1;
 287
 288	struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 289};
 290
 291/*
 292 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
 293 * apple_dart_domain.stream_maps
 294 *
 295 * @i int used as loop variable
 296 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
 297 * @stream pointer to the apple_dart_streams struct for each loop iteration
 298 */
 299#define for_each_stream_map(i, base, stream_map)                               \
 300	for (i = 0, stream_map = &(base)->stream_maps[0];                      \
 301	     i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
 302	     stream_map = &(base)->stream_maps[++i])
 303
 304static struct platform_driver apple_dart_driver;
 305static const struct iommu_ops apple_dart_iommu_ops;
 306
 307static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
 308{
 309	return container_of(dom, struct apple_dart_domain, domain);
 310}
 311
 312static void
 313apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels)
 314{
 315	struct apple_dart *dart = stream_map->dart;
 316	u32 tcr = dart->hw->tcr_enabled;
 317	int sid;
 318
 319	if (levels == 4)
 320		tcr |= dart->hw->tcr_4level;
 321
 322	WARN_ON(levels != 3 && levels != 4);
 323	WARN_ON(levels == 4 && !dart->four_level);
 324	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
 325		writel(tcr, dart->regs + DART_TCR(dart, sid));
 326}
 327
 328static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
 329{
 330	struct apple_dart *dart = stream_map->dart;
 331	int sid;
 332
 333	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
 334		writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid));
 335}
 336
 337static void
 338apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
 339{
 340	struct apple_dart *dart = stream_map->dart;
 341	int sid;
 342
 343	WARN_ON(!stream_map->dart->supports_bypass);
 344	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
 345		writel(dart->hw->tcr_bypass,
 346		       dart->regs + DART_TCR(dart, sid));
 347}
 348
 349static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
 350				   u8 idx, phys_addr_t paddr)
 351{
 352	struct apple_dart *dart = stream_map->dart;
 353	int sid;
 354
 355	WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1));
 356	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
 357		writel(dart->hw->ttbr_valid |
 358		       (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift,
 359		       dart->regs + DART_TTBR(dart, sid, idx));
 360}
 361
 362static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
 363				     u8 idx)
 364{
 365	struct apple_dart *dart = stream_map->dart;
 366	int sid;
 367
 368	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
 369		writel(0, dart->regs + DART_TTBR(dart, sid, idx));
 370}
 371
 372static void
 373apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
 374{
 375	int i;
 376
 377	for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
 378		apple_dart_hw_clear_ttbr(stream_map, i);
 379}
 380
 381static int
 382apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
 383			     u32 command)
 384{
 385	unsigned long flags;
 386	int ret, i;
 387	u32 command_reg;
 388
 389	spin_lock_irqsave(&stream_map->dart->lock, flags);
 390
 391	for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
 392		writel(stream_map->sidmap[i],
 393		       stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
 394	writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
 395
 396	ret = readl_poll_timeout_atomic(
 397		stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,
 398		!(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1,
 399		DART_STREAM_COMMAND_BUSY_TIMEOUT);
 400
 401	spin_unlock_irqrestore(&stream_map->dart->lock, flags);
 402
 403	if (ret) {
 404		dev_err(stream_map->dart->dev,
 405			"busy bit did not clear after command %x for streams %lx\n",
 406			command, stream_map->sidmap[0]);
 407		return ret;
 408	}
 409
 410	return 0;
 411}
 412
 413static int
 414apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,
 415				u32 command)
 416{
 417	struct apple_dart *dart = stream_map->dart;
 418	unsigned long flags;
 419	int ret = 0;
 420	int sid;
 421
 422	spin_lock_irqsave(&dart->lock, flags);
 423
 424	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
 425		u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) |
 426			FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid);
 427		writel(val, dart->regs + DART_T8110_TLB_CMD);
 428
 429		ret = readl_poll_timeout_atomic(
 430			dart->regs + DART_T8110_TLB_CMD, val,
 431			!(val & DART_T8110_TLB_CMD_BUSY), 1,
 432			DART_STREAM_COMMAND_BUSY_TIMEOUT);
 433
 434		if (ret)
 435			break;
 436
 437	}
 438
 439	spin_unlock_irqrestore(&dart->lock, flags);
 440
 441	if (ret) {
 442		dev_err(stream_map->dart->dev,
 443			"busy bit did not clear after command %x for stream %d\n",
 444			command, sid);
 445		return ret;
 446	}
 447
 448	return 0;
 449}
 450
 451static int
 452apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
 453{
 454	return apple_dart_t8020_hw_stream_command(
 455		stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);
 456}
 457
 458static int
 459apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
 460{
 461	return apple_dart_t8110_hw_tlb_command(
 462		stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);
 463}
 464
 465static int apple_dart_hw_reset(struct apple_dart *dart)
 466{
 467	u32 config;
 468	struct apple_dart_stream_map stream_map;
 469	int i;
 470
 471	config = readl(dart->regs + dart->hw->lock);
 472	if (config & dart->hw->lock_bit) {
 473		dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
 474			config);
 475		return -EINVAL;
 476	}
 477
 478	stream_map.dart = dart;
 479	bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);
 480	bitmap_set(stream_map.sidmap, 0, dart->num_streams);
 481	apple_dart_hw_disable_dma(&stream_map);
 482	apple_dart_hw_clear_all_ttbrs(&stream_map);
 483
 484	/* enable all streams globally since TCR is used to control isolation */
 485	for (i = 0; i < BITS_TO_U32(dart->num_streams); i++)
 486		writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i);
 487
 488	/* clear any pending errors before the interrupt is unmasked */
 489	writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error);
 490
 491	if (dart->hw->type == DART_T8110)
 492		writel(0,  dart->regs + DART_T8110_ERROR_MASK);
 493
 494	return dart->hw->invalidate_tlb(&stream_map);
 495}
 496
 497static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
 498{
 499	int i, j;
 500	struct apple_dart_atomic_stream_map *domain_stream_map;
 501	struct apple_dart_stream_map stream_map;
 502
 503	for_each_stream_map(i, domain, domain_stream_map) {
 504		stream_map.dart = domain_stream_map->dart;
 505
 506		for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)
 507			stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);
 508
 509		stream_map.dart->hw->invalidate_tlb(&stream_map);
 510	}
 511}
 512
 513static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
 514{
 515	apple_dart_domain_flush_tlb(to_dart_domain(domain));
 516}
 517
 518static void apple_dart_iotlb_sync(struct iommu_domain *domain,
 519				  struct iommu_iotlb_gather *gather)
 520{
 521	apple_dart_domain_flush_tlb(to_dart_domain(domain));
 522}
 523
 524static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
 525				     unsigned long iova, size_t size)
 526{
 527	apple_dart_domain_flush_tlb(to_dart_domain(domain));
 528	return 0;
 529}
 530
 531static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
 532					   dma_addr_t iova)
 533{
 534	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 535	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 536
 537	if (!ops)
 538		return 0;
 539
 540	return ops->iova_to_phys(ops, iova);
 541}
 542
 543static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
 544				phys_addr_t paddr, size_t pgsize,
 545				size_t pgcount, int prot, gfp_t gfp,
 546				size_t *mapped)
 547{
 548	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 549	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 550
 551	if (!ops)
 552		return -ENODEV;
 553
 554	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
 555			      mapped);
 556}
 557
 558static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
 559				     unsigned long iova, size_t pgsize,
 560				     size_t pgcount,
 561				     struct iommu_iotlb_gather *gather)
 562{
 563	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 564	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 565
 566	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
 567}
 568
 569static void
 570apple_dart_setup_translation(struct apple_dart_domain *domain,
 571			     struct apple_dart_stream_map *stream_map)
 572{
 573	int i;
 574	struct io_pgtable_cfg *pgtbl_cfg =
 575		&io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
 576
 577	for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
 578		apple_dart_hw_set_ttbr(stream_map, i,
 579				       pgtbl_cfg->apple_dart_cfg.ttbr[i]);
 580	for (; i < stream_map->dart->hw->ttbr_count; ++i)
 581		apple_dart_hw_clear_ttbr(stream_map, i);
 582
 583	apple_dart_hw_enable_translation(stream_map,
 584					 pgtbl_cfg->apple_dart_cfg.n_levels);
 585	stream_map->dart->hw->invalidate_tlb(stream_map);
 586}
 587
 588static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 589				      struct apple_dart_master_cfg *cfg)
 590{
 591	struct apple_dart *dart = cfg->stream_maps[0].dart;
 592	struct io_pgtable_cfg pgtbl_cfg;
 593	int ret = 0;
 594	int i, j;
 595
 596	if (dart->pgsize > PAGE_SIZE)
 597		return -EINVAL;
 598
 599	mutex_lock(&dart_domain->init_lock);
 600
 601	if (dart_domain->finalized)
 602		goto done;
 603
 604	for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 605		dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
 606		for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++)
 607			atomic_long_set(&dart_domain->stream_maps[i].sidmap[j],
 608					cfg->stream_maps[i].sidmap[j]);
 609	}
 610
 611	pgtbl_cfg = (struct io_pgtable_cfg){
 612		.pgsize_bitmap = dart->pgsize,
 613		.ias = dart->ias,
 614		.oas = dart->oas,
 615		.coherent_walk = 1,
 616		.iommu_dev = dart->dev,
 617	};
 618
 619	dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
 620						      &dart_domain->domain);
 621	if (!dart_domain->pgtbl_ops) {
 622		ret = -ENOMEM;
 623		goto done;
 624	}
 625
 626	dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 627	dart_domain->domain.geometry.aperture_start = 0;
 628	dart_domain->domain.geometry.aperture_end =
 629		(dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
 630	dart_domain->domain.geometry.force_aperture = true;
 631
 632	dart_domain->finalized = true;
 633
 634done:
 635	mutex_unlock(&dart_domain->init_lock);
 636	return ret;
 637}
 638
 639static int
 640apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
 641		       struct apple_dart_stream_map *master_maps,
 642		       bool add_streams)
 643{
 644	int i, j;
 645
 646	for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 647		if (domain_maps[i].dart != master_maps[i].dart)
 648			return -EINVAL;
 649	}
 650
 651	for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 652		if (!domain_maps[i].dart)
 653			break;
 654		for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) {
 655			if (add_streams)
 656				atomic_long_or(master_maps[i].sidmap[j],
 657					       &domain_maps[i].sidmap[j]);
 658			else
 659				atomic_long_and(~master_maps[i].sidmap[j],
 660						&domain_maps[i].sidmap[j]);
 661		}
 662	}
 663
 664	return 0;
 665}
 666
 667static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
 668					 struct apple_dart_master_cfg *cfg)
 669{
 670	return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
 671				      true);
 672}
 673
 674static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
 675					struct device *dev,
 676					struct iommu_domain *old)
 677{
 678	int ret, i;
 679	struct apple_dart_stream_map *stream_map;
 680	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 681	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 682
 683	ret = apple_dart_finalize_domain(dart_domain, cfg);
 684	if (ret)
 685		return ret;
 686
 687	ret = apple_dart_domain_add_streams(dart_domain, cfg);
 688	if (ret)
 689		return ret;
 690
 691	for_each_stream_map(i, cfg, stream_map)
 692		apple_dart_setup_translation(dart_domain, stream_map);
 693	return 0;
 694}
 695
 696static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
 697					  struct device *dev,
 698					  struct iommu_domain *old)
 699{
 700	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 701	struct apple_dart_stream_map *stream_map;
 702	int i;
 703
 704	if (!cfg->supports_bypass)
 705		return -EINVAL;
 706
 707	for_each_stream_map(i, cfg, stream_map)
 708		apple_dart_hw_enable_bypass(stream_map);
 709	return 0;
 710}
 711
 712static const struct iommu_domain_ops apple_dart_identity_ops = {
 713	.attach_dev = apple_dart_attach_dev_identity,
 714};
 715
 716static struct iommu_domain apple_dart_identity_domain = {
 717	.type = IOMMU_DOMAIN_IDENTITY,
 718	.ops = &apple_dart_identity_ops,
 719};
 720
 721static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
 722					 struct device *dev,
 723					 struct iommu_domain *old)
 724{
 725	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 726	struct apple_dart_stream_map *stream_map;
 727	int i;
 728
 729	for_each_stream_map(i, cfg, stream_map)
 730		apple_dart_hw_disable_dma(stream_map);
 731	return 0;
 732}
 733
 734static const struct iommu_domain_ops apple_dart_blocked_ops = {
 735	.attach_dev = apple_dart_attach_dev_blocked,
 736};
 737
 738static struct iommu_domain apple_dart_blocked_domain = {
 739	.type = IOMMU_DOMAIN_BLOCKED,
 740	.ops = &apple_dart_blocked_ops,
 741};
 742
 743static struct iommu_device *apple_dart_probe_device(struct device *dev)
 744{
 745	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 746	struct apple_dart_stream_map *stream_map;
 747	int i;
 748
 749	if (!cfg)
 750		return ERR_PTR(-ENODEV);
 751
 752	for_each_stream_map(i, cfg, stream_map)
 753		device_link_add(
 754			dev, stream_map->dart->dev,
 755			DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
 756
 757	return &cfg->stream_maps[0].dart->iommu;
 758}
 759
 760static void apple_dart_release_device(struct device *dev)
 761{
 762	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 763
 764	kfree(cfg);
 765}
 766
 767static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
 768{
 769	struct apple_dart_domain *dart_domain;
 770
 771	dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
 772	if (!dart_domain)
 773		return NULL;
 774
 775	mutex_init(&dart_domain->init_lock);
 776
 777	if (dev) {
 778		struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 779		int ret;
 780
 781		ret = apple_dart_finalize_domain(dart_domain, cfg);
 782		if (ret) {
 783			kfree(dart_domain);
 784			return ERR_PTR(ret);
 785		}
 786	}
 787	return &dart_domain->domain;
 788}
 789
 790static void apple_dart_domain_free(struct iommu_domain *domain)
 791{
 792	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 793
 794	free_io_pgtable_ops(dart_domain->pgtbl_ops);
 795
 796	kfree(dart_domain);
 797}
 798
 799static int apple_dart_of_xlate(struct device *dev,
 800			       const struct of_phandle_args *args)
 801{
 802	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 803	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
 804	struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
 805	struct apple_dart *cfg_dart;
 806	int i, sid;
 807
 808	put_device(&iommu_pdev->dev);
 809
 810	if (args->args_count != 1)
 811		return -EINVAL;
 812	sid = args->args[0];
 813
 814	if (!cfg) {
 815		cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
 816		if (!cfg)
 817			return -ENOMEM;
 818		/* Will be ANDed with DART capabilities */
 819		cfg->supports_bypass = true;
 820	}
 821	dev_iommu_priv_set(dev, cfg);
 822
 823	cfg_dart = cfg->stream_maps[0].dart;
 824	if (cfg_dart) {
 825		if (cfg_dart->pgsize != dart->pgsize)
 826			return -EINVAL;
 827		if (cfg_dart->ias != dart->ias)
 828			return -EINVAL;
 829	}
 830
 831	cfg->supports_bypass &= dart->supports_bypass;
 832
 833	for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 834		if (cfg->stream_maps[i].dart == dart) {
 835			set_bit(sid, cfg->stream_maps[i].sidmap);
 836			return 0;
 837		}
 838	}
 839	for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 840		if (!cfg->stream_maps[i].dart) {
 841			cfg->stream_maps[i].dart = dart;
 842			set_bit(sid, cfg->stream_maps[i].sidmap);
 843			return 0;
 844		}
 845	}
 846
 847	return -EINVAL;
 848}
 849
 850static DEFINE_MUTEX(apple_dart_groups_lock);
 851
 852static void apple_dart_release_group(void *iommu_data)
 853{
 854	int i, sid;
 855	struct apple_dart_stream_map *stream_map;
 856	struct apple_dart_master_cfg *group_master_cfg = iommu_data;
 857
 858	mutex_lock(&apple_dart_groups_lock);
 859
 860	for_each_stream_map(i, group_master_cfg, stream_map)
 861		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
 862			stream_map->dart->sid2group[sid] = NULL;
 863
 864	kfree(iommu_data);
 865	mutex_unlock(&apple_dart_groups_lock);
 866}
 867
 868static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst,
 869				       struct apple_dart_master_cfg *src)
 870{
 871	/*
 872	 * We know that this function is only called for groups returned from
 873	 * pci_device_group and that all Apple Silicon platforms never spread
 874	 * PCIe devices from the same bus across multiple DARTs such that we can
 875	 * just assume that both src and dst only have the same single DART.
 876	 */
 877	if (src->stream_maps[1].dart)
 878		return -EINVAL;
 879	if (dst->stream_maps[1].dart)
 880		return -EINVAL;
 881	if (src->stream_maps[0].dart != dst->stream_maps[0].dart)
 882		return -EINVAL;
 883
 884	bitmap_or(dst->stream_maps[0].sidmap,
 885		  dst->stream_maps[0].sidmap,
 886		  src->stream_maps[0].sidmap,
 887		  dst->stream_maps[0].dart->num_streams);
 888	return 0;
 889}
 890
 891static struct iommu_group *apple_dart_device_group(struct device *dev)
 892{
 893	int i, sid;
 894	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 895	struct apple_dart_stream_map *stream_map;
 896	struct apple_dart_master_cfg *group_master_cfg;
 897	struct iommu_group *group = NULL;
 898	struct iommu_group *res = ERR_PTR(-EINVAL);
 899
 900	mutex_lock(&apple_dart_groups_lock);
 901
 902	for_each_stream_map(i, cfg, stream_map) {
 903		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {
 904			struct iommu_group *stream_group =
 905				stream_map->dart->sid2group[sid];
 906
 907			if (group && group != stream_group) {
 908				res = ERR_PTR(-EINVAL);
 909				goto out;
 910			}
 911
 912			group = stream_group;
 913		}
 914	}
 915
 916	if (group) {
 917		res = iommu_group_ref_get(group);
 918		goto out;
 919	}
 920
 921#ifdef CONFIG_PCI
 922	if (dev_is_pci(dev))
 923		group = pci_device_group(dev);
 924	else
 925#endif
 926		group = generic_device_group(dev);
 927
 928	res = ERR_PTR(-ENOMEM);
 929	if (!group)
 930		goto out;
 931
 932	group_master_cfg = iommu_group_get_iommudata(group);
 933	if (group_master_cfg) {
 934		int ret;
 935
 936		ret = apple_dart_merge_master_cfg(group_master_cfg, cfg);
 937		if (ret) {
 938			dev_err(dev, "Failed to merge DART IOMMU groups.\n");
 939			iommu_group_put(group);
 940			res = ERR_PTR(ret);
 941			goto out;
 942		}
 943	} else {
 944		group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg),
 945					   GFP_KERNEL);
 946		if (!group_master_cfg) {
 947			iommu_group_put(group);
 948			goto out;
 949		}
 950
 951		iommu_group_set_iommudata(group, group_master_cfg,
 952			apple_dart_release_group);
 953	}
 954
 955	for_each_stream_map(i, cfg, stream_map)
 956		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
 957			stream_map->dart->sid2group[sid] = group;
 958
 959	res = group;
 960
 961out:
 962	mutex_unlock(&apple_dart_groups_lock);
 963	return res;
 964}
 965
 966static int apple_dart_def_domain_type(struct device *dev)
 967{
 968	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 969
 970	if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
 971		return IOMMU_DOMAIN_IDENTITY;
 972	if (!cfg->supports_bypass)
 973		return IOMMU_DOMAIN_DMA;
 974
 975	return 0;
 976}
 977
 978#ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
 979/* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
 980#define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR	0
 981#endif
 982#define DOORBELL_ADDR	(CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)
 983
 984static void apple_dart_get_resv_regions(struct device *dev,
 985					struct list_head *head)
 986{
 987	if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) {
 988		struct iommu_resv_region *region;
 989		int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
 990
 991		region = iommu_alloc_resv_region(DOORBELL_ADDR,
 992						 PAGE_SIZE, prot,
 993						 IOMMU_RESV_MSI, GFP_KERNEL);
 994		if (!region)
 995			return;
 996
 997		list_add_tail(&region->list, head);
 998	}
 999
1000	iommu_dma_get_resv_regions(dev, head);
1001}
1002
1003static const struct iommu_ops apple_dart_iommu_ops = {
1004	.identity_domain = &apple_dart_identity_domain,
1005	.blocked_domain = &apple_dart_blocked_domain,
1006	.domain_alloc_paging = apple_dart_domain_alloc_paging,
1007	.probe_device = apple_dart_probe_device,
1008	.release_device = apple_dart_release_device,
1009	.device_group = apple_dart_device_group,
1010	.of_xlate = apple_dart_of_xlate,
1011	.def_domain_type = apple_dart_def_domain_type,
1012	.get_resv_regions = apple_dart_get_resv_regions,
1013	.owner = THIS_MODULE,
1014	.default_domain_ops = &(const struct iommu_domain_ops) {
1015		.attach_dev	= apple_dart_attach_dev_paging,
1016		.map_pages	= apple_dart_map_pages,
1017		.unmap_pages	= apple_dart_unmap_pages,
1018		.flush_iotlb_all = apple_dart_flush_iotlb_all,
1019		.iotlb_sync	= apple_dart_iotlb_sync,
1020		.iotlb_sync_map	= apple_dart_iotlb_sync_map,
1021		.iova_to_phys	= apple_dart_iova_to_phys,
1022		.free		= apple_dart_domain_free,
1023	}
1024};
1025
1026static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
1027{
1028	struct apple_dart *dart = dev;
1029	const char *fault_name = NULL;
1030	u32 error = readl(dart->regs + DART_T8020_ERROR);
1031	u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error);
1032	u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO);
1033	u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI);
1034	u64 addr = addr_lo | (((u64)addr_hi) << 32);
1035	u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error);
1036
1037	if (!(error & DART_T8020_ERROR_FLAG))
1038		return IRQ_NONE;
1039
1040	/* there should only be a single bit set but let's use == to be sure */
1041	if (error_code == DART_T8020_ERROR_READ_FAULT)
1042		fault_name = "READ FAULT";
1043	else if (error_code == DART_T8020_ERROR_WRITE_FAULT)
1044		fault_name = "WRITE FAULT";
1045	else if (error_code == DART_T8020_ERROR_NO_PTE)
1046		fault_name = "NO PTE FOR IOVA";
1047	else if (error_code == DART_T8020_ERROR_NO_PMD)
1048		fault_name = "NO PMD FOR IOVA";
1049	else if (error_code == DART_T8020_ERROR_NO_TTBR)
1050		fault_name = "NO TTBR FOR IOVA";
1051	else
1052		fault_name = "unknown";
1053
1054	dev_err_ratelimited(
1055		dart->dev,
1056		"translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1057		error, stream_idx, error_code, fault_name, addr);
1058
1059	writel(error, dart->regs + DART_T8020_ERROR);
1060	return IRQ_HANDLED;
1061}
1062
1063static irqreturn_t apple_dart_t8110_irq(int irq, void *dev)
1064{
1065	struct apple_dart *dart = dev;
1066	const char *fault_name = NULL;
1067	u32 error = readl(dart->regs + DART_T8110_ERROR);
1068	u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error);
1069	u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO);
1070	u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI);
1071	u64 addr = addr_lo | (((u64)addr_hi) << 32);
1072	u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error);
1073
1074	if (!(error & DART_T8110_ERROR_FLAG))
1075		return IRQ_NONE;
1076
1077	/* there should only be a single bit set but let's use == to be sure */
1078	if (error_code == DART_T8110_ERROR_READ_FAULT)
1079		fault_name = "READ FAULT";
1080	else if (error_code == DART_T8110_ERROR_WRITE_FAULT)
1081		fault_name = "WRITE FAULT";
1082	else if (error_code == DART_T8110_ERROR_NO_PTE)
1083		fault_name = "NO PTE FOR IOVA";
1084	else if (error_code == DART_T8110_ERROR_NO_PMD)
1085		fault_name = "NO PMD FOR IOVA";
1086	else if (error_code == DART_T8110_ERROR_NO_PGD)
1087		fault_name = "NO PGD FOR IOVA";
1088	else if (error_code == DART_T8110_ERROR_NO_TTBR)
1089		fault_name = "NO TTBR FOR IOVA";
1090	else
1091		fault_name = "unknown";
1092
1093	dev_err_ratelimited(
1094		dart->dev,
1095		"translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1096		error, stream_idx, error_code, fault_name, addr);
1097
1098	writel(error, dart->regs + DART_T8110_ERROR);
1099	for (int i = 0; i < BITS_TO_U32(dart->num_streams); i++)
1100		writel(U32_MAX, dart->regs + DART_T8110_ERROR_STREAMS + 4 * i);
1101
1102	return IRQ_HANDLED;
1103}
1104
1105static int apple_dart_probe(struct platform_device *pdev)
1106{
1107	int ret;
1108	u32 dart_params[4];
1109	struct resource *res;
1110	struct apple_dart *dart;
1111	struct device *dev = &pdev->dev;
1112
1113	dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
1114	if (!dart)
1115		return -ENOMEM;
1116
1117	dart->dev = dev;
1118	dart->hw = of_device_get_match_data(dev);
1119	spin_lock_init(&dart->lock);
1120
1121	dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1122	if (IS_ERR(dart->regs))
1123		return PTR_ERR(dart->regs);
1124
1125	if (resource_size(res) < 0x4000) {
1126		dev_err(dev, "MMIO region too small (%pr)\n", res);
1127		return -EINVAL;
1128	}
1129
1130	dart->irq = platform_get_irq(pdev, 0);
1131	if (dart->irq < 0)
1132		return -ENODEV;
1133
1134	ret = devm_clk_bulk_get_all(dev, &dart->clks);
1135	if (ret < 0)
1136		return ret;
1137	dart->num_clks = ret;
1138
1139	ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
1140	if (ret)
1141		return ret;
1142
1143	dart_params[0] = readl(dart->regs + DART_PARAMS1);
1144	dart_params[1] = readl(dart->regs + DART_PARAMS2);
1145	dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]);
1146	dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT;
1147
1148	switch (dart->hw->type) {
1149	case DART_T8020:
1150	case DART_T6000:
1151		dart->ias = 32;
1152		dart->oas = dart->hw->oas;
1153		dart->num_streams = dart->hw->max_sid_count;
1154		break;
1155
1156	case DART_T8110:
1157		dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3);
1158		dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4);
1159		dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
1160		dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
1161		dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
1162		dart->four_level = dart->ias > 36;
1163		break;
1164	}
1165
1166	if (dart->num_streams > DART_MAX_STREAMS) {
1167		dev_err(&pdev->dev, "Too many streams (%d > %d)\n",
1168			dart->num_streams, DART_MAX_STREAMS);
1169		ret = -EINVAL;
1170		goto err_clk_disable;
1171	}
1172
1173	ret = apple_dart_hw_reset(dart);
1174	if (ret)
1175		goto err_clk_disable;
1176
1177	ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED,
1178			  "apple-dart fault handler", dart);
1179	if (ret)
1180		goto err_clk_disable;
1181
1182	platform_set_drvdata(pdev, dart);
1183
1184	ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
1185				     dev_name(&pdev->dev));
1186	if (ret)
1187		goto err_free_irq;
1188
1189	ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
1190	if (ret)
1191		goto err_sysfs_remove;
1192
1193	dev_info(
1194		&pdev->dev,
1195		"DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n",
1196		dart->pgsize, dart->num_streams, dart->supports_bypass,
1197		dart->pgsize > PAGE_SIZE, dart->ias, dart->oas);
1198	return 0;
1199
1200err_sysfs_remove:
1201	iommu_device_sysfs_remove(&dart->iommu);
1202err_free_irq:
1203	free_irq(dart->irq, dart);
1204err_clk_disable:
1205	clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1206
1207	return ret;
1208}
1209
1210static void apple_dart_remove(struct platform_device *pdev)
1211{
1212	struct apple_dart *dart = platform_get_drvdata(pdev);
1213
1214	apple_dart_hw_reset(dart);
1215	free_irq(dart->irq, dart);
1216
1217	iommu_device_unregister(&dart->iommu);
1218	iommu_device_sysfs_remove(&dart->iommu);
1219
1220	clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1221}
1222
1223static const struct apple_dart_hw apple_dart_hw_t8103 = {
1224	.type = DART_T8020,
1225	.irq_handler = apple_dart_t8020_irq,
1226	.invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1227	.oas = 36,
1228	.fmt = APPLE_DART,
1229	.max_sid_count = 16,
1230
1231	.enable_streams = DART_T8020_STREAMS_ENABLE,
1232	.lock = DART_T8020_CONFIG,
1233	.lock_bit = DART_T8020_CONFIG_LOCK,
1234
1235	.error = DART_T8020_ERROR,
1236
1237	.tcr = DART_T8020_TCR,
1238	.tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1239	.tcr_disabled = 0,
1240	.tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1241
1242	.ttbr = DART_T8020_TTBR,
1243	.ttbr_valid = DART_T8020_TTBR_VALID,
1244	.ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1245	.ttbr_shift = DART_T8020_TTBR_SHIFT,
1246	.ttbr_count = 4,
1247};
1248
1249static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
1250	.type = DART_T8020,
1251	.irq_handler = apple_dart_t8020_irq,
1252	.invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1253	.oas = 36,
1254	.fmt = APPLE_DART,
1255	.max_sid_count = 64,
1256
1257	.enable_streams = DART_T8020_STREAMS_ENABLE,
1258	.lock = DART_T8020_CONFIG,
1259	.lock_bit = DART_T8020_CONFIG_LOCK,
1260
1261	.error = DART_T8020_ERROR,
1262
1263	.tcr = DART_T8020_TCR,
1264	.tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1265	.tcr_disabled = 0,
1266	.tcr_bypass = 0,
1267
1268	.ttbr = DART_T8020_USB4_TTBR,
1269	.ttbr_valid = DART_T8020_TTBR_VALID,
1270	.ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1271	.ttbr_shift = DART_T8020_TTBR_SHIFT,
1272	.ttbr_count = 4,
1273};
1274
1275static const struct apple_dart_hw apple_dart_hw_t6000 = {
1276	.type = DART_T6000,
1277	.irq_handler = apple_dart_t8020_irq,
1278	.invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1279	.oas = 42,
1280	.fmt = APPLE_DART2,
1281	.max_sid_count = 16,
1282
1283	.enable_streams = DART_T8020_STREAMS_ENABLE,
1284	.lock = DART_T8020_CONFIG,
1285	.lock_bit = DART_T8020_CONFIG_LOCK,
1286
1287	.error = DART_T8020_ERROR,
1288
1289	.tcr = DART_T8020_TCR,
1290	.tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1291	.tcr_disabled = 0,
1292	.tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1293
1294	.ttbr = DART_T8020_TTBR,
1295	.ttbr_valid = DART_T8020_TTBR_VALID,
1296	.ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1297	.ttbr_shift = DART_T8020_TTBR_SHIFT,
1298	.ttbr_count = 4,
1299};
1300
1301static const struct apple_dart_hw apple_dart_hw_t8110 = {
1302	.type = DART_T8110,
1303	.irq_handler = apple_dart_t8110_irq,
1304	.invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb,
1305	.fmt = APPLE_DART2,
1306	.max_sid_count = 256,
1307
1308	.enable_streams = DART_T8110_ENABLE_STREAMS,
1309	.lock = DART_T8110_PROTECT,
1310	.lock_bit = DART_T8110_PROTECT_TTBR_TCR,
1311
1312	.error = DART_T8110_ERROR,
1313
1314	.tcr = DART_T8110_TCR,
1315	.tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
1316	.tcr_disabled = 0,
1317	.tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
1318	.tcr_4level = DART_T8110_TCR_FOUR_LEVEL,
1319
1320	.ttbr = DART_T8110_TTBR,
1321	.ttbr_valid = DART_T8110_TTBR_VALID,
1322	.ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT,
1323	.ttbr_shift = DART_T8110_TTBR_SHIFT,
1324	.ttbr_count = 1,
1325};
1326
1327static __maybe_unused int apple_dart_suspend(struct device *dev)
1328{
1329	struct apple_dart *dart = dev_get_drvdata(dev);
1330	unsigned int sid, idx;
1331
1332	for (sid = 0; sid < dart->num_streams; sid++) {
1333		dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid));
1334		for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1335			dart->save_ttbr[sid][idx] =
1336				readl(dart->regs + DART_TTBR(dart, sid, idx));
1337	}
1338
1339	return 0;
1340}
1341
1342static __maybe_unused int apple_dart_resume(struct device *dev)
1343{
1344	struct apple_dart *dart = dev_get_drvdata(dev);
1345	unsigned int sid, idx;
1346	int ret;
1347
1348	ret = apple_dart_hw_reset(dart);
1349	if (ret) {
1350		dev_err(dev, "Failed to reset DART on resume\n");
1351		return ret;
1352	}
1353
1354	for (sid = 0; sid < dart->num_streams; sid++) {
1355		for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1356			writel(dart->save_ttbr[sid][idx],
1357			       dart->regs + DART_TTBR(dart, sid, idx));
1358		writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid));
1359	}
1360
1361	return 0;
1362}
1363
1364static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume);
1365
1366static const struct of_device_id apple_dart_of_match[] = {
1367	{ .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1368	{ .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1369	{ .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1370	{ .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1371	{},
1372};
1373MODULE_DEVICE_TABLE(of, apple_dart_of_match);
1374
1375static struct platform_driver apple_dart_driver = {
1376	.driver	= {
1377		.name			= "apple-dart",
1378		.of_match_table		= apple_dart_of_match,
1379		.suppress_bind_attrs    = true,
1380		.pm			= pm_sleep_ptr(&apple_dart_pm_ops),
1381	},
1382	.probe	= apple_dart_probe,
1383	.remove = apple_dart_remove,
1384};
1385
1386module_platform_driver(apple_dart_driver);
1387
1388MODULE_DESCRIPTION("IOMMU API for Apple's DART");
1389MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1390MODULE_LICENSE("GPL v2");