iommu.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
   4 * Author: Joerg Roedel <jroedel@suse.de>
   5 */
   6
   7#define pr_fmt(fmt)    "iommu: " fmt
   8
   9#include <linux/amba/bus.h>
  10#include <linux/device.h>
  11#include <linux/kernel.h>
  12#include <linux/bits.h>
  13#include <linux/bug.h>
  14#include <linux/types.h>
  15#include <linux/init.h>
  16#include <linux/export.h>
  17#include <linux/slab.h>
  18#include <linux/errno.h>
  19#include <linux/host1x_context_bus.h>
  20#include <linux/iommu.h>
  21#include <linux/iommufd.h>
  22#include <linux/idr.h>
  23#include <linux/err.h>
  24#include <linux/pci.h>
  25#include <linux/pci-ats.h>
  26#include <linux/bitops.h>
  27#include <linux/platform_device.h>
  28#include <linux/property.h>
  29#include <linux/fsl/mc.h>
  30#include <linux/module.h>
  31#include <linux/cc_platform.h>
  32#include <linux/cdx/cdx_bus.h>
  33#include <trace/events/iommu.h>
  34#include <linux/sched/mm.h>
  35#include <linux/msi.h>
  36#include <uapi/linux/iommufd.h>
  37
  38#include "dma-iommu.h"
  39#include "iommu-priv.h"
  40
  41static struct kset *iommu_group_kset;
  42static DEFINE_IDA(iommu_group_ida);
  43static DEFINE_IDA(iommu_global_pasid_ida);
  44
  45static unsigned int iommu_def_domain_type __read_mostly;
  46static bool iommu_dma_strict __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_DMA_STRICT);
  47static u32 iommu_cmd_line __read_mostly;
  48
  49/* Tags used with xa_tag_pointer() in group->pasid_array */
  50enum { IOMMU_PASID_ARRAY_DOMAIN = 0, IOMMU_PASID_ARRAY_HANDLE = 1 };
  51
  52struct iommu_group {
  53	struct kobject kobj;
  54	struct kobject *devices_kobj;
  55	struct list_head devices;
  56	struct xarray pasid_array;
  57	struct mutex mutex;
  58	void *iommu_data;
  59	void (*iommu_data_release)(void *iommu_data);
  60	char *name;
  61	int id;
  62	struct iommu_domain *default_domain;
  63	struct iommu_domain *blocking_domain;
  64	struct iommu_domain *domain;
  65	struct list_head entry;
  66	unsigned int owner_cnt;
  67	void *owner;
  68};
  69
  70struct group_device {
  71	struct list_head list;
  72	struct device *dev;
  73	char *name;
  74};
  75
  76/* Iterate over each struct group_device in a struct iommu_group */
  77#define for_each_group_device(group, pos) \
  78	list_for_each_entry(pos, &(group)->devices, list)
  79
  80struct iommu_group_attribute {
  81	struct attribute attr;
  82	ssize_t (*show)(struct iommu_group *group, char *buf);
  83	ssize_t (*store)(struct iommu_group *group,
  84			 const char *buf, size_t count);
  85};
  86
  87static const char * const iommu_group_resv_type_string[] = {
  88	[IOMMU_RESV_DIRECT]			= "direct",
  89	[IOMMU_RESV_DIRECT_RELAXABLE]		= "direct-relaxable",
  90	[IOMMU_RESV_RESERVED]			= "reserved",
  91	[IOMMU_RESV_MSI]			= "msi",
  92	[IOMMU_RESV_SW_MSI]			= "msi",
  93};
  94
  95#define IOMMU_CMD_LINE_DMA_API		BIT(0)
  96#define IOMMU_CMD_LINE_STRICT		BIT(1)
  97
  98static int bus_iommu_probe(const struct bus_type *bus);
  99static int iommu_bus_notifier(struct notifier_block *nb,
 100			      unsigned long action, void *data);
 101static void iommu_release_device(struct device *dev);
 102static int __iommu_attach_device(struct iommu_domain *domain,
 103				 struct device *dev, struct iommu_domain *old);
 104static int __iommu_attach_group(struct iommu_domain *domain,
 105				struct iommu_group *group);
 106static struct iommu_domain *__iommu_paging_domain_alloc_flags(struct device *dev,
 107						       unsigned int type,
 108						       unsigned int flags);
 109
 110enum {
 111	IOMMU_SET_DOMAIN_MUST_SUCCEED = 1 << 0,
 112};
 113
 114static int __iommu_device_set_domain(struct iommu_group *group,
 115				     struct device *dev,
 116				     struct iommu_domain *new_domain,
 117				     struct iommu_domain *old_domain,
 118				     unsigned int flags);
 119static int __iommu_group_set_domain_internal(struct iommu_group *group,
 120					     struct iommu_domain *new_domain,
 121					     unsigned int flags);
 122static int __iommu_group_set_domain(struct iommu_group *group,
 123				    struct iommu_domain *new_domain)
 124{
 125	return __iommu_group_set_domain_internal(group, new_domain, 0);
 126}
 127static void __iommu_group_set_domain_nofail(struct iommu_group *group,
 128					    struct iommu_domain *new_domain)
 129{
 130	WARN_ON(__iommu_group_set_domain_internal(
 131		group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED));
 132}
 133
 134static int iommu_setup_default_domain(struct iommu_group *group,
 135				      int target_type);
 136static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
 137					       struct device *dev);
 138static ssize_t iommu_group_store_type(struct iommu_group *group,
 139				      const char *buf, size_t count);
 140static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
 141						     struct device *dev);
 142static void __iommu_group_free_device(struct iommu_group *group,
 143				      struct group_device *grp_dev);
 144static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
 145			      const struct iommu_ops *ops);
 146
 147#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store)		\
 148struct iommu_group_attribute iommu_group_attr_##_name =		\
 149	__ATTR(_name, _mode, _show, _store)
 150
 151#define to_iommu_group_attr(_attr)	\
 152	container_of(_attr, struct iommu_group_attribute, attr)
 153#define to_iommu_group(_kobj)		\
 154	container_of(_kobj, struct iommu_group, kobj)
 155
 156static LIST_HEAD(iommu_device_list);
 157static DEFINE_SPINLOCK(iommu_device_lock);
 158
 159static const struct bus_type * const iommu_buses[] = {
 160	&platform_bus_type,
 161#ifdef CONFIG_PCI
 162	&pci_bus_type,
 163#endif
 164#ifdef CONFIG_ARM_AMBA
 165	&amba_bustype,
 166#endif
 167#ifdef CONFIG_FSL_MC_BUS
 168	&fsl_mc_bus_type,
 169#endif
 170#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS
 171	&host1x_context_device_bus_type,
 172#endif
 173#ifdef CONFIG_CDX_BUS
 174	&cdx_bus_type,
 175#endif
 176};
 177
 178/*
 179 * Use a function instead of an array here because the domain-type is a
 180 * bit-field, so an array would waste memory.
 181 */
 182static const char *iommu_domain_type_str(unsigned int t)
 183{
 184	switch (t) {
 185	case IOMMU_DOMAIN_BLOCKED:
 186		return "Blocked";
 187	case IOMMU_DOMAIN_IDENTITY:
 188		return "Passthrough";
 189	case IOMMU_DOMAIN_UNMANAGED:
 190		return "Unmanaged";
 191	case IOMMU_DOMAIN_DMA:
 192	case IOMMU_DOMAIN_DMA_FQ:
 193		return "Translated";
 194	case IOMMU_DOMAIN_PLATFORM:
 195		return "Platform";
 196	default:
 197		return "Unknown";
 198	}
 199}
 200
 201static int __init iommu_subsys_init(void)
 202{
 203	struct notifier_block *nb;
 204
 205	if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) {
 206		if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
 207			iommu_set_default_passthrough(false);
 208		else
 209			iommu_set_default_translated(false);
 210
 211		if (iommu_default_passthrough() && cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
 212			pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
 213			iommu_set_default_translated(false);
 214		}
 215	}
 216
 217	if (!iommu_default_passthrough() && !iommu_dma_strict)
 218		iommu_def_domain_type = IOMMU_DOMAIN_DMA_FQ;
 219
 220	pr_info("Default domain type: %s%s\n",
 221		iommu_domain_type_str(iommu_def_domain_type),
 222		(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API) ?
 223			" (set via kernel command line)" : "");
 224
 225	if (!iommu_default_passthrough())
 226		pr_info("DMA domain TLB invalidation policy: %s mode%s\n",
 227			iommu_dma_strict ? "strict" : "lazy",
 228			(iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ?
 229				" (set via kernel command line)" : "");
 230
 231	nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL);
 232	if (!nb)
 233		return -ENOMEM;
 234
 235	for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) {
 236		nb[i].notifier_call = iommu_bus_notifier;
 237		bus_register_notifier(iommu_buses[i], &nb[i]);
 238	}
 239
 240	return 0;
 241}
 242subsys_initcall(iommu_subsys_init);
 243
 244static int remove_iommu_group(struct device *dev, void *data)
 245{
 246	if (dev->iommu && dev->iommu->iommu_dev == data)
 247		iommu_release_device(dev);
 248
 249	return 0;
 250}
 251
 252/**
 253 * iommu_device_register() - Register an IOMMU hardware instance
 254 * @iommu: IOMMU handle for the instance
 255 * @ops:   IOMMU ops to associate with the instance
 256 * @hwdev: (optional) actual instance device, used for fwnode lookup
 257 *
 258 * Return: 0 on success, or an error.
 259 */
 260int iommu_device_register(struct iommu_device *iommu,
 261			  const struct iommu_ops *ops, struct device *hwdev)
 262{
 263	int err = 0;
 264
 265	/* We need to be able to take module references appropriately */
 266	if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner))
 267		return -EINVAL;
 268
 269	iommu->ops = ops;
 270	if (hwdev)
 271		iommu->fwnode = dev_fwnode(hwdev);
 272
 273	spin_lock(&iommu_device_lock);
 274	list_add_tail(&iommu->list, &iommu_device_list);
 275	spin_unlock(&iommu_device_lock);
 276
 277	for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++)
 278		err = bus_iommu_probe(iommu_buses[i]);
 279	if (err)
 280		iommu_device_unregister(iommu);
 281	else
 282		WRITE_ONCE(iommu->ready, true);
 283	return err;
 284}
 285EXPORT_SYMBOL_GPL(iommu_device_register);
 286
 287void iommu_device_unregister(struct iommu_device *iommu)
 288{
 289	for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)
 290		bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group);
 291
 292	spin_lock(&iommu_device_lock);
 293	list_del(&iommu->list);
 294	spin_unlock(&iommu_device_lock);
 295
 296	/* Pairs with the alloc in generic_single_device_group() */
 297	iommu_group_put(iommu->singleton_group);
 298	iommu->singleton_group = NULL;
 299}
 300EXPORT_SYMBOL_GPL(iommu_device_unregister);
 301
 302#if IS_ENABLED(CONFIG_IOMMUFD_TEST)
 303void iommu_device_unregister_bus(struct iommu_device *iommu,
 304				 const struct bus_type *bus,
 305				 struct notifier_block *nb)
 306{
 307	bus_unregister_notifier(bus, nb);
 308	fwnode_remove_software_node(iommu->fwnode);
 309	iommu_device_unregister(iommu);
 310}
 311EXPORT_SYMBOL_GPL(iommu_device_unregister_bus);
 312
 313/*
 314 * Register an iommu driver against a single bus. This is only used by iommufd
 315 * selftest to create a mock iommu driver. The caller must provide
 316 * some memory to hold a notifier_block.
 317 */
 318int iommu_device_register_bus(struct iommu_device *iommu,
 319			      const struct iommu_ops *ops,
 320			      const struct bus_type *bus,
 321			      struct notifier_block *nb)
 322{
 323	int err;
 324
 325	iommu->ops = ops;
 326	nb->notifier_call = iommu_bus_notifier;
 327	err = bus_register_notifier(bus, nb);
 328	if (err)
 329		return err;
 330
 331	iommu->fwnode = fwnode_create_software_node(NULL, NULL);
 332	if (IS_ERR(iommu->fwnode)) {
 333		bus_unregister_notifier(bus, nb);
 334		return PTR_ERR(iommu->fwnode);
 335	}
 336
 337	spin_lock(&iommu_device_lock);
 338	list_add_tail(&iommu->list, &iommu_device_list);
 339	spin_unlock(&iommu_device_lock);
 340
 341	err = bus_iommu_probe(bus);
 342	if (err) {
 343		iommu_device_unregister_bus(iommu, bus, nb);
 344		return err;
 345	}
 346	WRITE_ONCE(iommu->ready, true);
 347	return 0;
 348}
 349EXPORT_SYMBOL_GPL(iommu_device_register_bus);
 350
 351int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu)
 352{
 353	int rc;
 354
 355	mutex_lock(&iommu_probe_device_lock);
 356	rc = iommu_fwspec_init(dev, iommu->fwnode);
 357	mutex_unlock(&iommu_probe_device_lock);
 358
 359	if (rc)
 360		return rc;
 361
 362	rc = device_add(dev);
 363	if (rc)
 364		iommu_fwspec_free(dev);
 365	return rc;
 366}
 367EXPORT_SYMBOL_GPL(iommu_mock_device_add);
 368#endif
 369
 370static struct dev_iommu *dev_iommu_get(struct device *dev)
 371{
 372	struct dev_iommu *param = dev->iommu;
 373
 374	lockdep_assert_held(&iommu_probe_device_lock);
 375
 376	if (param)
 377		return param;
 378
 379	param = kzalloc(sizeof(*param), GFP_KERNEL);
 380	if (!param)
 381		return NULL;
 382
 383	mutex_init(&param->lock);
 384	dev->iommu = param;
 385	return param;
 386}
 387
 388void dev_iommu_free(struct device *dev)
 389{
 390	struct dev_iommu *param = dev->iommu;
 391
 392	dev->iommu = NULL;
 393	if (param->fwspec) {
 394		fwnode_handle_put(param->fwspec->iommu_fwnode);
 395		kfree(param->fwspec);
 396	}
 397	kfree(param);
 398}
 399
 400/*
 401 * Internal equivalent of device_iommu_mapped() for when we care that a device
 402 * actually has API ops, and don't want false positives from VFIO-only groups.
 403 */
 404static bool dev_has_iommu(struct device *dev)
 405{
 406	return dev->iommu && dev->iommu->iommu_dev;
 407}
 408
 409static u32 dev_iommu_get_max_pasids(struct device *dev)
 410{
 411	u32 max_pasids = 0, bits = 0;
 412	int ret;
 413
 414	if (dev_is_pci(dev)) {
 415		ret = pci_max_pasids(to_pci_dev(dev));
 416		if (ret > 0)
 417			max_pasids = ret;
 418	} else {
 419		ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
 420		if (!ret)
 421			max_pasids = 1UL << bits;
 422	}
 423
 424	return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
 425}
 426
 427void dev_iommu_priv_set(struct device *dev, void *priv)
 428{
 429	/* FSL_PAMU does something weird */
 430	if (!IS_ENABLED(CONFIG_FSL_PAMU))
 431		lockdep_assert_held(&iommu_probe_device_lock);
 432	dev->iommu->priv = priv;
 433}
 434EXPORT_SYMBOL_GPL(dev_iommu_priv_set);
 435
 436/*
 437 * Init the dev->iommu and dev->iommu_group in the struct device and get the
 438 * driver probed
 439 */
 440static int iommu_init_device(struct device *dev)
 441{
 442	const struct iommu_ops *ops;
 443	struct iommu_device *iommu_dev;
 444	struct iommu_group *group;
 445	int ret;
 446
 447	if (!dev_iommu_get(dev))
 448		return -ENOMEM;
 449	/*
 450	 * For FDT-based systems and ACPI IORT/VIOT, the common firmware parsing
 451	 * is buried in the bus dma_configure path. Properly unpicking that is
 452	 * still a big job, so for now just invoke the whole thing. The device
 453	 * already having a driver bound means dma_configure has already run and
 454	 * found no IOMMU to wait for, so there's no point calling it again.
 455	 */
 456	if (!dev->iommu->fwspec && !dev->driver && dev->bus->dma_configure) {
 457		mutex_unlock(&iommu_probe_device_lock);
 458		dev->bus->dma_configure(dev);
 459		mutex_lock(&iommu_probe_device_lock);
 460		/* If another instance finished the job for us, skip it */
 461		if (!dev->iommu || dev->iommu_group)
 462			return -ENODEV;
 463	}
 464	/*
 465	 * At this point, relevant devices either now have a fwspec which will
 466	 * match ops registered with a non-NULL fwnode, or we can reasonably
 467	 * assume that only one of Intel, AMD, s390, PAMU or legacy SMMUv2 can
 468	 * be present, and that any of their registered instances has suitable
 469	 * ops for probing, and thus cheekily co-opt the same mechanism.
 470	 */
 471	ops = iommu_fwspec_ops(dev->iommu->fwspec);
 472	if (!ops) {
 473		ret = -ENODEV;
 474		goto err_free;
 475	}
 476
 477	if (!try_module_get(ops->owner)) {
 478		ret = -EINVAL;
 479		goto err_free;
 480	}
 481
 482	iommu_dev = ops->probe_device(dev);
 483	if (IS_ERR(iommu_dev)) {
 484		ret = PTR_ERR(iommu_dev);
 485		goto err_module_put;
 486	}
 487	dev->iommu->iommu_dev = iommu_dev;
 488
 489	ret = iommu_device_link(iommu_dev, dev);
 490	if (ret)
 491		goto err_release;
 492
 493	group = ops->device_group(dev);
 494	if (WARN_ON_ONCE(group == NULL))
 495		group = ERR_PTR(-EINVAL);
 496	if (IS_ERR(group)) {
 497		ret = PTR_ERR(group);
 498		goto err_unlink;
 499	}
 500	dev->iommu_group = group;
 501
 502	dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
 503	if (ops->is_attach_deferred)
 504		dev->iommu->attach_deferred = ops->is_attach_deferred(dev);
 505	return 0;
 506
 507err_unlink:
 508	iommu_device_unlink(iommu_dev, dev);
 509err_release:
 510	if (ops->release_device)
 511		ops->release_device(dev);
 512err_module_put:
 513	module_put(ops->owner);
 514err_free:
 515	dev->iommu->iommu_dev = NULL;
 516	dev_iommu_free(dev);
 517	return ret;
 518}
 519
 520static void iommu_deinit_device(struct device *dev)
 521{
 522	struct iommu_group *group = dev->iommu_group;
 523	const struct iommu_ops *ops = dev_iommu_ops(dev);
 524
 525	lockdep_assert_held(&group->mutex);
 526
 527	iommu_device_unlink(dev->iommu->iommu_dev, dev);
 528
 529	/*
 530	 * release_device() must stop using any attached domain on the device.
 531	 * If there are still other devices in the group, they are not affected
 532	 * by this callback.
 533	 *
 534	 * If the iommu driver provides release_domain, the core code ensures
 535	 * that domain is attached prior to calling release_device. Drivers can
 536	 * use this to enforce a translation on the idle iommu. Typically, the
 537	 * global static blocked_domain is a good choice.
 538	 *
 539	 * Otherwise, the iommu driver must set the device to either an identity
 540	 * or a blocking translation in release_device() and stop using any
 541	 * domain pointer, as it is going to be freed.
 542	 *
 543	 * Regardless, if a delayed attach never occurred, then the release
 544	 * should still avoid touching any hardware configuration either.
 545	 */
 546	if (!dev->iommu->attach_deferred && ops->release_domain) {
 547		struct iommu_domain *release_domain = ops->release_domain;
 548
 549		/*
 550		 * If the device requires direct mappings then it should not
 551		 * be parked on a BLOCKED domain during release as that would
 552		 * break the direct mappings.
 553		 */
 554		if (dev->iommu->require_direct && ops->identity_domain &&
 555		    release_domain == ops->blocked_domain)
 556			release_domain = ops->identity_domain;
 557
 558		release_domain->ops->attach_dev(release_domain, dev,
 559						group->domain);
 560	}
 561
 562	if (ops->release_device)
 563		ops->release_device(dev);
 564
 565	/*
 566	 * If this is the last driver to use the group then we must free the
 567	 * domains before we do the module_put().
 568	 */
 569	if (list_empty(&group->devices)) {
 570		if (group->default_domain) {
 571			iommu_domain_free(group->default_domain);
 572			group->default_domain = NULL;
 573		}
 574		if (group->blocking_domain) {
 575			iommu_domain_free(group->blocking_domain);
 576			group->blocking_domain = NULL;
 577		}
 578		group->domain = NULL;
 579	}
 580
 581	/* Caller must put iommu_group */
 582	dev->iommu_group = NULL;
 583	module_put(ops->owner);
 584	dev_iommu_free(dev);
 585#ifdef CONFIG_IOMMU_DMA
 586	dev->dma_iommu = false;
 587#endif
 588}
 589
 590static struct iommu_domain *pasid_array_entry_to_domain(void *entry)
 591{
 592	if (xa_pointer_tag(entry) == IOMMU_PASID_ARRAY_DOMAIN)
 593		return xa_untag_pointer(entry);
 594	return ((struct iommu_attach_handle *)xa_untag_pointer(entry))->domain;
 595}
 596
 597DEFINE_MUTEX(iommu_probe_device_lock);
 598
 599static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
 600{
 601	struct iommu_group *group;
 602	struct group_device *gdev;
 603	int ret;
 604
 605	/*
 606	 * Serialise to avoid races between IOMMU drivers registering in
 607	 * parallel and/or the "replay" calls from ACPI/OF code via client
 608	 * driver probe. Once the latter have been cleaned up we should
 609	 * probably be able to use device_lock() here to minimise the scope,
 610	 * but for now enforcing a simple global ordering is fine.
 611	 */
 612	lockdep_assert_held(&iommu_probe_device_lock);
 613
 614	/* Device is probed already if in a group */
 615	if (dev->iommu_group)
 616		return 0;
 617
 618	ret = iommu_init_device(dev);
 619	if (ret)
 620		return ret;
 621	/*
 622	 * And if we do now see any replay calls, they would indicate someone
 623	 * misusing the dma_configure path outside bus code.
 624	 */
 625	if (dev->driver)
 626		dev_WARN(dev, "late IOMMU probe at driver bind, something fishy here!\n");
 627
 628	group = dev->iommu_group;
 629	gdev = iommu_group_alloc_device(group, dev);
 630	mutex_lock(&group->mutex);
 631	if (IS_ERR(gdev)) {
 632		ret = PTR_ERR(gdev);
 633		goto err_put_group;
 634	}
 635
 636	/*
 637	 * The gdev must be in the list before calling
 638	 * iommu_setup_default_domain()
 639	 */
 640	list_add_tail(&gdev->list, &group->devices);
 641	WARN_ON(group->default_domain && !group->domain);
 642	if (group->default_domain)
 643		iommu_create_device_direct_mappings(group->default_domain, dev);
 644	if (group->domain) {
 645		ret = __iommu_device_set_domain(group, dev, group->domain, NULL,
 646						0);
 647		if (ret)
 648			goto err_remove_gdev;
 649	} else if (!group->default_domain && !group_list) {
 650		ret = iommu_setup_default_domain(group, 0);
 651		if (ret)
 652			goto err_remove_gdev;
 653	} else if (!group->default_domain) {
 654		/*
 655		 * With a group_list argument we defer the default_domain setup
 656		 * to the caller by providing a de-duplicated list of groups
 657		 * that need further setup.
 658		 */
 659		if (list_empty(&group->entry))
 660			list_add_tail(&group->entry, group_list);
 661	}
 662
 663	if (group->default_domain)
 664		iommu_setup_dma_ops(dev);
 665
 666	mutex_unlock(&group->mutex);
 667
 668	return 0;
 669
 670err_remove_gdev:
 671	list_del(&gdev->list);
 672	__iommu_group_free_device(group, gdev);
 673err_put_group:
 674	iommu_deinit_device(dev);
 675	mutex_unlock(&group->mutex);
 676	iommu_group_put(group);
 677
 678	return ret;
 679}
 680
 681int iommu_probe_device(struct device *dev)
 682{
 683	const struct iommu_ops *ops;
 684	int ret;
 685
 686	mutex_lock(&iommu_probe_device_lock);
 687	ret = __iommu_probe_device(dev, NULL);
 688	mutex_unlock(&iommu_probe_device_lock);
 689	if (ret)
 690		return ret;
 691
 692	ops = dev_iommu_ops(dev);
 693	if (ops->probe_finalize)
 694		ops->probe_finalize(dev);
 695
 696	return 0;
 697}
 698
 699static void __iommu_group_free_device(struct iommu_group *group,
 700				      struct group_device *grp_dev)
 701{
 702	struct device *dev = grp_dev->dev;
 703
 704	sysfs_remove_link(group->devices_kobj, grp_dev->name);
 705	sysfs_remove_link(&dev->kobj, "iommu_group");
 706
 707	trace_remove_device_from_group(group->id, dev);
 708
 709	/*
 710	 * If the group has become empty then ownership must have been
 711	 * released, and the current domain must be set back to NULL or
 712	 * the default domain.
 713	 */
 714	if (list_empty(&group->devices))
 715		WARN_ON(group->owner_cnt ||
 716			group->domain != group->default_domain);
 717
 718	kfree(grp_dev->name);
 719	kfree(grp_dev);
 720}
 721
 722/* Remove the iommu_group from the struct device. */
 723static void __iommu_group_remove_device(struct device *dev)
 724{
 725	struct iommu_group *group = dev->iommu_group;
 726	struct group_device *device;
 727
 728	mutex_lock(&group->mutex);
 729	for_each_group_device(group, device) {
 730		if (device->dev != dev)
 731			continue;
 732
 733		list_del(&device->list);
 734		__iommu_group_free_device(group, device);
 735		if (dev_has_iommu(dev))
 736			iommu_deinit_device(dev);
 737		else
 738			dev->iommu_group = NULL;
 739		break;
 740	}
 741	mutex_unlock(&group->mutex);
 742
 743	/*
 744	 * Pairs with the get in iommu_init_device() or
 745	 * iommu_group_add_device()
 746	 */
 747	iommu_group_put(group);
 748}
 749
 750static void iommu_release_device(struct device *dev)
 751{
 752	struct iommu_group *group = dev->iommu_group;
 753
 754	if (group)
 755		__iommu_group_remove_device(dev);
 756
 757	/* Free any fwspec if no iommu_driver was ever attached */
 758	if (dev->iommu)
 759		dev_iommu_free(dev);
 760}
 761
 762static int __init iommu_set_def_domain_type(char *str)
 763{
 764	bool pt;
 765	int ret;
 766
 767	ret = kstrtobool(str, &pt);
 768	if (ret)
 769		return ret;
 770
 771	if (pt)
 772		iommu_set_default_passthrough(true);
 773	else
 774		iommu_set_default_translated(true);
 775
 776	return 0;
 777}
 778early_param("iommu.passthrough", iommu_set_def_domain_type);
 779
 780static int __init iommu_dma_setup(char *str)
 781{
 782	int ret = kstrtobool(str, &iommu_dma_strict);
 783
 784	if (!ret)
 785		iommu_cmd_line |= IOMMU_CMD_LINE_STRICT;
 786	return ret;
 787}
 788early_param("iommu.strict", iommu_dma_setup);
 789
 790void iommu_set_dma_strict(void)
 791{
 792	iommu_dma_strict = true;
 793	if (iommu_def_domain_type == IOMMU_DOMAIN_DMA_FQ)
 794		iommu_def_domain_type = IOMMU_DOMAIN_DMA;
 795}
 796
 797static ssize_t iommu_group_attr_show(struct kobject *kobj,
 798				     struct attribute *__attr, char *buf)
 799{
 800	struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
 801	struct iommu_group *group = to_iommu_group(kobj);
 802	ssize_t ret = -EIO;
 803
 804	if (attr->show)
 805		ret = attr->show(group, buf);
 806	return ret;
 807}
 808
 809static ssize_t iommu_group_attr_store(struct kobject *kobj,
 810				      struct attribute *__attr,
 811				      const char *buf, size_t count)
 812{
 813	struct iommu_group_attribute *attr = to_iommu_group_attr(__attr);
 814	struct iommu_group *group = to_iommu_group(kobj);
 815	ssize_t ret = -EIO;
 816
 817	if (attr->store)
 818		ret = attr->store(group, buf, count);
 819	return ret;
 820}
 821
 822static const struct sysfs_ops iommu_group_sysfs_ops = {
 823	.show = iommu_group_attr_show,
 824	.store = iommu_group_attr_store,
 825};
 826
 827static int iommu_group_create_file(struct iommu_group *group,
 828				   struct iommu_group_attribute *attr)
 829{
 830	return sysfs_create_file(&group->kobj, &attr->attr);
 831}
 832
 833static void iommu_group_remove_file(struct iommu_group *group,
 834				    struct iommu_group_attribute *attr)
 835{
 836	sysfs_remove_file(&group->kobj, &attr->attr);
 837}
 838
 839static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
 840{
 841	return sysfs_emit(buf, "%s\n", group->name);
 842}
 843
 844/**
 845 * iommu_insert_resv_region - Insert a new region in the
 846 * list of reserved regions.
 847 * @new: new region to insert
 848 * @regions: list of regions
 849 *
 850 * Elements are sorted by start address and overlapping segments
 851 * of the same type are merged.
 852 */
 853static int iommu_insert_resv_region(struct iommu_resv_region *new,
 854				    struct list_head *regions)
 855{
 856	struct iommu_resv_region *iter, *tmp, *nr, *top;
 857	LIST_HEAD(stack);
 858
 859	nr = iommu_alloc_resv_region(new->start, new->length,
 860				     new->prot, new->type, GFP_KERNEL);
 861	if (!nr)
 862		return -ENOMEM;
 863
 864	/* First add the new element based on start address sorting */
 865	list_for_each_entry(iter, regions, list) {
 866		if (nr->start < iter->start ||
 867		    (nr->start == iter->start && nr->type <= iter->type))
 868			break;
 869	}
 870	list_add_tail(&nr->list, &iter->list);
 871
 872	/* Merge overlapping segments of type nr->type in @regions, if any */
 873	list_for_each_entry_safe(iter, tmp, regions, list) {
 874		phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
 875
 876		/* no merge needed on elements of different types than @new */
 877		if (iter->type != new->type) {
 878			list_move_tail(&iter->list, &stack);
 879			continue;
 880		}
 881
 882		/* look for the last stack element of same type as @iter */
 883		list_for_each_entry_reverse(top, &stack, list)
 884			if (top->type == iter->type)
 885				goto check_overlap;
 886
 887		list_move_tail(&iter->list, &stack);
 888		continue;
 889
 890check_overlap:
 891		top_end = top->start + top->length - 1;
 892
 893		if (iter->start > top_end + 1) {
 894			list_move_tail(&iter->list, &stack);
 895		} else {
 896			top->length = max(top_end, iter_end) - top->start + 1;
 897			list_del(&iter->list);
 898			kfree(iter);
 899		}
 900	}
 901	list_splice(&stack, regions);
 902	return 0;
 903}
 904
 905static int
 906iommu_insert_device_resv_regions(struct list_head *dev_resv_regions,
 907				 struct list_head *group_resv_regions)
 908{
 909	struct iommu_resv_region *entry;
 910	int ret = 0;
 911
 912	list_for_each_entry(entry, dev_resv_regions, list) {
 913		ret = iommu_insert_resv_region(entry, group_resv_regions);
 914		if (ret)
 915			break;
 916	}
 917	return ret;
 918}
 919
 920int iommu_get_group_resv_regions(struct iommu_group *group,
 921				 struct list_head *head)
 922{
 923	struct group_device *device;
 924	int ret = 0;
 925
 926	mutex_lock(&group->mutex);
 927	for_each_group_device(group, device) {
 928		struct list_head dev_resv_regions;
 929
 930		/*
 931		 * Non-API groups still expose reserved_regions in sysfs,
 932		 * so filter out calls that get here that way.
 933		 */
 934		if (!dev_has_iommu(device->dev))
 935			break;
 936
 937		INIT_LIST_HEAD(&dev_resv_regions);
 938		iommu_get_resv_regions(device->dev, &dev_resv_regions);
 939		ret = iommu_insert_device_resv_regions(&dev_resv_regions, head);
 940		iommu_put_resv_regions(device->dev, &dev_resv_regions);
 941		if (ret)
 942			break;
 943	}
 944	mutex_unlock(&group->mutex);
 945	return ret;
 946}
 947EXPORT_SYMBOL_GPL(iommu_get_group_resv_regions);
 948
 949static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,
 950					     char *buf)
 951{
 952	struct iommu_resv_region *region, *next;
 953	struct list_head group_resv_regions;
 954	int offset = 0;
 955
 956	INIT_LIST_HEAD(&group_resv_regions);
 957	iommu_get_group_resv_regions(group, &group_resv_regions);
 958
 959	list_for_each_entry_safe(region, next, &group_resv_regions, list) {
 960		offset += sysfs_emit_at(buf, offset, "0x%016llx 0x%016llx %s\n",
 961					(long long)region->start,
 962					(long long)(region->start +
 963						    region->length - 1),
 964					iommu_group_resv_type_string[region->type]);
 965		kfree(region);
 966	}
 967
 968	return offset;
 969}
 970
 971static ssize_t iommu_group_show_type(struct iommu_group *group,
 972				     char *buf)
 973{
 974	char *type = "unknown";
 975
 976	mutex_lock(&group->mutex);
 977	if (group->default_domain) {
 978		switch (group->default_domain->type) {
 979		case IOMMU_DOMAIN_BLOCKED:
 980			type = "blocked";
 981			break;
 982		case IOMMU_DOMAIN_IDENTITY:
 983			type = "identity";
 984			break;
 985		case IOMMU_DOMAIN_UNMANAGED:
 986			type = "unmanaged";
 987			break;
 988		case IOMMU_DOMAIN_DMA:
 989			type = "DMA";
 990			break;
 991		case IOMMU_DOMAIN_DMA_FQ:
 992			type = "DMA-FQ";
 993			break;
 994		}
 995	}
 996	mutex_unlock(&group->mutex);
 997
 998	return sysfs_emit(buf, "%s\n", type);
 999}
1000
1001static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL);
1002
1003static IOMMU_GROUP_ATTR(reserved_regions, 0444,
1004			iommu_group_show_resv_regions, NULL);
1005
1006static IOMMU_GROUP_ATTR(type, 0644, iommu_group_show_type,
1007			iommu_group_store_type);
1008
1009static void iommu_group_release(struct kobject *kobj)
1010{
1011	struct iommu_group *group = to_iommu_group(kobj);
1012
1013	pr_debug("Releasing group %d\n", group->id);
1014
1015	if (group->iommu_data_release)
1016		group->iommu_data_release(group->iommu_data);
1017
1018	ida_free(&iommu_group_ida, group->id);
1019
1020	/* Domains are free'd by iommu_deinit_device() */
1021	WARN_ON(group->default_domain);
1022	WARN_ON(group->blocking_domain);
1023
1024	kfree(group->name);
1025	kfree(group);
1026}
1027
1028static const struct kobj_type iommu_group_ktype = {
1029	.sysfs_ops = &iommu_group_sysfs_ops,
1030	.release = iommu_group_release,
1031};
1032
1033/**
1034 * iommu_group_alloc - Allocate a new group
1035 *
1036 * This function is called by an iommu driver to allocate a new iommu
1037 * group.  The iommu group represents the minimum granularity of the iommu.
1038 * Upon successful return, the caller holds a reference to the supplied
1039 * group in order to hold the group until devices are added.  Use
1040 * iommu_group_put() to release this extra reference count, allowing the
1041 * group to be automatically reclaimed once it has no devices or external
1042 * references.
1043 */
1044struct iommu_group *iommu_group_alloc(void)
1045{
1046	struct iommu_group *group;
1047	int ret;
1048
1049	group = kzalloc(sizeof(*group), GFP_KERNEL);
1050	if (!group)
1051		return ERR_PTR(-ENOMEM);
1052
1053	group->kobj.kset = iommu_group_kset;
1054	mutex_init(&group->mutex);
1055	INIT_LIST_HEAD(&group->devices);
1056	INIT_LIST_HEAD(&group->entry);
1057	xa_init(&group->pasid_array);
1058
1059	ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
1060	if (ret < 0) {
1061		kfree(group);
1062		return ERR_PTR(ret);
1063	}
1064	group->id = ret;
1065
1066	ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
1067				   NULL, "%d", group->id);
1068	if (ret) {
1069		kobject_put(&group->kobj);
1070		return ERR_PTR(ret);
1071	}
1072
1073	group->devices_kobj = kobject_create_and_add("devices", &group->kobj);
1074	if (!group->devices_kobj) {
1075		kobject_put(&group->kobj); /* triggers .release & free */
1076		return ERR_PTR(-ENOMEM);
1077	}
1078
1079	/*
1080	 * The devices_kobj holds a reference on the group kobject, so
1081	 * as long as that exists so will the group.  We can therefore
1082	 * use the devices_kobj for reference counting.
1083	 */
1084	kobject_put(&group->kobj);
1085
1086	ret = iommu_group_create_file(group,
1087				      &iommu_group_attr_reserved_regions);
1088	if (ret) {
1089		kobject_put(group->devices_kobj);
1090		return ERR_PTR(ret);
1091	}
1092
1093	ret = iommu_group_create_file(group, &iommu_group_attr_type);
1094	if (ret) {
1095		kobject_put(group->devices_kobj);
1096		return ERR_PTR(ret);
1097	}
1098
1099	pr_debug("Allocated group %d\n", group->id);
1100
1101	return group;
1102}
1103EXPORT_SYMBOL_GPL(iommu_group_alloc);
1104
1105/**
1106 * iommu_group_get_iommudata - retrieve iommu_data registered for a group
1107 * @group: the group
1108 *
1109 * iommu drivers can store data in the group for use when doing iommu
1110 * operations.  This function provides a way to retrieve it.  Caller
1111 * should hold a group reference.
1112 */
1113void *iommu_group_get_iommudata(struct iommu_group *group)
1114{
1115	return group->iommu_data;
1116}
1117EXPORT_SYMBOL_GPL(iommu_group_get_iommudata);
1118
1119/**
1120 * iommu_group_set_iommudata - set iommu_data for a group
1121 * @group: the group
1122 * @iommu_data: new data
1123 * @release: release function for iommu_data
1124 *
1125 * iommu drivers can store data in the group for use when doing iommu
1126 * operations.  This function provides a way to set the data after
1127 * the group has been allocated.  Caller should hold a group reference.
1128 */
1129void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,
1130			       void (*release)(void *iommu_data))
1131{
1132	group->iommu_data = iommu_data;
1133	group->iommu_data_release = release;
1134}
1135EXPORT_SYMBOL_GPL(iommu_group_set_iommudata);
1136
1137/**
1138 * iommu_group_set_name - set name for a group
1139 * @group: the group
1140 * @name: name
1141 *
1142 * Allow iommu driver to set a name for a group.  When set it will
1143 * appear in a name attribute file under the group in sysfs.
1144 */
1145int iommu_group_set_name(struct iommu_group *group, const char *name)
1146{
1147	int ret;
1148
1149	if (group->name) {
1150		iommu_group_remove_file(group, &iommu_group_attr_name);
1151		kfree(group->name);
1152		group->name = NULL;
1153		if (!name)
1154			return 0;
1155	}
1156
1157	group->name = kstrdup(name, GFP_KERNEL);
1158	if (!group->name)
1159		return -ENOMEM;
1160
1161	ret = iommu_group_create_file(group, &iommu_group_attr_name);
1162	if (ret) {
1163		kfree(group->name);
1164		group->name = NULL;
1165		return ret;
1166	}
1167
1168	return 0;
1169}
1170EXPORT_SYMBOL_GPL(iommu_group_set_name);
1171
1172static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
1173					       struct device *dev)
1174{
1175	struct iommu_resv_region *entry;
1176	struct list_head mappings;
1177	unsigned long pg_size;
1178	int ret = 0;
1179
1180	pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
1181	INIT_LIST_HEAD(&mappings);
1182
1183	if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))
1184		return -EINVAL;
1185
1186	iommu_get_resv_regions(dev, &mappings);
1187
1188	/* We need to consider overlapping regions for different devices */
1189	list_for_each_entry(entry, &mappings, list) {
1190		dma_addr_t start, end, addr;
1191		size_t map_size = 0;
1192
1193		if (entry->type == IOMMU_RESV_DIRECT)
1194			dev->iommu->require_direct = 1;
1195
1196		if ((entry->type != IOMMU_RESV_DIRECT &&
1197		     entry->type != IOMMU_RESV_DIRECT_RELAXABLE) ||
1198		    !iommu_is_dma_domain(domain))
1199			continue;
1200
1201		start = ALIGN(entry->start, pg_size);
1202		end   = ALIGN(entry->start + entry->length, pg_size);
1203
1204		for (addr = start; addr <= end; addr += pg_size) {
1205			phys_addr_t phys_addr;
1206
1207			if (addr == end)
1208				goto map_end;
1209
1210			/*
1211			 * Return address by iommu_iova_to_phys for 0 is
1212			 * ambiguous. Offset to address 1 if addr is 0.
1213			 */
1214			phys_addr = iommu_iova_to_phys(domain, addr ? addr : 1);
1215			if (!phys_addr) {
1216				map_size += pg_size;
1217				continue;
1218			}
1219
1220map_end:
1221			if (map_size) {
1222				ret = iommu_map(domain, addr - map_size,
1223						addr - map_size, map_size,
1224						entry->prot, GFP_KERNEL);
1225				if (ret)
1226					goto out;
1227				map_size = 0;
1228			}
1229		}
1230
1231	}
1232out:
1233	iommu_put_resv_regions(dev, &mappings);
1234
1235	return ret;
1236}
1237
1238/* This is undone by __iommu_group_free_device() */
1239static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
1240						     struct device *dev)
1241{
1242	int ret, i = 0;
1243	struct group_device *device;
1244
1245	device = kzalloc(sizeof(*device), GFP_KERNEL);
1246	if (!device)
1247		return ERR_PTR(-ENOMEM);
1248
1249	device->dev = dev;
1250
1251	ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");
1252	if (ret)
1253		goto err_free_device;
1254
1255	device->name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
1256rename:
1257	if (!device->name) {
1258		ret = -ENOMEM;
1259		goto err_remove_link;
1260	}
1261
1262	ret = sysfs_create_link_nowarn(group->devices_kobj,
1263				       &dev->kobj, device->name);
1264	if (ret) {
1265		if (ret == -EEXIST && i >= 0) {
1266			/*
1267			 * Account for the slim chance of collision
1268			 * and append an instance to the name.
1269			 */
1270			kfree(device->name);
1271			device->name = kasprintf(GFP_KERNEL, "%s.%d",
1272						 kobject_name(&dev->kobj), i++);
1273			goto rename;
1274		}
1275		goto err_free_name;
1276	}
1277
1278	trace_add_device_to_group(group->id, dev);
1279
1280	dev_info(dev, "Adding to iommu group %d\n", group->id);
1281
1282	return device;
1283
1284err_free_name:
1285	kfree(device->name);
1286err_remove_link:
1287	sysfs_remove_link(&dev->kobj, "iommu_group");
1288err_free_device:
1289	kfree(device);
1290	dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);
1291	return ERR_PTR(ret);
1292}
1293
1294/**
1295 * iommu_group_add_device - add a device to an iommu group
1296 * @group: the group into which to add the device (reference should be held)
1297 * @dev: the device
1298 *
1299 * This function is called by an iommu driver to add a device into a
1300 * group.  Adding a device increments the group reference count.
1301 */
1302int iommu_group_add_device(struct iommu_group *group, struct device *dev)
1303{
1304	struct group_device *gdev;
1305
1306	gdev = iommu_group_alloc_device(group, dev);
1307	if (IS_ERR(gdev))
1308		return PTR_ERR(gdev);
1309
1310	iommu_group_ref_get(group);
1311	dev->iommu_group = group;
1312
1313	mutex_lock(&group->mutex);
1314	list_add_tail(&gdev->list, &group->devices);
1315	mutex_unlock(&group->mutex);
1316	return 0;
1317}
1318EXPORT_SYMBOL_GPL(iommu_group_add_device);
1319
1320/**
1321 * iommu_group_remove_device - remove a device from it's current group
1322 * @dev: device to be removed
1323 *
1324 * This function is called by an iommu driver to remove the device from
1325 * it's current group.  This decrements the iommu group reference count.
1326 */
1327void iommu_group_remove_device(struct device *dev)
1328{
1329	struct iommu_group *group = dev->iommu_group;
1330
1331	if (!group)
1332		return;
1333
1334	dev_info(dev, "Removing from iommu group %d\n", group->id);
1335
1336	__iommu_group_remove_device(dev);
1337}
1338EXPORT_SYMBOL_GPL(iommu_group_remove_device);
1339
1340#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
1341/**
1342 * iommu_group_mutex_assert - Check device group mutex lock
1343 * @dev: the device that has group param set
1344 *
1345 * This function is called by an iommu driver to check whether it holds
1346 * group mutex lock for the given device or not.
1347 *
1348 * Note that this function must be called after device group param is set.
1349 */
1350void iommu_group_mutex_assert(struct device *dev)
1351{
1352	struct iommu_group *group = dev->iommu_group;
1353
1354	lockdep_assert_held(&group->mutex);
1355}
1356EXPORT_SYMBOL_GPL(iommu_group_mutex_assert);
1357#endif
1358
1359static struct device *iommu_group_first_dev(struct iommu_group *group)
1360{
1361	lockdep_assert_held(&group->mutex);
1362	return list_first_entry(&group->devices, struct group_device, list)->dev;
1363}
1364
1365/**
1366 * iommu_group_for_each_dev - iterate over each device in the group
1367 * @group: the group
1368 * @data: caller opaque data to be passed to callback function
1369 * @fn: caller supplied callback function
1370 *
1371 * This function is called by group users to iterate over group devices.
1372 * Callers should hold a reference count to the group during callback.
1373 * The group->mutex is held across callbacks, which will block calls to
1374 * iommu_group_add/remove_device.
1375 */
1376int iommu_group_for_each_dev(struct iommu_group *group, void *data,
1377			     int (*fn)(struct device *, void *))
1378{
1379	struct group_device *device;
1380	int ret = 0;
1381
1382	mutex_lock(&group->mutex);
1383	for_each_group_device(group, device) {
1384		ret = fn(device->dev, data);
1385		if (ret)
1386			break;
1387	}
1388	mutex_unlock(&group->mutex);
1389
1390	return ret;
1391}
1392EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
1393
1394/**
1395 * iommu_group_get - Return the group for a device and increment reference
1396 * @dev: get the group that this device belongs to
1397 *
1398 * This function is called by iommu drivers and users to get the group
1399 * for the specified device.  If found, the group is returned and the group
1400 * reference in incremented, else NULL.
1401 */
1402struct iommu_group *iommu_group_get(struct device *dev)
1403{
1404	struct iommu_group *group = dev->iommu_group;
1405
1406	if (group)
1407		kobject_get(group->devices_kobj);
1408
1409	return group;
1410}
1411EXPORT_SYMBOL_GPL(iommu_group_get);
1412
1413/**
1414 * iommu_group_ref_get - Increment reference on a group
1415 * @group: the group to use, must not be NULL
1416 *
1417 * This function is called by iommu drivers to take additional references on an
1418 * existing group.  Returns the given group for convenience.
1419 */
1420struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
1421{
1422	kobject_get(group->devices_kobj);
1423	return group;
1424}
1425EXPORT_SYMBOL_GPL(iommu_group_ref_get);
1426
1427/**
1428 * iommu_group_put - Decrement group reference
1429 * @group: the group to use
1430 *
1431 * This function is called by iommu drivers and users to release the
1432 * iommu group.  Once the reference count is zero, the group is released.
1433 */
1434void iommu_group_put(struct iommu_group *group)
1435{
1436	if (group)
1437		kobject_put(group->devices_kobj);
1438}
1439EXPORT_SYMBOL_GPL(iommu_group_put);
1440
1441/**
1442 * iommu_group_id - Return ID for a group
1443 * @group: the group to ID
1444 *
1445 * Return the unique ID for the group matching the sysfs group number.
1446 */
1447int iommu_group_id(struct iommu_group *group)
1448{
1449	return group->id;
1450}
1451EXPORT_SYMBOL_GPL(iommu_group_id);
1452
1453static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1454					       unsigned long *devfns);
1455
1456/*
1457 * To consider a PCI device isolated, we require ACS to support Source
1458 * Validation, Request Redirection, Completer Redirection, and Upstream
1459 * Forwarding.  This effectively means that devices cannot spoof their
1460 * requester ID, requests and completions cannot be redirected, and all
1461 * transactions are forwarded upstream, even as it passes through a
1462 * bridge where the target device is downstream.
1463 */
1464#define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
1465
1466/*
1467 * For multifunction devices which are not isolated from each other, find
1468 * all the other non-isolated functions and look for existing groups.  For
1469 * each function, we also need to look for aliases to or from other devices
1470 * that may already have a group.
1471 */
1472static struct iommu_group *get_pci_function_alias_group(struct pci_dev *pdev,
1473							unsigned long *devfns)
1474{
1475	struct pci_dev *tmp = NULL;
1476	struct iommu_group *group;
1477
1478	if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
1479		return NULL;
1480
1481	for_each_pci_dev(tmp) {
1482		if (tmp == pdev || tmp->bus != pdev->bus ||
1483		    PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
1484		    pci_acs_enabled(tmp, REQ_ACS_FLAGS))
1485			continue;
1486
1487		group = get_pci_alias_group(tmp, devfns);
1488		if (group) {
1489			pci_dev_put(tmp);
1490			return group;
1491		}
1492	}
1493
1494	return NULL;
1495}
1496
1497/*
1498 * Look for aliases to or from the given device for existing groups. DMA
1499 * aliases are only supported on the same bus, therefore the search
1500 * space is quite small (especially since we're really only looking at pcie
1501 * device, and therefore only expect multiple slots on the root complex or
1502 * downstream switch ports).  It's conceivable though that a pair of
1503 * multifunction devices could have aliases between them that would cause a
1504 * loop.  To prevent this, we use a bitmap to track where we've been.
1505 */
1506static struct iommu_group *get_pci_alias_group(struct pci_dev *pdev,
1507					       unsigned long *devfns)
1508{
1509	struct pci_dev *tmp = NULL;
1510	struct iommu_group *group;
1511
1512	if (test_and_set_bit(pdev->devfn & 0xff, devfns))
1513		return NULL;
1514
1515	group = iommu_group_get(&pdev->dev);
1516	if (group)
1517		return group;
1518
1519	for_each_pci_dev(tmp) {
1520		if (tmp == pdev || tmp->bus != pdev->bus)
1521			continue;
1522
1523		/* We alias them or they alias us */
1524		if (pci_devs_are_dma_aliases(pdev, tmp)) {
1525			group = get_pci_alias_group(tmp, devfns);
1526			if (group) {
1527				pci_dev_put(tmp);
1528				return group;
1529			}
1530
1531			group = get_pci_function_alias_group(tmp, devfns);
1532			if (group) {
1533				pci_dev_put(tmp);
1534				return group;
1535			}
1536		}
1537	}
1538
1539	return NULL;
1540}
1541
1542struct group_for_pci_data {
1543	struct pci_dev *pdev;
1544	struct iommu_group *group;
1545};
1546
1547/*
1548 * DMA alias iterator callback, return the last seen device.  Stop and return
1549 * the IOMMU group if we find one along the way.
1550 */
1551static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
1552{
1553	struct group_for_pci_data *data = opaque;
1554
1555	data->pdev = pdev;
1556	data->group = iommu_group_get(&pdev->dev);
1557
1558	return data->group != NULL;
1559}
1560
1561/*
1562 * Generic device_group call-back function. It just allocates one
1563 * iommu-group per device.
1564 */
1565struct iommu_group *generic_device_group(struct device *dev)
1566{
1567	return iommu_group_alloc();
1568}
1569EXPORT_SYMBOL_GPL(generic_device_group);
1570
1571/*
1572 * Generic device_group call-back function. It just allocates one
1573 * iommu-group per iommu driver instance shared by every device
1574 * probed by that iommu driver.
1575 */
1576struct iommu_group *generic_single_device_group(struct device *dev)
1577{
1578	struct iommu_device *iommu = dev->iommu->iommu_dev;
1579
1580	if (!iommu->singleton_group) {
1581		struct iommu_group *group;
1582
1583		group = iommu_group_alloc();
1584		if (IS_ERR(group))
1585			return group;
1586		iommu->singleton_group = group;
1587	}
1588	return iommu_group_ref_get(iommu->singleton_group);
1589}
1590EXPORT_SYMBOL_GPL(generic_single_device_group);
1591
1592/*
1593 * Use standard PCI bus topology, isolation features, and DMA alias quirks
1594 * to find or create an IOMMU group for a device.
1595 */
1596struct iommu_group *pci_device_group(struct device *dev)
1597{
1598	struct pci_dev *pdev = to_pci_dev(dev);
1599	struct group_for_pci_data data;
1600	struct pci_bus *bus;
1601	struct iommu_group *group = NULL;
1602	u64 devfns[4] = { 0 };
1603
1604	if (WARN_ON(!dev_is_pci(dev)))
1605		return ERR_PTR(-EINVAL);
1606
1607	/*
1608	 * Find the upstream DMA alias for the device.  A device must not
1609	 * be aliased due to topology in order to have its own IOMMU group.
1610	 * If we find an alias along the way that already belongs to a
1611	 * group, use it.
1612	 */
1613	if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
1614		return data.group;
1615
1616	pdev = data.pdev;
1617
1618	/*
1619	 * Continue upstream from the point of minimum IOMMU granularity
1620	 * due to aliases to the point where devices are protected from
1621	 * peer-to-peer DMA by PCI ACS.  Again, if we find an existing
1622	 * group, use it.
1623	 */
1624	for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
1625		if (!bus->self)
1626			continue;
1627
1628		if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1629			break;
1630
1631		pdev = bus->self;
1632
1633		group = iommu_group_get(&pdev->dev);
1634		if (group)
1635			return group;
1636	}
1637
1638	/*
1639	 * Look for existing groups on device aliases.  If we alias another
1640	 * device or another device aliases us, use the same group.
1641	 */
1642	group = get_pci_alias_group(pdev, (unsigned long *)devfns);
1643	if (group)
1644		return group;
1645
1646	/*
1647	 * Look for existing groups on non-isolated functions on the same
1648	 * slot and aliases of those funcions, if any.  No need to clear
1649	 * the search bitmap, the tested devfns are still valid.
1650	 */
1651	group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);
1652	if (group)
1653		return group;
1654
1655	/* No shared group found, allocate new */
1656	return iommu_group_alloc();
1657}
1658EXPORT_SYMBOL_GPL(pci_device_group);
1659
1660/* Get the IOMMU group for device on fsl-mc bus */
1661struct iommu_group *fsl_mc_device_group(struct device *dev)
1662{
1663	struct device *cont_dev = fsl_mc_cont_dev(dev);
1664	struct iommu_group *group;
1665
1666	group = iommu_group_get(cont_dev);
1667	if (!group)
1668		group = iommu_group_alloc();
1669	return group;
1670}
1671EXPORT_SYMBOL_GPL(fsl_mc_device_group);
1672
1673static struct iommu_domain *__iommu_alloc_identity_domain(struct device *dev)
1674{
1675	const struct iommu_ops *ops = dev_iommu_ops(dev);
1676	struct iommu_domain *domain;
1677
1678	if (ops->identity_domain)
1679		return ops->identity_domain;
1680
1681	if (ops->domain_alloc_identity) {
1682		domain = ops->domain_alloc_identity(dev);
1683		if (IS_ERR(domain))
1684			return domain;
1685	} else {
1686		return ERR_PTR(-EOPNOTSUPP);
1687	}
1688
1689	iommu_domain_init(domain, IOMMU_DOMAIN_IDENTITY, ops);
1690	return domain;
1691}
1692
1693static struct iommu_domain *
1694__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
1695{
1696	struct device *dev = iommu_group_first_dev(group);
1697	struct iommu_domain *dom;
1698
1699	if (group->default_domain && group->default_domain->type == req_type)
1700		return group->default_domain;
1701
1702	/*
1703	 * When allocating the DMA API domain assume that the driver is going to
1704	 * use PASID and make sure the RID's domain is PASID compatible.
1705	 */
1706	if (req_type & __IOMMU_DOMAIN_PAGING) {
1707		dom = __iommu_paging_domain_alloc_flags(dev, req_type,
1708			   dev->iommu->max_pasids ? IOMMU_HWPT_ALLOC_PASID : 0);
1709
1710		/*
1711		 * If driver does not support PASID feature then
1712		 * try to allocate non-PASID domain
1713		 */
1714		if (PTR_ERR(dom) == -EOPNOTSUPP)
1715			dom = __iommu_paging_domain_alloc_flags(dev, req_type, 0);
1716
1717		return dom;
1718	}
1719
1720	if (req_type == IOMMU_DOMAIN_IDENTITY)
1721		return __iommu_alloc_identity_domain(dev);
1722
1723	return ERR_PTR(-EINVAL);
1724}
1725
1726/*
1727 * req_type of 0 means "auto" which means to select a domain based on
1728 * iommu_def_domain_type or what the driver actually supports.
1729 */
1730static struct iommu_domain *
1731iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
1732{
1733	const struct iommu_ops *ops = dev_iommu_ops(iommu_group_first_dev(group));
1734	struct iommu_domain *dom;
1735
1736	lockdep_assert_held(&group->mutex);
1737
1738	/*
1739	 * Allow legacy drivers to specify the domain that will be the default
1740	 * domain. This should always be either an IDENTITY/BLOCKED/PLATFORM
1741	 * domain. Do not use in new drivers.
1742	 */
1743	if (ops->default_domain) {
1744		if (req_type != ops->default_domain->type)
1745			return ERR_PTR(-EINVAL);
1746		return ops->default_domain;
1747	}
1748
1749	if (req_type)
1750		return __iommu_group_alloc_default_domain(group, req_type);
1751
1752	/* The driver gave no guidance on what type to use, try the default */
1753	dom = __iommu_group_alloc_default_domain(group, iommu_def_domain_type);
1754	if (!IS_ERR(dom))
1755		return dom;
1756
1757	/* Otherwise IDENTITY and DMA_FQ defaults will try DMA */
1758	if (iommu_def_domain_type == IOMMU_DOMAIN_DMA)
1759		return ERR_PTR(-EINVAL);
1760	dom = __iommu_group_alloc_default_domain(group, IOMMU_DOMAIN_DMA);
1761	if (IS_ERR(dom))
1762		return dom;
1763
1764	pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
1765		iommu_def_domain_type, group->name);
1766	return dom;
1767}
1768
1769struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
1770{
1771	return group->default_domain;
1772}
1773
1774static int probe_iommu_group(struct device *dev, void *data)
1775{
1776	struct list_head *group_list = data;
1777	int ret;
1778
1779	mutex_lock(&iommu_probe_device_lock);
1780	ret = __iommu_probe_device(dev, group_list);
1781	mutex_unlock(&iommu_probe_device_lock);
1782	if (ret == -ENODEV)
1783		ret = 0;
1784
1785	return ret;
1786}
1787
1788static int iommu_bus_notifier(struct notifier_block *nb,
1789			      unsigned long action, void *data)
1790{
1791	struct device *dev = data;
1792
1793	if (action == BUS_NOTIFY_ADD_DEVICE) {
1794		int ret;
1795
1796		ret = iommu_probe_device(dev);
1797		return (ret) ? NOTIFY_DONE : NOTIFY_OK;
1798	} else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
1799		iommu_release_device(dev);
1800		return NOTIFY_OK;
1801	}
1802
1803	return 0;
1804}
1805
1806/*
1807 * Combine the driver's chosen def_domain_type across all the devices in a
1808 * group. Drivers must give a consistent result.
1809 */
1810static int iommu_get_def_domain_type(struct iommu_group *group,
1811				     struct device *dev, int cur_type)
1812{
1813	const struct iommu_ops *ops = dev_iommu_ops(dev);
1814	int type;
1815
1816	if (ops->default_domain) {
1817		/*
1818		 * Drivers that declare a global static default_domain will
1819		 * always choose that.
1820		 */
1821		type = ops->default_domain->type;
1822	} else {
1823		if (ops->def_domain_type)
1824			type = ops->def_domain_type(dev);
1825		else
1826			return cur_type;
1827	}
1828	if (!type || cur_type == type)
1829		return cur_type;
1830	if (!cur_type)
1831		return type;
1832
1833	dev_err_ratelimited(
1834		dev,
1835		"IOMMU driver error, requesting conflicting def_domain_type, %s and %s, for devices in group %u.\n",
1836		iommu_domain_type_str(cur_type), iommu_domain_type_str(type),
1837		group->id);
1838
1839	/*
1840	 * Try to recover, drivers are allowed to force IDENTITY or DMA, IDENTITY
1841	 * takes precedence.
1842	 */
1843	if (type == IOMMU_DOMAIN_IDENTITY)
1844		return type;
1845	return cur_type;
1846}
1847
1848/*
1849 * A target_type of 0 will select the best domain type. 0 can be returned in
1850 * this case meaning the global default should be used.
1851 */
1852static int iommu_get_default_domain_type(struct iommu_group *group,
1853					 int target_type)
1854{
1855	struct device *untrusted = NULL;
1856	struct group_device *gdev;
1857	int driver_type = 0;
1858
1859	lockdep_assert_held(&group->mutex);
1860
1861	/*
1862	 * ARM32 drivers supporting CONFIG_ARM_DMA_USE_IOMMU can declare an
1863	 * identity_domain and it will automatically become their default
1864	 * domain. Later on ARM_DMA_USE_IOMMU will install its UNMANAGED domain.
1865	 * Override the selection to IDENTITY.
1866	 */
1867	if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)) {
1868		static_assert(!(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) &&
1869				IS_ENABLED(CONFIG_IOMMU_DMA)));
1870		driver_type = IOMMU_DOMAIN_IDENTITY;
1871	}
1872
1873	for_each_group_device(group, gdev) {
1874		driver_type = iommu_get_def_domain_type(group, gdev->dev,
1875							driver_type);
1876
1877		if (dev_is_pci(gdev->dev) && to_pci_dev(gdev->dev)->untrusted) {
1878			/*
1879			 * No ARM32 using systems will set untrusted, it cannot
1880			 * work.
1881			 */
1882			if (WARN_ON(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)))
1883				return -1;
1884			untrusted = gdev->dev;
1885		}
1886	}
1887
1888	/*
1889	 * If the common dma ops are not selected in kconfig then we cannot use
1890	 * IOMMU_DOMAIN_DMA at all. Force IDENTITY if nothing else has been
1891	 * selected.
1892	 */
1893	if (!IS_ENABLED(CONFIG_IOMMU_DMA)) {
1894		if (WARN_ON(driver_type == IOMMU_DOMAIN_DMA))
1895			return -1;
1896		if (!driver_type)
1897			driver_type = IOMMU_DOMAIN_IDENTITY;
1898	}
1899
1900	if (untrusted) {
1901		if (driver_type && driver_type != IOMMU_DOMAIN_DMA) {
1902			dev_err_ratelimited(
1903				untrusted,
1904				"Device is not trusted, but driver is overriding group %u to %s, refusing to probe.\n",
1905				group->id, iommu_domain_type_str(driver_type));
1906			return -1;
1907		}
1908		driver_type = IOMMU_DOMAIN_DMA;
1909	}
1910
1911	if (target_type) {
1912		if (driver_type && target_type != driver_type)
1913			return -1;
1914		return target_type;
1915	}
1916	return driver_type;
1917}
1918
1919static void iommu_group_do_probe_finalize(struct device *dev)
1920{
1921	const struct iommu_ops *ops = dev_iommu_ops(dev);
1922
1923	if (ops->probe_finalize)
1924		ops->probe_finalize(dev);
1925}
1926
1927static int bus_iommu_probe(const struct bus_type *bus)
1928{
1929	struct iommu_group *group, *next;
1930	LIST_HEAD(group_list);
1931	int ret;
1932
1933	ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
1934	if (ret)
1935		return ret;
1936
1937	list_for_each_entry_safe(group, next, &group_list, entry) {
1938		struct group_device *gdev;
1939
1940		mutex_lock(&group->mutex);
1941
1942		/* Remove item from the list */
1943		list_del_init(&group->entry);
1944
1945		/*
1946		 * We go to the trouble of deferred default domain creation so
1947		 * that the cross-group default domain type and the setup of the
1948		 * IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios.
1949		 */
1950		ret = iommu_setup_default_domain(group, 0);
1951		if (ret) {
1952			mutex_unlock(&group->mutex);
1953			return ret;
1954		}
1955		for_each_group_device(group, gdev)
1956			iommu_setup_dma_ops(gdev->dev);
1957		mutex_unlock(&group->mutex);
1958
1959		/*
1960		 * FIXME: Mis-locked because the ops->probe_finalize() call-back
1961		 * of some IOMMU drivers calls arm_iommu_attach_device() which
1962		 * in-turn might call back into IOMMU core code, where it tries
1963		 * to take group->mutex, resulting in a deadlock.
1964		 */
1965		for_each_group_device(group, gdev)
1966			iommu_group_do_probe_finalize(gdev->dev);
1967	}
1968
1969	return 0;
1970}
1971
1972/**
1973 * device_iommu_capable() - check for a general IOMMU capability
1974 * @dev: device to which the capability would be relevant, if available
1975 * @cap: IOMMU capability
1976 *
1977 * Return: true if an IOMMU is present and supports the given capability
1978 * for the given device, otherwise false.
1979 */
1980bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
1981{
1982	const struct iommu_ops *ops;
1983
1984	if (!dev_has_iommu(dev))
1985		return false;
1986
1987	ops = dev_iommu_ops(dev);
1988	if (!ops->capable)
1989		return false;
1990
1991	return ops->capable(dev, cap);
1992}
1993EXPORT_SYMBOL_GPL(device_iommu_capable);
1994
1995/**
1996 * iommu_group_has_isolated_msi() - Compute msi_device_has_isolated_msi()
1997 *       for a group
1998 * @group: Group to query
1999 *
2000 * IOMMU groups should not have differing values of
2001 * msi_device_has_isolated_msi() for devices in a group. However nothing
2002 * directly prevents this, so ensure mistakes don't result in isolation failures
2003 * by checking that all the devices are the same.
2004 */
2005bool iommu_group_has_isolated_msi(struct iommu_group *group)
2006{
2007	struct group_device *group_dev;
2008	bool ret = true;
2009
2010	mutex_lock(&group->mutex);
2011	for_each_group_device(group, group_dev)
2012		ret &= msi_device_has_isolated_msi(group_dev->dev);
2013	mutex_unlock(&group->mutex);
2014	return ret;
2015}
2016EXPORT_SYMBOL_GPL(iommu_group_has_isolated_msi);
2017
2018/**
2019 * iommu_set_fault_handler() - set a fault handler for an iommu domain
2020 * @domain: iommu domain
2021 * @handler: fault handler
2022 * @token: user data, will be passed back to the fault handler
2023 *
2024 * This function should be used by IOMMU users which want to be notified
2025 * whenever an IOMMU fault happens.
2026 *
2027 * The fault handler itself should return 0 on success, and an appropriate
2028 * error code otherwise.
2029 */
2030void iommu_set_fault_handler(struct iommu_domain *domain,
2031					iommu_fault_handler_t handler,
2032					void *token)
2033{
2034	if (WARN_ON(!domain || domain->cookie_type != IOMMU_COOKIE_NONE))
2035		return;
2036
2037	domain->cookie_type = IOMMU_COOKIE_FAULT_HANDLER;
2038	domain->handler = handler;
2039	domain->handler_token = token;
2040}
2041EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
2042
2043static void iommu_domain_init(struct iommu_domain *domain, unsigned int type,
2044			      const struct iommu_ops *ops)
2045{
2046	domain->type = type;
2047	domain->owner = ops;
2048	if (!domain->ops)
2049		domain->ops = ops->default_domain_ops;
2050}
2051
2052static struct iommu_domain *
2053__iommu_paging_domain_alloc_flags(struct device *dev, unsigned int type,
2054				  unsigned int flags)
2055{
2056	const struct iommu_ops *ops;
2057	struct iommu_domain *domain;
2058
2059	if (!dev_has_iommu(dev))
2060		return ERR_PTR(-ENODEV);
2061
2062	ops = dev_iommu_ops(dev);
2063
2064	if (ops->domain_alloc_paging && !flags)
2065		domain = ops->domain_alloc_paging(dev);
2066	else if (ops->domain_alloc_paging_flags)
2067		domain = ops->domain_alloc_paging_flags(dev, flags, NULL);
2068#if IS_ENABLED(CONFIG_FSL_PAMU)
2069	else if (ops->domain_alloc && !flags)
2070		domain = ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
2071#endif
2072	else
2073		return ERR_PTR(-EOPNOTSUPP);
2074
2075	if (IS_ERR(domain))
2076		return domain;
2077	if (!domain)
2078		return ERR_PTR(-ENOMEM);
2079
2080	iommu_domain_init(domain, type, ops);
2081	return domain;
2082}
2083
2084/**
2085 * iommu_paging_domain_alloc_flags() - Allocate a paging domain
2086 * @dev: device for which the domain is allocated
2087 * @flags: Bitmap of iommufd_hwpt_alloc_flags
2088 *
2089 * Allocate a paging domain which will be managed by a kernel driver. Return
2090 * allocated domain if successful, or an ERR pointer for failure.
2091 */
2092struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
2093						     unsigned int flags)
2094{
2095	return __iommu_paging_domain_alloc_flags(dev,
2096					 IOMMU_DOMAIN_UNMANAGED, flags);
2097}
2098EXPORT_SYMBOL_GPL(iommu_paging_domain_alloc_flags);
2099
2100void iommu_domain_free(struct iommu_domain *domain)
2101{
2102	switch (domain->cookie_type) {
2103	case IOMMU_COOKIE_DMA_IOVA:
2104		iommu_put_dma_cookie(domain);
2105		break;
2106	case IOMMU_COOKIE_DMA_MSI:
2107		iommu_put_msi_cookie(domain);
2108		break;
2109	case IOMMU_COOKIE_SVA:
2110		mmdrop(domain->mm);
2111		break;
2112	default:
2113		break;
2114	}
2115	if (domain->ops->free)
2116		domain->ops->free(domain);
2117}
2118EXPORT_SYMBOL_GPL(iommu_domain_free);
2119
2120/*
2121 * Put the group's domain back to the appropriate core-owned domain - either the
2122 * standard kernel-mode DMA configuration or an all-DMA-blocked domain.
2123 */
2124static void __iommu_group_set_core_domain(struct iommu_group *group)
2125{
2126	struct iommu_domain *new_domain;
2127
2128	if (group->owner)
2129		new_domain = group->blocking_domain;
2130	else
2131		new_domain = group->default_domain;
2132
2133	__iommu_group_set_domain_nofail(group, new_domain);
2134}
2135
2136static int __iommu_attach_device(struct iommu_domain *domain,
2137				 struct device *dev, struct iommu_domain *old)
2138{
2139	int ret;
2140
2141	if (unlikely(domain->ops->attach_dev == NULL))
2142		return -ENODEV;
2143
2144	ret = domain->ops->attach_dev(domain, dev, old);
2145	if (ret)
2146		return ret;
2147	dev->iommu->attach_deferred = 0;
2148	trace_attach_device_to_domain(dev);
2149	return 0;
2150}
2151
2152/**
2153 * iommu_attach_device - Attach an IOMMU domain to a device
2154 * @domain: IOMMU domain to attach
2155 * @dev: Device that will be attached
2156 *
2157 * Returns 0 on success and error code on failure
2158 *
2159 * Note that EINVAL can be treated as a soft failure, indicating
2160 * that certain configuration of the domain is incompatible with
2161 * the device. In this case attaching a different domain to the
2162 * device may succeed.
2163 */
2164int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
2165{
2166	/* Caller must be a probed driver on dev */
2167	struct iommu_group *group = dev->iommu_group;
2168	int ret;
2169
2170	if (!group)
2171		return -ENODEV;
2172
2173	/*
2174	 * Lock the group to make sure the device-count doesn't
2175	 * change while we are attaching
2176	 */
2177	mutex_lock(&group->mutex);
2178	ret = -EINVAL;
2179	if (list_count_nodes(&group->devices) != 1)
2180		goto out_unlock;
2181
2182	ret = __iommu_attach_group(domain, group);
2183
2184out_unlock:
2185	mutex_unlock(&group->mutex);
2186	return ret;
2187}
2188EXPORT_SYMBOL_GPL(iommu_attach_device);
2189
2190int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
2191{
2192	if (dev->iommu && dev->iommu->attach_deferred)
2193		return __iommu_attach_device(domain, dev, NULL);
2194
2195	return 0;
2196}
2197
2198void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
2199{
2200	/* Caller must be a probed driver on dev */
2201	struct iommu_group *group = dev->iommu_group;
2202
2203	if (!group)
2204		return;
2205
2206	mutex_lock(&group->mutex);
2207	if (WARN_ON(domain != group->domain) ||
2208	    WARN_ON(list_count_nodes(&group->devices) != 1))
2209		goto out_unlock;
2210	__iommu_group_set_core_domain(group);
2211
2212out_unlock:
2213	mutex_unlock(&group->mutex);
2214}
2215EXPORT_SYMBOL_GPL(iommu_detach_device);
2216
2217struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
2218{
2219	/* Caller must be a probed driver on dev */
2220	struct iommu_group *group = dev->iommu_group;
2221
2222	if (!group)
2223		return NULL;
2224
2225	return group->domain;
2226}
2227EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
2228
2229/*
2230 * For IOMMU_DOMAIN_DMA implementations which already provide their own
2231 * guarantees that the group and its default domain are valid and correct.
2232 */
2233struct iommu_domain *iommu_get_dma_domain(struct device *dev)
2234{
2235	return dev->iommu_group->default_domain;
2236}
2237
2238static void *iommu_make_pasid_array_entry(struct iommu_domain *domain,
2239					  struct iommu_attach_handle *handle)
2240{
2241	if (handle) {
2242		handle->domain = domain;
2243		return xa_tag_pointer(handle, IOMMU_PASID_ARRAY_HANDLE);
2244	}
2245
2246	return xa_tag_pointer(domain, IOMMU_PASID_ARRAY_DOMAIN);
2247}
2248
2249static bool domain_iommu_ops_compatible(const struct iommu_ops *ops,
2250					struct iommu_domain *domain)
2251{
2252	if (domain->owner == ops)
2253		return true;
2254
2255	/* For static domains, owner isn't set. */
2256	if (domain == ops->blocked_domain || domain == ops->identity_domain)
2257		return true;
2258
2259	return false;
2260}
2261
2262static int __iommu_attach_group(struct iommu_domain *domain,
2263				struct iommu_group *group)
2264{
2265	struct device *dev;
2266
2267	if (group->domain && group->domain != group->default_domain &&
2268	    group->domain != group->blocking_domain)
2269		return -EBUSY;
2270
2271	dev = iommu_group_first_dev(group);
2272	if (!dev_has_iommu(dev) ||
2273	    !domain_iommu_ops_compatible(dev_iommu_ops(dev), domain))
2274		return -EINVAL;
2275
2276	return __iommu_group_set_domain(group, domain);
2277}
2278
2279/**
2280 * iommu_attach_group - Attach an IOMMU domain to an IOMMU group
2281 * @domain: IOMMU domain to attach
2282 * @group: IOMMU group that will be attached
2283 *
2284 * Returns 0 on success and error code on failure
2285 *
2286 * Note that EINVAL can be treated as a soft failure, indicating
2287 * that certain configuration of the domain is incompatible with
2288 * the group. In this case attaching a different domain to the
2289 * group may succeed.
2290 */
2291int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
2292{
2293	int ret;
2294
2295	mutex_lock(&group->mutex);
2296	ret = __iommu_attach_group(domain, group);
2297	mutex_unlock(&group->mutex);
2298
2299	return ret;
2300}
2301EXPORT_SYMBOL_GPL(iommu_attach_group);
2302
2303static int __iommu_device_set_domain(struct iommu_group *group,
2304				     struct device *dev,
2305				     struct iommu_domain *new_domain,
2306				     struct iommu_domain *old_domain,
2307				     unsigned int flags)
2308{
2309	int ret;
2310
2311	/*
2312	 * If the device requires IOMMU_RESV_DIRECT then we cannot allow
2313	 * the blocking domain to be attached as it does not contain the
2314	 * required 1:1 mapping. This test effectively excludes the device
2315	 * being used with iommu_group_claim_dma_owner() which will block
2316	 * vfio and iommufd as well.
2317	 */
2318	if (dev->iommu->require_direct &&
2319	    (new_domain->type == IOMMU_DOMAIN_BLOCKED ||
2320	     new_domain == group->blocking_domain)) {
2321		dev_warn(dev,
2322			 "Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n");
2323		return -EINVAL;
2324	}
2325
2326	if (dev->iommu->attach_deferred) {
2327		if (new_domain == group->default_domain)
2328			return 0;
2329		dev->iommu->attach_deferred = 0;
2330	}
2331
2332	ret = __iommu_attach_device(new_domain, dev, old_domain);
2333	if (ret) {
2334		/*
2335		 * If we have a blocking domain then try to attach that in hopes
2336		 * of avoiding a UAF. Modern drivers should implement blocking
2337		 * domains as global statics that cannot fail.
2338		 */
2339		if ((flags & IOMMU_SET_DOMAIN_MUST_SUCCEED) &&
2340		    group->blocking_domain &&
2341		    group->blocking_domain != new_domain)
2342			__iommu_attach_device(group->blocking_domain, dev,
2343					      old_domain);
2344		return ret;
2345	}
2346	return 0;
2347}
2348
2349/*
2350 * If 0 is returned the group's domain is new_domain. If an error is returned
2351 * then the group's domain will be set back to the existing domain unless
2352 * IOMMU_SET_DOMAIN_MUST_SUCCEED, otherwise an error is returned and the group's
2353 * domains is left inconsistent. This is a driver bug to fail attach with a
2354 * previously good domain. We try to avoid a kernel UAF because of this.
2355 *
2356 * IOMMU groups are really the natural working unit of the IOMMU, but the IOMMU
2357 * API works on domains and devices.  Bridge that gap by iterating over the
2358 * devices in a group.  Ideally we'd have a single device which represents the
2359 * requestor ID of the group, but we also allow IOMMU drivers to create policy
2360 * defined minimum sets, where the physical hardware may be able to distiguish
2361 * members, but we wish to group them at a higher level (ex. untrusted
2362 * multi-function PCI devices).  Thus we attach each device.
2363 */
2364static int __iommu_group_set_domain_internal(struct iommu_group *group,
2365					     struct iommu_domain *new_domain,
2366					     unsigned int flags)
2367{
2368	struct group_device *last_gdev;
2369	struct group_device *gdev;
2370	int result;
2371	int ret;
2372
2373	lockdep_assert_held(&group->mutex);
2374
2375	if (group->domain == new_domain)
2376		return 0;
2377
2378	if (WARN_ON(!new_domain))
2379		return -EINVAL;
2380
2381	/*
2382	 * Changing the domain is done by calling attach_dev() on the new
2383	 * domain. This switch does not have to be atomic and DMA can be
2384	 * discarded during the transition. DMA must only be able to access
2385	 * either new_domain or group->domain, never something else.
2386	 */
2387	result = 0;
2388	for_each_group_device(group, gdev) {
2389		ret = __iommu_device_set_domain(group, gdev->dev, new_domain,
2390						group->domain, flags);
2391		if (ret) {
2392			result = ret;
2393			/*
2394			 * Keep trying the other devices in the group. If a
2395			 * driver fails attach to an otherwise good domain, and
2396			 * does not support blocking domains, it should at least
2397			 * drop its reference on the current domain so we don't
2398			 * UAF.
2399			 */
2400			if (flags & IOMMU_SET_DOMAIN_MUST_SUCCEED)
2401				continue;
2402			goto err_revert;
2403		}
2404	}
2405	group->domain = new_domain;
2406	return result;
2407
2408err_revert:
2409	/*
2410	 * This is called in error unwind paths. A well behaved driver should
2411	 * always allow us to attach to a domain that was already attached.
2412	 */
2413	last_gdev = gdev;
2414	for_each_group_device(group, gdev) {
2415		/* No need to revert the last gdev that failed to set domain */
2416		if (gdev == last_gdev)
2417			break;
2418		/*
2419		 * A NULL domain can happen only for first probe, in which case
2420		 * we leave group->domain as NULL and let release clean
2421		 * everything up.
2422		 */
2423		if (group->domain)
2424			WARN_ON(__iommu_device_set_domain(
2425				group, gdev->dev, group->domain, new_domain,
2426				IOMMU_SET_DOMAIN_MUST_SUCCEED));
2427	}
2428	return ret;
2429}
2430
2431void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
2432{
2433	mutex_lock(&group->mutex);
2434	__iommu_group_set_core_domain(group);
2435	mutex_unlock(&group->mutex);
2436}
2437EXPORT_SYMBOL_GPL(iommu_detach_group);
2438
2439phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2440{
2441	if (domain->type == IOMMU_DOMAIN_IDENTITY)
2442		return iova;
2443
2444	if (domain->type == IOMMU_DOMAIN_BLOCKED)
2445		return 0;
2446
2447	return domain->ops->iova_to_phys(domain, iova);
2448}
2449EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
2450
2451static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova,
2452			   phys_addr_t paddr, size_t size, size_t *count)
2453{
2454	unsigned int pgsize_idx, pgsize_idx_next;
2455	unsigned long pgsizes;
2456	size_t offset, pgsize, pgsize_next;
2457	size_t offset_end;
2458	unsigned long addr_merge = paddr | iova;
2459
2460	/* Page sizes supported by the hardware and small enough for @size */
2461	pgsizes = domain->pgsize_bitmap & GENMASK(__fls(size), 0);
2462
2463	/* Constrain the page sizes further based on the maximum alignment */
2464	if (likely(addr_merge))
2465		pgsizes &= GENMASK(__ffs(addr_merge), 0);
2466
2467	/* Make sure we have at least one suitable page size */
2468	BUG_ON(!pgsizes);
2469
2470	/* Pick the biggest page size remaining */
2471	pgsize_idx = __fls(pgsizes);
2472	pgsize = BIT(pgsize_idx);
2473	if (!count)
2474		return pgsize;
2475
2476	/* Find the next biggest support page size, if it exists */
2477	pgsizes = domain->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
2478	if (!pgsizes)
2479		goto out_set_count;
2480
2481	pgsize_idx_next = __ffs(pgsizes);
2482	pgsize_next = BIT(pgsize_idx_next);
2483
2484	/*
2485	 * There's no point trying a bigger page size unless the virtual
2486	 * and physical addresses are similarly offset within the larger page.
2487	 */
2488	if ((iova ^ paddr) & (pgsize_next - 1))
2489		goto out_set_count;
2490
2491	/* Calculate the offset to the next page size alignment boundary */
2492	offset = pgsize_next - (addr_merge & (pgsize_next - 1));
2493
2494	/*
2495	 * If size is big enough to accommodate the larger page, reduce
2496	 * the number of smaller pages.
2497	 */
2498	if (!check_add_overflow(offset, pgsize_next, &offset_end) &&
2499	    offset_end <= size)
2500		size = offset;
2501
2502out_set_count:
2503	*count = size >> pgsize_idx;
2504	return pgsize;
2505}
2506
2507int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
2508		phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2509{
2510	const struct iommu_domain_ops *ops = domain->ops;
2511	unsigned long orig_iova = iova;
2512	unsigned int min_pagesz;
2513	size_t orig_size = size;
2514	phys_addr_t orig_paddr = paddr;
2515	int ret = 0;
2516
2517	might_sleep_if(gfpflags_allow_blocking(gfp));
2518
2519	if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2520		return -EINVAL;
2521
2522	if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL))
2523		return -ENODEV;
2524
2525	/* Discourage passing strange GFP flags */
2526	if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 |
2527				__GFP_HIGHMEM)))
2528		return -EINVAL;
2529
2530	/* find out the minimum page size supported */
2531	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2532
2533	/*
2534	 * both the virtual address and the physical one, as well as
2535	 * the size of the mapping, must be aligned (at least) to the
2536	 * size of the smallest page supported by the hardware
2537	 */
2538	if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
2539		pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
2540		       iova, &paddr, size, min_pagesz);
2541		return -EINVAL;
2542	}
2543
2544	pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
2545
2546	while (size) {
2547		size_t pgsize, count, mapped = 0;
2548
2549		pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
2550
2551		pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
2552			 iova, &paddr, pgsize, count);
2553		ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
2554				     gfp, &mapped);
2555		/*
2556		 * Some pages may have been mapped, even if an error occurred,
2557		 * so we should account for those so they can be unmapped.
2558		 */
2559		size -= mapped;
2560
2561		if (ret)
2562			break;
2563
2564		iova += mapped;
2565		paddr += mapped;
2566	}
2567
2568	/* unroll mapping in case something went wrong */
2569	if (ret)
2570		iommu_unmap(domain, orig_iova, orig_size - size);
2571	else
2572		trace_map(orig_iova, orig_paddr, orig_size);
2573
2574	return ret;
2575}
2576
2577int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size)
2578{
2579	const struct iommu_domain_ops *ops = domain->ops;
2580
2581	if (!ops->iotlb_sync_map)
2582		return 0;
2583	return ops->iotlb_sync_map(domain, iova, size);
2584}
2585
2586int iommu_map(struct iommu_domain *domain, unsigned long iova,
2587	      phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
2588{
2589	int ret;
2590
2591	ret = iommu_map_nosync(domain, iova, paddr, size, prot, gfp);
2592	if (ret)
2593		return ret;
2594
2595	ret = iommu_sync_map(domain, iova, size);
2596	if (ret)
2597		iommu_unmap(domain, iova, size);
2598
2599	return ret;
2600}
2601EXPORT_SYMBOL_GPL(iommu_map);
2602
2603static size_t __iommu_unmap(struct iommu_domain *domain,
2604			    unsigned long iova, size_t size,
2605			    struct iommu_iotlb_gather *iotlb_gather)
2606{
2607	const struct iommu_domain_ops *ops = domain->ops;
2608	size_t unmapped_page, unmapped = 0;
2609	unsigned long orig_iova = iova;
2610	unsigned int min_pagesz;
2611
2612	if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
2613		return 0;
2614
2615	if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL))
2616		return 0;
2617
2618	/* find out the minimum page size supported */
2619	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
2620
2621	/*
2622	 * The virtual address, as well as the size of the mapping, must be
2623	 * aligned (at least) to the size of the smallest page supported
2624	 * by the hardware
2625	 */
2626	if (!IS_ALIGNED(iova | size, min_pagesz)) {
2627		pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
2628		       iova, size, min_pagesz);
2629		return 0;
2630	}
2631
2632	pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size);
2633
2634	/*
2635	 * Keep iterating until we either unmap 'size' bytes (or more)
2636	 * or we hit an area that isn't mapped.
2637	 */
2638	while (unmapped < size) {
2639		size_t pgsize, count;
2640
2641		pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count);
2642		unmapped_page = ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather);
2643		if (!unmapped_page)
2644			break;
2645
2646		pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
2647			 iova, unmapped_page);
2648
2649		iova += unmapped_page;
2650		unmapped += unmapped_page;
2651	}
2652
2653	trace_unmap(orig_iova, size, unmapped);
2654	return unmapped;
2655}
2656
2657/**
2658 * iommu_unmap() - Remove mappings from a range of IOVA
2659 * @domain: Domain to manipulate
2660 * @iova: IO virtual address to start
2661 * @size: Length of the range starting from @iova
2662 *
2663 * iommu_unmap() will remove a translation created by iommu_map(). It cannot
2664 * subdivide a mapping created by iommu_map(), so it should be called with IOVA
2665 * ranges that match what was passed to iommu_map(). The range can aggregate
2666 * contiguous iommu_map() calls so long as no individual range is split.
2667 *
2668 * Returns: Number of bytes of IOVA unmapped. iova + res will be the point
2669 * unmapping stopped.
2670 */
2671size_t iommu_unmap(struct iommu_domain *domain,
2672		   unsigned long iova, size_t size)
2673{
2674	struct iommu_iotlb_gather iotlb_gather;
2675	size_t ret;
2676
2677	iommu_iotlb_gather_init(&iotlb_gather);
2678	ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
2679	iommu_iotlb_sync(domain, &iotlb_gather);
2680
2681	return ret;
2682}
2683EXPORT_SYMBOL_GPL(iommu_unmap);
2684
2685/**
2686 * iommu_unmap_fast() - Remove mappings from a range of IOVA without IOTLB sync
2687 * @domain: Domain to manipulate
2688 * @iova: IO virtual address to start
2689 * @size: Length of the range starting from @iova
2690 * @iotlb_gather: range information for a pending IOTLB flush
2691 *
2692 * iommu_unmap_fast() will remove a translation created by iommu_map().
2693 * It can't subdivide a mapping created by iommu_map(), so it should be
2694 * called with IOVA ranges that match what was passed to iommu_map(). The
2695 * range can aggregate contiguous iommu_map() calls so long as no individual
2696 * range is split.
2697 *
2698 * Basically iommu_unmap_fast() is the same as iommu_unmap() but for callers
2699 * which manage the IOTLB flushing externally to perform a batched sync.
2700 *
2701 * Returns: Number of bytes of IOVA unmapped. iova + res will be the point
2702 * unmapping stopped.
2703 */
2704size_t iommu_unmap_fast(struct iommu_domain *domain,
2705			unsigned long iova, size_t size,
2706			struct iommu_iotlb_gather *iotlb_gather)
2707{
2708	return __iommu_unmap(domain, iova, size, iotlb_gather);
2709}
2710EXPORT_SYMBOL_GPL(iommu_unmap_fast);
2711
2712ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
2713		     struct scatterlist *sg, unsigned int nents, int prot,
2714		     gfp_t gfp)
2715{
2716	size_t len = 0, mapped = 0;
2717	phys_addr_t start;
2718	unsigned int i = 0;
2719	int ret;
2720
2721	while (i <= nents) {
2722		phys_addr_t s_phys = sg_phys(sg);
2723
2724		if (len && s_phys != start + len) {
2725			ret = iommu_map_nosync(domain, iova + mapped, start,
2726					len, prot, gfp);
2727			if (ret)
2728				goto out_err;
2729
2730			mapped += len;
2731			len = 0;
2732		}
2733
2734		if (sg_dma_is_bus_address(sg))
2735			goto next;
2736
2737		if (len) {
2738			len += sg->length;
2739		} else {
2740			len = sg->length;
2741			start = s_phys;
2742		}
2743
2744next:
2745		if (++i < nents)
2746			sg = sg_next(sg);
2747	}
2748
2749	ret = iommu_sync_map(domain, iova, mapped);
2750	if (ret)
2751		goto out_err;
2752
2753	return mapped;
2754
2755out_err:
2756	/* undo mappings already done */
2757	iommu_unmap(domain, iova, mapped);
2758
2759	return ret;
2760}
2761EXPORT_SYMBOL_GPL(iommu_map_sg);
2762
2763/**
2764 * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
2765 * @domain: the iommu domain where the fault has happened
2766 * @dev: the device where the fault has happened
2767 * @iova: the faulting address
2768 * @flags: mmu fault flags (e.g. IOMMU_FAULT_READ/IOMMU_FAULT_WRITE/...)
2769 *
2770 * This function should be called by the low-level IOMMU implementations
2771 * whenever IOMMU faults happen, to allow high-level users, that are
2772 * interested in such events, to know about them.
2773 *
2774 * This event may be useful for several possible use cases:
2775 * - mere logging of the event
2776 * - dynamic TLB/PTE loading
2777 * - if restarting of the faulting device is required
2778 *
2779 * Returns 0 on success and an appropriate error code otherwise (if dynamic
2780 * PTE/TLB loading will one day be supported, implementations will be able
2781 * to tell whether it succeeded or not according to this return value).
2782 *
2783 * Specifically, -ENOSYS is returned if a fault handler isn't installed
2784 * (though fault handlers can also return -ENOSYS, in case they want to
2785 * elicit the default behavior of the IOMMU drivers).
2786 */
2787int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
2788		       unsigned long iova, int flags)
2789{
2790	int ret = -ENOSYS;
2791
2792	/*
2793	 * if upper layers showed interest and installed a fault handler,
2794	 * invoke it.
2795	 */
2796	if (domain->cookie_type == IOMMU_COOKIE_FAULT_HANDLER &&
2797	    domain->handler)
2798		ret = domain->handler(domain, dev, iova, flags,
2799						domain->handler_token);
2800
2801	trace_io_page_fault(dev, iova, flags);
2802	return ret;
2803}
2804EXPORT_SYMBOL_GPL(report_iommu_fault);
2805
2806static int __init iommu_init(void)
2807{
2808	iommu_group_kset = kset_create_and_add("iommu_groups",
2809					       NULL, kernel_kobj);
2810	BUG_ON(!iommu_group_kset);
2811
2812	iommu_debugfs_setup();
2813
2814	return 0;
2815}
2816core_initcall(iommu_init);
2817
2818int iommu_set_pgtable_quirks(struct iommu_domain *domain,
2819		unsigned long quirk)
2820{
2821	if (domain->type != IOMMU_DOMAIN_UNMANAGED)
2822		return -EINVAL;
2823	if (!domain->ops->set_pgtable_quirks)
2824		return -EINVAL;
2825	return domain->ops->set_pgtable_quirks(domain, quirk);
2826}
2827EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);
2828
2829/**
2830 * iommu_get_resv_regions - get reserved regions
2831 * @dev: device for which to get reserved regions
2832 * @list: reserved region list for device
2833 *
2834 * This returns a list of reserved IOVA regions specific to this device.
2835 * A domain user should not map IOVA in these ranges.
2836 */
2837void iommu_get_resv_regions(struct device *dev, struct list_head *list)
2838{
2839	const struct iommu_ops *ops = dev_iommu_ops(dev);
2840
2841	if (ops->get_resv_regions)
2842		ops->get_resv_regions(dev, list);
2843}
2844EXPORT_SYMBOL_GPL(iommu_get_resv_regions);
2845
2846/**
2847 * iommu_put_resv_regions - release reserved regions
2848 * @dev: device for which to free reserved regions
2849 * @list: reserved region list for device
2850 *
2851 * This releases a reserved region list acquired by iommu_get_resv_regions().
2852 */
2853void iommu_put_resv_regions(struct device *dev, struct list_head *list)
2854{
2855	struct iommu_resv_region *entry, *next;
2856
2857	list_for_each_entry_safe(entry, next, list, list) {
2858		if (entry->free)
2859			entry->free(dev, entry);
2860		else
2861			kfree(entry);
2862	}
2863}
2864EXPORT_SYMBOL(iommu_put_resv_regions);
2865
2866struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
2867						  size_t length, int prot,
2868						  enum iommu_resv_type type,
2869						  gfp_t gfp)
2870{
2871	struct iommu_resv_region *region;
2872
2873	region = kzalloc(sizeof(*region), gfp);
2874	if (!region)
2875		return NULL;
2876
2877	INIT_LIST_HEAD(&region->list);
2878	region->start = start;
2879	region->length = length;
2880	region->prot = prot;
2881	region->type = type;
2882	return region;
2883}
2884EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
2885
2886void iommu_set_default_passthrough(bool cmd_line)
2887{
2888	if (cmd_line)
2889		iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2890	iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
2891}
2892
2893void iommu_set_default_translated(bool cmd_line)
2894{
2895	if (cmd_line)
2896		iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
2897	iommu_def_domain_type = IOMMU_DOMAIN_DMA;
2898}
2899
2900bool iommu_default_passthrough(void)
2901{
2902	return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
2903}
2904EXPORT_SYMBOL_GPL(iommu_default_passthrough);
2905
2906static const struct iommu_device *iommu_from_fwnode(const struct fwnode_handle *fwnode)
2907{
2908	const struct iommu_device *iommu, *ret = NULL;
2909
2910	spin_lock(&iommu_device_lock);
2911	list_for_each_entry(iommu, &iommu_device_list, list)
2912		if (iommu->fwnode == fwnode) {
2913			ret = iommu;
2914			break;
2915		}
2916	spin_unlock(&iommu_device_lock);
2917	return ret;
2918}
2919
2920const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode)
2921{
2922	const struct iommu_device *iommu = iommu_from_fwnode(fwnode);
2923
2924	return iommu ? iommu->ops : NULL;
2925}
2926
2927int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode)
2928{
2929	const struct iommu_device *iommu = iommu_from_fwnode(iommu_fwnode);
2930	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2931
2932	if (!iommu)
2933		return driver_deferred_probe_check_state(dev);
2934	if (!dev->iommu && !READ_ONCE(iommu->ready))
2935		return -EPROBE_DEFER;
2936
2937	if (fwspec)
2938		return iommu->ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL;
2939
2940	if (!dev_iommu_get(dev))
2941		return -ENOMEM;
2942
2943	/* Preallocate for the overwhelmingly common case of 1 ID */
2944	fwspec = kzalloc(struct_size(fwspec, ids, 1), GFP_KERNEL);
2945	if (!fwspec)
2946		return -ENOMEM;
2947
2948	fwnode_handle_get(iommu_fwnode);
2949	fwspec->iommu_fwnode = iommu_fwnode;
2950	dev_iommu_fwspec_set(dev, fwspec);
2951	return 0;
2952}
2953EXPORT_SYMBOL_GPL(iommu_fwspec_init);
2954
2955void iommu_fwspec_free(struct device *dev)
2956{
2957	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2958
2959	if (fwspec) {
2960		fwnode_handle_put(fwspec->iommu_fwnode);
2961		kfree(fwspec);
2962		dev_iommu_fwspec_set(dev, NULL);
2963	}
2964}
2965
2966int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids)
2967{
2968	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2969	int i, new_num;
2970
2971	if (!fwspec)
2972		return -EINVAL;
2973
2974	new_num = fwspec->num_ids + num_ids;
2975	if (new_num > 1) {
2976		fwspec = krealloc(fwspec, struct_size(fwspec, ids, new_num),
2977				  GFP_KERNEL);
2978		if (!fwspec)
2979			return -ENOMEM;
2980
2981		dev_iommu_fwspec_set(dev, fwspec);
2982	}
2983
2984	for (i = 0; i < num_ids; i++)
2985		fwspec->ids[fwspec->num_ids + i] = ids[i];
2986
2987	fwspec->num_ids = new_num;
2988	return 0;
2989}
2990EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
2991
2992/**
2993 * iommu_setup_default_domain - Set the default_domain for the group
2994 * @group: Group to change
2995 * @target_type: Domain type to set as the default_domain
2996 *
2997 * Allocate a default domain and set it as the current domain on the group. If
2998 * the group already has a default domain it will be changed to the target_type.
2999 * When target_type is 0 the default domain is selected based on driver and
3000 * system preferences.
3001 */
3002static int iommu_setup_default_domain(struct iommu_group *group,
3003				      int target_type)
3004{
3005	struct iommu_domain *old_dom = group->default_domain;
3006	struct group_device *gdev;
3007	struct iommu_domain *dom;
3008	bool direct_failed;
3009	int req_type;
3010	int ret;
3011
3012	lockdep_assert_held(&group->mutex);
3013
3014	req_type = iommu_get_default_domain_type(group, target_type);
3015	if (req_type < 0)
3016		return -EINVAL;
3017
3018	dom = iommu_group_alloc_default_domain(group, req_type);
3019	if (IS_ERR(dom))
3020		return PTR_ERR(dom);
3021
3022	if (group->default_domain == dom)
3023		return 0;
3024
3025	if (iommu_is_dma_domain(dom)) {
3026		ret = iommu_get_dma_cookie(dom);
3027		if (ret) {
3028			iommu_domain_free(dom);
3029			return ret;
3030		}
3031	}
3032
3033	/*
3034	 * IOMMU_RESV_DIRECT and IOMMU_RESV_DIRECT_RELAXABLE regions must be
3035	 * mapped before their device is attached, in order to guarantee
3036	 * continuity with any FW activity
3037	 */
3038	direct_failed = false;
3039	for_each_group_device(group, gdev) {
3040		if (iommu_create_device_direct_mappings(dom, gdev->dev)) {
3041			direct_failed = true;
3042			dev_warn_once(
3043				gdev->dev->iommu->iommu_dev->dev,
3044				"IOMMU driver was not able to establish FW requested direct mapping.");
3045		}
3046	}
3047
3048	/* We must set default_domain early for __iommu_device_set_domain */
3049	group->default_domain = dom;
3050	if (!group->domain) {
3051		/*
3052		 * Drivers are not allowed to fail the first domain attach.
3053		 * The only way to recover from this is to fail attaching the
3054		 * iommu driver and call ops->release_device. Put the domain
3055		 * in group->default_domain so it is freed after.
3056		 */
3057		ret = __iommu_group_set_domain_internal(
3058			group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);
3059		if (WARN_ON(ret))
3060			goto out_free_old;
3061	} else {
3062		ret = __iommu_group_set_domain(group, dom);
3063		if (ret)
3064			goto err_restore_def_domain;
3065	}
3066
3067	/*
3068	 * Drivers are supposed to allow mappings to be installed in a domain
3069	 * before device attachment, but some don't. Hack around this defect by
3070	 * trying again after attaching. If this happens it means the device
3071	 * will not continuously have the IOMMU_RESV_DIRECT map.
3072	 */
3073	if (direct_failed) {
3074		for_each_group_device(group, gdev) {
3075			ret = iommu_create_device_direct_mappings(dom, gdev->dev);
3076			if (ret)
3077				goto err_restore_domain;
3078		}
3079	}
3080
3081out_free_old:
3082	if (old_dom)
3083		iommu_domain_free(old_dom);
3084	return ret;
3085
3086err_restore_domain:
3087	if (old_dom)
3088		__iommu_group_set_domain_internal(
3089			group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);
3090err_restore_def_domain:
3091	if (old_dom) {
3092		iommu_domain_free(dom);
3093		group->default_domain = old_dom;
3094	}
3095	return ret;
3096}
3097
3098/*
3099 * Changing the default domain through sysfs requires the users to unbind the
3100 * drivers from the devices in the iommu group, except for a DMA -> DMA-FQ
3101 * transition. Return failure if this isn't met.
3102 *
3103 * We need to consider the race between this and the device release path.
3104 * group->mutex is used here to guarantee that the device release path
3105 * will not be entered at the same time.
3106 */
3107static ssize_t iommu_group_store_type(struct iommu_group *group,
3108				      const char *buf, size_t count)
3109{
3110	struct group_device *gdev;
3111	int ret, req_type;
3112
3113	if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
3114		return -EACCES;
3115
3116	if (WARN_ON(!group) || !group->default_domain)
3117		return -EINVAL;
3118
3119	if (sysfs_streq(buf, "identity"))
3120		req_type = IOMMU_DOMAIN_IDENTITY;
3121	else if (sysfs_streq(buf, "DMA"))
3122		req_type = IOMMU_DOMAIN_DMA;
3123	else if (sysfs_streq(buf, "DMA-FQ"))
3124		req_type = IOMMU_DOMAIN_DMA_FQ;
3125	else if (sysfs_streq(buf, "auto"))
3126		req_type = 0;
3127	else
3128		return -EINVAL;
3129
3130	mutex_lock(&group->mutex);
3131	/* We can bring up a flush queue without tearing down the domain. */
3132	if (req_type == IOMMU_DOMAIN_DMA_FQ &&
3133	    group->default_domain->type == IOMMU_DOMAIN_DMA) {
3134		ret = iommu_dma_init_fq(group->default_domain);
3135		if (ret)
3136			goto out_unlock;
3137
3138		group->default_domain->type = IOMMU_DOMAIN_DMA_FQ;
3139		ret = count;
3140		goto out_unlock;
3141	}
3142
3143	/* Otherwise, ensure that device exists and no driver is bound. */
3144	if (list_empty(&group->devices) || group->owner_cnt) {
3145		ret = -EPERM;
3146		goto out_unlock;
3147	}
3148
3149	ret = iommu_setup_default_domain(group, req_type);
3150	if (ret)
3151		goto out_unlock;
3152
3153	/* Make sure dma_ops is appropriatley set */
3154	for_each_group_device(group, gdev)
3155		iommu_setup_dma_ops(gdev->dev);
3156
3157out_unlock:
3158	mutex_unlock(&group->mutex);
3159	return ret ?: count;
3160}
3161
3162/**
3163 * iommu_device_use_default_domain() - Device driver wants to handle device
3164 *                                     DMA through the kernel DMA API.
3165 * @dev: The device.
3166 *
3167 * The device driver about to bind @dev wants to do DMA through the kernel
3168 * DMA API. Return 0 if it is allowed, otherwise an error.
3169 */
3170int iommu_device_use_default_domain(struct device *dev)
3171{
3172	/* Caller is the driver core during the pre-probe path */
3173	struct iommu_group *group = dev->iommu_group;
3174	int ret = 0;
3175
3176	if (!group)
3177		return 0;
3178
3179	mutex_lock(&group->mutex);
3180	/* We may race against bus_iommu_probe() finalising groups here */
3181	if (!group->default_domain) {
3182		ret = -EPROBE_DEFER;
3183		goto unlock_out;
3184	}
3185	if (group->owner_cnt) {
3186		if (group->domain != group->default_domain || group->owner ||
3187		    !xa_empty(&group->pasid_array)) {
3188			ret = -EBUSY;
3189			goto unlock_out;
3190		}
3191	}
3192
3193	group->owner_cnt++;
3194
3195unlock_out:
3196	mutex_unlock(&group->mutex);
3197	return ret;
3198}
3199
3200/**
3201 * iommu_device_unuse_default_domain() - Device driver stops handling device
3202 *                                       DMA through the kernel DMA API.
3203 * @dev: The device.
3204 *
3205 * The device driver doesn't want to do DMA through kernel DMA API anymore.
3206 * It must be called after iommu_device_use_default_domain().
3207 */
3208void iommu_device_unuse_default_domain(struct device *dev)
3209{
3210	/* Caller is the driver core during the post-probe path */
3211	struct iommu_group *group = dev->iommu_group;
3212
3213	if (!group)
3214		return;
3215
3216	mutex_lock(&group->mutex);
3217	if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
3218		group->owner_cnt--;
3219
3220	mutex_unlock(&group->mutex);
3221}
3222
3223static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
3224{
3225	struct device *dev = iommu_group_first_dev(group);
3226	const struct iommu_ops *ops = dev_iommu_ops(dev);
3227	struct iommu_domain *domain;
3228
3229	if (group->blocking_domain)
3230		return 0;
3231
3232	if (ops->blocked_domain) {
3233		group->blocking_domain = ops->blocked_domain;
3234		return 0;
3235	}
3236
3237	/*
3238	 * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED create an
3239	 * empty PAGING domain instead.
3240	 */
3241	domain = iommu_paging_domain_alloc(dev);
3242	if (IS_ERR(domain))
3243		return PTR_ERR(domain);
3244	group->blocking_domain = domain;
3245	return 0;
3246}
3247
3248static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)
3249{
3250	int ret;
3251
3252	if ((group->domain && group->domain != group->default_domain) ||
3253	    !xa_empty(&group->pasid_array))
3254		return -EBUSY;
3255
3256	ret = __iommu_group_alloc_blocking_domain(group);
3257	if (ret)
3258		return ret;
3259	ret = __iommu_group_set_domain(group, group->blocking_domain);
3260	if (ret)
3261		return ret;
3262
3263	group->owner = owner;
3264	group->owner_cnt++;
3265	return 0;
3266}
3267
3268/**
3269 * iommu_group_claim_dma_owner() - Set DMA ownership of a group
3270 * @group: The group.
3271 * @owner: Caller specified pointer. Used for exclusive ownership.
3272 *
3273 * This is to support backward compatibility for vfio which manages the dma
3274 * ownership in iommu_group level. New invocations on this interface should be
3275 * prohibited. Only a single owner may exist for a group.
3276 */
3277int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
3278{
3279	int ret = 0;
3280
3281	if (WARN_ON(!owner))
3282		return -EINVAL;
3283
3284	mutex_lock(&group->mutex);
3285	if (group->owner_cnt) {
3286		ret = -EPERM;
3287		goto unlock_out;
3288	}
3289
3290	ret = __iommu_take_dma_ownership(group, owner);
3291unlock_out:
3292	mutex_unlock(&group->mutex);
3293
3294	return ret;
3295}
3296EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
3297
3298/**
3299 * iommu_device_claim_dma_owner() - Set DMA ownership of a device
3300 * @dev: The device.
3301 * @owner: Caller specified pointer. Used for exclusive ownership.
3302 *
3303 * Claim the DMA ownership of a device. Multiple devices in the same group may
3304 * concurrently claim ownership if they present the same owner value. Returns 0
3305 * on success and error code on failure
3306 */
3307int iommu_device_claim_dma_owner(struct device *dev, void *owner)
3308{
3309	/* Caller must be a probed driver on dev */
3310	struct iommu_group *group = dev->iommu_group;
3311	int ret = 0;
3312
3313	if (WARN_ON(!owner))
3314		return -EINVAL;
3315
3316	if (!group)
3317		return -ENODEV;
3318
3319	mutex_lock(&group->mutex);
3320	if (group->owner_cnt) {
3321		if (group->owner != owner) {
3322			ret = -EPERM;
3323			goto unlock_out;
3324		}
3325		group->owner_cnt++;
3326		goto unlock_out;
3327	}
3328
3329	ret = __iommu_take_dma_ownership(group, owner);
3330unlock_out:
3331	mutex_unlock(&group->mutex);
3332	return ret;
3333}
3334EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
3335
3336static void __iommu_release_dma_ownership(struct iommu_group *group)
3337{
3338	if (WARN_ON(!group->owner_cnt || !group->owner ||
3339		    !xa_empty(&group->pasid_array)))
3340		return;
3341
3342	group->owner_cnt = 0;
3343	group->owner = NULL;
3344	__iommu_group_set_domain_nofail(group, group->default_domain);
3345}
3346
3347/**
3348 * iommu_group_release_dma_owner() - Release DMA ownership of a group
3349 * @group: The group
3350 *
3351 * Release the DMA ownership claimed by iommu_group_claim_dma_owner().
3352 */
3353void iommu_group_release_dma_owner(struct iommu_group *group)
3354{
3355	mutex_lock(&group->mutex);
3356	__iommu_release_dma_ownership(group);
3357	mutex_unlock(&group->mutex);
3358}
3359EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
3360
3361/**
3362 * iommu_device_release_dma_owner() - Release DMA ownership of a device
3363 * @dev: The device.
3364 *
3365 * Release the DMA ownership claimed by iommu_device_claim_dma_owner().
3366 */
3367void iommu_device_release_dma_owner(struct device *dev)
3368{
3369	/* Caller must be a probed driver on dev */
3370	struct iommu_group *group = dev->iommu_group;
3371
3372	mutex_lock(&group->mutex);
3373	if (group->owner_cnt > 1)
3374		group->owner_cnt--;
3375	else
3376		__iommu_release_dma_ownership(group);
3377	mutex_unlock(&group->mutex);
3378}
3379EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
3380
3381/**
3382 * iommu_group_dma_owner_claimed() - Query group dma ownership status
3383 * @group: The group.
3384 *
3385 * This provides status query on a given group. It is racy and only for
3386 * non-binding status reporting.
3387 */
3388bool iommu_group_dma_owner_claimed(struct iommu_group *group)
3389{
3390	unsigned int user;
3391
3392	mutex_lock(&group->mutex);
3393	user = group->owner_cnt;
3394	mutex_unlock(&group->mutex);
3395
3396	return user;
3397}
3398EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3399
3400static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
3401				   struct iommu_domain *domain)
3402{
3403	const struct iommu_ops *ops = dev_iommu_ops(dev);
3404	struct iommu_domain *blocked_domain = ops->blocked_domain;
3405
3406	WARN_ON(blocked_domain->ops->set_dev_pasid(blocked_domain,
3407						   dev, pasid, domain));
3408}
3409
3410static int __iommu_set_group_pasid(struct iommu_domain *domain,
3411				   struct iommu_group *group, ioasid_t pasid,
3412				   struct iommu_domain *old)
3413{
3414	struct group_device *device, *last_gdev;
3415	int ret;
3416
3417	for_each_group_device(group, device) {
3418		if (device->dev->iommu->max_pasids > 0) {
3419			ret = domain->ops->set_dev_pasid(domain, device->dev,
3420							 pasid, old);
3421			if (ret)
3422				goto err_revert;
3423		}
3424	}
3425
3426	return 0;
3427
3428err_revert:
3429	last_gdev = device;
3430	for_each_group_device(group, device) {
3431		if (device == last_gdev)
3432			break;
3433		if (device->dev->iommu->max_pasids > 0) {
3434			/*
3435			 * If no old domain, undo the succeeded devices/pasid.
3436			 * Otherwise, rollback the succeeded devices/pasid to
3437			 * the old domain. And it is a driver bug to fail
3438			 * attaching with a previously good domain.
3439			 */
3440			if (!old ||
3441			    WARN_ON(old->ops->set_dev_pasid(old, device->dev,
3442							    pasid, domain)))
3443				iommu_remove_dev_pasid(device->dev, pasid, domain);
3444		}
3445	}
3446	return ret;
3447}
3448
3449static void __iommu_remove_group_pasid(struct iommu_group *group,
3450				       ioasid_t pasid,
3451				       struct iommu_domain *domain)
3452{
3453	struct group_device *device;
3454
3455	for_each_group_device(group, device) {
3456		if (device->dev->iommu->max_pasids > 0)
3457			iommu_remove_dev_pasid(device->dev, pasid, domain);
3458	}
3459}
3460
3461/*
3462 * iommu_attach_device_pasid() - Attach a domain to pasid of device
3463 * @domain: the iommu domain.
3464 * @dev: the attached device.
3465 * @pasid: the pasid of the device.
3466 * @handle: the attach handle.
3467 *
3468 * Caller should always provide a new handle to avoid race with the paths
3469 * that have lockless reference to handle if it intends to pass a valid handle.
3470 *
3471 * Return: 0 on success, or an error.
3472 */
3473int iommu_attach_device_pasid(struct iommu_domain *domain,
3474			      struct device *dev, ioasid_t pasid,
3475			      struct iommu_attach_handle *handle)
3476{
3477	/* Caller must be a probed driver on dev */
3478	struct iommu_group *group = dev->iommu_group;
3479	struct group_device *device;
3480	const struct iommu_ops *ops;
3481	void *entry;
3482	int ret;
3483
3484	if (!group)
3485		return -ENODEV;
3486
3487	ops = dev_iommu_ops(dev);
3488
3489	if (!domain->ops->set_dev_pasid ||
3490	    !ops->blocked_domain ||
3491	    !ops->blocked_domain->ops->set_dev_pasid)
3492		return -EOPNOTSUPP;
3493
3494	if (!domain_iommu_ops_compatible(ops, domain) ||
3495	    pasid == IOMMU_NO_PASID)
3496		return -EINVAL;
3497
3498	mutex_lock(&group->mutex);
3499	for_each_group_device(group, device) {
3500		/*
3501		 * Skip PASID validation for devices without PASID support
3502		 * (max_pasids = 0). These devices cannot issue transactions
3503		 * with PASID, so they don't affect group's PASID usage.
3504		 */
3505		if ((device->dev->iommu->max_pasids > 0) &&
3506		    (pasid >= device->dev->iommu->max_pasids)) {
3507			ret = -EINVAL;
3508			goto out_unlock;
3509		}
3510	}
3511
3512	entry = iommu_make_pasid_array_entry(domain, handle);
3513
3514	/*
3515	 * Entry present is a failure case. Use xa_insert() instead of
3516	 * xa_reserve().
3517	 */
3518	ret = xa_insert(&group->pasid_array, pasid, XA_ZERO_ENTRY, GFP_KERNEL);
3519	if (ret)
3520		goto out_unlock;
3521
3522	ret = __iommu_set_group_pasid(domain, group, pasid, NULL);
3523	if (ret) {
3524		xa_release(&group->pasid_array, pasid);
3525		goto out_unlock;
3526	}
3527
3528	/*
3529	 * The xa_insert() above reserved the memory, and the group->mutex is
3530	 * held, this cannot fail. The new domain cannot be visible until the
3531	 * operation succeeds as we cannot tolerate PRIs becoming concurrently
3532	 * queued and then failing attach.
3533	 */
3534	WARN_ON(xa_is_err(xa_store(&group->pasid_array,
3535				   pasid, entry, GFP_KERNEL)));
3536
3537out_unlock:
3538	mutex_unlock(&group->mutex);
3539	return ret;
3540}
3541EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3542
3543/**
3544 * iommu_replace_device_pasid - Replace the domain that a specific pasid
3545 *                              of the device is attached to
3546 * @domain: the new iommu domain
3547 * @dev: the attached device.
3548 * @pasid: the pasid of the device.
3549 * @handle: the attach handle.
3550 *
3551 * This API allows the pasid to switch domains. The @pasid should have been
3552 * attached. Otherwise, this fails. The pasid will keep the old configuration
3553 * if replacement failed.
3554 *
3555 * Caller should always provide a new handle to avoid race with the paths
3556 * that have lockless reference to handle if it intends to pass a valid handle.
3557 *
3558 * Return 0 on success, or an error.
3559 */
3560int iommu_replace_device_pasid(struct iommu_domain *domain,
3561			       struct device *dev, ioasid_t pasid,
3562			       struct iommu_attach_handle *handle)
3563{
3564	/* Caller must be a probed driver on dev */
3565	struct iommu_group *group = dev->iommu_group;
3566	struct iommu_attach_handle *entry;
3567	struct iommu_domain *curr_domain;
3568	void *curr;
3569	int ret;
3570
3571	if (!group)
3572		return -ENODEV;
3573
3574	if (!domain->ops->set_dev_pasid)
3575		return -EOPNOTSUPP;
3576
3577	if (!domain_iommu_ops_compatible(dev_iommu_ops(dev), domain) ||
3578	    pasid == IOMMU_NO_PASID || !handle)
3579		return -EINVAL;
3580
3581	mutex_lock(&group->mutex);
3582	entry = iommu_make_pasid_array_entry(domain, handle);
3583	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,
3584			  XA_ZERO_ENTRY, GFP_KERNEL);
3585	if (xa_is_err(curr)) {
3586		ret = xa_err(curr);
3587		goto out_unlock;
3588	}
3589
3590	/*
3591	 * No domain (with or without handle) attached, hence not
3592	 * a replace case.
3593	 */
3594	if (!curr) {
3595		xa_release(&group->pasid_array, pasid);
3596		ret = -EINVAL;
3597		goto out_unlock;
3598	}
3599
3600	/*
3601	 * Reusing handle is problematic as there are paths that refers
3602	 * the handle without lock. To avoid race, reject the callers that
3603	 * attempt it.
3604	 */
3605	if (curr == entry) {
3606		WARN_ON(1);
3607		ret = -EINVAL;
3608		goto out_unlock;
3609	}
3610
3611	curr_domain = pasid_array_entry_to_domain(curr);
3612	ret = 0;
3613
3614	if (curr_domain != domain) {
3615		ret = __iommu_set_group_pasid(domain, group,
3616					      pasid, curr_domain);
3617		if (ret)
3618			goto out_unlock;
3619	}
3620
3621	/*
3622	 * The above xa_cmpxchg() reserved the memory, and the
3623	 * group->mutex is held, this cannot fail.
3624	 */
3625	WARN_ON(xa_is_err(xa_store(&group->pasid_array,
3626				   pasid, entry, GFP_KERNEL)));
3627
3628out_unlock:
3629	mutex_unlock(&group->mutex);
3630	return ret;
3631}
3632EXPORT_SYMBOL_NS_GPL(iommu_replace_device_pasid, "IOMMUFD_INTERNAL");
3633
3634/*
3635 * iommu_detach_device_pasid() - Detach the domain from pasid of device
3636 * @domain: the iommu domain.
3637 * @dev: the attached device.
3638 * @pasid: the pasid of the device.
3639 *
3640 * The @domain must have been attached to @pasid of the @dev with
3641 * iommu_attach_device_pasid().
3642 */
3643void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3644			       ioasid_t pasid)
3645{
3646	/* Caller must be a probed driver on dev */
3647	struct iommu_group *group = dev->iommu_group;
3648
3649	mutex_lock(&group->mutex);
3650	__iommu_remove_group_pasid(group, pasid, domain);
3651	xa_erase(&group->pasid_array, pasid);
3652	mutex_unlock(&group->mutex);
3653}
3654EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3655
3656ioasid_t iommu_alloc_global_pasid(struct device *dev)
3657{
3658	int ret;
3659
3660	/* max_pasids == 0 means that the device does not support PASID */
3661	if (!dev->iommu->max_pasids)
3662		return IOMMU_PASID_INVALID;
3663
3664	/*
3665	 * max_pasids is set up by vendor driver based on number of PASID bits
3666	 * supported but the IDA allocation is inclusive.
3667	 */
3668	ret = ida_alloc_range(&iommu_global_pasid_ida, IOMMU_FIRST_GLOBAL_PASID,
3669			      dev->iommu->max_pasids - 1, GFP_KERNEL);
3670	return ret < 0 ? IOMMU_PASID_INVALID : ret;
3671}
3672EXPORT_SYMBOL_GPL(iommu_alloc_global_pasid);
3673
3674void iommu_free_global_pasid(ioasid_t pasid)
3675{
3676	if (WARN_ON(pasid == IOMMU_PASID_INVALID))
3677		return;
3678
3679	ida_free(&iommu_global_pasid_ida, pasid);
3680}
3681EXPORT_SYMBOL_GPL(iommu_free_global_pasid);
3682
3683/**
3684 * iommu_attach_handle_get - Return the attach handle
3685 * @group: the iommu group that domain was attached to
3686 * @pasid: the pasid within the group
3687 * @type: matched domain type, 0 for any match
3688 *
3689 * Return handle or ERR_PTR(-ENOENT) on none, ERR_PTR(-EBUSY) on mismatch.
3690 *
3691 * Return the attach handle to the caller. The life cycle of an iommu attach
3692 * handle is from the time when the domain is attached to the time when the
3693 * domain is detached. Callers are required to synchronize the call of
3694 * iommu_attach_handle_get() with domain attachment and detachment. The attach
3695 * handle can only be used during its life cycle.
3696 */
3697struct iommu_attach_handle *
3698iommu_attach_handle_get(struct iommu_group *group, ioasid_t pasid, unsigned int type)
3699{
3700	struct iommu_attach_handle *handle;
3701	void *entry;
3702
3703	xa_lock(&group->pasid_array);
3704	entry = xa_load(&group->pasid_array, pasid);
3705	if (!entry || xa_pointer_tag(entry) != IOMMU_PASID_ARRAY_HANDLE) {
3706		handle = ERR_PTR(-ENOENT);
3707	} else {
3708		handle = xa_untag_pointer(entry);
3709		if (type && handle->domain->type != type)
3710			handle = ERR_PTR(-EBUSY);
3711	}
3712	xa_unlock(&group->pasid_array);
3713
3714	return handle;
3715}
3716EXPORT_SYMBOL_NS_GPL(iommu_attach_handle_get, "IOMMUFD_INTERNAL");
3717
3718/**
3719 * iommu_attach_group_handle - Attach an IOMMU domain to an IOMMU group
3720 * @domain: IOMMU domain to attach
3721 * @group: IOMMU group that will be attached
3722 * @handle: attach handle
3723 *
3724 * Returns 0 on success and error code on failure.
3725 *
3726 * This is a variant of iommu_attach_group(). It allows the caller to provide
3727 * an attach handle and use it when the domain is attached. This is currently
3728 * used by IOMMUFD to deliver the I/O page faults.
3729 *
3730 * Caller should always provide a new handle to avoid race with the paths
3731 * that have lockless reference to handle.
3732 */
3733int iommu_attach_group_handle(struct iommu_domain *domain,
3734			      struct iommu_group *group,
3735			      struct iommu_attach_handle *handle)
3736{
3737	void *entry;
3738	int ret;
3739
3740	if (!handle)
3741		return -EINVAL;
3742
3743	mutex_lock(&group->mutex);
3744	entry = iommu_make_pasid_array_entry(domain, handle);
3745	ret = xa_insert(&group->pasid_array,
3746			IOMMU_NO_PASID, XA_ZERO_ENTRY, GFP_KERNEL);
3747	if (ret)
3748		goto out_unlock;
3749
3750	ret = __iommu_attach_group(domain, group);
3751	if (ret) {
3752		xa_release(&group->pasid_array, IOMMU_NO_PASID);
3753		goto out_unlock;
3754	}
3755
3756	/*
3757	 * The xa_insert() above reserved the memory, and the group->mutex is
3758	 * held, this cannot fail. The new domain cannot be visible until the
3759	 * operation succeeds as we cannot tolerate PRIs becoming concurrently
3760	 * queued and then failing attach.
3761	 */
3762	WARN_ON(xa_is_err(xa_store(&group->pasid_array,
3763				   IOMMU_NO_PASID, entry, GFP_KERNEL)));
3764
3765out_unlock:
3766	mutex_unlock(&group->mutex);
3767	return ret;
3768}
3769EXPORT_SYMBOL_NS_GPL(iommu_attach_group_handle, "IOMMUFD_INTERNAL");
3770
3771/**
3772 * iommu_detach_group_handle - Detach an IOMMU domain from an IOMMU group
3773 * @domain: IOMMU domain to attach
3774 * @group: IOMMU group that will be attached
3775 *
3776 * Detach the specified IOMMU domain from the specified IOMMU group.
3777 * It must be used in conjunction with iommu_attach_group_handle().
3778 */
3779void iommu_detach_group_handle(struct iommu_domain *domain,
3780			       struct iommu_group *group)
3781{
3782	mutex_lock(&group->mutex);
3783	__iommu_group_set_core_domain(group);
3784	xa_erase(&group->pasid_array, IOMMU_NO_PASID);
3785	mutex_unlock(&group->mutex);
3786}
3787EXPORT_SYMBOL_NS_GPL(iommu_detach_group_handle, "IOMMUFD_INTERNAL");
3788
3789/**
3790 * iommu_replace_group_handle - replace the domain that a group is attached to
3791 * @group: IOMMU group that will be attached to the new domain
3792 * @new_domain: new IOMMU domain to replace with
3793 * @handle: attach handle
3794 *
3795 * This API allows the group to switch domains without being forced to go to
3796 * the blocking domain in-between. It allows the caller to provide an attach
3797 * handle for the new domain and use it when the domain is attached.
3798 *
3799 * If the currently attached domain is a core domain (e.g. a default_domain),
3800 * it will act just like the iommu_attach_group_handle().
3801 *
3802 * Caller should always provide a new handle to avoid race with the paths
3803 * that have lockless reference to handle.
3804 */
3805int iommu_replace_group_handle(struct iommu_group *group,
3806			       struct iommu_domain *new_domain,
3807			       struct iommu_attach_handle *handle)
3808{
3809	void *curr, *entry;
3810	int ret;
3811
3812	if (!new_domain || !handle)
3813		return -EINVAL;
3814
3815	mutex_lock(&group->mutex);
3816	entry = iommu_make_pasid_array_entry(new_domain, handle);
3817	ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
3818	if (ret)
3819		goto err_unlock;
3820
3821	ret = __iommu_group_set_domain(group, new_domain);
3822	if (ret)
3823		goto err_release;
3824
3825	curr = xa_store(&group->pasid_array, IOMMU_NO_PASID, entry, GFP_KERNEL);
3826	WARN_ON(xa_is_err(curr));
3827
3828	mutex_unlock(&group->mutex);
3829
3830	return 0;
3831err_release:
3832	xa_release(&group->pasid_array, IOMMU_NO_PASID);
3833err_unlock:
3834	mutex_unlock(&group->mutex);
3835	return ret;
3836}
3837EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL");
3838
3839#if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
3840/**
3841 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
3842 * @desc: MSI descriptor, will store the MSI page
3843 * @msi_addr: MSI target address to be mapped
3844 *
3845 * The implementation of sw_msi() should take msi_addr and map it to
3846 * an IOVA in the domain and call msi_desc_set_iommu_msi_iova() with the
3847 * mapping information.
3848 *
3849 * Return: 0 on success or negative error code if the mapping failed.
3850 */
3851int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
3852{
3853	struct device *dev = msi_desc_to_dev(desc);
3854	struct iommu_group *group = dev->iommu_group;
3855	int ret = 0;
3856
3857	if (!group)
3858		return 0;
3859
3860	mutex_lock(&group->mutex);
3861	/* An IDENTITY domain must pass through */
3862	if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
3863		switch (group->domain->cookie_type) {
3864		case IOMMU_COOKIE_DMA_MSI:
3865		case IOMMU_COOKIE_DMA_IOVA:
3866			ret = iommu_dma_sw_msi(group->domain, desc, msi_addr);
3867			break;
3868		case IOMMU_COOKIE_IOMMUFD:
3869			ret = iommufd_sw_msi(group->domain, desc, msi_addr);
3870			break;
3871		default:
3872			ret = -EOPNOTSUPP;
3873			break;
3874		}
3875	}
3876	mutex_unlock(&group->mutex);
3877	return ret;
3878}
3879#endif /* CONFIG_IRQ_MSI_IOMMU */