nested.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * nested.c - nested mode translation support
  4 *
  5 * Copyright (C) 2023 Intel Corporation
  6 *
  7 * Author: Lu Baolu <baolu.lu@linux.intel.com>
  8 *         Jacob Pan <jacob.jun.pan@linux.intel.com>
  9 *         Yi Liu <yi.l.liu@intel.com>
 10 */
 11
 12#define pr_fmt(fmt)	"DMAR: " fmt
 13
 14#include <linux/iommu.h>
 15#include <linux/pci.h>
 16#include <linux/pci-ats.h>
 17
 18#include "iommu.h"
 19#include "pasid.h"
 20
 21static int intel_nested_attach_dev(struct iommu_domain *domain,
 22				   struct device *dev, struct iommu_domain *old)
 23{
 24	struct device_domain_info *info = dev_iommu_priv_get(dev);
 25	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 26	struct intel_iommu *iommu = info->iommu;
 27	unsigned long flags;
 28	int ret = 0;
 29
 30	device_block_translation(dev);
 31
 32	/*
 33	 * Stage-1 domain cannot work alone, it is nested on a s2_domain.
 34	 * The s2_domain will be used in nested translation, hence needs
 35	 * to ensure the s2_domain is compatible with this IOMMU.
 36	 */
 37	ret = paging_domain_compatible(&dmar_domain->s2_domain->domain, dev);
 38	if (ret) {
 39		dev_err_ratelimited(dev, "s2 domain is not compatible\n");
 40		return ret;
 41	}
 42
 43	ret = domain_attach_iommu(dmar_domain, iommu);
 44	if (ret) {
 45		dev_err_ratelimited(dev, "Failed to attach domain to iommu\n");
 46		return ret;
 47	}
 48
 49	ret = cache_tag_assign_domain(dmar_domain, dev, IOMMU_NO_PASID);
 50	if (ret)
 51		goto detach_iommu;
 52
 53	ret = iopf_for_domain_set(domain, dev);
 54	if (ret)
 55		goto unassign_tag;
 56
 57	ret = intel_pasid_setup_nested(iommu, dev,
 58				       IOMMU_NO_PASID, dmar_domain);
 59	if (ret)
 60		goto disable_iopf;
 61
 62	info->domain = dmar_domain;
 63	info->domain_attached = true;
 64	spin_lock_irqsave(&dmar_domain->lock, flags);
 65	list_add(&info->link, &dmar_domain->devices);
 66	spin_unlock_irqrestore(&dmar_domain->lock, flags);
 67
 68	return 0;
 69disable_iopf:
 70	iopf_for_domain_remove(domain, dev);
 71unassign_tag:
 72	cache_tag_unassign_domain(dmar_domain, dev, IOMMU_NO_PASID);
 73detach_iommu:
 74	domain_detach_iommu(dmar_domain, iommu);
 75
 76	return ret;
 77}
 78
 79static void intel_nested_domain_free(struct iommu_domain *domain)
 80{
 81	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 82	struct dmar_domain *s2_domain = dmar_domain->s2_domain;
 83
 84	spin_lock(&s2_domain->s1_lock);
 85	list_del(&dmar_domain->s2_link);
 86	spin_unlock(&s2_domain->s1_lock);
 87	kfree(dmar_domain->qi_batch);
 88	kfree(dmar_domain);
 89}
 90
 91static int intel_nested_cache_invalidate_user(struct iommu_domain *domain,
 92					      struct iommu_user_data_array *array)
 93{
 94	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 95	struct iommu_hwpt_vtd_s1_invalidate inv_entry;
 96	u32 index, processed = 0;
 97	int ret = 0;
 98
 99	if (array->type != IOMMU_HWPT_INVALIDATE_DATA_VTD_S1) {
100		ret = -EINVAL;
101		goto out;
102	}
103
104	for (index = 0; index < array->entry_num; index++) {
105		ret = iommu_copy_struct_from_user_array(&inv_entry, array,
106							IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
107							index, __reserved);
108		if (ret)
109			break;
110
111		if ((inv_entry.flags & ~IOMMU_VTD_INV_FLAGS_LEAF) ||
112		    inv_entry.__reserved) {
113			ret = -EOPNOTSUPP;
114			break;
115		}
116
117		if (!IS_ALIGNED(inv_entry.addr, VTD_PAGE_SIZE) ||
118		    ((inv_entry.npages == U64_MAX) && inv_entry.addr)) {
119			ret = -EINVAL;
120			break;
121		}
122
123		cache_tag_flush_range(dmar_domain, inv_entry.addr,
124				      inv_entry.addr + nrpages_to_size(inv_entry.npages) - 1,
125				      inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF);
126		processed++;
127	}
128
129out:
130	array->entry_num = processed;
131	return ret;
132}
133
134static int domain_setup_nested(struct intel_iommu *iommu,
135			       struct dmar_domain *domain,
136			       struct device *dev, ioasid_t pasid,
137			       struct iommu_domain *old)
138{
139	if (old)
140		intel_pasid_tear_down_entry(iommu, dev, pasid, false);
141
142	return intel_pasid_setup_nested(iommu, dev, pasid, domain);
143}
144
145static int intel_nested_set_dev_pasid(struct iommu_domain *domain,
146				      struct device *dev, ioasid_t pasid,
147				      struct iommu_domain *old)
148{
149	struct device_domain_info *info = dev_iommu_priv_get(dev);
150	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
151	struct intel_iommu *iommu = info->iommu;
152	struct dev_pasid_info *dev_pasid;
153	int ret;
154
155	if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
156		return -EOPNOTSUPP;
157
158	if (context_copied(iommu, info->bus, info->devfn))
159		return -EBUSY;
160
161	ret = paging_domain_compatible(&dmar_domain->s2_domain->domain, dev);
162	if (ret)
163		return ret;
164
165	dev_pasid = domain_add_dev_pasid(domain, dev, pasid);
166	if (IS_ERR(dev_pasid))
167		return PTR_ERR(dev_pasid);
168
169	ret = iopf_for_domain_replace(domain, old, dev);
170	if (ret)
171		goto out_remove_dev_pasid;
172
173	ret = domain_setup_nested(iommu, dmar_domain, dev, pasid, old);
174	if (ret)
175		goto out_unwind_iopf;
176
177	domain_remove_dev_pasid(old, dev, pasid);
178
179	return 0;
180
181out_unwind_iopf:
182	iopf_for_domain_replace(old, domain, dev);
183out_remove_dev_pasid:
184	domain_remove_dev_pasid(domain, dev, pasid);
185	return ret;
186}
187
188static const struct iommu_domain_ops intel_nested_domain_ops = {
189	.attach_dev		= intel_nested_attach_dev,
190	.set_dev_pasid		= intel_nested_set_dev_pasid,
191	.free			= intel_nested_domain_free,
192	.cache_invalidate_user	= intel_nested_cache_invalidate_user,
193};
194
195struct iommu_domain *
196intel_iommu_domain_alloc_nested(struct device *dev, struct iommu_domain *parent,
197				u32 flags,
198				const struct iommu_user_data *user_data)
199{
200	struct device_domain_info *info = dev_iommu_priv_get(dev);
201	struct dmar_domain *s2_domain = to_dmar_domain(parent);
202	struct intel_iommu *iommu = info->iommu;
203	struct iommu_hwpt_vtd_s1 vtd;
204	struct dmar_domain *domain;
205	int ret;
206
207	if (!nested_supported(iommu) || flags & ~IOMMU_HWPT_ALLOC_PASID)
208		return ERR_PTR(-EOPNOTSUPP);
209
210	/* Must be nested domain */
211	if (user_data->type != IOMMU_HWPT_DATA_VTD_S1)
212		return ERR_PTR(-EOPNOTSUPP);
213	if (!intel_domain_is_ss_paging(s2_domain) || !s2_domain->nested_parent)
214		return ERR_PTR(-EINVAL);
215
216	ret = iommu_copy_struct_from_user(&vtd, user_data,
217					  IOMMU_HWPT_DATA_VTD_S1, __reserved);
218	if (ret)
219		return ERR_PTR(ret);
220
221	domain = kzalloc(sizeof(*domain), GFP_KERNEL_ACCOUNT);
222	if (!domain)
223		return ERR_PTR(-ENOMEM);
224
225	domain->s2_domain = s2_domain;
226	domain->s1_cfg = vtd;
227	domain->domain.ops = &intel_nested_domain_ops;
228	domain->domain.type = IOMMU_DOMAIN_NESTED;
229	INIT_LIST_HEAD(&domain->devices);
230	INIT_LIST_HEAD(&domain->dev_pasids);
231	INIT_LIST_HEAD(&domain->cache_tags);
232	spin_lock_init(&domain->lock);
233	spin_lock_init(&domain->cache_lock);
234	xa_init(&domain->iommu_array);
235
236	spin_lock(&s2_domain->s1_lock);
237	list_add(&domain->s2_link, &s2_domain->s1_domains);
238	spin_unlock(&s2_domain->s1_lock);
239
240	return &domain->domain;
241}