driver.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
  3 */
  4#include "iommufd_private.h"
  5
  6/* Driver should use a per-structure helper in include/linux/iommufd.h */
  7int _iommufd_object_depend(struct iommufd_object *obj_dependent,
  8			   struct iommufd_object *obj_depended)
  9{
 10	/* Reject self dependency that dead locks */
 11	if (obj_dependent == obj_depended)
 12		return -EINVAL;
 13	/* Only support dependency between two objects of the same type */
 14	if (obj_dependent->type != obj_depended->type)
 15		return -EINVAL;
 16
 17	refcount_inc(&obj_depended->users);
 18	return 0;
 19}
 20EXPORT_SYMBOL_NS_GPL(_iommufd_object_depend, "IOMMUFD");
 21
 22/* Driver should use a per-structure helper in include/linux/iommufd.h */
 23void _iommufd_object_undepend(struct iommufd_object *obj_dependent,
 24			      struct iommufd_object *obj_depended)
 25{
 26	if (WARN_ON_ONCE(obj_dependent == obj_depended ||
 27			 obj_dependent->type != obj_depended->type))
 28		return;
 29
 30	refcount_dec(&obj_depended->users);
 31}
 32EXPORT_SYMBOL_NS_GPL(_iommufd_object_undepend, "IOMMUFD");
 33
 34/*
 35 * Allocate an @offset to return to user space to use for an mmap() syscall
 36 *
 37 * Driver should use a per-structure helper in include/linux/iommufd.h
 38 */
 39int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner,
 40			phys_addr_t mmio_addr, size_t length,
 41			unsigned long *offset)
 42{
 43	struct iommufd_mmap *immap;
 44	unsigned long startp;
 45	int rc;
 46
 47	if (!PAGE_ALIGNED(mmio_addr))
 48		return -EINVAL;
 49	if (!length || !PAGE_ALIGNED(length))
 50		return -EINVAL;
 51
 52	immap = kzalloc(sizeof(*immap), GFP_KERNEL);
 53	if (!immap)
 54		return -ENOMEM;
 55	immap->owner = owner;
 56	immap->length = length;
 57	immap->mmio_addr = mmio_addr;
 58
 59	/* Skip the first page to ease caller identifying the returned offset */
 60	rc = mtree_alloc_range(&ictx->mt_mmap, &startp, immap, immap->length,
 61			       PAGE_SIZE, ULONG_MAX, GFP_KERNEL);
 62	if (rc < 0) {
 63		kfree(immap);
 64		return rc;
 65	}
 66
 67	/* mmap() syscall will right-shift the offset in vma->vm_pgoff too */
 68	immap->vm_pgoff = startp >> PAGE_SHIFT;
 69	*offset = startp;
 70	return 0;
 71}
 72EXPORT_SYMBOL_NS_GPL(_iommufd_alloc_mmap, "IOMMUFD");
 73
 74/* Driver should use a per-structure helper in include/linux/iommufd.h */
 75void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
 76			   struct iommufd_object *owner, unsigned long offset)
 77{
 78	struct iommufd_mmap *immap;
 79
 80	immap = mtree_erase(&ictx->mt_mmap, offset);
 81	WARN_ON_ONCE(!immap || immap->owner != owner);
 82	kfree(immap);
 83}
 84EXPORT_SYMBOL_NS_GPL(_iommufd_destroy_mmap, "IOMMUFD");
 85
 86struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev)
 87{
 88	return vdev->idev->dev;
 89}
 90EXPORT_SYMBOL_NS_GPL(iommufd_vdevice_to_device, "IOMMUFD");
 91
 92/* Caller should xa_lock(&viommu->vdevs) to protect the return value */
 93struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu,
 94				       unsigned long vdev_id)
 95{
 96	struct iommufd_vdevice *vdev;
 97
 98	lockdep_assert_held(&viommu->vdevs.xa_lock);
 99
100	vdev = xa_load(&viommu->vdevs, vdev_id);
101	return vdev ? iommufd_vdevice_to_device(vdev) : NULL;
102}
103EXPORT_SYMBOL_NS_GPL(iommufd_viommu_find_dev, "IOMMUFD");
104
105/* Return -ENOENT if device is not associated to the vIOMMU */
106int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
107			       struct device *dev, unsigned long *vdev_id)
108{
109	struct iommufd_vdevice *vdev;
110	unsigned long index;
111	int rc = -ENOENT;
112
113	if (WARN_ON_ONCE(!vdev_id))
114		return -EINVAL;
115
116	xa_lock(&viommu->vdevs);
117	xa_for_each(&viommu->vdevs, index, vdev) {
118		if (iommufd_vdevice_to_device(vdev) == dev) {
119			*vdev_id = vdev->virt_id;
120			rc = 0;
121			break;
122		}
123	}
124	xa_unlock(&viommu->vdevs);
125	return rc;
126}
127EXPORT_SYMBOL_NS_GPL(iommufd_viommu_get_vdev_id, "IOMMUFD");
128
129/*
130 * Typically called in driver's threaded IRQ handler.
131 * The @type and @event_data must be defined in include/uapi/linux/iommufd.h
132 */
133int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
134				enum iommu_veventq_type type, void *event_data,
135				size_t data_len)
136{
137	struct iommufd_veventq *veventq;
138	struct iommufd_vevent *vevent;
139	int rc = 0;
140
141	if (WARN_ON_ONCE(!data_len || !event_data))
142		return -EINVAL;
143
144	down_read(&viommu->veventqs_rwsem);
145
146	veventq = iommufd_viommu_find_veventq(viommu, type);
147	if (!veventq) {
148		rc = -EOPNOTSUPP;
149		goto out_unlock_veventqs;
150	}
151
152	spin_lock(&veventq->common.lock);
153	if (veventq->num_events == veventq->depth) {
154		vevent = &veventq->lost_events_header;
155		goto out_set_header;
156	}
157
158	vevent = kzalloc(struct_size(vevent, event_data, data_len), GFP_ATOMIC);
159	if (!vevent) {
160		rc = -ENOMEM;
161		vevent = &veventq->lost_events_header;
162		goto out_set_header;
163	}
164	vevent->data_len = data_len;
165	memcpy(vevent->event_data, event_data, data_len);
166	veventq->num_events++;
167
168out_set_header:
169	iommufd_vevent_handler(veventq, vevent);
170	spin_unlock(&veventq->common.lock);
171out_unlock_veventqs:
172	up_read(&viommu->veventqs_rwsem);
173	return rc;
174}
175EXPORT_SYMBOL_NS_GPL(iommufd_viommu_report_event, "IOMMUFD");
176
177#ifdef CONFIG_IRQ_MSI_IOMMU
178/*
179 * Get a iommufd_sw_msi_map for the msi physical address requested by the irq
180 * layer. The mapping to IOVA is global to the iommufd file descriptor, every
181 * domain that is attached to a device using the same MSI parameters will use
182 * the same IOVA.
183 */
184static struct iommufd_sw_msi_map *
185iommufd_sw_msi_get_map(struct iommufd_ctx *ictx, phys_addr_t msi_addr,
186		       phys_addr_t sw_msi_start)
187{
188	struct iommufd_sw_msi_map *cur;
189	unsigned int max_pgoff = 0;
190
191	lockdep_assert_held(&ictx->sw_msi_lock);
192
193	list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) {
194		if (cur->sw_msi_start != sw_msi_start)
195			continue;
196		max_pgoff = max(max_pgoff, cur->pgoff + 1);
197		if (cur->msi_addr == msi_addr)
198			return cur;
199	}
200
201	if (ictx->sw_msi_id >=
202	    BITS_PER_BYTE * sizeof_field(struct iommufd_sw_msi_maps, bitmap))
203		return ERR_PTR(-EOVERFLOW);
204
205	cur = kzalloc(sizeof(*cur), GFP_KERNEL);
206	if (!cur)
207		return ERR_PTR(-ENOMEM);
208
209	cur->sw_msi_start = sw_msi_start;
210	cur->msi_addr = msi_addr;
211	cur->pgoff = max_pgoff;
212	cur->id = ictx->sw_msi_id++;
213	list_add_tail(&cur->sw_msi_item, &ictx->sw_msi_list);
214	return cur;
215}
216
217int iommufd_sw_msi_install(struct iommufd_ctx *ictx,
218			   struct iommufd_hwpt_paging *hwpt_paging,
219			   struct iommufd_sw_msi_map *msi_map)
220{
221	unsigned long iova;
222
223	lockdep_assert_held(&ictx->sw_msi_lock);
224
225	iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
226	if (!test_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap)) {
227		int rc;
228
229		rc = iommu_map(hwpt_paging->common.domain, iova,
230			       msi_map->msi_addr, PAGE_SIZE,
231			       IOMMU_WRITE | IOMMU_READ | IOMMU_MMIO,
232			       GFP_KERNEL_ACCOUNT);
233		if (rc)
234			return rc;
235		__set_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap);
236	}
237	return 0;
238}
239EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi_install, "IOMMUFD_INTERNAL");
240
241/*
242 * Called by the irq code if the platform translates the MSI address through the
243 * IOMMU. msi_addr is the physical address of the MSI page. iommufd will
244 * allocate a fd global iova for the physical page that is the same on all
245 * domains and devices.
246 */
247int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
248		   phys_addr_t msi_addr)
249{
250	struct device *dev = msi_desc_to_dev(desc);
251	struct iommufd_hwpt_paging *hwpt_paging;
252	struct iommu_attach_handle *raw_handle;
253	struct iommufd_attach_handle *handle;
254	struct iommufd_sw_msi_map *msi_map;
255	struct iommufd_ctx *ictx;
256	unsigned long iova;
257	int rc;
258
259	/*
260	 * It is safe to call iommu_attach_handle_get() here because the iommu
261	 * core code invokes this under the group mutex which also prevents any
262	 * change of the attach handle for the duration of this function.
263	 */
264	iommu_group_mutex_assert(dev);
265
266	raw_handle =
267		iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0);
268	if (IS_ERR(raw_handle))
269		return 0;
270	hwpt_paging = find_hwpt_paging(domain->iommufd_hwpt);
271
272	handle = to_iommufd_handle(raw_handle);
273	/* No IOMMU_RESV_SW_MSI means no change to the msi_msg */
274	if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX)
275		return 0;
276
277	ictx = handle->idev->ictx;
278	guard(mutex)(&ictx->sw_msi_lock);
279	/*
280	 * The input msi_addr is the exact byte offset of the MSI doorbell, we
281	 * assume the caller has checked that it is contained with a MMIO region
282	 * that is secure to map at PAGE_SIZE.
283	 */
284	msi_map = iommufd_sw_msi_get_map(handle->idev->ictx,
285					 msi_addr & PAGE_MASK,
286					 handle->idev->igroup->sw_msi_start);
287	if (IS_ERR(msi_map))
288		return PTR_ERR(msi_map);
289
290	rc = iommufd_sw_msi_install(ictx, hwpt_paging, msi_map);
291	if (rc)
292		return rc;
293	__set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap);
294
295	iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
296	msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT);
297	return 0;
298}
299EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi, "IOMMUFD");
300#endif
301
302MODULE_DESCRIPTION("iommufd code shared with builtin modules");
303MODULE_IMPORT_NS("IOMMUFD_INTERNAL");
304MODULE_LICENSE("GPL");