vfio_pci_dmabuf.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
  3 */
  4#include <linux/dma-buf-mapping.h>
  5#include <linux/pci-p2pdma.h>
  6#include <linux/dma-resv.h>
  7
  8#include "vfio_pci_priv.h"
  9
 10MODULE_IMPORT_NS("DMA_BUF");
 11
 12struct vfio_pci_dma_buf {
 13	struct dma_buf *dmabuf;
 14	struct vfio_pci_core_device *vdev;
 15	struct list_head dmabufs_elm;
 16	size_t size;
 17	struct dma_buf_phys_vec *phys_vec;
 18	struct p2pdma_provider *provider;
 19	u32 nr_ranges;
 20	u8 revoked : 1;
 21};
 22
 23static int vfio_pci_dma_buf_pin(struct dma_buf_attachment *attachment)
 24{
 25	return -EOPNOTSUPP;
 26}
 27
 28static void vfio_pci_dma_buf_unpin(struct dma_buf_attachment *attachment)
 29{
 30	/* Do nothing */
 31}
 32
 33static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
 34				   struct dma_buf_attachment *attachment)
 35{
 36	struct vfio_pci_dma_buf *priv = dmabuf->priv;
 37
 38	if (!attachment->peer2peer)
 39		return -EOPNOTSUPP;
 40
 41	if (priv->revoked)
 42		return -ENODEV;
 43
 44	return 0;
 45}
 46
 47static struct sg_table *
 48vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment,
 49		     enum dma_data_direction dir)
 50{
 51	struct vfio_pci_dma_buf *priv = attachment->dmabuf->priv;
 52
 53	dma_resv_assert_held(priv->dmabuf->resv);
 54
 55	if (priv->revoked)
 56		return ERR_PTR(-ENODEV);
 57
 58	return dma_buf_phys_vec_to_sgt(attachment, priv->provider,
 59				       priv->phys_vec, priv->nr_ranges,
 60				       priv->size, dir);
 61}
 62
 63static void vfio_pci_dma_buf_unmap(struct dma_buf_attachment *attachment,
 64				   struct sg_table *sgt,
 65				   enum dma_data_direction dir)
 66{
 67	dma_buf_free_sgt(attachment, sgt, dir);
 68}
 69
 70static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf)
 71{
 72	struct vfio_pci_dma_buf *priv = dmabuf->priv;
 73
 74	/*
 75	 * Either this or vfio_pci_dma_buf_cleanup() will remove from the list.
 76	 * The refcount prevents both.
 77	 */
 78	if (priv->vdev) {
 79		down_write(&priv->vdev->memory_lock);
 80		list_del_init(&priv->dmabufs_elm);
 81		up_write(&priv->vdev->memory_lock);
 82		vfio_device_put_registration(&priv->vdev->vdev);
 83	}
 84	kfree(priv->phys_vec);
 85	kfree(priv);
 86}
 87
 88static const struct dma_buf_ops vfio_pci_dmabuf_ops = {
 89	.pin = vfio_pci_dma_buf_pin,
 90	.unpin = vfio_pci_dma_buf_unpin,
 91	.attach = vfio_pci_dma_buf_attach,
 92	.map_dma_buf = vfio_pci_dma_buf_map,
 93	.unmap_dma_buf = vfio_pci_dma_buf_unmap,
 94	.release = vfio_pci_dma_buf_release,
 95};
 96
 97/*
 98 * This is a temporary "private interconnect" between VFIO DMABUF and iommufd.
 99 * It allows the two co-operating drivers to exchange the physical address of
100 * the BAR. This is to be replaced with a formal DMABUF system for negotiated
101 * interconnect types.
102 *
103 * If this function succeeds the following are true:
104 *  - There is one physical range and it is pointing to MMIO
105 *  - When move_notify is called it means revoke, not move, vfio_dma_buf_map
106 *    will fail if it is currently revoked
107 */
108int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment,
109				 struct dma_buf_phys_vec *phys)
110{
111	struct vfio_pci_dma_buf *priv;
112
113	dma_resv_assert_held(attachment->dmabuf->resv);
114
115	if (attachment->dmabuf->ops != &vfio_pci_dmabuf_ops)
116		return -EOPNOTSUPP;
117
118	priv = attachment->dmabuf->priv;
119	if (priv->revoked)
120		return -ENODEV;
121
122	/* More than one range to iommufd will require proper DMABUF support */
123	if (priv->nr_ranges != 1)
124		return -EOPNOTSUPP;
125
126	*phys = priv->phys_vec[0];
127	return 0;
128}
129EXPORT_SYMBOL_FOR_MODULES(vfio_pci_dma_buf_iommufd_map, "iommufd");
130
131int vfio_pci_core_fill_phys_vec(struct dma_buf_phys_vec *phys_vec,
132				struct vfio_region_dma_range *dma_ranges,
133				size_t nr_ranges, phys_addr_t start,
134				phys_addr_t len)
135{
136	phys_addr_t max_addr;
137	unsigned int i;
138
139	max_addr = start + len;
140	for (i = 0; i < nr_ranges; i++) {
141		phys_addr_t end;
142
143		if (!dma_ranges[i].length)
144			return -EINVAL;
145
146		if (check_add_overflow(start, dma_ranges[i].offset,
147				       &phys_vec[i].paddr) ||
148		    check_add_overflow(phys_vec[i].paddr,
149				       dma_ranges[i].length, &end))
150			return -EOVERFLOW;
151		if (end > max_addr)
152			return -EINVAL;
153
154		phys_vec[i].len = dma_ranges[i].length;
155	}
156	return 0;
157}
158EXPORT_SYMBOL_GPL(vfio_pci_core_fill_phys_vec);
159
160int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev,
161				  struct p2pdma_provider **provider,
162				  unsigned int region_index,
163				  struct dma_buf_phys_vec *phys_vec,
164				  struct vfio_region_dma_range *dma_ranges,
165				  size_t nr_ranges)
166{
167	struct pci_dev *pdev = vdev->pdev;
168
169	*provider = pcim_p2pdma_provider(pdev, region_index);
170	if (!*provider)
171		return -EINVAL;
172
173	return vfio_pci_core_fill_phys_vec(
174		phys_vec, dma_ranges, nr_ranges,
175		pci_resource_start(pdev, region_index),
176		pci_resource_len(pdev, region_index));
177}
178EXPORT_SYMBOL_GPL(vfio_pci_core_get_dmabuf_phys);
179
180static int validate_dmabuf_input(struct vfio_device_feature_dma_buf *dma_buf,
181				 struct vfio_region_dma_range *dma_ranges,
182				 size_t *lengthp)
183{
184	size_t length = 0;
185	u32 i;
186
187	for (i = 0; i < dma_buf->nr_ranges; i++) {
188		u64 offset = dma_ranges[i].offset;
189		u64 len = dma_ranges[i].length;
190
191		if (!len || !PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
192			return -EINVAL;
193
194		if (check_add_overflow(length, len, &length))
195			return -EINVAL;
196	}
197
198	/*
199	 * dma_iova_try_alloc() will WARN on if userspace proposes a size that
200	 * is too big, eg with lots of ranges.
201	 */
202	if ((u64)(length) & DMA_IOVA_USE_SWIOTLB)
203		return -EINVAL;
204
205	*lengthp = length;
206	return 0;
207}
208
209int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
210				  struct vfio_device_feature_dma_buf __user *arg,
211				  size_t argsz)
212{
213	struct vfio_device_feature_dma_buf get_dma_buf = {};
214	struct vfio_region_dma_range *dma_ranges;
215	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
216	struct vfio_pci_dma_buf *priv;
217	size_t length;
218	int ret;
219
220	if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys)
221		return -EOPNOTSUPP;
222
223	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
224				 sizeof(get_dma_buf));
225	if (ret != 1)
226		return ret;
227
228	if (copy_from_user(&get_dma_buf, arg, sizeof(get_dma_buf)))
229		return -EFAULT;
230
231	if (!get_dma_buf.nr_ranges || get_dma_buf.flags)
232		return -EINVAL;
233
234	/*
235	 * For PCI the region_index is the BAR number like everything else.
236	 */
237	if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX)
238		return -ENODEV;
239
240	dma_ranges = memdup_array_user(&arg->dma_ranges, get_dma_buf.nr_ranges,
241				       sizeof(*dma_ranges));
242	if (IS_ERR(dma_ranges))
243		return PTR_ERR(dma_ranges);
244
245	ret = validate_dmabuf_input(&get_dma_buf, dma_ranges, &length);
246	if (ret)
247		goto err_free_ranges;
248
249	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
250	if (!priv) {
251		ret = -ENOMEM;
252		goto err_free_ranges;
253	}
254	priv->phys_vec = kcalloc(get_dma_buf.nr_ranges, sizeof(*priv->phys_vec),
255				 GFP_KERNEL);
256	if (!priv->phys_vec) {
257		ret = -ENOMEM;
258		goto err_free_priv;
259	}
260
261	priv->vdev = vdev;
262	priv->nr_ranges = get_dma_buf.nr_ranges;
263	priv->size = length;
264	ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider,
265					     get_dma_buf.region_index,
266					     priv->phys_vec, dma_ranges,
267					     priv->nr_ranges);
268	if (ret)
269		goto err_free_phys;
270
271	kfree(dma_ranges);
272	dma_ranges = NULL;
273
274	if (!vfio_device_try_get_registration(&vdev->vdev)) {
275		ret = -ENODEV;
276		goto err_free_phys;
277	}
278
279	exp_info.ops = &vfio_pci_dmabuf_ops;
280	exp_info.size = priv->size;
281	exp_info.flags = get_dma_buf.open_flags;
282	exp_info.priv = priv;
283
284	priv->dmabuf = dma_buf_export(&exp_info);
285	if (IS_ERR(priv->dmabuf)) {
286		ret = PTR_ERR(priv->dmabuf);
287		goto err_dev_put;
288	}
289
290	/* dma_buf_put() now frees priv */
291	INIT_LIST_HEAD(&priv->dmabufs_elm);
292	down_write(&vdev->memory_lock);
293	dma_resv_lock(priv->dmabuf->resv, NULL);
294	priv->revoked = !__vfio_pci_memory_enabled(vdev);
295	list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs);
296	dma_resv_unlock(priv->dmabuf->resv);
297	up_write(&vdev->memory_lock);
298
299	/*
300	 * dma_buf_fd() consumes the reference, when the file closes the dmabuf
301	 * will be released.
302	 */
303	ret = dma_buf_fd(priv->dmabuf, get_dma_buf.open_flags);
304	if (ret < 0)
305		dma_buf_put(priv->dmabuf);
306
307	return ret;
308
309err_dev_put:
310	vfio_device_put_registration(&vdev->vdev);
311err_free_phys:
312	kfree(priv->phys_vec);
313err_free_priv:
314	kfree(priv);
315err_free_ranges:
316	kfree(dma_ranges);
317	return ret;
318}
319
320void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
321{
322	struct vfio_pci_dma_buf *priv;
323	struct vfio_pci_dma_buf *tmp;
324
325	lockdep_assert_held_write(&vdev->memory_lock);
326
327	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
328		if (!get_file_active(&priv->dmabuf->file))
329			continue;
330
331		if (priv->revoked != revoked) {
332			dma_resv_lock(priv->dmabuf->resv, NULL);
333			priv->revoked = revoked;
334			dma_buf_move_notify(priv->dmabuf);
335			dma_resv_unlock(priv->dmabuf->resv);
336		}
337		fput(priv->dmabuf->file);
338	}
339}
340
341void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
342{
343	struct vfio_pci_dma_buf *priv;
344	struct vfio_pci_dma_buf *tmp;
345
346	down_write(&vdev->memory_lock);
347	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
348		if (!get_file_active(&priv->dmabuf->file))
349			continue;
350
351		dma_resv_lock(priv->dmabuf->resv, NULL);
352		list_del_init(&priv->dmabufs_elm);
353		priv->vdev = NULL;
354		priv->revoked = true;
355		dma_buf_move_notify(priv->dmabuf);
356		dma_resv_unlock(priv->dmabuf->resv);
357		vfio_device_put_registration(&vdev->vdev);
358		fput(priv->dmabuf->file);
359	}
360	up_write(&vdev->memory_lock);
361}