grant-dma-ops.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Xen grant DMA-mapping layer - contains special DMA-mapping routines
  4 * for providing grant references as DMA addresses to be used by frontends
  5 * (e.g. virtio) in Xen guests
  6 *
  7 * Copyright (c) 2021, Juergen Gross <jgross@suse.com>
  8 */
  9
 10#include <linux/module.h>
 11#include <linux/dma-map-ops.h>
 12#include <linux/of.h>
 13#include <linux/pci.h>
 14#include <linux/pfn.h>
 15#include <linux/xarray.h>
 16#include <linux/virtio_anchor.h>
 17#include <linux/virtio.h>
 18#include <xen/xen.h>
 19#include <xen/xen-ops.h>
 20#include <xen/grant_table.h>
 21
 22struct xen_grant_dma_data {
 23	/* The ID of backend domain */
 24	domid_t backend_domid;
 25	/* Is device behaving sane? */
 26	bool broken;
 27};
 28
 29static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);
 30
 31#define XEN_GRANT_DMA_ADDR_OFF	(1ULL << 63)
 32
 33static inline dma_addr_t grant_to_dma(grant_ref_t grant)
 34{
 35	return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT);
 36}
 37
 38static inline grant_ref_t dma_to_grant(dma_addr_t dma)
 39{
 40	return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT);
 41}
 42
 43static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)
 44{
 45	struct xen_grant_dma_data *data;
 46	unsigned long flags;
 47
 48	xa_lock_irqsave(&xen_grant_dma_devices, flags);
 49	data = xa_load(&xen_grant_dma_devices, (unsigned long)dev);
 50	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
 51
 52	return data;
 53}
 54
 55static int store_xen_grant_dma_data(struct device *dev,
 56				    struct xen_grant_dma_data *data)
 57{
 58	unsigned long flags;
 59	int ret;
 60
 61	xa_lock_irqsave(&xen_grant_dma_devices, flags);
 62	ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,
 63			GFP_ATOMIC));
 64	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
 65
 66	return ret;
 67}
 68
 69/*
 70 * DMA ops for Xen frontends (e.g. virtio).
 71 *
 72 * Used to act as a kind of software IOMMU for Xen guests by using grants as
 73 * DMA addresses.
 74 * Such a DMA address is formed by using the grant reference as a frame
 75 * number and setting the highest address bit (this bit is for the backend
 76 * to be able to distinguish it from e.g. a mmio address).
 77 */
 78static void *xen_grant_dma_alloc(struct device *dev, size_t size,
 79				 dma_addr_t *dma_handle, gfp_t gfp,
 80				 unsigned long attrs)
 81{
 82	struct xen_grant_dma_data *data;
 83	unsigned int i, n_pages = XEN_PFN_UP(size);
 84	unsigned long pfn;
 85	grant_ref_t grant;
 86	void *ret;
 87
 88	data = find_xen_grant_dma_data(dev);
 89	if (!data)
 90		return NULL;
 91
 92	if (unlikely(data->broken))
 93		return NULL;
 94
 95	ret = alloc_pages_exact(n_pages * XEN_PAGE_SIZE, gfp);
 96	if (!ret)
 97		return NULL;
 98
 99	pfn = virt_to_pfn(ret);
100
101	if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) {
102		free_pages_exact(ret, n_pages * XEN_PAGE_SIZE);
103		return NULL;
104	}
105
106	for (i = 0; i < n_pages; i++) {
107		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
108				pfn_to_gfn(pfn + i), 0);
109	}
110
111	*dma_handle = grant_to_dma(grant);
112
113	return ret;
114}
115
116static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr,
117			       dma_addr_t dma_handle, unsigned long attrs)
118{
119	struct xen_grant_dma_data *data;
120	unsigned int i, n_pages = XEN_PFN_UP(size);
121	grant_ref_t grant;
122
123	data = find_xen_grant_dma_data(dev);
124	if (!data)
125		return;
126
127	if (unlikely(data->broken))
128		return;
129
130	grant = dma_to_grant(dma_handle);
131
132	for (i = 0; i < n_pages; i++) {
133		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
134			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
135			data->broken = true;
136			return;
137		}
138	}
139
140	gnttab_free_grant_reference_seq(grant, n_pages);
141
142	free_pages_exact(vaddr, n_pages * XEN_PAGE_SIZE);
143}
144
145static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size,
146					      dma_addr_t *dma_handle,
147					      enum dma_data_direction dir,
148					      gfp_t gfp)
149{
150	void *vaddr;
151
152	vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0);
153	if (!vaddr)
154		return NULL;
155
156	return virt_to_page(vaddr);
157}
158
159static void xen_grant_dma_free_pages(struct device *dev, size_t size,
160				     struct page *vaddr, dma_addr_t dma_handle,
161				     enum dma_data_direction dir)
162{
163	xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0);
164}
165
166static dma_addr_t xen_grant_dma_map_phys(struct device *dev, phys_addr_t phys,
167					 size_t size,
168					 enum dma_data_direction dir,
169					 unsigned long attrs)
170{
171	struct xen_grant_dma_data *data;
172	unsigned long offset = offset_in_page(phys);
173	unsigned long dma_offset = xen_offset_in_page(offset),
174			pfn_offset = XEN_PFN_DOWN(offset);
175	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
176	grant_ref_t grant;
177	dma_addr_t dma_handle;
178
179	if (unlikely(attrs & DMA_ATTR_MMIO))
180		return DMA_MAPPING_ERROR;
181
182	if (WARN_ON(dir == DMA_NONE))
183		return DMA_MAPPING_ERROR;
184
185	data = find_xen_grant_dma_data(dev);
186	if (!data)
187		return DMA_MAPPING_ERROR;
188
189	if (unlikely(data->broken))
190		return DMA_MAPPING_ERROR;
191
192	if (gnttab_alloc_grant_reference_seq(n_pages, &grant))
193		return DMA_MAPPING_ERROR;
194
195	for (i = 0; i < n_pages; i++) {
196		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
197				pfn_to_gfn(page_to_xen_pfn(phys_to_page(phys)) + i + pfn_offset),
198				dir == DMA_TO_DEVICE);
199	}
200
201	dma_handle = grant_to_dma(grant) + dma_offset;
202
203	return dma_handle;
204}
205
206static void xen_grant_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle,
207				     size_t size, enum dma_data_direction dir,
208				     unsigned long attrs)
209{
210	struct xen_grant_dma_data *data;
211	unsigned long dma_offset = xen_offset_in_page(dma_handle);
212	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
213	grant_ref_t grant;
214
215	if (WARN_ON(dir == DMA_NONE))
216		return;
217
218	data = find_xen_grant_dma_data(dev);
219	if (!data)
220		return;
221
222	if (unlikely(data->broken))
223		return;
224
225	grant = dma_to_grant(dma_handle);
226
227	for (i = 0; i < n_pages; i++) {
228		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
229			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
230			data->broken = true;
231			return;
232		}
233	}
234
235	gnttab_free_grant_reference_seq(grant, n_pages);
236}
237
238static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
239				   int nents, enum dma_data_direction dir,
240				   unsigned long attrs)
241{
242	struct scatterlist *s;
243	unsigned int i;
244
245	if (WARN_ON(dir == DMA_NONE))
246		return;
247
248	for_each_sg(sg, s, nents, i)
249		xen_grant_dma_unmap_phys(dev, s->dma_address, sg_dma_len(s), dir,
250				attrs);
251}
252
253static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg,
254				int nents, enum dma_data_direction dir,
255				unsigned long attrs)
256{
257	struct scatterlist *s;
258	unsigned int i;
259
260	if (WARN_ON(dir == DMA_NONE))
261		return -EINVAL;
262
263	for_each_sg(sg, s, nents, i) {
264		s->dma_address = xen_grant_dma_map_phys(dev, sg_phys(s),
265				s->length, dir, attrs);
266		if (s->dma_address == DMA_MAPPING_ERROR)
267			goto out;
268
269		sg_dma_len(s) = s->length;
270	}
271
272	return nents;
273
274out:
275	xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
276	sg_dma_len(sg) = 0;
277
278	return -EIO;
279}
280
281static int xen_grant_dma_supported(struct device *dev, u64 mask)
282{
283	return mask == DMA_BIT_MASK(64);
284}
285
286static const struct dma_map_ops xen_grant_dma_ops = {
287	.alloc = xen_grant_dma_alloc,
288	.free = xen_grant_dma_free,
289	.alloc_pages_op = xen_grant_dma_alloc_pages,
290	.free_pages = xen_grant_dma_free_pages,
291	.mmap = dma_common_mmap,
292	.get_sgtable = dma_common_get_sgtable,
293	.map_phys = xen_grant_dma_map_phys,
294	.unmap_phys = xen_grant_dma_unmap_phys,
295	.map_sg = xen_grant_dma_map_sg,
296	.unmap_sg = xen_grant_dma_unmap_sg,
297	.dma_supported = xen_grant_dma_supported,
298};
299
300static struct device_node *xen_dt_get_node(struct device *dev)
301{
302	if (dev_is_pci(dev)) {
303		struct pci_dev *pdev = to_pci_dev(dev);
304		struct pci_bus *bus = pdev->bus;
305
306		/* Walk up to the root bus to look for PCI Host controller */
307		while (!pci_is_root_bus(bus))
308			bus = bus->parent;
309
310		if (!bus->bridge->parent)
311			return NULL;
312		return of_node_get(bus->bridge->parent->of_node);
313	}
314
315	return of_node_get(dev->of_node);
316}
317
318static int xen_dt_grant_init_backend_domid(struct device *dev,
319					   struct device_node *np,
320					   domid_t *backend_domid)
321{
322	struct of_phandle_args iommu_spec = { .args_count = 1 };
323
324	if (dev_is_pci(dev)) {
325		struct pci_dev *pdev = to_pci_dev(dev);
326		u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
327
328		if (of_map_id(np, rid, "iommu-map", "iommu-map-mask", &iommu_spec.np,
329				iommu_spec.args)) {
330			dev_dbg(dev, "Cannot translate ID\n");
331			return -ESRCH;
332		}
333	} else {
334		if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells",
335				0, &iommu_spec)) {
336			dev_dbg(dev, "Cannot parse iommus property\n");
337			return -ESRCH;
338		}
339	}
340
341	if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") ||
342			iommu_spec.args_count != 1) {
343		dev_dbg(dev, "Incompatible IOMMU node\n");
344		of_node_put(iommu_spec.np);
345		return -ESRCH;
346	}
347
348	of_node_put(iommu_spec.np);
349
350	/*
351	 * The endpoint ID here means the ID of the domain where the
352	 * corresponding backend is running
353	 */
354	*backend_domid = iommu_spec.args[0];
355
356	return 0;
357}
358
359static int xen_grant_init_backend_domid(struct device *dev,
360					domid_t *backend_domid)
361{
362	struct device_node *np;
363	int ret = -ENODEV;
364
365	np = xen_dt_get_node(dev);
366	if (np) {
367		ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid);
368		of_node_put(np);
369	} else if (!xen_initial_domain() &&
370		   (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())) {
371		dev_info(dev, "Using dom0 as backend\n");
372		*backend_domid = 0;
373		ret = 0;
374	}
375
376	return ret;
377}
378
379static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid)
380{
381	struct xen_grant_dma_data *data;
382
383	data = find_xen_grant_dma_data(dev);
384	if (data) {
385		dev_err(dev, "Xen grant DMA data is already created\n");
386		return;
387	}
388
389	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
390	if (!data)
391		goto err;
392
393	data->backend_domid = backend_domid;
394
395	if (store_xen_grant_dma_data(dev, data)) {
396		dev_err(dev, "Cannot store Xen grant DMA data\n");
397		goto err;
398	}
399
400	dev->dma_ops = &xen_grant_dma_ops;
401
402	return;
403
404err:
405	devm_kfree(dev, data);
406	dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");
407}
408
409bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
410{
411	domid_t backend_domid;
412
413	if (!xen_grant_init_backend_domid(dev->dev.parent, &backend_domid)) {
414		xen_grant_setup_dma_ops(dev->dev.parent, backend_domid);
415		return true;
416	}
417
418	return false;
419}
420
421MODULE_DESCRIPTION("Xen grant DMA-mapping layer");
422MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");
423MODULE_LICENSE("GPL");