vfio_pci_rdwr.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * VFIO PCI I/O Port & MMIO access
  4 *
  5 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
  6 *     Author: Alex Williamson <alex.williamson@redhat.com>
  7 *
  8 * Derived from original vfio:
  9 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
 10 * Author: Tom Lyon, pugs@cisco.com
 11 */
 12
 13#include <linux/fs.h>
 14#include <linux/pci.h>
 15#include <linux/uaccess.h>
 16#include <linux/io.h>
 17#include <linux/vfio.h>
 18#include <linux/vgaarb.h>
 19#include <linux/io-64-nonatomic-lo-hi.h>
 20
 21#include "vfio_pci_priv.h"
 22
 23#ifdef __LITTLE_ENDIAN
 24#define vfio_ioread64	ioread64
 25#define vfio_iowrite64	iowrite64
 26#define vfio_ioread32	ioread32
 27#define vfio_iowrite32	iowrite32
 28#define vfio_ioread16	ioread16
 29#define vfio_iowrite16	iowrite16
 30#else
 31#define vfio_ioread64	ioread64be
 32#define vfio_iowrite64	iowrite64be
 33#define vfio_ioread32	ioread32be
 34#define vfio_iowrite32	iowrite32be
 35#define vfio_ioread16	ioread16be
 36#define vfio_iowrite16	iowrite16be
 37#endif
 38#define vfio_ioread8	ioread8
 39#define vfio_iowrite8	iowrite8
 40
 41#define VFIO_IOWRITE(size) \
 42int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev,	\
 43			bool test_mem, u##size val, void __iomem *io)	\
 44{									\
 45	if (test_mem) {							\
 46		down_read(&vdev->memory_lock);				\
 47		if (!__vfio_pci_memory_enabled(vdev)) {			\
 48			up_read(&vdev->memory_lock);			\
 49			return -EIO;					\
 50		}							\
 51	}								\
 52									\
 53	vfio_iowrite##size(val, io);					\
 54									\
 55	if (test_mem)							\
 56		up_read(&vdev->memory_lock);				\
 57									\
 58	return 0;							\
 59}									\
 60EXPORT_SYMBOL_GPL(vfio_pci_core_iowrite##size);
 61
 62VFIO_IOWRITE(8)
 63VFIO_IOWRITE(16)
 64VFIO_IOWRITE(32)
 65VFIO_IOWRITE(64)
 66
 67#define VFIO_IOREAD(size) \
 68int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev,	\
 69			bool test_mem, u##size *val, void __iomem *io)	\
 70{									\
 71	if (test_mem) {							\
 72		down_read(&vdev->memory_lock);				\
 73		if (!__vfio_pci_memory_enabled(vdev)) {			\
 74			up_read(&vdev->memory_lock);			\
 75			return -EIO;					\
 76		}							\
 77	}								\
 78									\
 79	*val = vfio_ioread##size(io);					\
 80									\
 81	if (test_mem)							\
 82		up_read(&vdev->memory_lock);				\
 83									\
 84	return 0;							\
 85}									\
 86EXPORT_SYMBOL_GPL(vfio_pci_core_ioread##size);
 87
 88VFIO_IOREAD(8)
 89VFIO_IOREAD(16)
 90VFIO_IOREAD(32)
 91VFIO_IOREAD(64)
 92
 93#define VFIO_IORDWR(size)						\
 94static int vfio_pci_iordwr##size(struct vfio_pci_core_device *vdev,\
 95				bool iswrite, bool test_mem,		\
 96				void __iomem *io, char __user *buf,	\
 97				loff_t off, size_t *filled)		\
 98{									\
 99	u##size val;							\
100	int ret;							\
101									\
102	if (iswrite) {							\
103		if (copy_from_user(&val, buf, sizeof(val)))		\
104			return -EFAULT;					\
105									\
106		ret = vfio_pci_core_iowrite##size(vdev, test_mem,	\
107						  val, io + off);	\
108		if (ret)						\
109			return ret;					\
110	} else {							\
111		ret = vfio_pci_core_ioread##size(vdev, test_mem,	\
112						 &val, io + off);	\
113		if (ret)						\
114			return ret;					\
115									\
116		if (copy_to_user(buf, &val, sizeof(val)))		\
117			return -EFAULT;					\
118	}								\
119									\
120	*filled = sizeof(val);						\
121	return 0;							\
122}									\
123
124VFIO_IORDWR(8)
125VFIO_IORDWR(16)
126VFIO_IORDWR(32)
127VFIO_IORDWR(64)
128
129/*
130 * Read or write from an __iomem region (MMIO or I/O port) with an excluded
131 * range which is inaccessible.  The excluded range drops writes and fills
132 * reads with -1.  This is intended for handling MSI-X vector tables and
133 * leftover space for ROM BARs.
134 */
135ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
136			       void __iomem *io, char __user *buf,
137			       loff_t off, size_t count, size_t x_start,
138			       size_t x_end, bool iswrite,
139			       enum vfio_pci_io_width max_width)
140{
141	ssize_t done = 0;
142	int ret;
143
144	while (count) {
145		size_t fillable, filled;
146
147		if (off < x_start)
148			fillable = min(count, (size_t)(x_start - off));
149		else if (off >= x_end)
150			fillable = count;
151		else
152			fillable = 0;
153
154		if (fillable >= 8 && !(off % 8) && max_width >= 8) {
155			ret = vfio_pci_iordwr64(vdev, iswrite, test_mem,
156						io, buf, off, &filled);
157			if (ret)
158				return ret;
159
160		} else if (fillable >= 4 && !(off % 4) && max_width >= 4) {
161			ret = vfio_pci_iordwr32(vdev, iswrite, test_mem,
162						io, buf, off, &filled);
163			if (ret)
164				return ret;
165
166		} else if (fillable >= 2 && !(off % 2) && max_width >= 2) {
167			ret = vfio_pci_iordwr16(vdev, iswrite, test_mem,
168						io, buf, off, &filled);
169			if (ret)
170				return ret;
171
172		} else if (fillable) {
173			ret = vfio_pci_iordwr8(vdev, iswrite, test_mem,
174					       io, buf, off, &filled);
175			if (ret)
176				return ret;
177
178		} else {
179			/* Fill reads with -1, drop writes */
180			filled = min(count, (size_t)(x_end - off));
181			if (!iswrite) {
182				u8 val = 0xFF;
183				size_t i;
184
185				for (i = 0; i < filled; i++)
186					if (copy_to_user(buf + i, &val, 1))
187						return -EFAULT;
188			}
189		}
190
191		count -= filled;
192		done += filled;
193		off += filled;
194		buf += filled;
195	}
196
197	return done;
198}
199EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw);
200
201int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
202{
203	struct pci_dev *pdev = vdev->pdev;
204	int ret;
205	void __iomem *io;
206
207	if (vdev->barmap[bar])
208		return 0;
209
210	ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
211	if (ret)
212		return ret;
213
214	io = pci_iomap(pdev, bar, 0);
215	if (!io) {
216		pci_release_selected_regions(pdev, 1 << bar);
217		return -ENOMEM;
218	}
219
220	vdev->barmap[bar] = io;
221
222	return 0;
223}
224EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap);
225
226ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
227			size_t count, loff_t *ppos, bool iswrite)
228{
229	struct pci_dev *pdev = vdev->pdev;
230	loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
231	int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
232	size_t x_start = 0, x_end = 0;
233	resource_size_t end;
234	void __iomem *io;
235	struct resource *res = &vdev->pdev->resource[bar];
236	ssize_t done;
237	enum vfio_pci_io_width max_width = VFIO_PCI_IO_WIDTH_8;
238
239	if (pci_resource_start(pdev, bar))
240		end = pci_resource_len(pdev, bar);
241	else if (bar == PCI_ROM_RESOURCE && pdev->rom && pdev->romlen)
242		end = roundup_pow_of_two(pdev->romlen);
243	else
244		return -EINVAL;
245
246	if (pos >= end)
247		return -EINVAL;
248
249	count = min(count, (size_t)(end - pos));
250
251	if (bar == PCI_ROM_RESOURCE) {
252		/*
253		 * The ROM can fill less space than the BAR, so we start the
254		 * excluded range at the end of the actual ROM.  This makes
255		 * filling large ROM BARs much faster.
256		 */
257		if (pci_resource_start(pdev, bar)) {
258			io = pci_map_rom(pdev, &x_start);
259		} else {
260			io = ioremap(pdev->rom, pdev->romlen);
261			x_start = pdev->romlen;
262		}
263		if (!io)
264			return -ENOMEM;
265		x_end = end;
266
267		/*
268		 * Certain devices (e.g. Intel X710) don't support qword
269		 * access to the ROM bar. Otherwise PCI AER errors might be
270		 * triggered.
271		 *
272		 * Disable qword access to the ROM bar universally, which
273		 * worked reliably for years before qword access is enabled.
274		 */
275		max_width = VFIO_PCI_IO_WIDTH_4;
276	} else {
277		int ret = vfio_pci_core_setup_barmap(vdev, bar);
278		if (ret) {
279			done = ret;
280			goto out;
281		}
282
283		io = vdev->barmap[bar];
284	}
285
286	if (bar == vdev->msix_bar) {
287		x_start = vdev->msix_offset;
288		x_end = vdev->msix_offset + vdev->msix_size;
289	}
290
291	done = vfio_pci_core_do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
292				      count, x_start, x_end, iswrite, max_width);
293
294	if (done >= 0)
295		*ppos += done;
296
297	if (bar == PCI_ROM_RESOURCE) {
298		if (pci_resource_start(pdev, bar))
299			pci_unmap_rom(pdev, io);
300		else
301			iounmap(io);
302	}
303
304out:
305	return done;
306}
307
308#ifdef CONFIG_VFIO_PCI_VGA
309ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
310			       size_t count, loff_t *ppos, bool iswrite)
311{
312	int ret;
313	loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
314	void __iomem *iomem = NULL;
315	unsigned int rsrc;
316	bool is_ioport;
317	ssize_t done;
318
319	if (!vdev->has_vga)
320		return -EINVAL;
321
322	if (pos > 0xbfffful)
323		return -EINVAL;
324
325	switch ((u32)pos) {
326	case 0xa0000 ... 0xbffff:
327		count = min(count, (size_t)(0xc0000 - pos));
328		iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
329		off = pos - 0xa0000;
330		rsrc = VGA_RSRC_LEGACY_MEM;
331		is_ioport = false;
332		break;
333	case 0x3b0 ... 0x3bb:
334		count = min(count, (size_t)(0x3bc - pos));
335		iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
336		off = pos - 0x3b0;
337		rsrc = VGA_RSRC_LEGACY_IO;
338		is_ioport = true;
339		break;
340	case 0x3c0 ... 0x3df:
341		count = min(count, (size_t)(0x3e0 - pos));
342		iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
343		off = pos - 0x3c0;
344		rsrc = VGA_RSRC_LEGACY_IO;
345		is_ioport = true;
346		break;
347	default:
348		return -EINVAL;
349	}
350
351	if (!iomem)
352		return -ENOMEM;
353
354	ret = vga_get_interruptible(vdev->pdev, rsrc);
355	if (ret) {
356		is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
357		return ret;
358	}
359
360	/*
361	 * VGA MMIO is a legacy, non-BAR resource that hopefully allows
362	 * probing, so we don't currently worry about access in relation
363	 * to the memory enable bit in the command register.
364	 */
365	done = vfio_pci_core_do_io_rw(vdev, false, iomem, buf, off, count,
366				      0, 0, iswrite, VFIO_PCI_IO_WIDTH_4);
367
368	vga_put(vdev->pdev, rsrc);
369
370	is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
371
372	if (done >= 0)
373		*ppos += done;
374
375	return done;
376}
377#endif
378
379static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
380					bool test_mem)
381{
382	switch (ioeventfd->count) {
383	case 1:
384		vfio_pci_core_iowrite8(ioeventfd->vdev, test_mem,
385				       ioeventfd->data, ioeventfd->addr);
386		break;
387	case 2:
388		vfio_pci_core_iowrite16(ioeventfd->vdev, test_mem,
389					ioeventfd->data, ioeventfd->addr);
390		break;
391	case 4:
392		vfio_pci_core_iowrite32(ioeventfd->vdev, test_mem,
393					ioeventfd->data, ioeventfd->addr);
394		break;
395	case 8:
396		vfio_pci_core_iowrite64(ioeventfd->vdev, test_mem,
397					ioeventfd->data, ioeventfd->addr);
398		break;
399	}
400}
401
402static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
403{
404	struct vfio_pci_ioeventfd *ioeventfd = opaque;
405	struct vfio_pci_core_device *vdev = ioeventfd->vdev;
406
407	if (ioeventfd->test_mem) {
408		if (!down_read_trylock(&vdev->memory_lock))
409			return 1; /* Lock contended, use thread */
410		if (!__vfio_pci_memory_enabled(vdev)) {
411			up_read(&vdev->memory_lock);
412			return 0;
413		}
414	}
415
416	vfio_pci_ioeventfd_do_write(ioeventfd, false);
417
418	if (ioeventfd->test_mem)
419		up_read(&vdev->memory_lock);
420
421	return 0;
422}
423
424static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
425{
426	struct vfio_pci_ioeventfd *ioeventfd = opaque;
427
428	vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
429}
430
431int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
432		       uint64_t data, int count, int fd)
433{
434	struct pci_dev *pdev = vdev->pdev;
435	loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
436	int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
437	struct vfio_pci_ioeventfd *ioeventfd;
438
439	/* Only support ioeventfds into BARs */
440	if (bar > VFIO_PCI_BAR5_REGION_INDEX)
441		return -EINVAL;
442
443	if (pos + count > pci_resource_len(pdev, bar))
444		return -EINVAL;
445
446	/* Disallow ioeventfds working around MSI-X table writes */
447	if (bar == vdev->msix_bar &&
448	    !(pos + count <= vdev->msix_offset ||
449	      pos >= vdev->msix_offset + vdev->msix_size))
450		return -EINVAL;
451
452	if (count == 8)
453		return -EINVAL;
454
455	ret = vfio_pci_core_setup_barmap(vdev, bar);
456	if (ret)
457		return ret;
458
459	mutex_lock(&vdev->ioeventfds_lock);
460
461	list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
462		if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
463		    ioeventfd->data == data && ioeventfd->count == count) {
464			if (fd == -1) {
465				vfio_virqfd_disable(&ioeventfd->virqfd);
466				list_del(&ioeventfd->next);
467				vdev->ioeventfds_nr--;
468				kfree(ioeventfd);
469				ret = 0;
470			} else
471				ret = -EEXIST;
472
473			goto out_unlock;
474		}
475	}
476
477	if (fd < 0) {
478		ret = -ENODEV;
479		goto out_unlock;
480	}
481
482	if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
483		ret = -ENOSPC;
484		goto out_unlock;
485	}
486
487	ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT);
488	if (!ioeventfd) {
489		ret = -ENOMEM;
490		goto out_unlock;
491	}
492
493	ioeventfd->vdev = vdev;
494	ioeventfd->addr = vdev->barmap[bar] + pos;
495	ioeventfd->data = data;
496	ioeventfd->pos = pos;
497	ioeventfd->bar = bar;
498	ioeventfd->count = count;
499	ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
500
501	ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
502				 vfio_pci_ioeventfd_thread, NULL,
503				 &ioeventfd->virqfd, fd);
504	if (ret) {
505		kfree(ioeventfd);
506		goto out_unlock;
507	}
508
509	list_add(&ioeventfd->next, &vdev->ioeventfds_list);
510	vdev->ioeventfds_nr++;
511
512out_unlock:
513	mutex_unlock(&vdev->ioeventfds_lock);
514
515	return ret;
516}