pci_mmio.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Access to PCI I/O memory from user space programs.
  4 *
  5 * Copyright IBM Corp. 2014
  6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
  7 */
  8#include <linux/kernel.h>
  9#include <linux/syscalls.h>
 10#include <linux/init.h>
 11#include <linux/mm.h>
 12#include <linux/errno.h>
 13#include <linux/pci.h>
 14#include <asm/asm-extable.h>
 15#include <asm/pci_io.h>
 16#include <asm/pci_debug.h>
 17#include <asm/asm.h>
 18
 19static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
 20{
 21	struct {
 22		u64 offset;
 23		u8 cc;
 24		u8 status;
 25	} data = {offset, cc, status};
 26
 27	zpci_err_hex(&data, sizeof(data));
 28}
 29
 30static inline int __pcistb_mio_inuser(
 31		void __iomem *ioaddr, const void __user *src,
 32		u64 len, u8 *status)
 33{
 34	int cc, exception;
 35	bool sacf_flag;
 36
 37	exception = 1;
 38	sacf_flag = enable_sacf_uaccess();
 39	asm_inline volatile (
 40		"	sacf	256\n"
 41		"0:	.insn	rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
 42		"1:	lhi	%[exc],0\n"
 43		"2:	sacf	768\n"
 44		CC_IPM(cc)
 45		EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
 46		: CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception)
 47		: [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
 48		: CC_CLOBBER_LIST("memory"));
 49	disable_sacf_uaccess(sacf_flag);
 50	*status = len >> 24 & 0xff;
 51	return exception ? -ENXIO : CC_TRANSFORM(cc);
 52}
 53
 54static inline int __pcistg_mio_inuser(
 55		void __iomem *ioaddr, const void __user *src,
 56		u64 ulen, u8 *status)
 57{
 58	union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
 59	int cc, exception;
 60	bool sacf_flag;
 61	u64 val = 0;
 62	u64 cnt = ulen;
 63	u8 tmp;
 64
 65	/*
 66	 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
 67	 * a register, then store it to PCI at @ioaddr while in secondary
 68	 * address space. pcistg then uses the user mappings.
 69	 */
 70	exception = 1;
 71	sacf_flag = enable_sacf_uaccess();
 72	asm_inline volatile (
 73		"	sacf	256\n"
 74		"0:	llgc	%[tmp],0(%[src])\n"
 75		"4:	sllg	%[val],%[val],8\n"
 76		"	aghi	%[src],1\n"
 77		"	ogr	%[val],%[tmp]\n"
 78		"	brctg	%[cnt],0b\n"
 79		"1:	.insn	rre,0xb9d40000,%[val],%[ioaddr_len]\n"
 80		"2:	lhi	%[exc],0\n"
 81		"3:	sacf	768\n"
 82		CC_IPM(cc)
 83		EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
 84		: [src] "+a" (src), [cnt] "+d" (cnt),
 85		  [val] "+d" (val), [tmp] "=d" (tmp), [exc] "+d" (exception),
 86		  CC_OUT(cc, cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
 87		:
 88		: CC_CLOBBER_LIST("memory"));
 89	disable_sacf_uaccess(sacf_flag);
 90	*status = ioaddr_len.odd >> 24 & 0xff;
 91
 92	cc = exception ? -ENXIO : CC_TRANSFORM(cc);
 93	/* did we read everything from user memory? */
 94	if (!cc && cnt != 0)
 95		cc = -EFAULT;
 96
 97	return cc;
 98}
 99
100static inline int __memcpy_toio_inuser(void __iomem *dst,
101				   const void __user *src, size_t n)
102{
103	int size, rc = 0;
104	u8 status = 0;
105
106	if (!src)
107		return -EINVAL;
108
109	while (n > 0) {
110		size = zpci_get_max_io_size((u64 __force) dst,
111					    (u64 __force) src, n,
112					    ZPCI_MAX_WRITE_SIZE);
113		if (size > 8) /* main path */
114			rc = __pcistb_mio_inuser(dst, src, size, &status);
115		else
116			rc = __pcistg_mio_inuser(dst, src, size, &status);
117		if (rc)
118			break;
119		src += size;
120		dst += size;
121		n -= size;
122	}
123	if (rc)
124		zpci_err_mmio(rc, status, (__force u64) dst);
125	return rc;
126}
127
128SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
129		const void __user *, user_buffer, size_t, length)
130{
131	struct follow_pfnmap_args args = { };
132	u8 local_buf[64];
133	void __iomem *io_addr;
134	void *buf;
135	struct vm_area_struct *vma;
136	long ret;
137
138	if (!zpci_is_enabled())
139		return -ENODEV;
140
141	if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
142		return -EINVAL;
143
144	/*
145	 * We only support write access to MIO capable devices if we are on
146	 * a MIO enabled system. Otherwise we would have to check for every
147	 * address if it is a special ZPCI_ADDR and would have to do
148	 * a pfn lookup which we don't need for MIO capable devices.  Currently
149	 * ISM devices are the only devices without MIO support and there is no
150	 * known need for accessing these from userspace.
151	 */
152	if (static_branch_likely(&have_mio)) {
153		ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
154					user_buffer,
155					length);
156		return ret;
157	}
158
159	if (length > 64) {
160		buf = kmalloc(length, GFP_KERNEL);
161		if (!buf)
162			return -ENOMEM;
163	} else
164		buf = local_buf;
165
166	ret = -EFAULT;
167	if (copy_from_user(buf, user_buffer, length))
168		goto out_free;
169
170	mmap_read_lock(current->mm);
171	ret = -EINVAL;
172	vma = vma_lookup(current->mm, mmio_addr);
173	if (!vma)
174		goto out_unlock_mmap;
175	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
176		goto out_unlock_mmap;
177	ret = -EACCES;
178	if (!(vma->vm_flags & VM_WRITE))
179		goto out_unlock_mmap;
180
181	args.address = mmio_addr;
182	args.vma = vma;
183	ret = follow_pfnmap_start(&args);
184	if (ret) {
185		fixup_user_fault(current->mm, mmio_addr, FAULT_FLAG_WRITE, NULL);
186		ret = follow_pfnmap_start(&args);
187		if (ret)
188			goto out_unlock_mmap;
189	}
190
191	io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) |
192			(mmio_addr & ~PAGE_MASK));
193
194	if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
195		goto out_unlock_pt;
196
197	ret = zpci_memcpy_toio(io_addr, buf, length);
198out_unlock_pt:
199	follow_pfnmap_end(&args);
200out_unlock_mmap:
201	mmap_read_unlock(current->mm);
202out_free:
203	if (buf != local_buf)
204		kfree(buf);
205	return ret;
206}
207
208static inline int __pcilg_mio_inuser(
209		void __user *dst, const void __iomem *ioaddr,
210		u64 ulen, u8 *status)
211{
212	union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
213	bool sacf_flag;
214	u64 cnt = ulen;
215	int shift = ulen * 8;
216	int cc, exception;
217	u64 val, tmp;
218
219	/*
220	 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
221	 * user space) into a register using pcilg then store these bytes at
222	 * user address @dst
223	 */
224	exception = 1;
225	sacf_flag = enable_sacf_uaccess();
226	asm_inline volatile (
227		"	sacf	256\n"
228		"0:	.insn	rre,0xb9d60000,%[val],%[ioaddr_len]\n"
229		"1:	lhi	%[exc],0\n"
230		"	jne	4f\n"
231		"2:	ahi	%[shift],-8\n"
232		"	srlg	%[tmp],%[val],0(%[shift])\n"
233		"3:	stc	%[tmp],0(%[dst])\n"
234		"5:	aghi	%[dst],1\n"
235		"	brctg	%[cnt],2b\n"
236		/*
237		 * Use xr to clear exc and set condition code to zero
238		 * to ensure flag output is correct for this branch.
239		 */
240		"	xr	%[exc],%[exc]\n"
241		"4:	sacf	768\n"
242		CC_IPM(cc)
243		EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
244		: [ioaddr_len] "+&d" (ioaddr_len.pair), [exc] "+d" (exception),
245		  CC_OUT(cc, cc), [val] "=d" (val),
246		  [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
247		  [shift] "+a" (shift)
248		:
249		: CC_CLOBBER_LIST("memory"));
250	disable_sacf_uaccess(sacf_flag);
251	cc = exception ? -ENXIO : CC_TRANSFORM(cc);
252	/* did we write everything to the user space buffer? */
253	if (!cc && cnt != 0)
254		cc = -EFAULT;
255
256	*status = ioaddr_len.odd >> 24 & 0xff;
257	return cc;
258}
259
260static inline int __memcpy_fromio_inuser(void __user *dst,
261				     const void __iomem *src,
262				     unsigned long n)
263{
264	int size, rc = 0;
265	u8 status;
266
267	while (n > 0) {
268		size = zpci_get_max_io_size((u64 __force) src,
269					    (u64 __force) dst, n,
270					    ZPCI_MAX_READ_SIZE);
271		rc = __pcilg_mio_inuser(dst, src, size, &status);
272		if (rc)
273			break;
274		src += size;
275		dst += size;
276		n -= size;
277	}
278	if (rc)
279		zpci_err_mmio(rc, status, (__force u64) dst);
280	return rc;
281}
282
283SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
284		void __user *, user_buffer, size_t, length)
285{
286	struct follow_pfnmap_args args = { };
287	u8 local_buf[64];
288	void __iomem *io_addr;
289	void *buf;
290	struct vm_area_struct *vma;
291	long ret;
292
293	if (!zpci_is_enabled())
294		return -ENODEV;
295
296	if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
297		return -EINVAL;
298
299	/*
300	 * We only support read access to MIO capable devices if we are on
301	 * a MIO enabled system. Otherwise we would have to check for every
302	 * address if it is a special ZPCI_ADDR and would have to do
303	 * a pfn lookup which we don't need for MIO capable devices.  Currently
304	 * ISM devices are the only devices without MIO support and there is no
305	 * known need for accessing these from userspace.
306	 */
307	if (static_branch_likely(&have_mio)) {
308		ret = __memcpy_fromio_inuser(
309				user_buffer, (const void __iomem *)mmio_addr,
310				length);
311		return ret;
312	}
313
314	if (length > 64) {
315		buf = kmalloc(length, GFP_KERNEL);
316		if (!buf)
317			return -ENOMEM;
318	} else {
319		buf = local_buf;
320	}
321
322	mmap_read_lock(current->mm);
323	ret = -EINVAL;
324	vma = vma_lookup(current->mm, mmio_addr);
325	if (!vma)
326		goto out_unlock_mmap;
327	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
328		goto out_unlock_mmap;
329	ret = -EACCES;
330	if (!(vma->vm_flags & VM_READ))
331		goto out_unlock_mmap;
332
333	args.vma = vma;
334	args.address = mmio_addr;
335	ret = follow_pfnmap_start(&args);
336	if (ret) {
337		fixup_user_fault(current->mm, mmio_addr, 0, NULL);
338		ret = follow_pfnmap_start(&args);
339		if (ret)
340			goto out_unlock_mmap;
341	}
342
343	io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) |
344			(mmio_addr & ~PAGE_MASK));
345
346	if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
347		ret = -EFAULT;
348		goto out_unlock_pt;
349	}
350	ret = zpci_memcpy_fromio(buf, io_addr, length);
351
352out_unlock_pt:
353	follow_pfnmap_end(&args);
354out_unlock_mmap:
355	mmap_read_unlock(current->mm);
356
357	if (!ret && copy_to_user(user_buffer, buf, length))
358		ret = -EFAULT;
359
360	if (buf != local_buf)
361		kfree(buf);
362	return ret;
363}