ipu6-dma.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2013--2024 Intel Corporation
  4 */
  5
  6#include <linux/cacheflush.h>
  7#include <linux/dma-mapping.h>
  8#include <linux/iova.h>
  9#include <linux/list.h>
 10#include <linux/mm.h>
 11#include <linux/vmalloc.h>
 12#include <linux/scatterlist.h>
 13#include <linux/slab.h>
 14#include <linux/types.h>
 15
 16#include "ipu6.h"
 17#include "ipu6-bus.h"
 18#include "ipu6-dma.h"
 19#include "ipu6-mmu.h"
 20
 21struct vm_info {
 22	struct list_head list;
 23	struct page **pages;
 24	dma_addr_t ipu6_iova;
 25	void *vaddr;
 26	unsigned long size;
 27};
 28
 29static struct vm_info *get_vm_info(struct ipu6_mmu *mmu, dma_addr_t iova)
 30{
 31	struct vm_info *info, *save;
 32
 33	list_for_each_entry_safe(info, save, &mmu->vma_list, list) {
 34		if (iova >= info->ipu6_iova &&
 35		    iova < (info->ipu6_iova + info->size))
 36			return info;
 37	}
 38
 39	return NULL;
 40}
 41
 42static void __clear_buffer(struct page *page, size_t size, unsigned long attrs)
 43{
 44	void *ptr;
 45
 46	if (!page)
 47		return;
 48	/*
 49	 * Ensure that the allocated pages are zeroed, and that any data
 50	 * lurking in the kernel direct-mapped region is invalidated.
 51	 */
 52	ptr = page_address(page);
 53	memset(ptr, 0, size);
 54	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
 55		clflush_cache_range(ptr, size);
 56}
 57
 58static struct page **__alloc_buffer(size_t size, gfp_t gfp, unsigned long attrs)
 59{
 60	int count = PHYS_PFN(size);
 61	int array_size = count * sizeof(struct page *);
 62	struct page **pages;
 63	int i = 0;
 64
 65	pages = kvzalloc(array_size, GFP_KERNEL);
 66	if (!pages)
 67		return NULL;
 68
 69	gfp |= __GFP_NOWARN;
 70
 71	while (count) {
 72		int j, order = __fls(count);
 73
 74		pages[i] = alloc_pages(gfp, order);
 75		while (!pages[i] && order)
 76			pages[i] = alloc_pages(gfp, --order);
 77		if (!pages[i])
 78			goto error;
 79
 80		if (order) {
 81			split_page(pages[i], order);
 82			j = 1 << order;
 83			while (j--)
 84				pages[i + j] = pages[i] + j;
 85		}
 86
 87		__clear_buffer(pages[i], PAGE_SIZE << order, attrs);
 88		i += 1 << order;
 89		count -= 1 << order;
 90	}
 91
 92	return pages;
 93error:
 94	while (i--)
 95		if (pages[i])
 96			__free_pages(pages[i], 0);
 97	kvfree(pages);
 98	return NULL;
 99}
100
101static void __free_buffer(struct page **pages, size_t size, unsigned long attrs)
102{
103	int count = PHYS_PFN(size);
104	unsigned int i;
105
106	for (i = 0; i < count && pages[i]; i++) {
107		__clear_buffer(pages[i], PAGE_SIZE, attrs);
108		__free_pages(pages[i], 0);
109	}
110
111	kvfree(pages);
112}
113
114void ipu6_dma_sync_single(struct ipu6_bus_device *sys, dma_addr_t dma_handle,
115			  size_t size)
116{
117	void *vaddr;
118	u32 offset;
119	struct vm_info *info;
120	struct ipu6_mmu *mmu = sys->mmu;
121
122	info = get_vm_info(mmu, dma_handle);
123	if (WARN_ON(!info))
124		return;
125
126	offset = dma_handle - info->ipu6_iova;
127	if (WARN_ON(size > (info->size - offset)))
128		return;
129
130	vaddr = info->vaddr + offset;
131	clflush_cache_range(vaddr, size);
132}
133EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_single, "INTEL_IPU6");
134
135void ipu6_dma_sync_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
136		      int nents)
137{
138	struct scatterlist *sg;
139	int i;
140
141	for_each_sg(sglist, sg, nents, i)
142		clflush_cache_range(sg_virt(sg), sg->length);
143}
144EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_sg, "INTEL_IPU6");
145
146void ipu6_dma_sync_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt)
147{
148	ipu6_dma_sync_sg(sys, sgt->sgl, sgt->orig_nents);
149}
150EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_sgtable, "INTEL_IPU6");
151
152void *ipu6_dma_alloc(struct ipu6_bus_device *sys, size_t size,
153		     dma_addr_t *dma_handle, gfp_t gfp,
154		     unsigned long attrs)
155{
156	struct device *dev = &sys->auxdev.dev;
157	struct pci_dev *pdev = sys->isp->pdev;
158	dma_addr_t pci_dma_addr, ipu6_iova;
159	struct ipu6_mmu *mmu = sys->mmu;
160	struct vm_info *info;
161	unsigned long count;
162	struct page **pages;
163	struct iova *iova;
164	unsigned int i;
165	int ret;
166
167	info = kzalloc(sizeof(*info), GFP_KERNEL);
168	if (!info)
169		return NULL;
170
171	size = PAGE_ALIGN(size);
172	count = PHYS_PFN(size);
173
174	iova = alloc_iova(&mmu->dmap->iovad, count,
175			  PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
176	if (!iova)
177		goto out_kfree;
178
179	pages = __alloc_buffer(size, gfp, attrs);
180	if (!pages)
181		goto out_free_iova;
182
183	dev_dbg(dev, "dma_alloc: size %zu iova low pfn %lu, high pfn %lu\n",
184		size, iova->pfn_lo, iova->pfn_hi);
185	for (i = 0; iova->pfn_lo + i <= iova->pfn_hi; i++) {
186		pci_dma_addr = dma_map_page_attrs(&pdev->dev, pages[i], 0,
187						  PAGE_SIZE, DMA_BIDIRECTIONAL,
188						  attrs);
189		dev_dbg(dev, "dma_alloc: mapped pci_dma_addr %pad\n",
190			&pci_dma_addr);
191		if (dma_mapping_error(&pdev->dev, pci_dma_addr)) {
192			dev_err(dev, "pci_dma_mapping for page[%d] failed", i);
193			goto out_unmap;
194		}
195
196		ret = ipu6_mmu_map(mmu->dmap->mmu_info,
197				   PFN_PHYS(iova->pfn_lo + i), pci_dma_addr,
198				   PAGE_SIZE);
199		if (ret) {
200			dev_err(dev, "ipu6_mmu_map for pci_dma[%d] %pad failed",
201				i, &pci_dma_addr);
202			dma_unmap_page_attrs(&pdev->dev, pci_dma_addr,
203					     PAGE_SIZE, DMA_BIDIRECTIONAL,
204					     attrs);
205			goto out_unmap;
206		}
207	}
208
209	info->vaddr = vmap(pages, count, VM_USERMAP, PAGE_KERNEL);
210	if (!info->vaddr)
211		goto out_unmap;
212
213	*dma_handle = PFN_PHYS(iova->pfn_lo);
214
215	info->pages = pages;
216	info->ipu6_iova = *dma_handle;
217	info->size = size;
218	list_add(&info->list, &mmu->vma_list);
219
220	return info->vaddr;
221
222out_unmap:
223	while (i--) {
224		ipu6_iova = PFN_PHYS(iova->pfn_lo + i);
225		pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
226						     ipu6_iova);
227		dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,
228				     DMA_BIDIRECTIONAL, attrs);
229
230		ipu6_mmu_unmap(mmu->dmap->mmu_info, ipu6_iova, PAGE_SIZE);
231	}
232
233	__free_buffer(pages, size, attrs);
234
235out_free_iova:
236	__free_iova(&mmu->dmap->iovad, iova);
237out_kfree:
238	kfree(info);
239
240	return NULL;
241}
242EXPORT_SYMBOL_NS_GPL(ipu6_dma_alloc, "INTEL_IPU6");
243
244void ipu6_dma_free(struct ipu6_bus_device *sys, size_t size, void *vaddr,
245		   dma_addr_t dma_handle, unsigned long attrs)
246{
247	struct ipu6_mmu *mmu = sys->mmu;
248	struct pci_dev *pdev = sys->isp->pdev;
249	struct iova *iova = find_iova(&mmu->dmap->iovad, PHYS_PFN(dma_handle));
250	dma_addr_t pci_dma_addr, ipu6_iova;
251	struct vm_info *info;
252	struct page **pages;
253	unsigned int i;
254
255	if (WARN_ON(!iova))
256		return;
257
258	info = get_vm_info(mmu, dma_handle);
259	if (WARN_ON(!info))
260		return;
261
262	if (WARN_ON(!info->vaddr))
263		return;
264
265	if (WARN_ON(!info->pages))
266		return;
267
268	list_del(&info->list);
269
270	size = PAGE_ALIGN(size);
271
272	pages = info->pages;
273
274	vunmap(vaddr);
275
276	for (i = 0; i < PHYS_PFN(size); i++) {
277		ipu6_iova = PFN_PHYS(iova->pfn_lo + i);
278		pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
279						     ipu6_iova);
280		dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,
281				     DMA_BIDIRECTIONAL, attrs);
282	}
283
284	ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
285		       PFN_PHYS(iova_size(iova)));
286
287	__free_buffer(pages, size, attrs);
288
289	mmu->tlb_invalidate(mmu);
290
291	__free_iova(&mmu->dmap->iovad, iova);
292
293	kfree(info);
294}
295EXPORT_SYMBOL_NS_GPL(ipu6_dma_free, "INTEL_IPU6");
296
297int ipu6_dma_mmap(struct ipu6_bus_device *sys, struct vm_area_struct *vma,
298		  void *addr, dma_addr_t iova, size_t size,
299		  unsigned long attrs)
300{
301	struct ipu6_mmu *mmu = sys->mmu;
302	size_t count = PFN_UP(size);
303	struct vm_info *info;
304	size_t i;
305	int ret;
306
307	info = get_vm_info(mmu, iova);
308	if (!info)
309		return -EFAULT;
310
311	if (!info->vaddr)
312		return -EFAULT;
313
314	if (vma->vm_start & ~PAGE_MASK)
315		return -EINVAL;
316
317	if (size > info->size)
318		return -EFAULT;
319
320	for (i = 0; i < count; i++) {
321		ret = vm_insert_page(vma, vma->vm_start + PFN_PHYS(i),
322				     info->pages[i]);
323		if (ret < 0)
324			return ret;
325	}
326
327	return 0;
328}
329
330void ipu6_dma_unmap_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
331		       int nents, enum dma_data_direction dir,
332		       unsigned long attrs)
333{
334	struct device *dev = &sys->auxdev.dev;
335	struct ipu6_mmu *mmu = sys->mmu;
336	struct iova *iova = find_iova(&mmu->dmap->iovad,
337				      PHYS_PFN(sg_dma_address(sglist)));
338	struct scatterlist *sg;
339	dma_addr_t pci_dma_addr;
340	unsigned int i;
341
342	if (!nents)
343		return;
344
345	if (WARN_ON(!iova))
346		return;
347
348	/*
349	 * Before IPU6 mmu unmap, return the pci dma address back to sg
350	 * assume the nents is less than orig_nents as the least granule
351	 * is 1 SZ_4K page
352	 */
353	dev_dbg(dev, "trying to unmap concatenated %u ents\n", nents);
354	for_each_sg(sglist, sg, nents, i) {
355		dev_dbg(dev, "unmap sg[%d] %pad size %u\n", i,
356			&sg_dma_address(sg), sg_dma_len(sg));
357		pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
358						     sg_dma_address(sg));
359		dev_dbg(dev, "return pci_dma_addr %pad back to sg[%d]\n",
360			&pci_dma_addr, i);
361		sg_dma_address(sg) = pci_dma_addr;
362	}
363
364	dev_dbg(dev, "ipu6_mmu_unmap low pfn %lu high pfn %lu\n",
365		iova->pfn_lo, iova->pfn_hi);
366	ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
367		       PFN_PHYS(iova_size(iova)));
368
369	mmu->tlb_invalidate(mmu);
370	__free_iova(&mmu->dmap->iovad, iova);
371}
372EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sg, "INTEL_IPU6");
373
374int ipu6_dma_map_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
375		    int nents, enum dma_data_direction dir,
376		    unsigned long attrs)
377{
378	struct device *dev = &sys->auxdev.dev;
379	struct ipu6_mmu *mmu = sys->mmu;
380	struct scatterlist *sg;
381	struct iova *iova;
382	size_t npages = 0;
383	unsigned long iova_addr;
384	int i;
385
386	for_each_sg(sglist, sg, nents, i) {
387		if (sg->offset) {
388			dev_err(dev, "Unsupported non-zero sg[%d].offset %x\n",
389				i, sg->offset);
390			return -EFAULT;
391		}
392	}
393
394	for_each_sg(sglist, sg, nents, i)
395		npages += PFN_UP(sg_dma_len(sg));
396
397	dev_dbg(dev, "dmamap trying to map %d ents %zu pages\n",
398		nents, npages);
399
400	iova = alloc_iova(&mmu->dmap->iovad, npages,
401			  PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
402	if (!iova)
403		return 0;
404
405	dev_dbg(dev, "dmamap: iova low pfn %lu, high pfn %lu\n", iova->pfn_lo,
406		iova->pfn_hi);
407
408	iova_addr = iova->pfn_lo;
409	for_each_sg(sglist, sg, nents, i) {
410		phys_addr_t iova_pa;
411		int ret;
412
413		iova_pa = PFN_PHYS(iova_addr);
414		dev_dbg(dev, "mapping entry %d: iova %pap phy %pap size %d\n",
415			i, &iova_pa, &sg_dma_address(sg), sg_dma_len(sg));
416
417		ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),
418				   sg_dma_address(sg),
419				   PAGE_ALIGN(sg_dma_len(sg)));
420		if (ret)
421			goto out_fail;
422
423		sg_dma_address(sg) = PFN_PHYS(iova_addr);
424
425		iova_addr += PFN_UP(sg_dma_len(sg));
426	}
427
428	dev_dbg(dev, "dmamap %d ents %zu pages mapped\n", nents, npages);
429
430	return nents;
431
432out_fail:
433	ipu6_dma_unmap_sg(sys, sglist, i, dir, attrs);
434
435	return 0;
436}
437EXPORT_SYMBOL_NS_GPL(ipu6_dma_map_sg, "INTEL_IPU6");
438
439int ipu6_dma_map_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
440			 enum dma_data_direction dir, unsigned long attrs)
441{
442	int nents;
443
444	nents = ipu6_dma_map_sg(sys, sgt->sgl, sgt->nents, dir, attrs);
445	if (nents < 0)
446		return nents;
447
448	sgt->nents = nents;
449
450	return 0;
451}
452EXPORT_SYMBOL_NS_GPL(ipu6_dma_map_sgtable, "INTEL_IPU6");
453
454void ipu6_dma_unmap_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
455			    enum dma_data_direction dir, unsigned long attrs)
456{
457	ipu6_dma_unmap_sg(sys, sgt->sgl, sgt->nents, dir, attrs);
458}
459EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sgtable, "INTEL_IPU6");