bcm_vk_sg.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright 2018-2020 Broadcom.
  4 */
  5#include <linux/dma-mapping.h>
  6#include <linux/mm.h>
  7#include <linux/pagemap.h>
  8#include <linux/pgtable.h>
  9#include <linux/vmalloc.h>
 10
 11#include <asm/page.h>
 12#include <linux/unaligned.h>
 13
 14#include <uapi/linux/misc/bcm_vk.h>
 15
 16#include "bcm_vk.h"
 17#include "bcm_vk_msg.h"
 18#include "bcm_vk_sg.h"
 19
 20/*
 21 * Valkyrie has a hardware limitation of 16M transfer size.
 22 * So limit the SGL chunks to 16M.
 23 */
 24#define BCM_VK_MAX_SGL_CHUNK SZ_16M
 25
 26static int bcm_vk_dma_alloc(struct device *dev,
 27			    struct bcm_vk_dma *dma,
 28			    int dir,
 29			    struct _vk_data *vkdata);
 30static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
 31
 32/* Uncomment to dump SGLIST */
 33/* #define BCM_VK_DUMP_SGLIST */
 34
 35static int bcm_vk_dma_alloc(struct device *dev,
 36			    struct bcm_vk_dma *dma,
 37			    int direction,
 38			    struct _vk_data *vkdata)
 39{
 40	dma_addr_t addr, sg_addr;
 41	int err;
 42	int i;
 43	int offset;
 44	u32 size;
 45	u32 remaining_size;
 46	u32 transfer_size;
 47	u64 data;
 48	unsigned long first, last;
 49	struct _vk_data *sgdata;
 50
 51	/* Get 64-bit user address */
 52	data = get_unaligned(&vkdata->address);
 53
 54	/* offset into first page */
 55	offset = offset_in_page(data);
 56
 57	/* Calculate number of pages */
 58	first = (data & PAGE_MASK) >> PAGE_SHIFT;
 59	last  = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
 60	dma->nr_pages = last - first + 1;
 61
 62	/* Allocate DMA pages */
 63	dma->pages = kmalloc_array(dma->nr_pages,
 64				   sizeof(struct page *),
 65				   GFP_KERNEL);
 66	if (!dma->pages)
 67		return -ENOMEM;
 68
 69	dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
 70		data, vkdata->size, dma->nr_pages);
 71
 72	dma->direction = direction;
 73
 74	/* Get user pages into memory */
 75	err = get_user_pages_fast(data & PAGE_MASK,
 76				  dma->nr_pages,
 77				  direction == DMA_FROM_DEVICE,
 78				  dma->pages);
 79	if (err != dma->nr_pages) {
 80		dma->nr_pages = (err >= 0) ? err : 0;
 81		dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
 82			err, dma->nr_pages);
 83		return err < 0 ? err : -EINVAL;
 84	}
 85
 86	/* Max size of sg list is 1 per mapped page + fields at start */
 87	dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
 88		     (sizeof(u32) * SGLIST_VKDATA_START);
 89
 90	/* Allocate sglist */
 91	dma->sglist = dma_alloc_coherent(dev,
 92					 dma->sglen,
 93					 &dma->handle,
 94					 GFP_KERNEL);
 95	if (!dma->sglist)
 96		return -ENOMEM;
 97
 98	dma->sglist[SGLIST_NUM_SG] = 0;
 99	dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
100	remaining_size = vkdata->size;
101	sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
102
103	/* Map all pages into DMA */
104	size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
105	remaining_size -= size;
106	sg_addr = dma_map_page(dev,
107			       dma->pages[0],
108			       offset,
109			       size,
110			       dma->direction);
111	transfer_size = size;
112	if (unlikely(dma_mapping_error(dev, sg_addr))) {
113		__free_page(dma->pages[0]);
114		return -EIO;
115	}
116
117	for (i = 1; i < dma->nr_pages; i++) {
118		size = min_t(size_t, PAGE_SIZE, remaining_size);
119		remaining_size -= size;
120		addr = dma_map_page(dev,
121				    dma->pages[i],
122				    0,
123				    size,
124				    dma->direction);
125		if (unlikely(dma_mapping_error(dev, addr))) {
126			__free_page(dma->pages[i]);
127			return -EIO;
128		}
129
130		/*
131		 * Compress SG list entry when pages are contiguous
132		 * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
133		 */
134		if ((addr == (sg_addr + transfer_size)) &&
135		    ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
136			/* pages are contiguous, add to same sg entry */
137			transfer_size += size;
138		} else {
139			/* pages are not contiguous, write sg entry */
140			sgdata->size = transfer_size;
141			put_unaligned(sg_addr, (u64 *)&sgdata->address);
142			dma->sglist[SGLIST_NUM_SG]++;
143
144			/* start new sg entry */
145			sgdata++;
146			sg_addr = addr;
147			transfer_size = size;
148		}
149	}
150	/* Write last sg list entry */
151	sgdata->size = transfer_size;
152	put_unaligned(sg_addr, (u64 *)&sgdata->address);
153	dma->sglist[SGLIST_NUM_SG]++;
154
155	/* Update pointers and size field to point to sglist */
156	put_unaligned((u64)dma->handle, &vkdata->address);
157	vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
158		       (sizeof(u32) * SGLIST_VKDATA_START);
159
160#ifdef BCM_VK_DUMP_SGLIST
161	dev_dbg(dev,
162		"sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
163		(u64)dma->sglist,
164		dma->handle,
165		dma->sglen,
166		vkdata->size);
167	for (i = 0; i < vkdata->size / sizeof(u32); i++)
168		dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
169#endif
170
171	return 0;
172}
173
174int bcm_vk_sg_alloc(struct device *dev,
175		    struct bcm_vk_dma *dma,
176		    int dir,
177		    struct _vk_data *vkdata,
178		    int num)
179{
180	int i;
181	int rc = -EINVAL;
182
183	/* Convert user addresses to DMA SG List */
184	for (i = 0; i < num; i++) {
185		if (vkdata[i].size && vkdata[i].address) {
186			/*
187			 * If both size and address are non-zero
188			 * then DMA alloc.
189			 */
190			rc = bcm_vk_dma_alloc(dev,
191					      &dma[i],
192					      dir,
193					      &vkdata[i]);
194		} else if (vkdata[i].size ||
195			   vkdata[i].address) {
196			/*
197			 * If one of size and address are zero
198			 * there is a problem.
199			 */
200			dev_err(dev,
201				"Invalid vkdata %x 0x%x 0x%llx\n",
202				i, vkdata[i].size, vkdata[i].address);
203			rc = -EINVAL;
204		} else {
205			/*
206			 * If size and address are both zero
207			 * don't convert, but return success.
208			 */
209			rc = 0;
210		}
211
212		if (rc)
213			goto fail_alloc;
214	}
215	return rc;
216
217fail_alloc:
218	while (i > 0) {
219		i--;
220		if (dma[i].sglist)
221			bcm_vk_dma_free(dev, &dma[i]);
222	}
223	return rc;
224}
225
226static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
227{
228	dma_addr_t addr;
229	int i;
230	int num_sg;
231	u32 size;
232	struct _vk_data *vkdata;
233
234	dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
235
236	/* Unmap all pages in the sglist */
237	num_sg = dma->sglist[SGLIST_NUM_SG];
238	vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
239	for (i = 0; i < num_sg; i++) {
240		size = vkdata[i].size;
241		addr = get_unaligned(&vkdata[i].address);
242
243		dma_unmap_page(dev, addr, size, dma->direction);
244	}
245
246	/* Free allocated sglist */
247	dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
248
249	/* Release lock on all pages */
250	for (i = 0; i < dma->nr_pages; i++)
251		put_page(dma->pages[i]);
252
253	/* Free allocated dma pages */
254	kfree(dma->pages);
255	dma->sglist = NULL;
256
257	return 0;
258}
259
260int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
261		   int *proc_cnt)
262{
263	int i;
264
265	*proc_cnt = 0;
266	/* Unmap and free all pages and sglists */
267	for (i = 0; i < num; i++) {
268		if (dma[i].sglist) {
269			bcm_vk_dma_free(dev, &dma[i]);
270			*proc_cnt += 1;
271		}
272	}
273
274	return 0;
275}