rockchip_drm_gem.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) Rockchip Electronics Co., Ltd.
  4 * Author:Mark Yao <mark.yao@rock-chips.com>
  5 */
  6
  7#include <linux/dma-buf.h>
  8#include <linux/iommu.h>
  9#include <linux/vmalloc.h>
 10
 11#include <drm/drm.h>
 12#include <drm/drm_dumb_buffers.h>
 13#include <drm/drm_fb_helper.h>
 14#include <drm/drm_gem.h>
 15#include <drm/drm_gem_dma_helper.h>
 16#include <drm/drm_prime.h>
 17#include <drm/drm_print.h>
 18#include <drm/drm_vma_manager.h>
 19
 20#include "rockchip_drm_drv.h"
 21#include "rockchip_drm_gem.h"
 22
 23static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj)
 24{
 25	struct drm_device *drm = rk_obj->base.dev;
 26	struct rockchip_drm_private *private = drm->dev_private;
 27	int prot = IOMMU_READ | IOMMU_WRITE;
 28	ssize_t ret;
 29
 30	mutex_lock(&private->mm_lock);
 31	ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm,
 32					 rk_obj->base.size, PAGE_SIZE,
 33					 0, 0);
 34	mutex_unlock(&private->mm_lock);
 35
 36	if (ret < 0) {
 37		DRM_ERROR("out of I/O virtual memory: %zd\n", ret);
 38		return ret;
 39	}
 40
 41	rk_obj->dma_addr = rk_obj->mm.start;
 42
 43	ret = iommu_map_sgtable(private->domain, rk_obj->dma_addr, rk_obj->sgt,
 44				prot);
 45	if (ret < (ssize_t)rk_obj->base.size) {
 46		DRM_ERROR("failed to map buffer: size=%zd request_size=%zd\n",
 47			  ret, rk_obj->base.size);
 48		ret = -ENOMEM;
 49		goto err_remove_node;
 50	}
 51
 52	rk_obj->size = ret;
 53
 54	return 0;
 55
 56err_remove_node:
 57	mutex_lock(&private->mm_lock);
 58	drm_mm_remove_node(&rk_obj->mm);
 59	mutex_unlock(&private->mm_lock);
 60
 61	return ret;
 62}
 63
 64static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj)
 65{
 66	struct drm_device *drm = rk_obj->base.dev;
 67	struct rockchip_drm_private *private = drm->dev_private;
 68
 69	iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size);
 70
 71	mutex_lock(&private->mm_lock);
 72
 73	drm_mm_remove_node(&rk_obj->mm);
 74
 75	mutex_unlock(&private->mm_lock);
 76
 77	return 0;
 78}
 79
 80static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj)
 81{
 82	struct drm_device *drm = rk_obj->base.dev;
 83	int ret, i;
 84	struct scatterlist *s;
 85
 86	rk_obj->pages = drm_gem_get_pages(&rk_obj->base);
 87	if (IS_ERR(rk_obj->pages))
 88		return PTR_ERR(rk_obj->pages);
 89
 90	rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT;
 91
 92	rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->base.dev,
 93					    rk_obj->pages, rk_obj->num_pages);
 94	if (IS_ERR(rk_obj->sgt)) {
 95		ret = PTR_ERR(rk_obj->sgt);
 96		goto err_put_pages;
 97	}
 98
 99	/*
100	 * Fake up the SG table so that dma_sync_sg_for_device() can be used
101	 * to flush the pages associated with it.
102	 *
103	 * TODO: Replace this by drm_clflush_sg() once it can be implemented
104	 * without relying on symbols that are not exported.
105	 */
106	for_each_sgtable_sg(rk_obj->sgt, s, i)
107		sg_dma_address(s) = sg_phys(s);
108
109	dma_sync_sgtable_for_device(drm->dev, rk_obj->sgt, DMA_TO_DEVICE);
110
111	return 0;
112
113err_put_pages:
114	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false);
115	return ret;
116}
117
118static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj)
119{
120	sg_free_table(rk_obj->sgt);
121	kfree(rk_obj->sgt);
122	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true);
123}
124
125static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj,
126				    bool alloc_kmap)
127{
128	int ret;
129
130	ret = rockchip_gem_get_pages(rk_obj);
131	if (ret < 0)
132		return ret;
133
134	ret = rockchip_gem_iommu_map(rk_obj);
135	if (ret < 0)
136		goto err_free;
137
138	if (alloc_kmap) {
139		rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
140				      pgprot_writecombine(PAGE_KERNEL));
141		if (!rk_obj->kvaddr) {
142			DRM_ERROR("failed to vmap() buffer\n");
143			ret = -ENOMEM;
144			goto err_unmap;
145		}
146	}
147
148	return 0;
149
150err_unmap:
151	rockchip_gem_iommu_unmap(rk_obj);
152err_free:
153	rockchip_gem_put_pages(rk_obj);
154
155	return ret;
156}
157
158static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj,
159				  bool alloc_kmap)
160{
161	struct drm_gem_object *obj = &rk_obj->base;
162	struct drm_device *drm = obj->dev;
163
164	rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;
165
166	if (!alloc_kmap)
167		rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;
168
169	rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size,
170					 &rk_obj->dma_addr, GFP_KERNEL,
171					 rk_obj->dma_attrs);
172	if (!rk_obj->kvaddr) {
173		DRM_ERROR("failed to allocate %zu byte dma buffer", obj->size);
174		return -ENOMEM;
175	}
176
177	return 0;
178}
179
180static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,
181				  bool alloc_kmap)
182{
183	struct drm_gem_object *obj = &rk_obj->base;
184	struct drm_device *drm = obj->dev;
185	struct rockchip_drm_private *private = drm->dev_private;
186
187	if (private->domain)
188		return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap);
189	else
190		return rockchip_gem_alloc_dma(rk_obj, alloc_kmap);
191}
192
193static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj)
194{
195	vunmap(rk_obj->kvaddr);
196	rockchip_gem_iommu_unmap(rk_obj);
197	rockchip_gem_put_pages(rk_obj);
198}
199
200static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj)
201{
202	struct drm_gem_object *obj = &rk_obj->base;
203	struct drm_device *drm = obj->dev;
204
205	dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr,
206		       rk_obj->dma_attrs);
207}
208
209static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj)
210{
211	if (rk_obj->pages)
212		rockchip_gem_free_iommu(rk_obj);
213	else
214		rockchip_gem_free_dma(rk_obj);
215}
216
217static int rockchip_drm_gem_object_mmap_iommu(struct drm_gem_object *obj,
218					      struct vm_area_struct *vma)
219{
220	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
221	unsigned int count = obj->size >> PAGE_SHIFT;
222	unsigned long user_count = vma_pages(vma);
223
224	if (user_count == 0)
225		return -ENXIO;
226
227	return vm_map_pages(vma, rk_obj->pages, count);
228}
229
230static int rockchip_drm_gem_object_mmap_dma(struct drm_gem_object *obj,
231					    struct vm_area_struct *vma)
232{
233	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
234	struct drm_device *drm = obj->dev;
235
236	return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr,
237			      obj->size, rk_obj->dma_attrs);
238}
239
240static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj,
241					struct vm_area_struct *vma)
242{
243	int ret;
244	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
245
246	/*
247	 * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the
248	 * whole buffer from the start.
249	 */
250	vma->vm_pgoff = 0;
251
252	/*
253	 * We allocated a struct page table for rk_obj, so clear
254	 * VM_PFNMAP flag that was set by drm_gem_mmap_obj()/drm_gem_mmap().
255	 */
256	vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP);
257
258	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
259	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
260
261	if (rk_obj->pages)
262		ret = rockchip_drm_gem_object_mmap_iommu(obj, vma);
263	else
264		ret = rockchip_drm_gem_object_mmap_dma(obj, vma);
265
266	return ret;
267}
268
269static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj)
270{
271	drm_gem_object_release(&rk_obj->base);
272	kfree(rk_obj);
273}
274
275static const struct drm_gem_object_funcs rockchip_gem_object_funcs = {
276	.free = rockchip_gem_free_object,
277	.get_sg_table = rockchip_gem_prime_get_sg_table,
278	.vmap = rockchip_gem_prime_vmap,
279	.vunmap	= rockchip_gem_prime_vunmap,
280	.mmap = rockchip_drm_gem_object_mmap,
281	.vm_ops = &drm_gem_dma_vm_ops,
282};
283
284static struct rockchip_gem_object *
285	rockchip_gem_alloc_object(struct drm_device *drm, unsigned int size)
286{
287	struct rockchip_gem_object *rk_obj;
288	struct drm_gem_object *obj;
289
290	size = round_up(size, PAGE_SIZE);
291
292	rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL);
293	if (!rk_obj)
294		return ERR_PTR(-ENOMEM);
295
296	obj = &rk_obj->base;
297
298	obj->funcs = &rockchip_gem_object_funcs;
299
300	drm_gem_object_init(drm, obj, size);
301
302	return rk_obj;
303}
304
305struct rockchip_gem_object *
306rockchip_gem_create_object(struct drm_device *drm, unsigned int size,
307			   bool alloc_kmap)
308{
309	struct rockchip_gem_object *rk_obj;
310	int ret;
311
312	rk_obj = rockchip_gem_alloc_object(drm, size);
313	if (IS_ERR(rk_obj))
314		return rk_obj;
315
316	ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap);
317	if (ret)
318		goto err_free_rk_obj;
319
320	return rk_obj;
321
322err_free_rk_obj:
323	rockchip_gem_release_object(rk_obj);
324	return ERR_PTR(ret);
325}
326
327/*
328 * rockchip_gem_free_object - (struct drm_gem_object_funcs)->free
329 * callback function
330 */
331void rockchip_gem_free_object(struct drm_gem_object *obj)
332{
333	struct drm_device *drm = obj->dev;
334	struct rockchip_drm_private *private = drm->dev_private;
335	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
336
337	if (obj->import_attach) {
338		if (private->domain) {
339			rockchip_gem_iommu_unmap(rk_obj);
340		} else {
341			dma_unmap_sgtable(drm->dev, rk_obj->sgt,
342					  DMA_BIDIRECTIONAL, 0);
343		}
344		drm_prime_gem_destroy(obj, rk_obj->sgt);
345	} else {
346		rockchip_gem_free_buf(rk_obj);
347	}
348
349	rockchip_gem_release_object(rk_obj);
350}
351
352/*
353 * rockchip_gem_create_with_handle - allocate an object with the given
354 * size and create a gem handle on it
355 *
356 * returns a struct rockchip_gem_object* on success or ERR_PTR values
357 * on failure.
358 */
359static struct rockchip_gem_object *
360rockchip_gem_create_with_handle(struct drm_file *file_priv,
361				struct drm_device *drm, unsigned int size,
362				unsigned int *handle)
363{
364	struct rockchip_gem_object *rk_obj;
365	struct drm_gem_object *obj;
366	bool is_framebuffer;
367	int ret;
368
369	is_framebuffer = drm->fb_helper && file_priv == drm->fb_helper->client.file;
370
371	rk_obj = rockchip_gem_create_object(drm, size, is_framebuffer);
372	if (IS_ERR(rk_obj))
373		return ERR_CAST(rk_obj);
374
375	obj = &rk_obj->base;
376
377	/*
378	 * allocate a id of idr table where the obj is registered
379	 * and handle has the id what user can see.
380	 */
381	ret = drm_gem_handle_create(file_priv, obj, handle);
382	if (ret)
383		goto err_handle_create;
384
385	/* drop reference from allocate - handle holds it now. */
386	drm_gem_object_put(obj);
387
388	return rk_obj;
389
390err_handle_create:
391	rockchip_gem_free_object(obj);
392
393	return ERR_PTR(ret);
394}
395
396/*
397 * rockchip_gem_dumb_create - (struct drm_driver)->dumb_create callback
398 * function
399 *
400 * This aligns the pitch and size arguments to the minimum required. wrap
401 * this into your own function if you need bigger alignment.
402 */
403int rockchip_gem_dumb_create(struct drm_file *file_priv,
404			     struct drm_device *dev,
405			     struct drm_mode_create_dumb *args)
406{
407	struct rockchip_gem_object *rk_obj;
408	int ret;
409
410	/* 64-byte alignment required by Mali */
411	ret = drm_mode_size_dumb(dev, args, SZ_64, 0);
412	if (ret)
413		return ret;
414
415	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,
416						 &args->handle);
417
418	return PTR_ERR_OR_ZERO(rk_obj);
419}
420
421/*
422 * Allocate a sg_table for this GEM object.
423 * Note: Both the table's contents, and the sg_table itself must be freed by
424 *       the caller.
425 * Returns a pointer to the newly allocated sg_table, or an ERR_PTR() error.
426 */
427struct sg_table *rockchip_gem_prime_get_sg_table(struct drm_gem_object *obj)
428{
429	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
430	struct drm_device *drm = obj->dev;
431	struct sg_table *sgt;
432	int ret;
433
434	if (rk_obj->pages)
435		return drm_prime_pages_to_sg(obj->dev, rk_obj->pages, rk_obj->num_pages);
436
437	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
438	if (!sgt)
439		return ERR_PTR(-ENOMEM);
440
441	ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr,
442				    rk_obj->dma_addr, obj->size,
443				    rk_obj->dma_attrs);
444	if (ret) {
445		DRM_ERROR("failed to allocate sgt, %d\n", ret);
446		kfree(sgt);
447		return ERR_PTR(ret);
448	}
449
450	return sgt;
451}
452
453static int
454rockchip_gem_iommu_map_sg(struct drm_device *drm,
455			  struct dma_buf_attachment *attach,
456			  struct sg_table *sg,
457			  struct rockchip_gem_object *rk_obj)
458{
459	rk_obj->sgt = sg;
460	return rockchip_gem_iommu_map(rk_obj);
461}
462
463static int
464rockchip_gem_dma_map_sg(struct drm_device *drm,
465			struct dma_buf_attachment *attach,
466			struct sg_table *sg,
467			struct rockchip_gem_object *rk_obj)
468{
469	int err = dma_map_sgtable(drm->dev, sg, DMA_BIDIRECTIONAL, 0);
470	if (err)
471		return err;
472
473	if (drm_prime_get_contiguous_size(sg) < attach->dmabuf->size) {
474		DRM_ERROR("failed to map sg_table to contiguous linear address.\n");
475		dma_unmap_sgtable(drm->dev, sg, DMA_BIDIRECTIONAL, 0);
476		return -EINVAL;
477	}
478
479	rk_obj->dma_addr = sg_dma_address(sg->sgl);
480	rk_obj->sgt = sg;
481	return 0;
482}
483
484struct drm_gem_object *
485rockchip_gem_prime_import_sg_table(struct drm_device *drm,
486				   struct dma_buf_attachment *attach,
487				   struct sg_table *sg)
488{
489	struct rockchip_drm_private *private = drm->dev_private;
490	struct rockchip_gem_object *rk_obj;
491	int ret;
492
493	rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size);
494	if (IS_ERR(rk_obj))
495		return ERR_CAST(rk_obj);
496
497	if (private->domain)
498		ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj);
499	else
500		ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj);
501
502	if (ret < 0) {
503		DRM_ERROR("failed to import sg table: %d\n", ret);
504		goto err_free_rk_obj;
505	}
506
507	return &rk_obj->base;
508
509err_free_rk_obj:
510	rockchip_gem_release_object(rk_obj);
511	return ERR_PTR(ret);
512}
513
514int rockchip_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
515{
516	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
517
518	if (rk_obj->pages) {
519		void *vaddr;
520
521		if (rk_obj->kvaddr)
522			vaddr = rk_obj->kvaddr;
523		else
524			vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
525				     pgprot_writecombine(PAGE_KERNEL));
526
527		if (!vaddr)
528			return -ENOMEM;
529		iosys_map_set_vaddr(map, vaddr);
530		return 0;
531	}
532
533	if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)
534		return -ENOMEM;
535	iosys_map_set_vaddr(map, rk_obj->kvaddr);
536
537	return 0;
538}
539
540void rockchip_gem_prime_vunmap(struct drm_gem_object *obj,
541			       struct iosys_map *map)
542{
543	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
544
545	if (rk_obj->pages) {
546		if (map->vaddr != rk_obj->kvaddr)
547			vunmap(map->vaddr);
548		return;
549	}
550
551	/* Nothing to do if allocated by DMA mapping API. */
552}