ivpu_gem.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2020-2023 Intel Corporation
  4 */
  5
  6#include <linux/dma-buf.h>
  7#include <linux/highmem.h>
  8#include <linux/module.h>
  9#include <linux/set_memory.h>
 10#include <linux/xarray.h>
 11
 12#include <drm/drm_cache.h>
 13#include <drm/drm_debugfs.h>
 14#include <drm/drm_file.h>
 15#include <drm/drm_utils.h>
 16
 17#include "ivpu_drv.h"
 18#include "ivpu_fw.h"
 19#include "ivpu_gem.h"
 20#include "ivpu_hw.h"
 21#include "ivpu_mmu.h"
 22#include "ivpu_mmu_context.h"
 23
 24MODULE_IMPORT_NS("DMA_BUF");
 25
 26static const struct drm_gem_object_funcs ivpu_gem_funcs;
 27
 28static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
 29{
 30	ivpu_dbg(vdev, BO,
 31		 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n",
 32		 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr,
 33		 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
 34		 (bool)drm_gem_is_imported(&bo->base.base));
 35}
 36
 37static inline int ivpu_bo_lock(struct ivpu_bo *bo)
 38{
 39	return dma_resv_lock(bo->base.base.resv, NULL);
 40}
 41
 42static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
 43{
 44	dma_resv_unlock(bo->base.base.resv);
 45}
 46
 47static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo)
 48{
 49	struct sg_table *sgt;
 50
 51	drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach);
 52
 53	ivpu_bo_lock(bo);
 54
 55	sgt = bo->base.sgt;
 56	if (!sgt) {
 57		sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL);
 58		if (IS_ERR(sgt))
 59			ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt));
 60		else
 61			bo->base.sgt = sgt;
 62	}
 63
 64	ivpu_bo_unlock(bo);
 65
 66	return sgt;
 67}
 68
 69/*
 70 * ivpu_bo_bind() - pin the backing physical pages and map them to VPU.
 71 *
 72 * This function pins physical memory pages, then maps the physical pages
 73 * to IOMMU address space and finally updates the VPU MMU page tables
 74 * to allow the VPU to translate VPU address to IOMMU address.
 75 */
 76int __must_check ivpu_bo_bind(struct ivpu_bo *bo)
 77{
 78	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
 79	struct sg_table *sgt;
 80	int ret = 0;
 81
 82	ivpu_dbg_bo(vdev, bo, "bind");
 83
 84	if (bo->base.base.import_attach)
 85		sgt = ivpu_bo_map_attachment(vdev, bo);
 86	else
 87		sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
 88	if (IS_ERR(sgt)) {
 89		ret = PTR_ERR(sgt);
 90		ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
 91		return ret;
 92	}
 93
 94	ivpu_bo_lock(bo);
 95
 96	if (!bo->mmu_mapped) {
 97		drm_WARN_ON(&vdev->drm, !bo->ctx);
 98		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
 99					       ivpu_bo_is_snooped(bo), ivpu_bo_is_read_only(bo));
100		if (ret) {
101			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
102			goto unlock;
103		}
104		bo->mmu_mapped = true;
105	}
106
107unlock:
108	ivpu_bo_unlock(bo);
109
110	return ret;
111}
112
113static int
114ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
115		       const struct ivpu_addr_range *range)
116{
117	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
118	int idx, ret;
119
120	if (!drm_dev_enter(&vdev->drm, &idx))
121		return -ENODEV;
122
123	ivpu_bo_lock(bo);
124
125	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
126	if (!ret) {
127		bo->ctx = ctx;
128		bo->ctx_id = ctx->id;
129		bo->vpu_addr = bo->mm_node.start;
130		ivpu_dbg_bo(vdev, bo, "vaddr");
131	}
132
133	ivpu_bo_unlock(bo);
134
135	drm_dev_exit(idx);
136
137	return ret;
138}
139
140static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
141{
142	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
143
144	dma_resv_assert_held(bo->base.base.resv);
145
146	if (bo->mmu_mapped) {
147		drm_WARN_ON(&vdev->drm, !bo->ctx);
148		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
149		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
150		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
151		bo->mmu_mapped = false;
152	}
153
154	if (bo->ctx) {
155		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
156		bo->ctx = NULL;
157	}
158
159	if (bo->base.sgt) {
160		if (bo->base.base.import_attach) {
161			dma_buf_unmap_attachment(bo->base.base.import_attach,
162						 bo->base.sgt, DMA_BIDIRECTIONAL);
163		} else {
164			dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
165			sg_free_table(bo->base.sgt);
166			kfree(bo->base.sgt);
167		}
168		bo->base.sgt = NULL;
169	}
170}
171
172void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
173{
174	struct ivpu_bo *bo;
175
176	if (drm_WARN_ON(&vdev->drm, !ctx))
177		return;
178
179	mutex_lock(&vdev->bo_list_lock);
180	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
181		ivpu_bo_lock(bo);
182		if (bo->ctx == ctx) {
183			ivpu_dbg_bo(vdev, bo, "unbind");
184			ivpu_bo_unbind_locked(bo);
185		}
186		ivpu_bo_unlock(bo);
187	}
188	mutex_unlock(&vdev->bo_list_lock);
189}
190
191struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
192{
193	struct ivpu_bo *bo;
194
195	if (size == 0 || !PAGE_ALIGNED(size))
196		return ERR_PTR(-EINVAL);
197
198	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
199	if (!bo)
200		return ERR_PTR(-ENOMEM);
201
202	bo->base.base.funcs = &ivpu_gem_funcs;
203	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
204
205	INIT_LIST_HEAD(&bo->bo_list_node);
206
207	return &bo->base.base;
208}
209
210struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
211					     struct dma_buf *dma_buf)
212{
213	struct ivpu_device *vdev = to_ivpu_device(dev);
214	struct device *attach_dev = dev->dev;
215	struct dma_buf_attachment *attach;
216	struct drm_gem_object *obj;
217	struct ivpu_bo *bo;
218	int ret;
219
220	attach = dma_buf_attach(dma_buf, attach_dev);
221	if (IS_ERR(attach))
222		return ERR_CAST(attach);
223
224	get_dma_buf(dma_buf);
225
226	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL);
227	if (IS_ERR(obj)) {
228		ret = PTR_ERR(obj);
229		goto fail_detach;
230	}
231
232	obj->import_attach = attach;
233	obj->resv = dma_buf->resv;
234
235	bo = to_ivpu_bo(obj);
236
237	mutex_lock(&vdev->bo_list_lock);
238	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
239	mutex_unlock(&vdev->bo_list_lock);
240
241	ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo));
242
243	return obj;
244
245fail_detach:
246	dma_buf_detach(dma_buf, attach);
247	dma_buf_put(dma_buf);
248
249	return ERR_PTR(ret);
250}
251
252static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
253{
254	struct drm_gem_shmem_object *shmem;
255	struct ivpu_bo *bo;
256
257	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
258	case DRM_IVPU_BO_CACHED:
259	case DRM_IVPU_BO_WC:
260		break;
261	default:
262		return ERR_PTR(-EINVAL);
263	}
264
265	shmem = drm_gem_shmem_create(&vdev->drm, size);
266	if (IS_ERR(shmem))
267		return ERR_CAST(shmem);
268
269	bo = to_ivpu_bo(&shmem->base);
270	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
271	bo->flags = flags;
272
273	mutex_lock(&vdev->bo_list_lock);
274	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
275	mutex_unlock(&vdev->bo_list_lock);
276
277	ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size);
278
279	return bo;
280}
281
282static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
283{
284	struct ivpu_file_priv *file_priv = file->driver_priv;
285	struct ivpu_device *vdev = file_priv->vdev;
286	struct ivpu_bo *bo = to_ivpu_bo(obj);
287	struct ivpu_addr_range *range;
288
289	if (bo->ctx) {
290		ivpu_dbg(vdev, IOCTL, "Can't add BO %pe to ctx %u: already in ctx %u\n",
291			 bo, file_priv->ctx.id, bo->ctx->id);
292		return -EALREADY;
293	}
294
295	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
296		range = &vdev->hw->ranges.shave;
297	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
298		range = &vdev->hw->ranges.dma;
299	else
300		range = &vdev->hw->ranges.user;
301
302	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
303}
304
305static void ivpu_gem_bo_free(struct drm_gem_object *obj)
306{
307	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
308	struct ivpu_bo *bo = to_ivpu_bo(obj);
309
310	ivpu_dbg_bo(vdev, bo, "free");
311
312	drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node));
313
314	mutex_lock(&vdev->bo_list_lock);
315	list_del(&bo->bo_list_node);
316
317	drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
318		    !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
319	drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
320	drm_WARN_ON(&vdev->drm, bo->base.vaddr);
321
322	ivpu_bo_lock(bo);
323	ivpu_bo_unbind_locked(bo);
324	ivpu_bo_unlock(bo);
325
326	mutex_unlock(&vdev->bo_list_lock);
327
328	drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
329	drm_WARN_ON(&vdev->drm, bo->ctx);
330
331	drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
332	drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node);
333	drm_gem_shmem_free(&bo->base);
334}
335
336static enum drm_gem_object_status ivpu_gem_status(struct drm_gem_object *obj)
337{
338	struct ivpu_bo *bo = to_ivpu_bo(obj);
339	enum drm_gem_object_status status = 0;
340
341	if (ivpu_bo_is_resident(bo))
342		status |= DRM_GEM_OBJECT_RESIDENT;
343
344	return status;
345}
346
347static const struct drm_gem_object_funcs ivpu_gem_funcs = {
348	.free = ivpu_gem_bo_free,
349	.open = ivpu_gem_bo_open,
350	.print_info = drm_gem_shmem_object_print_info,
351	.pin = drm_gem_shmem_object_pin,
352	.unpin = drm_gem_shmem_object_unpin,
353	.get_sg_table = drm_gem_shmem_object_get_sg_table,
354	.vmap = drm_gem_shmem_object_vmap,
355	.vunmap = drm_gem_shmem_object_vunmap,
356	.mmap = drm_gem_shmem_object_mmap,
357	.status = ivpu_gem_status,
358	.vm_ops = &drm_gem_shmem_vm_ops,
359};
360
361int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
362{
363	struct ivpu_file_priv *file_priv = file->driver_priv;
364	struct ivpu_device *vdev = file_priv->vdev;
365	struct drm_ivpu_bo_create *args = data;
366	u64 size = PAGE_ALIGN(args->size);
367	struct ivpu_bo *bo;
368	int ret;
369
370	if (args->flags & ~DRM_IVPU_BO_FLAGS) {
371		ivpu_dbg(vdev, IOCTL, "Invalid BO flags 0x%x\n", args->flags);
372		return -EINVAL;
373	}
374
375	if (size == 0) {
376		ivpu_dbg(vdev, IOCTL, "Invalid BO size %llu\n", args->size);
377		return -EINVAL;
378	}
379
380	bo = ivpu_bo_alloc(vdev, size, args->flags);
381	if (IS_ERR(bo)) {
382		ivpu_dbg(vdev, IOCTL, "Failed to allocate BO: %pe ctx %u size %llu flags 0x%x\n",
383			 bo, file_priv->ctx.id, args->size, args->flags);
384		return PTR_ERR(bo);
385	}
386
387	drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0);
388
389	ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
390	if (ret) {
391		ivpu_dbg(vdev, IOCTL, "Failed to create handle for BO: %pe ctx %u size %llu flags 0x%x\n",
392			 bo, file_priv->ctx.id, args->size, args->flags);
393	} else {
394		args->vpu_addr = bo->vpu_addr;
395		drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1);
396	}
397
398	drm_gem_object_put(&bo->base.base);
399
400	return ret;
401}
402
403struct ivpu_bo *
404ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
405	       struct ivpu_addr_range *range, u64 size, u32 flags)
406{
407	struct iosys_map map;
408	struct ivpu_bo *bo;
409	int ret;
410
411	if (drm_WARN_ON(&vdev->drm, !range))
412		return NULL;
413
414	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
415	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
416	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
417
418	bo = ivpu_bo_alloc(vdev, size, flags);
419	if (IS_ERR(bo)) {
420		ivpu_err(vdev, "Failed to allocate BO: %pe vpu_addr 0x%llx size %llu flags 0x%x\n",
421			 bo, range->start, size, flags);
422		return NULL;
423	}
424
425	ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
426	if (ret) {
427		ivpu_err(vdev, "Failed to allocate NPU address for BO: %pe ctx %u size %llu: %d\n",
428			 bo, ctx->id, size, ret);
429		goto err_put;
430	}
431
432	ret = ivpu_bo_bind(bo);
433	if (ret)
434		goto err_put;
435
436	if (flags & DRM_IVPU_BO_MAPPABLE) {
437		ivpu_bo_lock(bo);
438		ret = drm_gem_shmem_vmap_locked(&bo->base, &map);
439		ivpu_bo_unlock(bo);
440
441		if (ret)
442			goto err_put;
443	}
444
445	return bo;
446
447err_put:
448	drm_gem_object_put(&bo->base.base);
449	return NULL;
450}
451
452struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags)
453{
454	struct ivpu_addr_range range;
455
456	if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) {
457		ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size);
458		return NULL;
459	}
460
461	if (ivpu_hw_range_init(vdev, &range, addr, size))
462		return NULL;
463
464	return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags);
465}
466
467struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
468{
469	return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
470}
471
472void ivpu_bo_free(struct ivpu_bo *bo)
473{
474	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
475
476	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
477		ivpu_bo_lock(bo);
478		drm_gem_shmem_vunmap_locked(&bo->base, &map);
479		ivpu_bo_unlock(bo);
480	}
481
482	drm_gem_object_put(&bo->base.base);
483}
484
485int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
486{
487	struct drm_ivpu_bo_info *args = data;
488	struct drm_gem_object *obj;
489	struct ivpu_bo *bo;
490	int ret = 0;
491
492	obj = drm_gem_object_lookup(file, args->handle);
493	if (!obj)
494		return -ENOENT;
495
496	bo = to_ivpu_bo(obj);
497
498	ivpu_bo_lock(bo);
499	args->flags = bo->flags;
500	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
501	args->vpu_addr = bo->vpu_addr;
502	args->size = obj->size;
503	ivpu_bo_unlock(bo);
504
505	drm_gem_object_put(obj);
506	return ret;
507}
508
509int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
510{
511	struct drm_ivpu_bo_wait *args = data;
512	struct drm_gem_object *obj;
513	unsigned long timeout;
514	long ret;
515
516	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
517
518	/* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
519	timeout += 1;
520
521	obj = drm_gem_object_lookup(file, args->handle);
522	if (!obj)
523		return -EINVAL;
524
525	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
526	if (ret == 0) {
527		ret = -ETIMEDOUT;
528	} else if (ret > 0) {
529		ret = 0;
530		args->job_status = to_ivpu_bo(obj)->job_status;
531	}
532
533	drm_gem_object_put(obj);
534
535	return ret;
536}
537
538static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
539{
540	ivpu_bo_lock(bo);
541
542	drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
543		   bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
544		   bo->flags, kref_read(&bo->base.base.refcount));
545
546	if (bo->base.pages)
547		drm_printf(p, " has_pages");
548
549	if (bo->mmu_mapped)
550		drm_printf(p, " mmu_mapped");
551
552	if (drm_gem_is_imported(&bo->base.base))
553		drm_printf(p, " imported");
554
555	drm_printf(p, "\n");
556
557	ivpu_bo_unlock(bo);
558}
559
560void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
561{
562	struct ivpu_device *vdev = to_ivpu_device(dev);
563	struct ivpu_bo *bo;
564
565	drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
566		   "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
567
568	mutex_lock(&vdev->bo_list_lock);
569	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
570		ivpu_bo_print_info(bo, p);
571	mutex_unlock(&vdev->bo_list_lock);
572}
573
574void ivpu_bo_list_print(struct drm_device *dev)
575{
576	struct drm_printer p = drm_info_printer(dev->dev);
577
578	ivpu_bo_list(dev, &p);
579}