amdxdna_gem.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2024, Advanced Micro Devices, Inc.
  4 */
  5
  6#include <drm/amdxdna_accel.h>
  7#include <drm/drm_cache.h>
  8#include <drm/drm_device.h>
  9#include <drm/drm_gem.h>
 10#include <drm/drm_gem_shmem_helper.h>
 11#include <drm/drm_print.h>
 12#include <drm/gpu_scheduler.h>
 13#include <linux/dma-buf.h>
 14#include <linux/dma-direct.h>
 15#include <linux/iosys-map.h>
 16#include <linux/pagemap.h>
 17#include <linux/vmalloc.h>
 18
 19#include "amdxdna_ctx.h"
 20#include "amdxdna_gem.h"
 21#include "amdxdna_pci_drv.h"
 22#include "amdxdna_ubuf.h"
 23
 24MODULE_IMPORT_NS("DMA_BUF");
 25
 26static int
 27amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
 28{
 29	struct amdxdna_client *client = abo->client;
 30	struct amdxdna_dev *xdna = client->xdna;
 31	struct amdxdna_mem *mem = &abo->mem;
 32	struct amdxdna_gem_obj *heap;
 33	u64 offset;
 34	u32 align;
 35	int ret;
 36
 37	mutex_lock(&client->mm_lock);
 38
 39	heap = client->dev_heap;
 40	if (!heap) {
 41		ret = -EINVAL;
 42		goto unlock_out;
 43	}
 44
 45	if (heap->mem.userptr == AMDXDNA_INVALID_ADDR) {
 46		XDNA_ERR(xdna, "Invalid dev heap userptr");
 47		ret = -EINVAL;
 48		goto unlock_out;
 49	}
 50
 51	if (mem->size == 0 || mem->size > heap->mem.size) {
 52		XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, limit 0x%lx",
 53			 mem->size, heap->mem.size);
 54		ret = -EINVAL;
 55		goto unlock_out;
 56	}
 57
 58	align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift);
 59	ret = drm_mm_insert_node_generic(&heap->mm, &abo->mm_node,
 60					 mem->size, align,
 61					 0, DRM_MM_INSERT_BEST);
 62	if (ret) {
 63		XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
 64		goto unlock_out;
 65	}
 66
 67	mem->dev_addr = abo->mm_node.start;
 68	offset = mem->dev_addr - heap->mem.dev_addr;
 69	mem->userptr = heap->mem.userptr + offset;
 70	mem->kva = heap->mem.kva + offset;
 71
 72	drm_gem_object_get(to_gobj(heap));
 73
 74unlock_out:
 75	mutex_unlock(&client->mm_lock);
 76
 77	return ret;
 78}
 79
 80static void
 81amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo)
 82{
 83	mutex_destroy(&abo->lock);
 84	kfree(abo);
 85}
 86
 87static void
 88amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo)
 89{
 90	struct amdxdna_gem_obj *heap;
 91
 92	mutex_lock(&abo->client->mm_lock);
 93
 94	drm_mm_remove_node(&abo->mm_node);
 95
 96	heap = abo->client->dev_heap;
 97	drm_gem_object_put(to_gobj(heap));
 98
 99	mutex_unlock(&abo->client->mm_lock);
100}
101
102static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
103				   const struct mmu_notifier_range *range,
104				   unsigned long cur_seq)
105{
106	struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier);
107	struct amdxdna_gem_obj *abo = mapp->abo;
108	struct amdxdna_dev *xdna;
109
110	xdna = to_xdna_dev(to_gobj(abo)->dev);
111	XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d",
112		 mapp->vma->vm_start, mapp->vma->vm_end, abo->type);
113
114	if (!mmu_notifier_range_blockable(range))
115		return false;
116
117	down_write(&xdna->notifier_lock);
118	abo->mem.map_invalid = true;
119	mapp->invalid = true;
120	mmu_interval_set_seq(&mapp->notifier, cur_seq);
121	up_write(&xdna->notifier_lock);
122
123	xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
124
125	if (range->event == MMU_NOTIFY_UNMAP) {
126		down_write(&xdna->notifier_lock);
127		if (!mapp->unmapped) {
128			queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
129			mapp->unmapped = true;
130		}
131		up_write(&xdna->notifier_lock);
132	}
133
134	return true;
135}
136
137static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = {
138	.invalidate = amdxdna_hmm_invalidate,
139};
140
141static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo,
142				   struct vm_area_struct *vma)
143{
144	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
145	struct amdxdna_umap *mapp;
146
147	down_read(&xdna->notifier_lock);
148	list_for_each_entry(mapp, &abo->mem.umap_list, node) {
149		if (!vma || mapp->vma == vma) {
150			if (!mapp->unmapped) {
151				queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
152				mapp->unmapped = true;
153			}
154			if (vma)
155				break;
156		}
157	}
158	up_read(&xdna->notifier_lock);
159}
160
161static void amdxdna_umap_release(struct kref *ref)
162{
163	struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
164	struct vm_area_struct *vma = mapp->vma;
165	struct amdxdna_dev *xdna;
166
167	mmu_interval_notifier_remove(&mapp->notifier);
168	if (is_import_bo(mapp->abo) && vma->vm_file && vma->vm_file->f_mapping)
169		mapping_clear_unevictable(vma->vm_file->f_mapping);
170
171	xdna = to_xdna_dev(to_gobj(mapp->abo)->dev);
172	down_write(&xdna->notifier_lock);
173	list_del(&mapp->node);
174	up_write(&xdna->notifier_lock);
175
176	kvfree(mapp->range.hmm_pfns);
177	kfree(mapp);
178}
179
180void amdxdna_umap_put(struct amdxdna_umap *mapp)
181{
182	kref_put(&mapp->refcnt, amdxdna_umap_release);
183}
184
185static void amdxdna_hmm_unreg_work(struct work_struct *work)
186{
187	struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap,
188						 hmm_unreg_work);
189
190	amdxdna_umap_put(mapp);
191}
192
193static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
194				struct vm_area_struct *vma)
195{
196	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
197	unsigned long len = vma->vm_end - vma->vm_start;
198	unsigned long addr = vma->vm_start;
199	struct amdxdna_umap *mapp;
200	u32 nr_pages;
201	int ret;
202
203	if (!xdna->dev_info->ops->hmm_invalidate)
204		return 0;
205
206	mapp = kzalloc(sizeof(*mapp), GFP_KERNEL);
207	if (!mapp)
208		return -ENOMEM;
209
210	nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT;
211	mapp->range.hmm_pfns = kvcalloc(nr_pages, sizeof(*mapp->range.hmm_pfns),
212					GFP_KERNEL);
213	if (!mapp->range.hmm_pfns) {
214		ret = -ENOMEM;
215		goto free_map;
216	}
217
218	ret = mmu_interval_notifier_insert_locked(&mapp->notifier,
219						  current->mm,
220						  addr,
221						  len,
222						  &amdxdna_hmm_ops);
223	if (ret) {
224		XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret);
225		goto free_pfns;
226	}
227
228	mapp->range.notifier = &mapp->notifier;
229	mapp->range.start = vma->vm_start;
230	mapp->range.end = vma->vm_end;
231	mapp->range.default_flags = HMM_PFN_REQ_FAULT;
232	mapp->vma = vma;
233	mapp->abo = abo;
234	kref_init(&mapp->refcnt);
235
236	if (abo->mem.userptr == AMDXDNA_INVALID_ADDR)
237		abo->mem.userptr = addr;
238	INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work);
239	if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping)
240		mapping_set_unevictable(vma->vm_file->f_mapping);
241
242	down_write(&xdna->notifier_lock);
243	list_add_tail(&mapp->node, &abo->mem.umap_list);
244	up_write(&xdna->notifier_lock);
245
246	return 0;
247
248free_pfns:
249	kvfree(mapp->range.hmm_pfns);
250free_map:
251	kfree(mapp);
252	return ret;
253}
254
255static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj)
256{
257	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
258	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
259
260	XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr);
261	if (abo->pinned)
262		amdxdna_gem_unpin(abo);
263
264	amdxdna_gem_heap_free(abo);
265	drm_gem_object_release(gobj);
266	amdxdna_gem_destroy_obj(abo);
267}
268
269static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo,
270				struct vm_area_struct *vma)
271{
272	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
273	unsigned long num_pages = vma_pages(vma);
274	unsigned long offset = 0;
275	int ret;
276
277	if (!is_import_bo(abo)) {
278		ret = drm_gem_shmem_mmap(&abo->base, vma);
279		if (ret) {
280			XDNA_ERR(xdna, "Failed shmem mmap %d", ret);
281			return ret;
282		}
283
284		/* The buffer is based on memory pages. Fix the flag. */
285		vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
286		ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages,
287				      &num_pages);
288		if (ret) {
289			XDNA_ERR(xdna, "Failed insert pages %d", ret);
290			vma->vm_ops->close(vma);
291			return ret;
292		}
293
294		return 0;
295	}
296
297	vma->vm_private_data = NULL;
298	vma->vm_ops = NULL;
299	ret = dma_buf_mmap(abo->dma_buf, vma, 0);
300	if (ret) {
301		XDNA_ERR(xdna, "Failed to mmap dma buf %d", ret);
302		return ret;
303	}
304
305	do {
306		vm_fault_t fault_ret;
307
308		fault_ret = handle_mm_fault(vma, vma->vm_start + offset,
309					    FAULT_FLAG_WRITE, NULL);
310		if (fault_ret & VM_FAULT_ERROR) {
311			vma->vm_ops->close(vma);
312			XDNA_ERR(xdna, "Fault in page failed");
313			return -EFAULT;
314		}
315
316		offset += PAGE_SIZE;
317	} while (--num_pages);
318
319	/* Drop the reference drm_gem_mmap_obj() acquired.*/
320	drm_gem_object_put(to_gobj(abo));
321
322	return 0;
323}
324
325static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
326				struct vm_area_struct *vma)
327{
328	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
329	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
330	int ret;
331
332	ret = amdxdna_hmm_register(abo, vma);
333	if (ret)
334		return ret;
335
336	ret = amdxdna_insert_pages(abo, vma);
337	if (ret) {
338		XDNA_ERR(xdna, "Failed insert pages, ret %d", ret);
339		goto hmm_unreg;
340	}
341
342	XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx",
343		 drm_vma_node_offset_addr(&gobj->vma_node), abo->type,
344		 vma->vm_start, gobj->size);
345	return 0;
346
347hmm_unreg:
348	amdxdna_hmm_unregister(abo, vma);
349	return ret;
350}
351
352static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
353{
354	struct drm_gem_object *gobj = dma_buf->priv;
355	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
356	unsigned long num_pages = vma_pages(vma);
357	int ret;
358
359	vma->vm_ops = &drm_gem_shmem_vm_ops;
360	vma->vm_private_data = gobj;
361
362	drm_gem_object_get(gobj);
363	ret = drm_gem_shmem_mmap(&abo->base, vma);
364	if (ret)
365		goto put_obj;
366
367	/* The buffer is based on memory pages. Fix the flag. */
368	vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
369	ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages,
370			      &num_pages);
371	if (ret)
372		goto close_vma;
373
374	return 0;
375
376close_vma:
377	vma->vm_ops->close(vma);
378put_obj:
379	drm_gem_object_put(gobj);
380	return ret;
381}
382
383static const struct dma_buf_ops amdxdna_dmabuf_ops = {
384	.attach = drm_gem_map_attach,
385	.detach = drm_gem_map_detach,
386	.map_dma_buf = drm_gem_map_dma_buf,
387	.unmap_dma_buf = drm_gem_unmap_dma_buf,
388	.release = drm_gem_dmabuf_release,
389	.mmap = amdxdna_gem_dmabuf_mmap,
390	.vmap = drm_gem_dmabuf_vmap,
391	.vunmap = drm_gem_dmabuf_vunmap,
392};
393
394static int amdxdna_gem_obj_vmap(struct amdxdna_gem_obj *abo, void **vaddr)
395{
396	struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL);
397	int ret;
398
399	if (is_import_bo(abo))
400		ret = dma_buf_vmap_unlocked(abo->dma_buf, &map);
401	else
402		ret = drm_gem_vmap(to_gobj(abo), &map);
403
404	*vaddr = map.vaddr;
405	return ret;
406}
407
408static void amdxdna_gem_obj_vunmap(struct amdxdna_gem_obj *abo)
409{
410	struct iosys_map map;
411
412	if (!abo->mem.kva)
413		return;
414
415	iosys_map_set_vaddr(&map, abo->mem.kva);
416
417	if (is_import_bo(abo))
418		dma_buf_vunmap_unlocked(abo->dma_buf, &map);
419	else
420		drm_gem_vunmap(to_gobj(abo), &map);
421}
422
423static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags)
424{
425	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
426	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
427
428	if (abo->dma_buf) {
429		get_dma_buf(abo->dma_buf);
430		return abo->dma_buf;
431	}
432
433	exp_info.ops = &amdxdna_dmabuf_ops;
434	exp_info.size = gobj->size;
435	exp_info.flags = flags;
436	exp_info.priv = gobj;
437	exp_info.resv = gobj->resv;
438
439	return drm_gem_dmabuf_export(gobj->dev, &exp_info);
440}
441
442static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo)
443{
444	dma_buf_unmap_attachment_unlocked(abo->attach, abo->base.sgt, DMA_BIDIRECTIONAL);
445	dma_buf_detach(abo->dma_buf, abo->attach);
446	dma_buf_put(abo->dma_buf);
447	drm_gem_object_release(to_gobj(abo));
448	kfree(abo);
449}
450
451static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
452{
453	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
454	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
455
456	XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr);
457
458	amdxdna_hmm_unregister(abo, NULL);
459	flush_workqueue(xdna->notifier_wq);
460
461	if (abo->pinned)
462		amdxdna_gem_unpin(abo);
463
464	if (abo->type == AMDXDNA_BO_DEV_HEAP)
465		drm_mm_takedown(&abo->mm);
466
467	amdxdna_gem_obj_vunmap(abo);
468	mutex_destroy(&abo->lock);
469
470	if (is_import_bo(abo)) {
471		amdxdna_imported_obj_free(abo);
472		return;
473	}
474
475	drm_gem_shmem_free(&abo->base);
476}
477
478static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
479	.free = amdxdna_gem_dev_obj_free,
480};
481
482static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
483	.free = amdxdna_gem_obj_free,
484	.print_info = drm_gem_shmem_object_print_info,
485	.pin = drm_gem_shmem_object_pin,
486	.unpin = drm_gem_shmem_object_unpin,
487	.get_sg_table = drm_gem_shmem_object_get_sg_table,
488	.vmap = drm_gem_shmem_object_vmap,
489	.vunmap = drm_gem_shmem_object_vunmap,
490	.mmap = amdxdna_gem_obj_mmap,
491	.vm_ops = &drm_gem_shmem_vm_ops,
492	.export = amdxdna_gem_prime_export,
493};
494
495static struct amdxdna_gem_obj *
496amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
497{
498	struct amdxdna_gem_obj *abo;
499
500	abo = kzalloc(sizeof(*abo), GFP_KERNEL);
501	if (!abo)
502		return ERR_PTR(-ENOMEM);
503
504	abo->pinned = false;
505	abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
506	mutex_init(&abo->lock);
507
508	abo->mem.userptr = AMDXDNA_INVALID_ADDR;
509	abo->mem.dev_addr = AMDXDNA_INVALID_ADDR;
510	abo->mem.size = size;
511	INIT_LIST_HEAD(&abo->mem.umap_list);
512
513	return abo;
514}
515
516/* For drm_driver->gem_create_object callback */
517struct drm_gem_object *
518amdxdna_gem_create_object_cb(struct drm_device *dev, size_t size)
519{
520	struct amdxdna_gem_obj *abo;
521
522	abo = amdxdna_gem_create_obj(dev, size);
523	if (IS_ERR(abo))
524		return ERR_CAST(abo);
525
526	to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs;
527
528	return to_gobj(abo);
529}
530
531static struct amdxdna_gem_obj *
532amdxdna_gem_create_shmem_object(struct drm_device *dev, size_t size)
533{
534	struct drm_gem_shmem_object *shmem = drm_gem_shmem_create(dev, size);
535
536	if (IS_ERR(shmem))
537		return ERR_CAST(shmem);
538
539	shmem->map_wc = false;
540	return to_xdna_obj(&shmem->base);
541}
542
543static struct amdxdna_gem_obj *
544amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args)
545{
546	struct amdxdna_dev *xdna = to_xdna_dev(dev);
547	enum amdxdna_ubuf_flag flags = 0;
548	struct amdxdna_drm_va_tbl va_tbl;
549	struct drm_gem_object *gobj;
550	struct dma_buf *dma_buf;
551
552	if (copy_from_user(&va_tbl, u64_to_user_ptr(args->vaddr), sizeof(va_tbl))) {
553		XDNA_DBG(xdna, "Access va table failed");
554		return ERR_PTR(-EINVAL);
555	}
556
557	if (va_tbl.num_entries) {
558		if (args->type == AMDXDNA_BO_CMD)
559			flags |= AMDXDNA_UBUF_FLAG_MAP_DMA;
560
561		dma_buf = amdxdna_get_ubuf(dev, flags, va_tbl.num_entries,
562					   u64_to_user_ptr(args->vaddr + sizeof(va_tbl)));
563	} else {
564		dma_buf = dma_buf_get(va_tbl.dmabuf_fd);
565	}
566
567	if (IS_ERR(dma_buf))
568		return ERR_CAST(dma_buf);
569
570	gobj = amdxdna_gem_prime_import(dev, dma_buf);
571	if (IS_ERR(gobj)) {
572		dma_buf_put(dma_buf);
573		return ERR_CAST(gobj);
574	}
575
576	dma_buf_put(dma_buf);
577
578	return to_xdna_obj(gobj);
579}
580
581static struct amdxdna_gem_obj *
582amdxdna_gem_create_object(struct drm_device *dev,
583			  struct amdxdna_drm_create_bo *args)
584{
585	size_t aligned_sz = PAGE_ALIGN(args->size);
586
587	if (args->vaddr)
588		return amdxdna_gem_create_ubuf_object(dev, args);
589
590	return amdxdna_gem_create_shmem_object(dev, aligned_sz);
591}
592
593struct drm_gem_object *
594amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf)
595{
596	struct dma_buf_attachment *attach;
597	struct amdxdna_gem_obj *abo;
598	struct drm_gem_object *gobj;
599	struct sg_table *sgt;
600	int ret;
601
602	get_dma_buf(dma_buf);
603
604	attach = dma_buf_attach(dma_buf, dev->dev);
605	if (IS_ERR(attach)) {
606		ret = PTR_ERR(attach);
607		goto put_buf;
608	}
609
610	sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
611	if (IS_ERR(sgt)) {
612		ret = PTR_ERR(sgt);
613		goto fail_detach;
614	}
615
616	gobj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
617	if (IS_ERR(gobj)) {
618		ret = PTR_ERR(gobj);
619		goto fail_unmap;
620	}
621
622	abo = to_xdna_obj(gobj);
623	abo->attach = attach;
624	abo->dma_buf = dma_buf;
625
626	return gobj;
627
628fail_unmap:
629	dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
630fail_detach:
631	dma_buf_detach(dma_buf, attach);
632put_buf:
633	dma_buf_put(dma_buf);
634
635	return ERR_PTR(ret);
636}
637
638static struct amdxdna_gem_obj *
639amdxdna_drm_alloc_shmem(struct drm_device *dev,
640			struct amdxdna_drm_create_bo *args,
641			struct drm_file *filp)
642{
643	struct amdxdna_client *client = filp->driver_priv;
644	struct amdxdna_gem_obj *abo;
645
646	abo = amdxdna_gem_create_object(dev, args);
647	if (IS_ERR(abo))
648		return ERR_CAST(abo);
649
650	abo->client = client;
651	abo->type = AMDXDNA_BO_SHMEM;
652
653	return abo;
654}
655
656static struct amdxdna_gem_obj *
657amdxdna_drm_create_dev_heap(struct drm_device *dev,
658			    struct amdxdna_drm_create_bo *args,
659			    struct drm_file *filp)
660{
661	struct amdxdna_client *client = filp->driver_priv;
662	struct amdxdna_dev *xdna = to_xdna_dev(dev);
663	struct amdxdna_gem_obj *abo;
664	int ret;
665
666	if (args->size > xdna->dev_info->dev_mem_size) {
667		XDNA_DBG(xdna, "Invalid dev heap size 0x%llx, limit 0x%lx",
668			 args->size, xdna->dev_info->dev_mem_size);
669		return ERR_PTR(-EINVAL);
670	}
671
672	mutex_lock(&client->mm_lock);
673	if (client->dev_heap) {
674		XDNA_DBG(client->xdna, "dev heap is already created");
675		ret = -EBUSY;
676		goto mm_unlock;
677	}
678
679	abo = amdxdna_gem_create_object(dev, args);
680	if (IS_ERR(abo)) {
681		ret = PTR_ERR(abo);
682		goto mm_unlock;
683	}
684
685	abo->type = AMDXDNA_BO_DEV_HEAP;
686	abo->client = client;
687	abo->mem.dev_addr = client->xdna->dev_info->dev_mem_base;
688	drm_mm_init(&abo->mm, abo->mem.dev_addr, abo->mem.size);
689
690	ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva);
691	if (ret) {
692		XDNA_ERR(xdna, "Vmap heap bo failed, ret %d", ret);
693		goto release_obj;
694	}
695
696	client->dev_heap = abo;
697	drm_gem_object_get(to_gobj(abo));
698	mutex_unlock(&client->mm_lock);
699
700	return abo;
701
702release_obj:
703	drm_gem_object_put(to_gobj(abo));
704mm_unlock:
705	mutex_unlock(&client->mm_lock);
706	return ERR_PTR(ret);
707}
708
709struct amdxdna_gem_obj *
710amdxdna_drm_alloc_dev_bo(struct drm_device *dev,
711			 struct amdxdna_drm_create_bo *args,
712			 struct drm_file *filp)
713{
714	struct amdxdna_client *client = filp->driver_priv;
715	struct amdxdna_dev *xdna = to_xdna_dev(dev);
716	size_t aligned_sz = PAGE_ALIGN(args->size);
717	struct amdxdna_gem_obj *abo;
718	int ret;
719
720	abo = amdxdna_gem_create_obj(&xdna->ddev, aligned_sz);
721	if (IS_ERR(abo))
722		return abo;
723
724	to_gobj(abo)->funcs = &amdxdna_gem_dev_obj_funcs;
725	abo->type = AMDXDNA_BO_DEV;
726	abo->client = client;
727
728	ret = amdxdna_gem_heap_alloc(abo);
729	if (ret) {
730		XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
731		amdxdna_gem_destroy_obj(abo);
732		return ERR_PTR(ret);
733	}
734
735	drm_gem_private_object_init(&xdna->ddev, to_gobj(abo), aligned_sz);
736
737	return abo;
738}
739
740static struct amdxdna_gem_obj *
741amdxdna_drm_create_cmd_bo(struct drm_device *dev,
742			  struct amdxdna_drm_create_bo *args,
743			  struct drm_file *filp)
744{
745	struct amdxdna_dev *xdna = to_xdna_dev(dev);
746	struct amdxdna_gem_obj *abo;
747
748	if (args->size < sizeof(struct amdxdna_cmd)) {
749		XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size);
750		return ERR_PTR(-EINVAL);
751	}
752
753	abo = amdxdna_gem_create_object(dev, args);
754	if (IS_ERR(abo))
755		return ERR_CAST(abo);
756
757	abo->type = AMDXDNA_BO_CMD;
758	abo->client = filp->driver_priv;
759
760	return abo;
761}
762
763int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
764{
765	struct amdxdna_dev *xdna = to_xdna_dev(dev);
766	struct amdxdna_drm_create_bo *args = data;
767	struct amdxdna_gem_obj *abo;
768	int ret;
769
770	if (args->flags)
771		return -EINVAL;
772
773	XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx",
774		 args->type, args->vaddr, args->size, args->flags);
775	switch (args->type) {
776	case AMDXDNA_BO_SHMEM:
777		abo = amdxdna_drm_alloc_shmem(dev, args, filp);
778		break;
779	case AMDXDNA_BO_DEV_HEAP:
780		abo = amdxdna_drm_create_dev_heap(dev, args, filp);
781		break;
782	case AMDXDNA_BO_DEV:
783		abo = amdxdna_drm_alloc_dev_bo(dev, args, filp);
784		break;
785	case AMDXDNA_BO_CMD:
786		abo = amdxdna_drm_create_cmd_bo(dev, args, filp);
787		break;
788	default:
789		return -EINVAL;
790	}
791	if (IS_ERR(abo))
792		return PTR_ERR(abo);
793
794	/* ready to publish object to userspace */
795	ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle);
796	if (ret) {
797		XDNA_ERR(xdna, "Create handle failed");
798		goto put_obj;
799	}
800
801	XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx",
802		 args->handle, args->type, abo->mem.userptr,
803		 abo->mem.dev_addr, abo->mem.size);
804put_obj:
805	/* Dereference object reference. Handle holds it now. */
806	drm_gem_object_put(to_gobj(abo));
807	return ret;
808}
809
810int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
811{
812	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
813	int ret;
814
815	if (abo->type == AMDXDNA_BO_DEV)
816		abo = abo->client->dev_heap;
817
818	if (is_import_bo(abo))
819		return 0;
820
821	ret = drm_gem_shmem_pin(&abo->base);
822
823	XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret);
824	return ret;
825}
826
827int amdxdna_gem_pin(struct amdxdna_gem_obj *abo)
828{
829	int ret;
830
831	mutex_lock(&abo->lock);
832	ret = amdxdna_gem_pin_nolock(abo);
833	mutex_unlock(&abo->lock);
834
835	return ret;
836}
837
838void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo)
839{
840	if (abo->type == AMDXDNA_BO_DEV)
841		abo = abo->client->dev_heap;
842
843	if (is_import_bo(abo))
844		return;
845
846	mutex_lock(&abo->lock);
847	drm_gem_shmem_unpin(&abo->base);
848	mutex_unlock(&abo->lock);
849}
850
851struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client,
852					    u32 bo_hdl, u8 bo_type)
853{
854	struct amdxdna_dev *xdna = client->xdna;
855	struct amdxdna_gem_obj *abo;
856	struct drm_gem_object *gobj;
857	int ret;
858
859	gobj = drm_gem_object_lookup(client->filp, bo_hdl);
860	if (!gobj) {
861		XDNA_DBG(xdna, "Can not find bo %d", bo_hdl);
862		return NULL;
863	}
864
865	abo = to_xdna_obj(gobj);
866	if (bo_type != AMDXDNA_BO_INVALID && abo->type != bo_type)
867		goto put_obj;
868
869	if (bo_type != AMDXDNA_BO_CMD || abo->mem.kva)
870		return abo;
871
872	if (abo->mem.size > SZ_32K) {
873		XDNA_ERR(xdna, "Cmd bo is too big %ld", abo->mem.size);
874		goto put_obj;
875	}
876
877	ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva);
878	if (ret) {
879		XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret);
880		goto put_obj;
881	}
882
883	return abo;
884
885put_obj:
886	drm_gem_object_put(gobj);
887	return NULL;
888}
889
890int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
891{
892	struct amdxdna_drm_get_bo_info *args = data;
893	struct amdxdna_dev *xdna = to_xdna_dev(dev);
894	struct amdxdna_gem_obj *abo;
895	struct drm_gem_object *gobj;
896	int ret = 0;
897
898	if (args->ext || args->ext_flags || args->pad)
899		return -EINVAL;
900
901	gobj = drm_gem_object_lookup(filp, args->handle);
902	if (!gobj) {
903		XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle);
904		return -ENOENT;
905	}
906
907	abo = to_xdna_obj(gobj);
908	args->vaddr = abo->mem.userptr;
909	args->xdna_addr = abo->mem.dev_addr;
910
911	if (abo->type != AMDXDNA_BO_DEV)
912		args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node);
913	else
914		args->map_offset = AMDXDNA_INVALID_ADDR;
915
916	XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx",
917		 args->handle, args->map_offset, args->vaddr, args->xdna_addr);
918
919	drm_gem_object_put(gobj);
920	return ret;
921}
922
923/*
924 * The sync bo ioctl is to make sure the CPU cache is in sync with memory.
925 * This is required because NPU is not cache coherent device. CPU cache
926 * flushing/invalidation is expensive so it is best to handle this outside
927 * of the command submission path. This ioctl allows explicit cache
928 * flushing/invalidation outside of the critical path.
929 */
930int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
931			      void *data, struct drm_file *filp)
932{
933	struct amdxdna_dev *xdna = to_xdna_dev(dev);
934	struct amdxdna_drm_sync_bo *args = data;
935	struct amdxdna_gem_obj *abo;
936	struct drm_gem_object *gobj;
937	int ret;
938
939	gobj = drm_gem_object_lookup(filp, args->handle);
940	if (!gobj) {
941		XDNA_ERR(xdna, "Lookup GEM object failed");
942		return -ENOENT;
943	}
944	abo = to_xdna_obj(gobj);
945
946	ret = amdxdna_gem_pin(abo);
947	if (ret) {
948		XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret);
949		goto put_obj;
950	}
951
952	if (is_import_bo(abo))
953		drm_clflush_sg(abo->base.sgt);
954	else if (abo->mem.kva)
955		drm_clflush_virt_range(abo->mem.kva + args->offset, args->size);
956	else if (abo->base.pages)
957		drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT);
958	else
959		drm_WARN(&xdna->ddev, 1, "Can not get flush memory");
960
961	amdxdna_gem_unpin(abo);
962
963	XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n",
964		 args->handle, args->offset, args->size);
965
966	if (args->direction == SYNC_DIRECT_FROM_DEVICE)
967		ret = amdxdna_hwctx_sync_debug_bo(abo->client, args->handle);
968
969put_obj:
970	drm_gem_object_put(gobj);
971	return ret;
972}