lsdc_ttm.c

  1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * Copyright (C) 2023 Loongson Technology Corporation Limited
  4 */
  5
  6#include <drm/drm_drv.h>
  7#include <drm/drm_file.h>
  8#include <drm/drm_gem.h>
  9#include <drm/drm_managed.h>
 10#include <drm/drm_prime.h>
 11#include <drm/drm_print.h>
 12
 13#include "lsdc_drv.h"
 14#include "lsdc_ttm.h"
 15
 16const char *lsdc_mem_type_to_str(uint32_t mem_type)
 17{
 18	switch (mem_type) {
 19	case TTM_PL_VRAM:
 20		return "VRAM";
 21	case TTM_PL_TT:
 22		return "GTT";
 23	case TTM_PL_SYSTEM:
 24		return "SYSTEM";
 25	default:
 26		break;
 27	}
 28
 29	return "Unknown";
 30}
 31
 32const char *lsdc_domain_to_str(u32 domain)
 33{
 34	switch (domain) {
 35	case LSDC_GEM_DOMAIN_VRAM:
 36		return "VRAM";
 37	case LSDC_GEM_DOMAIN_GTT:
 38		return "GTT";
 39	case LSDC_GEM_DOMAIN_SYSTEM:
 40		return "SYSTEM";
 41	default:
 42		break;
 43	}
 44
 45	return "Unknown";
 46}
 47
 48static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
 49{
 50	u32 c = 0;
 51	u32 pflags = 0;
 52	u32 i;
 53
 54	if (lbo->tbo.base.size <= PAGE_SIZE)
 55		pflags |= TTM_PL_FLAG_TOPDOWN;
 56
 57	lbo->placement.placement = lbo->placements;
 58
 59	if (domain & LSDC_GEM_DOMAIN_VRAM) {
 60		lbo->placements[c].mem_type = TTM_PL_VRAM;
 61		lbo->placements[c++].flags = pflags;
 62	}
 63
 64	if (domain & LSDC_GEM_DOMAIN_GTT) {
 65		lbo->placements[c].mem_type = TTM_PL_TT;
 66		lbo->placements[c++].flags = pflags;
 67	}
 68
 69	if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
 70		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
 71		lbo->placements[c++].flags = 0;
 72	}
 73
 74	if (!c) {
 75		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
 76		lbo->placements[c++].flags = 0;
 77	}
 78
 79	lbo->placement.num_placement = c;
 80
 81	for (i = 0; i < c; ++i) {
 82		lbo->placements[i].fpfn = 0;
 83		lbo->placements[i].lpfn = 0;
 84	}
 85}
 86
 87static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
 88{
 89	ttm_tt_fini(tt);
 90	kfree(tt);
 91}
 92
 93static struct ttm_tt *
 94lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
 95{
 96	struct ttm_tt *tt;
 97	int ret;
 98
 99	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
100	if (!tt)
101		return NULL;
102
103	ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
104	if (ret < 0) {
105		kfree(tt);
106		return NULL;
107	}
108
109	return tt;
110}
111
112static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
113				struct ttm_tt *ttm,
114				struct ttm_operation_ctx *ctx)
115{
116	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
117
118	if (slave && ttm->sg) {
119		drm_prime_sg_to_dma_addr_array(ttm->sg,
120					       ttm->dma_address,
121					       ttm->num_pages);
122
123		return 0;
124	}
125
126	return ttm_pool_alloc(&bdev->pool, ttm, ctx);
127}
128
129static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
130				   struct ttm_tt *ttm)
131{
132	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
133
134	if (slave)
135		return;
136
137	return ttm_pool_free(&bdev->pool, ttm);
138}
139
140static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
141				struct ttm_placement *tplacement)
142{
143	struct ttm_resource *resource = tbo->resource;
144	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
145
146	switch (resource->mem_type) {
147	case TTM_PL_VRAM:
148		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
149		break;
150	case TTM_PL_TT:
151	default:
152		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
153		break;
154	}
155
156	*tplacement = lbo->placement;
157}
158
159static int lsdc_bo_move(struct ttm_buffer_object *tbo,
160			bool evict,
161			struct ttm_operation_ctx *ctx,
162			struct ttm_resource *new_mem,
163			struct ttm_place *hop)
164{
165	struct drm_device *ddev = tbo->base.dev;
166	struct ttm_resource *old_mem = tbo->resource;
167	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
168	int ret;
169
170	if (unlikely(tbo->pin_count > 0)) {
171		drm_warn(ddev, "Can't move a pinned BO\n");
172		return -EINVAL;
173	}
174
175	ret = ttm_bo_wait_ctx(tbo, ctx);
176	if (ret)
177		return ret;
178
179	if (!old_mem) {
180		drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
181			lbo, lsdc_mem_type_to_str(new_mem->mem_type),
182			lsdc_bo_size(lbo));
183		ttm_bo_move_null(tbo, new_mem);
184		return 0;
185	}
186
187	if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
188		ttm_bo_move_null(tbo, new_mem);
189		drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
190			lbo, lsdc_bo_size(lbo));
191		return 0;
192	}
193
194	if (old_mem->mem_type == TTM_PL_SYSTEM &&
195	    new_mem->mem_type == TTM_PL_TT) {
196		drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
197			lbo, lsdc_bo_size(lbo));
198		ttm_bo_move_null(tbo, new_mem);
199		return 0;
200	}
201
202	if (old_mem->mem_type == TTM_PL_TT &&
203	    new_mem->mem_type == TTM_PL_SYSTEM) {
204		drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
205			lbo, lsdc_bo_size(lbo));
206		ttm_resource_free(tbo, &tbo->resource);
207		ttm_bo_assign_mem(tbo, new_mem);
208		return 0;
209	}
210
211	drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
212		lbo,
213		lsdc_mem_type_to_str(old_mem->mem_type),
214		lsdc_mem_type_to_str(new_mem->mem_type),
215		lsdc_bo_size(lbo));
216
217	return ttm_bo_move_memcpy(tbo, ctx, new_mem);
218}
219
220static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
221				  struct ttm_resource *mem)
222{
223	struct lsdc_device *ldev = tdev_to_ldev(bdev);
224
225	switch (mem->mem_type) {
226	case TTM_PL_SYSTEM:
227		break;
228	case TTM_PL_TT:
229		break;
230	case TTM_PL_VRAM:
231		mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
232		mem->bus.is_iomem = true;
233		mem->bus.caching = ttm_write_combined;
234		break;
235	default:
236		return -EINVAL;
237	}
238
239	return 0;
240}
241
242static struct ttm_device_funcs lsdc_bo_driver = {
243	.ttm_tt_create = lsdc_ttm_tt_create,
244	.ttm_tt_populate = lsdc_ttm_tt_populate,
245	.ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
246	.ttm_tt_destroy = lsdc_ttm_tt_destroy,
247	.eviction_valuable = ttm_bo_eviction_valuable,
248	.evict_flags = lsdc_bo_evict_flags,
249	.move = lsdc_bo_move,
250	.io_mem_reserve = lsdc_bo_reserve_io_mem,
251};
252
253u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
254{
255	struct ttm_buffer_object *tbo = &lbo->tbo;
256	struct drm_device *ddev = tbo->base.dev;
257	struct ttm_resource *resource = tbo->resource;
258
259	if (unlikely(!tbo->pin_count)) {
260		drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
261		return 0;
262	}
263
264	if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
265		return 0;
266
267	return resource->start << PAGE_SHIFT;
268}
269
270size_t lsdc_bo_size(struct lsdc_bo *lbo)
271{
272	struct ttm_buffer_object *tbo = &lbo->tbo;
273
274	return tbo->base.size;
275}
276
277int lsdc_bo_reserve(struct lsdc_bo *lbo)
278{
279	return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
280}
281
282void lsdc_bo_unreserve(struct lsdc_bo *lbo)
283{
284	return ttm_bo_unreserve(&lbo->tbo);
285}
286
287int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
288{
289	struct ttm_operation_ctx ctx = { false, false };
290	struct ttm_buffer_object *tbo = &lbo->tbo;
291	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
292	int ret;
293
294	if (tbo->pin_count)
295		goto bo_pinned;
296
297	if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
298		return -EINVAL;
299
300	if (domain)
301		lsdc_bo_set_placement(lbo, domain);
302
303	ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
304	if (unlikely(ret)) {
305		drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
306		return ret;
307	}
308
309	if (domain == LSDC_GEM_DOMAIN_VRAM)
310		ldev->vram_pinned_size += lsdc_bo_size(lbo);
311	else if (domain == LSDC_GEM_DOMAIN_GTT)
312		ldev->gtt_pinned_size += lsdc_bo_size(lbo);
313
314bo_pinned:
315	ttm_bo_pin(tbo);
316
317	if (gpu_addr)
318		*gpu_addr = lsdc_bo_gpu_offset(lbo);
319
320	return 0;
321}
322
323void lsdc_bo_unpin(struct lsdc_bo *lbo)
324{
325	struct ttm_buffer_object *tbo = &lbo->tbo;
326	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
327
328	if (unlikely(!tbo->pin_count)) {
329		drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
330		return;
331	}
332
333	ttm_bo_unpin(tbo);
334
335	if (!tbo->pin_count) {
336		if (tbo->resource->mem_type == TTM_PL_VRAM)
337			ldev->vram_pinned_size -= lsdc_bo_size(lbo);
338		else if (tbo->resource->mem_type == TTM_PL_TT)
339			ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
340	}
341}
342
343void lsdc_bo_ref(struct lsdc_bo *lbo)
344{
345	drm_gem_object_get(&lbo->tbo.base);
346}
347
348void lsdc_bo_unref(struct lsdc_bo *lbo)
349{
350	drm_gem_object_put(&lbo->tbo.base);
351}
352
353int lsdc_bo_kmap(struct lsdc_bo *lbo)
354{
355	struct ttm_buffer_object *tbo = &lbo->tbo;
356	struct drm_gem_object *gem = &tbo->base;
357	struct drm_device *ddev = gem->dev;
358	long ret;
359	int err;
360
361	ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
362				    MAX_SCHEDULE_TIMEOUT);
363	if (ret < 0) {
364		drm_warn(ddev, "wait fence timeout\n");
365		return ret;
366	}
367
368	if (lbo->kptr)
369		return 0;
370
371	err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
372	if (err) {
373		drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
374		return err;
375	}
376
377	lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
378
379	return 0;
380}
381
382void lsdc_bo_kunmap(struct lsdc_bo *lbo)
383{
384	if (!lbo->kptr)
385		return;
386
387	lbo->kptr = NULL;
388	ttm_bo_kunmap(&lbo->kmap);
389}
390
391void lsdc_bo_clear(struct lsdc_bo *lbo)
392{
393	lsdc_bo_kmap(lbo);
394
395	if (lbo->is_iomem)
396		memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
397	else
398		memset(lbo->kptr, 0, lbo->size);
399
400	lsdc_bo_kunmap(lbo);
401}
402
403int lsdc_bo_evict_vram(struct drm_device *ddev)
404{
405	struct lsdc_device *ldev = to_lsdc(ddev);
406	struct ttm_device *bdev = &ldev->bdev;
407	struct ttm_resource_manager *man;
408
409	man = ttm_manager_type(bdev, TTM_PL_VRAM);
410	if (unlikely(!man))
411		return 0;
412
413	return ttm_resource_manager_evict_all(bdev, man);
414}
415
416static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
417{
418	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
419	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
420
421	mutex_lock(&ldev->gem.mutex);
422	list_del_init(&lbo->list);
423	mutex_unlock(&ldev->gem.mutex);
424
425	drm_gem_object_release(&tbo->base);
426
427	kfree(lbo);
428}
429
430struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
431			       u32 domain,
432			       size_t size,
433			       bool kernel,
434			       struct sg_table *sg,
435			       struct dma_resv *resv)
436{
437	struct lsdc_device *ldev = to_lsdc(ddev);
438	struct ttm_device *bdev = &ldev->bdev;
439	struct ttm_buffer_object *tbo;
440	struct lsdc_bo *lbo;
441	enum ttm_bo_type bo_type;
442	int ret;
443
444	lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
445	if (!lbo)
446		return ERR_PTR(-ENOMEM);
447
448	INIT_LIST_HEAD(&lbo->list);
449
450	lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
451					LSDC_GEM_DOMAIN_GTT |
452					LSDC_GEM_DOMAIN_SYSTEM);
453
454	tbo = &lbo->tbo;
455
456	size = ALIGN(size, PAGE_SIZE);
457
458	ret = drm_gem_object_init(ddev, &tbo->base, size);
459	if (ret) {
460		kfree(lbo);
461		return ERR_PTR(ret);
462	}
463
464	tbo->bdev = bdev;
465
466	if (kernel)
467		bo_type = ttm_bo_type_kernel;
468	else if (sg)
469		bo_type = ttm_bo_type_sg;
470	else
471		bo_type = ttm_bo_type_device;
472
473	lsdc_bo_set_placement(lbo, domain);
474	lbo->size = size;
475
476	ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
477				   false, sg, resv, lsdc_bo_destroy);
478	if (ret) {
479		kfree(lbo);
480		return ERR_PTR(ret);
481	}
482
483	return lbo;
484}
485
486struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
487					     u32 domain,
488					     size_t size)
489{
490	struct lsdc_bo *lbo;
491	int ret;
492
493	lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
494	if (IS_ERR(lbo))
495		return ERR_CAST(lbo);
496
497	ret = lsdc_bo_reserve(lbo);
498	if (unlikely(ret)) {
499		lsdc_bo_unref(lbo);
500		return ERR_PTR(ret);
501	}
502
503	ret = lsdc_bo_pin(lbo, domain, NULL);
504	lsdc_bo_unreserve(lbo);
505	if (unlikely(ret)) {
506		lsdc_bo_unref(lbo);
507		return ERR_PTR(ret);
508	}
509
510	return lbo;
511}
512
513void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
514{
515	int ret;
516
517	ret = lsdc_bo_reserve(lbo);
518	if (unlikely(ret))
519		return;
520
521	lsdc_bo_unpin(lbo);
522	lsdc_bo_unreserve(lbo);
523
524	lsdc_bo_unref(lbo);
525}
526
527static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
528{
529	struct lsdc_device *ldev = (struct lsdc_device *)data;
530
531	ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
532	ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
533
534	ttm_device_fini(&ldev->bdev);
535
536	drm_dbg(ddev, "ttm finished\n");
537}
538
539int lsdc_ttm_init(struct lsdc_device *ldev)
540{
541	struct drm_device *ddev = &ldev->base;
542	unsigned long num_vram_pages;
543	unsigned long num_gtt_pages;
544	int ret;
545
546	ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
547			      ddev->anon_inode->i_mapping,
548			      ddev->vma_offset_manager,
549			      TTM_ALLOCATION_POOL_USE_DMA32);
550	if (ret)
551		return ret;
552
553	num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
554
555	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
556	if (unlikely(ret))
557		return ret;
558
559	drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
560
561	/* 512M is far enough for us now */
562	ldev->gtt_size = 512 << 20;
563
564	num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
565
566	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
567	if (unlikely(ret))
568		return ret;
569
570	drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
571
572	return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
573}
574
575void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
576{
577	struct ttm_device *bdev = &ldev->bdev;
578	struct drm_device *ddev = &ldev->base;
579	struct drm_minor *minor = ddev->primary;
580	struct dentry *root = minor->debugfs_root;
581	struct ttm_resource_manager *vram_man;
582	struct ttm_resource_manager *gtt_man;
583
584	vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
585	gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
586
587	ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
588	ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
589}