xe_ttm_vram_mgr.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021-2022 Intel Corporation
  4 * Copyright (C) 2021-2022 Red Hat
  5 */
  6
  7#include <drm/drm_managed.h>
  8#include <drm/drm_drv.h>
  9
 10#include <drm/ttm/ttm_placement.h>
 11#include <drm/ttm/ttm_range_manager.h>
 12
 13#include "xe_bo.h"
 14#include "xe_device.h"
 15#include "xe_gt.h"
 16#include "xe_res_cursor.h"
 17#include "xe_ttm_vram_mgr.h"
 18#include "xe_vram_types.h"
 19
 20static inline struct drm_buddy_block *
 21xe_ttm_vram_mgr_first_block(struct list_head *list)
 22{
 23	return list_first_entry_or_null(list, struct drm_buddy_block, link);
 24}
 25
 26static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
 27						    struct list_head *head)
 28{
 29	struct drm_buddy_block *block;
 30	u64 start, size;
 31
 32	block = xe_ttm_vram_mgr_first_block(head);
 33	if (!block)
 34		return false;
 35
 36	while (head != block->link.next) {
 37		start = drm_buddy_block_offset(block);
 38		size = drm_buddy_block_size(mm, block);
 39
 40		block = list_entry(block->link.next, struct drm_buddy_block,
 41				   link);
 42		if (start + size != drm_buddy_block_offset(block))
 43			return false;
 44	}
 45
 46	return true;
 47}
 48
 49static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 50			       struct ttm_buffer_object *tbo,
 51			       const struct ttm_place *place,
 52			       struct ttm_resource **res)
 53{
 54	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
 55	struct xe_ttm_vram_mgr_resource *vres;
 56	struct drm_buddy *mm = &mgr->mm;
 57	u64 size, min_page_size;
 58	unsigned long lpfn;
 59	int err;
 60
 61	lpfn = place->lpfn;
 62	if (!lpfn || lpfn > man->size >> PAGE_SHIFT)
 63		lpfn = man->size >> PAGE_SHIFT;
 64
 65	if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn))
 66		return -E2BIG; /* don't trigger eviction for the impossible */
 67
 68	vres = kzalloc(sizeof(*vres), GFP_KERNEL);
 69	if (!vres)
 70		return -ENOMEM;
 71
 72	ttm_resource_init(tbo, place, &vres->base);
 73
 74	/* bail out quickly if there's likely not enough VRAM for this BO */
 75	if (ttm_resource_manager_usage(man) > man->size) {
 76		err = -ENOSPC;
 77		goto error_fini;
 78	}
 79
 80	INIT_LIST_HEAD(&vres->blocks);
 81
 82	if (place->flags & TTM_PL_FLAG_TOPDOWN)
 83		vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
 84
 85	if (place->fpfn || lpfn != man->size >> PAGE_SHIFT)
 86		vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
 87
 88	if (WARN_ON(!vres->base.size)) {
 89		err = -EINVAL;
 90		goto error_fini;
 91	}
 92	size = vres->base.size;
 93
 94	min_page_size = mgr->default_page_size;
 95	if (tbo->page_alignment)
 96		min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT;
 97
 98	if (WARN_ON(min_page_size < mm->chunk_size)) {
 99		err = -EINVAL;
100		goto error_fini;
101	}
102
103	if (WARN_ON(!IS_ALIGNED(size, min_page_size))) {
104		err = -EINVAL;
105		goto error_fini;
106	}
107
108	mutex_lock(&mgr->lock);
109	if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) {
110		err = -ENOSPC;
111		goto error_unlock;
112	}
113
114	if (place->fpfn + (size >> PAGE_SHIFT) != lpfn &&
115	    place->flags & TTM_PL_FLAG_CONTIGUOUS) {
116		size = roundup_pow_of_two(size);
117		min_page_size = size;
118
119		lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn);
120	}
121
122	err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
123				     (u64)lpfn << PAGE_SHIFT, size,
124				     min_page_size, &vres->blocks, vres->flags);
125	if (err)
126		goto error_unlock;
127
128	if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
129		if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks))
130			size = vres->base.size;
131	}
132
133	if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {
134		vres->used_visible_size = size;
135	} else {
136		struct drm_buddy_block *block;
137
138		list_for_each_entry(block, &vres->blocks, link) {
139			u64 start = drm_buddy_block_offset(block);
140
141			if (start < mgr->visible_size) {
142				u64 end = start + drm_buddy_block_size(mm, block);
143
144				vres->used_visible_size +=
145					min(end, mgr->visible_size) - start;
146			}
147		}
148	}
149
150	mgr->visible_avail -= vres->used_visible_size;
151	mutex_unlock(&mgr->lock);
152
153	if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) &&
154	    xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks))
155		vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
156
157	/*
158	 * For some kernel objects we still rely on the start when io mapping
159	 * the object.
160	 */
161	if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) {
162		struct drm_buddy_block *block = list_first_entry(&vres->blocks,
163								 typeof(*block),
164								 link);
165
166		vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT;
167	} else {
168		vres->base.start = XE_BO_INVALID_OFFSET;
169	}
170
171	*res = &vres->base;
172	return 0;
173error_unlock:
174	mutex_unlock(&mgr->lock);
175error_fini:
176	ttm_resource_fini(man, &vres->base);
177	kfree(vres);
178
179	return err;
180}
181
182static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
183				struct ttm_resource *res)
184{
185	struct xe_ttm_vram_mgr_resource *vres =
186		to_xe_ttm_vram_mgr_resource(res);
187	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
188	struct drm_buddy *mm = &mgr->mm;
189
190	mutex_lock(&mgr->lock);
191	drm_buddy_free_list(mm, &vres->blocks, 0);
192	mgr->visible_avail += vres->used_visible_size;
193	mutex_unlock(&mgr->lock);
194
195	ttm_resource_fini(man, res);
196
197	kfree(vres);
198}
199
200static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,
201				  struct drm_printer *printer)
202{
203	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
204	struct drm_buddy *mm = &mgr->mm;
205
206	mutex_lock(&mgr->lock);
207	drm_printf(printer, "default_page_size: %lluKiB\n",
208		   mgr->default_page_size >> 10);
209	drm_printf(printer, "visible_avail: %lluMiB\n",
210		   (u64)mgr->visible_avail >> 20);
211	drm_printf(printer, "visible_size: %lluMiB\n",
212		   (u64)mgr->visible_size >> 20);
213
214	drm_buddy_print(mm, printer);
215	mutex_unlock(&mgr->lock);
216	drm_printf(printer, "man size:%llu\n", man->size);
217}
218
219static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,
220				       struct ttm_resource *res,
221				       const struct ttm_place *place,
222				       size_t size)
223{
224	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
225	struct xe_ttm_vram_mgr_resource *vres =
226		to_xe_ttm_vram_mgr_resource(res);
227	struct drm_buddy *mm = &mgr->mm;
228	struct drm_buddy_block *block;
229
230	if (!place->fpfn && !place->lpfn)
231		return true;
232
233	if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT)
234		return vres->used_visible_size > 0;
235
236	list_for_each_entry(block, &vres->blocks, link) {
237		unsigned long fpfn =
238			drm_buddy_block_offset(block) >> PAGE_SHIFT;
239		unsigned long lpfn = fpfn +
240			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
241
242		if (place->fpfn < lpfn && place->lpfn > fpfn)
243			return true;
244	}
245
246	return false;
247}
248
249static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,
250				       struct ttm_resource *res,
251				       const struct ttm_place *place,
252				       size_t size)
253{
254	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
255	struct xe_ttm_vram_mgr_resource *vres =
256		to_xe_ttm_vram_mgr_resource(res);
257	struct drm_buddy *mm = &mgr->mm;
258	struct drm_buddy_block *block;
259
260	if (!place->fpfn && !place->lpfn)
261		return true;
262
263	if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT)
264		return vres->used_visible_size == size;
265
266	list_for_each_entry(block, &vres->blocks, link) {
267		unsigned long fpfn =
268			drm_buddy_block_offset(block) >> PAGE_SHIFT;
269		unsigned long lpfn = fpfn +
270			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
271
272		if (fpfn < place->fpfn || lpfn > place->lpfn)
273			return false;
274	}
275
276	return true;
277}
278
279static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = {
280	.alloc	= xe_ttm_vram_mgr_new,
281	.free	= xe_ttm_vram_mgr_del,
282	.intersects = xe_ttm_vram_mgr_intersects,
283	.compatible = xe_ttm_vram_mgr_compatible,
284	.debug	= xe_ttm_vram_mgr_debug
285};
286
287static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
288{
289	struct xe_device *xe = to_xe_device(dev);
290	struct xe_ttm_vram_mgr *mgr = arg;
291	struct ttm_resource_manager *man = &mgr->manager;
292
293	ttm_resource_manager_set_used(man, false);
294
295	if (ttm_resource_manager_evict_all(&xe->ttm, man))
296		return;
297
298	WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size);
299
300	drm_buddy_fini(&mgr->mm);
301
302	ttm_resource_manager_cleanup(&mgr->manager);
303
304	ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL);
305
306	mutex_destroy(&mgr->lock);
307}
308
309int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
310			   u32 mem_type, u64 size, u64 io_size,
311			   u64 default_page_size)
312{
313	struct ttm_resource_manager *man = &mgr->manager;
314	int err;
315
316	if (mem_type != XE_PL_STOLEN) {
317		const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1";
318		man->cg = drmm_cgroup_register_region(&xe->drm, name, size);
319		if (IS_ERR(man->cg))
320			return PTR_ERR(man->cg);
321	}
322
323	man->func = &xe_ttm_vram_mgr_func;
324	mgr->mem_type = mem_type;
325	mutex_init(&mgr->lock);
326	mgr->default_page_size = default_page_size;
327	mgr->visible_size = io_size;
328	mgr->visible_avail = io_size;
329
330	ttm_resource_manager_init(man, &xe->ttm, size);
331	err = drm_buddy_init(&mgr->mm, man->size, default_page_size);
332	if (err)
333		return err;
334
335	ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager);
336	ttm_resource_manager_set_used(&mgr->manager, true);
337
338	return drmm_add_action_or_reset(&xe->drm, xe_ttm_vram_mgr_fini, mgr);
339}
340
341/**
342 * xe_ttm_vram_mgr_init - initialize TTM VRAM region
343 * @xe: pointer to Xe device
344 * @vram: pointer to xe_vram_region that contains the memory region attributes
345 *
346 * Initialize the Xe TTM for given @vram region using the given parameters.
347 *
348 * Returns 0 for success, negative error code otherwise.
349 */
350int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram)
351{
352	return __xe_ttm_vram_mgr_init(xe, &vram->ttm, vram->placement,
353				      xe_vram_region_usable_size(vram),
354				      xe_vram_region_io_size(vram),
355				      PAGE_SIZE);
356}
357
358int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
359			      struct ttm_resource *res,
360			      u64 offset, u64 length,
361			      struct device *dev,
362			      enum dma_data_direction dir,
363			      struct sg_table **sgt)
364{
365	struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0];
366	struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res);
367	struct xe_res_cursor cursor;
368	struct scatterlist *sg;
369	int num_entries = 0;
370	int i, r;
371
372	if (vres->used_visible_size < res->size)
373		return -EOPNOTSUPP;
374
375	*sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
376	if (!*sgt)
377		return -ENOMEM;
378
379	/* Determine the number of DRM_BUDDY blocks to export */
380	xe_res_first(res, offset, length, &cursor);
381	while (cursor.remaining) {
382		num_entries++;
383		/* Limit maximum size to 2GiB due to SG table limitations. */
384		xe_res_next(&cursor, min_t(u64, cursor.size, SZ_2G));
385	}
386
387	r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
388	if (r)
389		goto error_free;
390
391	/* Initialize scatterlist nodes of sg_table */
392	for_each_sgtable_sg((*sgt), sg, i)
393		sg->length = 0;
394
395	/*
396	 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
397	 * @note: Use iterator api to get first the DRM_BUDDY block
398	 * and the number of bytes from it. Access the following
399	 * DRM_BUDDY block(s) if more buffer needs to exported
400	 */
401	xe_res_first(res, offset, length, &cursor);
402	for_each_sgtable_sg((*sgt), sg, i) {
403		phys_addr_t phys = cursor.start + xe_vram_region_io_start(tile->mem.vram);
404		size_t size = min_t(u64, cursor.size, SZ_2G);
405		dma_addr_t addr;
406
407		addr = dma_map_resource(dev, phys, size, dir,
408					DMA_ATTR_SKIP_CPU_SYNC);
409		r = dma_mapping_error(dev, addr);
410		if (r)
411			goto error_unmap;
412
413		sg_set_page(sg, NULL, size, 0);
414		sg_dma_address(sg) = addr;
415		sg_dma_len(sg) = size;
416
417		xe_res_next(&cursor, size);
418	}
419
420	return 0;
421
422error_unmap:
423	for_each_sgtable_sg((*sgt), sg, i) {
424		if (!sg->length)
425			continue;
426
427		dma_unmap_resource(dev, sg->dma_address,
428				   sg->length, dir,
429				   DMA_ATTR_SKIP_CPU_SYNC);
430	}
431	sg_free_table(*sgt);
432
433error_free:
434	kfree(*sgt);
435	return r;
436}
437
438void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,
439			      struct sg_table *sgt)
440{
441	struct scatterlist *sg;
442	int i;
443
444	for_each_sgtable_sg(sgt, sg, i)
445		dma_unmap_resource(dev, sg->dma_address,
446				   sg->length, dir,
447				   DMA_ATTR_SKIP_CPU_SYNC);
448	sg_free_table(sgt);
449	kfree(sgt);
450}
451
452u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man)
453{
454	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
455
456	return mgr->visible_size;
457}
458
459void xe_ttm_vram_get_used(struct ttm_resource_manager *man,
460			  u64 *used, u64 *used_visible)
461{
462	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
463
464	mutex_lock(&mgr->lock);
465	*used = mgr->mm.size - mgr->mm.avail;
466	*used_visible = mgr->visible_size - mgr->visible_avail;
467	mutex_unlock(&mgr->lock);
468}
469
470u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man)
471{
472	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
473	u64 avail;
474
475	mutex_lock(&mgr->lock);
476	avail =  mgr->mm.avail;
477	mutex_unlock(&mgr->lock);
478
479	return avail;
480}