ttm_resource.c

  1/*
  2 * Copyright 2020 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 * Authors: Christian König
 23 */
 24
 25#include <linux/debugfs.h>
 26#include <linux/export.h>
 27#include <linux/io-mapping.h>
 28#include <linux/iosys-map.h>
 29#include <linux/scatterlist.h>
 30#include <linux/cgroup_dmem.h>
 31
 32#include <drm/ttm/ttm_bo.h>
 33#include <drm/ttm/ttm_placement.h>
 34#include <drm/ttm/ttm_resource.h>
 35#include <drm/ttm/ttm_tt.h>
 36
 37#include <drm/drm_print.h>
 38#include <drm/drm_util.h>
 39
 40/* Detach the cursor from the bulk move list*/
 41static void
 42ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
 43{
 44	lockdep_assert_held(&cursor->man->bdev->lru_lock);
 45
 46	cursor->bulk = NULL;
 47	list_del_init(&cursor->bulk_link);
 48}
 49
 50/* Move the cursor to the end of the bulk move list it's in */
 51static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk,
 52					       struct ttm_resource_cursor *cursor)
 53{
 54	struct ttm_lru_bulk_move_pos *pos;
 55
 56	lockdep_assert_held(&cursor->man->bdev->lru_lock);
 57
 58	if (WARN_ON_ONCE(bulk != cursor->bulk)) {
 59		list_del_init(&cursor->bulk_link);
 60		return;
 61	}
 62
 63	pos = &bulk->pos[cursor->mem_type][cursor->priority];
 64	if (pos->last)
 65		list_move(&cursor->hitch.link, &pos->last->lru.link);
 66	ttm_resource_cursor_clear_bulk(cursor);
 67}
 68
 69/* Move all cursors attached to a bulk move to its end */
 70static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk)
 71{
 72	struct ttm_resource_cursor *cursor, *next;
 73
 74	list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
 75		ttm_resource_cursor_move_bulk_tail(bulk, cursor);
 76}
 77
 78/* Remove a cursor from an empty bulk move list */
 79static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk)
 80{
 81	struct ttm_resource_cursor *cursor, *next;
 82
 83	list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
 84		ttm_resource_cursor_clear_bulk(cursor);
 85}
 86
 87/**
 88 * ttm_resource_cursor_init() - Initialize a struct ttm_resource_cursor
 89 * @cursor: The cursor to initialize.
 90 * @man: The resource manager.
 91 *
 92 * Initialize the cursor before using it for iteration.
 93 */
 94void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor,
 95			      struct ttm_resource_manager *man)
 96{
 97	cursor->priority = 0;
 98	cursor->man = man;
 99	ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH);
100	INIT_LIST_HEAD(&cursor->bulk_link);
101	INIT_LIST_HEAD(&cursor->hitch.link);
102}
103
104/**
105 * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
106 * @cursor: The struct ttm_resource_cursor to finalize.
107 *
108 * The function pulls the LRU list cursor off any lists it was previusly
109 * attached to. Needs to be called with the LRU lock held. The function
110 * can be called multiple times after eachother.
111 */
112void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
113{
114	lockdep_assert_held(&cursor->man->bdev->lru_lock);
115	list_del_init(&cursor->hitch.link);
116	ttm_resource_cursor_clear_bulk(cursor);
117}
118
119/**
120 * ttm_lru_bulk_move_init - initialize a bulk move structure
121 * @bulk: the structure to init
122 *
123 * For now just memset the structure to zero.
124 */
125void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
126{
127	memset(bulk, 0, sizeof(*bulk));
128	INIT_LIST_HEAD(&bulk->cursor_list);
129}
130EXPORT_SYMBOL(ttm_lru_bulk_move_init);
131
132/**
133 * ttm_lru_bulk_move_fini - finalize a bulk move structure
134 * @bdev: The struct ttm_device
135 * @bulk: the structure to finalize
136 *
137 * Sanity checks that bulk moves don't have any
138 * resources left and hence no cursors attached.
139 */
140void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
141			    struct ttm_lru_bulk_move *bulk)
142{
143	spin_lock(&bdev->lru_lock);
144	ttm_bulk_move_drop_cursors(bulk);
145	spin_unlock(&bdev->lru_lock);
146}
147EXPORT_SYMBOL(ttm_lru_bulk_move_fini);
148
149/**
150 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
151 *
152 * @bulk: bulk move structure
153 *
154 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
155 * resource order never changes. Should be called with &ttm_device.lru_lock held.
156 */
157void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
158{
159	unsigned i, j;
160
161	ttm_bulk_move_adjust_cursors(bulk);
162	for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
163		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
164			struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
165			struct ttm_resource_manager *man;
166
167			if (!pos->first)
168				continue;
169
170			lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
171			dma_resv_assert_held(pos->first->bo->base.resv);
172			dma_resv_assert_held(pos->last->bo->base.resv);
173
174			man = ttm_manager_type(pos->first->bo->bdev, i);
175			list_bulk_move_tail(&man->lru[j], &pos->first->lru.link,
176					    &pos->last->lru.link);
177		}
178	}
179}
180EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
181
182/* Return the bulk move pos object for this resource */
183static struct ttm_lru_bulk_move_pos *
184ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
185{
186	return &bulk->pos[res->mem_type][res->bo->priority];
187}
188
189/* Return the previous resource on the list (skip over non-resource list items) */
190static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)
191{
192	struct ttm_lru_item *lru = &cur->lru;
193
194	do {
195		lru = list_prev_entry(lru, link);
196	} while (!ttm_lru_item_is_res(lru));
197
198	return ttm_lru_item_to_res(lru);
199}
200
201/* Return the next resource on the list (skip over non-resource list items) */
202static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)
203{
204	struct ttm_lru_item *lru = &cur->lru;
205
206	do {
207		lru = list_next_entry(lru, link);
208	} while (!ttm_lru_item_is_res(lru));
209
210	return ttm_lru_item_to_res(lru);
211}
212
213/* Move the resource to the tail of the bulk move range */
214static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
215				       struct ttm_resource *res)
216{
217	if (pos->last != res) {
218		if (pos->first == res)
219			pos->first = ttm_lru_next_res(res);
220		list_move(&res->lru.link, &pos->last->lru.link);
221		pos->last = res;
222	}
223}
224
225/* Add the resource to a bulk_move cursor */
226static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
227				  struct ttm_resource *res)
228{
229	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
230
231	if (!pos->first) {
232		pos->first = res;
233		pos->last = res;
234	} else {
235		WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);
236		ttm_lru_bulk_move_pos_tail(pos, res);
237	}
238}
239
240/* Remove the resource from a bulk_move range */
241static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
242				  struct ttm_resource *res)
243{
244	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
245
246	if (unlikely(WARN_ON(!pos->first || !pos->last) ||
247		     (pos->first == res && pos->last == res))) {
248		pos->first = NULL;
249		pos->last = NULL;
250	} else if (pos->first == res) {
251		pos->first = ttm_lru_next_res(res);
252	} else if (pos->last == res) {
253		pos->last = ttm_lru_prev_res(res);
254	} else {
255		list_move(&res->lru.link, &pos->last->lru.link);
256	}
257}
258
259static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_object *bo)
260{
261	/*
262	 * Take care when creating a new resource for a bo, that it is not considered
263	 * swapped if it's not the current resource for the bo and is thus logically
264	 * associated with the ttm_tt. Think a VRAM resource created to move a
265	 * swapped-out bo to VRAM.
266	 */
267	if (bo->resource != res || !bo->ttm)
268		return false;
269
270	dma_resv_assert_held(bo->base.resv);
271	return ttm_tt_is_swapped(bo->ttm);
272}
273
274static bool ttm_resource_unevictable(struct ttm_resource *res, struct ttm_buffer_object *bo)
275{
276	return bo->pin_count || ttm_resource_is_swapped(res, bo);
277}
278
279/* Add the resource to a bulk move if the BO is configured for it */
280void ttm_resource_add_bulk_move(struct ttm_resource *res,
281				struct ttm_buffer_object *bo)
282{
283	if (bo->bulk_move && !ttm_resource_unevictable(res, bo))
284		ttm_lru_bulk_move_add(bo->bulk_move, res);
285}
286
287/* Remove the resource from a bulk move if the BO is configured for it */
288void ttm_resource_del_bulk_move(struct ttm_resource *res,
289				struct ttm_buffer_object *bo)
290{
291	if (bo->bulk_move && !ttm_resource_unevictable(res, bo))
292		ttm_lru_bulk_move_del(bo->bulk_move, res);
293}
294
295/* Move a resource to the LRU or bulk tail */
296void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
297{
298	struct ttm_buffer_object *bo = res->bo;
299	struct ttm_device *bdev = bo->bdev;
300
301	lockdep_assert_held(&bo->bdev->lru_lock);
302
303	if (ttm_resource_unevictable(res, bo)) {
304		list_move_tail(&res->lru.link, &bdev->unevictable);
305
306	} else if (bo->bulk_move) {
307		struct ttm_lru_bulk_move_pos *pos =
308			ttm_lru_bulk_move_pos(bo->bulk_move, res);
309
310		ttm_lru_bulk_move_pos_tail(pos, res);
311	} else {
312		struct ttm_resource_manager *man;
313
314		man = ttm_manager_type(bdev, res->mem_type);
315		list_move_tail(&res->lru.link, &man->lru[bo->priority]);
316	}
317}
318
319/**
320 * ttm_resource_init - resource object constructure
321 * @bo: buffer object this resources is allocated for
322 * @place: placement of the resource
323 * @res: the resource object to inistilize
324 *
325 * Initialize a new resource object. Counterpart of ttm_resource_fini().
326 */
327void ttm_resource_init(struct ttm_buffer_object *bo,
328                       const struct ttm_place *place,
329                       struct ttm_resource *res)
330{
331	struct ttm_resource_manager *man;
332
333	res->start = 0;
334	res->size = bo->base.size;
335	res->mem_type = place->mem_type;
336	res->placement = place->flags;
337	res->bus.addr = NULL;
338	res->bus.offset = 0;
339	res->bus.is_iomem = false;
340	res->bus.caching = ttm_cached;
341	res->bo = bo;
342
343	man = ttm_manager_type(bo->bdev, place->mem_type);
344	spin_lock(&bo->bdev->lru_lock);
345	if (ttm_resource_unevictable(res, bo))
346		list_add_tail(&res->lru.link, &bo->bdev->unevictable);
347	else
348		list_add_tail(&res->lru.link, &man->lru[bo->priority]);
349	man->usage += res->size;
350	spin_unlock(&bo->bdev->lru_lock);
351}
352EXPORT_SYMBOL(ttm_resource_init);
353
354/**
355 * ttm_resource_fini - resource destructor
356 * @man: the resource manager this resource belongs to
357 * @res: the resource to clean up
358 *
359 * Should be used by resource manager backends to clean up the TTM resource
360 * objects before freeing the underlying structure. Makes sure the resource is
361 * removed from the LRU before destruction.
362 * Counterpart of ttm_resource_init().
363 */
364void ttm_resource_fini(struct ttm_resource_manager *man,
365		       struct ttm_resource *res)
366{
367	struct ttm_device *bdev = man->bdev;
368
369	spin_lock(&bdev->lru_lock);
370	list_del_init(&res->lru.link);
371	man->usage -= res->size;
372	spin_unlock(&bdev->lru_lock);
373}
374EXPORT_SYMBOL(ttm_resource_fini);
375
376int ttm_resource_alloc(struct ttm_buffer_object *bo,
377		       const struct ttm_place *place,
378		       struct ttm_resource **res_ptr,
379		       struct dmem_cgroup_pool_state **ret_limit_pool)
380{
381	struct ttm_resource_manager *man =
382		ttm_manager_type(bo->bdev, place->mem_type);
383	struct dmem_cgroup_pool_state *pool = NULL;
384	int ret;
385
386	if (man->cg) {
387		ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool);
388		if (ret)
389			return ret;
390	}
391
392	ret = man->func->alloc(man, bo, place, res_ptr);
393	if (ret) {
394		if (pool)
395			dmem_cgroup_uncharge(pool, bo->base.size);
396		return ret;
397	}
398
399	(*res_ptr)->css = pool;
400
401	spin_lock(&bo->bdev->lru_lock);
402	ttm_resource_add_bulk_move(*res_ptr, bo);
403	spin_unlock(&bo->bdev->lru_lock);
404	return 0;
405}
406EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);
407
408void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
409{
410	struct ttm_resource_manager *man;
411	struct dmem_cgroup_pool_state *pool;
412
413	if (!*res)
414		return;
415
416	spin_lock(&bo->bdev->lru_lock);
417	ttm_resource_del_bulk_move(*res, bo);
418	spin_unlock(&bo->bdev->lru_lock);
419
420	pool = (*res)->css;
421	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
422	man->func->free(man, *res);
423	*res = NULL;
424	if (man->cg)
425		dmem_cgroup_uncharge(pool, bo->base.size);
426}
427EXPORT_SYMBOL(ttm_resource_free);
428
429/**
430 * ttm_resource_intersects - test for intersection
431 *
432 * @bdev: TTM device structure
433 * @res: The resource to test
434 * @place: The placement to test
435 * @size: How many bytes the new allocation needs.
436 *
437 * Test if @res intersects with @place and @size. Used for testing if evictions
438 * are valueable or not.
439 *
440 * Returns true if the res placement intersects with @place and @size.
441 */
442bool ttm_resource_intersects(struct ttm_device *bdev,
443			     struct ttm_resource *res,
444			     const struct ttm_place *place,
445			     size_t size)
446{
447	struct ttm_resource_manager *man;
448
449	if (!res)
450		return false;
451
452	man = ttm_manager_type(bdev, res->mem_type);
453	if (!place || !man->func->intersects)
454		return true;
455
456	return man->func->intersects(man, res, place, size);
457}
458
459/**
460 * ttm_resource_compatible - check if resource is compatible with placement
461 *
462 * @res: the resource to check
463 * @placement: the placement to check against
464 * @evicting: true if the caller is doing evictions
465 *
466 * Returns true if the placement is compatible.
467 */
468bool ttm_resource_compatible(struct ttm_resource *res,
469			     struct ttm_placement *placement,
470			     bool evicting)
471{
472	struct ttm_buffer_object *bo = res->bo;
473	struct ttm_device *bdev = bo->bdev;
474	unsigned i;
475
476	if (res->placement & TTM_PL_FLAG_TEMPORARY)
477		return false;
478
479	for (i = 0; i < placement->num_placement; i++) {
480		const struct ttm_place *place = &placement->placement[i];
481		struct ttm_resource_manager *man;
482
483		if (res->mem_type != place->mem_type)
484			continue;
485
486		if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :
487				    TTM_PL_FLAG_FALLBACK))
488			continue;
489
490		if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&
491		    !(res->placement & TTM_PL_FLAG_CONTIGUOUS))
492			continue;
493
494		man = ttm_manager_type(bdev, res->mem_type);
495		if (man->func->compatible &&
496		    !man->func->compatible(man, res, place, bo->base.size))
497			continue;
498
499		return true;
500	}
501	return false;
502}
503
504void ttm_resource_set_bo(struct ttm_resource *res,
505			 struct ttm_buffer_object *bo)
506{
507	spin_lock(&bo->bdev->lru_lock);
508	res->bo = bo;
509	spin_unlock(&bo->bdev->lru_lock);
510}
511
512/**
513 * ttm_resource_manager_init
514 *
515 * @man: memory manager object to init
516 * @bdev: ttm device this manager belongs to
517 * @size: size of managed resources in arbitrary units
518 *
519 * Initialise core parts of a manager object.
520 */
521void ttm_resource_manager_init(struct ttm_resource_manager *man,
522			       struct ttm_device *bdev,
523			       uint64_t size)
524{
525	unsigned i;
526
527	man->bdev = bdev;
528	man->size = size;
529	man->usage = 0;
530
531	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
532		INIT_LIST_HEAD(&man->lru[i]);
533	spin_lock_init(&man->eviction_lock);
534	for (i = 0; i < TTM_NUM_MOVE_FENCES; i++)
535		man->eviction_fences[i] = NULL;
536}
537EXPORT_SYMBOL(ttm_resource_manager_init);
538
539/*
540 * ttm_resource_manager_evict_all
541 *
542 * @bdev - device to use
543 * @man - manager to use
544 *
545 * Evict all the objects out of a memory manager until it is empty.
546 * Part of memory manager cleanup sequence.
547 */
548int ttm_resource_manager_evict_all(struct ttm_device *bdev,
549				   struct ttm_resource_manager *man)
550{
551	struct ttm_operation_ctx ctx = {
552		.interruptible = false,
553		.no_wait_gpu = false,
554	};
555	struct dma_fence *fence;
556	int ret, i;
557
558	do {
559		ret = ttm_bo_evict_first(bdev, man, &ctx);
560		cond_resched();
561	} while (!ret);
562
563	if (ret && ret != -ENOENT)
564		return ret;
565
566	ret = 0;
567
568	spin_lock(&man->eviction_lock);
569	for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) {
570		fence = man->eviction_fences[i];
571		if (fence && !dma_fence_is_signaled(fence)) {
572			dma_fence_get(fence);
573			spin_unlock(&man->eviction_lock);
574			ret = dma_fence_wait(fence, false);
575			dma_fence_put(fence);
576			if (ret)
577				return ret;
578			spin_lock(&man->eviction_lock);
579		}
580	}
581	spin_unlock(&man->eviction_lock);
582
583	return ret;
584}
585EXPORT_SYMBOL(ttm_resource_manager_evict_all);
586
587/**
588 * ttm_resource_manager_usage
589 *
590 * @man: A memory manager object.
591 *
592 * Return how many resources are currently used.
593 */
594uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
595{
596	uint64_t usage;
597
598	if (WARN_ON_ONCE(!man->bdev))
599		return 0;
600
601	spin_lock(&man->bdev->lru_lock);
602	usage = man->usage;
603	spin_unlock(&man->bdev->lru_lock);
604	return usage;
605}
606EXPORT_SYMBOL(ttm_resource_manager_usage);
607
608/**
609 * ttm_resource_manager_debug
610 *
611 * @man: manager type to dump.
612 * @p: printer to use for debug.
613 */
614void ttm_resource_manager_debug(struct ttm_resource_manager *man,
615				struct drm_printer *p)
616{
617	drm_printf(p, "  use_type: %d\n", man->use_type);
618	drm_printf(p, "  use_tt: %d\n", man->use_tt);
619	drm_printf(p, "  size: %llu\n", man->size);
620	drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
621	if (man->func->debug)
622		man->func->debug(man, p);
623}
624EXPORT_SYMBOL(ttm_resource_manager_debug);
625
626static void
627ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
628			       struct ttm_lru_item *next_lru)
629{
630	struct ttm_resource *next = ttm_lru_item_to_res(next_lru);
631	struct ttm_lru_bulk_move *bulk = NULL;
632	struct ttm_buffer_object *bo = next->bo;
633
634	lockdep_assert_held(&cursor->man->bdev->lru_lock);
635	bulk = bo->bulk_move;
636
637	if (cursor->bulk != bulk) {
638		if (bulk) {
639			list_move_tail(&cursor->bulk_link, &bulk->cursor_list);
640			cursor->mem_type = next->mem_type;
641		} else {
642			list_del_init(&cursor->bulk_link);
643		}
644		cursor->bulk = bulk;
645	}
646}
647
648/**
649 * ttm_resource_manager_first() - Start iterating over the resources
650 * of a resource manager
651 * @cursor: cursor to record the position
652 *
653 * Initializes the cursor and starts iterating. When done iterating,
654 * the caller must explicitly call ttm_resource_cursor_fini().
655 *
656 * Return: The first resource from the resource manager.
657 */
658struct ttm_resource *
659ttm_resource_manager_first(struct ttm_resource_cursor *cursor)
660{
661	struct ttm_resource_manager *man = cursor->man;
662
663	if (WARN_ON_ONCE(!man))
664		return NULL;
665
666	lockdep_assert_held(&man->bdev->lru_lock);
667
668	list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
669	return ttm_resource_manager_next(cursor);
670}
671
672/**
673 * ttm_resource_manager_next() - Continue iterating over the resource manager
674 * resources
675 * @cursor: cursor to record the position
676 *
677 * Return: the next resource from the resource manager.
678 */
679struct ttm_resource *
680ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
681{
682	struct ttm_resource_manager *man = cursor->man;
683	struct ttm_lru_item *lru;
684
685	lockdep_assert_held(&man->bdev->lru_lock);
686
687	for (;;) {
688		lru = &cursor->hitch;
689		list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
690			if (ttm_lru_item_is_res(lru)) {
691				ttm_resource_cursor_check_bulk(cursor, lru);
692				list_move(&cursor->hitch.link, &lru->link);
693				return ttm_lru_item_to_res(lru);
694			}
695		}
696
697		if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
698			break;
699
700		list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
701		ttm_resource_cursor_clear_bulk(cursor);
702	}
703
704	return NULL;
705}
706
707/**
708 * ttm_lru_first_res_or_null() - Return the first resource on an lru list
709 * @head: The list head of the lru list.
710 *
711 * Return: Pointer to the first resource on the lru list or NULL if
712 * there is none.
713 */
714struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head)
715{
716	struct ttm_lru_item *lru;
717
718	list_for_each_entry(lru, head, link) {
719		if (ttm_lru_item_is_res(lru))
720			return ttm_lru_item_to_res(lru);
721	}
722
723	return NULL;
724}
725
726static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
727					  struct iosys_map *dmap,
728					  pgoff_t i)
729{
730	struct ttm_kmap_iter_iomap *iter_io =
731		container_of(iter, typeof(*iter_io), base);
732	void __iomem *addr;
733
734retry:
735	while (i >= iter_io->cache.end) {
736		iter_io->cache.sg = iter_io->cache.sg ?
737			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
738		iter_io->cache.i = iter_io->cache.end;
739		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
740			PAGE_SHIFT;
741		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
742			iter_io->start;
743	}
744
745	if (i < iter_io->cache.i) {
746		iter_io->cache.end = 0;
747		iter_io->cache.sg = NULL;
748		goto retry;
749	}
750
751	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
752				       (((resource_size_t)i - iter_io->cache.i)
753					<< PAGE_SHIFT));
754	iosys_map_set_vaddr_iomem(dmap, addr);
755}
756
757static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
758					    struct iosys_map *map)
759{
760	io_mapping_unmap_local(map->vaddr_iomem);
761}
762
763static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
764	.map_local =  ttm_kmap_iter_iomap_map_local,
765	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
766	.maps_tt = false,
767};
768
769/**
770 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
771 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
772 * @iomap: The struct io_mapping representing the underlying linear io_memory.
773 * @st: sg_table into @iomap, representing the memory of the struct
774 * ttm_resource.
775 * @start: Offset that needs to be subtracted from @st to make
776 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
777 *
778 * Return: Pointer to the embedded struct ttm_kmap_iter.
779 */
780struct ttm_kmap_iter *
781ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
782			 struct io_mapping *iomap,
783			 struct sg_table *st,
784			 resource_size_t start)
785{
786	iter_io->base.ops = &ttm_kmap_iter_io_ops;
787	iter_io->iomap = iomap;
788	iter_io->st = st;
789	iter_io->start = start;
790	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
791
792	return &iter_io->base;
793}
794EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
795
796/**
797 * DOC: Linear io iterator
798 *
799 * This code should die in the not too near future. Best would be if we could
800 * make io-mapping use memremap for all io memory, and have memremap
801 * implement a kmap_local functionality. We could then strip a huge amount of
802 * code. These linear io iterators are implemented to mimic old functionality,
803 * and they don't use kmap_local semantics at all internally. Rather ioremap or
804 * friends, and at least on 32-bit they add global TLB flushes and points
805 * of failure.
806 */
807
808static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
809					      struct iosys_map *dmap,
810					      pgoff_t i)
811{
812	struct ttm_kmap_iter_linear_io *iter_io =
813		container_of(iter, typeof(*iter_io), base);
814
815	*dmap = iter_io->dmap;
816	iosys_map_incr(dmap, i * PAGE_SIZE);
817}
818
819static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
820	.map_local =  ttm_kmap_iter_linear_io_map_local,
821	.maps_tt = false,
822};
823
824/**
825 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
826 * @iter_io: The iterator to initialize
827 * @bdev: The TTM device
828 * @mem: The ttm resource representing the iomap.
829 *
830 * This function is for internal TTM use only. It sets up a memcpy kmap iterator
831 * pointing at a linear chunk of io memory.
832 *
833 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
834 * failure.
835 */
836struct ttm_kmap_iter *
837ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
838			     struct ttm_device *bdev,
839			     struct ttm_resource *mem)
840{
841	int ret;
842
843	ret = ttm_mem_io_reserve(bdev, mem);
844	if (ret)
845		goto out_err;
846	if (!mem->bus.is_iomem) {
847		ret = -EINVAL;
848		goto out_io_free;
849	}
850
851	if (mem->bus.addr) {
852		iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
853		iter_io->needs_unmap = false;
854	} else {
855		iter_io->needs_unmap = true;
856		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
857		if (mem->bus.caching == ttm_write_combined)
858			iosys_map_set_vaddr_iomem(&iter_io->dmap,
859						  ioremap_wc(mem->bus.offset,
860							     mem->size));
861		else if (mem->bus.caching == ttm_cached)
862			iosys_map_set_vaddr(&iter_io->dmap,
863					    memremap(mem->bus.offset, mem->size,
864						     MEMREMAP_WB |
865						     MEMREMAP_WT |
866						     MEMREMAP_WC));
867
868		/* If uncached requested or if mapping cached or wc failed */
869		if (iosys_map_is_null(&iter_io->dmap))
870			iosys_map_set_vaddr_iomem(&iter_io->dmap,
871						  ioremap(mem->bus.offset,
872							  mem->size));
873
874		if (iosys_map_is_null(&iter_io->dmap)) {
875			ret = -ENOMEM;
876			goto out_io_free;
877		}
878	}
879
880	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
881	return &iter_io->base;
882
883out_io_free:
884	ttm_mem_io_free(bdev, mem);
885out_err:
886	return ERR_PTR(ret);
887}
888
889/**
890 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
891 * @iter_io: The iterator to initialize
892 * @bdev: The TTM device
893 * @mem: The ttm resource representing the iomap.
894 *
895 * This function is for internal TTM use only. It cleans up a memcpy kmap
896 * iterator initialized by ttm_kmap_iter_linear_io_init.
897 */
898void
899ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
900			     struct ttm_device *bdev,
901			     struct ttm_resource *mem)
902{
903	if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
904		if (iter_io->dmap.is_iomem)
905			iounmap(iter_io->dmap.vaddr_iomem);
906		else
907			memunmap(iter_io->dmap.vaddr);
908	}
909
910	ttm_mem_io_free(bdev, mem);
911}
912
913#if defined(CONFIG_DEBUG_FS)
914
915static int ttm_resource_manager_show(struct seq_file *m, void *unused)
916{
917	struct ttm_resource_manager *man =
918		(struct ttm_resource_manager *)m->private;
919	struct drm_printer p = drm_seq_file_printer(m);
920	ttm_resource_manager_debug(man, &p);
921	return 0;
922}
923DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
924
925#endif
926
927/**
928 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
929 * resource manager.
930 * @man: The TTM resource manager for which the debugfs stats file be creates
931 * @parent: debugfs directory in which the file will reside
932 * @name: The filename to create.
933 *
934 * This function setups up a debugfs file that can be used to look
935 * at debug statistics of the specified ttm_resource_manager.
936 */
937void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
938					 struct dentry * parent,
939					 const char *name)
940{
941#if defined(CONFIG_DEBUG_FS)
942	debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
943#endif
944}
945EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);