xe_bo.c

  1// SPDX-License-Identifier: GPL-2.0 AND MIT
  2/*
  3 * Copyright © 2022 Intel Corporation
  4 */
  5
  6#include <kunit/test.h>
  7#include <kunit/visibility.h>
  8
  9#include <linux/iosys-map.h>
 10#include <linux/math64.h>
 11#include <linux/prandom.h>
 12#include <linux/swap.h>
 13
 14#include <uapi/linux/sysinfo.h>
 15
 16#include "tests/xe_kunit_helpers.h"
 17#include "tests/xe_pci_test.h"
 18#include "tests/xe_test.h"
 19
 20#include "xe_bo_evict.h"
 21#include "xe_pci.h"
 22#include "xe_pm.h"
 23
 24static int ccs_test_migrate(struct xe_tile *tile, struct xe_bo *bo,
 25			    bool clear, u64 get_val, u64 assign_val,
 26			    struct kunit *test, struct drm_exec *exec)
 27{
 28	struct dma_fence *fence;
 29	struct ttm_tt *ttm;
 30	struct page *page;
 31	pgoff_t ccs_page;
 32	long timeout;
 33	u64 *cpu_map;
 34	int ret;
 35	u32 offset;
 36
 37	/* Move bo to VRAM if not already there. */
 38	ret = xe_bo_validate(bo, NULL, false, exec);
 39	if (ret) {
 40		KUNIT_FAIL(test, "Failed to validate bo.\n");
 41		return ret;
 42	}
 43
 44	/* Optionally clear bo *and* CCS data in VRAM. */
 45	if (clear) {
 46		fence = xe_migrate_clear(tile->migrate, bo, bo->ttm.resource,
 47					 XE_MIGRATE_CLEAR_FLAG_FULL);
 48		if (IS_ERR(fence)) {
 49			KUNIT_FAIL(test, "Failed to submit bo clear.\n");
 50			return PTR_ERR(fence);
 51		}
 52
 53		if (dma_fence_wait_timeout(fence, false, 5 * HZ) <= 0) {
 54			dma_fence_put(fence);
 55			KUNIT_FAIL(test, "Timeout while clearing bo.\n");
 56			return  -ETIME;
 57		}
 58
 59		dma_fence_put(fence);
 60	}
 61
 62	/* Evict to system. CCS data should be copied. */
 63	ret = xe_bo_evict(bo, exec);
 64	if (ret) {
 65		KUNIT_FAIL(test, "Failed to evict bo.\n");
 66		return ret;
 67	}
 68
 69	/* Sync all migration blits */
 70	timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
 71					DMA_RESV_USAGE_KERNEL,
 72					true,
 73					5 * HZ);
 74	if (timeout <= 0) {
 75		KUNIT_FAIL(test, "Failed to sync bo eviction.\n");
 76		return -ETIME;
 77	}
 78
 79	/*
 80	 * Bo with CCS data is now in system memory. Verify backing store
 81	 * and data integrity. Then assign for the next testing round while
 82	 * we still have a CPU map.
 83	 */
 84	ttm = bo->ttm.ttm;
 85	if (!ttm || !ttm_tt_is_populated(ttm)) {
 86		KUNIT_FAIL(test, "Bo was not in expected placement.\n");
 87		return -EINVAL;
 88	}
 89
 90	ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT;
 91	if (ccs_page >= ttm->num_pages) {
 92		KUNIT_FAIL(test, "No TTM CCS pages present.\n");
 93		return -EINVAL;
 94	}
 95
 96	page = ttm->pages[ccs_page];
 97	cpu_map = kmap_local_page(page);
 98
 99	/* Check first CCS value */
100	if (cpu_map[0] != get_val) {
101		KUNIT_FAIL(test,
102			   "Expected CCS readout 0x%016llx, got 0x%016llx.\n",
103			   (unsigned long long)get_val,
104			   (unsigned long long)cpu_map[0]);
105		ret = -EINVAL;
106	}
107
108	/* Check last CCS value, or at least last value in page. */
109	offset = xe_device_ccs_bytes(tile_to_xe(tile), xe_bo_size(bo));
110	offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1;
111	if (cpu_map[offset] != get_val) {
112		KUNIT_FAIL(test,
113			   "Expected CCS readout 0x%016llx, got 0x%016llx.\n",
114			   (unsigned long long)get_val,
115			   (unsigned long long)cpu_map[offset]);
116		ret = -EINVAL;
117	}
118
119	cpu_map[0] = assign_val;
120	cpu_map[offset] = assign_val;
121	kunmap_local(cpu_map);
122
123	return ret;
124}
125
126static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile,
127			      struct kunit *test)
128{
129	struct xe_bo *bo;
130
131	int ret;
132
133	/* TODO: Sanity check */
134	unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
135	struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
136
137	if (IS_DGFX(xe))
138		kunit_info(test, "Testing vram id %u\n", tile->id);
139	else
140		kunit_info(test, "Testing system memory\n");
141
142	bo = xe_bo_create_user(xe, NULL, SZ_1M, DRM_XE_GEM_CPU_CACHING_WC,
143			       bo_flags, exec);
144	if (IS_ERR(bo)) {
145		KUNIT_FAIL(test, "Failed to create bo.\n");
146		return;
147	}
148
149	xe_bo_lock(bo, false);
150
151	kunit_info(test, "Verifying that CCS data is cleared on creation.\n");
152	ret = ccs_test_migrate(tile, bo, false, 0ULL, 0xdeadbeefdeadbeefULL,
153			       test, exec);
154	if (ret)
155		goto out_unlock;
156
157	kunit_info(test, "Verifying that CCS data survives migration.\n");
158	ret = ccs_test_migrate(tile, bo, false, 0xdeadbeefdeadbeefULL,
159			       0xdeadbeefdeadbeefULL, test, exec);
160	if (ret)
161		goto out_unlock;
162
163	kunit_info(test, "Verifying that CCS data can be properly cleared.\n");
164	ret = ccs_test_migrate(tile, bo, true, 0ULL, 0ULL, test, exec);
165
166out_unlock:
167	xe_bo_unlock(bo);
168	xe_bo_put(bo);
169}
170
171static int ccs_test_run_device(struct xe_device *xe)
172{
173	struct kunit *test = kunit_get_current_test();
174	struct xe_tile *tile;
175	int id;
176
177	if (!xe_device_has_flat_ccs(xe)) {
178		kunit_skip(test, "non-flat-ccs device\n");
179		return 0;
180	}
181
182	/* For xe2+ dgfx, we don't handle ccs metadata */
183	if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) {
184		kunit_skip(test, "xe2+ dgfx device\n");
185		return 0;
186	}
187
188	xe_pm_runtime_get(xe);
189
190	for_each_tile(tile, xe, id) {
191		/* For igfx run only for primary tile */
192		if (!IS_DGFX(xe) && id > 0)
193			continue;
194		ccs_test_run_tile(xe, tile, test);
195	}
196
197	xe_pm_runtime_put(xe);
198
199	return 0;
200}
201
202static void xe_ccs_migrate_kunit(struct kunit *test)
203{
204	struct xe_device *xe = test->priv;
205
206	ccs_test_run_device(xe);
207}
208
209static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test)
210{
211	struct xe_bo *bo, *external;
212	unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
213	struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate);
214	struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
215	struct xe_gt *__gt;
216	int err, i, id;
217
218	kunit_info(test, "Testing device %s vram id %u\n",
219		   dev_name(xe->drm.dev), tile->id);
220
221	for (i = 0; i < 2; ++i) {
222		xe_vm_lock(vm, false);
223		bo = xe_bo_create_user(xe, vm, 0x10000,
224				       DRM_XE_GEM_CPU_CACHING_WC,
225				       bo_flags, exec);
226		xe_vm_unlock(vm);
227		if (IS_ERR(bo)) {
228			KUNIT_FAIL(test, "bo create err=%pe\n", bo);
229			break;
230		}
231
232		external = xe_bo_create_user(xe, NULL, 0x10000,
233					     DRM_XE_GEM_CPU_CACHING_WC,
234					     bo_flags, NULL);
235		if (IS_ERR(external)) {
236			KUNIT_FAIL(test, "external bo create err=%pe\n", external);
237			goto cleanup_bo;
238		}
239
240		xe_bo_lock(external, false);
241		err = xe_bo_pin_external(external, false, exec);
242		xe_bo_unlock(external);
243		if (err) {
244			KUNIT_FAIL(test, "external bo pin err=%pe\n",
245				   ERR_PTR(err));
246			goto cleanup_external;
247		}
248
249		err = xe_bo_evict_all(xe);
250		if (err) {
251			KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err));
252			goto cleanup_all;
253		}
254
255		for_each_gt(__gt, xe, id)
256			xe_gt_sanitize(__gt);
257		err = xe_bo_restore_early(xe);
258		/*
259		 * Snapshotting the CTB and copying back a potentially old
260		 * version seems risky, depending on what might have been
261		 * inflight. Also it seems snapshotting the ADS object and
262		 * copying back results in serious breakage. Normally when
263		 * calling xe_bo_restore_kernel() we always fully restart the
264		 * GT, which re-intializes such things.  We could potentially
265		 * skip saving and restoring such objects in xe_bo_evict_all()
266		 * however seems quite fragile not to also restart the GT. Try
267		 * to do that here by triggering a GT reset.
268		 */
269		for_each_gt(__gt, xe, id)
270			xe_gt_reset(__gt);
271
272		if (err) {
273			KUNIT_FAIL(test, "restore kernel err=%pe\n",
274				   ERR_PTR(err));
275			goto cleanup_all;
276		}
277
278		err = xe_bo_restore_late(xe);
279		if (err) {
280			KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err));
281			goto cleanup_all;
282		}
283
284		if (!xe_bo_is_vram(external)) {
285			KUNIT_FAIL(test, "external bo is not vram\n");
286			err = -EPROTO;
287			goto cleanup_all;
288		}
289
290		if (xe_bo_is_vram(bo)) {
291			KUNIT_FAIL(test, "bo is vram\n");
292			err = -EPROTO;
293			goto cleanup_all;
294		}
295
296		if (i) {
297			down_read(&vm->lock);
298			xe_vm_lock(vm, false);
299			err = xe_bo_validate(bo, bo->vm, false, exec);
300			xe_vm_unlock(vm);
301			up_read(&vm->lock);
302			if (err) {
303				KUNIT_FAIL(test, "bo valid err=%pe\n",
304					   ERR_PTR(err));
305				goto cleanup_all;
306			}
307			xe_bo_lock(external, false);
308			err = xe_bo_validate(external, NULL, false, exec);
309			xe_bo_unlock(external);
310			if (err) {
311				KUNIT_FAIL(test, "external bo valid err=%pe\n",
312					   ERR_PTR(err));
313				goto cleanup_all;
314			}
315		}
316
317		xe_bo_lock(external, false);
318		xe_bo_unpin_external(external);
319		xe_bo_unlock(external);
320
321		xe_bo_put(external);
322
323		xe_bo_lock(bo, false);
324		__xe_bo_unset_bulk_move(bo);
325		xe_bo_unlock(bo);
326		xe_bo_put(bo);
327		continue;
328
329cleanup_all:
330		xe_bo_lock(external, false);
331		xe_bo_unpin_external(external);
332		xe_bo_unlock(external);
333cleanup_external:
334		xe_bo_put(external);
335cleanup_bo:
336		xe_bo_lock(bo, false);
337		__xe_bo_unset_bulk_move(bo);
338		xe_bo_unlock(bo);
339		xe_bo_put(bo);
340		break;
341	}
342
343	xe_vm_put(vm);
344
345	return 0;
346}
347
348static int evict_test_run_device(struct xe_device *xe)
349{
350	struct kunit *test = kunit_get_current_test();
351	struct xe_tile *tile;
352	int id;
353
354	if (!IS_DGFX(xe)) {
355		kunit_skip(test, "non-discrete device\n");
356		return 0;
357	}
358
359	xe_pm_runtime_get(xe);
360
361	for_each_tile(tile, xe, id)
362		evict_test_run_tile(xe, tile, test);
363
364	xe_pm_runtime_put(xe);
365
366	return 0;
367}
368
369static void xe_bo_evict_kunit(struct kunit *test)
370{
371	struct xe_device *xe = test->priv;
372
373	evict_test_run_device(xe);
374}
375
376struct xe_bo_link {
377	struct list_head link;
378	struct xe_bo *bo;
379	u32 val;
380};
381
382#define XE_BO_SHRINK_SIZE ((unsigned long)SZ_64M)
383
384static int shrink_test_fill_random(struct xe_bo *bo, struct rnd_state *state,
385				   struct xe_bo_link *link)
386{
387	struct iosys_map map;
388	int ret = ttm_bo_vmap(&bo->ttm, &map);
389	size_t __maybe_unused i;
390
391	if (ret)
392		return ret;
393
394	for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) {
395		u32 val = prandom_u32_state(state);
396
397		iosys_map_wr(&map, i, u32, val);
398		if (i == 0)
399			link->val = val;
400	}
401
402	ttm_bo_vunmap(&bo->ttm, &map);
403	return 0;
404}
405
406static bool shrink_test_verify(struct kunit *test, struct xe_bo *bo,
407			       unsigned int bo_nr, struct rnd_state *state,
408			       struct xe_bo_link *link)
409{
410	struct iosys_map map;
411	int ret = ttm_bo_vmap(&bo->ttm, &map);
412	size_t i;
413	bool failed = false;
414
415	if (ret) {
416		KUNIT_FAIL(test, "Error mapping bo %u for content check.\n", bo_nr);
417		return true;
418	}
419
420	for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) {
421		u32 val = prandom_u32_state(state);
422
423		if (iosys_map_rd(&map, i, u32) != val) {
424			KUNIT_FAIL(test, "Content not preserved, bo %u offset 0x%016llx",
425				   bo_nr, (unsigned long long)i);
426			kunit_info(test, "Failed value is 0x%08x, recorded 0x%08x\n",
427				   (unsigned int)iosys_map_rd(&map, i, u32), val);
428			if (i == 0 && val != link->val)
429				kunit_info(test, "Looks like PRNG is out of sync.\n");
430			failed = true;
431			break;
432		}
433	}
434
435	ttm_bo_vunmap(&bo->ttm, &map);
436
437	return failed;
438}
439
440/*
441 * Try to create system bos corresponding to twice the amount
442 * of available system memory to test shrinker functionality.
443 * If no swap space is available to accommodate the
444 * memory overcommit, mark bos purgeable.
445 */
446static int shrink_test_run_device(struct xe_device *xe)
447{
448	struct kunit *test = kunit_get_current_test();
449	LIST_HEAD(bos);
450	struct xe_bo_link *link, *next;
451	struct sysinfo si;
452	u64 ram, ram_and_swap, purgeable = 0, alloced, to_alloc, limit;
453	unsigned int interrupted = 0, successful = 0, count = 0;
454	struct rnd_state prng;
455	u64 rand_seed;
456	bool failed = false;
457
458	rand_seed = get_random_u64();
459	prandom_seed_state(&prng, rand_seed);
460	kunit_info(test, "Random seed is 0x%016llx.\n",
461		   (unsigned long long)rand_seed);
462
463	/* Skip if execution time is expected to be too long. */
464
465	limit = SZ_32G;
466	/* IGFX with flat CCS needs to copy when swapping / shrinking */
467	if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))
468		limit = SZ_16G;
469
470	si_meminfo(&si);
471	ram = (size_t)si.freeram * si.mem_unit;
472	if (ram > limit) {
473		kunit_skip(test, "Too long expected execution time.\n");
474		return 0;
475	}
476	to_alloc = ram * 2;
477
478	ram_and_swap = ram + get_nr_swap_pages() * PAGE_SIZE;
479	if (to_alloc > ram_and_swap)
480		purgeable = to_alloc - ram_and_swap;
481	purgeable += div64_u64(purgeable, 5);
482
483	kunit_info(test, "Free ram is %lu bytes. Will allocate twice of that.\n",
484		   (unsigned long)ram);
485	for (alloced = 0; alloced < to_alloc; alloced += XE_BO_SHRINK_SIZE) {
486		struct xe_bo *bo;
487		unsigned int mem_type;
488		struct xe_ttm_tt *xe_tt;
489
490		link = kzalloc(sizeof(*link), GFP_KERNEL);
491		if (!link) {
492			KUNIT_FAIL(test, "Unexpected link allocation failure\n");
493			failed = true;
494			break;
495		}
496
497		INIT_LIST_HEAD(&link->link);
498
499		/* We can create bos using WC caching here. But it is slower. */
500		bo = xe_bo_create_user(xe, NULL, XE_BO_SHRINK_SIZE,
501				       DRM_XE_GEM_CPU_CACHING_WB,
502				       XE_BO_FLAG_SYSTEM, NULL);
503		if (IS_ERR(bo)) {
504			if (bo != ERR_PTR(-ENOMEM) && bo != ERR_PTR(-ENOSPC) &&
505			    bo != ERR_PTR(-EINTR) && bo != ERR_PTR(-ERESTARTSYS))
506				KUNIT_FAIL(test, "Error creating bo: %pe\n", bo);
507			kfree(link);
508			failed = true;
509			break;
510		}
511		xe_bo_lock(bo, false);
512		xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm);
513
514		/*
515		 * Allocate purgeable bos first, because if we do it the
516		 * other way around, they may not be subject to swapping...
517		 */
518		if (alloced < purgeable) {
519			xe_ttm_tt_account_subtract(xe, &xe_tt->ttm);
520			xe_tt->purgeable = true;
521			xe_ttm_tt_account_add(xe, &xe_tt->ttm);
522			bo->ttm.priority = 0;
523			spin_lock(&bo->ttm.bdev->lru_lock);
524			ttm_bo_move_to_lru_tail(&bo->ttm);
525			spin_unlock(&bo->ttm.bdev->lru_lock);
526		} else {
527			int ret = shrink_test_fill_random(bo, &prng, link);
528
529			if (ret) {
530				xe_bo_unlock(bo);
531				xe_bo_put(bo);
532				KUNIT_FAIL(test, "Error filling bo with random data: %pe\n",
533					   ERR_PTR(ret));
534				kfree(link);
535				failed = true;
536				break;
537			}
538		}
539
540		mem_type = bo->ttm.resource->mem_type;
541		xe_bo_unlock(bo);
542		link->bo = bo;
543		list_add_tail(&link->link, &bos);
544
545		if (mem_type != XE_PL_TT) {
546			KUNIT_FAIL(test, "Bo in incorrect memory type: %u\n",
547				   bo->ttm.resource->mem_type);
548			failed = true;
549		}
550		cond_resched();
551		if (signal_pending(current))
552			break;
553	}
554
555	/*
556	 * Read back and destroy bos. Reset the pseudo-random seed to get an
557	 * identical pseudo-random number sequence for readback.
558	 */
559	prandom_seed_state(&prng, rand_seed);
560	list_for_each_entry_safe(link, next, &bos, link) {
561		static struct ttm_operation_ctx ctx = {.interruptible = true};
562		struct xe_bo *bo = link->bo;
563		struct xe_ttm_tt *xe_tt;
564		int ret;
565
566		count++;
567		if (!signal_pending(current) && !failed) {
568			bool purgeable, intr = false;
569
570			xe_bo_lock(bo, NULL);
571
572			/* xe_tt->purgeable is cleared on validate. */
573			xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm);
574			purgeable = xe_tt->purgeable;
575			do {
576				ret = ttm_bo_validate(&bo->ttm, &tt_placement, &ctx);
577				if (ret == -EINTR)
578					intr = true;
579			} while (ret == -EINTR && !signal_pending(current));
580			if (!ret && !purgeable)
581				failed = shrink_test_verify(test, bo, count, &prng, link);
582
583			xe_bo_unlock(bo);
584			if (ret) {
585				KUNIT_FAIL(test, "Validation failed: %pe\n",
586					   ERR_PTR(ret));
587				failed = true;
588			} else if (intr) {
589				interrupted++;
590			} else {
591				successful++;
592			}
593		}
594		xe_bo_put(link->bo);
595		list_del(&link->link);
596		kfree(link);
597	}
598	kunit_info(test, "Readbacks interrupted: %u successful: %u\n",
599		   interrupted, successful);
600
601	return 0;
602}
603
604static void xe_bo_shrink_kunit(struct kunit *test)
605{
606	struct xe_device *xe = test->priv;
607
608	shrink_test_run_device(xe);
609}
610
611static struct kunit_case xe_bo_tests[] = {
612	KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param),
613	KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param),
614	{}
615};
616
617VISIBLE_IF_KUNIT
618struct kunit_suite xe_bo_test_suite = {
619	.name = "xe_bo",
620	.test_cases = xe_bo_tests,
621	.init = xe_kunit_helper_xe_device_live_test_init,
622};
623EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite);
624
625static struct kunit_case xe_bo_shrink_test[] = {
626	KUNIT_CASE_PARAM_ATTR(xe_bo_shrink_kunit, xe_pci_live_device_gen_param,
627			      {.speed = KUNIT_SPEED_SLOW}),
628	{}
629};
630
631VISIBLE_IF_KUNIT
632struct kunit_suite xe_bo_shrink_test_suite = {
633	.name = "xe_bo_shrink",
634	.test_cases = xe_bo_shrink_test,
635	.init = xe_kunit_helper_xe_device_live_test_init,
636};
637EXPORT_SYMBOL_IF_KUNIT(xe_bo_shrink_test_suite);