xe_migrate.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2020-2022 Intel Corporation
  4 */
  5
  6#include <kunit/test.h>
  7#include <kunit/visibility.h>
  8
  9#include "tests/xe_kunit_helpers.h"
 10#include "tests/xe_pci_test.h"
 11
 12#include "xe_pci.h"
 13#include "xe_pm.h"
 14
 15static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
 16				const char *str, struct kunit *test)
 17{
 18	long ret;
 19
 20	if (IS_ERR(fence)) {
 21		KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
 22			   PTR_ERR(fence));
 23		return true;
 24	}
 25	if (!fence)
 26		return true;
 27
 28	ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
 29	if (ret <= 0) {
 30		KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
 31		return true;
 32	}
 33
 34	return false;
 35}
 36
 37static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe,
 38			  struct xe_bb *bb, u32 second_idx, const char *str,
 39			  struct kunit *test)
 40{
 41	u64 batch_base = xe_migrate_batch_base(m, xe->info.has_usm);
 42	struct xe_sched_job *job = xe_bb_create_migration_job(m->q, bb,
 43							      batch_base,
 44							      second_idx);
 45	struct dma_fence *fence;
 46
 47	if (IS_ERR(job)) {
 48		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 49			   PTR_ERR(job));
 50		return PTR_ERR(job);
 51	}
 52
 53	xe_sched_job_arm(job);
 54	fence = dma_fence_get(&job->drm.s_fence->finished);
 55	xe_sched_job_push(job);
 56
 57	if (sanity_fence_failed(xe, fence, str, test))
 58		return -ETIMEDOUT;
 59
 60	dma_fence_put(fence);
 61	kunit_info(test, "%s: Job completed\n", str);
 62	return 0;
 63}
 64
 65#define check(_retval, _expected, str, _test)				\
 66	do { if ((_retval) != (_expected)) {				\
 67			KUNIT_FAIL(_test, "Sanity check failed: " str	\
 68				   " expected %llx, got %llx\n",	\
 69				   (u64)(_expected), (u64)(_retval));	\
 70		} } while (0)
 71
 72static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
 73		      struct kunit *test, u32 region, struct drm_exec *exec)
 74{
 75	struct xe_device *xe = tile_to_xe(m->tile);
 76	u64 retval, expected = 0;
 77	bool big = xe_bo_size(bo) >= SZ_2M;
 78	struct dma_fence *fence;
 79	const char *str = big ? "Copying big bo" : "Copying small bo";
 80	int err;
 81
 82	struct xe_bo *remote = xe_bo_create_locked(xe, m->tile, NULL,
 83						   xe_bo_size(bo),
 84						   ttm_bo_type_kernel,
 85						   region |
 86						   XE_BO_FLAG_NEEDS_CPU_ACCESS |
 87						   XE_BO_FLAG_PINNED,
 88						   exec);
 89	if (IS_ERR(remote)) {
 90		KUNIT_FAIL(test, "Failed to allocate remote bo for %s: %pe\n",
 91			   str, remote);
 92		return;
 93	}
 94
 95	err = xe_bo_validate(remote, NULL, false, exec);
 96	if (err) {
 97		KUNIT_FAIL(test, "Failed to validate system bo for %s: %i\n",
 98			   str, err);
 99		goto out_unlock;
100	}
101
102	err = xe_bo_vmap(remote);
103	if (err) {
104		KUNIT_FAIL(test, "Failed to vmap system bo for %s: %i\n",
105			   str, err);
106		goto out_unlock;
107	}
108
109	xe_map_memset(xe, &remote->vmap, 0, 0xd0, xe_bo_size(remote));
110	fence = xe_migrate_clear(m, remote, remote->ttm.resource,
111				 XE_MIGRATE_CLEAR_FLAG_FULL);
112	if (!sanity_fence_failed(xe, fence, big ? "Clearing remote big bo" :
113				 "Clearing remote small bo", test)) {
114		retval = xe_map_rd(xe, &remote->vmap, 0, u64);
115		check(retval, expected, "remote first offset should be cleared",
116		      test);
117		retval = xe_map_rd(xe, &remote->vmap, xe_bo_size(remote) - 8, u64);
118		check(retval, expected, "remote last offset should be cleared",
119		      test);
120	}
121	dma_fence_put(fence);
122
123	/* Try to copy 0xc0 from remote to vram with 2MB or 64KiB/4KiB pages */
124	xe_map_memset(xe, &remote->vmap, 0, 0xc0, xe_bo_size(remote));
125	xe_map_memset(xe, &bo->vmap, 0, 0xd0, xe_bo_size(bo));
126
127	expected = 0xc0c0c0c0c0c0c0c0;
128	fence = xe_migrate_copy(m, remote, bo, remote->ttm.resource,
129				bo->ttm.resource, false);
130	if (!sanity_fence_failed(xe, fence, big ? "Copying big bo remote -> vram" :
131				 "Copying small bo remote -> vram", test)) {
132		retval = xe_map_rd(xe, &bo->vmap, 0, u64);
133		check(retval, expected,
134		      "remote -> vram bo first offset should be copied", test);
135		retval = xe_map_rd(xe, &bo->vmap, xe_bo_size(bo) - 8, u64);
136		check(retval, expected,
137		      "remote -> vram bo offset should be copied", test);
138	}
139	dma_fence_put(fence);
140
141	/* And other way around.. slightly hacky.. */
142	xe_map_memset(xe, &remote->vmap, 0, 0xd0, xe_bo_size(remote));
143	xe_map_memset(xe, &bo->vmap, 0, 0xc0, xe_bo_size(bo));
144
145	fence = xe_migrate_copy(m, bo, remote, bo->ttm.resource,
146				remote->ttm.resource, false);
147	if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> remote" :
148				 "Copying small bo vram -> remote", test)) {
149		retval = xe_map_rd(xe, &remote->vmap, 0, u64);
150		check(retval, expected,
151		      "vram -> remote bo first offset should be copied", test);
152		retval = xe_map_rd(xe, &remote->vmap, xe_bo_size(bo) - 8, u64);
153		check(retval, expected,
154		      "vram -> remote bo last offset should be copied", test);
155	}
156	dma_fence_put(fence);
157
158	xe_bo_vunmap(remote);
159out_unlock:
160	xe_bo_unlock(remote);
161	xe_bo_put(remote);
162}
163
164static void test_copy_sysmem(struct xe_migrate *m, struct xe_bo *bo,
165			     struct drm_exec *exec, struct kunit *test)
166{
167	test_copy(m, bo, test, XE_BO_FLAG_SYSTEM, exec);
168}
169
170static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo,
171			   struct drm_exec *exec, struct kunit *test)
172{
173	u32 region;
174
175	if (bo->ttm.resource->mem_type == XE_PL_SYSTEM)
176		return;
177
178	if (bo->ttm.resource->mem_type == XE_PL_VRAM0)
179		region = XE_BO_FLAG_VRAM1;
180	else
181		region = XE_BO_FLAG_VRAM0;
182	test_copy(m, bo, test, region, exec);
183}
184
185static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test,
186				   struct drm_exec *exec)
187{
188	struct xe_tile *tile = m->tile;
189	struct xe_device *xe = tile_to_xe(tile);
190	struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny;
191	struct xe_res_cursor src_it;
192	struct dma_fence *fence;
193	u64 retval, expected;
194	struct xe_bb *bb;
195	int err;
196	u8 id = tile->id;
197
198	err = xe_bo_vmap(bo);
199	if (err) {
200		KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
201			   PTR_ERR(bo));
202		return;
203	}
204
205	big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M,
206				   ttm_bo_type_kernel,
207				   XE_BO_FLAG_VRAM_IF_DGFX(tile),
208				   exec);
209	if (IS_ERR(big)) {
210		KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
211		goto vunmap;
212	}
213
214	pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE,
215				  ttm_bo_type_kernel,
216				  XE_BO_FLAG_VRAM_IF_DGFX(tile),
217				  exec);
218	if (IS_ERR(pt)) {
219		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
220			   PTR_ERR(pt));
221		goto free_big;
222	}
223
224	tiny = xe_bo_create_pin_map(xe, tile, m->q->vm,
225				    2 * SZ_4K,
226				    ttm_bo_type_kernel,
227				    XE_BO_FLAG_VRAM_IF_DGFX(tile),
228				    exec);
229	if (IS_ERR(tiny)) {
230		KUNIT_FAIL(test, "Failed to allocate tiny fake pt: %li\n",
231			   PTR_ERR(tiny));
232		goto free_pt;
233	}
234
235	bb = xe_bb_new(tile->primary_gt, 32, xe->info.has_usm);
236	if (IS_ERR(bb)) {
237		KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
238			   PTR_ERR(bb));
239		goto free_tiny;
240	}
241
242	kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
243		   (unsigned long)xe_bo_main_addr(m->q->vm->pt_root[id]->bo, XE_PAGE_SIZE),
244		   (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE));
245
246	/* First part of the test, are we updating our pagetable bo with a new entry? */
247	xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
248		  0xdeaddeadbeefbeef);
249	expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0);
250	if (m->q->vm->flags & XE_VM_FLAG_64K)
251		expected |= XE_PTE_PS64;
252	if (xe_bo_is_vram(pt))
253		xe_res_first(pt->ttm.resource, 0, xe_bo_size(pt), &src_it);
254	else
255		xe_res_first_sg(xe_bo_sg(pt), 0, xe_bo_size(pt), &src_it);
256
257	emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), false,
258		 &src_it, XE_PAGE_SIZE, pt->ttm.resource);
259
260	run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test);
261
262	retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
263			   u64);
264	check(retval, expected, "PTE entry write", test);
265
266	/* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
267	bb->len = 0;
268	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
269	xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
270	expected = 0;
271
272	emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
273		   IS_DGFX(xe));
274	run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable",
275		       test);
276
277	retval = xe_map_rd(xe, &pt->vmap, 0, u32);
278	check(retval, expected, "Write to PT after adding PTE", test);
279
280	/* Sanity checks passed, try the full ones! */
281
282	/* Clear a small bo */
283	kunit_info(test, "Clearing small buffer object\n");
284	xe_map_memset(xe, &tiny->vmap, 0, 0x22, xe_bo_size(tiny));
285	expected = 0;
286	fence = xe_migrate_clear(m, tiny, tiny->ttm.resource,
287				 XE_MIGRATE_CLEAR_FLAG_FULL);
288	if (sanity_fence_failed(xe, fence, "Clearing small bo", test))
289		goto out;
290
291	dma_fence_put(fence);
292	retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
293	check(retval, expected, "Command clear small first value", test);
294	retval = xe_map_rd(xe, &tiny->vmap, xe_bo_size(tiny) - 4, u32);
295	check(retval, expected, "Command clear small last value", test);
296
297	kunit_info(test, "Copying small buffer object to system\n");
298	test_copy_sysmem(m, tiny, exec, test);
299	if (xe->info.tile_count > 1) {
300		kunit_info(test, "Copying small buffer object to other vram\n");
301		test_copy_vram(m, tiny, exec, test);
302	}
303
304	/* Clear a big bo */
305	kunit_info(test, "Clearing big buffer object\n");
306	xe_map_memset(xe, &big->vmap, 0, 0x11, xe_bo_size(big));
307	expected = 0;
308	fence = xe_migrate_clear(m, big, big->ttm.resource,
309				 XE_MIGRATE_CLEAR_FLAG_FULL);
310	if (sanity_fence_failed(xe, fence, "Clearing big bo", test))
311		goto out;
312
313	dma_fence_put(fence);
314	retval = xe_map_rd(xe, &big->vmap, 0, u32);
315	check(retval, expected, "Command clear big first value", test);
316	retval = xe_map_rd(xe, &big->vmap, xe_bo_size(big) - 4, u32);
317	check(retval, expected, "Command clear big last value", test);
318
319	kunit_info(test, "Copying big buffer object to system\n");
320	test_copy_sysmem(m, big, exec, test);
321	if (xe->info.tile_count > 1) {
322		kunit_info(test, "Copying big buffer object to other vram\n");
323		test_copy_vram(m, big, exec, test);
324	}
325
326out:
327	xe_bb_free(bb, NULL);
328free_tiny:
329	xe_bo_unpin(tiny);
330	xe_bo_put(tiny);
331free_pt:
332	xe_bo_unpin(pt);
333	xe_bo_put(pt);
334free_big:
335	xe_bo_unpin(big);
336	xe_bo_put(big);
337vunmap:
338	xe_bo_vunmap(m->pt_bo);
339}
340
341static int migrate_test_run_device(struct xe_device *xe)
342{
343	struct kunit *test = kunit_get_current_test();
344	struct xe_tile *tile;
345	int id;
346
347	xe_pm_runtime_get(xe);
348
349	for_each_tile(tile, xe, id) {
350		struct xe_migrate *m = tile->migrate;
351		struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
352
353		kunit_info(test, "Testing tile id %d.\n", id);
354		xe_vm_lock(m->q->vm, false);
355		xe_migrate_sanity_test(m, test, exec);
356		xe_vm_unlock(m->q->vm);
357	}
358
359	xe_pm_runtime_put(xe);
360
361	return 0;
362}
363
364static void xe_migrate_sanity_kunit(struct kunit *test)
365{
366	struct xe_device *xe = test->priv;
367
368	migrate_test_run_device(xe);
369}
370
371static struct dma_fence *blt_copy(struct xe_tile *tile,
372				  struct xe_bo *src_bo, struct xe_bo *dst_bo,
373				  bool copy_only_ccs, const char *str, struct kunit *test)
374{
375	struct xe_gt *gt = tile->primary_gt;
376	struct xe_migrate *m = tile->migrate;
377	struct xe_device *xe = gt_to_xe(gt);
378	struct dma_fence *fence = NULL;
379	u64 size = xe_bo_size(src_bo);
380	struct xe_res_cursor src_it, dst_it;
381	struct ttm_resource *src = src_bo->ttm.resource, *dst = dst_bo->ttm.resource;
382	u64 src_L0_ofs, dst_L0_ofs;
383	u32 src_L0_pt, dst_L0_pt;
384	u64 src_L0, dst_L0;
385	int err;
386	bool src_is_vram = mem_type_is_vram(src->mem_type);
387	bool dst_is_vram = mem_type_is_vram(dst->mem_type);
388
389	if (!src_is_vram)
390		xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
391	else
392		xe_res_first(src, 0, size, &src_it);
393
394	if (!dst_is_vram)
395		xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);
396	else
397		xe_res_first(dst, 0, size, &dst_it);
398
399	while (size) {
400		u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
401		struct xe_sched_job *job;
402		struct xe_bb *bb;
403		u32 flush_flags = 0;
404		u32 update_idx;
405		u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
406		u32 pte_flags;
407
408		src_L0 = xe_migrate_res_sizes(m, &src_it);
409		dst_L0 = xe_migrate_res_sizes(m, &dst_it);
410
411		src_L0 = min(src_L0, dst_L0);
412
413		pte_flags = src_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM |
414					   PTE_UPDATE_FLAG_IS_COMP_PTE) : 0;
415		batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,
416					      &src_L0_ofs, &src_L0_pt, 0, 0,
417					      avail_pts);
418
419		pte_flags = dst_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM |
420					   PTE_UPDATE_FLAG_IS_COMP_PTE) : 0;
421		batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0,
422					      &dst_L0_ofs, &dst_L0_pt, 0,
423					      avail_pts, avail_pts);
424
425		/* Add copy commands size here */
426		batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
427			((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0);
428
429		bb = xe_bb_new(gt, batch_size, xe->info.has_usm);
430		if (IS_ERR(bb)) {
431			err = PTR_ERR(bb);
432			goto err_sync;
433		}
434
435		if (src_is_vram)
436			xe_res_next(&src_it, src_L0);
437		else
438			emit_pte(m, bb, src_L0_pt, src_is_vram, false,
439				 &src_it, src_L0, src);
440
441		if (dst_is_vram)
442			xe_res_next(&dst_it, src_L0);
443		else
444			emit_pte(m, bb, dst_L0_pt, dst_is_vram, false,
445				 &dst_it, src_L0, dst);
446
447		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
448		update_idx = bb->len;
449		if (!copy_only_ccs)
450			emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
451
452		if (copy_only_ccs)
453			flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
454							  src_is_vram, dst_L0_ofs,
455							  dst_is_vram, src_L0, dst_L0_ofs,
456							  copy_only_ccs);
457
458		job = xe_bb_create_migration_job(m->q, bb,
459						 xe_migrate_batch_base(m, xe->info.has_usm),
460						 update_idx);
461		if (IS_ERR(job)) {
462			err = PTR_ERR(job);
463			goto err;
464		}
465
466		xe_sched_job_add_migrate_flush(job, flush_flags);
467
468		mutex_lock(&m->job_mutex);
469		xe_sched_job_arm(job);
470		dma_fence_put(fence);
471		fence = dma_fence_get(&job->drm.s_fence->finished);
472		xe_sched_job_push(job);
473
474		dma_fence_put(m->fence);
475		m->fence = dma_fence_get(fence);
476
477		mutex_unlock(&m->job_mutex);
478
479		xe_bb_free(bb, fence);
480		size -= src_L0;
481		continue;
482
483err:
484		xe_bb_free(bb, NULL);
485
486err_sync:
487		if (fence) {
488			dma_fence_wait(fence, false);
489			dma_fence_put(fence);
490		}
491		return ERR_PTR(err);
492	}
493
494	return fence;
495}
496
497static void test_migrate(struct xe_device *xe, struct xe_tile *tile,
498			 struct xe_bo *sys_bo, struct xe_bo *vram_bo, struct xe_bo *ccs_bo,
499			 struct drm_exec *exec, struct kunit *test)
500{
501	struct dma_fence *fence;
502	u64 expected, retval;
503	long timeout;
504	long ret;
505
506	expected = 0xd0d0d0d0d0d0d0d0;
507	xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, xe_bo_size(sys_bo));
508
509	fence = blt_copy(tile, sys_bo, vram_bo, false, "Blit copy from sysmem to vram", test);
510	if (!sanity_fence_failed(xe, fence, "Blit copy from sysmem to vram", test)) {
511		retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
512		if (retval == expected)
513			KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n");
514	}
515	dma_fence_put(fence);
516
517	kunit_info(test, "Evict vram buffer object\n");
518	ret = xe_bo_evict(vram_bo, exec);
519	if (ret) {
520		KUNIT_FAIL(test, "Failed to evict bo.\n");
521		return;
522	}
523
524	ret = xe_bo_vmap(vram_bo);
525	if (ret) {
526		KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
527		return;
528	}
529
530	retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
531	check(retval, expected, "Clear evicted vram data first value", test);
532	retval = xe_map_rd(xe, &vram_bo->vmap, xe_bo_size(vram_bo) - 8, u64);
533	check(retval, expected, "Clear evicted vram data last value", test);
534
535	fence = blt_copy(tile, vram_bo, ccs_bo,
536			 true, "Blit surf copy from vram to sysmem", test);
537	if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) {
538		retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64);
539		check(retval, 0, "Clear ccs data first value", test);
540
541		retval = xe_map_rd(xe, &ccs_bo->vmap, xe_bo_size(ccs_bo) - 8, u64);
542		check(retval, 0, "Clear ccs data last value", test);
543	}
544	dma_fence_put(fence);
545
546	kunit_info(test, "Restore vram buffer object\n");
547	ret = xe_bo_validate(vram_bo, NULL, false, exec);
548	if (ret) {
549		KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret);
550		return;
551	}
552
553	/* Sync all migration blits */
554	timeout = dma_resv_wait_timeout(vram_bo->ttm.base.resv,
555					DMA_RESV_USAGE_KERNEL,
556					true,
557					5 * HZ);
558	if (timeout <= 0) {
559		KUNIT_FAIL(test, "Failed to sync bo eviction.\n");
560		return;
561	}
562
563	ret = xe_bo_vmap(vram_bo);
564	if (ret) {
565		KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
566		return;
567	}
568
569	retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
570	check(retval, expected, "Restored value must be equal to initial value", test);
571	retval = xe_map_rd(xe, &vram_bo->vmap, xe_bo_size(vram_bo) - 8, u64);
572	check(retval, expected, "Restored value must be equal to initial value", test);
573
574	fence = blt_copy(tile, vram_bo, ccs_bo,
575			 true, "Blit surf copy from vram to sysmem", test);
576	if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) {
577		retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64);
578		check(retval, 0, "Clear ccs data first value", test);
579		retval = xe_map_rd(xe, &ccs_bo->vmap, xe_bo_size(ccs_bo) - 8, u64);
580		check(retval, 0, "Clear ccs data last value", test);
581	}
582	dma_fence_put(fence);
583}
584
585static void test_clear(struct xe_device *xe, struct xe_tile *tile,
586		       struct xe_bo *sys_bo, struct xe_bo *vram_bo, struct kunit *test)
587{
588	struct dma_fence *fence;
589	u64 expected, retval;
590
591	expected = 0xd0d0d0d0d0d0d0d0;
592	xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, xe_bo_size(sys_bo));
593
594	fence = blt_copy(tile, sys_bo, vram_bo, false, "Blit copy from sysmem to vram", test);
595	if (!sanity_fence_failed(xe, fence, "Blit copy from sysmem to vram", test)) {
596		retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
597		if (retval == expected)
598			KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n");
599	}
600	dma_fence_put(fence);
601
602	fence = blt_copy(tile, vram_bo, sys_bo, false, "Blit copy from vram to sysmem", test);
603	if (!sanity_fence_failed(xe, fence, "Blit copy from vram to sysmem", test)) {
604		retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
605		check(retval, expected, "Decompressed value must be equal to initial value", test);
606		retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64);
607		check(retval, expected, "Decompressed value must be equal to initial value", test);
608	}
609	dma_fence_put(fence);
610
611	kunit_info(test, "Clear vram buffer object\n");
612	expected = 0x0000000000000000;
613	fence = xe_migrate_clear(tile->migrate, vram_bo, vram_bo->ttm.resource,
614				 XE_MIGRATE_CLEAR_FLAG_FULL);
615	if (sanity_fence_failed(xe, fence, "Clear vram_bo", test))
616		return;
617	dma_fence_put(fence);
618
619	fence = blt_copy(tile, vram_bo, sys_bo,
620			 false, "Blit copy from vram to sysmem", test);
621	if (!sanity_fence_failed(xe, fence, "Clear main buffer data", test)) {
622		retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
623		check(retval, expected, "Clear main buffer first value", test);
624		retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64);
625		check(retval, expected, "Clear main buffer last value", test);
626	}
627	dma_fence_put(fence);
628
629	fence = blt_copy(tile, vram_bo, sys_bo,
630			 true, "Blit surf copy from vram to sysmem", test);
631	if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) {
632		retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
633		check(retval, expected, "Clear ccs data first value", test);
634		retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64);
635		check(retval, expected, "Clear ccs data last value", test);
636	}
637	dma_fence_put(fence);
638}
639
640static void validate_ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile,
641				       struct kunit *test)
642{
643	struct xe_bo *sys_bo, *vram_bo = NULL, *ccs_bo = NULL;
644	unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
645	struct drm_exec *exec;
646	long ret;
647
648	sys_bo = xe_bo_create_user(xe, NULL, SZ_4M,
649				   DRM_XE_GEM_CPU_CACHING_WC,
650				   XE_BO_FLAG_SYSTEM |
651				   XE_BO_FLAG_NEEDS_CPU_ACCESS |
652				   XE_BO_FLAG_PINNED, NULL);
653
654	if (IS_ERR(sys_bo)) {
655		KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
656			   PTR_ERR(sys_bo));
657		return;
658	}
659
660	exec = XE_VALIDATION_OPT_OUT;
661	xe_bo_lock(sys_bo, false);
662	ret = xe_bo_validate(sys_bo, NULL, false, exec);
663	if (ret) {
664		KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret);
665		goto free_sysbo;
666	}
667
668	ret = xe_bo_vmap(sys_bo);
669	if (ret) {
670		KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret);
671		goto free_sysbo;
672	}
673	xe_bo_unlock(sys_bo);
674
675	ccs_bo = xe_bo_create_user(xe, NULL, SZ_4M,
676				   DRM_XE_GEM_CPU_CACHING_WC,
677				   bo_flags | XE_BO_FLAG_NEEDS_CPU_ACCESS |
678				   XE_BO_FLAG_PINNED, NULL);
679
680	if (IS_ERR(ccs_bo)) {
681		KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
682			   PTR_ERR(ccs_bo));
683		return;
684	}
685
686	xe_bo_lock(ccs_bo, false);
687	ret = xe_bo_validate(ccs_bo, NULL, false, exec);
688	if (ret) {
689		KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret);
690		goto free_ccsbo;
691	}
692
693	ret = xe_bo_vmap(ccs_bo);
694	if (ret) {
695		KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret);
696		goto free_ccsbo;
697	}
698	xe_bo_unlock(ccs_bo);
699
700	vram_bo = xe_bo_create_user(xe, NULL, SZ_4M,
701				    DRM_XE_GEM_CPU_CACHING_WC,
702				    bo_flags | XE_BO_FLAG_NEEDS_CPU_ACCESS |
703				    XE_BO_FLAG_PINNED, NULL);
704	if (IS_ERR(vram_bo)) {
705		KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
706			   PTR_ERR(vram_bo));
707		return;
708	}
709
710	xe_bo_lock(vram_bo, false);
711	ret = xe_bo_validate(vram_bo, NULL, false, exec);
712	if (ret) {
713		KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret);
714		goto free_vrambo;
715	}
716
717	ret = xe_bo_vmap(vram_bo);
718	if (ret) {
719		KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
720		goto free_vrambo;
721	}
722
723	test_clear(xe, tile, sys_bo, vram_bo, test);
724	test_migrate(xe, tile, sys_bo, vram_bo, ccs_bo, exec, test);
725	xe_bo_unlock(vram_bo);
726
727	xe_bo_lock(vram_bo, false);
728	xe_bo_vunmap(vram_bo);
729	xe_bo_unlock(vram_bo);
730
731	xe_bo_lock(ccs_bo, false);
732	xe_bo_vunmap(ccs_bo);
733	xe_bo_unlock(ccs_bo);
734
735	xe_bo_lock(sys_bo, false);
736	xe_bo_vunmap(sys_bo);
737	xe_bo_unlock(sys_bo);
738free_vrambo:
739	xe_bo_put(vram_bo);
740free_ccsbo:
741	xe_bo_put(ccs_bo);
742free_sysbo:
743	xe_bo_put(sys_bo);
744}
745
746static int validate_ccs_test_run_device(struct xe_device *xe)
747{
748	struct kunit *test = kunit_get_current_test();
749	struct xe_tile *tile;
750	int id;
751
752	if (!xe_device_has_flat_ccs(xe)) {
753		kunit_skip(test, "non-flat-ccs device\n");
754		return 0;
755	}
756
757	if (!(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))) {
758		kunit_skip(test, "non-xe2 discrete device\n");
759		return 0;
760	}
761
762	xe_pm_runtime_get(xe);
763
764	for_each_tile(tile, xe, id)
765		validate_ccs_test_run_tile(xe, tile, test);
766
767	xe_pm_runtime_put(xe);
768
769	return 0;
770}
771
772static void xe_validate_ccs_kunit(struct kunit *test)
773{
774	struct xe_device *xe = test->priv;
775
776	validate_ccs_test_run_device(xe);
777}
778
779static struct kunit_case xe_migrate_tests[] = {
780	KUNIT_CASE_PARAM(xe_migrate_sanity_kunit, xe_pci_live_device_gen_param),
781	KUNIT_CASE_PARAM(xe_validate_ccs_kunit, xe_pci_live_device_gen_param),
782	{}
783};
784
785VISIBLE_IF_KUNIT
786struct kunit_suite xe_migrate_test_suite = {
787	.name = "xe_migrate",
788	.test_cases = xe_migrate_tests,
789	.init = xe_kunit_helper_xe_device_live_test_init,
790};
791EXPORT_SYMBOL_IF_KUNIT(xe_migrate_test_suite);