i915_gem_mman.c

   1/*
   2 * SPDX-License-Identifier: MIT
   3 *
   4 * Copyright © 2016 Intel Corporation
   5 */
   6
   7#include <linux/highmem.h>
   8#include <linux/prime_numbers.h>
   9
  10#include <drm/drm_print.h>
  11
  12#include "gem/i915_gem_internal.h"
  13#include "gem/i915_gem_lmem.h"
  14#include "gem/i915_gem_region.h"
  15#include "gem/i915_gem_ttm.h"
  16#include "gem/i915_gem_ttm_move.h"
  17#include "gt/intel_engine_pm.h"
  18#include "gt/intel_gpu_commands.h"
  19#include "gt/intel_gt.h"
  20#include "gt/intel_gt_pm.h"
  21#include "gt/intel_migrate.h"
  22#include "i915_reg.h"
  23#include "i915_ttm_buddy_manager.h"
  24
  25#include "huge_gem_object.h"
  26#include "i915_selftest.h"
  27#include "selftests/i915_random.h"
  28#include "selftests/igt_flush_test.h"
  29#include "selftests/igt_reset.h"
  30#include "selftests/igt_mmap.h"
  31
  32struct tile {
  33	unsigned int width;
  34	unsigned int height;
  35	unsigned int stride;
  36	unsigned int size;
  37	unsigned int tiling;
  38	unsigned int swizzle;
  39};
  40
  41static u64 swizzle_bit(unsigned int bit, u64 offset)
  42{
  43	return (offset & BIT_ULL(bit)) >> (bit - 6);
  44}
  45
  46static u64 tiled_offset(const struct tile *tile, u64 v)
  47{
  48	u64 x, y;
  49
  50	if (tile->tiling == I915_TILING_NONE)
  51		return v;
  52
  53	y = div64_u64_rem(v, tile->stride, &x);
  54	v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height;
  55
  56	if (tile->tiling == I915_TILING_X) {
  57		v += y * tile->width;
  58		v += div64_u64_rem(x, tile->width, &x) << tile->size;
  59		v += x;
  60	} else if (tile->width == 128) {
  61		const unsigned int ytile_span = 16;
  62		const unsigned int ytile_height = 512;
  63
  64		v += y * ytile_span;
  65		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
  66		v += x;
  67	} else {
  68		const unsigned int ytile_span = 32;
  69		const unsigned int ytile_height = 256;
  70
  71		v += y * ytile_span;
  72		v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
  73		v += x;
  74	}
  75
  76	switch (tile->swizzle) {
  77	case I915_BIT_6_SWIZZLE_9:
  78		v ^= swizzle_bit(9, v);
  79		break;
  80	case I915_BIT_6_SWIZZLE_9_10:
  81		v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
  82		break;
  83	case I915_BIT_6_SWIZZLE_9_11:
  84		v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
  85		break;
  86	case I915_BIT_6_SWIZZLE_9_10_11:
  87		v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
  88		break;
  89	}
  90
  91	return v;
  92}
  93
  94static int check_partial_mapping(struct drm_i915_gem_object *obj,
  95				 const struct tile *tile,
  96				 struct rnd_state *prng)
  97{
  98	const unsigned long npages = obj->base.size / PAGE_SIZE;
  99	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 100	struct i915_gtt_view view;
 101	struct i915_vma *vma;
 102	unsigned long offset;
 103	unsigned long page;
 104	u32 __iomem *io;
 105	struct page *p;
 106	unsigned int n;
 107	u32 *cpu;
 108	int err;
 109
 110	err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
 111	if (err) {
 112		pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
 113		       tile->tiling, tile->stride, err);
 114		return err;
 115	}
 116
 117	GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
 118	GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
 119
 120	i915_gem_object_lock(obj, NULL);
 121	err = i915_gem_object_set_to_gtt_domain(obj, true);
 122	i915_gem_object_unlock(obj);
 123	if (err) {
 124		pr_err("Failed to flush to GTT write domain; err=%d\n", err);
 125		return err;
 126	}
 127
 128	page = i915_prandom_u32_max_state(npages, prng);
 129	view = compute_partial_view(obj, page, MIN_CHUNK_PAGES);
 130
 131	vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
 132	if (IS_ERR(vma)) {
 133		pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
 134		       page, (int)PTR_ERR(vma));
 135		return PTR_ERR(vma);
 136	}
 137
 138	n = page - view.partial.offset;
 139	GEM_BUG_ON(n >= view.partial.size);
 140
 141	io = i915_vma_pin_iomap(vma);
 142	i915_vma_unpin(vma);
 143	if (IS_ERR(io)) {
 144		pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
 145		       page, (int)PTR_ERR(io));
 146		err = PTR_ERR(io);
 147		goto out;
 148	}
 149
 150	iowrite32(page, io + n * PAGE_SIZE / sizeof(*io));
 151	i915_vma_unpin_iomap(vma);
 152
 153	offset = tiled_offset(tile, page << PAGE_SHIFT);
 154	if (offset >= obj->base.size)
 155		goto out;
 156
 157	intel_gt_flush_ggtt_writes(to_gt(i915));
 158
 159	p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
 160	cpu = kmap(p) + offset_in_page(offset);
 161	drm_clflush_virt_range(cpu, sizeof(*cpu));
 162	if (*cpu != (u32)page) {
 163		pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%lu + %u [0x%lx]) of 0x%x, found 0x%x\n",
 164		       page, n,
 165		       view.partial.offset,
 166		       view.partial.size,
 167		       vma->size >> PAGE_SHIFT,
 168		       tile->tiling ? tile_row_pages(obj) : 0,
 169		       vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
 170		       offset >> PAGE_SHIFT,
 171		       (unsigned int)offset_in_page(offset),
 172		       offset,
 173		       (u32)page, *cpu);
 174		err = -EINVAL;
 175	}
 176	*cpu = 0;
 177	drm_clflush_virt_range(cpu, sizeof(*cpu));
 178	kunmap(p);
 179
 180out:
 181	i915_gem_object_lock(obj, NULL);
 182	i915_vma_destroy(vma);
 183	i915_gem_object_unlock(obj);
 184	return err;
 185}
 186
 187static int check_partial_mappings(struct drm_i915_gem_object *obj,
 188				  const struct tile *tile,
 189				  unsigned long end_time)
 190{
 191	const unsigned int nreal = obj->scratch / PAGE_SIZE;
 192	const unsigned long npages = obj->base.size / PAGE_SIZE;
 193	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 194	struct i915_vma *vma;
 195	unsigned long page;
 196	int err;
 197
 198	err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride);
 199	if (err) {
 200		pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n",
 201		       tile->tiling, tile->stride, err);
 202		return err;
 203	}
 204
 205	GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling);
 206	GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride);
 207
 208	i915_gem_object_lock(obj, NULL);
 209	err = i915_gem_object_set_to_gtt_domain(obj, true);
 210	i915_gem_object_unlock(obj);
 211	if (err) {
 212		pr_err("Failed to flush to GTT write domain; err=%d\n", err);
 213		return err;
 214	}
 215
 216	for_each_prime_number_from(page, 1, npages) {
 217		struct i915_gtt_view view =
 218			compute_partial_view(obj, page, MIN_CHUNK_PAGES);
 219		unsigned long offset;
 220		u32 __iomem *io;
 221		struct page *p;
 222		unsigned int n;
 223		u32 *cpu;
 224
 225		GEM_BUG_ON(view.partial.size > nreal);
 226		cond_resched();
 227
 228		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
 229		if (IS_ERR(vma)) {
 230			pr_err("Failed to pin partial view: offset=%lu; err=%d\n",
 231			       page, (int)PTR_ERR(vma));
 232			return PTR_ERR(vma);
 233		}
 234
 235		n = page - view.partial.offset;
 236		GEM_BUG_ON(n >= view.partial.size);
 237
 238		io = i915_vma_pin_iomap(vma);
 239		i915_vma_unpin(vma);
 240		if (IS_ERR(io)) {
 241			pr_err("Failed to iomap partial view: offset=%lu; err=%d\n",
 242			       page, (int)PTR_ERR(io));
 243			return PTR_ERR(io);
 244		}
 245
 246		iowrite32(page, io + n * PAGE_SIZE / sizeof(*io));
 247		i915_vma_unpin_iomap(vma);
 248
 249		offset = tiled_offset(tile, page << PAGE_SHIFT);
 250		if (offset >= obj->base.size)
 251			continue;
 252
 253		intel_gt_flush_ggtt_writes(to_gt(i915));
 254
 255		p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
 256		cpu = kmap(p) + offset_in_page(offset);
 257		drm_clflush_virt_range(cpu, sizeof(*cpu));
 258		if (*cpu != (u32)page) {
 259			pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%lu + %u [0x%lx]) of 0x%x, found 0x%x\n",
 260			       page, n,
 261			       view.partial.offset,
 262			       view.partial.size,
 263			       vma->size >> PAGE_SHIFT,
 264			       tile->tiling ? tile_row_pages(obj) : 0,
 265			       vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride,
 266			       offset >> PAGE_SHIFT,
 267			       (unsigned int)offset_in_page(offset),
 268			       offset,
 269			       (u32)page, *cpu);
 270			err = -EINVAL;
 271		}
 272		*cpu = 0;
 273		drm_clflush_virt_range(cpu, sizeof(*cpu));
 274		kunmap(p);
 275		if (err)
 276			return err;
 277
 278		i915_gem_object_lock(obj, NULL);
 279		i915_vma_destroy(vma);
 280		i915_gem_object_unlock(obj);
 281
 282		if (igt_timeout(end_time,
 283				"%s: timed out after tiling=%d stride=%d\n",
 284				__func__, tile->tiling, tile->stride))
 285			return -EINTR;
 286	}
 287
 288	return 0;
 289}
 290
 291static unsigned int
 292setup_tile_size(struct tile *tile, struct drm_i915_private *i915)
 293{
 294	if (GRAPHICS_VER(i915) <= 2) {
 295		tile->height = 16;
 296		tile->width = 128;
 297		tile->size = 11;
 298	} else if (tile->tiling == I915_TILING_Y &&
 299		   HAS_128_BYTE_Y_TILING(i915)) {
 300		tile->height = 32;
 301		tile->width = 128;
 302		tile->size = 12;
 303	} else {
 304		tile->height = 8;
 305		tile->width = 512;
 306		tile->size = 12;
 307	}
 308
 309	if (GRAPHICS_VER(i915) < 4)
 310		return 8192 / tile->width;
 311	else if (GRAPHICS_VER(i915) < 7)
 312		return 128 * I965_FENCE_MAX_PITCH_VAL / tile->width;
 313	else
 314		return 128 * GEN7_FENCE_MAX_PITCH_VAL / tile->width;
 315}
 316
 317static int igt_partial_tiling(void *arg)
 318{
 319	const unsigned int nreal = 1 << 12; /* largest tile row x2 */
 320	struct drm_i915_private *i915 = arg;
 321	struct drm_i915_gem_object *obj;
 322	intel_wakeref_t wakeref;
 323	int tiling;
 324	int err;
 325
 326	if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
 327		return 0;
 328
 329	/* We want to check the page mapping and fencing of a large object
 330	 * mmapped through the GTT. The object we create is larger than can
 331	 * possibly be mmaped as a whole, and so we must use partial GGTT vma.
 332	 * We then check that a write through each partial GGTT vma ends up
 333	 * in the right set of pages within the object, and with the expected
 334	 * tiling, which we verify by manual swizzling.
 335	 */
 336
 337	obj = huge_gem_object(i915,
 338			      nreal << PAGE_SHIFT,
 339			      (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
 340	if (IS_ERR(obj))
 341		return PTR_ERR(obj);
 342
 343	err = i915_gem_object_pin_pages_unlocked(obj);
 344	if (err) {
 345		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
 346		       nreal, obj->base.size / PAGE_SIZE, err);
 347		goto out;
 348	}
 349
 350	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 351
 352	if (1) {
 353		IGT_TIMEOUT(end);
 354		struct tile tile;
 355
 356		tile.height = 1;
 357		tile.width = 1;
 358		tile.size = 0;
 359		tile.stride = 0;
 360		tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
 361		tile.tiling = I915_TILING_NONE;
 362
 363		err = check_partial_mappings(obj, &tile, end);
 364		if (err && err != -EINTR)
 365			goto out_unlock;
 366	}
 367
 368	for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) {
 369		IGT_TIMEOUT(end);
 370		unsigned int max_pitch;
 371		unsigned int pitch;
 372		struct tile tile;
 373
 374		if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 375			/*
 376			 * The swizzling pattern is actually unknown as it
 377			 * varies based on physical address of each page.
 378			 * See i915_gem_detect_bit_6_swizzle().
 379			 */
 380			break;
 381
 382		tile.tiling = tiling;
 383		switch (tiling) {
 384		case I915_TILING_X:
 385			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x;
 386			break;
 387		case I915_TILING_Y:
 388			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y;
 389			break;
 390		}
 391
 392		GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
 393		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
 394		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
 395			continue;
 396
 397		max_pitch = setup_tile_size(&tile, i915);
 398
 399		for (pitch = max_pitch; pitch; pitch >>= 1) {
 400			tile.stride = tile.width * pitch;
 401			err = check_partial_mappings(obj, &tile, end);
 402			if (err == -EINTR)
 403				goto next_tiling;
 404			if (err)
 405				goto out_unlock;
 406
 407			if (pitch > 2 && GRAPHICS_VER(i915) >= 4) {
 408				tile.stride = tile.width * (pitch - 1);
 409				err = check_partial_mappings(obj, &tile, end);
 410				if (err == -EINTR)
 411					goto next_tiling;
 412				if (err)
 413					goto out_unlock;
 414			}
 415
 416			if (pitch < max_pitch && GRAPHICS_VER(i915) >= 4) {
 417				tile.stride = tile.width * (pitch + 1);
 418				err = check_partial_mappings(obj, &tile, end);
 419				if (err == -EINTR)
 420					goto next_tiling;
 421				if (err)
 422					goto out_unlock;
 423			}
 424		}
 425
 426		if (GRAPHICS_VER(i915) >= 4) {
 427			for_each_prime_number(pitch, max_pitch) {
 428				tile.stride = tile.width * pitch;
 429				err = check_partial_mappings(obj, &tile, end);
 430				if (err == -EINTR)
 431					goto next_tiling;
 432				if (err)
 433					goto out_unlock;
 434			}
 435		}
 436
 437next_tiling: ;
 438	}
 439
 440out_unlock:
 441	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 442	i915_gem_object_unpin_pages(obj);
 443out:
 444	i915_gem_object_put(obj);
 445	return err;
 446}
 447
 448static int igt_smoke_tiling(void *arg)
 449{
 450	const unsigned int nreal = 1 << 12; /* largest tile row x2 */
 451	struct drm_i915_private *i915 = arg;
 452	struct drm_i915_gem_object *obj;
 453	intel_wakeref_t wakeref;
 454	I915_RND_STATE(prng);
 455	unsigned long count;
 456	IGT_TIMEOUT(end);
 457	int err;
 458
 459	if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
 460		return 0;
 461
 462	/*
 463	 * igt_partial_tiling() does an exhastive check of partial tiling
 464	 * chunking, but will undoubtably run out of time. Here, we do a
 465	 * randomised search and hope over many runs of 1s with different
 466	 * seeds we will do a thorough check.
 467	 *
 468	 * Remember to look at the st_seed if we see a flip-flop in BAT!
 469	 */
 470
 471	if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 472		return 0;
 473
 474	obj = huge_gem_object(i915,
 475			      nreal << PAGE_SHIFT,
 476			      (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT);
 477	if (IS_ERR(obj))
 478		return PTR_ERR(obj);
 479
 480	err = i915_gem_object_pin_pages_unlocked(obj);
 481	if (err) {
 482		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
 483		       nreal, obj->base.size / PAGE_SIZE, err);
 484		goto out;
 485	}
 486
 487	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 488
 489	count = 0;
 490	do {
 491		struct tile tile;
 492
 493		tile.tiling =
 494			i915_prandom_u32_max_state(I915_TILING_Y + 1, &prng);
 495		switch (tile.tiling) {
 496		case I915_TILING_NONE:
 497			tile.height = 1;
 498			tile.width = 1;
 499			tile.size = 0;
 500			tile.stride = 0;
 501			tile.swizzle = I915_BIT_6_SWIZZLE_NONE;
 502			break;
 503
 504		case I915_TILING_X:
 505			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x;
 506			break;
 507		case I915_TILING_Y:
 508			tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y;
 509			break;
 510		}
 511
 512		if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 ||
 513		    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
 514			continue;
 515
 516		if (tile.tiling != I915_TILING_NONE) {
 517			unsigned int max_pitch = setup_tile_size(&tile, i915);
 518
 519			tile.stride =
 520				i915_prandom_u32_max_state(max_pitch, &prng);
 521			tile.stride = (1 + tile.stride) * tile.width;
 522			if (GRAPHICS_VER(i915) < 4)
 523				tile.stride = rounddown_pow_of_two(tile.stride);
 524		}
 525
 526		err = check_partial_mapping(obj, &tile, &prng);
 527		if (err)
 528			break;
 529
 530		count++;
 531	} while (!__igt_timeout(end, NULL));
 532
 533	pr_info("%s: Completed %lu trials\n", __func__, count);
 534
 535	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 536	i915_gem_object_unpin_pages(obj);
 537out:
 538	i915_gem_object_put(obj);
 539	return err;
 540}
 541
 542static int make_obj_busy(struct drm_i915_gem_object *obj)
 543{
 544	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 545	struct intel_engine_cs *engine;
 546
 547	for_each_uabi_engine(engine, i915) {
 548		struct i915_request *rq;
 549		struct i915_vma *vma;
 550		struct i915_gem_ww_ctx ww;
 551		int err;
 552
 553		vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
 554		if (IS_ERR(vma))
 555			return PTR_ERR(vma);
 556
 557		i915_gem_ww_ctx_init(&ww, false);
 558retry:
 559		err = i915_gem_object_lock(obj, &ww);
 560		if (!err)
 561			err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
 562		if (err)
 563			goto err;
 564
 565		rq = intel_engine_create_kernel_request(engine);
 566		if (IS_ERR(rq)) {
 567			err = PTR_ERR(rq);
 568			goto err_unpin;
 569		}
 570
 571		err = i915_vma_move_to_active(vma, rq,
 572					      EXEC_OBJECT_WRITE);
 573
 574		i915_request_add(rq);
 575err_unpin:
 576		i915_vma_unpin(vma);
 577err:
 578		if (err == -EDEADLK) {
 579			err = i915_gem_ww_ctx_backoff(&ww);
 580			if (!err)
 581				goto retry;
 582		}
 583		i915_gem_ww_ctx_fini(&ww);
 584		if (err)
 585			return err;
 586	}
 587
 588	i915_gem_object_put(obj); /* leave it only alive via its active ref */
 589	return 0;
 590}
 591
 592static enum i915_mmap_type default_mapping(struct drm_i915_private *i915)
 593{
 594	if (HAS_LMEM(i915))
 595		return I915_MMAP_TYPE_FIXED;
 596
 597	return I915_MMAP_TYPE_GTT;
 598}
 599
 600static struct drm_i915_gem_object *
 601create_sys_or_internal(struct drm_i915_private *i915,
 602		       unsigned long size)
 603{
 604	if (HAS_LMEM(i915)) {
 605		struct intel_memory_region *sys_region =
 606			i915->mm.regions[INTEL_REGION_SMEM];
 607
 608		return __i915_gem_object_create_user(i915, size, &sys_region, 1);
 609	}
 610
 611	return i915_gem_object_create_internal(i915, size);
 612}
 613
 614static bool assert_mmap_offset(struct drm_i915_private *i915,
 615			       unsigned long size,
 616			       int expected)
 617{
 618	struct drm_i915_gem_object *obj;
 619	u64 offset;
 620	int ret;
 621
 622	obj = create_sys_or_internal(i915, size);
 623	if (IS_ERR(obj))
 624		return expected && expected == PTR_ERR(obj);
 625
 626	ret = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL);
 627	i915_gem_object_put(obj);
 628
 629	return ret == expected;
 630}
 631
 632static void disable_retire_worker(struct drm_i915_private *i915)
 633{
 634	i915_gem_driver_unregister__shrinker(i915);
 635	intel_gt_pm_get_untracked(to_gt(i915));
 636	cancel_delayed_work_sync(&to_gt(i915)->requests.retire_work);
 637}
 638
 639static void restore_retire_worker(struct drm_i915_private *i915)
 640{
 641	igt_flush_test(i915);
 642	intel_gt_pm_put_untracked(to_gt(i915));
 643	i915_gem_driver_register__shrinker(i915);
 644}
 645
 646static void mmap_offset_lock(struct drm_i915_private *i915)
 647	__acquires(&i915->drm.vma_offset_manager->vm_lock)
 648{
 649	write_lock(&i915->drm.vma_offset_manager->vm_lock);
 650}
 651
 652static void mmap_offset_unlock(struct drm_i915_private *i915)
 653	__releases(&i915->drm.vma_offset_manager->vm_lock)
 654{
 655	write_unlock(&i915->drm.vma_offset_manager->vm_lock);
 656}
 657
 658static int igt_mmap_offset_exhaustion(void *arg)
 659{
 660	struct drm_i915_private *i915 = arg;
 661	struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm;
 662	struct drm_i915_gem_object *obj;
 663	struct drm_mm_node *hole, *next;
 664	int loop, err = 0;
 665	u64 offset;
 666	int enospc = HAS_LMEM(i915) ? -ENXIO : -ENOSPC;
 667
 668	/* Disable background reaper */
 669	disable_retire_worker(i915);
 670	GEM_BUG_ON(!to_gt(i915)->awake);
 671	intel_gt_retire_requests(to_gt(i915));
 672	i915_gem_drain_freed_objects(i915);
 673
 674	/* Trim the device mmap space to only a page */
 675	mmap_offset_lock(i915);
 676	loop = 1; /* PAGE_SIZE units */
 677	list_for_each_entry_safe(hole, next, &mm->hole_stack, hole_stack) {
 678		struct drm_mm_node *resv;
 679
 680		resv = kzalloc(sizeof(*resv), GFP_NOWAIT);
 681		if (!resv) {
 682			err = -ENOMEM;
 683			goto out_park;
 684		}
 685
 686		resv->start = drm_mm_hole_node_start(hole) + loop;
 687		resv->size = hole->hole_size - loop;
 688		resv->color = -1ul;
 689		loop = 0;
 690
 691		if (!resv->size) {
 692			kfree(resv);
 693			continue;
 694		}
 695
 696		pr_debug("Reserving hole [%llx + %llx]\n",
 697			 resv->start, resv->size);
 698
 699		err = drm_mm_reserve_node(mm, resv);
 700		if (err) {
 701			pr_err("Failed to trim VMA manager, err=%d\n", err);
 702			kfree(resv);
 703			goto out_park;
 704		}
 705	}
 706	GEM_BUG_ON(!list_is_singular(&mm->hole_stack));
 707	mmap_offset_unlock(i915);
 708
 709	/* Just fits! */
 710	if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) {
 711		pr_err("Unable to insert object into single page hole\n");
 712		err = -EINVAL;
 713		goto out;
 714	}
 715
 716	/* Too large */
 717	if (!assert_mmap_offset(i915, 2 * PAGE_SIZE, enospc)) {
 718		pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n");
 719		err = -EINVAL;
 720		goto out;
 721	}
 722
 723	/* Fill the hole, further allocation attempts should then fail */
 724	obj = create_sys_or_internal(i915, PAGE_SIZE);
 725	if (IS_ERR(obj)) {
 726		err = PTR_ERR(obj);
 727		pr_err("Unable to create object for reclaimed hole\n");
 728		goto out;
 729	}
 730
 731	err = __assign_mmap_offset(obj, default_mapping(i915), &offset, NULL);
 732	if (err) {
 733		pr_err("Unable to insert object into reclaimed hole\n");
 734		goto err_obj;
 735	}
 736
 737	if (!assert_mmap_offset(i915, PAGE_SIZE, enospc)) {
 738		pr_err("Unexpectedly succeeded in inserting object into no holes!\n");
 739		err = -EINVAL;
 740		goto err_obj;
 741	}
 742
 743	i915_gem_object_put(obj);
 744
 745	/* Now fill with busy dead objects that we expect to reap */
 746	for (loop = 0; loop < 3; loop++) {
 747		if (intel_gt_is_wedged(to_gt(i915)))
 748			break;
 749
 750		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 751		if (IS_ERR(obj)) {
 752			err = PTR_ERR(obj);
 753			goto out;
 754		}
 755
 756		err = make_obj_busy(obj);
 757		if (err) {
 758			pr_err("[loop %d] Failed to busy the object\n", loop);
 759			goto err_obj;
 760		}
 761	}
 762
 763out:
 764	mmap_offset_lock(i915);
 765out_park:
 766	drm_mm_for_each_node_safe(hole, next, mm) {
 767		if (hole->color != -1ul)
 768			continue;
 769
 770		drm_mm_remove_node(hole);
 771		kfree(hole);
 772	}
 773	mmap_offset_unlock(i915);
 774	restore_retire_worker(i915);
 775	return err;
 776err_obj:
 777	i915_gem_object_put(obj);
 778	goto out;
 779}
 780
 781static int gtt_set(struct drm_i915_gem_object *obj)
 782{
 783	intel_wakeref_t wakeref;
 784	struct i915_vma *vma;
 785	void __iomem *map;
 786	int err = 0;
 787
 788	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
 789	if (IS_ERR(vma))
 790		return PTR_ERR(vma);
 791
 792	wakeref = intel_gt_pm_get(vma->vm->gt);
 793	map = i915_vma_pin_iomap(vma);
 794	i915_vma_unpin(vma);
 795	if (IS_ERR(map)) {
 796		err = PTR_ERR(map);
 797		goto out;
 798	}
 799
 800	memset_io(map, POISON_INUSE, obj->base.size);
 801	i915_vma_unpin_iomap(vma);
 802
 803out:
 804	intel_gt_pm_put(vma->vm->gt, wakeref);
 805	return err;
 806}
 807
 808static int gtt_check(struct drm_i915_gem_object *obj)
 809{
 810	intel_wakeref_t wakeref;
 811	struct i915_vma *vma;
 812	void __iomem *map;
 813	int err = 0;
 814
 815	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
 816	if (IS_ERR(vma))
 817		return PTR_ERR(vma);
 818
 819	wakeref = intel_gt_pm_get(vma->vm->gt);
 820	map = i915_vma_pin_iomap(vma);
 821	i915_vma_unpin(vma);
 822	if (IS_ERR(map)) {
 823		err = PTR_ERR(map);
 824		goto out;
 825	}
 826
 827	if (memchr_inv((void __force *)map, POISON_FREE, obj->base.size)) {
 828		pr_err("%s: Write via mmap did not land in backing store (GTT)\n",
 829		       obj->mm.region->name);
 830		err = -EINVAL;
 831	}
 832	i915_vma_unpin_iomap(vma);
 833
 834out:
 835	intel_gt_pm_put(vma->vm->gt, wakeref);
 836	return err;
 837}
 838
 839static int wc_set(struct drm_i915_gem_object *obj)
 840{
 841	void *vaddr;
 842
 843	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 844	if (IS_ERR(vaddr))
 845		return PTR_ERR(vaddr);
 846
 847	memset(vaddr, POISON_INUSE, obj->base.size);
 848	i915_gem_object_flush_map(obj);
 849	i915_gem_object_unpin_map(obj);
 850
 851	return 0;
 852}
 853
 854static int wc_check(struct drm_i915_gem_object *obj)
 855{
 856	void *vaddr;
 857	int err = 0;
 858
 859	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 860	if (IS_ERR(vaddr))
 861		return PTR_ERR(vaddr);
 862
 863	if (memchr_inv(vaddr, POISON_FREE, obj->base.size)) {
 864		pr_err("%s: Write via mmap did not land in backing store (WC)\n",
 865		       obj->mm.region->name);
 866		err = -EINVAL;
 867	}
 868	i915_gem_object_unpin_map(obj);
 869
 870	return err;
 871}
 872
 873static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type)
 874{
 875	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 876	bool no_map;
 877
 878	if (obj->ops->mmap_offset)
 879		return type == I915_MMAP_TYPE_FIXED;
 880	else if (type == I915_MMAP_TYPE_FIXED)
 881		return false;
 882
 883	if (type == I915_MMAP_TYPE_GTT &&
 884	    !i915_ggtt_has_aperture(to_gt(i915)->ggtt))
 885		return false;
 886
 887	i915_gem_object_lock(obj, NULL);
 888	no_map = (type != I915_MMAP_TYPE_GTT &&
 889		  !i915_gem_object_has_struct_page(obj) &&
 890		  !i915_gem_object_has_iomem(obj));
 891	i915_gem_object_unlock(obj);
 892
 893	return !no_map;
 894}
 895
 896#define expand32(x) (((x) << 0) | ((x) << 8) | ((x) << 16) | ((x) << 24))
 897static int __igt_mmap(struct drm_i915_private *i915,
 898		      struct drm_i915_gem_object *obj,
 899		      enum i915_mmap_type type)
 900{
 901	struct vm_area_struct *area;
 902	unsigned long addr;
 903	int err, i;
 904	u64 offset;
 905
 906	if (!can_mmap(obj, type))
 907		return 0;
 908
 909	err = wc_set(obj);
 910	if (err == -ENXIO)
 911		err = gtt_set(obj);
 912	if (err)
 913		return err;
 914
 915	err = __assign_mmap_offset(obj, type, &offset, NULL);
 916	if (err)
 917		return err;
 918
 919	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
 920	if (IS_ERR_VALUE(addr))
 921		return addr;
 922
 923	pr_debug("igt_mmap(%s, %d) @ %lx\n", obj->mm.region->name, type, addr);
 924
 925	mmap_read_lock(current->mm);
 926	area = vma_lookup(current->mm, addr);
 927	mmap_read_unlock(current->mm);
 928	if (!area) {
 929		pr_err("%s: Did not create a vm_area_struct for the mmap\n",
 930		       obj->mm.region->name);
 931		err = -EINVAL;
 932		goto out_unmap;
 933	}
 934
 935	for (i = 0; i < obj->base.size / sizeof(u32); i++) {
 936		u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux)));
 937		u32 x;
 938
 939		if (get_user(x, ux)) {
 940			pr_err("%s: Unable to read from mmap, offset:%zd\n",
 941			       obj->mm.region->name, i * sizeof(x));
 942			err = -EFAULT;
 943			goto out_unmap;
 944		}
 945
 946		if (x != expand32(POISON_INUSE)) {
 947			pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n",
 948			       obj->mm.region->name,
 949			       i * sizeof(x), x, expand32(POISON_INUSE));
 950			err = -EINVAL;
 951			goto out_unmap;
 952		}
 953
 954		x = expand32(POISON_FREE);
 955		if (put_user(x, ux)) {
 956			pr_err("%s: Unable to write to mmap, offset:%zd\n",
 957			       obj->mm.region->name, i * sizeof(x));
 958			err = -EFAULT;
 959			goto out_unmap;
 960		}
 961	}
 962
 963	if (type == I915_MMAP_TYPE_GTT)
 964		intel_gt_flush_ggtt_writes(to_gt(i915));
 965
 966	err = wc_check(obj);
 967	if (err == -ENXIO)
 968		err = gtt_check(obj);
 969out_unmap:
 970	vm_munmap(addr, obj->base.size);
 971	return err;
 972}
 973
 974static int igt_mmap(void *arg)
 975{
 976	struct drm_i915_private *i915 = arg;
 977	struct intel_memory_region *mr;
 978	enum intel_region_id id;
 979
 980	for_each_memory_region(mr, i915, id) {
 981		unsigned long sizes[] = {
 982			PAGE_SIZE,
 983			mr->min_page_size,
 984			SZ_4M,
 985		};
 986		int i;
 987
 988		if (mr->private)
 989			continue;
 990
 991		for (i = 0; i < ARRAY_SIZE(sizes); i++) {
 992			struct drm_i915_gem_object *obj;
 993			int err;
 994
 995			obj = __i915_gem_object_create_user(i915, sizes[i], &mr, 1);
 996			if (obj == ERR_PTR(-ENODEV))
 997				continue;
 998
 999			if (IS_ERR(obj))
1000				return PTR_ERR(obj);
1001
1002			err = __igt_mmap(i915, obj, I915_MMAP_TYPE_GTT);
1003			if (err == 0)
1004				err = __igt_mmap(i915, obj, I915_MMAP_TYPE_WC);
1005			if (err == 0)
1006				err = __igt_mmap(i915, obj, I915_MMAP_TYPE_FIXED);
1007
1008			i915_gem_object_put(obj);
1009			if (err)
1010				return err;
1011		}
1012	}
1013
1014	return 0;
1015}
1016
1017static void igt_close_objects(struct drm_i915_private *i915,
1018			      struct list_head *objects)
1019{
1020	struct drm_i915_gem_object *obj, *on;
1021
1022	list_for_each_entry_safe(obj, on, objects, st_link) {
1023		i915_gem_object_lock(obj, NULL);
1024		if (i915_gem_object_has_pinned_pages(obj))
1025			i915_gem_object_unpin_pages(obj);
1026		/* No polluting the memory region between tests */
1027		__i915_gem_object_put_pages(obj);
1028		i915_gem_object_unlock(obj);
1029		list_del(&obj->st_link);
1030		i915_gem_object_put(obj);
1031	}
1032
1033	cond_resched();
1034
1035	i915_gem_drain_freed_objects(i915);
1036}
1037
1038static void igt_make_evictable(struct list_head *objects)
1039{
1040	struct drm_i915_gem_object *obj;
1041
1042	list_for_each_entry(obj, objects, st_link) {
1043		i915_gem_object_lock(obj, NULL);
1044		if (i915_gem_object_has_pinned_pages(obj))
1045			i915_gem_object_unpin_pages(obj);
1046		i915_gem_object_unlock(obj);
1047	}
1048
1049	cond_resched();
1050}
1051
1052static int igt_fill_mappable(struct intel_memory_region *mr,
1053			     struct list_head *objects)
1054{
1055	u64 size, total;
1056	int err;
1057
1058	total = 0;
1059	size = resource_size(&mr->io);
1060	do {
1061		struct drm_i915_gem_object *obj;
1062
1063		obj = i915_gem_object_create_region(mr, size, 0, 0);
1064		if (IS_ERR(obj)) {
1065			err = PTR_ERR(obj);
1066			goto err_close;
1067		}
1068
1069		list_add(&obj->st_link, objects);
1070
1071		err = i915_gem_object_pin_pages_unlocked(obj);
1072		if (err) {
1073			if (err != -ENXIO && err != -ENOMEM)
1074				goto err_close;
1075
1076			if (size == mr->min_page_size) {
1077				err = 0;
1078				break;
1079			}
1080
1081			size >>= 1;
1082			continue;
1083		}
1084
1085		total += obj->base.size;
1086	} while (1);
1087
1088	pr_info("%s filled=%lluMiB\n", __func__, total >> 20);
1089	return 0;
1090
1091err_close:
1092	igt_close_objects(mr->i915, objects);
1093	return err;
1094}
1095
1096static int ___igt_mmap_migrate(struct drm_i915_private *i915,
1097			       struct drm_i915_gem_object *obj,
1098			       unsigned long addr,
1099			       bool unfaultable)
1100{
1101	int i;
1102
1103	pr_info("igt_mmap(%s, %d) @ %lx\n",
1104		obj->mm.region->name, I915_MMAP_TYPE_FIXED, addr);
1105
1106	for (i = 0; i < obj->base.size / sizeof(u32); i++) {
1107		u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux)));
1108		u32 x;
1109
1110		if (get_user(x, ux)) {
1111			if (!unfaultable) {
1112				pr_err("%s: Unable to read from mmap, offset:%zd\n",
1113				       obj->mm.region->name, i * sizeof(x));
1114				return -EFAULT;
1115			}
1116
1117			continue;
1118		}
1119
1120		if (unfaultable) {
1121			pr_err("%s: Faulted unmappable memory\n",
1122			       obj->mm.region->name);
1123			return -EINVAL;
1124		}
1125
1126		if (x != expand32(POISON_INUSE)) {
1127			pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n",
1128			       obj->mm.region->name,
1129			       i * sizeof(x), x, expand32(POISON_INUSE));
1130			return -EINVAL;
1131		}
1132
1133		x = expand32(POISON_FREE);
1134		if (put_user(x, ux)) {
1135			pr_err("%s: Unable to write to mmap, offset:%zd\n",
1136			       obj->mm.region->name, i * sizeof(x));
1137			return -EFAULT;
1138		}
1139	}
1140
1141	if (unfaultable)
1142		return 0;
1143
1144	obj->flags &= ~I915_BO_ALLOC_GPU_ONLY;
1145	return wc_check(obj);
1146}
1147
1148#define IGT_MMAP_MIGRATE_TOPDOWN     (1 << 0)
1149#define IGT_MMAP_MIGRATE_FILL        (1 << 1)
1150#define IGT_MMAP_MIGRATE_EVICTABLE   (1 << 2)
1151#define IGT_MMAP_MIGRATE_UNFAULTABLE (1 << 3)
1152#define IGT_MMAP_MIGRATE_FAIL_GPU    (1 << 4)
1153static int __igt_mmap_migrate(struct intel_memory_region **placements,
1154			      int n_placements,
1155			      struct intel_memory_region *expected_mr,
1156			      unsigned int flags)
1157{
1158	struct drm_i915_private *i915 = placements[0]->i915;
1159	struct drm_i915_gem_object *obj;
1160	struct i915_request *rq = NULL;
1161	struct vm_area_struct *area;
1162	unsigned long addr;
1163	LIST_HEAD(objects);
1164	u64 offset;
1165	int err;
1166
1167	obj = __i915_gem_object_create_user(i915, PAGE_SIZE,
1168					    placements,
1169					    n_placements);
1170	if (IS_ERR(obj))
1171		return PTR_ERR(obj);
1172
1173	if (flags & IGT_MMAP_MIGRATE_TOPDOWN)
1174		obj->flags |= I915_BO_ALLOC_GPU_ONLY;
1175
1176	err = __assign_mmap_offset(obj, I915_MMAP_TYPE_FIXED, &offset, NULL);
1177	if (err)
1178		goto out_put;
1179
1180	/*
1181	 * This will eventually create a GEM context, due to opening dummy drm
1182	 * file, which needs a tiny amount of mappable device memory for the top
1183	 * level paging structures(and perhaps scratch), so make sure we
1184	 * allocate early, to avoid tears.
1185	 */
1186	addr = igt_mmap_offset(i915, offset, obj->base.size,
1187			       PROT_WRITE, MAP_SHARED);
1188	if (IS_ERR_VALUE(addr)) {
1189		err = addr;
1190		goto out_put;
1191	}
1192
1193	mmap_read_lock(current->mm);
1194	area = vma_lookup(current->mm, addr);
1195	mmap_read_unlock(current->mm);
1196	if (!area) {
1197		pr_err("%s: Did not create a vm_area_struct for the mmap\n",
1198		       obj->mm.region->name);
1199		err = -EINVAL;
1200		goto out_addr;
1201	}
1202
1203	if (flags & IGT_MMAP_MIGRATE_FILL) {
1204		err = igt_fill_mappable(placements[0], &objects);
1205		if (err)
1206			goto out_addr;
1207	}
1208
1209	err = i915_gem_object_lock(obj, NULL);
1210	if (err)
1211		goto out_addr;
1212
1213	err = i915_gem_object_pin_pages(obj);
1214	if (err) {
1215		i915_gem_object_unlock(obj);
1216		goto out_addr;
1217	}
1218
1219	err = intel_context_migrate_clear(to_gt(i915)->migrate.context, NULL,
1220					  obj->mm.pages->sgl, obj->pat_index,
1221					  i915_gem_object_is_lmem(obj),
1222					  expand32(POISON_INUSE), &rq);
1223	i915_gem_object_unpin_pages(obj);
1224	if (rq && !err) {
1225		err = dma_resv_reserve_fences(obj->base.resv, 1);
1226		if (!err)
1227			dma_resv_add_fence(obj->base.resv, &rq->fence,
1228					   DMA_RESV_USAGE_KERNEL);
1229		i915_request_put(rq);
1230	}
1231	i915_gem_object_unlock(obj);
1232	if (err)
1233		goto out_addr;
1234
1235	if (flags & IGT_MMAP_MIGRATE_EVICTABLE)
1236		igt_make_evictable(&objects);
1237
1238	if (flags & IGT_MMAP_MIGRATE_FAIL_GPU) {
1239		err = i915_gem_object_lock(obj, NULL);
1240		if (err)
1241			goto out_addr;
1242
1243		/*
1244		 * Ensure we only simulate the gpu failure when faulting the
1245		 * pages.
1246		 */
1247		err = i915_gem_object_wait_moving_fence(obj, true);
1248		i915_gem_object_unlock(obj);
1249		if (err)
1250			goto out_addr;
1251		i915_ttm_migrate_set_failure_modes(true, false);
1252	}
1253
1254	err = ___igt_mmap_migrate(i915, obj, addr,
1255				  flags & IGT_MMAP_MIGRATE_UNFAULTABLE);
1256
1257	if (!err && obj->mm.region != expected_mr) {
1258		pr_err("%s region mismatch %s\n", __func__, expected_mr->name);
1259		err = -EINVAL;
1260	}
1261
1262	if (flags & IGT_MMAP_MIGRATE_FAIL_GPU) {
1263		struct intel_gt *gt;
1264		unsigned int id;
1265
1266		i915_ttm_migrate_set_failure_modes(false, false);
1267
1268		for_each_gt(gt, i915, id) {
1269			intel_wakeref_t wakeref;
1270			bool wedged;
1271
1272			mutex_lock(&gt->reset.mutex);
1273			wedged = test_bit(I915_WEDGED, &gt->reset.flags);
1274			mutex_unlock(&gt->reset.mutex);
1275			if (!wedged) {
1276				pr_err("gt(%u) not wedged\n", id);
1277				err = -EINVAL;
1278				continue;
1279			}
1280
1281			wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1282			igt_global_reset_lock(gt);
1283			intel_gt_reset(gt, ALL_ENGINES, NULL);
1284			igt_global_reset_unlock(gt);
1285			intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1286		}
1287
1288		if (!i915_gem_object_has_unknown_state(obj)) {
1289			pr_err("object missing unknown_state\n");
1290			err = -EINVAL;
1291		}
1292	}
1293
1294out_addr:
1295	vm_munmap(addr, obj->base.size);
1296
1297out_put:
1298	i915_gem_object_put(obj);
1299	igt_close_objects(i915, &objects);
1300	return err;
1301}
1302
1303static int igt_mmap_migrate(void *arg)
1304{
1305	struct drm_i915_private *i915 = arg;
1306	struct intel_memory_region *system = i915->mm.regions[INTEL_REGION_SMEM];
1307	struct intel_memory_region *mr;
1308	enum intel_region_id id;
1309
1310	for_each_memory_region(mr, i915, id) {
1311		struct intel_memory_region *mixed[] = { mr, system };
1312		struct intel_memory_region *single[] = { mr };
1313		struct ttm_resource_manager *man = mr->region_private;
1314		struct resource saved_io;
1315		int err;
1316
1317		if (mr->private)
1318			continue;
1319
1320		if (!resource_size(&mr->io))
1321			continue;
1322
1323		/*
1324		 * For testing purposes let's force small BAR, if not already
1325		 * present.
1326		 */
1327		saved_io = mr->io;
1328		if (resource_size(&mr->io) == mr->total) {
1329			resource_size_t io_size = resource_size(&mr->io);
1330
1331			io_size = rounddown_pow_of_two(io_size >> 1);
1332			if (io_size < PAGE_SIZE)
1333				continue;
1334
1335			mr->io = DEFINE_RES_MEM(mr->io.start, io_size);
1336			i915_ttm_buddy_man_force_visible_size(man,
1337							      io_size >> PAGE_SHIFT);
1338		}
1339
1340		/*
1341		 * Allocate in the mappable portion, should be no surprises here.
1342		 */
1343		err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), mr, 0);
1344		if (err)
1345			goto out_io_size;
1346
1347		/*
1348		 * Allocate in the non-mappable portion, but force migrating to
1349		 * the mappable portion on fault (LMEM -> LMEM)
1350		 */
1351		err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr,
1352					 IGT_MMAP_MIGRATE_TOPDOWN |
1353					 IGT_MMAP_MIGRATE_FILL |
1354					 IGT_MMAP_MIGRATE_EVICTABLE);
1355		if (err)
1356			goto out_io_size;
1357
1358		/*
1359		 * Allocate in the non-mappable portion, but force spilling into
1360		 * system memory on fault (LMEM -> SMEM)
1361		 */
1362		err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), system,
1363					 IGT_MMAP_MIGRATE_TOPDOWN |
1364					 IGT_MMAP_MIGRATE_FILL);
1365		if (err)
1366			goto out_io_size;
1367
1368		/*
1369		 * Allocate in the non-mappable portion, but since the mappable
1370		 * portion is already full, and we can't spill to system memory,
1371		 * then we should expect the fault to fail.
1372		 */
1373		err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr,
1374					 IGT_MMAP_MIGRATE_TOPDOWN |
1375					 IGT_MMAP_MIGRATE_FILL |
1376					 IGT_MMAP_MIGRATE_UNFAULTABLE);
1377		if (err)
1378			goto out_io_size;
1379
1380		/*
1381		 * Allocate in the non-mappable portion, but force migrating to
1382		 * the mappable portion on fault (LMEM -> LMEM). We then also
1383		 * simulate a gpu error when moving the pages when faulting the
1384		 * pages, which should result in wedging the gpu and returning
1385		 * SIGBUS in the fault handler, since we can't fallback to
1386		 * memcpy.
1387		 */
1388		err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr,
1389					 IGT_MMAP_MIGRATE_TOPDOWN |
1390					 IGT_MMAP_MIGRATE_FILL |
1391					 IGT_MMAP_MIGRATE_EVICTABLE |
1392					 IGT_MMAP_MIGRATE_FAIL_GPU |
1393					 IGT_MMAP_MIGRATE_UNFAULTABLE);
1394out_io_size:
1395		mr->io = saved_io;
1396		i915_ttm_buddy_man_force_visible_size(man,
1397						      resource_size(&mr->io) >> PAGE_SHIFT);
1398		if (err)
1399			return err;
1400	}
1401
1402	return 0;
1403}
1404
1405static const char *repr_mmap_type(enum i915_mmap_type type)
1406{
1407	switch (type) {
1408	case I915_MMAP_TYPE_GTT: return "gtt";
1409	case I915_MMAP_TYPE_WB: return "wb";
1410	case I915_MMAP_TYPE_WC: return "wc";
1411	case I915_MMAP_TYPE_UC: return "uc";
1412	case I915_MMAP_TYPE_FIXED: return "fixed";
1413	default: return "unknown";
1414	}
1415}
1416
1417static bool can_access(struct drm_i915_gem_object *obj)
1418{
1419	bool access;
1420
1421	i915_gem_object_lock(obj, NULL);
1422	access = i915_gem_object_has_struct_page(obj) ||
1423		i915_gem_object_has_iomem(obj);
1424	i915_gem_object_unlock(obj);
1425
1426	return access;
1427}
1428
1429static int __igt_mmap_access(struct drm_i915_private *i915,
1430			     struct drm_i915_gem_object *obj,
1431			     enum i915_mmap_type type)
1432{
1433	unsigned long __user *ptr;
1434	unsigned long A, B;
1435	unsigned long x, y;
1436	unsigned long addr;
1437	int err;
1438	u64 offset;
1439
1440	memset(&A, 0xAA, sizeof(A));
1441	memset(&B, 0xBB, sizeof(B));
1442
1443	if (!can_mmap(obj, type) || !can_access(obj))
1444		return 0;
1445
1446	err = __assign_mmap_offset(obj, type, &offset, NULL);
1447	if (err)
1448		return err;
1449
1450	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
1451	if (IS_ERR_VALUE(addr))
1452		return addr;
1453	ptr = (unsigned long __user *)addr;
1454
1455	err = __put_user(A, ptr);
1456	if (err) {
1457		pr_err("%s(%s): failed to write into user mmap\n",
1458		       obj->mm.region->name, repr_mmap_type(type));
1459		goto out_unmap;
1460	}
1461
1462	intel_gt_flush_ggtt_writes(to_gt(i915));
1463
1464	err = access_process_vm(current, addr, &x, sizeof(x), 0);
1465	if (err != sizeof(x)) {
1466		pr_err("%s(%s): access_process_vm() read failed\n",
1467		       obj->mm.region->name, repr_mmap_type(type));
1468		goto out_unmap;
1469	}
1470
1471	err = access_process_vm(current, addr, &B, sizeof(B), FOLL_WRITE);
1472	if (err != sizeof(B)) {
1473		pr_err("%s(%s): access_process_vm() write failed\n",
1474		       obj->mm.region->name, repr_mmap_type(type));
1475		goto out_unmap;
1476	}
1477
1478	intel_gt_flush_ggtt_writes(to_gt(i915));
1479
1480	err = __get_user(y, ptr);
1481	if (err) {
1482		pr_err("%s(%s): failed to read from user mmap\n",
1483		       obj->mm.region->name, repr_mmap_type(type));
1484		goto out_unmap;
1485	}
1486
1487	if (x != A || y != B) {
1488		pr_err("%s(%s): failed to read/write values, found (%lx, %lx)\n",
1489		       obj->mm.region->name, repr_mmap_type(type),
1490		       x, y);
1491		err = -EINVAL;
1492		goto out_unmap;
1493	}
1494
1495out_unmap:
1496	vm_munmap(addr, obj->base.size);
1497	return err;
1498}
1499
1500static int igt_mmap_access(void *arg)
1501{
1502	struct drm_i915_private *i915 = arg;
1503	struct intel_memory_region *mr;
1504	enum intel_region_id id;
1505
1506	for_each_memory_region(mr, i915, id) {
1507		struct drm_i915_gem_object *obj;
1508		int err;
1509
1510		if (mr->private)
1511			continue;
1512
1513		obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1);
1514		if (obj == ERR_PTR(-ENODEV))
1515			continue;
1516
1517		if (IS_ERR(obj))
1518			return PTR_ERR(obj);
1519
1520		err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT);
1521		if (err == 0)
1522			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB);
1523		if (err == 0)
1524			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WC);
1525		if (err == 0)
1526			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_UC);
1527		if (err == 0)
1528			err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_FIXED);
1529
1530		i915_gem_object_put(obj);
1531		if (err)
1532			return err;
1533	}
1534
1535	return 0;
1536}
1537
1538static int __igt_mmap_gpu(struct drm_i915_private *i915,
1539			  struct drm_i915_gem_object *obj,
1540			  enum i915_mmap_type type)
1541{
1542	struct intel_engine_cs *engine;
1543	unsigned long addr;
1544	u32 __user *ux;
1545	u32 bbe;
1546	int err;
1547	u64 offset;
1548
1549	/*
1550	 * Verify that the mmap access into the backing store aligns with
1551	 * that of the GPU, i.e. that mmap is indeed writing into the same
1552	 * page as being read by the GPU.
1553	 */
1554
1555	if (!can_mmap(obj, type))
1556		return 0;
1557
1558	err = wc_set(obj);
1559	if (err == -ENXIO)
1560		err = gtt_set(obj);
1561	if (err)
1562		return err;
1563
1564	err = __assign_mmap_offset(obj, type, &offset, NULL);
1565	if (err)
1566		return err;
1567
1568	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
1569	if (IS_ERR_VALUE(addr))
1570		return addr;
1571
1572	ux = u64_to_user_ptr((u64)addr);
1573	bbe = MI_BATCH_BUFFER_END;
1574	if (put_user(bbe, ux)) {
1575		pr_err("%s: Unable to write to mmap\n", obj->mm.region->name);
1576		err = -EFAULT;
1577		goto out_unmap;
1578	}
1579
1580	if (type == I915_MMAP_TYPE_GTT)
1581		intel_gt_flush_ggtt_writes(to_gt(i915));
1582
1583	for_each_uabi_engine(engine, i915) {
1584		struct i915_request *rq;
1585		struct i915_vma *vma;
1586		struct i915_gem_ww_ctx ww;
1587
1588		vma = i915_vma_instance(obj, engine->kernel_context->vm, NULL);
1589		if (IS_ERR(vma)) {
1590			err = PTR_ERR(vma);
1591			goto out_unmap;
1592		}
1593
1594		i915_gem_ww_ctx_init(&ww, false);
1595retry:
1596		err = i915_gem_object_lock(obj, &ww);
1597		if (!err)
1598			err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
1599		if (err)
1600			goto out_ww;
1601
1602		rq = i915_request_create(engine->kernel_context);
1603		if (IS_ERR(rq)) {
1604			err = PTR_ERR(rq);
1605			goto out_unpin;
1606		}
1607
1608		err = i915_vma_move_to_active(vma, rq, 0);
1609
1610		err = engine->emit_bb_start(rq, i915_vma_offset(vma), 0, 0);
1611		i915_request_get(rq);
1612		i915_request_add(rq);
1613
1614		if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1615			struct drm_printer p =
1616				drm_info_printer(engine->i915->drm.dev);
1617
1618			pr_err("%s(%s, %s): Failed to execute batch\n",
1619			       __func__, engine->name, obj->mm.region->name);
1620			intel_engine_dump(engine, &p,
1621					  "%s\n", engine->name);
1622
1623			intel_gt_set_wedged(engine->gt);
1624			err = -EIO;
1625		}
1626		i915_request_put(rq);
1627
1628out_unpin:
1629		i915_vma_unpin(vma);
1630out_ww:
1631		if (err == -EDEADLK) {
1632			err = i915_gem_ww_ctx_backoff(&ww);
1633			if (!err)
1634				goto retry;
1635		}
1636		i915_gem_ww_ctx_fini(&ww);
1637		if (err)
1638			goto out_unmap;
1639	}
1640
1641out_unmap:
1642	vm_munmap(addr, obj->base.size);
1643	return err;
1644}
1645
1646static int igt_mmap_gpu(void *arg)
1647{
1648	struct drm_i915_private *i915 = arg;
1649	struct intel_memory_region *mr;
1650	enum intel_region_id id;
1651
1652	for_each_memory_region(mr, i915, id) {
1653		struct drm_i915_gem_object *obj;
1654		int err;
1655
1656		if (mr->private)
1657			continue;
1658
1659		obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1);
1660		if (obj == ERR_PTR(-ENODEV))
1661			continue;
1662
1663		if (IS_ERR(obj))
1664			return PTR_ERR(obj);
1665
1666		err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_GTT);
1667		if (err == 0)
1668			err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_WC);
1669		if (err == 0)
1670			err = __igt_mmap_gpu(i915, obj, I915_MMAP_TYPE_FIXED);
1671
1672		i915_gem_object_put(obj);
1673		if (err)
1674			return err;
1675	}
1676
1677	return 0;
1678}
1679
1680static int check_present_pte(pte_t *pte, unsigned long addr, void *data)
1681{
1682	pte_t ptent = ptep_get(pte);
1683
1684	if (!pte_present(ptent) || pte_none(ptent)) {
1685		pr_err("missing PTE:%lx\n",
1686		       (addr - (unsigned long)data) >> PAGE_SHIFT);
1687		return -EINVAL;
1688	}
1689
1690	return 0;
1691}
1692
1693static int check_absent_pte(pte_t *pte, unsigned long addr, void *data)
1694{
1695	pte_t ptent = ptep_get(pte);
1696
1697	if (pte_present(ptent) && !pte_none(ptent)) {
1698		pr_err("present PTE:%lx; expected to be revoked\n",
1699		       (addr - (unsigned long)data) >> PAGE_SHIFT);
1700		return -EINVAL;
1701	}
1702
1703	return 0;
1704}
1705
1706static int check_present(unsigned long addr, unsigned long len)
1707{
1708	return apply_to_page_range(current->mm, addr, len,
1709				   check_present_pte, (void *)addr);
1710}
1711
1712static int check_absent(unsigned long addr, unsigned long len)
1713{
1714	return apply_to_page_range(current->mm, addr, len,
1715				   check_absent_pte, (void *)addr);
1716}
1717
1718static int prefault_range(u64 start, u64 len)
1719{
1720	const char __user *addr, *end;
1721	char __maybe_unused c;
1722	int err;
1723
1724	addr = u64_to_user_ptr(start);
1725	end = addr + len;
1726
1727	for (; addr < end; addr += PAGE_SIZE) {
1728		err = __get_user(c, addr);
1729		if (err)
1730			return err;
1731	}
1732
1733	return __get_user(c, end - 1);
1734}
1735
1736static int __igt_mmap_revoke(struct drm_i915_private *i915,
1737			     struct drm_i915_gem_object *obj,
1738			     enum i915_mmap_type type)
1739{
1740	unsigned long addr;
1741	int err;
1742	u64 offset;
1743
1744	if (!can_mmap(obj, type))
1745		return 0;
1746
1747	err = __assign_mmap_offset(obj, type, &offset, NULL);
1748	if (err)
1749		return err;
1750
1751	addr = igt_mmap_offset(i915, offset, obj->base.size, PROT_WRITE, MAP_SHARED);
1752	if (IS_ERR_VALUE(addr))
1753		return addr;
1754
1755	err = prefault_range(addr, obj->base.size);
1756	if (err)
1757		goto out_unmap;
1758
1759	err = check_present(addr, obj->base.size);
1760	if (err) {
1761		pr_err("%s: was not present\n", obj->mm.region->name);
1762		goto out_unmap;
1763	}
1764
1765	/*
1766	 * After unbinding the object from the GGTT, its address may be reused
1767	 * for other objects. Ergo we have to revoke the previous mmap PTE
1768	 * access as it no longer points to the same object.
1769	 */
1770	i915_gem_object_lock(obj, NULL);
1771	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
1772	i915_gem_object_unlock(obj);
1773	if (err) {
1774		pr_err("Failed to unbind object!\n");
1775		goto out_unmap;
1776	}
1777
1778	if (type != I915_MMAP_TYPE_GTT) {
1779		i915_gem_object_lock(obj, NULL);
1780		__i915_gem_object_put_pages(obj);
1781		i915_gem_object_unlock(obj);
1782		if (i915_gem_object_has_pages(obj)) {
1783			pr_err("Failed to put-pages object!\n");
1784			err = -EINVAL;
1785			goto out_unmap;
1786		}
1787	}
1788
1789	err = check_absent(addr, obj->base.size);
1790	if (err) {
1791		pr_err("%s: was not absent\n", obj->mm.region->name);
1792		goto out_unmap;
1793	}
1794
1795out_unmap:
1796	vm_munmap(addr, obj->base.size);
1797	return err;
1798}
1799
1800static int igt_mmap_revoke(void *arg)
1801{
1802	struct drm_i915_private *i915 = arg;
1803	struct intel_memory_region *mr;
1804	enum intel_region_id id;
1805
1806	for_each_memory_region(mr, i915, id) {
1807		struct drm_i915_gem_object *obj;
1808		int err;
1809
1810		if (mr->private)
1811			continue;
1812
1813		obj = __i915_gem_object_create_user(i915, PAGE_SIZE, &mr, 1);
1814		if (obj == ERR_PTR(-ENODEV))
1815			continue;
1816
1817		if (IS_ERR(obj))
1818			return PTR_ERR(obj);
1819
1820		err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_GTT);
1821		if (err == 0)
1822			err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_WC);
1823		if (err == 0)
1824			err = __igt_mmap_revoke(i915, obj, I915_MMAP_TYPE_FIXED);
1825
1826		i915_gem_object_put(obj);
1827		if (err)
1828			return err;
1829	}
1830
1831	return 0;
1832}
1833
1834int i915_gem_mman_live_selftests(struct drm_i915_private *i915)
1835{
1836	int ret;
1837	bool unuse_mm = false;
1838	static const struct i915_subtest tests[] = {
1839		SUBTEST(igt_partial_tiling),
1840		SUBTEST(igt_smoke_tiling),
1841		SUBTEST(igt_mmap_offset_exhaustion),
1842		SUBTEST(igt_mmap),
1843		SUBTEST(igt_mmap_migrate),
1844		SUBTEST(igt_mmap_access),
1845		SUBTEST(igt_mmap_revoke),
1846		SUBTEST(igt_mmap_gpu),
1847	};
1848
1849	if (!current->mm) {
1850		kthread_use_mm(current->active_mm);
1851		unuse_mm = true;
1852	}
1853
1854	ret = i915_live_subtests(tests, i915);
1855
1856	if (unuse_mm)
1857		kthread_unuse_mm(current->active_mm);
1858
1859	return ret;
1860}