i915_vma.c

   1/*
   2 * Copyright © 2016 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/prime_numbers.h>
  26
  27#include "gem/i915_gem_context.h"
  28#include "gem/i915_gem_internal.h"
  29#include "gem/selftests/mock_context.h"
  30
  31#include "i915_scatterlist.h"
  32#include "i915_selftest.h"
  33
  34#include "mock_gem_device.h"
  35#include "mock_gtt.h"
  36
  37static bool assert_vma(struct i915_vma *vma,
  38		       struct drm_i915_gem_object *obj,
  39		       struct i915_gem_context *ctx)
  40{
  41	bool ok = true;
  42
  43	if (vma->vm != ctx->vm) {
  44		pr_err("VMA created with wrong VM\n");
  45		ok = false;
  46	}
  47
  48	if (vma->size != obj->base.size) {
  49		pr_err("VMA created with wrong size, found %llu, expected %zu\n",
  50		       vma->size, obj->base.size);
  51		ok = false;
  52	}
  53
  54	if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
  55		pr_err("VMA created with wrong type [%d]\n",
  56		       vma->gtt_view.type);
  57		ok = false;
  58	}
  59
  60	return ok;
  61}
  62
  63static struct i915_vma *
  64checked_vma_instance(struct drm_i915_gem_object *obj,
  65		     struct i915_address_space *vm,
  66		     const struct i915_gtt_view *view)
  67{
  68	struct i915_vma *vma;
  69	bool ok = true;
  70
  71	vma = i915_vma_instance(obj, vm, view);
  72	if (IS_ERR(vma))
  73		return vma;
  74
  75	/* Manual checks, will be reinforced by i915_vma_compare! */
  76	if (vma->vm != vm) {
  77		pr_err("VMA's vm [%p] does not match request [%p]\n",
  78		       vma->vm, vm);
  79		ok = false;
  80	}
  81
  82	if (i915_is_ggtt(vm) != i915_vma_is_ggtt(vma)) {
  83		pr_err("VMA ggtt status [%d] does not match parent [%d]\n",
  84		       i915_vma_is_ggtt(vma), i915_is_ggtt(vm));
  85		ok = false;
  86	}
  87
  88	if (i915_vma_compare(vma, vm, view)) {
  89		pr_err("i915_vma_compare failed with create parameters!\n");
  90		return ERR_PTR(-EINVAL);
  91	}
  92
  93	if (i915_vma_compare(vma, vma->vm,
  94			     i915_vma_is_ggtt(vma) ? &vma->gtt_view : NULL)) {
  95		pr_err("i915_vma_compare failed with itself\n");
  96		return ERR_PTR(-EINVAL);
  97	}
  98
  99	if (!ok) {
 100		pr_err("i915_vma_compare failed to detect the difference!\n");
 101		return ERR_PTR(-EINVAL);
 102	}
 103
 104	return vma;
 105}
 106
 107static int create_vmas(struct drm_i915_private *i915,
 108		       struct list_head *objects,
 109		       struct list_head *contexts)
 110{
 111	struct drm_i915_gem_object *obj;
 112	struct i915_gem_context *ctx;
 113	int pinned;
 114
 115	list_for_each_entry(obj, objects, st_link) {
 116		for (pinned = 0; pinned <= 1; pinned++) {
 117			list_for_each_entry(ctx, contexts, link) {
 118				struct i915_address_space *vm;
 119				struct i915_vma *vma;
 120				int err;
 121
 122				vm = i915_gem_context_get_eb_vm(ctx);
 123				vma = checked_vma_instance(obj, vm, NULL);
 124				i915_vm_put(vm);
 125				if (IS_ERR(vma))
 126					return PTR_ERR(vma);
 127
 128				if (!assert_vma(vma, obj, ctx)) {
 129					pr_err("VMA lookup/create failed\n");
 130					return -EINVAL;
 131				}
 132
 133				if (!pinned) {
 134					err = i915_vma_pin(vma, 0, 0, PIN_USER);
 135					if (err) {
 136						pr_err("Failed to pin VMA\n");
 137						return err;
 138					}
 139				} else {
 140					i915_vma_unpin(vma);
 141				}
 142			}
 143		}
 144	}
 145
 146	return 0;
 147}
 148
 149static int igt_vma_create(void *arg)
 150{
 151	struct i915_ggtt *ggtt = arg;
 152	struct drm_i915_private *i915 = ggtt->vm.i915;
 153	struct drm_i915_gem_object *obj, *on;
 154	struct i915_gem_context *ctx, *cn;
 155	unsigned long num_obj, num_ctx;
 156	unsigned long no, nc;
 157	IGT_TIMEOUT(end_time);
 158	LIST_HEAD(contexts);
 159	LIST_HEAD(objects);
 160	int err = -ENOMEM;
 161
 162	/*
 163	 * Exercise creating many vma amongst many objections, checking the
 164	 * vma creation and lookup routines.
 165	 */
 166
 167	no = 0;
 168	for_each_prime_number(num_obj, ULONG_MAX - 1) {
 169		for (; no < num_obj; no++) {
 170			obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 171			if (IS_ERR(obj))
 172				goto out;
 173
 174			list_add(&obj->st_link, &objects);
 175		}
 176
 177		nc = 0;
 178		for_each_prime_number(num_ctx, 2 * BITS_PER_LONG) {
 179			for (; nc < num_ctx; nc++) {
 180				ctx = mock_context(i915, "mock");
 181				if (!ctx)
 182					goto out;
 183
 184				list_move(&ctx->link, &contexts);
 185			}
 186
 187			err = create_vmas(i915, &objects, &contexts);
 188			if (err)
 189				goto out;
 190
 191			if (igt_timeout(end_time,
 192					"%s timed out: after %lu objects in %lu contexts\n",
 193					__func__, no, nc))
 194				goto end;
 195		}
 196
 197		list_for_each_entry_safe(ctx, cn, &contexts, link) {
 198			list_del_init(&ctx->link);
 199			mock_context_close(ctx);
 200		}
 201
 202		cond_resched();
 203	}
 204
 205end:
 206	/* Final pass to lookup all created contexts */
 207	err = create_vmas(i915, &objects, &contexts);
 208out:
 209	list_for_each_entry_safe(ctx, cn, &contexts, link) {
 210		list_del_init(&ctx->link);
 211		mock_context_close(ctx);
 212	}
 213
 214	list_for_each_entry_safe(obj, on, &objects, st_link)
 215		i915_gem_object_put(obj);
 216	return err;
 217}
 218
 219struct pin_mode {
 220	u64 size;
 221	u64 flags;
 222	bool (*assert)(const struct i915_vma *,
 223		       const struct pin_mode *mode,
 224		       int result);
 225	const char *string;
 226};
 227
 228static bool assert_pin_valid(const struct i915_vma *vma,
 229			     const struct pin_mode *mode,
 230			     int result)
 231{
 232	if (result)
 233		return false;
 234
 235	if (i915_vma_misplaced(vma, mode->size, 0, mode->flags))
 236		return false;
 237
 238	return true;
 239}
 240
 241__maybe_unused
 242static bool assert_pin_enospc(const struct i915_vma *vma,
 243			      const struct pin_mode *mode,
 244			      int result)
 245{
 246	return result == -ENOSPC;
 247}
 248
 249__maybe_unused
 250static bool assert_pin_einval(const struct i915_vma *vma,
 251			      const struct pin_mode *mode,
 252			      int result)
 253{
 254	return result == -EINVAL;
 255}
 256
 257static int igt_vma_pin1(void *arg)
 258{
 259	struct i915_ggtt *ggtt = arg;
 260	const struct pin_mode modes[] = {
 261#define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " }
 262#define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" }
 263#define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL)
 264#define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC)
 265		VALID(0, PIN_GLOBAL),
 266		VALID(0, PIN_GLOBAL | PIN_MAPPABLE),
 267
 268		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 4096),
 269		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 8192),
 270		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
 271		VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
 272		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
 273
 274		VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
 275		INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | ggtt->mappable_end),
 276		VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
 277		INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | ggtt->vm.total),
 278		INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | round_down(U64_MAX, PAGE_SIZE)),
 279
 280		VALID(4096, PIN_GLOBAL),
 281		VALID(8192, PIN_GLOBAL),
 282		VALID(ggtt->mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE),
 283		VALID(ggtt->mappable_end, PIN_GLOBAL | PIN_MAPPABLE),
 284		NOSPACE(ggtt->mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
 285		VALID(ggtt->vm.total - 4096, PIN_GLOBAL),
 286		VALID(ggtt->vm.total, PIN_GLOBAL),
 287		NOSPACE(ggtt->vm.total + 4096, PIN_GLOBAL),
 288		NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
 289		INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
 290		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
 291		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)),
 292
 293		VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
 294
 295#if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 296		/*
 297		 * Misusing BIAS is a programming error (it is not controllable
 298		 * from userspace) so when debugging is enabled, it explodes.
 299		 * However, the tests are still quite interesting for checking
 300		 * variable start, end and size.
 301		 */
 302		NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | ggtt->mappable_end),
 303		NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | ggtt->vm.total),
 304		NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
 305		NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
 306#endif
 307		{ },
 308#undef NOSPACE
 309#undef INVALID
 310#undef __INVALID
 311#undef VALID
 312	}, *m;
 313	struct drm_i915_gem_object *obj;
 314	struct i915_vma *vma;
 315	int err = -EINVAL;
 316
 317	/*
 318	 * Exercise all the weird and wonderful i915_vma_pin requests,
 319	 * focusing on error handling of boundary conditions.
 320	 */
 321
 322	GEM_BUG_ON(!drm_mm_clean(&ggtt->vm.mm));
 323
 324	obj = i915_gem_object_create_internal(ggtt->vm.i915, PAGE_SIZE);
 325	if (IS_ERR(obj))
 326		return PTR_ERR(obj);
 327
 328	vma = checked_vma_instance(obj, &ggtt->vm, NULL);
 329	if (IS_ERR(vma))
 330		goto out;
 331
 332	for (m = modes; m->assert; m++) {
 333		err = i915_vma_pin(vma, m->size, 0, m->flags);
 334		if (!m->assert(vma, m, err)) {
 335			pr_err("%s to pin single page into GGTT with mode[%d:%s]: size=%llx flags=%llx, err=%d\n",
 336			       m->assert == assert_pin_valid ? "Failed" : "Unexpectedly succeeded",
 337			       (int)(m - modes), m->string, m->size, m->flags,
 338			       err);
 339			if (!err)
 340				i915_vma_unpin(vma);
 341			err = -EINVAL;
 342			goto out;
 343		}
 344
 345		if (!err) {
 346			i915_vma_unpin(vma);
 347			err = i915_vma_unbind_unlocked(vma);
 348			if (err) {
 349				pr_err("Failed to unbind single page from GGTT, err=%d\n", err);
 350				goto out;
 351			}
 352		}
 353
 354		cond_resched();
 355	}
 356
 357	err = 0;
 358out:
 359	i915_gem_object_put(obj);
 360	return err;
 361}
 362
 363static unsigned long rotated_index(const struct intel_rotation_info *r,
 364				   unsigned int n,
 365				   unsigned int x,
 366				   unsigned int y)
 367{
 368	return (r->plane[n].src_stride * (r->plane[n].height - y - 1) +
 369		r->plane[n].offset + x);
 370}
 371
 372static struct scatterlist *
 373assert_rotated(struct drm_i915_gem_object *obj,
 374	       const struct intel_rotation_info *r, unsigned int n,
 375	       struct scatterlist *sg)
 376{
 377	unsigned int x, y;
 378
 379	for (x = 0; x < r->plane[n].width; x++) {
 380		unsigned int left;
 381
 382		for (y = 0; y < r->plane[n].height; y++) {
 383			unsigned long src_idx;
 384			dma_addr_t src;
 385
 386			if (!sg) {
 387				pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
 388				       n, x, y);
 389				return ERR_PTR(-EINVAL);
 390			}
 391
 392			src_idx = rotated_index(r, n, x, y);
 393			src = i915_gem_object_get_dma_address(obj, src_idx);
 394
 395			if (sg_dma_len(sg) != PAGE_SIZE) {
 396				pr_err("Invalid sg.length, found %d, expected %lu for rotated page (%d, %d) [src index %lu]\n",
 397				       sg_dma_len(sg), PAGE_SIZE,
 398				       x, y, src_idx);
 399				return ERR_PTR(-EINVAL);
 400			}
 401
 402			if (sg_dma_address(sg) != src) {
 403				pr_err("Invalid address for rotated page (%d, %d) [src index %lu]\n",
 404				       x, y, src_idx);
 405				return ERR_PTR(-EINVAL);
 406			}
 407
 408			sg = sg_next(sg);
 409		}
 410
 411		left = (r->plane[n].dst_stride - y) * PAGE_SIZE;
 412
 413		if (!left)
 414			continue;
 415
 416		if (!sg) {
 417			pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
 418			       n, x, y);
 419			return ERR_PTR(-EINVAL);
 420		}
 421
 422		if (sg_dma_len(sg) != left) {
 423			pr_err("Invalid sg.length, found %d, expected %u for rotated page (%d, %d)\n",
 424			       sg_dma_len(sg), left, x, y);
 425			return ERR_PTR(-EINVAL);
 426		}
 427
 428		if (sg_dma_address(sg) != 0) {
 429			pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
 430			       &sg_dma_address(sg), x, y);
 431			return ERR_PTR(-EINVAL);
 432		}
 433
 434		sg = sg_next(sg);
 435	}
 436
 437	return sg;
 438}
 439
 440static unsigned long remapped_index(const struct intel_remapped_info *r,
 441				    unsigned int n,
 442				    unsigned int x,
 443				    unsigned int y)
 444{
 445	return (r->plane[n].src_stride * y +
 446		r->plane[n].offset + x);
 447}
 448
 449static struct scatterlist *
 450assert_remapped(struct drm_i915_gem_object *obj,
 451		const struct intel_remapped_info *r, unsigned int n,
 452		struct scatterlist *sg)
 453{
 454	unsigned int x, y;
 455	unsigned int left = 0;
 456	unsigned int offset;
 457
 458	for (y = 0; y < r->plane[n].height; y++) {
 459		for (x = 0; x < r->plane[n].width; x++) {
 460			unsigned long src_idx;
 461			dma_addr_t src;
 462
 463			if (!sg) {
 464				pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
 465				       n, x, y);
 466				return ERR_PTR(-EINVAL);
 467			}
 468			if (!left) {
 469				offset = 0;
 470				left = sg_dma_len(sg);
 471			}
 472
 473			src_idx = remapped_index(r, n, x, y);
 474			src = i915_gem_object_get_dma_address(obj, src_idx);
 475
 476			if (left < PAGE_SIZE || left & (PAGE_SIZE-1)) {
 477				pr_err("Invalid sg.length, found %d, expected %lu for remapped page (%d, %d) [src index %lu]\n",
 478				       sg_dma_len(sg), PAGE_SIZE,
 479				       x, y, src_idx);
 480				return ERR_PTR(-EINVAL);
 481			}
 482
 483			if (sg_dma_address(sg) + offset != src) {
 484				pr_err("Invalid address for remapped page (%d, %d) [src index %lu]\n",
 485				       x, y, src_idx);
 486				return ERR_PTR(-EINVAL);
 487			}
 488
 489			left -= PAGE_SIZE;
 490			offset += PAGE_SIZE;
 491
 492
 493			if (!left)
 494				sg = sg_next(sg);
 495		}
 496
 497		if (left) {
 498			pr_err("Unexpected sg tail with %d size for remapped page (%d, %d)\n",
 499			       left,
 500			       x, y);
 501			return ERR_PTR(-EINVAL);
 502		}
 503
 504		left = (r->plane[n].dst_stride - r->plane[n].width) * PAGE_SIZE;
 505
 506		if (!left)
 507			continue;
 508
 509		if (!sg) {
 510			pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
 511			       n, x, y);
 512			return ERR_PTR(-EINVAL);
 513		}
 514
 515		if (sg_dma_len(sg) != left) {
 516			pr_err("Invalid sg.length, found %u, expected %u for remapped page (%d, %d)\n",
 517			       sg_dma_len(sg), left,
 518			       x, y);
 519			return ERR_PTR(-EINVAL);
 520		}
 521
 522		if (sg_dma_address(sg) != 0) {
 523			pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
 524			       &sg_dma_address(sg),
 525			       x, y);
 526			return ERR_PTR(-EINVAL);
 527		}
 528
 529		sg = sg_next(sg);
 530		left = 0;
 531	}
 532
 533	return sg;
 534}
 535
 536static unsigned int remapped_size(enum i915_gtt_view_type view_type,
 537				  const struct intel_remapped_plane_info *a,
 538				  const struct intel_remapped_plane_info *b)
 539{
 540
 541	if (view_type == I915_GTT_VIEW_ROTATED)
 542		return a->dst_stride * a->width + b->dst_stride * b->width;
 543	else
 544		return a->dst_stride * a->height + b->dst_stride * b->height;
 545}
 546
 547static int igt_vma_rotate_remap(void *arg)
 548{
 549	struct i915_ggtt *ggtt = arg;
 550	struct i915_address_space *vm = &ggtt->vm;
 551	struct drm_i915_gem_object *obj;
 552	const struct intel_remapped_plane_info planes[] = {
 553		{ .width = 1, .height = 1, .src_stride = 1 },
 554		{ .width = 2, .height = 2, .src_stride = 2 },
 555		{ .width = 4, .height = 4, .src_stride = 4 },
 556		{ .width = 8, .height = 8, .src_stride = 8 },
 557
 558		{ .width = 3, .height = 5, .src_stride = 3 },
 559		{ .width = 3, .height = 5, .src_stride = 4 },
 560		{ .width = 3, .height = 5, .src_stride = 5 },
 561
 562		{ .width = 5, .height = 3, .src_stride = 5 },
 563		{ .width = 5, .height = 3, .src_stride = 7 },
 564		{ .width = 5, .height = 3, .src_stride = 9 },
 565
 566		{ .width = 4, .height = 6, .src_stride = 6 },
 567		{ .width = 6, .height = 4, .src_stride = 6 },
 568
 569		{ .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
 570		{ .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
 571		{ .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
 572
 573		{ }
 574	}, *a, *b;
 575	enum i915_gtt_view_type types[] = {
 576		I915_GTT_VIEW_ROTATED,
 577		I915_GTT_VIEW_REMAPPED,
 578		0,
 579	}, *t;
 580	const unsigned int max_pages = 64;
 581	int err = -ENOMEM;
 582
 583	/*
 584	 * Create VMA for many different combinations of planes and check
 585	 * that the page layout within the rotated VMA match our expectations.
 586	 */
 587
 588	obj = i915_gem_object_create_internal(vm->i915, max_pages * PAGE_SIZE);
 589	if (IS_ERR(obj))
 590		goto out;
 591
 592	for (t = types; *t; t++) {
 593	for (a = planes; a->width; a++) {
 594		for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) {
 595			struct i915_gtt_view view = {
 596				.type = *t,
 597				.remapped.plane[0] = *a,
 598				.remapped.plane[1] = *b,
 599			};
 600			struct intel_remapped_plane_info *plane_info = view.remapped.plane;
 601			unsigned int n, max_offset;
 602
 603			max_offset = max(plane_info[0].src_stride * plane_info[0].height,
 604					 plane_info[1].src_stride * plane_info[1].height);
 605			GEM_BUG_ON(max_offset > max_pages);
 606			max_offset = max_pages - max_offset;
 607
 608			if (!plane_info[0].dst_stride)
 609				plane_info[0].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
 610									plane_info[0].height :
 611									plane_info[0].width;
 612			if (!plane_info[1].dst_stride)
 613				plane_info[1].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
 614									plane_info[1].height :
 615									plane_info[1].width;
 616
 617			for_each_prime_number_from(plane_info[0].offset, 0, max_offset) {
 618				for_each_prime_number_from(plane_info[1].offset, 0, max_offset) {
 619					struct scatterlist *sg;
 620					struct i915_vma *vma;
 621					unsigned int expected_pages;
 622
 623					vma = checked_vma_instance(obj, vm, &view);
 624					if (IS_ERR(vma)) {
 625						err = PTR_ERR(vma);
 626						goto out_object;
 627					}
 628
 629					err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 630					if (err) {
 631						pr_err("Failed to pin VMA, err=%d\n", err);
 632						goto out_object;
 633					}
 634
 635					expected_pages = remapped_size(view.type, &plane_info[0], &plane_info[1]);
 636
 637					if (view.type == I915_GTT_VIEW_ROTATED &&
 638					    vma->size != expected_pages * PAGE_SIZE) {
 639						pr_err("VMA is wrong size, expected %lu, found %llu\n",
 640						       PAGE_SIZE * expected_pages, vma->size);
 641						err = -EINVAL;
 642						goto out_object;
 643					}
 644
 645					if (view.type == I915_GTT_VIEW_REMAPPED &&
 646					    vma->size > expected_pages * PAGE_SIZE) {
 647						pr_err("VMA is wrong size, expected %lu, found %llu\n",
 648						       PAGE_SIZE * expected_pages, vma->size);
 649						err = -EINVAL;
 650						goto out_object;
 651					}
 652
 653					if (vma->pages->nents > expected_pages) {
 654						pr_err("sg table is wrong sizeo, expected %u, found %u nents\n",
 655						       expected_pages, vma->pages->nents);
 656						err = -EINVAL;
 657						goto out_object;
 658					}
 659
 660					if (vma->node.size < vma->size) {
 661						pr_err("VMA binding too small, expected %llu, found %llu\n",
 662						       vma->size, vma->node.size);
 663						err = -EINVAL;
 664						goto out_object;
 665					}
 666
 667					if (vma->pages == obj->mm.pages) {
 668						pr_err("VMA using unrotated object pages!\n");
 669						err = -EINVAL;
 670						goto out_object;
 671					}
 672
 673					sg = vma->pages->sgl;
 674					for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) {
 675						if (view.type == I915_GTT_VIEW_ROTATED)
 676							sg = assert_rotated(obj, &view.rotated, n, sg);
 677						else
 678							sg = assert_remapped(obj, &view.remapped, n, sg);
 679						if (IS_ERR(sg)) {
 680							pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n",
 681							       view.type == I915_GTT_VIEW_ROTATED ?
 682							       "rotated" : "remapped", n,
 683							       plane_info[0].width,
 684							       plane_info[0].height,
 685							       plane_info[0].src_stride,
 686							       plane_info[0].dst_stride,
 687							       plane_info[0].offset,
 688							       plane_info[1].width,
 689							       plane_info[1].height,
 690							       plane_info[1].src_stride,
 691							       plane_info[1].dst_stride,
 692							       plane_info[1].offset);
 693							err = -EINVAL;
 694							goto out_object;
 695						}
 696					}
 697
 698					i915_vma_unpin(vma);
 699					err = i915_vma_unbind_unlocked(vma);
 700					if (err) {
 701						pr_err("Unbinding returned %i\n", err);
 702						goto out_object;
 703					}
 704					cond_resched();
 705				}
 706			}
 707		}
 708	}
 709	}
 710
 711out_object:
 712	i915_gem_object_put(obj);
 713out:
 714	return err;
 715}
 716
 717static bool assert_partial(struct drm_i915_gem_object *obj,
 718			   struct i915_vma *vma,
 719			   unsigned long offset,
 720			   unsigned long size)
 721{
 722	struct sgt_iter sgt;
 723	dma_addr_t dma;
 724
 725	for_each_sgt_daddr(dma, sgt, vma->pages) {
 726		dma_addr_t src;
 727
 728		if (!size) {
 729			pr_err("Partial scattergather list too long\n");
 730			return false;
 731		}
 732
 733		src = i915_gem_object_get_dma_address(obj, offset);
 734		if (src != dma) {
 735			pr_err("DMA mismatch for partial page offset %lu\n",
 736			       offset);
 737			return false;
 738		}
 739
 740		offset++;
 741		size--;
 742	}
 743
 744	return true;
 745}
 746
 747static bool assert_pin(struct i915_vma *vma,
 748		       struct i915_gtt_view *view,
 749		       u64 size,
 750		       const char *name)
 751{
 752	bool ok = true;
 753
 754	if (vma->size != size) {
 755		pr_err("(%s) VMA is wrong size, expected %llu, found %llu\n",
 756		       name, size, vma->size);
 757		ok = false;
 758	}
 759
 760	if (vma->node.size < vma->size) {
 761		pr_err("(%s) VMA binding too small, expected %llu, found %llu\n",
 762		       name, vma->size, vma->node.size);
 763		ok = false;
 764	}
 765
 766	if (view && view->type != I915_GTT_VIEW_NORMAL) {
 767		if (memcmp(&vma->gtt_view, view, sizeof(*view))) {
 768			pr_err("(%s) VMA mismatch upon creation!\n",
 769			       name);
 770			ok = false;
 771		}
 772
 773		if (vma->pages == vma->obj->mm.pages) {
 774			pr_err("(%s) VMA using original object pages!\n",
 775			       name);
 776			ok = false;
 777		}
 778	} else {
 779		if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
 780			pr_err("Not the normal ggtt view! Found %d\n",
 781			       vma->gtt_view.type);
 782			ok = false;
 783		}
 784
 785		if (vma->pages != vma->obj->mm.pages) {
 786			pr_err("VMA not using object pages!\n");
 787			ok = false;
 788		}
 789	}
 790
 791	return ok;
 792}
 793
 794static int igt_vma_partial(void *arg)
 795{
 796	struct i915_ggtt *ggtt = arg;
 797	struct i915_address_space *vm = &ggtt->vm;
 798	const unsigned int npages = 1021; /* prime! */
 799	struct drm_i915_gem_object *obj;
 800	const struct phase {
 801		const char *name;
 802	} phases[] = {
 803		{ "create" },
 804		{ "lookup" },
 805		{ },
 806	}, *p;
 807	unsigned int sz, offset;
 808	struct i915_vma *vma;
 809	int err = -ENOMEM;
 810
 811	/*
 812	 * Create lots of different VMA for the object and check that
 813	 * we are returned the same VMA when we later request the same range.
 814	 */
 815
 816	obj = i915_gem_object_create_internal(vm->i915, npages * PAGE_SIZE);
 817	if (IS_ERR(obj))
 818		goto out;
 819
 820	for (p = phases; p->name; p++) { /* exercise both create/lookup */
 821		unsigned int count, nvma;
 822
 823		nvma = 0;
 824		for_each_prime_number_from(sz, 1, npages) {
 825			for_each_prime_number_from(offset, 0, npages - sz) {
 826				struct i915_gtt_view view;
 827
 828				view.type = I915_GTT_VIEW_PARTIAL;
 829				view.partial.offset = offset;
 830				view.partial.size = sz;
 831
 832				if (sz == npages)
 833					view.type = I915_GTT_VIEW_NORMAL;
 834
 835				vma = checked_vma_instance(obj, vm, &view);
 836				if (IS_ERR(vma)) {
 837					err = PTR_ERR(vma);
 838					goto out_object;
 839				}
 840
 841				err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 842				if (err)
 843					goto out_object;
 844
 845				if (!assert_pin(vma, &view, sz*PAGE_SIZE, p->name)) {
 846					pr_err("(%s) Inconsistent partial pinning for (offset=%d, size=%d)\n",
 847					       p->name, offset, sz);
 848					err = -EINVAL;
 849					goto out_object;
 850				}
 851
 852				if (!assert_partial(obj, vma, offset, sz)) {
 853					pr_err("(%s) Inconsistent partial pages for (offset=%d, size=%d)\n",
 854					       p->name, offset, sz);
 855					err = -EINVAL;
 856					goto out_object;
 857				}
 858
 859				i915_vma_unpin(vma);
 860				nvma++;
 861				err = i915_vma_unbind_unlocked(vma);
 862				if (err) {
 863					pr_err("Unbinding returned %i\n", err);
 864					goto out_object;
 865				}
 866
 867				cond_resched();
 868			}
 869		}
 870
 871		count = 0;
 872		list_for_each_entry(vma, &obj->vma.list, obj_link)
 873			count++;
 874		if (count != nvma) {
 875			pr_err("(%s) All partial vma were not recorded on the obj->vma_list: found %u, expected %u\n",
 876			       p->name, count, nvma);
 877			err = -EINVAL;
 878			goto out_object;
 879		}
 880
 881		/* Check that we did create the whole object mapping */
 882		vma = checked_vma_instance(obj, vm, NULL);
 883		if (IS_ERR(vma)) {
 884			err = PTR_ERR(vma);
 885			goto out_object;
 886		}
 887
 888		err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 889		if (err)
 890			goto out_object;
 891
 892		if (!assert_pin(vma, NULL, obj->base.size, p->name)) {
 893			pr_err("(%s) inconsistent full pin\n", p->name);
 894			err = -EINVAL;
 895			goto out_object;
 896		}
 897
 898		i915_vma_unpin(vma);
 899
 900		err = i915_vma_unbind_unlocked(vma);
 901		if (err) {
 902			pr_err("Unbinding returned %i\n", err);
 903			goto out_object;
 904		}
 905
 906		count = 0;
 907		list_for_each_entry(vma, &obj->vma.list, obj_link)
 908			count++;
 909		if (count != nvma) {
 910			pr_err("(%s) allocated an extra full vma!\n", p->name);
 911			err = -EINVAL;
 912			goto out_object;
 913		}
 914	}
 915
 916out_object:
 917	i915_gem_object_put(obj);
 918out:
 919	return err;
 920}
 921
 922int i915_vma_mock_selftests(void)
 923{
 924	static const struct i915_subtest tests[] = {
 925		SUBTEST(igt_vma_create),
 926		SUBTEST(igt_vma_pin1),
 927		SUBTEST(igt_vma_rotate_remap),
 928		SUBTEST(igt_vma_partial),
 929	};
 930	struct drm_i915_private *i915;
 931	struct intel_gt *gt;
 932	int err;
 933
 934	i915 = mock_gem_device();
 935	if (!i915)
 936		return -ENOMEM;
 937
 938	/* allocate the ggtt */
 939	err = intel_gt_assign_ggtt(to_gt(i915));
 940	if (err)
 941		goto out_put;
 942
 943	gt = to_gt(i915);
 944
 945	mock_init_ggtt(gt);
 946
 947	err = i915_subtests(tests, gt->ggtt);
 948
 949	mock_device_flush(i915);
 950	i915_gem_drain_freed_objects(i915);
 951	mock_fini_ggtt(gt->ggtt);
 952
 953out_put:
 954	mock_destroy_device(i915);
 955	return err;
 956}
 957
 958static int igt_vma_remapped_gtt(void *arg)
 959{
 960	struct drm_i915_private *i915 = arg;
 961	const struct intel_remapped_plane_info planes[] = {
 962		{ .width = 1, .height = 1, .src_stride = 1 },
 963		{ .width = 2, .height = 2, .src_stride = 2 },
 964		{ .width = 4, .height = 4, .src_stride = 4 },
 965		{ .width = 8, .height = 8, .src_stride = 8 },
 966
 967		{ .width = 3, .height = 5, .src_stride = 3 },
 968		{ .width = 3, .height = 5, .src_stride = 4 },
 969		{ .width = 3, .height = 5, .src_stride = 5 },
 970
 971		{ .width = 5, .height = 3, .src_stride = 5 },
 972		{ .width = 5, .height = 3, .src_stride = 7 },
 973		{ .width = 5, .height = 3, .src_stride = 9 },
 974
 975		{ .width = 4, .height = 6, .src_stride = 6 },
 976		{ .width = 6, .height = 4, .src_stride = 6 },
 977
 978		{ .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
 979		{ .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
 980		{ .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
 981
 982		{ }
 983	}, *p;
 984	enum i915_gtt_view_type types[] = {
 985		I915_GTT_VIEW_ROTATED,
 986		I915_GTT_VIEW_REMAPPED,
 987		0,
 988	}, *t;
 989	struct drm_i915_gem_object *obj;
 990	intel_wakeref_t wakeref;
 991	int err = 0;
 992
 993	if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
 994		return 0;
 995
 996	obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE);
 997	if (IS_ERR(obj))
 998		return PTR_ERR(obj);
 999
1000	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1001
1002	for (t = types; *t; t++) {
1003		for (p = planes; p->width; p++) {
1004			struct i915_gtt_view view = {
1005				.type = *t,
1006				.rotated.plane[0] = *p,
1007			};
1008			struct intel_remapped_plane_info *plane_info = view.rotated.plane;
1009			struct i915_vma *vma;
1010			u32 __iomem *map;
1011			unsigned int x, y;
1012
1013			i915_gem_object_lock(obj, NULL);
1014			err = i915_gem_object_set_to_gtt_domain(obj, true);
1015			i915_gem_object_unlock(obj);
1016			if (err)
1017				goto out;
1018
1019			if (!plane_info[0].dst_stride)
1020				plane_info[0].dst_stride = *t == I915_GTT_VIEW_ROTATED ?
1021								 p->height : p->width;
1022
1023			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1024			if (IS_ERR(vma)) {
1025				err = PTR_ERR(vma);
1026				goto out;
1027			}
1028
1029			GEM_BUG_ON(vma->gtt_view.type != *t);
1030
1031			map = i915_vma_pin_iomap(vma);
1032			i915_vma_unpin(vma);
1033			if (IS_ERR(map)) {
1034				err = PTR_ERR(map);
1035				goto out;
1036			}
1037
1038			for (y = 0 ; y < plane_info[0].height; y++) {
1039				for (x = 0 ; x < plane_info[0].width; x++) {
1040					unsigned int offset;
1041					u32 val = y << 16 | x;
1042
1043					if (*t == I915_GTT_VIEW_ROTATED)
1044						offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE;
1045					else
1046						offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE;
1047
1048					iowrite32(val, &map[offset / sizeof(*map)]);
1049				}
1050			}
1051
1052			i915_vma_unpin_iomap(vma);
1053
1054			vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
1055			if (IS_ERR(vma)) {
1056				err = PTR_ERR(vma);
1057				goto out;
1058			}
1059
1060			GEM_BUG_ON(vma->gtt_view.type != I915_GTT_VIEW_NORMAL);
1061
1062			map = i915_vma_pin_iomap(vma);
1063			i915_vma_unpin(vma);
1064			if (IS_ERR(map)) {
1065				err = PTR_ERR(map);
1066				goto out;
1067			}
1068
1069			for (y = 0 ; y < plane_info[0].height; y++) {
1070				for (x = 0 ; x < plane_info[0].width; x++) {
1071					unsigned int offset, src_idx;
1072					u32 exp = y << 16 | x;
1073					u32 val;
1074
1075					if (*t == I915_GTT_VIEW_ROTATED)
1076						src_idx = rotated_index(&view.rotated, 0, x, y);
1077					else
1078						src_idx = remapped_index(&view.remapped, 0, x, y);
1079					offset = src_idx * PAGE_SIZE;
1080
1081					val = ioread32(&map[offset / sizeof(*map)]);
1082					if (val != exp) {
1083						pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n",
1084						       *t == I915_GTT_VIEW_ROTATED ? "Rotated" : "Remapped",
1085						       exp, val);
1086						i915_vma_unpin_iomap(vma);
1087						err = -EINVAL;
1088						goto out;
1089					}
1090				}
1091			}
1092			i915_vma_unpin_iomap(vma);
1093
1094			cond_resched();
1095		}
1096	}
1097
1098out:
1099	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1100	i915_gem_object_put(obj);
1101
1102	return err;
1103}
1104
1105int i915_vma_live_selftests(struct drm_i915_private *i915)
1106{
1107	static const struct i915_subtest tests[] = {
1108		SUBTEST(igt_vma_remapped_gtt),
1109	};
1110
1111	return i915_live_subtests(tests, i915);
1112}