i915_gem_context.c

   1/*
   2 * SPDX-License-Identifier: MIT
   3 *
   4 * Copyright © 2017 Intel Corporation
   5 */
   6
   7#include <linux/prime_numbers.h>
   8#include <linux/string_helpers.h>
   9
  10#include "gem/i915_gem_internal.h"
  11#include "gem/i915_gem_pm.h"
  12#include "gt/intel_engine_pm.h"
  13#include "gt/intel_engine_regs.h"
  14#include "gt/intel_gt.h"
  15#include "gt/intel_gt_requests.h"
  16#include "gt/intel_reset.h"
  17#include "i915_selftest.h"
  18
  19#include "gem/selftests/igt_gem_utils.h"
  20#include "selftests/i915_random.h"
  21#include "selftests/igt_flush_test.h"
  22#include "selftests/igt_live_test.h"
  23#include "selftests/igt_reset.h"
  24#include "selftests/igt_spinner.h"
  25#include "selftests/mock_drm.h"
  26#include "selftests/mock_gem_device.h"
  27
  28#include "huge_gem_object.h"
  29#include "igt_gem_utils.h"
  30
  31#define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
  32
  33static int live_nop_switch(void *arg)
  34{
  35	const unsigned int nctx = 1024;
  36	struct drm_i915_private *i915 = arg;
  37	struct intel_engine_cs *engine;
  38	struct i915_gem_context **ctx;
  39	struct igt_live_test t;
  40	struct file *file;
  41	unsigned long n;
  42	int err = -ENODEV;
  43
  44	/*
  45	 * Create as many contexts as we can feasibly get away with
  46	 * and check we can switch between them rapidly.
  47	 *
  48	 * Serves as very simple stress test for submission and HW switching
  49	 * between contexts.
  50	 */
  51
  52	if (!DRIVER_CAPS(i915)->has_logical_contexts)
  53		return 0;
  54
  55	file = mock_file(i915);
  56	if (IS_ERR(file))
  57		return PTR_ERR(file);
  58
  59	ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL);
  60	if (!ctx) {
  61		err = -ENOMEM;
  62		goto out_file;
  63	}
  64
  65	for (n = 0; n < nctx; n++) {
  66		ctx[n] = live_context(i915, file);
  67		if (IS_ERR(ctx[n])) {
  68			err = PTR_ERR(ctx[n]);
  69			goto out_ctx;
  70		}
  71	}
  72
  73	for_each_uabi_engine(engine, i915) {
  74		struct i915_request *rq = NULL;
  75		unsigned long end_time, prime;
  76		ktime_t times[2] = {};
  77
  78		times[0] = ktime_get_raw();
  79		for (n = 0; n < nctx; n++) {
  80			struct i915_request *this;
  81
  82			this = igt_request_alloc(ctx[n], engine);
  83			if (IS_ERR(this)) {
  84				err = PTR_ERR(this);
  85				goto out_ctx;
  86			}
  87			if (rq) {
  88				i915_request_await_dma_fence(this, &rq->fence);
  89				i915_request_put(rq);
  90			}
  91			rq = i915_request_get(this);
  92			i915_request_add(this);
  93		}
  94		if (i915_request_wait(rq, 0, 10 * HZ) < 0) {
  95			pr_err("Failed to populated %d contexts\n", nctx);
  96			intel_gt_set_wedged(engine->gt);
  97			i915_request_put(rq);
  98			err = -EIO;
  99			goto out_ctx;
 100		}
 101		i915_request_put(rq);
 102
 103		times[1] = ktime_get_raw();
 104
 105		pr_info("Populated %d contexts on %s in %lluns\n",
 106			nctx, engine->name, ktime_to_ns(times[1] - times[0]));
 107
 108		err = igt_live_test_begin(&t, i915, __func__, engine->name);
 109		if (err)
 110			goto out_ctx;
 111
 112		end_time = jiffies + i915_selftest.timeout_jiffies;
 113		for_each_prime_number_from(prime, 2, 8192) {
 114			times[1] = ktime_get_raw();
 115
 116			rq = NULL;
 117			for (n = 0; n < prime; n++) {
 118				struct i915_request *this;
 119
 120				this = igt_request_alloc(ctx[n % nctx], engine);
 121				if (IS_ERR(this)) {
 122					err = PTR_ERR(this);
 123					goto out_ctx;
 124				}
 125
 126				if (rq) { /* Force submission order */
 127					i915_request_await_dma_fence(this, &rq->fence);
 128					i915_request_put(rq);
 129				}
 130
 131				/*
 132				 * This space is left intentionally blank.
 133				 *
 134				 * We do not actually want to perform any
 135				 * action with this request, we just want
 136				 * to measure the latency in allocation
 137				 * and submission of our breadcrumbs -
 138				 * ensuring that the bare request is sufficient
 139				 * for the system to work (i.e. proper HEAD
 140				 * tracking of the rings, interrupt handling,
 141				 * etc). It also gives us the lowest bounds
 142				 * for latency.
 143				 */
 144
 145				rq = i915_request_get(this);
 146				i915_request_add(this);
 147			}
 148			GEM_BUG_ON(!rq);
 149			if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 150				pr_err("Switching between %ld contexts timed out\n",
 151				       prime);
 152				intel_gt_set_wedged(engine->gt);
 153				i915_request_put(rq);
 154				break;
 155			}
 156			i915_request_put(rq);
 157
 158			times[1] = ktime_sub(ktime_get_raw(), times[1]);
 159			if (prime == 2)
 160				times[0] = times[1];
 161
 162			if (__igt_timeout(end_time, NULL))
 163				break;
 164		}
 165
 166		err = igt_live_test_end(&t);
 167		if (err)
 168			goto out_ctx;
 169
 170		pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n",
 171			engine->name,
 172			ktime_to_ns(times[0]),
 173			prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1));
 174	}
 175
 176out_ctx:
 177	kfree(ctx);
 178out_file:
 179	fput(file);
 180	return err;
 181}
 182
 183struct parallel_switch {
 184	struct kthread_worker *worker;
 185	struct kthread_work work;
 186	struct intel_context *ce[2];
 187	int result;
 188};
 189
 190static void __live_parallel_switch1(struct kthread_work *work)
 191{
 192	struct parallel_switch *arg =
 193		container_of(work, typeof(*arg), work);
 194	IGT_TIMEOUT(end_time);
 195	unsigned long count;
 196
 197	count = 0;
 198	arg->result = 0;
 199	do {
 200		struct i915_request *rq = NULL;
 201		int n;
 202
 203		for (n = 0; !arg->result && n < ARRAY_SIZE(arg->ce); n++) {
 204			struct i915_request *prev = rq;
 205
 206			rq = i915_request_create(arg->ce[n]);
 207			if (IS_ERR(rq)) {
 208				i915_request_put(prev);
 209				arg->result = PTR_ERR(rq);
 210				break;
 211			}
 212
 213			i915_request_get(rq);
 214			if (prev) {
 215				arg->result =
 216					i915_request_await_dma_fence(rq,
 217								     &prev->fence);
 218				i915_request_put(prev);
 219			}
 220
 221			i915_request_add(rq);
 222		}
 223
 224		if (IS_ERR_OR_NULL(rq))
 225			break;
 226
 227		if (i915_request_wait(rq, 0, HZ) < 0)
 228			arg->result = -ETIME;
 229
 230		i915_request_put(rq);
 231
 232		count++;
 233	} while (!arg->result && !__igt_timeout(end_time, NULL));
 234
 235	pr_info("%s: %lu switches (sync) <%d>\n",
 236		arg->ce[0]->engine->name, count, arg->result);
 237}
 238
 239static void __live_parallel_switchN(struct kthread_work *work)
 240{
 241	struct parallel_switch *arg =
 242		container_of(work, typeof(*arg), work);
 243	struct i915_request *rq = NULL;
 244	IGT_TIMEOUT(end_time);
 245	unsigned long count;
 246	int n;
 247
 248	count = 0;
 249	arg->result = 0;
 250	do {
 251		for (n = 0; !arg->result && n < ARRAY_SIZE(arg->ce); n++) {
 252			struct i915_request *prev = rq;
 253
 254			rq = i915_request_create(arg->ce[n]);
 255			if (IS_ERR(rq)) {
 256				i915_request_put(prev);
 257				arg->result = PTR_ERR(rq);
 258				break;
 259			}
 260
 261			i915_request_get(rq);
 262			if (prev) {
 263				arg->result =
 264					i915_request_await_dma_fence(rq,
 265								     &prev->fence);
 266				i915_request_put(prev);
 267			}
 268
 269			i915_request_add(rq);
 270		}
 271
 272		count++;
 273	} while (!arg->result && !__igt_timeout(end_time, NULL));
 274
 275	if (!IS_ERR_OR_NULL(rq))
 276		i915_request_put(rq);
 277
 278	pr_info("%s: %lu switches (many) <%d>\n",
 279		arg->ce[0]->engine->name, count, arg->result);
 280}
 281
 282static int live_parallel_switch(void *arg)
 283{
 284	struct drm_i915_private *i915 = arg;
 285	static void (* const func[])(struct kthread_work *) = {
 286		__live_parallel_switch1,
 287		__live_parallel_switchN,
 288		NULL,
 289	};
 290	struct parallel_switch *data = NULL;
 291	struct i915_gem_engines *engines;
 292	struct i915_gem_engines_iter it;
 293	void (* const *fn)(struct kthread_work *);
 294	struct i915_gem_context *ctx;
 295	struct intel_context *ce;
 296	struct file *file;
 297	int n, m, count;
 298	int err = 0;
 299
 300	/*
 301	 * Check we can process switches on all engines simultaneously.
 302	 */
 303
 304	if (!DRIVER_CAPS(i915)->has_logical_contexts)
 305		return 0;
 306
 307	file = mock_file(i915);
 308	if (IS_ERR(file))
 309		return PTR_ERR(file);
 310
 311	ctx = live_context(i915, file);
 312	if (IS_ERR(ctx)) {
 313		err = PTR_ERR(ctx);
 314		goto out_file;
 315	}
 316
 317	engines = i915_gem_context_lock_engines(ctx);
 318	count = engines->num_engines;
 319
 320	data = kcalloc(count, sizeof(*data), GFP_KERNEL);
 321	if (!data) {
 322		i915_gem_context_unlock_engines(ctx);
 323		err = -ENOMEM;
 324		goto out_file;
 325	}
 326
 327	m = 0; /* Use the first context as our template for the engines */
 328	for_each_gem_engine(ce, engines, it) {
 329		err = intel_context_pin(ce);
 330		if (err) {
 331			i915_gem_context_unlock_engines(ctx);
 332			goto out;
 333		}
 334		data[m++].ce[0] = intel_context_get(ce);
 335	}
 336	i915_gem_context_unlock_engines(ctx);
 337
 338	/* Clone the same set of engines into the other contexts */
 339	for (n = 1; n < ARRAY_SIZE(data->ce); n++) {
 340		ctx = live_context(i915, file);
 341		if (IS_ERR(ctx)) {
 342			err = PTR_ERR(ctx);
 343			goto out;
 344		}
 345
 346		for (m = 0; m < count; m++) {
 347			if (!data[m].ce[0])
 348				continue;
 349
 350			ce = intel_context_create(data[m].ce[0]->engine);
 351			if (IS_ERR(ce)) {
 352				err = PTR_ERR(ce);
 353				goto out;
 354			}
 355
 356			err = intel_context_pin(ce);
 357			if (err) {
 358				intel_context_put(ce);
 359				goto out;
 360			}
 361
 362			data[m].ce[n] = ce;
 363		}
 364	}
 365
 366	for (n = 0; n < count; n++) {
 367		struct kthread_worker *worker;
 368
 369		if (!data[n].ce[0])
 370			continue;
 371
 372		worker = kthread_run_worker(0, "igt/parallel:%s",
 373					       data[n].ce[0]->engine->name);
 374		if (IS_ERR(worker)) {
 375			err = PTR_ERR(worker);
 376			goto out;
 377		}
 378
 379		data[n].worker = worker;
 380	}
 381
 382	for (fn = func; !err && *fn; fn++) {
 383		struct igt_live_test t;
 384
 385		err = igt_live_test_begin(&t, i915, __func__, "");
 386		if (err)
 387			break;
 388
 389		for (n = 0; n < count; n++) {
 390			if (!data[n].ce[0])
 391				continue;
 392
 393			data[n].result = 0;
 394			kthread_init_work(&data[n].work, *fn);
 395			kthread_queue_work(data[n].worker, &data[n].work);
 396		}
 397
 398		for (n = 0; n < count; n++) {
 399			if (data[n].ce[0]) {
 400				kthread_flush_work(&data[n].work);
 401				if (data[n].result && !err)
 402					err = data[n].result;
 403			}
 404		}
 405
 406		if (igt_live_test_end(&t)) {
 407			err = err ?: -EIO;
 408			break;
 409		}
 410	}
 411
 412out:
 413	for (n = 0; n < count; n++) {
 414		for (m = 0; m < ARRAY_SIZE(data->ce); m++) {
 415			if (!data[n].ce[m])
 416				continue;
 417
 418			intel_context_unpin(data[n].ce[m]);
 419			intel_context_put(data[n].ce[m]);
 420		}
 421
 422		if (data[n].worker)
 423			kthread_destroy_worker(data[n].worker);
 424	}
 425	kfree(data);
 426out_file:
 427	fput(file);
 428	return err;
 429}
 430
 431static unsigned long real_page_count(struct drm_i915_gem_object *obj)
 432{
 433	return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
 434}
 435
 436static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
 437{
 438	return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
 439}
 440
 441static int gpu_fill(struct intel_context *ce,
 442		    struct drm_i915_gem_object *obj,
 443		    unsigned int dw)
 444{
 445	struct i915_vma *vma;
 446	int err;
 447
 448	GEM_BUG_ON(obj->base.size > ce->vm->total);
 449	GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
 450
 451	vma = i915_vma_instance(obj, ce->vm, NULL);
 452	if (IS_ERR(vma))
 453		return PTR_ERR(vma);
 454
 455	err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
 456	if (err)
 457		return err;
 458
 459	/*
 460	 * Within the GTT the huge objects maps every page onto
 461	 * its 1024 real pages (using phys_pfn = dma_pfn % 1024).
 462	 * We set the nth dword within the page using the nth
 463	 * mapping via the GTT - this should exercise the GTT mapping
 464	 * whilst checking that each context provides a unique view
 465	 * into the object.
 466	 */
 467	err = igt_gpu_fill_dw(ce, vma,
 468			      (dw * real_page_count(obj)) << PAGE_SHIFT |
 469			      (dw * sizeof(u32)),
 470			      real_page_count(obj),
 471			      dw);
 472	i915_vma_unpin(vma);
 473
 474	return err;
 475}
 476
 477static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
 478{
 479	const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
 480	unsigned int need_flush;
 481	unsigned long n, m;
 482	int err;
 483
 484	i915_gem_object_lock(obj, NULL);
 485	err = i915_gem_object_prepare_write(obj, &need_flush);
 486	if (err)
 487		goto out;
 488
 489	for (n = 0; n < real_page_count(obj); n++) {
 490		u32 *map;
 491
 492		map = kmap_local_page(i915_gem_object_get_page(obj, n));
 493		for (m = 0; m < DW_PER_PAGE; m++)
 494			map[m] = value;
 495		if (!has_llc)
 496			drm_clflush_virt_range(map, PAGE_SIZE);
 497		kunmap_local(map);
 498	}
 499
 500	i915_gem_object_finish_access(obj);
 501	obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
 502	obj->write_domain = 0;
 503out:
 504	i915_gem_object_unlock(obj);
 505	return err;
 506}
 507
 508static noinline int cpu_check(struct drm_i915_gem_object *obj,
 509			      unsigned int idx, unsigned int max)
 510{
 511	unsigned int needs_flush;
 512	unsigned long n;
 513	int err;
 514
 515	i915_gem_object_lock(obj, NULL);
 516	err = i915_gem_object_prepare_read(obj, &needs_flush);
 517	if (err)
 518		goto out_unlock;
 519
 520	for (n = 0; n < real_page_count(obj); n++) {
 521		u32 *map, m;
 522
 523		map = kmap_local_page(i915_gem_object_get_page(obj, n));
 524		if (needs_flush & CLFLUSH_BEFORE)
 525			drm_clflush_virt_range(map, PAGE_SIZE);
 526
 527		for (m = 0; m < max; m++) {
 528			if (map[m] != m) {
 529				pr_err("%pS: Invalid value at object %d page %ld/%ld, offset %d/%d: found %x expected %x\n",
 530				       __builtin_return_address(0), idx,
 531				       n, real_page_count(obj), m, max,
 532				       map[m], m);
 533				err = -EINVAL;
 534				goto out_unmap;
 535			}
 536		}
 537
 538		for (; m < DW_PER_PAGE; m++) {
 539			if (map[m] != STACK_MAGIC) {
 540				pr_err("%pS: Invalid value at object %d page %ld, offset %d: found %x expected %x (uninitialised)\n",
 541				       __builtin_return_address(0), idx, n, m,
 542				       map[m], STACK_MAGIC);
 543				err = -EINVAL;
 544				goto out_unmap;
 545			}
 546		}
 547
 548out_unmap:
 549		kunmap_local(map);
 550		if (err)
 551			break;
 552	}
 553
 554	i915_gem_object_finish_access(obj);
 555out_unlock:
 556	i915_gem_object_unlock(obj);
 557	return err;
 558}
 559
 560static int file_add_object(struct file *file, struct drm_i915_gem_object *obj)
 561{
 562	int err;
 563
 564	GEM_BUG_ON(obj->base.handle_count);
 565
 566	/* tie the object to the drm_file for easy reaping */
 567	err = idr_alloc(&to_drm_file(file)->object_idr,
 568			&obj->base, 1, 0, GFP_KERNEL);
 569	if (err < 0)
 570		return err;
 571
 572	i915_gem_object_get(obj);
 573	obj->base.handle_count++;
 574	return 0;
 575}
 576
 577static struct drm_i915_gem_object *
 578create_test_object(struct i915_address_space *vm,
 579		   struct file *file,
 580		   struct list_head *objects)
 581{
 582	struct drm_i915_gem_object *obj;
 583	u64 size;
 584	int err;
 585
 586	/* Keep in GEM's good graces */
 587	intel_gt_retire_requests(vm->gt);
 588
 589	size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
 590	size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
 591
 592	obj = huge_gem_object(vm->i915, DW_PER_PAGE * PAGE_SIZE, size);
 593	if (IS_ERR(obj))
 594		return obj;
 595
 596	err = file_add_object(file, obj);
 597	i915_gem_object_put(obj);
 598	if (err)
 599		return ERR_PTR(err);
 600
 601	err = cpu_fill(obj, STACK_MAGIC);
 602	if (err) {
 603		pr_err("Failed to fill object with cpu, err=%d\n",
 604		       err);
 605		return ERR_PTR(err);
 606	}
 607
 608	list_add_tail(&obj->st_link, objects);
 609	return obj;
 610}
 611
 612static unsigned long max_dwords(struct drm_i915_gem_object *obj)
 613{
 614	unsigned long npages = fake_page_count(obj);
 615
 616	GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
 617	return npages / DW_PER_PAGE;
 618}
 619
 620static void throttle_release(struct i915_request **q, int count)
 621{
 622	int i;
 623
 624	for (i = 0; i < count; i++) {
 625		if (IS_ERR_OR_NULL(q[i]))
 626			continue;
 627
 628		i915_request_put(fetch_and_zero(&q[i]));
 629	}
 630}
 631
 632static int throttle(struct intel_context *ce,
 633		    struct i915_request **q, int count)
 634{
 635	int i;
 636
 637	if (!IS_ERR_OR_NULL(q[0])) {
 638		if (i915_request_wait(q[0],
 639				      I915_WAIT_INTERRUPTIBLE,
 640				      MAX_SCHEDULE_TIMEOUT) < 0)
 641			return -EINTR;
 642
 643		i915_request_put(q[0]);
 644	}
 645
 646	for (i = 0; i < count - 1; i++)
 647		q[i] = q[i + 1];
 648
 649	q[i] = intel_context_create_request(ce);
 650	if (IS_ERR(q[i]))
 651		return PTR_ERR(q[i]);
 652
 653	i915_request_get(q[i]);
 654	i915_request_add(q[i]);
 655
 656	return 0;
 657}
 658
 659static int igt_ctx_exec(void *arg)
 660{
 661	struct drm_i915_private *i915 = arg;
 662	struct intel_engine_cs *engine;
 663	int err = -ENODEV;
 664
 665	/*
 666	 * Create a few different contexts (with different mm) and write
 667	 * through each ctx/mm using the GPU making sure those writes end
 668	 * up in the expected pages of our obj.
 669	 */
 670
 671	if (!DRIVER_CAPS(i915)->has_logical_contexts)
 672		return 0;
 673
 674	for_each_uabi_engine(engine, i915) {
 675		struct drm_i915_gem_object *obj = NULL;
 676		unsigned long ncontexts, ndwords, dw;
 677		struct i915_request *tq[5] = {};
 678		struct igt_live_test t;
 679		IGT_TIMEOUT(end_time);
 680		LIST_HEAD(objects);
 681		struct file *file;
 682
 683		if (!intel_engine_can_store_dword(engine))
 684			continue;
 685
 686		if (!engine->context_size)
 687			continue; /* No logical context support in HW */
 688
 689		file = mock_file(i915);
 690		if (IS_ERR(file))
 691			return PTR_ERR(file);
 692
 693		err = igt_live_test_begin(&t, i915, __func__, engine->name);
 694		if (err)
 695			goto out_file;
 696
 697		ncontexts = 0;
 698		ndwords = 0;
 699		dw = 0;
 700		while (!time_after(jiffies, end_time)) {
 701			struct i915_gem_context *ctx;
 702			struct intel_context *ce;
 703
 704			ctx = kernel_context(i915, NULL);
 705			if (IS_ERR(ctx)) {
 706				err = PTR_ERR(ctx);
 707				goto out_file;
 708			}
 709
 710			ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
 711			GEM_BUG_ON(IS_ERR(ce));
 712
 713			if (!obj) {
 714				obj = create_test_object(ce->vm, file, &objects);
 715				if (IS_ERR(obj)) {
 716					err = PTR_ERR(obj);
 717					intel_context_put(ce);
 718					kernel_context_close(ctx);
 719					goto out_file;
 720				}
 721			}
 722
 723			err = gpu_fill(ce, obj, dw);
 724			if (err) {
 725				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
 726				       ndwords, dw, max_dwords(obj),
 727				       engine->name,
 728				       str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
 729				       err);
 730				intel_context_put(ce);
 731				kernel_context_close(ctx);
 732				goto out_file;
 733			}
 734
 735			err = throttle(ce, tq, ARRAY_SIZE(tq));
 736			if (err) {
 737				intel_context_put(ce);
 738				kernel_context_close(ctx);
 739				goto out_file;
 740			}
 741
 742			if (++dw == max_dwords(obj)) {
 743				obj = NULL;
 744				dw = 0;
 745			}
 746
 747			ndwords++;
 748			ncontexts++;
 749
 750			intel_context_put(ce);
 751			kernel_context_close(ctx);
 752		}
 753
 754		pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
 755			ncontexts, engine->name, ndwords);
 756
 757		ncontexts = dw = 0;
 758		list_for_each_entry(obj, &objects, st_link) {
 759			unsigned int rem =
 760				min_t(unsigned int, ndwords - dw, max_dwords(obj));
 761
 762			err = cpu_check(obj, ncontexts++, rem);
 763			if (err)
 764				break;
 765
 766			dw += rem;
 767		}
 768
 769out_file:
 770		throttle_release(tq, ARRAY_SIZE(tq));
 771		if (igt_live_test_end(&t))
 772			err = -EIO;
 773
 774		fput(file);
 775		if (err)
 776			return err;
 777
 778		i915_gem_drain_freed_objects(i915);
 779	}
 780
 781	return 0;
 782}
 783
 784static int igt_shared_ctx_exec(void *arg)
 785{
 786	struct drm_i915_private *i915 = arg;
 787	struct i915_request *tq[5] = {};
 788	struct i915_gem_context *parent;
 789	struct intel_engine_cs *engine;
 790	struct igt_live_test t;
 791	struct file *file;
 792	int err = 0;
 793
 794	/*
 795	 * Create a few different contexts with the same mm and write
 796	 * through each ctx using the GPU making sure those writes end
 797	 * up in the expected pages of our obj.
 798	 */
 799	if (!DRIVER_CAPS(i915)->has_logical_contexts)
 800		return 0;
 801
 802	file = mock_file(i915);
 803	if (IS_ERR(file))
 804		return PTR_ERR(file);
 805
 806	parent = live_context(i915, file);
 807	if (IS_ERR(parent)) {
 808		err = PTR_ERR(parent);
 809		goto out_file;
 810	}
 811
 812	if (!parent->vm) { /* not full-ppgtt; nothing to share */
 813		err = 0;
 814		goto out_file;
 815	}
 816
 817	err = igt_live_test_begin(&t, i915, __func__, "");
 818	if (err)
 819		goto out_file;
 820
 821	for_each_uabi_engine(engine, i915) {
 822		unsigned long ncontexts, ndwords, dw;
 823		struct drm_i915_gem_object *obj = NULL;
 824		IGT_TIMEOUT(end_time);
 825		LIST_HEAD(objects);
 826
 827		if (!intel_engine_can_store_dword(engine))
 828			continue;
 829
 830		dw = 0;
 831		ndwords = 0;
 832		ncontexts = 0;
 833		while (!time_after(jiffies, end_time)) {
 834			struct i915_gem_context *ctx;
 835			struct intel_context *ce;
 836
 837			ctx = kernel_context(i915, parent->vm);
 838			if (IS_ERR(ctx)) {
 839				err = PTR_ERR(ctx);
 840				goto out_test;
 841			}
 842
 843			ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
 844			GEM_BUG_ON(IS_ERR(ce));
 845
 846			if (!obj) {
 847				obj = create_test_object(parent->vm,
 848							 file, &objects);
 849				if (IS_ERR(obj)) {
 850					err = PTR_ERR(obj);
 851					intel_context_put(ce);
 852					kernel_context_close(ctx);
 853					goto out_test;
 854				}
 855			}
 856
 857			err = gpu_fill(ce, obj, dw);
 858			if (err) {
 859				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
 860				       ndwords, dw, max_dwords(obj),
 861				       engine->name,
 862				       str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
 863				       err);
 864				intel_context_put(ce);
 865				kernel_context_close(ctx);
 866				goto out_test;
 867			}
 868
 869			err = throttle(ce, tq, ARRAY_SIZE(tq));
 870			if (err) {
 871				intel_context_put(ce);
 872				kernel_context_close(ctx);
 873				goto out_test;
 874			}
 875
 876			if (++dw == max_dwords(obj)) {
 877				obj = NULL;
 878				dw = 0;
 879			}
 880
 881			ndwords++;
 882			ncontexts++;
 883
 884			intel_context_put(ce);
 885			kernel_context_close(ctx);
 886		}
 887		pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
 888			ncontexts, engine->name, ndwords);
 889
 890		ncontexts = dw = 0;
 891		list_for_each_entry(obj, &objects, st_link) {
 892			unsigned int rem =
 893				min_t(unsigned int, ndwords - dw, max_dwords(obj));
 894
 895			err = cpu_check(obj, ncontexts++, rem);
 896			if (err)
 897				goto out_test;
 898
 899			dw += rem;
 900		}
 901
 902		i915_gem_drain_freed_objects(i915);
 903	}
 904out_test:
 905	throttle_release(tq, ARRAY_SIZE(tq));
 906	if (igt_live_test_end(&t))
 907		err = -EIO;
 908out_file:
 909	fput(file);
 910	return err;
 911}
 912
 913static int rpcs_query_batch(struct drm_i915_gem_object *rpcs,
 914			    struct i915_vma *vma,
 915			    struct intel_engine_cs *engine)
 916{
 917	u32 *cmd;
 918
 919	GEM_BUG_ON(GRAPHICS_VER(vma->vm->i915) < 8);
 920
 921	cmd = i915_gem_object_pin_map(rpcs, I915_MAP_WB);
 922	if (IS_ERR(cmd))
 923		return PTR_ERR(cmd);
 924
 925	*cmd++ = MI_STORE_REGISTER_MEM_GEN8;
 926	*cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE(engine->mmio_base));
 927	*cmd++ = lower_32_bits(i915_vma_offset(vma));
 928	*cmd++ = upper_32_bits(i915_vma_offset(vma));
 929	*cmd = MI_BATCH_BUFFER_END;
 930
 931	__i915_gem_object_flush_map(rpcs, 0, 64);
 932	i915_gem_object_unpin_map(rpcs);
 933
 934	intel_gt_chipset_flush(vma->vm->gt);
 935
 936	return 0;
 937}
 938
 939static int
 940emit_rpcs_query(struct drm_i915_gem_object *obj,
 941		struct intel_context *ce,
 942		struct i915_request **rq_out)
 943{
 944	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 945	struct i915_request *rq;
 946	struct i915_gem_ww_ctx ww;
 947	struct i915_vma *batch;
 948	struct i915_vma *vma;
 949	struct drm_i915_gem_object *rpcs;
 950	int err;
 951
 952	GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
 953
 954	if (GRAPHICS_VER(i915) < 8)
 955		return -EINVAL;
 956
 957	vma = i915_vma_instance(obj, ce->vm, NULL);
 958	if (IS_ERR(vma))
 959		return PTR_ERR(vma);
 960
 961	rpcs = i915_gem_object_create_internal(i915, PAGE_SIZE);
 962	if (IS_ERR(rpcs))
 963		return PTR_ERR(rpcs);
 964
 965	i915_gem_ww_ctx_init(&ww, false);
 966
 967	batch = i915_vma_instance(rpcs, ce->vm, NULL);
 968	if (IS_ERR(batch)) {
 969		err = PTR_ERR(batch);
 970		goto err_put;
 971	}
 972
 973retry:
 974	err = i915_gem_object_lock(obj, &ww);
 975	if (!err)
 976		err = i915_gem_object_lock(rpcs, &ww);
 977	if (!err)
 978		err = i915_gem_object_set_to_gtt_domain(obj, false);
 979	if (!err)
 980		err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
 981	if (err)
 982		goto err_put;
 983
 984	err = i915_vma_pin_ww(batch, &ww, 0, 0, PIN_USER);
 985	if (err)
 986		goto err_vma;
 987
 988	err = rpcs_query_batch(rpcs, vma, ce->engine);
 989	if (err)
 990		goto err_batch;
 991
 992	rq = i915_request_create(ce);
 993	if (IS_ERR(rq)) {
 994		err = PTR_ERR(rq);
 995		goto err_batch;
 996	}
 997
 998	err = i915_vma_move_to_active(batch, rq, 0);
 999	if (err)
1000		goto skip_request;
1001
1002	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1003	if (err)
1004		goto skip_request;
1005
1006	if (rq->engine->emit_init_breadcrumb) {
1007		err = rq->engine->emit_init_breadcrumb(rq);
1008		if (err)
1009			goto skip_request;
1010	}
1011
1012	err = rq->engine->emit_bb_start(rq,
1013					i915_vma_offset(batch),
1014					i915_vma_size(batch),
1015					0);
1016	if (err)
1017		goto skip_request;
1018
1019	*rq_out = i915_request_get(rq);
1020
1021skip_request:
1022	if (err)
1023		i915_request_set_error_once(rq, err);
1024	i915_request_add(rq);
1025err_batch:
1026	i915_vma_unpin(batch);
1027err_vma:
1028	i915_vma_unpin(vma);
1029err_put:
1030	if (err == -EDEADLK) {
1031		err = i915_gem_ww_ctx_backoff(&ww);
1032		if (!err)
1033			goto retry;
1034	}
1035	i915_gem_ww_ctx_fini(&ww);
1036	i915_gem_object_put(rpcs);
1037	return err;
1038}
1039
1040#define TEST_IDLE	BIT(0)
1041#define TEST_BUSY	BIT(1)
1042#define TEST_RESET	BIT(2)
1043
1044static int
1045__sseu_prepare(const char *name,
1046	       unsigned int flags,
1047	       struct intel_context *ce,
1048	       struct igt_spinner **spin)
1049{
1050	struct i915_request *rq;
1051	int ret;
1052
1053	*spin = NULL;
1054	if (!(flags & (TEST_BUSY | TEST_RESET)))
1055		return 0;
1056
1057	*spin = kzalloc(sizeof(**spin), GFP_KERNEL);
1058	if (!*spin)
1059		return -ENOMEM;
1060
1061	ret = igt_spinner_init(*spin, ce->engine->gt);
1062	if (ret)
1063		goto err_free;
1064
1065	rq = igt_spinner_create_request(*spin, ce, MI_NOOP);
1066	if (IS_ERR(rq)) {
1067		ret = PTR_ERR(rq);
1068		goto err_fini;
1069	}
1070
1071	i915_request_add(rq);
1072
1073	if (!igt_wait_for_spinner(*spin, rq)) {
1074		pr_err("%s: Spinner failed to start!\n", name);
1075		ret = -ETIMEDOUT;
1076		goto err_end;
1077	}
1078
1079	return 0;
1080
1081err_end:
1082	igt_spinner_end(*spin);
1083err_fini:
1084	igt_spinner_fini(*spin);
1085err_free:
1086	kfree(fetch_and_zero(spin));
1087	return ret;
1088}
1089
1090static int
1091__read_slice_count(struct intel_context *ce,
1092		   struct drm_i915_gem_object *obj,
1093		   struct igt_spinner *spin,
1094		   u32 *rpcs)
1095{
1096	struct i915_request *rq = NULL;
1097	u32 s_mask, s_shift;
1098	unsigned int cnt;
1099	u32 *buf, val;
1100	long ret;
1101
1102	ret = emit_rpcs_query(obj, ce, &rq);
1103	if (ret)
1104		return ret;
1105
1106	if (spin)
1107		igt_spinner_end(spin);
1108
1109	ret = i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1110	i915_request_put(rq);
1111	if (ret < 0)
1112		return ret;
1113
1114	buf = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1115	if (IS_ERR(buf)) {
1116		ret = PTR_ERR(buf);
1117		return ret;
1118	}
1119
1120	if (GRAPHICS_VER(ce->engine->i915) >= 11) {
1121		s_mask = GEN11_RPCS_S_CNT_MASK;
1122		s_shift = GEN11_RPCS_S_CNT_SHIFT;
1123	} else {
1124		s_mask = GEN8_RPCS_S_CNT_MASK;
1125		s_shift = GEN8_RPCS_S_CNT_SHIFT;
1126	}
1127
1128	val = *buf;
1129	cnt = (val & s_mask) >> s_shift;
1130	*rpcs = val;
1131
1132	i915_gem_object_unpin_map(obj);
1133
1134	return cnt;
1135}
1136
1137static int
1138__check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected,
1139	     const char *prefix, const char *suffix)
1140{
1141	if (slices == expected)
1142		return 0;
1143
1144	if (slices < 0) {
1145		pr_err("%s: %s read slice count failed with %d%s\n",
1146		       name, prefix, slices, suffix);
1147		return slices;
1148	}
1149
1150	pr_err("%s: %s slice count %d is not %u%s\n",
1151	       name, prefix, slices, expected, suffix);
1152
1153	pr_info("RPCS=0x%x; %u%sx%u%s\n",
1154		rpcs, slices,
1155		(rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "",
1156		(rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT,
1157		(rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : "");
1158
1159	return -EINVAL;
1160}
1161
1162static int
1163__sseu_finish(const char *name,
1164	      unsigned int flags,
1165	      struct intel_context *ce,
1166	      struct drm_i915_gem_object *obj,
1167	      unsigned int expected,
1168	      struct igt_spinner *spin)
1169{
1170	unsigned int slices = hweight32(ce->engine->sseu.slice_mask);
1171	u32 rpcs = 0;
1172	int ret = 0;
1173
1174	if (flags & TEST_RESET) {
1175		ret = intel_engine_reset(ce->engine, "sseu");
1176		if (ret)
1177			goto out;
1178	}
1179
1180	ret = __read_slice_count(ce, obj,
1181				 flags & TEST_RESET ? NULL : spin, &rpcs);
1182	ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!");
1183	if (ret)
1184		goto out;
1185
1186	ret = __read_slice_count(ce->engine->kernel_context, obj, NULL, &rpcs);
1187	ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!");
1188
1189out:
1190	if (spin)
1191		igt_spinner_end(spin);
1192
1193	if ((flags & TEST_IDLE) && ret == 0) {
1194		ret = igt_flush_test(ce->engine->i915);
1195		if (ret)
1196			return ret;
1197
1198		ret = __read_slice_count(ce, obj, NULL, &rpcs);
1199		ret = __check_rpcs(name, rpcs, ret, expected,
1200				   "Context", " after idle!");
1201	}
1202
1203	return ret;
1204}
1205
1206static int
1207__sseu_test(const char *name,
1208	    unsigned int flags,
1209	    struct intel_context *ce,
1210	    struct drm_i915_gem_object *obj,
1211	    struct intel_sseu sseu)
1212{
1213	struct igt_spinner *spin = NULL;
1214	int ret;
1215
1216	intel_engine_pm_get(ce->engine);
1217
1218	ret = __sseu_prepare(name, flags, ce, &spin);
1219	if (ret)
1220		goto out_pm;
1221
1222	ret = intel_context_reconfigure_sseu(ce, sseu);
1223	if (ret)
1224		goto out_spin;
1225
1226	ret = __sseu_finish(name, flags, ce, obj,
1227			    hweight32(sseu.slice_mask), spin);
1228
1229out_spin:
1230	if (spin) {
1231		igt_spinner_end(spin);
1232		igt_spinner_fini(spin);
1233		kfree(spin);
1234	}
1235out_pm:
1236	intel_engine_pm_put(ce->engine);
1237	return ret;
1238}
1239
1240static int
1241__igt_ctx_sseu(struct drm_i915_private *i915,
1242	       const char *name,
1243	       unsigned int flags)
1244{
1245	struct drm_i915_gem_object *obj;
1246	int inst = 0;
1247	int ret = 0;
1248
1249	if (GRAPHICS_VER(i915) < 9)
1250		return 0;
1251
1252	if (flags & TEST_RESET)
1253		igt_global_reset_lock(to_gt(i915));
1254
1255	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1256	if (IS_ERR(obj)) {
1257		ret = PTR_ERR(obj);
1258		goto out_unlock;
1259	}
1260
1261	do {
1262		struct intel_engine_cs *engine;
1263		struct intel_context *ce;
1264		struct intel_sseu pg_sseu;
1265
1266		engine = intel_engine_lookup_user(i915,
1267						  I915_ENGINE_CLASS_RENDER,
1268						  inst++);
1269		if (!engine)
1270			break;
1271
1272		if (hweight32(engine->sseu.slice_mask) < 2)
1273			continue;
1274
1275		if (!engine->gt->info.sseu.has_slice_pg)
1276			continue;
1277
1278		/*
1279		 * Gen11 VME friendly power-gated configuration with
1280		 * half enabled sub-slices.
1281		 */
1282		pg_sseu = engine->sseu;
1283		pg_sseu.slice_mask = 1;
1284		pg_sseu.subslice_mask =
1285			~(~0 << (hweight32(engine->sseu.subslice_mask) / 2));
1286
1287		pr_info("%s: SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n",
1288			engine->name, name, flags,
1289			hweight32(engine->sseu.slice_mask),
1290			hweight32(pg_sseu.slice_mask));
1291
1292		ce = intel_context_create(engine);
1293		if (IS_ERR(ce)) {
1294			ret = PTR_ERR(ce);
1295			goto out_put;
1296		}
1297
1298		ret = intel_context_pin(ce);
1299		if (ret)
1300			goto out_ce;
1301
1302		/* First set the default mask. */
1303		ret = __sseu_test(name, flags, ce, obj, engine->sseu);
1304		if (ret)
1305			goto out_unpin;
1306
1307		/* Then set a power-gated configuration. */
1308		ret = __sseu_test(name, flags, ce, obj, pg_sseu);
1309		if (ret)
1310			goto out_unpin;
1311
1312		/* Back to defaults. */
1313		ret = __sseu_test(name, flags, ce, obj, engine->sseu);
1314		if (ret)
1315			goto out_unpin;
1316
1317		/* One last power-gated configuration for the road. */
1318		ret = __sseu_test(name, flags, ce, obj, pg_sseu);
1319		if (ret)
1320			goto out_unpin;
1321
1322out_unpin:
1323		intel_context_unpin(ce);
1324out_ce:
1325		intel_context_put(ce);
1326	} while (!ret);
1327
1328	if (igt_flush_test(i915))
1329		ret = -EIO;
1330
1331out_put:
1332	i915_gem_object_put(obj);
1333
1334out_unlock:
1335	if (flags & TEST_RESET)
1336		igt_global_reset_unlock(to_gt(i915));
1337
1338	if (ret)
1339		pr_err("%s: Failed with %d!\n", name, ret);
1340
1341	return ret;
1342}
1343
1344static int igt_ctx_sseu(void *arg)
1345{
1346	struct {
1347		const char *name;
1348		unsigned int flags;
1349	} *phase, phases[] = {
1350		{ .name = "basic", .flags = 0 },
1351		{ .name = "idle", .flags = TEST_IDLE },
1352		{ .name = "busy", .flags = TEST_BUSY },
1353		{ .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET },
1354		{ .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE },
1355		{ .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE },
1356	};
1357	unsigned int i;
1358	int ret = 0;
1359
1360	for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases);
1361	     i++, phase++)
1362		ret = __igt_ctx_sseu(arg, phase->name, phase->flags);
1363
1364	return ret;
1365}
1366
1367static int igt_ctx_readonly(void *arg)
1368{
1369	struct drm_i915_private *i915 = arg;
1370	unsigned long idx, ndwords, dw, num_engines;
1371	struct drm_i915_gem_object *obj = NULL;
1372	struct i915_request *tq[5] = {};
1373	struct i915_gem_engines_iter it;
1374	struct i915_address_space *vm;
1375	struct i915_gem_context *ctx;
1376	struct intel_context *ce;
1377	struct igt_live_test t;
1378	I915_RND_STATE(prng);
1379	IGT_TIMEOUT(end_time);
1380	LIST_HEAD(objects);
1381	struct file *file;
1382	int err = -ENODEV;
1383
1384	/*
1385	 * Create a few read-only objects (with the occasional writable object)
1386	 * and try to write into these object checking that the GPU discards
1387	 * any write to a read-only object.
1388	 */
1389
1390	file = mock_file(i915);
1391	if (IS_ERR(file))
1392		return PTR_ERR(file);
1393
1394	err = igt_live_test_begin(&t, i915, __func__, "");
1395	if (err)
1396		goto out_file;
1397
1398	ctx = live_context(i915, file);
1399	if (IS_ERR(ctx)) {
1400		err = PTR_ERR(ctx);
1401		goto out_file;
1402	}
1403
1404	vm = ctx->vm ?: &to_gt(i915)->ggtt->alias->vm;
1405	if (!vm || !vm->has_read_only) {
1406		err = 0;
1407		goto out_file;
1408	}
1409
1410	num_engines = 0;
1411	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it)
1412		if (intel_engine_can_store_dword(ce->engine))
1413			num_engines++;
1414	i915_gem_context_unlock_engines(ctx);
1415
1416	ndwords = 0;
1417	dw = 0;
1418	while (!time_after(jiffies, end_time)) {
1419		for_each_gem_engine(ce,
1420				    i915_gem_context_lock_engines(ctx), it) {
1421			if (!intel_engine_can_store_dword(ce->engine))
1422				continue;
1423
1424			if (!obj) {
1425				obj = create_test_object(ce->vm, file, &objects);
1426				if (IS_ERR(obj)) {
1427					err = PTR_ERR(obj);
1428					i915_gem_context_unlock_engines(ctx);
1429					goto out_file;
1430				}
1431
1432				if (prandom_u32_state(&prng) & 1)
1433					i915_gem_object_set_readonly(obj);
1434			}
1435
1436			err = gpu_fill(ce, obj, dw);
1437			if (err) {
1438				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
1439				       ndwords, dw, max_dwords(obj),
1440				       ce->engine->name,
1441				       str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
1442				       err);
1443				i915_gem_context_unlock_engines(ctx);
1444				goto out_file;
1445			}
1446
1447			err = throttle(ce, tq, ARRAY_SIZE(tq));
1448			if (err) {
1449				i915_gem_context_unlock_engines(ctx);
1450				goto out_file;
1451			}
1452
1453			if (++dw == max_dwords(obj)) {
1454				obj = NULL;
1455				dw = 0;
1456			}
1457			ndwords++;
1458		}
1459		i915_gem_context_unlock_engines(ctx);
1460	}
1461	pr_info("Submitted %lu dwords (across %lu engines)\n",
1462		ndwords, num_engines);
1463
1464	dw = 0;
1465	idx = 0;
1466	list_for_each_entry(obj, &objects, st_link) {
1467		unsigned int rem =
1468			min_t(unsigned int, ndwords - dw, max_dwords(obj));
1469		unsigned int num_writes;
1470
1471		num_writes = rem;
1472		if (i915_gem_object_is_readonly(obj))
1473			num_writes = 0;
1474
1475		err = cpu_check(obj, idx++, num_writes);
1476		if (err)
1477			break;
1478
1479		dw += rem;
1480	}
1481
1482out_file:
1483	throttle_release(tq, ARRAY_SIZE(tq));
1484	if (igt_live_test_end(&t))
1485		err = -EIO;
1486
1487	fput(file);
1488	return err;
1489}
1490
1491static int check_scratch(struct i915_address_space *vm, u64 offset)
1492{
1493	struct drm_mm_node *node;
1494
1495	mutex_lock(&vm->mutex);
1496	node = __drm_mm_interval_first(&vm->mm,
1497				       offset, offset + sizeof(u32) - 1);
1498	mutex_unlock(&vm->mutex);
1499	if (!node || node->start > offset)
1500		return 0;
1501
1502	GEM_BUG_ON(offset >= node->start + node->size);
1503
1504	pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n",
1505	       upper_32_bits(offset), lower_32_bits(offset));
1506	return -EINVAL;
1507}
1508
1509static int write_to_scratch(struct i915_gem_context *ctx,
1510			    struct intel_engine_cs *engine,
1511			    struct drm_i915_gem_object *obj,
1512			    u64 offset, u32 value)
1513{
1514	struct drm_i915_private *i915 = ctx->i915;
1515	struct i915_address_space *vm;
1516	struct i915_request *rq;
1517	struct i915_vma *vma;
1518	u32 *cmd;
1519	int err;
1520
1521	GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1522
1523	err = check_scratch(ctx->vm, offset);
1524	if (err)
1525		return err;
1526
1527	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1528	if (IS_ERR(cmd))
1529		return PTR_ERR(cmd);
1530
1531	*cmd++ = MI_STORE_DWORD_IMM_GEN4;
1532	if (GRAPHICS_VER(i915) >= 8) {
1533		*cmd++ = lower_32_bits(offset);
1534		*cmd++ = upper_32_bits(offset);
1535	} else {
1536		*cmd++ = 0;
1537		*cmd++ = offset;
1538	}
1539	*cmd++ = value;
1540	*cmd = MI_BATCH_BUFFER_END;
1541	__i915_gem_object_flush_map(obj, 0, 64);
1542	i915_gem_object_unpin_map(obj);
1543
1544	intel_gt_chipset_flush(engine->gt);
1545
1546	vm = i915_gem_context_get_eb_vm(ctx);
1547	vma = i915_vma_instance(obj, vm, NULL);
1548	if (IS_ERR(vma)) {
1549		err = PTR_ERR(vma);
1550		goto out_vm;
1551	}
1552
1553	err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1554	if (err)
1555		goto out_vm;
1556
1557	rq = igt_request_alloc(ctx, engine);
1558	if (IS_ERR(rq)) {
1559		err = PTR_ERR(rq);
1560		goto err_unpin;
1561	}
1562
1563	err = igt_vma_move_to_active_unlocked(vma, rq, 0);
1564	if (err)
1565		goto skip_request;
1566
1567	if (rq->engine->emit_init_breadcrumb) {
1568		err = rq->engine->emit_init_breadcrumb(rq);
1569		if (err)
1570			goto skip_request;
1571	}
1572
1573	err = engine->emit_bb_start(rq, i915_vma_offset(vma),
1574				    i915_vma_size(vma), 0);
1575	if (err)
1576		goto skip_request;
1577
1578	i915_vma_unpin(vma);
1579
1580	i915_request_add(rq);
1581
1582	goto out_vm;
1583skip_request:
1584	i915_request_set_error_once(rq, err);
1585	i915_request_add(rq);
1586err_unpin:
1587	i915_vma_unpin(vma);
1588out_vm:
1589	i915_vm_put(vm);
1590
1591	if (!err)
1592		err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1593
1594	return err;
1595}
1596
1597static int read_from_scratch(struct i915_gem_context *ctx,
1598			     struct intel_engine_cs *engine,
1599			     struct drm_i915_gem_object *obj,
1600			     u64 offset, u32 *value)
1601{
1602	struct drm_i915_private *i915 = ctx->i915;
1603	struct i915_address_space *vm;
1604	const u32 result = 0x100;
1605	struct i915_request *rq;
1606	struct i915_vma *vma;
1607	unsigned int flags;
1608	u32 *cmd;
1609	int err;
1610
1611	GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1612
1613	err = check_scratch(ctx->vm, offset);
1614	if (err)
1615		return err;
1616
1617	if (GRAPHICS_VER(i915) >= 8) {
1618		const u32 GPR0 = engine->mmio_base + 0x600;
1619
1620		vm = i915_gem_context_get_eb_vm(ctx);
1621		vma = i915_vma_instance(obj, vm, NULL);
1622		if (IS_ERR(vma)) {
1623			err = PTR_ERR(vma);
1624			goto out_vm;
1625		}
1626
1627		err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1628		if (err)
1629			goto out_vm;
1630
1631		cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1632		if (IS_ERR(cmd)) {
1633			err = PTR_ERR(cmd);
1634			goto err_unpin;
1635		}
1636
1637		memset(cmd, POISON_INUSE, PAGE_SIZE);
1638		*cmd++ = MI_LOAD_REGISTER_MEM_GEN8;
1639		*cmd++ = GPR0;
1640		*cmd++ = lower_32_bits(offset);
1641		*cmd++ = upper_32_bits(offset);
1642		*cmd++ = MI_STORE_REGISTER_MEM_GEN8;
1643		*cmd++ = GPR0;
1644		*cmd++ = result;
1645		*cmd++ = 0;
1646		*cmd = MI_BATCH_BUFFER_END;
1647
1648		i915_gem_object_flush_map(obj);
1649		i915_gem_object_unpin_map(obj);
1650
1651		flags = 0;
1652	} else {
1653		const u32 reg = engine->mmio_base + 0x420;
1654
1655		/* hsw: register access even to 3DPRIM! is protected */
1656		vm = i915_vm_get(&engine->gt->ggtt->vm);
1657		vma = i915_vma_instance(obj, vm, NULL);
1658		if (IS_ERR(vma)) {
1659			err = PTR_ERR(vma);
1660			goto out_vm;
1661		}
1662
1663		err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1664		if (err)
1665			goto out_vm;
1666
1667		cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1668		if (IS_ERR(cmd)) {
1669			err = PTR_ERR(cmd);
1670			goto err_unpin;
1671		}
1672
1673		memset(cmd, POISON_INUSE, PAGE_SIZE);
1674		*cmd++ = MI_LOAD_REGISTER_MEM;
1675		*cmd++ = reg;
1676		*cmd++ = offset;
1677		*cmd++ = MI_STORE_REGISTER_MEM | MI_USE_GGTT;
1678		*cmd++ = reg;
1679		*cmd++ = i915_vma_offset(vma) + result;
1680		*cmd = MI_BATCH_BUFFER_END;
1681
1682		i915_gem_object_flush_map(obj);
1683		i915_gem_object_unpin_map(obj);
1684
1685		flags = I915_DISPATCH_SECURE;
1686	}
1687
1688	intel_gt_chipset_flush(engine->gt);
1689
1690	rq = igt_request_alloc(ctx, engine);
1691	if (IS_ERR(rq)) {
1692		err = PTR_ERR(rq);
1693		goto err_unpin;
1694	}
1695
1696	err = igt_vma_move_to_active_unlocked(vma, rq, EXEC_OBJECT_WRITE);
1697	if (err)
1698		goto skip_request;
1699
1700	if (rq->engine->emit_init_breadcrumb) {
1701		err = rq->engine->emit_init_breadcrumb(rq);
1702		if (err)
1703			goto skip_request;
1704	}
1705
1706	err = engine->emit_bb_start(rq, i915_vma_offset(vma),
1707				    i915_vma_size(vma), flags);
1708	if (err)
1709		goto skip_request;
1710
1711	i915_vma_unpin(vma);
1712
1713	i915_request_add(rq);
1714
1715	i915_gem_object_lock(obj, NULL);
1716	err = i915_gem_object_set_to_cpu_domain(obj, false);
1717	i915_gem_object_unlock(obj);
1718	if (err)
1719		goto out_vm;
1720
1721	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1722	if (IS_ERR(cmd)) {
1723		err = PTR_ERR(cmd);
1724		goto out_vm;
1725	}
1726
1727	*value = cmd[result / sizeof(*cmd)];
1728	i915_gem_object_unpin_map(obj);
1729
1730	goto out_vm;
1731skip_request:
1732	i915_request_set_error_once(rq, err);
1733	i915_request_add(rq);
1734err_unpin:
1735	i915_vma_unpin(vma);
1736out_vm:
1737	i915_vm_put(vm);
1738
1739	if (!err)
1740		err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1741
1742	return err;
1743}
1744
1745static int check_scratch_page(struct i915_gem_context *ctx, u32 *out)
1746{
1747	struct i915_address_space *vm;
1748	u32 *vaddr;
1749	int err = 0;
1750
1751	vm = ctx->vm;
1752	if (!vm)
1753		return -ENODEV;
1754
1755	if (!vm->scratch[0]) {
1756		pr_err("No scratch page!\n");
1757		return -EINVAL;
1758	}
1759
1760	vaddr = __px_vaddr(vm->scratch[0]);
1761
1762	memcpy(out, vaddr, sizeof(*out));
1763	if (memchr_inv(vaddr, *out, PAGE_SIZE)) {
1764		pr_err("Inconsistent initial state of scratch page!\n");
1765		err = -EINVAL;
1766	}
1767
1768	return err;
1769}
1770
1771static int igt_vm_isolation(void *arg)
1772{
1773	struct drm_i915_private *i915 = arg;
1774	struct i915_gem_context *ctx_a, *ctx_b;
1775	struct drm_i915_gem_object *obj_a, *obj_b;
1776	unsigned long num_engines, count;
1777	struct intel_engine_cs *engine;
1778	struct igt_live_test t;
1779	I915_RND_STATE(prng);
1780	struct file *file;
1781	u64 vm_total;
1782	u32 expected;
1783	int err;
1784
1785	if (GRAPHICS_VER(i915) < 7)
1786		return 0;
1787
1788	/*
1789	 * The simple goal here is that a write into one context is not
1790	 * observed in a second (separate page tables and scratch).
1791	 */
1792
1793	file = mock_file(i915);
1794	if (IS_ERR(file))
1795		return PTR_ERR(file);
1796
1797	err = igt_live_test_begin(&t, i915, __func__, "");
1798	if (err)
1799		goto out_file;
1800
1801	ctx_a = live_context(i915, file);
1802	if (IS_ERR(ctx_a)) {
1803		err = PTR_ERR(ctx_a);
1804		goto out_file;
1805	}
1806
1807	ctx_b = live_context(i915, file);
1808	if (IS_ERR(ctx_b)) {
1809		err = PTR_ERR(ctx_b);
1810		goto out_file;
1811	}
1812
1813	/* We can only test vm isolation, if the vm are distinct */
1814	if (ctx_a->vm == ctx_b->vm)
1815		goto out_file;
1816
1817	/* Read the initial state of the scratch page */
1818	err = check_scratch_page(ctx_a, &expected);
1819	if (err)
1820		goto out_file;
1821
1822	err = check_scratch_page(ctx_b, &expected);
1823	if (err)
1824		goto out_file;
1825
1826	vm_total = ctx_a->vm->total;
1827	GEM_BUG_ON(ctx_b->vm->total != vm_total);
1828
1829	obj_a = i915_gem_object_create_internal(i915, PAGE_SIZE);
1830	if (IS_ERR(obj_a)) {
1831		err = PTR_ERR(obj_a);
1832		goto out_file;
1833	}
1834
1835	obj_b = i915_gem_object_create_internal(i915, PAGE_SIZE);
1836	if (IS_ERR(obj_b)) {
1837		err = PTR_ERR(obj_b);
1838		goto put_a;
1839	}
1840
1841	count = 0;
1842	num_engines = 0;
1843	for_each_uabi_engine(engine, i915) {
1844		IGT_TIMEOUT(end_time);
1845		unsigned long this = 0;
1846
1847		if (!intel_engine_can_store_dword(engine))
1848			continue;
1849
1850		/* Not all engines have their own GPR! */
1851		if (GRAPHICS_VER(i915) < 8 && engine->class != RENDER_CLASS)
1852			continue;
1853
1854		while (!__igt_timeout(end_time, NULL)) {
1855			u32 value = 0xc5c5c5c5;
1856			u64 offset;
1857
1858			/* Leave enough space at offset 0 for the batch */
1859			offset = igt_random_offset(&prng,
1860						   I915_GTT_PAGE_SIZE, vm_total,
1861						   sizeof(u32), alignof_dword);
1862
1863			err = write_to_scratch(ctx_a, engine, obj_a,
1864					       offset, 0xdeadbeef);
1865			if (err == 0)
1866				err = read_from_scratch(ctx_b, engine, obj_b,
1867							offset, &value);
1868			if (err)
1869				goto put_b;
1870
1871			if (value != expected) {
1872				pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n",
1873				       engine->name, value,
1874				       upper_32_bits(offset),
1875				       lower_32_bits(offset),
1876				       this);
1877				err = -EINVAL;
1878				goto put_b;
1879			}
1880
1881			this++;
1882		}
1883		count += this;
1884		num_engines++;
1885	}
1886	pr_info("Checked %lu scratch offsets across %lu engines\n",
1887		count, num_engines);
1888
1889put_b:
1890	i915_gem_object_put(obj_b);
1891put_a:
1892	i915_gem_object_put(obj_a);
1893out_file:
1894	if (igt_live_test_end(&t))
1895		err = -EIO;
1896	fput(file);
1897	return err;
1898}
1899
1900int i915_gem_context_live_selftests(struct drm_i915_private *i915)
1901{
1902	static const struct i915_subtest tests[] = {
1903		SUBTEST(live_nop_switch),
1904		SUBTEST(live_parallel_switch),
1905		SUBTEST(igt_ctx_exec),
1906		SUBTEST(igt_ctx_readonly),
1907		SUBTEST(igt_ctx_sseu),
1908		SUBTEST(igt_shared_ctx_exec),
1909		SUBTEST(igt_vm_isolation),
1910	};
1911
1912	if (intel_gt_is_wedged(to_gt(i915)))
1913		return 0;
1914
1915	return i915_live_subtests(tests, i915);
1916}