amdgpu_gfx.c

   1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 */
  25
  26#include <linux/firmware.h>
  27#include <linux/pm_runtime.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_gfx.h"
  31#include "amdgpu_rlc.h"
  32#include "amdgpu_ras.h"
  33#include "amdgpu_reset.h"
  34#include "amdgpu_xcp.h"
  35#include "amdgpu_xgmi.h"
  36#include "amdgpu_mes.h"
  37#include "nvd.h"
  38
  39/* delay 0.1 second to enable gfx off feature */
  40#define GFX_OFF_DELAY_ENABLE         msecs_to_jiffies(100)
  41
  42#define GFX_OFF_NO_DELAY 0
  43
  44/*
  45 * GPU GFX IP block helpers function.
  46 */
  47
  48int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
  49				int pipe, int queue)
  50{
  51	int bit = 0;
  52
  53	bit += mec * adev->gfx.mec.num_pipe_per_mec
  54		* adev->gfx.mec.num_queue_per_pipe;
  55	bit += pipe * adev->gfx.mec.num_queue_per_pipe;
  56	bit += queue;
  57
  58	return bit;
  59}
  60
  61void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
  62				 int *mec, int *pipe, int *queue)
  63{
  64	*queue = bit % adev->gfx.mec.num_queue_per_pipe;
  65	*pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
  66		% adev->gfx.mec.num_pipe_per_mec;
  67	*mec = (bit / adev->gfx.mec.num_queue_per_pipe)
  68	       / adev->gfx.mec.num_pipe_per_mec;
  69
  70}
  71
  72bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
  73				     int xcc_id, int mec, int pipe, int queue)
  74{
  75	return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
  76			adev->gfx.mec_bitmap[xcc_id].queue_bitmap);
  77}
  78
  79static int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
  80				      int me, int pipe, int queue)
  81{
  82	int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
  83	int bit = 0;
  84
  85	bit += me * adev->gfx.me.num_pipe_per_me
  86		* num_queue_per_pipe;
  87	bit += pipe * num_queue_per_pipe;
  88	bit += queue;
  89
  90	return bit;
  91}
  92
  93bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
  94				    int me, int pipe, int queue)
  95{
  96	return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
  97			adev->gfx.me.queue_bitmap);
  98}
  99
 100/**
 101 * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
 102 *
 103 * @mask: array in which the per-shader array disable masks will be stored
 104 * @max_se: number of SEs
 105 * @max_sh: number of SHs
 106 *
 107 * The bitmask of CUs to be disabled in the shader array determined by se and
 108 * sh is stored in mask[se * max_sh + sh].
 109 */
 110void amdgpu_gfx_parse_disable_cu(unsigned int *mask, unsigned int max_se, unsigned int max_sh)
 111{
 112	unsigned int se, sh, cu;
 113	const char *p;
 114
 115	memset(mask, 0, sizeof(*mask) * max_se * max_sh);
 116
 117	if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
 118		return;
 119
 120	p = amdgpu_disable_cu;
 121	for (;;) {
 122		char *next;
 123		int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
 124
 125		if (ret < 3) {
 126			DRM_ERROR("amdgpu: could not parse disable_cu\n");
 127			return;
 128		}
 129
 130		if (se < max_se && sh < max_sh && cu < 16) {
 131			DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu);
 132			mask[se * max_sh + sh] |= 1u << cu;
 133		} else {
 134			DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n",
 135				  se, sh, cu);
 136		}
 137
 138		next = strchr(p, ',');
 139		if (!next)
 140			break;
 141		p = next + 1;
 142	}
 143}
 144
 145static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev)
 146{
 147	return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1;
 148}
 149
 150static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev)
 151{
 152	if (amdgpu_compute_multipipe != -1) {
 153		dev_info(adev->dev, "amdgpu: forcing compute pipe policy %d\n",
 154			 amdgpu_compute_multipipe);
 155		return amdgpu_compute_multipipe == 1;
 156	}
 157
 158	if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
 159		return true;
 160
 161	/* FIXME: spreading the queues across pipes causes perf regressions
 162	 * on POLARIS11 compute workloads */
 163	if (adev->asic_type == CHIP_POLARIS11)
 164		return false;
 165
 166	return adev->gfx.mec.num_mec > 1;
 167}
 168
 169bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev,
 170						struct amdgpu_ring *ring)
 171{
 172	int queue = ring->queue;
 173	int pipe = ring->pipe;
 174
 175	/* Policy: use pipe1 queue0 as high priority graphics queue if we
 176	 * have more than one gfx pipe.
 177	 */
 178	if (amdgpu_gfx_is_graphics_multipipe_capable(adev) &&
 179	    adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) {
 180		int me = ring->me;
 181		int bit;
 182
 183		bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue);
 184		if (ring == &adev->gfx.gfx_ring[bit])
 185			return true;
 186	}
 187
 188	return false;
 189}
 190
 191bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev,
 192					       struct amdgpu_ring *ring)
 193{
 194	/* Policy: use 1st queue as high priority compute queue if we
 195	 * have more than one compute queue.
 196	 */
 197	if (adev->gfx.num_compute_rings > 1 &&
 198	    ring == &adev->gfx.compute_ring[0])
 199		return true;
 200
 201	return false;
 202}
 203
 204void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
 205{
 206	int i, j, queue, pipe;
 207	bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev);
 208	int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *
 209				     adev->gfx.mec.num_queue_per_pipe,
 210				     adev->gfx.num_compute_rings);
 211	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 212
 213	if (multipipe_policy) {
 214		/* policy: make queues evenly cross all pipes on MEC1 only
 215		 * for multiple xcc, just use the original policy for simplicity */
 216		for (j = 0; j < num_xcc; j++) {
 217			for (i = 0; i < max_queues_per_mec; i++) {
 218				pipe = i % adev->gfx.mec.num_pipe_per_mec;
 219				queue = (i / adev->gfx.mec.num_pipe_per_mec) %
 220					 adev->gfx.mec.num_queue_per_pipe;
 221
 222				set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue,
 223					adev->gfx.mec_bitmap[j].queue_bitmap);
 224			}
 225		}
 226	} else {
 227		/* policy: amdgpu owns all queues in the given pipe */
 228		for (j = 0; j < num_xcc; j++) {
 229			for (i = 0; i < max_queues_per_mec; ++i)
 230				set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap);
 231		}
 232	}
 233
 234	for (j = 0; j < num_xcc; j++) {
 235		dev_dbg(adev->dev, "mec queue bitmap weight=%d\n",
 236			bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES));
 237	}
 238}
 239
 240void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
 241{
 242	int i, queue, pipe;
 243	bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev);
 244	int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
 245	int max_queues_per_me = adev->gfx.me.num_pipe_per_me * num_queue_per_pipe;
 246
 247	if (multipipe_policy) {
 248		/* policy: amdgpu owns the first queue per pipe at this stage
 249		 * will extend to mulitple queues per pipe later */
 250		for (i = 0; i < max_queues_per_me; i++) {
 251			pipe = i % adev->gfx.me.num_pipe_per_me;
 252			queue = (i / adev->gfx.me.num_pipe_per_me) %
 253				num_queue_per_pipe;
 254
 255			set_bit(pipe * num_queue_per_pipe + queue,
 256				adev->gfx.me.queue_bitmap);
 257		}
 258	} else {
 259		for (i = 0; i < max_queues_per_me; ++i)
 260			set_bit(i, adev->gfx.me.queue_bitmap);
 261	}
 262
 263	/* update the number of active graphics rings */
 264	if (adev->gfx.num_gfx_rings)
 265		adev->gfx.num_gfx_rings =
 266			bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
 267}
 268
 269static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
 270				  struct amdgpu_ring *ring, int xcc_id)
 271{
 272	int queue_bit;
 273	int mec, pipe, queue;
 274
 275	queue_bit = adev->gfx.mec.num_mec
 276		    * adev->gfx.mec.num_pipe_per_mec
 277		    * adev->gfx.mec.num_queue_per_pipe;
 278
 279	while (--queue_bit >= 0) {
 280		if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
 281			continue;
 282
 283		amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
 284
 285		/*
 286		 * 1. Using pipes 2/3 from MEC 2 seems cause problems.
 287		 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
 288		 * only can be issued on queue 0.
 289		 */
 290		if ((mec == 1 && pipe > 1) || queue != 0)
 291			continue;
 292
 293		ring->me = mec + 1;
 294		ring->pipe = pipe;
 295		ring->queue = queue;
 296
 297		return 0;
 298	}
 299
 300	dev_err(adev->dev, "Failed to find a queue for KIQ\n");
 301	return -EINVAL;
 302}
 303
 304int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id)
 305{
 306	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 307	struct amdgpu_irq_src *irq = &kiq->irq;
 308	struct amdgpu_ring *ring = &kiq->ring;
 309	int r = 0;
 310
 311	spin_lock_init(&kiq->ring_lock);
 312
 313	ring->adev = NULL;
 314	ring->ring_obj = NULL;
 315	ring->use_doorbell = true;
 316	ring->xcc_id = xcc_id;
 317	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
 318	ring->doorbell_index =
 319		(adev->doorbell_index.kiq +
 320		 xcc_id * adev->doorbell_index.xcc_doorbell_range)
 321		<< 1;
 322
 323	r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id);
 324	if (r)
 325		return r;
 326
 327	ring->eop_gpu_addr = kiq->eop_gpu_addr;
 328	ring->no_scheduler = true;
 329	snprintf(ring->name, sizeof(ring->name), "kiq_%hhu.%hhu.%hhu.%hhu",
 330		 (unsigned char)xcc_id, (unsigned char)ring->me,
 331		 (unsigned char)ring->pipe, (unsigned char)ring->queue);
 332	r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0,
 333			     AMDGPU_RING_PRIO_DEFAULT, NULL);
 334	if (r)
 335		dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
 336
 337	return r;
 338}
 339
 340void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring)
 341{
 342	amdgpu_ring_fini(ring);
 343}
 344
 345void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id)
 346{
 347	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 348
 349	amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
 350}
 351
 352int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
 353			unsigned int hpd_size, int xcc_id)
 354{
 355	int r;
 356	u32 *hpd;
 357	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 358
 359	r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE,
 360				    AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
 361				    &kiq->eop_gpu_addr, (void **)&hpd);
 362	if (r) {
 363		dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
 364		return r;
 365	}
 366
 367	memset(hpd, 0, hpd_size);
 368
 369	r = amdgpu_bo_reserve(kiq->eop_obj, true);
 370	if (unlikely(r != 0))
 371		dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
 372	amdgpu_bo_kunmap(kiq->eop_obj);
 373	amdgpu_bo_unreserve(kiq->eop_obj);
 374
 375	return 0;
 376}
 377
 378/* create MQD for each compute/gfx queue */
 379int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 380			   unsigned int mqd_size, int xcc_id)
 381{
 382	int r, i, j;
 383	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 384	struct amdgpu_ring *ring = &kiq->ring;
 385	u32 domain = AMDGPU_GEM_DOMAIN_GTT;
 386
 387#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
 388	/* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */
 389	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0))
 390		domain |= AMDGPU_GEM_DOMAIN_VRAM;
 391#endif
 392
 393	/* create MQD for KIQ */
 394	if (!adev->enable_mes_kiq && !ring->mqd_obj) {
 395		/* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
 396		 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
 397		 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
 398		 * KIQ MQD no matter SRIOV or Bare-metal
 399		 */
 400		r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 401					    AMDGPU_GEM_DOMAIN_VRAM |
 402					    AMDGPU_GEM_DOMAIN_GTT,
 403					    &ring->mqd_obj,
 404					    &ring->mqd_gpu_addr,
 405					    &ring->mqd_ptr);
 406		if (r) {
 407			dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
 408			return r;
 409		}
 410
 411		/* prepare MQD backup */
 412		kiq->mqd_backup = kzalloc(mqd_size, GFP_KERNEL);
 413		if (!kiq->mqd_backup) {
 414			dev_warn(adev->dev,
 415				 "no memory to create MQD backup for ring %s\n", ring->name);
 416			return -ENOMEM;
 417		}
 418	}
 419
 420	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
 421		/* create MQD for each KGQ */
 422		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 423			ring = &adev->gfx.gfx_ring[i];
 424			if (!ring->mqd_obj) {
 425				r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 426							    domain, &ring->mqd_obj,
 427							    &ring->mqd_gpu_addr, &ring->mqd_ptr);
 428				if (r) {
 429					dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
 430					return r;
 431				}
 432
 433				ring->mqd_size = mqd_size;
 434				/* prepare MQD backup */
 435				adev->gfx.me.mqd_backup[i] = kzalloc(mqd_size, GFP_KERNEL);
 436				if (!adev->gfx.me.mqd_backup[i]) {
 437					dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 438					return -ENOMEM;
 439				}
 440			}
 441		}
 442	}
 443
 444	/* create MQD for each KCQ */
 445	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 446		j = i + xcc_id * adev->gfx.num_compute_rings;
 447		ring = &adev->gfx.compute_ring[j];
 448		if (!ring->mqd_obj) {
 449			r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 450						    domain, &ring->mqd_obj,
 451						    &ring->mqd_gpu_addr, &ring->mqd_ptr);
 452			if (r) {
 453				dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
 454				return r;
 455			}
 456
 457			ring->mqd_size = mqd_size;
 458			/* prepare MQD backup */
 459			adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
 460			if (!adev->gfx.mec.mqd_backup[j]) {
 461				dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
 462				return -ENOMEM;
 463			}
 464		}
 465	}
 466
 467	return 0;
 468}
 469
 470void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
 471{
 472	struct amdgpu_ring *ring = NULL;
 473	int i, j;
 474	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 475
 476	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
 477		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 478			ring = &adev->gfx.gfx_ring[i];
 479			kfree(adev->gfx.me.mqd_backup[i]);
 480			amdgpu_bo_free_kernel(&ring->mqd_obj,
 481					      &ring->mqd_gpu_addr,
 482					      &ring->mqd_ptr);
 483		}
 484	}
 485
 486	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 487		j = i + xcc_id * adev->gfx.num_compute_rings;
 488		ring = &adev->gfx.compute_ring[j];
 489		kfree(adev->gfx.mec.mqd_backup[j]);
 490		amdgpu_bo_free_kernel(&ring->mqd_obj,
 491				      &ring->mqd_gpu_addr,
 492				      &ring->mqd_ptr);
 493	}
 494
 495	ring = &kiq->ring;
 496	kfree(kiq->mqd_backup);
 497	amdgpu_bo_free_kernel(&ring->mqd_obj,
 498			      &ring->mqd_gpu_addr,
 499			      &ring->mqd_ptr);
 500}
 501
 502int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
 503{
 504	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 505	struct amdgpu_ring *kiq_ring = &kiq->ring;
 506	int i, r = 0;
 507	int j;
 508
 509	if (adev->enable_mes) {
 510		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 511			j = i + xcc_id * adev->gfx.num_compute_rings;
 512			amdgpu_mes_unmap_legacy_queue(adev,
 513						   &adev->gfx.compute_ring[j],
 514						   RESET_QUEUES, 0, 0);
 515		}
 516		return 0;
 517	}
 518
 519	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
 520		return -EINVAL;
 521
 522	if (!kiq_ring->sched.ready || amdgpu_in_reset(adev))
 523		return 0;
 524
 525	spin_lock(&kiq->ring_lock);
 526	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
 527					adev->gfx.num_compute_rings)) {
 528		spin_unlock(&kiq->ring_lock);
 529		return -ENOMEM;
 530	}
 531
 532	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 533		j = i + xcc_id * adev->gfx.num_compute_rings;
 534		kiq->pmf->kiq_unmap_queues(kiq_ring,
 535					   &adev->gfx.compute_ring[j],
 536					   RESET_QUEUES, 0, 0);
 537	}
 538	/* Submit unmap queue packet */
 539	amdgpu_ring_commit(kiq_ring);
 540	/*
 541	 * Ring test will do a basic scratch register change check. Just run
 542	 * this to ensure that unmap queues that is submitted before got
 543	 * processed successfully before returning.
 544	 */
 545	r = amdgpu_ring_test_helper(kiq_ring);
 546
 547	spin_unlock(&kiq->ring_lock);
 548
 549	return r;
 550}
 551
 552int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
 553{
 554	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 555	struct amdgpu_ring *kiq_ring = &kiq->ring;
 556	int i, r = 0;
 557	int j;
 558
 559	if (adev->enable_mes) {
 560		if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 561			for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 562				j = i + xcc_id * adev->gfx.num_gfx_rings;
 563				amdgpu_mes_unmap_legacy_queue(adev,
 564						      &adev->gfx.gfx_ring[j],
 565						      PREEMPT_QUEUES, 0, 0);
 566			}
 567		}
 568		return 0;
 569	}
 570
 571	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
 572		return -EINVAL;
 573
 574	if (!adev->gfx.kiq[0].ring.sched.ready || amdgpu_in_reset(adev))
 575		return 0;
 576
 577	if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 578		spin_lock(&kiq->ring_lock);
 579		if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
 580						adev->gfx.num_gfx_rings)) {
 581			spin_unlock(&kiq->ring_lock);
 582			return -ENOMEM;
 583		}
 584
 585		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 586			j = i + xcc_id * adev->gfx.num_gfx_rings;
 587			kiq->pmf->kiq_unmap_queues(kiq_ring,
 588						   &adev->gfx.gfx_ring[j],
 589						   PREEMPT_QUEUES, 0, 0);
 590		}
 591		/* Submit unmap queue packet */
 592		amdgpu_ring_commit(kiq_ring);
 593
 594		/*
 595		 * Ring test will do a basic scratch register change check.
 596		 * Just run this to ensure that unmap queues that is submitted
 597		 * before got processed successfully before returning.
 598		 */
 599		r = amdgpu_ring_test_helper(kiq_ring);
 600		spin_unlock(&kiq->ring_lock);
 601	}
 602
 603	return r;
 604}
 605
 606int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
 607					int queue_bit)
 608{
 609	int mec, pipe, queue;
 610	int set_resource_bit = 0;
 611
 612	amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
 613
 614	set_resource_bit = mec * 4 * 8 + pipe * 8 + queue;
 615
 616	return set_resource_bit;
 617}
 618
 619static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 620{
 621	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 622	struct amdgpu_ring *kiq_ring = &kiq->ring;
 623	uint64_t queue_mask = ~0ULL;
 624	int r, i, j;
 625
 626	amdgpu_device_flush_hdp(adev, NULL);
 627
 628	if (!adev->enable_uni_mes) {
 629		spin_lock(&kiq->ring_lock);
 630		r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->set_resources_size);
 631		if (r) {
 632			dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
 633			spin_unlock(&kiq->ring_lock);
 634			return r;
 635		}
 636
 637		kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
 638		r = amdgpu_ring_test_helper(kiq_ring);
 639		spin_unlock(&kiq->ring_lock);
 640		if (r)
 641			dev_err(adev->dev, "KIQ failed to set resources\n");
 642	}
 643
 644	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 645		j = i + xcc_id * adev->gfx.num_compute_rings;
 646		r = amdgpu_mes_map_legacy_queue(adev,
 647						&adev->gfx.compute_ring[j]);
 648		if (r) {
 649			dev_err(adev->dev, "failed to map compute queue\n");
 650			return r;
 651		}
 652	}
 653
 654	return 0;
 655}
 656
 657int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 658{
 659	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 660	struct amdgpu_ring *kiq_ring = &kiq->ring;
 661	uint64_t queue_mask = 0;
 662	int r, i, j;
 663
 664	if (adev->mes.enable_legacy_queue_map)
 665		return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
 666
 667	if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
 668		return -EINVAL;
 669
 670	for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
 671		if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
 672			continue;
 673
 674		/* This situation may be hit in the future if a new HW
 675		 * generation exposes more than 64 queues. If so, the
 676		 * definition of queue_mask needs updating */
 677		if (WARN_ON(i > (sizeof(queue_mask)*8))) {
 678			dev_err(adev->dev, "Invalid KCQ enabled: %d\n", i);
 679			break;
 680		}
 681
 682		queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i));
 683	}
 684
 685	amdgpu_device_flush_hdp(adev, NULL);
 686
 687	dev_info(adev->dev, "kiq ring mec %d pipe %d q %d\n", kiq_ring->me,
 688		 kiq_ring->pipe, kiq_ring->queue);
 689
 690	spin_lock(&kiq->ring_lock);
 691	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
 692					adev->gfx.num_compute_rings +
 693					kiq->pmf->set_resources_size);
 694	if (r) {
 695		dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
 696		spin_unlock(&kiq->ring_lock);
 697		return r;
 698	}
 699
 700	kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
 701	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 702		j = i + xcc_id * adev->gfx.num_compute_rings;
 703		kiq->pmf->kiq_map_queues(kiq_ring,
 704					 &adev->gfx.compute_ring[j]);
 705	}
 706	/* Submit map queue packet */
 707	amdgpu_ring_commit(kiq_ring);
 708	/*
 709	 * Ring test will do a basic scratch register change check. Just run
 710	 * this to ensure that map queues that is submitted before got
 711	 * processed successfully before returning.
 712	 */
 713	r = amdgpu_ring_test_helper(kiq_ring);
 714	spin_unlock(&kiq->ring_lock);
 715	if (r)
 716		dev_err(adev->dev, "KCQ enable failed\n");
 717
 718	return r;
 719}
 720
 721int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
 722{
 723	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 724	struct amdgpu_ring *kiq_ring = &kiq->ring;
 725	int r, i, j;
 726
 727	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
 728		return -EINVAL;
 729
 730	amdgpu_device_flush_hdp(adev, NULL);
 731
 732	if (adev->mes.enable_legacy_queue_map) {
 733		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 734			j = i + xcc_id * adev->gfx.num_gfx_rings;
 735			r = amdgpu_mes_map_legacy_queue(adev,
 736							&adev->gfx.gfx_ring[j]);
 737			if (r) {
 738				dev_err(adev->dev, "failed to map gfx queue\n");
 739				return r;
 740			}
 741		}
 742
 743		return 0;
 744	}
 745
 746	spin_lock(&kiq->ring_lock);
 747	/* No need to map kcq on the slave */
 748	if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
 749		r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
 750						adev->gfx.num_gfx_rings);
 751		if (r) {
 752			dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
 753			spin_unlock(&kiq->ring_lock);
 754			return r;
 755		}
 756
 757		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 758			j = i + xcc_id * adev->gfx.num_gfx_rings;
 759			kiq->pmf->kiq_map_queues(kiq_ring,
 760						 &adev->gfx.gfx_ring[j]);
 761		}
 762	}
 763	/* Submit map queue packet */
 764	amdgpu_ring_commit(kiq_ring);
 765	/*
 766	 * Ring test will do a basic scratch register change check. Just run
 767	 * this to ensure that map queues that is submitted before got
 768	 * processed successfully before returning.
 769	 */
 770	r = amdgpu_ring_test_helper(kiq_ring);
 771	spin_unlock(&kiq->ring_lock);
 772	if (r)
 773		dev_err(adev->dev, "KGQ enable failed\n");
 774
 775	return r;
 776}
 777
 778static void amdgpu_gfx_do_off_ctrl(struct amdgpu_device *adev, bool enable,
 779				   bool no_delay)
 780{
 781	unsigned long delay = GFX_OFF_DELAY_ENABLE;
 782
 783	if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
 784		return;
 785
 786	mutex_lock(&adev->gfx.gfx_off_mutex);
 787
 788	if (enable) {
 789		/* If the count is already 0, it means there's an imbalance bug somewhere.
 790		 * Note that the bug may be in a different caller than the one which triggers the
 791		 * WARN_ON_ONCE.
 792		 */
 793		if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))
 794			goto unlock;
 795
 796		adev->gfx.gfx_off_req_count--;
 797
 798		if (adev->gfx.gfx_off_req_count == 0 &&
 799		    !adev->gfx.gfx_off_state) {
 800			/* If going to s2idle, no need to wait */
 801			if (no_delay) {
 802				if (!amdgpu_dpm_set_powergating_by_smu(adev,
 803						AMD_IP_BLOCK_TYPE_GFX, true, 0))
 804					adev->gfx.gfx_off_state = true;
 805			} else {
 806				schedule_delayed_work(&adev->gfx.gfx_off_delay_work,
 807					      delay);
 808			}
 809		}
 810	} else {
 811		if (adev->gfx.gfx_off_req_count == 0) {
 812			cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
 813
 814			if (adev->gfx.gfx_off_state &&
 815			    !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false, 0)) {
 816				adev->gfx.gfx_off_state = false;
 817
 818				if (adev->gfx.funcs->init_spm_golden) {
 819					dev_dbg(adev->dev,
 820						"GFXOFF is disabled, re-init SPM golden settings\n");
 821					amdgpu_gfx_init_spm_golden(adev);
 822				}
 823			}
 824		}
 825
 826		adev->gfx.gfx_off_req_count++;
 827	}
 828
 829unlock:
 830	mutex_unlock(&adev->gfx.gfx_off_mutex);
 831}
 832
 833/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
 834 *
 835 * @adev: amdgpu_device pointer
 836 * @bool enable true: enable gfx off feature, false: disable gfx off feature
 837 *
 838 * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.
 839 * 2. other client can send request to disable gfx off feature, the request should be honored.
 840 * 3. other client can cancel their request of disable gfx off feature
 841 * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
 842 *
 843 * gfx off allow will be delayed by GFX_OFF_DELAY_ENABLE ms.
 844 */
 845void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
 846{
 847	/* If going to s2idle, no need to wait */
 848	bool no_delay = adev->in_s0ix ? true : false;
 849
 850	amdgpu_gfx_do_off_ctrl(adev, enable, no_delay);
 851}
 852
 853/* amdgpu_gfx_off_ctrl_immediate - Handle gfx off feature enable/disable
 854 *
 855 * @adev: amdgpu_device pointer
 856 * @bool enable true: enable gfx off feature, false: disable gfx off feature
 857 *
 858 * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.
 859 * 2. other client can send request to disable gfx off feature, the request should be honored.
 860 * 3. other client can cancel their request of disable gfx off feature
 861 * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
 862 *
 863 * gfx off allow will be issued immediately.
 864 */
 865void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable)
 866{
 867	amdgpu_gfx_do_off_ctrl(adev, enable, true);
 868}
 869
 870int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value)
 871{
 872	int r = 0;
 873
 874	mutex_lock(&adev->gfx.gfx_off_mutex);
 875
 876	r = amdgpu_dpm_set_residency_gfxoff(adev, value);
 877
 878	mutex_unlock(&adev->gfx.gfx_off_mutex);
 879
 880	return r;
 881}
 882
 883int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value)
 884{
 885	int r = 0;
 886
 887	mutex_lock(&adev->gfx.gfx_off_mutex);
 888
 889	r = amdgpu_dpm_get_residency_gfxoff(adev, value);
 890
 891	mutex_unlock(&adev->gfx.gfx_off_mutex);
 892
 893	return r;
 894}
 895
 896int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value)
 897{
 898	int r = 0;
 899
 900	mutex_lock(&adev->gfx.gfx_off_mutex);
 901
 902	r = amdgpu_dpm_get_entrycount_gfxoff(adev, value);
 903
 904	mutex_unlock(&adev->gfx.gfx_off_mutex);
 905
 906	return r;
 907}
 908
 909int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value)
 910{
 911
 912	int r = 0;
 913
 914	mutex_lock(&adev->gfx.gfx_off_mutex);
 915
 916	r = amdgpu_dpm_get_status_gfxoff(adev, value);
 917
 918	mutex_unlock(&adev->gfx.gfx_off_mutex);
 919
 920	return r;
 921}
 922
 923int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 924{
 925	int r;
 926
 927	if (amdgpu_ras_is_supported(adev, ras_block->block)) {
 928		if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
 929			r = amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX);
 930			if (r)
 931				return r;
 932		}
 933
 934		r = amdgpu_ras_block_late_init(adev, ras_block);
 935		if (r)
 936			return r;
 937
 938		if (amdgpu_sriov_vf(adev))
 939			return r;
 940
 941		if (adev->gfx.cp_ecc_error_irq.funcs) {
 942			r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
 943			if (r)
 944				goto late_fini;
 945		}
 946	} else {
 947		amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
 948	}
 949
 950	return 0;
 951late_fini:
 952	amdgpu_ras_block_late_fini(adev, ras_block);
 953	return r;
 954}
 955
 956int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev)
 957{
 958	int err = 0;
 959	struct amdgpu_gfx_ras *ras = NULL;
 960
 961	/* adev->gfx.ras is NULL, which means gfx does not
 962	 * support ras function, then do nothing here.
 963	 */
 964	if (!adev->gfx.ras)
 965		return 0;
 966
 967	ras = adev->gfx.ras;
 968
 969	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
 970	if (err) {
 971		dev_err(adev->dev, "Failed to register gfx ras block!\n");
 972		return err;
 973	}
 974
 975	strcpy(ras->ras_block.ras_comm.name, "gfx");
 976	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX;
 977	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
 978	adev->gfx.ras_if = &ras->ras_block.ras_comm;
 979
 980	/* If not define special ras_late_init function, use gfx default ras_late_init */
 981	if (!ras->ras_block.ras_late_init)
 982		ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init;
 983
 984	/* If not defined special ras_cb function, use default ras_cb */
 985	if (!ras->ras_block.ras_cb)
 986		ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb;
 987
 988	return 0;
 989}
 990
 991int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev,
 992						struct amdgpu_iv_entry *entry)
 993{
 994	if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler)
 995		return adev->gfx.ras->poison_consumption_handler(adev, entry);
 996
 997	return 0;
 998}
 999
1000int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
1001		void *err_data,
1002		struct amdgpu_iv_entry *entry)
1003{
1004	/* TODO ue will trigger an interrupt.
1005	 *
1006	 * When “Full RAS” is enabled, the per-IP interrupt sources should
1007	 * be disabled and the driver should only look for the aggregated
1008	 * interrupt via sync flood
1009	 */
1010	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) {
1011		kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
1012		if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops &&
1013		    adev->gfx.ras->ras_block.hw_ops->query_ras_error_count)
1014			adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1015		amdgpu_ras_reset_gpu(adev);
1016	}
1017	return AMDGPU_RAS_SUCCESS;
1018}
1019
1020int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
1021				  struct amdgpu_irq_src *source,
1022				  struct amdgpu_iv_entry *entry)
1023{
1024	struct ras_common_if *ras_if = adev->gfx.ras_if;
1025	struct ras_dispatch_if ih_data = {
1026		.entry = entry,
1027	};
1028
1029	if (!ras_if)
1030		return 0;
1031
1032	ih_data.head = *ras_if;
1033
1034	dev_err(adev->dev, "CP ECC ERROR IRQ\n");
1035	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
1036	return 0;
1037}
1038
1039void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev,
1040		void *ras_error_status,
1041		void (*func)(struct amdgpu_device *adev, void *ras_error_status,
1042				int xcc_id))
1043{
1044	int i;
1045	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
1046	uint32_t xcc_mask = GENMASK(num_xcc - 1, 0);
1047	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1048
1049	if (err_data) {
1050		err_data->ue_count = 0;
1051		err_data->ce_count = 0;
1052	}
1053
1054	for_each_inst(i, xcc_mask)
1055		func(adev, ras_error_status, i);
1056}
1057
1058uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id)
1059{
1060	signed long r, cnt = 0;
1061	unsigned long flags;
1062	uint32_t seq, reg_val_offs = 0, value = 0;
1063	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1064	struct amdgpu_ring *ring = &kiq->ring;
1065
1066	if (amdgpu_device_skip_hw_access(adev))
1067		return 0;
1068
1069	if (adev->mes.ring[0].sched.ready)
1070		return amdgpu_mes_rreg(adev, reg);
1071
1072	BUG_ON(!ring->funcs->emit_rreg);
1073
1074	spin_lock_irqsave(&kiq->ring_lock, flags);
1075	if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
1076		pr_err("critical bug! too many kiq readers\n");
1077		goto failed_unlock;
1078	}
1079	r = amdgpu_ring_alloc(ring, 32);
1080	if (r)
1081		goto failed_unlock;
1082
1083	amdgpu_ring_emit_rreg(ring, reg, reg_val_offs);
1084	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1085	if (r)
1086		goto failed_undo;
1087
1088	amdgpu_ring_commit(ring);
1089	spin_unlock_irqrestore(&kiq->ring_lock, flags);
1090
1091	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1092
1093	/* don't wait anymore for gpu reset case because this way may
1094	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1095	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1096	 * never return if we keep waiting in virt_kiq_rreg, which cause
1097	 * gpu_recover() hang there.
1098	 *
1099	 * also don't wait anymore for IRQ context
1100	 * */
1101	if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1102		goto failed_kiq_read;
1103
1104	might_sleep();
1105	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1106		if (amdgpu_in_reset(adev))
1107			goto failed_kiq_read;
1108
1109		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1110		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1111	}
1112
1113	if (cnt > MAX_KIQ_REG_TRY)
1114		goto failed_kiq_read;
1115
1116	mb();
1117	value = adev->wb.wb[reg_val_offs];
1118	amdgpu_device_wb_free(adev, reg_val_offs);
1119	return value;
1120
1121failed_undo:
1122	amdgpu_ring_undo(ring);
1123failed_unlock:
1124	spin_unlock_irqrestore(&kiq->ring_lock, flags);
1125failed_kiq_read:
1126	if (reg_val_offs)
1127		amdgpu_device_wb_free(adev, reg_val_offs);
1128	dev_err(adev->dev, "failed to read reg:%x\n", reg);
1129	return ~0;
1130}
1131
1132void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id)
1133{
1134	signed long r, cnt = 0;
1135	unsigned long flags;
1136	uint32_t seq;
1137	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1138	struct amdgpu_ring *ring = &kiq->ring;
1139
1140	BUG_ON(!ring->funcs->emit_wreg);
1141
1142	if (amdgpu_device_skip_hw_access(adev))
1143		return;
1144
1145	if (adev->mes.ring[0].sched.ready) {
1146		amdgpu_mes_wreg(adev, reg, v);
1147		return;
1148	}
1149
1150	spin_lock_irqsave(&kiq->ring_lock, flags);
1151	r = amdgpu_ring_alloc(ring, 32);
1152	if (r)
1153		goto failed_unlock;
1154
1155	amdgpu_ring_emit_wreg(ring, reg, v);
1156	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1157	if (r)
1158		goto failed_undo;
1159
1160	amdgpu_ring_commit(ring);
1161	spin_unlock_irqrestore(&kiq->ring_lock, flags);
1162
1163	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1164
1165	/* don't wait anymore for gpu reset case because this way may
1166	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1167	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1168	 * never return if we keep waiting in virt_kiq_rreg, which cause
1169	 * gpu_recover() hang there.
1170	 *
1171	 * also don't wait anymore for IRQ context
1172	 * */
1173	if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1174		goto failed_kiq_write;
1175
1176	might_sleep();
1177	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1178		if (amdgpu_in_reset(adev))
1179			goto failed_kiq_write;
1180
1181		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1182		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1183	}
1184
1185	if (cnt > MAX_KIQ_REG_TRY)
1186		goto failed_kiq_write;
1187
1188	return;
1189
1190failed_undo:
1191	amdgpu_ring_undo(ring);
1192failed_unlock:
1193	spin_unlock_irqrestore(&kiq->ring_lock, flags);
1194failed_kiq_write:
1195	dev_err(adev->dev, "failed to write reg:%x\n", reg);
1196}
1197
1198int amdgpu_kiq_hdp_flush(struct amdgpu_device *adev)
1199{
1200	signed long r, cnt = 0;
1201	unsigned long flags;
1202	uint32_t seq;
1203	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1204	struct amdgpu_ring *ring = &kiq->ring;
1205
1206	if (amdgpu_device_skip_hw_access(adev))
1207		return 0;
1208
1209	if (adev->enable_mes_kiq && adev->mes.ring[0].sched.ready)
1210		return amdgpu_mes_hdp_flush(adev);
1211
1212	if (!ring->funcs->emit_hdp_flush) {
1213		return -EOPNOTSUPP;
1214	}
1215
1216	spin_lock_irqsave(&kiq->ring_lock, flags);
1217	r = amdgpu_ring_alloc(ring, 32);
1218	if (r)
1219		goto failed_unlock;
1220
1221	amdgpu_ring_emit_hdp_flush(ring);
1222	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1223	if (r)
1224		goto failed_undo;
1225
1226	amdgpu_ring_commit(ring);
1227	spin_unlock_irqrestore(&kiq->ring_lock, flags);
1228
1229	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1230
1231	/* don't wait anymore for gpu reset case because this way may
1232	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1233	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1234	 * never return if we keep waiting in virt_kiq_rreg, which cause
1235	 * gpu_recover() hang there.
1236	 *
1237	 * also don't wait anymore for IRQ context
1238	 * */
1239	if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1240		goto failed_kiq_hdp_flush;
1241
1242	might_sleep();
1243	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1244		if (amdgpu_in_reset(adev))
1245			goto failed_kiq_hdp_flush;
1246
1247		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1248		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1249	}
1250
1251	if (cnt > MAX_KIQ_REG_TRY) {
1252		dev_err(adev->dev, "failed to flush HDP via KIQ timeout\n");
1253		return -ETIMEDOUT;
1254	}
1255
1256	return 0;
1257
1258failed_undo:
1259	amdgpu_ring_undo(ring);
1260failed_unlock:
1261	spin_unlock_irqrestore(&kiq->ring_lock, flags);
1262failed_kiq_hdp_flush:
1263	dev_err(adev->dev, "failed to flush HDP via KIQ\n");
1264	return r < 0 ? r : -EIO;
1265}
1266
1267int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev)
1268{
1269	if (amdgpu_num_kcq == -1) {
1270		return 8;
1271	} else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) {
1272		dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n");
1273		return 8;
1274	}
1275	return amdgpu_num_kcq;
1276}
1277
1278void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev,
1279				  uint32_t ucode_id)
1280{
1281	const struct gfx_firmware_header_v1_0 *cp_hdr;
1282	const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
1283	struct amdgpu_firmware_info *info = NULL;
1284	const struct firmware *ucode_fw;
1285	unsigned int fw_size;
1286
1287	switch (ucode_id) {
1288	case AMDGPU_UCODE_ID_CP_PFP:
1289		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1290			adev->gfx.pfp_fw->data;
1291		adev->gfx.pfp_fw_version =
1292			le32_to_cpu(cp_hdr->header.ucode_version);
1293		adev->gfx.pfp_feature_version =
1294			le32_to_cpu(cp_hdr->ucode_feature_version);
1295		ucode_fw = adev->gfx.pfp_fw;
1296		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1297		break;
1298	case AMDGPU_UCODE_ID_CP_RS64_PFP:
1299		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1300			adev->gfx.pfp_fw->data;
1301		adev->gfx.pfp_fw_version =
1302			le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1303		adev->gfx.pfp_feature_version =
1304			le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1305		ucode_fw = adev->gfx.pfp_fw;
1306		fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1307		break;
1308	case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
1309	case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
1310		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1311			adev->gfx.pfp_fw->data;
1312		ucode_fw = adev->gfx.pfp_fw;
1313		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1314		break;
1315	case AMDGPU_UCODE_ID_CP_ME:
1316		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1317			adev->gfx.me_fw->data;
1318		adev->gfx.me_fw_version =
1319			le32_to_cpu(cp_hdr->header.ucode_version);
1320		adev->gfx.me_feature_version =
1321			le32_to_cpu(cp_hdr->ucode_feature_version);
1322		ucode_fw = adev->gfx.me_fw;
1323		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1324		break;
1325	case AMDGPU_UCODE_ID_CP_RS64_ME:
1326		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1327			adev->gfx.me_fw->data;
1328		adev->gfx.me_fw_version =
1329			le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1330		adev->gfx.me_feature_version =
1331			le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1332		ucode_fw = adev->gfx.me_fw;
1333		fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1334		break;
1335	case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
1336	case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
1337		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1338			adev->gfx.me_fw->data;
1339		ucode_fw = adev->gfx.me_fw;
1340		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1341		break;
1342	case AMDGPU_UCODE_ID_CP_CE:
1343		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1344			adev->gfx.ce_fw->data;
1345		adev->gfx.ce_fw_version =
1346			le32_to_cpu(cp_hdr->header.ucode_version);
1347		adev->gfx.ce_feature_version =
1348			le32_to_cpu(cp_hdr->ucode_feature_version);
1349		ucode_fw = adev->gfx.ce_fw;
1350		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1351		break;
1352	case AMDGPU_UCODE_ID_CP_MEC1:
1353		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1354			adev->gfx.mec_fw->data;
1355		adev->gfx.mec_fw_version =
1356			le32_to_cpu(cp_hdr->header.ucode_version);
1357		adev->gfx.mec_feature_version =
1358			le32_to_cpu(cp_hdr->ucode_feature_version);
1359		ucode_fw = adev->gfx.mec_fw;
1360		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1361			  le32_to_cpu(cp_hdr->jt_size) * 4;
1362		break;
1363	case AMDGPU_UCODE_ID_CP_MEC1_JT:
1364		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1365			adev->gfx.mec_fw->data;
1366		ucode_fw = adev->gfx.mec_fw;
1367		fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1368		break;
1369	case AMDGPU_UCODE_ID_CP_MEC2:
1370		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1371			adev->gfx.mec2_fw->data;
1372		adev->gfx.mec2_fw_version =
1373			le32_to_cpu(cp_hdr->header.ucode_version);
1374		adev->gfx.mec2_feature_version =
1375			le32_to_cpu(cp_hdr->ucode_feature_version);
1376		ucode_fw = adev->gfx.mec2_fw;
1377		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1378			  le32_to_cpu(cp_hdr->jt_size) * 4;
1379		break;
1380	case AMDGPU_UCODE_ID_CP_MEC2_JT:
1381		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1382			adev->gfx.mec2_fw->data;
1383		ucode_fw = adev->gfx.mec2_fw;
1384		fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1385		break;
1386	case AMDGPU_UCODE_ID_CP_RS64_MEC:
1387		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1388			adev->gfx.mec_fw->data;
1389		adev->gfx.mec_fw_version =
1390			le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1391		adev->gfx.mec_feature_version =
1392			le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1393		ucode_fw = adev->gfx.mec_fw;
1394		fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1395		break;
1396	case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
1397	case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
1398	case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
1399	case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
1400		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1401			adev->gfx.mec_fw->data;
1402		ucode_fw = adev->gfx.mec_fw;
1403		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1404		break;
1405	default:
1406		dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id);
1407		return;
1408	}
1409
1410	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1411		info = &adev->firmware.ucode[ucode_id];
1412		info->ucode_id = ucode_id;
1413		info->fw = ucode_fw;
1414		adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE);
1415	}
1416}
1417
1418bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id)
1419{
1420	return !(xcc_id % (adev->gfx.num_xcc_per_xcp ?
1421			adev->gfx.num_xcc_per_xcp : 1));
1422}
1423
1424static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev,
1425						struct device_attribute *addr,
1426						char *buf)
1427{
1428	struct drm_device *ddev = dev_get_drvdata(dev);
1429	struct amdgpu_device *adev = drm_to_adev(ddev);
1430	int mode;
1431
1432	/* Only minimal precaution taken to reject requests while in reset.*/
1433	if (amdgpu_in_reset(adev))
1434		return -EPERM;
1435
1436	mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
1437					       AMDGPU_XCP_FL_NONE);
1438
1439	return sysfs_emit(buf, "%s\n", amdgpu_gfx_compute_mode_desc(mode));
1440}
1441
1442static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev,
1443						struct device_attribute *addr,
1444						const char *buf, size_t count)
1445{
1446	struct drm_device *ddev = dev_get_drvdata(dev);
1447	struct amdgpu_device *adev = drm_to_adev(ddev);
1448	enum amdgpu_gfx_partition mode;
1449	int ret = 0, num_xcc;
1450
1451	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1452	if (num_xcc % 2 != 0)
1453		return -EINVAL;
1454
1455	if (!strncasecmp("SPX", buf, strlen("SPX"))) {
1456		mode = AMDGPU_SPX_PARTITION_MODE;
1457	} else if (!strncasecmp("DPX", buf, strlen("DPX"))) {
1458		/*
1459		 * DPX mode needs AIDs to be in multiple of 2.
1460		 * Each AID connects 2 XCCs.
1461		 */
1462		if (num_xcc%4)
1463			return -EINVAL;
1464		mode = AMDGPU_DPX_PARTITION_MODE;
1465	} else if (!strncasecmp("TPX", buf, strlen("TPX"))) {
1466		if (num_xcc != 6)
1467			return -EINVAL;
1468		mode = AMDGPU_TPX_PARTITION_MODE;
1469	} else if (!strncasecmp("QPX", buf, strlen("QPX"))) {
1470		if (num_xcc != 8)
1471			return -EINVAL;
1472		mode = AMDGPU_QPX_PARTITION_MODE;
1473	} else if (!strncasecmp("CPX", buf, strlen("CPX"))) {
1474		mode = AMDGPU_CPX_PARTITION_MODE;
1475	} else {
1476		return -EINVAL;
1477	}
1478
1479	/* Don't allow a switch while under reset */
1480	if (!down_read_trylock(&adev->reset_domain->sem))
1481		return -EPERM;
1482
1483	ret = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, mode);
1484
1485	up_read(&adev->reset_domain->sem);
1486
1487	if (ret)
1488		return ret;
1489
1490	return count;
1491}
1492
1493static const char *xcp_desc[] = {
1494	[AMDGPU_SPX_PARTITION_MODE] = "SPX",
1495	[AMDGPU_DPX_PARTITION_MODE] = "DPX",
1496	[AMDGPU_TPX_PARTITION_MODE] = "TPX",
1497	[AMDGPU_QPX_PARTITION_MODE] = "QPX",
1498	[AMDGPU_CPX_PARTITION_MODE] = "CPX",
1499};
1500
1501static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev,
1502						struct device_attribute *addr,
1503						char *buf)
1504{
1505	struct drm_device *ddev = dev_get_drvdata(dev);
1506	struct amdgpu_device *adev = drm_to_adev(ddev);
1507	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1508	int size = 0, mode;
1509	char *sep = "";
1510
1511	if (!xcp_mgr || !xcp_mgr->avail_xcp_modes)
1512		return sysfs_emit(buf, "Not supported\n");
1513
1514	for_each_inst(mode, xcp_mgr->avail_xcp_modes) {
1515		size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]);
1516		sep = ", ";
1517	}
1518
1519	size += sysfs_emit_at(buf, size, "\n");
1520
1521	return size;
1522}
1523
1524static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring)
1525{
1526	struct amdgpu_device *adev = ring->adev;
1527	struct drm_gpu_scheduler *sched = &ring->sched;
1528	struct drm_sched_entity entity;
1529	static atomic_t counter;
1530	struct dma_fence *f;
1531	struct amdgpu_job *job;
1532	struct amdgpu_ib *ib;
1533	void *owner;
1534	int i, r;
1535
1536	/* Initialize the scheduler entity */
1537	r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL,
1538				  &sched, 1, NULL);
1539	if (r) {
1540		dev_err(adev->dev, "Failed setting up GFX kernel entity.\n");
1541		goto err;
1542	}
1543
1544	/*
1545	 * Use some unique dummy value as the owner to make sure we execute
1546	 * the cleaner shader on each submission. The value just need to change
1547	 * for each submission and is otherwise meaningless.
1548	 */
1549	owner = (void *)(unsigned long)atomic_inc_return(&counter);
1550
1551	r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner,
1552				     64, 0, &job,
1553				     AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER);
1554	if (r)
1555		goto err;
1556
1557	job->enforce_isolation = true;
1558	/* always run the cleaner shader */
1559	job->run_cleaner_shader = true;
1560
1561	ib = &job->ibs[0];
1562	for (i = 0; i <= ring->funcs->align_mask; ++i)
1563		ib->ptr[i] = ring->funcs->nop;
1564	ib->length_dw = ring->funcs->align_mask + 1;
1565
1566	f = amdgpu_job_submit(job);
1567
1568	r = dma_fence_wait(f, false);
1569	if (r)
1570		goto err;
1571
1572	dma_fence_put(f);
1573
1574	/* Clean up the scheduler entity */
1575	drm_sched_entity_destroy(&entity);
1576	return 0;
1577
1578err:
1579	return r;
1580}
1581
1582static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id)
1583{
1584	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1585	struct amdgpu_ring *ring;
1586	int num_xcc_to_clear;
1587	int i, r, xcc_id;
1588
1589	if (adev->gfx.num_xcc_per_xcp)
1590		num_xcc_to_clear = adev->gfx.num_xcc_per_xcp;
1591	else
1592		num_xcc_to_clear = 1;
1593
1594	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1595		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1596			ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
1597			if ((ring->xcp_id == xcp_id) && ring->sched.ready) {
1598				r = amdgpu_gfx_run_cleaner_shader_job(ring);
1599				if (r)
1600					return r;
1601				num_xcc_to_clear--;
1602				break;
1603			}
1604		}
1605	}
1606
1607	if (num_xcc_to_clear)
1608		return -ENOENT;
1609
1610	return 0;
1611}
1612
1613/**
1614 * amdgpu_gfx_set_run_cleaner_shader - Execute the AMDGPU GFX Cleaner Shader
1615 * @dev: The device structure
1616 * @attr: The device attribute structure
1617 * @buf: The buffer containing the input data
1618 * @count: The size of the input data
1619 *
1620 * Provides the sysfs interface to manually run a cleaner shader, which is
1621 * used to clear the GPU state between different tasks. Writing a value to the
1622 * 'run_cleaner_shader' sysfs file triggers the cleaner shader execution.
1623 * The value written corresponds to the partition index on multi-partition
1624 * devices. On single-partition devices, the value should be '0'.
1625 *
1626 * The cleaner shader clears the Local Data Store (LDS) and General Purpose
1627 * Registers (GPRs) to ensure data isolation between GPU workloads.
1628 *
1629 * Return: The number of bytes written to the sysfs file.
1630 */
1631static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
1632						 struct device_attribute *attr,
1633						 const char *buf,
1634						 size_t count)
1635{
1636	struct drm_device *ddev = dev_get_drvdata(dev);
1637	struct amdgpu_device *adev = drm_to_adev(ddev);
1638	int ret;
1639	long value;
1640
1641	if (amdgpu_in_reset(adev))
1642		return -EPERM;
1643	if (adev->in_suspend && !adev->in_runpm)
1644		return -EPERM;
1645
1646	if (adev->gfx.disable_kq)
1647		return -EPERM;
1648
1649	ret = kstrtol(buf, 0, &value);
1650
1651	if (ret)
1652		return -EINVAL;
1653
1654	if (value < 0)
1655		return -EINVAL;
1656
1657	if (adev->xcp_mgr) {
1658		if (value >= adev->xcp_mgr->num_xcps)
1659			return -EINVAL;
1660	} else {
1661		if (value > 1)
1662			return -EINVAL;
1663	}
1664
1665	ret = pm_runtime_get_sync(ddev->dev);
1666	if (ret < 0) {
1667		pm_runtime_put_autosuspend(ddev->dev);
1668		return ret;
1669	}
1670
1671	ret = amdgpu_gfx_run_cleaner_shader(adev, value);
1672
1673	pm_runtime_put_autosuspend(ddev->dev);
1674
1675	if (ret)
1676		return ret;
1677
1678	return count;
1679}
1680
1681/**
1682 * amdgpu_gfx_get_enforce_isolation - Query AMDGPU GFX Enforce Isolation Settings
1683 * @dev: The device structure
1684 * @attr: The device attribute structure
1685 * @buf: The buffer to store the output data
1686 *
1687 * Provides the sysfs read interface to get the current settings of the 'enforce_isolation'
1688 * feature for each GPU partition. Reading from the 'enforce_isolation'
1689 * sysfs file returns the isolation settings for all partitions, where '0'
1690 * indicates disabled, '1' indicates enabled, and '2' indicates enabled in legacy mode,
1691 * and '3' indicates enabled without cleaner shader.
1692 *
1693 * Return: The number of bytes read from the sysfs file.
1694 */
1695static ssize_t amdgpu_gfx_get_enforce_isolation(struct device *dev,
1696						struct device_attribute *attr,
1697						char *buf)
1698{
1699	struct drm_device *ddev = dev_get_drvdata(dev);
1700	struct amdgpu_device *adev = drm_to_adev(ddev);
1701	int i;
1702	ssize_t size = 0;
1703
1704	if (adev->xcp_mgr) {
1705		for (i = 0; i < adev->xcp_mgr->num_xcps; i++) {
1706			size += sysfs_emit_at(buf, size, "%u", adev->enforce_isolation[i]);
1707			if (i < (adev->xcp_mgr->num_xcps - 1))
1708				size += sysfs_emit_at(buf, size, " ");
1709		}
1710		buf[size++] = '\n';
1711	} else {
1712		size = sysfs_emit_at(buf, 0, "%u\n", adev->enforce_isolation[0]);
1713	}
1714
1715	return size;
1716}
1717
1718/**
1719 * amdgpu_gfx_set_enforce_isolation - Control AMDGPU GFX Enforce Isolation
1720 * @dev: The device structure
1721 * @attr: The device attribute structure
1722 * @buf: The buffer containing the input data
1723 * @count: The size of the input data
1724 *
1725 * This function allows control over the 'enforce_isolation' feature, which
1726 * serializes access to the graphics engine. Writing '0' to disable, '1' to
1727 * enable isolation with cleaner shader, '2' to enable legacy isolation without
1728 * cleaner shader, or '3' to enable process isolation without submitting the
1729 * cleaner shader to the 'enforce_isolation' sysfs file sets the isolation mode
1730 * for each partition. The input should specify the setting for all
1731 * partitions.
1732 *
1733 * Return: The number of bytes written to the sysfs file.
1734 */
1735static ssize_t amdgpu_gfx_set_enforce_isolation(struct device *dev,
1736						struct device_attribute *attr,
1737						const char *buf, size_t count)
1738{
1739	struct drm_device *ddev = dev_get_drvdata(dev);
1740	struct amdgpu_device *adev = drm_to_adev(ddev);
1741	long partition_values[MAX_XCP] = {0};
1742	int ret, i, num_partitions;
1743	const char *input_buf = buf;
1744
1745	for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) {
1746		ret = sscanf(input_buf, "%ld", &partition_values[i]);
1747		if (ret <= 0)
1748			break;
1749
1750		/* Move the pointer to the next value in the string */
1751		input_buf = strchr(input_buf, ' ');
1752		if (input_buf) {
1753			input_buf++;
1754		} else {
1755			i++;
1756			break;
1757		}
1758	}
1759	num_partitions = i;
1760
1761	if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps)
1762		return -EINVAL;
1763
1764	if (!adev->xcp_mgr && num_partitions != 1)
1765		return -EINVAL;
1766
1767	for (i = 0; i < num_partitions; i++) {
1768		if (partition_values[i] != 0 &&
1769		    partition_values[i] != 1 &&
1770		    partition_values[i] != 2 &&
1771		    partition_values[i] != 3)
1772			return -EINVAL;
1773	}
1774
1775	mutex_lock(&adev->enforce_isolation_mutex);
1776	for (i = 0; i < num_partitions; i++) {
1777		switch (partition_values[i]) {
1778		case 0:
1779		default:
1780			adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE;
1781			break;
1782		case 1:
1783			adev->enforce_isolation[i] =
1784				AMDGPU_ENFORCE_ISOLATION_ENABLE;
1785			break;
1786		case 2:
1787			adev->enforce_isolation[i] =
1788				AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY;
1789			break;
1790		case 3:
1791			adev->enforce_isolation[i] =
1792				AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER;
1793			break;
1794		}
1795	}
1796	mutex_unlock(&adev->enforce_isolation_mutex);
1797
1798	amdgpu_mes_update_enforce_isolation(adev);
1799
1800	return count;
1801}
1802
1803static ssize_t amdgpu_gfx_get_gfx_reset_mask(struct device *dev,
1804						struct device_attribute *attr,
1805						char *buf)
1806{
1807	struct drm_device *ddev = dev_get_drvdata(dev);
1808	struct amdgpu_device *adev = drm_to_adev(ddev);
1809
1810	if (!adev)
1811		return -ENODEV;
1812
1813	return amdgpu_show_reset_mask(buf, adev->gfx.gfx_supported_reset);
1814}
1815
1816static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
1817						struct device_attribute *attr,
1818						char *buf)
1819{
1820	struct drm_device *ddev = dev_get_drvdata(dev);
1821	struct amdgpu_device *adev = drm_to_adev(ddev);
1822
1823	if (!adev)
1824		return -ENODEV;
1825
1826	return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset);
1827}
1828
1829static DEVICE_ATTR(run_cleaner_shader, 0200,
1830		   NULL, amdgpu_gfx_set_run_cleaner_shader);
1831
1832static DEVICE_ATTR(enforce_isolation, 0644,
1833		   amdgpu_gfx_get_enforce_isolation,
1834		   amdgpu_gfx_set_enforce_isolation);
1835
1836static DEVICE_ATTR(current_compute_partition, 0644,
1837		   amdgpu_gfx_get_current_compute_partition,
1838		   amdgpu_gfx_set_compute_partition);
1839
1840static DEVICE_ATTR(available_compute_partition, 0444,
1841		   amdgpu_gfx_get_available_compute_partition, NULL);
1842static DEVICE_ATTR(gfx_reset_mask, 0444,
1843		   amdgpu_gfx_get_gfx_reset_mask, NULL);
1844
1845static DEVICE_ATTR(compute_reset_mask, 0444,
1846		   amdgpu_gfx_get_compute_reset_mask, NULL);
1847
1848static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev)
1849{
1850	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1851	bool xcp_switch_supported;
1852	int r;
1853
1854	if (!xcp_mgr)
1855		return 0;
1856
1857	xcp_switch_supported =
1858		(xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
1859
1860	if (!xcp_switch_supported)
1861		dev_attr_current_compute_partition.attr.mode &=
1862			~(S_IWUSR | S_IWGRP | S_IWOTH);
1863
1864	r = device_create_file(adev->dev, &dev_attr_current_compute_partition);
1865	if (r)
1866		return r;
1867
1868	if (xcp_switch_supported)
1869		r = device_create_file(adev->dev,
1870				       &dev_attr_available_compute_partition);
1871
1872	return r;
1873}
1874
1875static void amdgpu_gfx_sysfs_xcp_fini(struct amdgpu_device *adev)
1876{
1877	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1878	bool xcp_switch_supported;
1879
1880	if (!xcp_mgr)
1881		return;
1882
1883	xcp_switch_supported =
1884		(xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
1885	device_remove_file(adev->dev, &dev_attr_current_compute_partition);
1886
1887	if (xcp_switch_supported)
1888		device_remove_file(adev->dev,
1889				   &dev_attr_available_compute_partition);
1890}
1891
1892static int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev)
1893{
1894	int r;
1895
1896	r = device_create_file(adev->dev, &dev_attr_enforce_isolation);
1897	if (r)
1898		return r;
1899	if (adev->gfx.enable_cleaner_shader)
1900		r = device_create_file(adev->dev, &dev_attr_run_cleaner_shader);
1901
1902	return r;
1903}
1904
1905static void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev)
1906{
1907	device_remove_file(adev->dev, &dev_attr_enforce_isolation);
1908	if (adev->gfx.enable_cleaner_shader)
1909		device_remove_file(adev->dev, &dev_attr_run_cleaner_shader);
1910}
1911
1912static int amdgpu_gfx_sysfs_reset_mask_init(struct amdgpu_device *adev)
1913{
1914	int r = 0;
1915
1916	if (!amdgpu_gpu_recovery)
1917		return r;
1918
1919	if (adev->gfx.num_gfx_rings) {
1920		r = device_create_file(adev->dev, &dev_attr_gfx_reset_mask);
1921		if (r)
1922			return r;
1923	}
1924
1925	if (adev->gfx.num_compute_rings) {
1926		r = device_create_file(adev->dev, &dev_attr_compute_reset_mask);
1927		if (r)
1928			return r;
1929	}
1930
1931	return r;
1932}
1933
1934static void amdgpu_gfx_sysfs_reset_mask_fini(struct amdgpu_device *adev)
1935{
1936	if (!amdgpu_gpu_recovery)
1937		return;
1938
1939	if (adev->gfx.num_gfx_rings)
1940		device_remove_file(adev->dev, &dev_attr_gfx_reset_mask);
1941
1942	if (adev->gfx.num_compute_rings)
1943		device_remove_file(adev->dev, &dev_attr_compute_reset_mask);
1944}
1945
1946int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev)
1947{
1948	int r;
1949
1950	r = amdgpu_gfx_sysfs_xcp_init(adev);
1951	if (r) {
1952		dev_err(adev->dev, "failed to create xcp sysfs files");
1953		return r;
1954	}
1955
1956	r = amdgpu_gfx_sysfs_isolation_shader_init(adev);
1957	if (r)
1958		dev_err(adev->dev, "failed to create isolation sysfs files");
1959
1960	r = amdgpu_gfx_sysfs_reset_mask_init(adev);
1961	if (r)
1962		dev_err(adev->dev, "failed to create reset mask sysfs files");
1963
1964	return r;
1965}
1966
1967void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev)
1968{
1969	if (adev->dev->kobj.sd) {
1970		amdgpu_gfx_sysfs_xcp_fini(adev);
1971		amdgpu_gfx_sysfs_isolation_shader_fini(adev);
1972		amdgpu_gfx_sysfs_reset_mask_fini(adev);
1973	}
1974}
1975
1976int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev,
1977				      unsigned int cleaner_shader_size)
1978{
1979	if (!adev->gfx.enable_cleaner_shader)
1980		return -EOPNOTSUPP;
1981
1982	return amdgpu_bo_create_kernel(adev, cleaner_shader_size, PAGE_SIZE,
1983				       AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
1984				       &adev->gfx.cleaner_shader_obj,
1985				       &adev->gfx.cleaner_shader_gpu_addr,
1986				       (void **)&adev->gfx.cleaner_shader_cpu_ptr);
1987}
1988
1989void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev)
1990{
1991	if (!adev->gfx.enable_cleaner_shader)
1992		return;
1993
1994	amdgpu_bo_free_kernel(&adev->gfx.cleaner_shader_obj,
1995			      &adev->gfx.cleaner_shader_gpu_addr,
1996			      (void **)&adev->gfx.cleaner_shader_cpu_ptr);
1997}
1998
1999void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev,
2000				    unsigned int cleaner_shader_size,
2001				    const void *cleaner_shader_ptr)
2002{
2003	if (!adev->gfx.enable_cleaner_shader)
2004		return;
2005
2006	if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr)
2007		memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr,
2008			    cleaner_shader_size);
2009}
2010
2011/**
2012 * amdgpu_gfx_kfd_sch_ctrl - Control the KFD scheduler from the KGD (Graphics Driver)
2013 * @adev: amdgpu_device pointer
2014 * @idx: Index of the scheduler to control
2015 * @enable: Whether to enable or disable the KFD scheduler
2016 *
2017 * This function is used to control the KFD (Kernel Fusion Driver) scheduler
2018 * from the KGD. It is part of the cleaner shader feature. This function plays
2019 * a key role in enforcing process isolation on the GPU.
2020 *
2021 * The function uses a reference count mechanism (kfd_sch_req_count) to keep
2022 * track of the number of requests to enable the KFD scheduler. When a request
2023 * to enable the KFD scheduler is made, the reference count is decremented.
2024 * When the reference count reaches zero, a delayed work is scheduled to
2025 * enforce isolation after a delay of GFX_SLICE_PERIOD.
2026 *
2027 * When a request to disable the KFD scheduler is made, the function first
2028 * checks if the reference count is zero. If it is, it cancels the delayed work
2029 * for enforcing isolation and checks if the KFD scheduler is active. If the
2030 * KFD scheduler is active, it sends a request to stop the KFD scheduler and
2031 * sets the KFD scheduler state to inactive. Then, it increments the reference
2032 * count.
2033 *
2034 * The function is synchronized using the kfd_sch_mutex to ensure that the KFD
2035 * scheduler state and reference count are updated atomically.
2036 *
2037 * Note: If the reference count is already zero when a request to enable the
2038 * KFD scheduler is made, it means there's an imbalance bug somewhere. The
2039 * function triggers a warning in this case.
2040 */
2041static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx,
2042				    bool enable)
2043{
2044	mutex_lock(&adev->gfx.userq_sch_mutex);
2045
2046	if (enable) {
2047		/* If the count is already 0, it means there's an imbalance bug somewhere.
2048		 * Note that the bug may be in a different caller than the one which triggers the
2049		 * WARN_ON_ONCE.
2050		 */
2051		if (WARN_ON_ONCE(adev->gfx.userq_sch_req_count[idx] == 0)) {
2052			dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n");
2053			goto unlock;
2054		}
2055
2056		adev->gfx.userq_sch_req_count[idx]--;
2057
2058		if (adev->gfx.userq_sch_req_count[idx] == 0 &&
2059		    adev->gfx.userq_sch_inactive[idx]) {
2060			schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work,
2061					      msecs_to_jiffies(adev->gfx.enforce_isolation_time[idx]));
2062		}
2063	} else {
2064		if (adev->gfx.userq_sch_req_count[idx] == 0) {
2065			cancel_delayed_work_sync(&adev->gfx.enforce_isolation[idx].work);
2066			if (!adev->gfx.userq_sch_inactive[idx]) {
2067				amdgpu_userq_stop_sched_for_enforce_isolation(adev, idx);
2068				if (adev->kfd.init_complete)
2069					amdgpu_amdkfd_stop_sched(adev, idx);
2070				adev->gfx.userq_sch_inactive[idx] = true;
2071			}
2072		}
2073
2074		adev->gfx.userq_sch_req_count[idx]++;
2075	}
2076
2077unlock:
2078	mutex_unlock(&adev->gfx.userq_sch_mutex);
2079}
2080
2081/**
2082 * amdgpu_gfx_enforce_isolation_handler - work handler for enforcing shader isolation
2083 *
2084 * @work: work_struct.
2085 *
2086 * This function is the work handler for enforcing shader isolation on AMD GPUs.
2087 * It counts the number of emitted fences for each GFX and compute ring. If there
2088 * are any fences, it schedules the `enforce_isolation_work` to be run after a
2089 * delay of `GFX_SLICE_PERIOD`. If there are no fences, it signals the Kernel Fusion
2090 * Driver (KFD) to resume the runqueue. The function is synchronized using the
2091 * `enforce_isolation_mutex`.
2092 */
2093void amdgpu_gfx_enforce_isolation_handler(struct work_struct *work)
2094{
2095	struct amdgpu_isolation_work *isolation_work =
2096		container_of(work, struct amdgpu_isolation_work, work.work);
2097	struct amdgpu_device *adev = isolation_work->adev;
2098	u32 i, idx, fences = 0;
2099
2100	if (isolation_work->xcp_id == AMDGPU_XCP_NO_PARTITION)
2101		idx = 0;
2102	else
2103		idx = isolation_work->xcp_id;
2104
2105	if (idx >= MAX_XCP)
2106		return;
2107
2108	mutex_lock(&adev->enforce_isolation_mutex);
2109	for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) {
2110		if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id)
2111			fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]);
2112	}
2113	for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) {
2114		if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id)
2115			fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]);
2116	}
2117	if (fences) {
2118		/* we've already had our timeslice, so let's wrap this up */
2119		schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work,
2120				      msecs_to_jiffies(1));
2121	} else {
2122		/* Tell KFD to resume the runqueue */
2123		WARN_ON_ONCE(!adev->gfx.userq_sch_inactive[idx]);
2124		WARN_ON_ONCE(adev->gfx.userq_sch_req_count[idx]);
2125
2126		amdgpu_userq_start_sched_for_enforce_isolation(adev, idx);
2127		if (adev->kfd.init_complete)
2128			amdgpu_amdkfd_start_sched(adev, idx);
2129		adev->gfx.userq_sch_inactive[idx] = false;
2130	}
2131	mutex_unlock(&adev->enforce_isolation_mutex);
2132}
2133
2134/**
2135 * amdgpu_gfx_enforce_isolation_wait_for_kfd - Manage KFD wait period for process isolation
2136 * @adev: amdgpu_device pointer
2137 * @idx: Index of the GPU partition
2138 *
2139 * When kernel submissions come in, the jobs are given a time slice and once
2140 * that time slice is up, if there are KFD user queues active, kernel
2141 * submissions are blocked until KFD has had its time slice. Once the KFD time
2142 * slice is up, KFD user queues are preempted and kernel submissions are
2143 * unblocked and allowed to run again.
2144 */
2145static void
2146amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev,
2147					  u32 idx)
2148{
2149	unsigned long cjiffies;
2150	bool wait = false;
2151
2152	mutex_lock(&adev->enforce_isolation_mutex);
2153	if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2154		/* set the initial values if nothing is set */
2155		if (!adev->gfx.enforce_isolation_jiffies[idx]) {
2156			adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2157			adev->gfx.enforce_isolation_time[idx] =	GFX_SLICE_PERIOD_MS;
2158		}
2159		/* Make sure KFD gets a chance to run */
2160		if (amdgpu_amdkfd_compute_active(adev, idx)) {
2161			cjiffies = jiffies;
2162			if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) {
2163				cjiffies -= adev->gfx.enforce_isolation_jiffies[idx];
2164				if ((jiffies_to_msecs(cjiffies) >= GFX_SLICE_PERIOD_MS)) {
2165					/* if our time is up, let KGD work drain before scheduling more */
2166					wait = true;
2167					/* reset the timer period */
2168					adev->gfx.enforce_isolation_time[idx] =	GFX_SLICE_PERIOD_MS;
2169				} else {
2170					/* set the timer period to what's left in our time slice */
2171					adev->gfx.enforce_isolation_time[idx] =
2172						GFX_SLICE_PERIOD_MS - jiffies_to_msecs(cjiffies);
2173				}
2174			} else {
2175				/* if jiffies wrap around we will just wait a little longer */
2176				adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2177			}
2178		} else {
2179			/* if there is no KFD work, then set the full slice period */
2180			adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2181			adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2182		}
2183	}
2184	mutex_unlock(&adev->enforce_isolation_mutex);
2185
2186	if (wait)
2187		msleep(GFX_SLICE_PERIOD_MS);
2188}
2189
2190/**
2191 * amdgpu_gfx_enforce_isolation_ring_begin_use - Begin use of a ring with enforced isolation
2192 * @ring: Pointer to the amdgpu_ring structure
2193 *
2194 * Ring begin_use helper implementation for gfx which serializes access to the
2195 * gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2196 * enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2197 * each get a time slice when both are active.
2198 */
2199void amdgpu_gfx_enforce_isolation_ring_begin_use(struct amdgpu_ring *ring)
2200{
2201	struct amdgpu_device *adev = ring->adev;
2202	u32 idx;
2203	bool sched_work = false;
2204
2205	if (!adev->gfx.enable_cleaner_shader)
2206		return;
2207
2208	if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2209		idx = 0;
2210	else
2211		idx = ring->xcp_id;
2212
2213	if (idx >= MAX_XCP)
2214		return;
2215
2216	/* Don't submit more work until KFD has had some time */
2217	amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx);
2218
2219	mutex_lock(&adev->enforce_isolation_mutex);
2220	if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2221		if (adev->kfd.init_complete)
2222			sched_work = true;
2223	}
2224	mutex_unlock(&adev->enforce_isolation_mutex);
2225
2226	if (sched_work)
2227		amdgpu_gfx_kfd_sch_ctrl(adev, idx, false);
2228}
2229
2230/**
2231 * amdgpu_gfx_enforce_isolation_ring_end_use - End use of a ring with enforced isolation
2232 * @ring: Pointer to the amdgpu_ring structure
2233 *
2234 * Ring end_use helper implementation for gfx which serializes access to the
2235 * gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2236 * enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2237 * each get a time slice when both are active.
2238 */
2239void amdgpu_gfx_enforce_isolation_ring_end_use(struct amdgpu_ring *ring)
2240{
2241	struct amdgpu_device *adev = ring->adev;
2242	u32 idx;
2243	bool sched_work = false;
2244
2245	if (!adev->gfx.enable_cleaner_shader)
2246		return;
2247
2248	if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2249		idx = 0;
2250	else
2251		idx = ring->xcp_id;
2252
2253	if (idx >= MAX_XCP)
2254		return;
2255
2256	mutex_lock(&adev->enforce_isolation_mutex);
2257	if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2258		if (adev->kfd.init_complete)
2259			sched_work = true;
2260	}
2261	mutex_unlock(&adev->enforce_isolation_mutex);
2262
2263	if (sched_work)
2264		amdgpu_gfx_kfd_sch_ctrl(adev, idx, true);
2265}
2266
2267void amdgpu_gfx_profile_idle_work_handler(struct work_struct *work)
2268{
2269	struct amdgpu_device *adev =
2270		container_of(work, struct amdgpu_device, gfx.idle_work.work);
2271	enum PP_SMC_POWER_PROFILE profile;
2272	u32 i, fences = 0;
2273	int r;
2274
2275	if (adev->gfx.num_gfx_rings)
2276		profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
2277	else
2278		profile = PP_SMC_POWER_PROFILE_COMPUTE;
2279
2280	for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i)
2281		fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]);
2282	for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i)
2283		fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]);
2284	if (!fences && !atomic_read(&adev->gfx.total_submission_cnt)) {
2285		mutex_lock(&adev->gfx.workload_profile_mutex);
2286		if (adev->gfx.workload_profile_active) {
2287			r = amdgpu_dpm_switch_power_profile(adev, profile, false);
2288			if (r)
2289				dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,
2290					 profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?
2291					 "fullscreen 3D" : "compute");
2292			adev->gfx.workload_profile_active = false;
2293		}
2294		mutex_unlock(&adev->gfx.workload_profile_mutex);
2295	} else {
2296		schedule_delayed_work(&adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);
2297	}
2298}
2299
2300void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring)
2301{
2302	struct amdgpu_device *adev = ring->adev;
2303	enum PP_SMC_POWER_PROFILE profile;
2304	int r;
2305
2306	if (amdgpu_dpm_is_overdrive_enabled(adev))
2307		return;
2308
2309	if (adev->gfx.num_gfx_rings)
2310		profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
2311	else
2312		profile = PP_SMC_POWER_PROFILE_COMPUTE;
2313
2314	atomic_inc(&adev->gfx.total_submission_cnt);
2315
2316	cancel_delayed_work_sync(&adev->gfx.idle_work);
2317
2318	/* We can safely return early here because we've cancelled the
2319	 * the delayed work so there is no one else to set it to false
2320	 * and we don't care if someone else sets it to true.
2321	 */
2322	if (adev->gfx.workload_profile_active)
2323		return;
2324
2325	mutex_lock(&adev->gfx.workload_profile_mutex);
2326	if (!adev->gfx.workload_profile_active) {
2327		r = amdgpu_dpm_switch_power_profile(adev, profile, true);
2328		if (r)
2329			dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,
2330				 profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?
2331				 "fullscreen 3D" : "compute");
2332		adev->gfx.workload_profile_active = true;
2333	}
2334	mutex_unlock(&adev->gfx.workload_profile_mutex);
2335}
2336
2337void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring)
2338{
2339	struct amdgpu_device *adev = ring->adev;
2340
2341	if (amdgpu_dpm_is_overdrive_enabled(adev))
2342		return;
2343
2344	atomic_dec(&ring->adev->gfx.total_submission_cnt);
2345
2346	schedule_delayed_work(&ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);
2347}
2348
2349/**
2350 * amdgpu_gfx_csb_preamble_start - Set CSB preamble start
2351 *
2352 * @buffer: This is an output variable that gets the PACKET3 preamble setup.
2353 *
2354 * Return:
2355 * return the latest index.
2356 */
2357u32 amdgpu_gfx_csb_preamble_start(u32 *buffer)
2358{
2359	u32 count = 0;
2360
2361	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2362	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2363
2364	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2365	buffer[count++] = cpu_to_le32(0x80000000);
2366	buffer[count++] = cpu_to_le32(0x80000000);
2367
2368	return count;
2369}
2370
2371/**
2372 * amdgpu_gfx_csb_data_parser - Parser CS data
2373 *
2374 * @adev: amdgpu_device pointer used to get the CS data and other gfx info.
2375 * @buffer: This is an output variable that gets the PACKET3 preamble end.
2376 * @count: Index to start set the preemble end.
2377 *
2378 * Return:
2379 * return the latest index.
2380 */
2381u32 amdgpu_gfx_csb_data_parser(struct amdgpu_device *adev, u32 *buffer, u32 count)
2382{
2383	const struct cs_section_def *sect = NULL;
2384	const struct cs_extent_def *ext = NULL;
2385	u32 i;
2386
2387	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2388		for (ext = sect->section; ext->extent != NULL; ++ext) {
2389			if (sect->id == SECT_CONTEXT) {
2390				buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2391				buffer[count++] = cpu_to_le32(ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2392
2393				for (i = 0; i < ext->reg_count; i++)
2394					buffer[count++] = cpu_to_le32(ext->extent[i]);
2395			}
2396		}
2397	}
2398
2399	return count;
2400}
2401
2402/**
2403 * amdgpu_gfx_csb_preamble_end - Set CSB preamble end
2404 *
2405 * @buffer: This is an output variable that gets the PACKET3 preamble end.
2406 * @count: Index to start set the preemble end.
2407 */
2408void amdgpu_gfx_csb_preamble_end(u32 *buffer, u32 count)
2409{
2410	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2411	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
2412
2413	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
2414	buffer[count++] = cpu_to_le32(0);
2415}
2416
2417/*
2418 * debugfs for to enable/disable gfx job submission to specific core.
2419 */
2420#if defined(CONFIG_DEBUG_FS)
2421static int amdgpu_debugfs_gfx_sched_mask_set(void *data, u64 val)
2422{
2423	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2424	u32 i;
2425	u64 mask = 0;
2426	struct amdgpu_ring *ring;
2427
2428	if (!adev)
2429		return -ENODEV;
2430
2431	mask = (1ULL << adev->gfx.num_gfx_rings) - 1;
2432	if ((val & mask) == 0)
2433		return -EINVAL;
2434
2435	for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2436		ring = &adev->gfx.gfx_ring[i];
2437		if (val & (1 << i))
2438			ring->sched.ready = true;
2439		else
2440			ring->sched.ready = false;
2441	}
2442	/* publish sched.ready flag update effective immediately across smp */
2443	smp_rmb();
2444	return 0;
2445}
2446
2447static int amdgpu_debugfs_gfx_sched_mask_get(void *data, u64 *val)
2448{
2449	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2450	u32 i;
2451	u64 mask = 0;
2452	struct amdgpu_ring *ring;
2453
2454	if (!adev)
2455		return -ENODEV;
2456	for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2457		ring = &adev->gfx.gfx_ring[i];
2458		if (ring->sched.ready)
2459			mask |= 1ULL << i;
2460	}
2461
2462	*val = mask;
2463	return 0;
2464}
2465
2466DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gfx_sched_mask_fops,
2467			 amdgpu_debugfs_gfx_sched_mask_get,
2468			 amdgpu_debugfs_gfx_sched_mask_set, "%llx\n");
2469
2470#endif
2471
2472void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev)
2473{
2474#if defined(CONFIG_DEBUG_FS)
2475	struct drm_minor *minor = adev_to_drm(adev)->primary;
2476	struct dentry *root = minor->debugfs_root;
2477	char name[32];
2478
2479	if (!(adev->gfx.num_gfx_rings > 1))
2480		return;
2481	sprintf(name, "amdgpu_gfx_sched_mask");
2482	debugfs_create_file(name, 0600, root, adev,
2483			    &amdgpu_debugfs_gfx_sched_mask_fops);
2484#endif
2485}
2486
2487/*
2488 * debugfs for to enable/disable compute job submission to specific core.
2489 */
2490#if defined(CONFIG_DEBUG_FS)
2491static int amdgpu_debugfs_compute_sched_mask_set(void *data, u64 val)
2492{
2493	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2494	u32 i;
2495	u64 mask = 0;
2496	struct amdgpu_ring *ring;
2497
2498	if (!adev)
2499		return -ENODEV;
2500
2501	mask = (1ULL << adev->gfx.num_compute_rings) - 1;
2502	if ((val & mask) == 0)
2503		return -EINVAL;
2504
2505	for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2506		ring = &adev->gfx.compute_ring[i];
2507		if (val & (1 << i))
2508			ring->sched.ready = true;
2509		else
2510			ring->sched.ready = false;
2511	}
2512
2513	/* publish sched.ready flag update effective immediately across smp */
2514	smp_rmb();
2515	return 0;
2516}
2517
2518static int amdgpu_debugfs_compute_sched_mask_get(void *data, u64 *val)
2519{
2520	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2521	u32 i;
2522	u64 mask = 0;
2523	struct amdgpu_ring *ring;
2524
2525	if (!adev)
2526		return -ENODEV;
2527	for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2528		ring = &adev->gfx.compute_ring[i];
2529		if (ring->sched.ready)
2530			mask |= 1ULL << i;
2531	}
2532
2533	*val = mask;
2534	return 0;
2535}
2536
2537DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_compute_sched_mask_fops,
2538			 amdgpu_debugfs_compute_sched_mask_get,
2539			 amdgpu_debugfs_compute_sched_mask_set, "%llx\n");
2540
2541#endif
2542
2543void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev)
2544{
2545#if defined(CONFIG_DEBUG_FS)
2546	struct drm_minor *minor = adev_to_drm(adev)->primary;
2547	struct dentry *root = minor->debugfs_root;
2548	char name[32];
2549
2550	if (!(adev->gfx.num_compute_rings > 1))
2551		return;
2552	sprintf(name, "amdgpu_compute_sched_mask");
2553	debugfs_create_file(name, 0600, root, adev,
2554			    &amdgpu_debugfs_compute_sched_mask_fops);
2555#endif
2556}
2557