amdgpu_kms.c

   1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include "amdgpu.h"
  30#include <drm/amdgpu_drm.h>
  31#include <drm/drm_drv.h>
  32#include <drm/drm_fb_helper.h>
  33#include "amdgpu_uvd.h"
  34#include "amdgpu_vce.h"
  35#include "atom.h"
  36
  37#include <linux/vga_switcheroo.h>
  38#include <linux/slab.h>
  39#include <linux/uaccess.h>
  40#include <linux/pci.h>
  41#include <linux/pm_runtime.h>
  42#include "amdgpu_amdkfd.h"
  43#include "amdgpu_gem.h"
  44#include "amdgpu_display.h"
  45#include "amdgpu_ras.h"
  46#include "amdgpu_reset.h"
  47#include "amd_pcie.h"
  48#include "amdgpu_userq.h"
  49
  50void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
  51{
  52	struct amdgpu_gpu_instance *gpu_instance;
  53	int i;
  54
  55	mutex_lock(&mgpu_info.mutex);
  56
  57	for (i = 0; i < mgpu_info.num_gpu; i++) {
  58		gpu_instance = &(mgpu_info.gpu_ins[i]);
  59		if (gpu_instance->adev == adev) {
  60			mgpu_info.gpu_ins[i] =
  61				mgpu_info.gpu_ins[mgpu_info.num_gpu - 1];
  62			mgpu_info.num_gpu--;
  63			if (adev->flags & AMD_IS_APU)
  64				mgpu_info.num_apu--;
  65			else
  66				mgpu_info.num_dgpu--;
  67			break;
  68		}
  69	}
  70
  71	mutex_unlock(&mgpu_info.mutex);
  72}
  73
  74/**
  75 * amdgpu_driver_unload_kms - Main unload function for KMS.
  76 *
  77 * @dev: drm dev pointer
  78 *
  79 * This is the main unload function for KMS (all asics).
  80 * Returns 0 on success.
  81 */
  82void amdgpu_driver_unload_kms(struct drm_device *dev)
  83{
  84	struct amdgpu_device *adev = drm_to_adev(dev);
  85
  86	if (adev == NULL || !adev->num_ip_blocks)
  87		return;
  88
  89	amdgpu_unregister_gpu_instance(adev);
  90
  91	if (adev->rmmio == NULL)
  92		return;
  93
  94	if (amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DRV_UNLOAD))
  95		DRM_WARN("smart shift update failed\n");
  96
  97	amdgpu_acpi_fini(adev);
  98	amdgpu_device_fini_hw(adev);
  99}
 100
 101void amdgpu_register_gpu_instance(struct amdgpu_device *adev)
 102{
 103	struct amdgpu_gpu_instance *gpu_instance;
 104
 105	mutex_lock(&mgpu_info.mutex);
 106
 107	if (mgpu_info.num_gpu >= MAX_GPU_INSTANCE) {
 108		DRM_ERROR("Cannot register more gpu instance\n");
 109		mutex_unlock(&mgpu_info.mutex);
 110		return;
 111	}
 112
 113	gpu_instance = &(mgpu_info.gpu_ins[mgpu_info.num_gpu]);
 114	gpu_instance->adev = adev;
 115	gpu_instance->mgpu_fan_enabled = 0;
 116
 117	mgpu_info.num_gpu++;
 118	if (adev->flags & AMD_IS_APU)
 119		mgpu_info.num_apu++;
 120	else
 121		mgpu_info.num_dgpu++;
 122
 123	mutex_unlock(&mgpu_info.mutex);
 124}
 125
 126/**
 127 * amdgpu_driver_load_kms - Main load function for KMS.
 128 *
 129 * @adev: pointer to struct amdgpu_device
 130 * @flags: device flags
 131 *
 132 * This is the main load function for KMS (all asics).
 133 * Returns 0 on success, error on failure.
 134 */
 135int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags)
 136{
 137	struct drm_device *dev;
 138	int r, acpi_status;
 139
 140	dev = adev_to_drm(adev);
 141
 142	/* amdgpu_device_init should report only fatal error
 143	 * like memory allocation failure or iomapping failure,
 144	 * or memory manager initialization failure, it must
 145	 * properly initialize the GPU MC controller and permit
 146	 * VRAM allocation
 147	 */
 148	r = amdgpu_device_init(adev, flags);
 149	if (r) {
 150		dev_err(dev->dev, "Fatal error during GPU init\n");
 151		goto out;
 152	}
 153
 154	amdgpu_device_detect_runtime_pm_mode(adev);
 155
 156	/* Call ACPI methods: require modeset init
 157	 * but failure is not fatal
 158	 */
 159
 160	acpi_status = amdgpu_acpi_init(adev);
 161	if (acpi_status)
 162		dev_dbg(dev->dev, "Error during ACPI methods call\n");
 163
 164	if (amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DRV_LOAD))
 165		DRM_WARN("smart shift update failed\n");
 166
 167out:
 168	if (r)
 169		amdgpu_driver_unload_kms(dev);
 170
 171	return r;
 172}
 173
 174static enum amd_ip_block_type
 175	amdgpu_ip_get_block_type(struct amdgpu_device *adev, uint32_t ip)
 176{
 177	enum amd_ip_block_type type;
 178
 179	switch (ip) {
 180	case AMDGPU_HW_IP_GFX:
 181		type = AMD_IP_BLOCK_TYPE_GFX;
 182		break;
 183	case AMDGPU_HW_IP_COMPUTE:
 184		type = AMD_IP_BLOCK_TYPE_GFX;
 185		break;
 186	case AMDGPU_HW_IP_DMA:
 187		type = AMD_IP_BLOCK_TYPE_SDMA;
 188		break;
 189	case AMDGPU_HW_IP_UVD:
 190	case AMDGPU_HW_IP_UVD_ENC:
 191		type = AMD_IP_BLOCK_TYPE_UVD;
 192		break;
 193	case AMDGPU_HW_IP_VCE:
 194		type = AMD_IP_BLOCK_TYPE_VCE;
 195		break;
 196	case AMDGPU_HW_IP_VCN_DEC:
 197	case AMDGPU_HW_IP_VCN_ENC:
 198		type = AMD_IP_BLOCK_TYPE_VCN;
 199		break;
 200	case AMDGPU_HW_IP_VCN_JPEG:
 201		type = (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_JPEG)) ?
 202				   AMD_IP_BLOCK_TYPE_JPEG : AMD_IP_BLOCK_TYPE_VCN;
 203		break;
 204	case AMDGPU_HW_IP_VPE:
 205		type = AMD_IP_BLOCK_TYPE_VPE;
 206		break;
 207	default:
 208		type = AMD_IP_BLOCK_TYPE_NUM;
 209		break;
 210	}
 211
 212	return type;
 213}
 214
 215static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 216				struct drm_amdgpu_query_fw *query_fw,
 217				struct amdgpu_device *adev)
 218{
 219	switch (query_fw->fw_type) {
 220	case AMDGPU_INFO_FW_VCE:
 221		fw_info->ver = adev->vce.fw_version;
 222		fw_info->feature = adev->vce.fb_version;
 223		break;
 224	case AMDGPU_INFO_FW_UVD:
 225		fw_info->ver = adev->uvd.fw_version;
 226		fw_info->feature = 0;
 227		break;
 228	case AMDGPU_INFO_FW_VCN:
 229		fw_info->ver = adev->vcn.fw_version;
 230		fw_info->feature = 0;
 231		break;
 232	case AMDGPU_INFO_FW_GMC:
 233		fw_info->ver = adev->gmc.fw_version;
 234		fw_info->feature = 0;
 235		break;
 236	case AMDGPU_INFO_FW_GFX_ME:
 237		fw_info->ver = adev->gfx.me_fw_version;
 238		fw_info->feature = adev->gfx.me_feature_version;
 239		break;
 240	case AMDGPU_INFO_FW_GFX_PFP:
 241		fw_info->ver = adev->gfx.pfp_fw_version;
 242		fw_info->feature = adev->gfx.pfp_feature_version;
 243		break;
 244	case AMDGPU_INFO_FW_GFX_CE:
 245		fw_info->ver = adev->gfx.ce_fw_version;
 246		fw_info->feature = adev->gfx.ce_feature_version;
 247		break;
 248	case AMDGPU_INFO_FW_GFX_RLC:
 249		fw_info->ver = adev->gfx.rlc_fw_version;
 250		fw_info->feature = adev->gfx.rlc_feature_version;
 251		break;
 252	case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL:
 253		fw_info->ver = adev->gfx.rlc_srlc_fw_version;
 254		fw_info->feature = adev->gfx.rlc_srlc_feature_version;
 255		break;
 256	case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM:
 257		fw_info->ver = adev->gfx.rlc_srlg_fw_version;
 258		fw_info->feature = adev->gfx.rlc_srlg_feature_version;
 259		break;
 260	case AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM:
 261		fw_info->ver = adev->gfx.rlc_srls_fw_version;
 262		fw_info->feature = adev->gfx.rlc_srls_feature_version;
 263		break;
 264	case AMDGPU_INFO_FW_GFX_RLCP:
 265		fw_info->ver = adev->gfx.rlcp_ucode_version;
 266		fw_info->feature = adev->gfx.rlcp_ucode_feature_version;
 267		break;
 268	case AMDGPU_INFO_FW_GFX_RLCV:
 269		fw_info->ver = adev->gfx.rlcv_ucode_version;
 270		fw_info->feature = adev->gfx.rlcv_ucode_feature_version;
 271		break;
 272	case AMDGPU_INFO_FW_GFX_MEC:
 273		if (query_fw->index == 0) {
 274			fw_info->ver = adev->gfx.mec_fw_version;
 275			fw_info->feature = adev->gfx.mec_feature_version;
 276		} else if (query_fw->index == 1) {
 277			fw_info->ver = adev->gfx.mec2_fw_version;
 278			fw_info->feature = adev->gfx.mec2_feature_version;
 279		} else
 280			return -EINVAL;
 281		break;
 282	case AMDGPU_INFO_FW_SMC:
 283		fw_info->ver = adev->pm.fw_version;
 284		fw_info->feature = 0;
 285		break;
 286	case AMDGPU_INFO_FW_TA:
 287		switch (query_fw->index) {
 288		case TA_FW_TYPE_PSP_XGMI:
 289			fw_info->ver = adev->psp.xgmi_context.context.bin_desc.fw_version;
 290			fw_info->feature = adev->psp.xgmi_context.context
 291						   .bin_desc.feature_version;
 292			break;
 293		case TA_FW_TYPE_PSP_RAS:
 294			fw_info->ver = adev->psp.ras_context.context.bin_desc.fw_version;
 295			fw_info->feature = adev->psp.ras_context.context
 296						   .bin_desc.feature_version;
 297			break;
 298		case TA_FW_TYPE_PSP_HDCP:
 299			fw_info->ver = adev->psp.hdcp_context.context.bin_desc.fw_version;
 300			fw_info->feature = adev->psp.hdcp_context.context
 301						   .bin_desc.feature_version;
 302			break;
 303		case TA_FW_TYPE_PSP_DTM:
 304			fw_info->ver = adev->psp.dtm_context.context.bin_desc.fw_version;
 305			fw_info->feature = adev->psp.dtm_context.context
 306						   .bin_desc.feature_version;
 307			break;
 308		case TA_FW_TYPE_PSP_RAP:
 309			fw_info->ver = adev->psp.rap_context.context.bin_desc.fw_version;
 310			fw_info->feature = adev->psp.rap_context.context
 311						   .bin_desc.feature_version;
 312			break;
 313		case TA_FW_TYPE_PSP_SECUREDISPLAY:
 314			fw_info->ver = adev->psp.securedisplay_context.context.bin_desc.fw_version;
 315			fw_info->feature =
 316				adev->psp.securedisplay_context.context.bin_desc
 317					.feature_version;
 318			break;
 319		default:
 320			return -EINVAL;
 321		}
 322		break;
 323	case AMDGPU_INFO_FW_SDMA:
 324		if (query_fw->index >= adev->sdma.num_instances)
 325			return -EINVAL;
 326		fw_info->ver = adev->sdma.instance[query_fw->index].fw_version;
 327		fw_info->feature = adev->sdma.instance[query_fw->index].feature_version;
 328		break;
 329	case AMDGPU_INFO_FW_SOS:
 330		fw_info->ver = adev->psp.sos.fw_version;
 331		fw_info->feature = adev->psp.sos.feature_version;
 332		break;
 333	case AMDGPU_INFO_FW_ASD:
 334		fw_info->ver = adev->psp.asd_context.bin_desc.fw_version;
 335		fw_info->feature = adev->psp.asd_context.bin_desc.feature_version;
 336		break;
 337	case AMDGPU_INFO_FW_DMCU:
 338		fw_info->ver = adev->dm.dmcu_fw_version;
 339		fw_info->feature = 0;
 340		break;
 341	case AMDGPU_INFO_FW_DMCUB:
 342		fw_info->ver = adev->dm.dmcub_fw_version;
 343		fw_info->feature = 0;
 344		break;
 345	case AMDGPU_INFO_FW_TOC:
 346		fw_info->ver = adev->psp.toc.fw_version;
 347		fw_info->feature = adev->psp.toc.feature_version;
 348		break;
 349	case AMDGPU_INFO_FW_CAP:
 350		fw_info->ver = adev->psp.cap_fw_version;
 351		fw_info->feature = adev->psp.cap_feature_version;
 352		break;
 353	case AMDGPU_INFO_FW_MES_KIQ:
 354		fw_info->ver = adev->mes.kiq_version & AMDGPU_MES_VERSION_MASK;
 355		fw_info->feature = (adev->mes.kiq_version & AMDGPU_MES_FEAT_VERSION_MASK)
 356					>> AMDGPU_MES_FEAT_VERSION_SHIFT;
 357		break;
 358	case AMDGPU_INFO_FW_MES:
 359		fw_info->ver = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;
 360		fw_info->feature = (adev->mes.sched_version & AMDGPU_MES_FEAT_VERSION_MASK)
 361					>> AMDGPU_MES_FEAT_VERSION_SHIFT;
 362		break;
 363	case AMDGPU_INFO_FW_IMU:
 364		fw_info->ver = adev->gfx.imu_fw_version;
 365		fw_info->feature = 0;
 366		break;
 367	case AMDGPU_INFO_FW_VPE:
 368		fw_info->ver = adev->vpe.fw_version;
 369		fw_info->feature = adev->vpe.feature_version;
 370		break;
 371	default:
 372		return -EINVAL;
 373	}
 374	return 0;
 375}
 376
 377static int amdgpu_userq_metadata_info_gfx(struct amdgpu_device *adev,
 378					  struct drm_amdgpu_info *info,
 379					  struct drm_amdgpu_info_uq_metadata_gfx *meta)
 380{
 381	int ret = -EOPNOTSUPP;
 382
 383	if (adev->gfx.funcs->get_gfx_shadow_info) {
 384		struct amdgpu_gfx_shadow_info shadow = {};
 385
 386		adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow, true);
 387		meta->shadow_size = shadow.shadow_size;
 388		meta->shadow_alignment = shadow.shadow_alignment;
 389		meta->csa_size = shadow.csa_size;
 390		meta->csa_alignment = shadow.csa_alignment;
 391		ret = 0;
 392	}
 393
 394	return ret;
 395}
 396
 397static int amdgpu_hw_ip_info(struct amdgpu_device *adev,
 398			     struct drm_amdgpu_info *info,
 399			     struct drm_amdgpu_info_hw_ip *result)
 400{
 401	uint32_t ib_start_alignment = 0;
 402	uint32_t ib_size_alignment = 0;
 403	enum amd_ip_block_type type;
 404	unsigned int num_rings = 0;
 405	uint32_t num_slots = 0;
 406	unsigned int i, j;
 407
 408	if (info->query_hw_ip.ip_instance >= AMDGPU_HW_IP_INSTANCE_MAX_COUNT)
 409		return -EINVAL;
 410
 411	switch (info->query_hw_ip.type) {
 412	case AMDGPU_HW_IP_GFX:
 413		type = AMD_IP_BLOCK_TYPE_GFX;
 414		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
 415			if (adev->gfx.gfx_ring[i].sched.ready &&
 416			    !adev->gfx.gfx_ring[i].no_user_submission)
 417				++num_rings;
 418
 419		if (!adev->gfx.disable_uq) {
 420			for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
 421				num_slots += hweight32(adev->mes.gfx_hqd_mask[i]);
 422		}
 423
 424		ib_start_alignment = 32;
 425		ib_size_alignment = 32;
 426		break;
 427	case AMDGPU_HW_IP_COMPUTE:
 428		type = AMD_IP_BLOCK_TYPE_GFX;
 429		for (i = 0; i < adev->gfx.num_compute_rings; i++)
 430			if (adev->gfx.compute_ring[i].sched.ready &&
 431			    !adev->gfx.compute_ring[i].no_user_submission)
 432				++num_rings;
 433
 434		if (!adev->sdma.disable_uq) {
 435			for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
 436				num_slots += hweight32(adev->mes.compute_hqd_mask[i]);
 437		}
 438
 439		ib_start_alignment = 32;
 440		ib_size_alignment = 32;
 441		break;
 442	case AMDGPU_HW_IP_DMA:
 443		type = AMD_IP_BLOCK_TYPE_SDMA;
 444		for (i = 0; i < adev->sdma.num_instances; i++)
 445			if (adev->sdma.instance[i].ring.sched.ready &&
 446			    !adev->sdma.instance[i].ring.no_user_submission)
 447				++num_rings;
 448
 449		if (!adev->gfx.disable_uq) {
 450			for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
 451				num_slots += hweight32(adev->mes.sdma_hqd_mask[i]);
 452		}
 453
 454		ib_start_alignment = 256;
 455		ib_size_alignment = 4;
 456		break;
 457	case AMDGPU_HW_IP_UVD:
 458		type = AMD_IP_BLOCK_TYPE_UVD;
 459		for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
 460			if (adev->uvd.harvest_config & (1 << i))
 461				continue;
 462
 463			if (adev->uvd.inst[i].ring.sched.ready &&
 464			    !adev->uvd.inst[i].ring.no_user_submission)
 465				++num_rings;
 466		}
 467		ib_start_alignment = 256;
 468		ib_size_alignment = 64;
 469		break;
 470	case AMDGPU_HW_IP_VCE:
 471		type = AMD_IP_BLOCK_TYPE_VCE;
 472		for (i = 0; i < adev->vce.num_rings; i++)
 473			if (adev->vce.ring[i].sched.ready &&
 474			    !adev->vce.ring[i].no_user_submission)
 475				++num_rings;
 476		ib_start_alignment = 256;
 477		ib_size_alignment = 4;
 478		break;
 479	case AMDGPU_HW_IP_UVD_ENC:
 480		type = AMD_IP_BLOCK_TYPE_UVD;
 481		for (i = 0; i < adev->uvd.num_uvd_inst; i++) {
 482			if (adev->uvd.harvest_config & (1 << i))
 483				continue;
 484
 485			for (j = 0; j < adev->uvd.num_enc_rings; j++)
 486				if (adev->uvd.inst[i].ring_enc[j].sched.ready &&
 487				    !adev->uvd.inst[i].ring_enc[j].no_user_submission)
 488					++num_rings;
 489		}
 490		ib_start_alignment = 256;
 491		ib_size_alignment = 4;
 492		break;
 493	case AMDGPU_HW_IP_VCN_DEC:
 494		type = AMD_IP_BLOCK_TYPE_VCN;
 495		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 496			if (adev->vcn.harvest_config & (1 << i))
 497				continue;
 498
 499			if (adev->vcn.inst[i].ring_dec.sched.ready &&
 500			    !adev->vcn.inst[i].ring_dec.no_user_submission)
 501				++num_rings;
 502		}
 503		ib_start_alignment = 256;
 504		ib_size_alignment = 64;
 505		break;
 506	case AMDGPU_HW_IP_VCN_ENC:
 507		type = AMD_IP_BLOCK_TYPE_VCN;
 508		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 509			if (adev->vcn.harvest_config & (1 << i))
 510				continue;
 511
 512			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)
 513				if (adev->vcn.inst[i].ring_enc[j].sched.ready &&
 514				    !adev->vcn.inst[i].ring_enc[j].no_user_submission)
 515					++num_rings;
 516		}
 517		ib_start_alignment = 256;
 518		ib_size_alignment = 4;
 519		break;
 520	case AMDGPU_HW_IP_VCN_JPEG:
 521		type = (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_JPEG)) ?
 522			AMD_IP_BLOCK_TYPE_JPEG : AMD_IP_BLOCK_TYPE_VCN;
 523
 524		for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
 525			if (adev->jpeg.harvest_config & (1 << i))
 526				continue;
 527
 528			for (j = 0; j < adev->jpeg.num_jpeg_rings; j++)
 529				if (adev->jpeg.inst[i].ring_dec[j].sched.ready &&
 530				    !adev->jpeg.inst[i].ring_dec[j].no_user_submission)
 531					++num_rings;
 532		}
 533		ib_start_alignment = 256;
 534		ib_size_alignment = 64;
 535		break;
 536	case AMDGPU_HW_IP_VPE:
 537		type = AMD_IP_BLOCK_TYPE_VPE;
 538		if (adev->vpe.ring.sched.ready &&
 539		    !adev->vpe.ring.no_user_submission)
 540			++num_rings;
 541		ib_start_alignment = 256;
 542		ib_size_alignment = 4;
 543		break;
 544	default:
 545		return -EINVAL;
 546	}
 547
 548	for (i = 0; i < adev->num_ip_blocks; i++)
 549		if (adev->ip_blocks[i].version->type == type &&
 550		    adev->ip_blocks[i].status.valid)
 551			break;
 552
 553	if (i == adev->num_ip_blocks)
 554		return 0;
 555
 556	num_rings = min(amdgpu_ctx_num_entities[info->query_hw_ip.type],
 557			num_rings);
 558
 559	result->hw_ip_version_major = adev->ip_blocks[i].version->major;
 560	result->hw_ip_version_minor = adev->ip_blocks[i].version->minor;
 561
 562	if (adev->asic_type >= CHIP_VEGA10) {
 563		switch (type) {
 564		case AMD_IP_BLOCK_TYPE_GFX:
 565			result->ip_discovery_version =
 566				IP_VERSION_MAJ_MIN_REV(amdgpu_ip_version(adev, GC_HWIP, 0));
 567			break;
 568		case AMD_IP_BLOCK_TYPE_SDMA:
 569			result->ip_discovery_version =
 570				IP_VERSION_MAJ_MIN_REV(amdgpu_ip_version(adev, SDMA0_HWIP, 0));
 571			break;
 572		case AMD_IP_BLOCK_TYPE_UVD:
 573		case AMD_IP_BLOCK_TYPE_VCN:
 574		case AMD_IP_BLOCK_TYPE_JPEG:
 575			result->ip_discovery_version =
 576				IP_VERSION_MAJ_MIN_REV(amdgpu_ip_version(adev, UVD_HWIP, 0));
 577			break;
 578		case AMD_IP_BLOCK_TYPE_VCE:
 579			result->ip_discovery_version =
 580				IP_VERSION_MAJ_MIN_REV(amdgpu_ip_version(adev, VCE_HWIP, 0));
 581			break;
 582		case AMD_IP_BLOCK_TYPE_VPE:
 583			result->ip_discovery_version =
 584				IP_VERSION_MAJ_MIN_REV(amdgpu_ip_version(adev, VPE_HWIP, 0));
 585			break;
 586		default:
 587			result->ip_discovery_version = 0;
 588			break;
 589		}
 590	} else {
 591		result->ip_discovery_version = 0;
 592	}
 593	result->capabilities_flags = 0;
 594	result->available_rings = (1 << num_rings) - 1;
 595	result->userq_num_slots = num_slots;
 596	result->ib_start_alignment = ib_start_alignment;
 597	result->ib_size_alignment = ib_size_alignment;
 598	return 0;
 599}
 600
 601/*
 602 * Userspace get information ioctl
 603 */
 604/**
 605 * amdgpu_info_ioctl - answer a device specific request.
 606 *
 607 * @dev: drm device pointer
 608 * @data: request object
 609 * @filp: drm filp
 610 *
 611 * This function is used to pass device specific parameters to the userspace
 612 * drivers.  Examples include: pci device id, pipeline parms, tiling params,
 613 * etc. (all asics).
 614 * Returns 0 on success, -EINVAL on failure.
 615 */
 616int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 617{
 618	struct amdgpu_device *adev = drm_to_adev(dev);
 619	struct drm_amdgpu_info *info = data;
 620	struct amdgpu_mode_info *minfo = &adev->mode_info;
 621	void __user *out = (void __user *)(uintptr_t)info->return_pointer;
 622	struct amdgpu_fpriv *fpriv;
 623	struct amdgpu_ip_block *ip_block;
 624	enum amd_ip_block_type type;
 625	struct amdgpu_xcp *xcp;
 626	u32 count, inst_mask;
 627	uint32_t size = info->return_size;
 628	struct drm_crtc *crtc;
 629	uint32_t ui32 = 0;
 630	uint64_t ui64 = 0;
 631	int i, found, ret;
 632	int ui32_size = sizeof(ui32);
 633
 634	if (!info->return_size || !info->return_pointer)
 635		return -EINVAL;
 636
 637	switch (info->query) {
 638	case AMDGPU_INFO_ACCEL_WORKING:
 639		ui32 = adev->accel_working;
 640		return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
 641	case AMDGPU_INFO_CRTC_FROM_ID:
 642		for (i = 0, found = 0; i < adev->mode_info.num_crtc; i++) {
 643			crtc = (struct drm_crtc *)minfo->crtcs[i];
 644			if (crtc && crtc->base.id == info->mode_crtc.id) {
 645				struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 646
 647				ui32 = amdgpu_crtc->crtc_id;
 648				found = 1;
 649				break;
 650			}
 651		}
 652		if (!found) {
 653			DRM_DEBUG_KMS("unknown crtc id %d\n", info->mode_crtc.id);
 654			return -EINVAL;
 655		}
 656		return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
 657	case AMDGPU_INFO_HW_IP_INFO: {
 658		struct drm_amdgpu_info_hw_ip ip = {};
 659
 660		ret = amdgpu_hw_ip_info(adev, info, &ip);
 661		if (ret)
 662			return ret;
 663
 664		ret = copy_to_user(out, &ip, min_t(size_t, size, sizeof(ip)));
 665		return ret ? -EFAULT : 0;
 666	}
 667	case AMDGPU_INFO_HW_IP_COUNT: {
 668		fpriv = (struct amdgpu_fpriv *)filp->driver_priv;
 669		type = amdgpu_ip_get_block_type(adev, info->query_hw_ip.type);
 670		ip_block = amdgpu_device_ip_get_ip_block(adev, type);
 671
 672		if (!ip_block || !ip_block->status.valid)
 673			return -EINVAL;
 674
 675		if (adev->xcp_mgr && adev->xcp_mgr->num_xcps > 0 &&
 676		    fpriv->xcp_id < adev->xcp_mgr->num_xcps) {
 677			xcp = &adev->xcp_mgr->xcp[fpriv->xcp_id];
 678			switch (type) {
 679			case AMD_IP_BLOCK_TYPE_GFX:
 680				ret = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask);
 681				if (ret)
 682					return ret;
 683				count = hweight32(inst_mask);
 684				break;
 685			case AMD_IP_BLOCK_TYPE_SDMA:
 686				ret = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_SDMA, &inst_mask);
 687				if (ret)
 688					return ret;
 689				count = hweight32(inst_mask);
 690				break;
 691			case AMD_IP_BLOCK_TYPE_JPEG:
 692				ret = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_VCN, &inst_mask);
 693				if (ret)
 694					return ret;
 695				count = hweight32(inst_mask) * adev->jpeg.num_jpeg_rings;
 696				break;
 697			case AMD_IP_BLOCK_TYPE_VCN:
 698				ret = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_VCN, &inst_mask);
 699				if (ret)
 700					return ret;
 701				count = hweight32(inst_mask);
 702				break;
 703			default:
 704				return -EINVAL;
 705			}
 706
 707			return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
 708		}
 709
 710		switch (type) {
 711		case AMD_IP_BLOCK_TYPE_GFX:
 712		case AMD_IP_BLOCK_TYPE_VCE:
 713			count = 1;
 714			break;
 715		case AMD_IP_BLOCK_TYPE_SDMA:
 716			count = adev->sdma.num_instances;
 717			break;
 718		case AMD_IP_BLOCK_TYPE_JPEG:
 719			count = adev->jpeg.num_jpeg_inst * adev->jpeg.num_jpeg_rings;
 720			break;
 721		case AMD_IP_BLOCK_TYPE_VCN:
 722			count = adev->vcn.num_vcn_inst;
 723			break;
 724		case AMD_IP_BLOCK_TYPE_UVD:
 725			count = adev->uvd.num_uvd_inst;
 726			break;
 727		case AMD_IP_BLOCK_TYPE_VPE:
 728			count = adev->vpe.num_instances;
 729			break;
 730		/* For all other IP block types not listed in the switch statement
 731		 * the ip status is valid here and the instance count is one.
 732		 */
 733		default:
 734			count = 1;
 735			break;
 736		}
 737
 738		return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
 739	}
 740	case AMDGPU_INFO_TIMESTAMP:
 741		ui64 = amdgpu_gfx_get_gpu_clock_counter(adev);
 742		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 743	case AMDGPU_INFO_FW_VERSION: {
 744		struct drm_amdgpu_info_firmware fw_info;
 745
 746		/* We only support one instance of each IP block right now. */
 747		if (info->query_fw.ip_instance != 0)
 748			return -EINVAL;
 749
 750		ret = amdgpu_firmware_info(&fw_info, &info->query_fw, adev);
 751		if (ret)
 752			return ret;
 753
 754		return copy_to_user(out, &fw_info,
 755				    min((size_t)size, sizeof(fw_info))) ? -EFAULT : 0;
 756	}
 757	case AMDGPU_INFO_NUM_BYTES_MOVED:
 758		ui64 = atomic64_read(&adev->num_bytes_moved);
 759		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 760	case AMDGPU_INFO_NUM_EVICTIONS:
 761		ui64 = atomic64_read(&adev->num_evictions);
 762		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 763	case AMDGPU_INFO_NUM_VRAM_CPU_PAGE_FAULTS:
 764		ui64 = atomic64_read(&adev->num_vram_cpu_page_faults);
 765		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 766	case AMDGPU_INFO_VRAM_USAGE:
 767		ui64 = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ?
 768			ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) : 0;
 769		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 770	case AMDGPU_INFO_VIS_VRAM_USAGE:
 771		ui64 = amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);
 772		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 773	case AMDGPU_INFO_GTT_USAGE:
 774		ui64 = ttm_resource_manager_usage(&adev->mman.gtt_mgr.manager);
 775		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 776	case AMDGPU_INFO_GDS_CONFIG: {
 777		struct drm_amdgpu_info_gds gds_info;
 778
 779		memset(&gds_info, 0, sizeof(gds_info));
 780		gds_info.compute_partition_size = adev->gds.gds_size;
 781		gds_info.gds_total_size = adev->gds.gds_size;
 782		gds_info.gws_per_compute_partition = adev->gds.gws_size;
 783		gds_info.oa_per_compute_partition = adev->gds.oa_size;
 784		return copy_to_user(out, &gds_info,
 785				    min((size_t)size, sizeof(gds_info))) ? -EFAULT : 0;
 786	}
 787	case AMDGPU_INFO_VRAM_GTT: {
 788		struct drm_amdgpu_info_vram_gtt vram_gtt;
 789
 790		vram_gtt.vram_size = adev->gmc.real_vram_size -
 791			atomic64_read(&adev->vram_pin_size) -
 792			AMDGPU_VM_RESERVED_VRAM;
 793		vram_gtt.vram_cpu_accessible_size =
 794			min(adev->gmc.visible_vram_size -
 795			    atomic64_read(&adev->visible_pin_size),
 796			    vram_gtt.vram_size);
 797		vram_gtt.gtt_size = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT)->size;
 798		vram_gtt.gtt_size -= atomic64_read(&adev->gart_pin_size);
 799		return copy_to_user(out, &vram_gtt,
 800				    min((size_t)size, sizeof(vram_gtt))) ? -EFAULT : 0;
 801	}
 802	case AMDGPU_INFO_MEMORY: {
 803		struct drm_amdgpu_memory_info mem;
 804		struct ttm_resource_manager *gtt_man =
 805			&adev->mman.gtt_mgr.manager;
 806		struct ttm_resource_manager *vram_man =
 807			&adev->mman.vram_mgr.manager;
 808
 809		memset(&mem, 0, sizeof(mem));
 810		mem.vram.total_heap_size = adev->gmc.real_vram_size;
 811		mem.vram.usable_heap_size = adev->gmc.real_vram_size -
 812			atomic64_read(&adev->vram_pin_size) -
 813			AMDGPU_VM_RESERVED_VRAM;
 814		mem.vram.heap_usage = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ?
 815				ttm_resource_manager_usage(vram_man) : 0;
 816		mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4;
 817
 818		mem.cpu_accessible_vram.total_heap_size =
 819			adev->gmc.visible_vram_size;
 820		mem.cpu_accessible_vram.usable_heap_size =
 821			min(adev->gmc.visible_vram_size -
 822			    atomic64_read(&adev->visible_pin_size),
 823			    mem.vram.usable_heap_size);
 824		mem.cpu_accessible_vram.heap_usage =
 825			amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);
 826		mem.cpu_accessible_vram.max_allocation =
 827			mem.cpu_accessible_vram.usable_heap_size * 3 / 4;
 828
 829		mem.gtt.total_heap_size = gtt_man->size;
 830		mem.gtt.usable_heap_size = mem.gtt.total_heap_size -
 831			atomic64_read(&adev->gart_pin_size);
 832		mem.gtt.heap_usage = ttm_resource_manager_usage(gtt_man);
 833		mem.gtt.max_allocation = mem.gtt.usable_heap_size * 3 / 4;
 834
 835		return copy_to_user(out, &mem,
 836				    min((size_t)size, sizeof(mem)))
 837				    ? -EFAULT : 0;
 838	}
 839	case AMDGPU_INFO_READ_MMR_REG: {
 840		int ret = 0;
 841		unsigned int n, alloc_size;
 842		uint32_t *regs;
 843		unsigned int se_num = (info->read_mmr_reg.instance >>
 844				   AMDGPU_INFO_MMR_SE_INDEX_SHIFT) &
 845				  AMDGPU_INFO_MMR_SE_INDEX_MASK;
 846		unsigned int sh_num = (info->read_mmr_reg.instance >>
 847				   AMDGPU_INFO_MMR_SH_INDEX_SHIFT) &
 848				  AMDGPU_INFO_MMR_SH_INDEX_MASK;
 849
 850		if (!down_read_trylock(&adev->reset_domain->sem))
 851			return -ENOENT;
 852
 853		/* set full masks if the userspace set all bits
 854		 * in the bitfields
 855		 */
 856		if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) {
 857			se_num = 0xffffffff;
 858		} else if (se_num >= AMDGPU_GFX_MAX_SE) {
 859			ret = -EINVAL;
 860			goto out;
 861		}
 862
 863		if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) {
 864			sh_num = 0xffffffff;
 865		} else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE) {
 866			ret = -EINVAL;
 867			goto out;
 868		}
 869
 870		if (info->read_mmr_reg.count > 128) {
 871			ret = -EINVAL;
 872			goto out;
 873		}
 874
 875		regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL);
 876		if (!regs) {
 877			ret = -ENOMEM;
 878			goto out;
 879		}
 880
 881		alloc_size = info->read_mmr_reg.count * sizeof(*regs);
 882
 883		amdgpu_gfx_off_ctrl(adev, false);
 884		for (i = 0; i < info->read_mmr_reg.count; i++) {
 885			if (amdgpu_asic_read_register(adev, se_num, sh_num,
 886						      info->read_mmr_reg.dword_offset + i,
 887						      &regs[i])) {
 888				DRM_DEBUG_KMS("unallowed offset %#x\n",
 889					      info->read_mmr_reg.dword_offset + i);
 890				kfree(regs);
 891				amdgpu_gfx_off_ctrl(adev, true);
 892				ret = -EFAULT;
 893				goto out;
 894			}
 895		}
 896		amdgpu_gfx_off_ctrl(adev, true);
 897		n = copy_to_user(out, regs, min(size, alloc_size));
 898		kfree(regs);
 899		ret = (n ? -EFAULT : 0);
 900out:
 901		up_read(&adev->reset_domain->sem);
 902		return ret;
 903	}
 904	case AMDGPU_INFO_DEV_INFO: {
 905		struct drm_amdgpu_info_device *dev_info;
 906		uint64_t vm_size;
 907		uint32_t pcie_gen_mask, pcie_width_mask;
 908
 909		dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
 910		if (!dev_info)
 911			return -ENOMEM;
 912
 913		dev_info->device_id = adev->pdev->device;
 914		dev_info->chip_rev = adev->rev_id;
 915		dev_info->external_rev = adev->external_rev_id;
 916		dev_info->pci_rev = adev->pdev->revision;
 917		dev_info->family = adev->family;
 918		dev_info->num_shader_engines = adev->gfx.config.max_shader_engines;
 919		dev_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se;
 920		/* return all clocks in KHz */
 921		dev_info->gpu_counter_freq = amdgpu_asic_get_xclk(adev) * 10;
 922		if (adev->pm.dpm_enabled) {
 923			dev_info->max_engine_clock = amdgpu_dpm_get_sclk(adev, false) * 10;
 924			dev_info->max_memory_clock = amdgpu_dpm_get_mclk(adev, false) * 10;
 925			dev_info->min_engine_clock = amdgpu_dpm_get_sclk(adev, true) * 10;
 926			dev_info->min_memory_clock = amdgpu_dpm_get_mclk(adev, true) * 10;
 927		} else {
 928			dev_info->max_engine_clock =
 929				dev_info->min_engine_clock =
 930					adev->clock.default_sclk * 10;
 931			dev_info->max_memory_clock =
 932				dev_info->min_memory_clock =
 933					adev->clock.default_mclk * 10;
 934		}
 935		dev_info->enabled_rb_pipes_mask = adev->gfx.config.backend_enable_mask;
 936		dev_info->num_rb_pipes = adev->gfx.config.max_backends_per_se *
 937			adev->gfx.config.max_shader_engines;
 938		dev_info->num_hw_gfx_contexts = adev->gfx.config.max_hw_contexts;
 939		dev_info->ids_flags = 0;
 940		if (adev->flags & AMD_IS_APU)
 941			dev_info->ids_flags |= AMDGPU_IDS_FLAGS_FUSION;
 942		if (adev->gfx.mcbp)
 943			dev_info->ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
 944		if (amdgpu_is_tmz(adev))
 945			dev_info->ids_flags |= AMDGPU_IDS_FLAGS_TMZ;
 946		if (adev->gfx.config.ta_cntl2_truncate_coord_mode)
 947			dev_info->ids_flags |= AMDGPU_IDS_FLAGS_CONFORMANT_TRUNC_COORD;
 948
 949		/* Gang submit is not supported under SRIOV currently */
 950		if (!amdgpu_sriov_vf(adev))
 951			dev_info->ids_flags |= AMDGPU_IDS_FLAGS_GANG_SUBMIT;
 952
 953		if (amdgpu_passthrough(adev))
 954			dev_info->ids_flags |= (AMDGPU_IDS_FLAGS_MODE_PT <<
 955						AMDGPU_IDS_FLAGS_MODE_SHIFT) &
 956						AMDGPU_IDS_FLAGS_MODE_MASK;
 957		else if (amdgpu_sriov_vf(adev))
 958			dev_info->ids_flags |= (AMDGPU_IDS_FLAGS_MODE_VF <<
 959						AMDGPU_IDS_FLAGS_MODE_SHIFT) &
 960						AMDGPU_IDS_FLAGS_MODE_MASK;
 961
 962		vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
 963		vm_size -= AMDGPU_VA_RESERVED_TOP;
 964
 965		/* Older VCE FW versions are buggy and can handle only 40bits */
 966		if (adev->vce.fw_version &&
 967		    adev->vce.fw_version < AMDGPU_VCE_FW_53_45)
 968			vm_size = min(vm_size, 1ULL << 40);
 969
 970		dev_info->virtual_address_offset = AMDGPU_VA_RESERVED_BOTTOM;
 971		dev_info->virtual_address_max =
 972			min(vm_size, AMDGPU_GMC_HOLE_START);
 973
 974		if (vm_size > AMDGPU_GMC_HOLE_START) {
 975			dev_info->high_va_offset = AMDGPU_GMC_HOLE_END;
 976			dev_info->high_va_max = AMDGPU_GMC_HOLE_END | vm_size;
 977		}
 978		dev_info->virtual_address_alignment = max_t(u32, PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
 979		dev_info->pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE;
 980		dev_info->gart_page_size = max_t(u32, PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
 981		dev_info->cu_active_number = adev->gfx.cu_info.number;
 982		dev_info->cu_ao_mask = adev->gfx.cu_info.ao_cu_mask;
 983		dev_info->ce_ram_size = adev->gfx.ce_ram_size;
 984		memcpy(&dev_info->cu_ao_bitmap[0], &adev->gfx.cu_info.ao_cu_bitmap[0],
 985		       sizeof(adev->gfx.cu_info.ao_cu_bitmap));
 986		memcpy(&dev_info->cu_bitmap[0], &adev->gfx.cu_info.bitmap[0],
 987		       sizeof(dev_info->cu_bitmap));
 988		dev_info->vram_type = adev->gmc.vram_type;
 989		dev_info->vram_bit_width = adev->gmc.vram_width;
 990		dev_info->vce_harvest_config = adev->vce.harvest_config;
 991		dev_info->gc_double_offchip_lds_buf =
 992			adev->gfx.config.double_offchip_lds_buf;
 993		dev_info->wave_front_size = adev->gfx.cu_info.wave_front_size;
 994		dev_info->num_shader_visible_vgprs = adev->gfx.config.max_gprs;
 995		dev_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh;
 996		dev_info->num_tcc_blocks = adev->gfx.config.max_texture_channel_caches;
 997		dev_info->gs_vgt_table_depth = adev->gfx.config.gs_vgt_table_depth;
 998		dev_info->gs_prim_buffer_depth = adev->gfx.config.gs_prim_buffer_depth;
 999		dev_info->max_gs_waves_per_vgt = adev->gfx.config.max_gs_threads;
1000
1001		if (adev->family >= AMDGPU_FAMILY_NV)
1002			dev_info->pa_sc_tile_steering_override =
1003				adev->gfx.config.pa_sc_tile_steering_override;
1004
1005		dev_info->tcc_disabled_mask = adev->gfx.config.tcc_disabled_mask;
1006
1007		/* Combine the chip gen mask with the platform (CPU/mobo) mask. */
1008		pcie_gen_mask = adev->pm.pcie_gen_mask &
1009			(adev->pm.pcie_gen_mask >> CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT);
1010		pcie_width_mask = adev->pm.pcie_mlw_mask &
1011			(adev->pm.pcie_mlw_mask >> CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT);
1012		dev_info->pcie_gen = fls(pcie_gen_mask);
1013		dev_info->pcie_num_lanes =
1014			pcie_width_mask & CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 ? 32 :
1015			pcie_width_mask & CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 ? 16 :
1016			pcie_width_mask & CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 ? 12 :
1017			pcie_width_mask & CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 ? 8 :
1018			pcie_width_mask & CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 ? 4 :
1019			pcie_width_mask & CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 ? 2 : 1;
1020
1021		dev_info->tcp_cache_size = adev->gfx.config.gc_tcp_l1_size;
1022		dev_info->num_sqc_per_wgp = adev->gfx.config.gc_num_sqc_per_wgp;
1023		dev_info->sqc_data_cache_size = adev->gfx.config.gc_l1_data_cache_size_per_sqc;
1024		dev_info->sqc_inst_cache_size = adev->gfx.config.gc_l1_instruction_cache_size_per_sqc;
1025		dev_info->gl1c_cache_size = adev->gfx.config.gc_gl1c_size_per_instance *
1026					    adev->gfx.config.gc_gl1c_per_sa;
1027		dev_info->gl2c_cache_size = adev->gfx.config.gc_gl2c_per_gpu;
1028		dev_info->mall_size = adev->gmc.mall_size;
1029
1030
1031		if (adev->gfx.funcs->get_gfx_shadow_info) {
1032			struct amdgpu_gfx_shadow_info shadow_info;
1033
1034			ret = amdgpu_gfx_get_gfx_shadow_info(adev, &shadow_info);
1035			if (!ret) {
1036				dev_info->shadow_size = shadow_info.shadow_size;
1037				dev_info->shadow_alignment = shadow_info.shadow_alignment;
1038				dev_info->csa_size = shadow_info.csa_size;
1039				dev_info->csa_alignment = shadow_info.csa_alignment;
1040			}
1041		}
1042
1043		dev_info->userq_ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1044
1045		ret = copy_to_user(out, dev_info,
1046				   min((size_t)size, sizeof(*dev_info))) ? -EFAULT : 0;
1047		kfree(dev_info);
1048		return ret;
1049	}
1050	case AMDGPU_INFO_VCE_CLOCK_TABLE: {
1051		unsigned int i;
1052		struct drm_amdgpu_info_vce_clock_table vce_clk_table = {};
1053		struct amd_vce_state *vce_state;
1054
1055		for (i = 0; i < AMDGPU_VCE_CLOCK_TABLE_ENTRIES; i++) {
1056			vce_state = amdgpu_dpm_get_vce_clock_state(adev, i);
1057			if (vce_state) {
1058				vce_clk_table.entries[i].sclk = vce_state->sclk;
1059				vce_clk_table.entries[i].mclk = vce_state->mclk;
1060				vce_clk_table.entries[i].eclk = vce_state->evclk;
1061				vce_clk_table.num_valid_entries++;
1062			}
1063		}
1064
1065		return copy_to_user(out, &vce_clk_table,
1066				    min((size_t)size, sizeof(vce_clk_table))) ? -EFAULT : 0;
1067	}
1068	case AMDGPU_INFO_VBIOS: {
1069		uint32_t bios_size = adev->bios_size;
1070
1071		switch (info->vbios_info.type) {
1072		case AMDGPU_INFO_VBIOS_SIZE:
1073			return copy_to_user(out, &bios_size,
1074					min((size_t)size, sizeof(bios_size)))
1075					? -EFAULT : 0;
1076		case AMDGPU_INFO_VBIOS_IMAGE: {
1077			uint8_t *bios;
1078			uint32_t bios_offset = info->vbios_info.offset;
1079
1080			if (bios_offset >= bios_size)
1081				return -EINVAL;
1082
1083			bios = adev->bios + bios_offset;
1084			return copy_to_user(out, bios,
1085					    min((size_t)size, (size_t)(bios_size - bios_offset)))
1086					? -EFAULT : 0;
1087		}
1088		case AMDGPU_INFO_VBIOS_INFO: {
1089			struct drm_amdgpu_info_vbios vbios_info = {};
1090			struct atom_context *atom_context;
1091
1092			atom_context = adev->mode_info.atom_context;
1093			if (atom_context) {
1094				memcpy(vbios_info.name, atom_context->name,
1095				       sizeof(atom_context->name));
1096				memcpy(vbios_info.vbios_pn, atom_context->vbios_pn,
1097				       sizeof(atom_context->vbios_pn));
1098				vbios_info.version = atom_context->version;
1099				memcpy(vbios_info.vbios_ver_str, atom_context->vbios_ver_str,
1100				       sizeof(atom_context->vbios_ver_str));
1101				memcpy(vbios_info.date, atom_context->date,
1102				       sizeof(atom_context->date));
1103			}
1104
1105			return copy_to_user(out, &vbios_info,
1106						min((size_t)size, sizeof(vbios_info))) ? -EFAULT : 0;
1107		}
1108		default:
1109			DRM_DEBUG_KMS("Invalid request %d\n",
1110					info->vbios_info.type);
1111			return -EINVAL;
1112		}
1113	}
1114	case AMDGPU_INFO_NUM_HANDLES: {
1115		struct drm_amdgpu_info_num_handles handle;
1116
1117		switch (info->query_hw_ip.type) {
1118		case AMDGPU_HW_IP_UVD:
1119			/* Starting Polaris, we support unlimited UVD handles */
1120			if (adev->asic_type < CHIP_POLARIS10) {
1121				handle.uvd_max_handles = adev->uvd.max_handles;
1122				handle.uvd_used_handles = amdgpu_uvd_used_handles(adev);
1123
1124				return copy_to_user(out, &handle,
1125					min((size_t)size, sizeof(handle))) ? -EFAULT : 0;
1126			} else {
1127				return -ENODATA;
1128			}
1129
1130			break;
1131		default:
1132			return -EINVAL;
1133		}
1134	}
1135	case AMDGPU_INFO_SENSOR: {
1136		if (!adev->pm.dpm_enabled)
1137			return -ENOENT;
1138
1139		switch (info->sensor_info.type) {
1140		case AMDGPU_INFO_SENSOR_GFX_SCLK:
1141			/* get sclk in Mhz */
1142			if (amdgpu_dpm_read_sensor(adev,
1143						   AMDGPU_PP_SENSOR_GFX_SCLK,
1144						   (void *)&ui32, &ui32_size)) {
1145				return -EINVAL;
1146			}
1147			ui32 /= 100;
1148			break;
1149		case AMDGPU_INFO_SENSOR_GFX_MCLK:
1150			/* get mclk in Mhz */
1151			if (amdgpu_dpm_read_sensor(adev,
1152						   AMDGPU_PP_SENSOR_GFX_MCLK,
1153						   (void *)&ui32, &ui32_size)) {
1154				return -EINVAL;
1155			}
1156			ui32 /= 100;
1157			break;
1158		case AMDGPU_INFO_SENSOR_GPU_TEMP:
1159			/* get temperature in millidegrees C */
1160			if (amdgpu_dpm_read_sensor(adev,
1161						   AMDGPU_PP_SENSOR_GPU_TEMP,
1162						   (void *)&ui32, &ui32_size)) {
1163				return -EINVAL;
1164			}
1165			break;
1166		case AMDGPU_INFO_SENSOR_GPU_LOAD:
1167			/* get GPU load */
1168			if (amdgpu_dpm_read_sensor(adev,
1169						   AMDGPU_PP_SENSOR_GPU_LOAD,
1170						   (void *)&ui32, &ui32_size)) {
1171				return -EINVAL;
1172			}
1173			break;
1174		case AMDGPU_INFO_SENSOR_GPU_AVG_POWER:
1175			/* get average GPU power */
1176			if (amdgpu_dpm_read_sensor(adev,
1177						   AMDGPU_PP_SENSOR_GPU_AVG_POWER,
1178						   (void *)&ui32, &ui32_size)) {
1179				/* fall back to input power for backwards compat */
1180				if (amdgpu_dpm_read_sensor(adev,
1181							   AMDGPU_PP_SENSOR_GPU_INPUT_POWER,
1182							   (void *)&ui32, &ui32_size)) {
1183					return -EINVAL;
1184				}
1185			}
1186			ui32 >>= 8;
1187			break;
1188		case AMDGPU_INFO_SENSOR_GPU_INPUT_POWER:
1189			/* get input GPU power */
1190			if (amdgpu_dpm_read_sensor(adev,
1191						   AMDGPU_PP_SENSOR_GPU_INPUT_POWER,
1192						   (void *)&ui32, &ui32_size)) {
1193				return -EINVAL;
1194			}
1195			ui32 >>= 8;
1196			break;
1197		case AMDGPU_INFO_SENSOR_VDDNB:
1198			/* get VDDNB in millivolts */
1199			if (amdgpu_dpm_read_sensor(adev,
1200						   AMDGPU_PP_SENSOR_VDDNB,
1201						   (void *)&ui32, &ui32_size)) {
1202				return -EINVAL;
1203			}
1204			break;
1205		case AMDGPU_INFO_SENSOR_VDDGFX:
1206			/* get VDDGFX in millivolts */
1207			if (amdgpu_dpm_read_sensor(adev,
1208						   AMDGPU_PP_SENSOR_VDDGFX,
1209						   (void *)&ui32, &ui32_size)) {
1210				return -EINVAL;
1211			}
1212			break;
1213		case AMDGPU_INFO_SENSOR_STABLE_PSTATE_GFX_SCLK:
1214			/* get stable pstate sclk in Mhz */
1215			if (amdgpu_dpm_read_sensor(adev,
1216						   AMDGPU_PP_SENSOR_STABLE_PSTATE_SCLK,
1217						   (void *)&ui32, &ui32_size)) {
1218				return -EINVAL;
1219			}
1220			ui32 /= 100;
1221			break;
1222		case AMDGPU_INFO_SENSOR_STABLE_PSTATE_GFX_MCLK:
1223			/* get stable pstate mclk in Mhz */
1224			if (amdgpu_dpm_read_sensor(adev,
1225						   AMDGPU_PP_SENSOR_STABLE_PSTATE_MCLK,
1226						   (void *)&ui32, &ui32_size)) {
1227				return -EINVAL;
1228			}
1229			ui32 /= 100;
1230			break;
1231		case AMDGPU_INFO_SENSOR_PEAK_PSTATE_GFX_SCLK:
1232			/* get peak pstate sclk in Mhz */
1233			if (amdgpu_dpm_read_sensor(adev,
1234						   AMDGPU_PP_SENSOR_PEAK_PSTATE_SCLK,
1235						   (void *)&ui32, &ui32_size)) {
1236				return -EINVAL;
1237			}
1238			ui32 /= 100;
1239			break;
1240		case AMDGPU_INFO_SENSOR_PEAK_PSTATE_GFX_MCLK:
1241			/* get peak pstate mclk in Mhz */
1242			if (amdgpu_dpm_read_sensor(adev,
1243						   AMDGPU_PP_SENSOR_PEAK_PSTATE_MCLK,
1244						   (void *)&ui32, &ui32_size)) {
1245				return -EINVAL;
1246			}
1247			ui32 /= 100;
1248			break;
1249		default:
1250			DRM_DEBUG_KMS("Invalid request %d\n",
1251				      info->sensor_info.type);
1252			return -EINVAL;
1253		}
1254		return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
1255	}
1256	case AMDGPU_INFO_VRAM_LOST_COUNTER:
1257		ui32 = atomic_read(&adev->vram_lost_counter);
1258		return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
1259	case AMDGPU_INFO_RAS_ENABLED_FEATURES: {
1260		struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1261		uint64_t ras_mask;
1262
1263		if (!ras)
1264			return -EINVAL;
1265		ras_mask = (uint64_t)adev->ras_enabled << 32 | ras->features;
1266
1267		return copy_to_user(out, &ras_mask,
1268				min_t(u64, size, sizeof(ras_mask))) ?
1269			-EFAULT : 0;
1270	}
1271	case AMDGPU_INFO_VIDEO_CAPS: {
1272		const struct amdgpu_video_codecs *codecs;
1273		struct drm_amdgpu_info_video_caps *caps;
1274		int r;
1275
1276		if (!adev->asic_funcs->query_video_codecs)
1277			return -EINVAL;
1278
1279		switch (info->video_cap.type) {
1280		case AMDGPU_INFO_VIDEO_CAPS_DECODE:
1281			r = amdgpu_asic_query_video_codecs(adev, false, &codecs);
1282			if (r)
1283				return -EINVAL;
1284			break;
1285		case AMDGPU_INFO_VIDEO_CAPS_ENCODE:
1286			r = amdgpu_asic_query_video_codecs(adev, true, &codecs);
1287			if (r)
1288				return -EINVAL;
1289			break;
1290		default:
1291			DRM_DEBUG_KMS("Invalid request %d\n",
1292				      info->video_cap.type);
1293			return -EINVAL;
1294		}
1295
1296		caps = kzalloc(sizeof(*caps), GFP_KERNEL);
1297		if (!caps)
1298			return -ENOMEM;
1299
1300		for (i = 0; i < codecs->codec_count; i++) {
1301			int idx = codecs->codec_array[i].codec_type;
1302
1303			switch (idx) {
1304			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2:
1305			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4:
1306			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1:
1307			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC:
1308			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_HEVC:
1309			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_JPEG:
1310			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VP9:
1311			case AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1:
1312				caps->codec_info[idx].valid = 1;
1313				caps->codec_info[idx].max_width =
1314					codecs->codec_array[i].max_width;
1315				caps->codec_info[idx].max_height =
1316					codecs->codec_array[i].max_height;
1317				caps->codec_info[idx].max_pixels_per_frame =
1318					codecs->codec_array[i].max_pixels_per_frame;
1319				caps->codec_info[idx].max_level =
1320					codecs->codec_array[i].max_level;
1321				break;
1322			default:
1323				break;
1324			}
1325		}
1326		r = copy_to_user(out, caps,
1327				 min((size_t)size, sizeof(*caps))) ? -EFAULT : 0;
1328		kfree(caps);
1329		return r;
1330	}
1331	case AMDGPU_INFO_MAX_IBS: {
1332		uint32_t max_ibs[AMDGPU_HW_IP_NUM];
1333
1334		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
1335			max_ibs[i] = amdgpu_ring_max_ibs(i);
1336
1337		return copy_to_user(out, max_ibs,
1338				    min((size_t)size, sizeof(max_ibs))) ? -EFAULT : 0;
1339	}
1340	case AMDGPU_INFO_GPUVM_FAULT: {
1341		struct amdgpu_fpriv *fpriv = filp->driver_priv;
1342		struct amdgpu_vm *vm = &fpriv->vm;
1343		struct drm_amdgpu_info_gpuvm_fault gpuvm_fault;
1344		unsigned long flags;
1345
1346		if (!vm)
1347			return -EINVAL;
1348
1349		memset(&gpuvm_fault, 0, sizeof(gpuvm_fault));
1350
1351		xa_lock_irqsave(&adev->vm_manager.pasids, flags);
1352		gpuvm_fault.addr = vm->fault_info.addr;
1353		gpuvm_fault.status = vm->fault_info.status;
1354		gpuvm_fault.vmhub = vm->fault_info.vmhub;
1355		xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
1356
1357		return copy_to_user(out, &gpuvm_fault,
1358				    min((size_t)size, sizeof(gpuvm_fault))) ? -EFAULT : 0;
1359	}
1360	case AMDGPU_INFO_UQ_FW_AREAS: {
1361		struct drm_amdgpu_info_uq_metadata meta_info = {};
1362
1363		switch (info->query_hw_ip.type) {
1364		case AMDGPU_HW_IP_GFX:
1365			ret = amdgpu_userq_metadata_info_gfx(adev, info, &meta_info.gfx);
1366			if (ret)
1367				return ret;
1368
1369			ret = copy_to_user(out, &meta_info,
1370						min((size_t)size, sizeof(meta_info))) ? -EFAULT : 0;
1371			return 0;
1372		default:
1373			return -EINVAL;
1374		}
1375	}
1376	default:
1377		DRM_DEBUG_KMS("Invalid request %d\n", info->query);
1378		return -EINVAL;
1379	}
1380	return 0;
1381}
1382
1383/**
1384 * amdgpu_driver_open_kms - drm callback for open
1385 *
1386 * @dev: drm dev pointer
1387 * @file_priv: drm file
1388 *
1389 * On device open, init vm on cayman+ (all asics).
1390 * Returns 0 on success, error on failure.
1391 */
1392int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
1393{
1394	struct amdgpu_device *adev = drm_to_adev(dev);
1395	struct amdgpu_fpriv *fpriv;
1396	int r, pasid;
1397
1398	/* Ensure IB tests are run on ring */
1399	flush_delayed_work(&adev->delayed_init_work);
1400
1401
1402	if (amdgpu_ras_intr_triggered()) {
1403		DRM_ERROR("RAS Intr triggered, device disabled!!");
1404		return -EHWPOISON;
1405	}
1406
1407	file_priv->driver_priv = NULL;
1408
1409	r = pm_runtime_get_sync(dev->dev);
1410	if (r < 0)
1411		goto pm_put;
1412
1413	fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
1414	if (unlikely(!fpriv)) {
1415		r = -ENOMEM;
1416		goto out_suspend;
1417	}
1418
1419	pasid = amdgpu_pasid_alloc(16);
1420	if (pasid < 0) {
1421		dev_warn(adev->dev, "No more PASIDs available!");
1422		pasid = 0;
1423	}
1424
1425	r = amdgpu_xcp_open_device(adev, fpriv, file_priv);
1426	if (r)
1427		goto error_pasid;
1428
1429	amdgpu_debugfs_vm_init(file_priv);
1430
1431	r = amdgpu_vm_init(adev, &fpriv->vm, fpriv->xcp_id, pasid);
1432	if (r)
1433		goto error_pasid;
1434
1435	fpriv->prt_va = amdgpu_vm_bo_add(adev, &fpriv->vm, NULL);
1436	if (!fpriv->prt_va) {
1437		r = -ENOMEM;
1438		goto error_vm;
1439	}
1440
1441	if (adev->gfx.mcbp) {
1442		uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK;
1443
1444		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
1445						&fpriv->csa_va, csa_addr, AMDGPU_CSA_SIZE);
1446		if (r)
1447			goto error_vm;
1448	}
1449
1450	r = amdgpu_seq64_map(adev, &fpriv->vm, &fpriv->seq64_va);
1451	if (r)
1452		goto error_vm;
1453
1454	mutex_init(&fpriv->bo_list_lock);
1455	idr_init_base(&fpriv->bo_list_handles, 1);
1456
1457	r = amdgpu_userq_mgr_init(&fpriv->userq_mgr, file_priv, adev);
1458	if (r)
1459		DRM_WARN("Can't setup usermode queues, use legacy workload submission only\n");
1460
1461	r = amdgpu_eviction_fence_init(&fpriv->evf_mgr);
1462	if (r)
1463		goto error_vm;
1464
1465	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
1466
1467	file_priv->driver_priv = fpriv;
1468	goto out_suspend;
1469
1470error_vm:
1471	amdgpu_vm_fini(adev, &fpriv->vm);
1472
1473error_pasid:
1474	if (pasid)
1475		amdgpu_pasid_free(pasid);
1476
1477	kfree(fpriv);
1478
1479out_suspend:
1480pm_put:
1481	pm_runtime_put_autosuspend(dev->dev);
1482
1483	return r;
1484}
1485
1486/**
1487 * amdgpu_driver_postclose_kms - drm callback for post close
1488 *
1489 * @dev: drm dev pointer
1490 * @file_priv: drm file
1491 *
1492 * On device post close, tear down vm on cayman+ (all asics).
1493 */
1494void amdgpu_driver_postclose_kms(struct drm_device *dev,
1495				 struct drm_file *file_priv)
1496{
1497	struct amdgpu_device *adev = drm_to_adev(dev);
1498	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1499	struct amdgpu_bo_list *list;
1500	struct amdgpu_bo *pd;
1501	u32 pasid;
1502	int handle;
1503
1504	if (!fpriv)
1505		return;
1506
1507	pm_runtime_get_sync(dev->dev);
1508
1509	if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_UVD) != NULL)
1510		amdgpu_uvd_free_handles(adev, file_priv);
1511	if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_VCE) != NULL)
1512		amdgpu_vce_free_handles(adev, file_priv);
1513
1514	if (fpriv->csa_va) {
1515		uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK;
1516
1517		WARN_ON(amdgpu_unmap_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
1518						fpriv->csa_va, csa_addr));
1519		fpriv->csa_va = NULL;
1520	}
1521
1522	amdgpu_seq64_unmap(adev, fpriv);
1523
1524	pasid = fpriv->vm.pasid;
1525	pd = amdgpu_bo_ref(fpriv->vm.root.bo);
1526	if (!WARN_ON(amdgpu_bo_reserve(pd, true))) {
1527		amdgpu_vm_bo_del(adev, fpriv->prt_va);
1528		amdgpu_bo_unreserve(pd);
1529	}
1530
1531	amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
1532	amdgpu_vm_fini(adev, &fpriv->vm);
1533
1534	if (pasid)
1535		amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
1536	amdgpu_bo_unref(&pd);
1537
1538	idr_for_each_entry(&fpriv->bo_list_handles, list, handle)
1539		amdgpu_bo_list_put(list);
1540
1541	idr_destroy(&fpriv->bo_list_handles);
1542	mutex_destroy(&fpriv->bo_list_lock);
1543
1544	kfree(fpriv);
1545	file_priv->driver_priv = NULL;
1546
1547	pm_runtime_put_autosuspend(dev->dev);
1548}
1549
1550
1551void amdgpu_driver_release_kms(struct drm_device *dev)
1552{
1553	struct amdgpu_device *adev = drm_to_adev(dev);
1554
1555	amdgpu_device_fini_sw(adev);
1556	pci_set_drvdata(adev->pdev, NULL);
1557}
1558
1559/*
1560 * VBlank related functions.
1561 */
1562/**
1563 * amdgpu_get_vblank_counter_kms - get frame count
1564 *
1565 * @crtc: crtc to get the frame count from
1566 *
1567 * Gets the frame count on the requested crtc (all asics).
1568 * Returns frame count on success, -EINVAL on failure.
1569 */
1570u32 amdgpu_get_vblank_counter_kms(struct drm_crtc *crtc)
1571{
1572	struct drm_device *dev = crtc->dev;
1573	unsigned int pipe = crtc->index;
1574	struct amdgpu_device *adev = drm_to_adev(dev);
1575	int vpos, hpos, stat;
1576	u32 count;
1577
1578	if (pipe >= adev->mode_info.num_crtc) {
1579		DRM_ERROR("Invalid crtc %u\n", pipe);
1580		return -EINVAL;
1581	}
1582
1583	/* The hw increments its frame counter at start of vsync, not at start
1584	 * of vblank, as is required by DRM core vblank counter handling.
1585	 * Cook the hw count here to make it appear to the caller as if it
1586	 * incremented at start of vblank. We measure distance to start of
1587	 * vblank in vpos. vpos therefore will be >= 0 between start of vblank
1588	 * and start of vsync, so vpos >= 0 means to bump the hw frame counter
1589	 * result by 1 to give the proper appearance to caller.
1590	 */
1591	if (adev->mode_info.crtcs[pipe]) {
1592		/* Repeat readout if needed to provide stable result if
1593		 * we cross start of vsync during the queries.
1594		 */
1595		do {
1596			count = amdgpu_display_vblank_get_counter(adev, pipe);
1597			/* Ask amdgpu_display_get_crtc_scanoutpos to return
1598			 * vpos as distance to start of vblank, instead of
1599			 * regular vertical scanout pos.
1600			 */
1601			stat = amdgpu_display_get_crtc_scanoutpos(
1602				dev, pipe, GET_DISTANCE_TO_VBLANKSTART,
1603				&vpos, &hpos, NULL, NULL,
1604				&adev->mode_info.crtcs[pipe]->base.hwmode);
1605		} while (count != amdgpu_display_vblank_get_counter(adev, pipe));
1606
1607		if (((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) !=
1608		    (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE))) {
1609			DRM_DEBUG_VBL("Query failed! stat %d\n", stat);
1610		} else {
1611			DRM_DEBUG_VBL("crtc %d: dist from vblank start %d\n",
1612				      pipe, vpos);
1613
1614			/* Bump counter if we are at >= leading edge of vblank,
1615			 * but before vsync where vpos would turn negative and
1616			 * the hw counter really increments.
1617			 */
1618			if (vpos >= 0)
1619				count++;
1620		}
1621	} else {
1622		/* Fallback to use value as is. */
1623		count = amdgpu_display_vblank_get_counter(adev, pipe);
1624		DRM_DEBUG_VBL("NULL mode info! Returned count may be wrong.\n");
1625	}
1626
1627	return count;
1628}
1629
1630/**
1631 * amdgpu_enable_vblank_kms - enable vblank interrupt
1632 *
1633 * @crtc: crtc to enable vblank interrupt for
1634 *
1635 * Enable the interrupt on the requested crtc (all asics).
1636 * Returns 0 on success, -EINVAL on failure.
1637 */
1638int amdgpu_enable_vblank_kms(struct drm_crtc *crtc)
1639{
1640	struct drm_device *dev = crtc->dev;
1641	unsigned int pipe = crtc->index;
1642	struct amdgpu_device *adev = drm_to_adev(dev);
1643	int idx = amdgpu_display_crtc_idx_to_irq_type(adev, pipe);
1644
1645	return amdgpu_irq_get(adev, &adev->crtc_irq, idx);
1646}
1647
1648/**
1649 * amdgpu_disable_vblank_kms - disable vblank interrupt
1650 *
1651 * @crtc: crtc to disable vblank interrupt for
1652 *
1653 * Disable the interrupt on the requested crtc (all asics).
1654 */
1655void amdgpu_disable_vblank_kms(struct drm_crtc *crtc)
1656{
1657	struct drm_device *dev = crtc->dev;
1658	unsigned int pipe = crtc->index;
1659	struct amdgpu_device *adev = drm_to_adev(dev);
1660	int idx = amdgpu_display_crtc_idx_to_irq_type(adev, pipe);
1661
1662	amdgpu_irq_put(adev, &adev->crtc_irq, idx);
1663}
1664
1665/*
1666 * Debugfs info
1667 */
1668#if defined(CONFIG_DEBUG_FS)
1669
1670static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused)
1671{
1672	struct amdgpu_device *adev = m->private;
1673	struct drm_amdgpu_info_firmware fw_info;
1674	struct drm_amdgpu_query_fw query_fw;
1675	struct atom_context *ctx = adev->mode_info.atom_context;
1676	uint8_t smu_program, smu_major, smu_minor, smu_debug;
1677	int ret, i;
1678
1679	static const char *ta_fw_name[TA_FW_TYPE_MAX_INDEX] = {
1680#define TA_FW_NAME(type)[TA_FW_TYPE_PSP_##type] = #type
1681		TA_FW_NAME(XGMI),
1682		TA_FW_NAME(RAS),
1683		TA_FW_NAME(HDCP),
1684		TA_FW_NAME(DTM),
1685		TA_FW_NAME(RAP),
1686		TA_FW_NAME(SECUREDISPLAY),
1687#undef TA_FW_NAME
1688	};
1689
1690	/* VCE */
1691	query_fw.fw_type = AMDGPU_INFO_FW_VCE;
1692	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1693	if (ret)
1694		return ret;
1695	seq_printf(m, "VCE feature version: %u, firmware version: 0x%08x\n",
1696		   fw_info.feature, fw_info.ver);
1697
1698	/* UVD */
1699	query_fw.fw_type = AMDGPU_INFO_FW_UVD;
1700	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1701	if (ret)
1702		return ret;
1703	seq_printf(m, "UVD feature version: %u, firmware version: 0x%08x\n",
1704		   fw_info.feature, fw_info.ver);
1705
1706	/* GMC */
1707	query_fw.fw_type = AMDGPU_INFO_FW_GMC;
1708	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1709	if (ret)
1710		return ret;
1711	seq_printf(m, "MC feature version: %u, firmware version: 0x%08x\n",
1712		   fw_info.feature, fw_info.ver);
1713
1714	/* ME */
1715	query_fw.fw_type = AMDGPU_INFO_FW_GFX_ME;
1716	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1717	if (ret)
1718		return ret;
1719	seq_printf(m, "ME feature version: %u, firmware version: 0x%08x\n",
1720		   fw_info.feature, fw_info.ver);
1721
1722	/* PFP */
1723	query_fw.fw_type = AMDGPU_INFO_FW_GFX_PFP;
1724	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1725	if (ret)
1726		return ret;
1727	seq_printf(m, "PFP feature version: %u, firmware version: 0x%08x\n",
1728		   fw_info.feature, fw_info.ver);
1729
1730	/* CE */
1731	query_fw.fw_type = AMDGPU_INFO_FW_GFX_CE;
1732	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1733	if (ret)
1734		return ret;
1735	seq_printf(m, "CE feature version: %u, firmware version: 0x%08x\n",
1736		   fw_info.feature, fw_info.ver);
1737
1738	/* RLC */
1739	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC;
1740	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1741	if (ret)
1742		return ret;
1743	seq_printf(m, "RLC feature version: %u, firmware version: 0x%08x\n",
1744		   fw_info.feature, fw_info.ver);
1745
1746	/* RLC SAVE RESTORE LIST CNTL */
1747	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_CNTL;
1748	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1749	if (ret)
1750		return ret;
1751	seq_printf(m, "RLC SRLC feature version: %u, firmware version: 0x%08x\n",
1752		   fw_info.feature, fw_info.ver);
1753
1754	/* RLC SAVE RESTORE LIST GPM MEM */
1755	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_GPM_MEM;
1756	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1757	if (ret)
1758		return ret;
1759	seq_printf(m, "RLC SRLG feature version: %u, firmware version: 0x%08x\n",
1760		   fw_info.feature, fw_info.ver);
1761
1762	/* RLC SAVE RESTORE LIST SRM MEM */
1763	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM;
1764	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1765	if (ret)
1766		return ret;
1767	seq_printf(m, "RLC SRLS feature version: %u, firmware version: 0x%08x\n",
1768		   fw_info.feature, fw_info.ver);
1769
1770	/* RLCP */
1771	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLCP;
1772	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1773	if (ret)
1774		return ret;
1775	seq_printf(m, "RLCP feature version: %u, firmware version: 0x%08x\n",
1776		   fw_info.feature, fw_info.ver);
1777
1778	/* RLCV */
1779	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLCV;
1780	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1781	if (ret)
1782		return ret;
1783	seq_printf(m, "RLCV feature version: %u, firmware version: 0x%08x\n",
1784		   fw_info.feature, fw_info.ver);
1785
1786	/* MEC */
1787	query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC;
1788	query_fw.index = 0;
1789	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1790	if (ret)
1791		return ret;
1792	seq_printf(m, "MEC feature version: %u, firmware version: 0x%08x\n",
1793		   fw_info.feature, fw_info.ver);
1794
1795	/* MEC2 */
1796	if (adev->gfx.mec2_fw) {
1797		query_fw.index = 1;
1798		ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1799		if (ret)
1800			return ret;
1801		seq_printf(m, "MEC2 feature version: %u, firmware version: 0x%08x\n",
1802			   fw_info.feature, fw_info.ver);
1803	}
1804
1805	/* IMU */
1806	query_fw.fw_type = AMDGPU_INFO_FW_IMU;
1807	query_fw.index = 0;
1808	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1809	if (ret)
1810		return ret;
1811	seq_printf(m, "IMU feature version: %u, firmware version: 0x%08x\n",
1812		   fw_info.feature, fw_info.ver);
1813
1814	/* PSP SOS */
1815	query_fw.fw_type = AMDGPU_INFO_FW_SOS;
1816	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1817	if (ret)
1818		return ret;
1819	seq_printf(m, "SOS feature version: %u, firmware version: 0x%08x\n",
1820		   fw_info.feature, fw_info.ver);
1821
1822
1823	/* PSP ASD */
1824	query_fw.fw_type = AMDGPU_INFO_FW_ASD;
1825	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1826	if (ret)
1827		return ret;
1828	seq_printf(m, "ASD feature version: %u, firmware version: 0x%08x\n",
1829		   fw_info.feature, fw_info.ver);
1830
1831	query_fw.fw_type = AMDGPU_INFO_FW_TA;
1832	for (i = TA_FW_TYPE_PSP_XGMI; i < TA_FW_TYPE_MAX_INDEX; i++) {
1833		query_fw.index = i;
1834		ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1835		if (ret)
1836			continue;
1837
1838		seq_printf(m, "TA %s feature version: 0x%08x, firmware version: 0x%08x\n",
1839			   ta_fw_name[i], fw_info.feature, fw_info.ver);
1840	}
1841
1842	/* SMC */
1843	query_fw.fw_type = AMDGPU_INFO_FW_SMC;
1844	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1845	if (ret)
1846		return ret;
1847	smu_program = (fw_info.ver >> 24) & 0xff;
1848	smu_major = (fw_info.ver >> 16) & 0xff;
1849	smu_minor = (fw_info.ver >> 8) & 0xff;
1850	smu_debug = (fw_info.ver >> 0) & 0xff;
1851	seq_printf(m, "SMC feature version: %u, program: %d, firmware version: 0x%08x (%d.%d.%d)\n",
1852		   fw_info.feature, smu_program, fw_info.ver, smu_major, smu_minor, smu_debug);
1853
1854	/* SDMA */
1855	query_fw.fw_type = AMDGPU_INFO_FW_SDMA;
1856	for (i = 0; i < adev->sdma.num_instances; i++) {
1857		query_fw.index = i;
1858		ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1859		if (ret)
1860			return ret;
1861		seq_printf(m, "SDMA%d feature version: %u, firmware version: 0x%08x\n",
1862			   i, fw_info.feature, fw_info.ver);
1863	}
1864
1865	/* VCN */
1866	query_fw.fw_type = AMDGPU_INFO_FW_VCN;
1867	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1868	if (ret)
1869		return ret;
1870	seq_printf(m, "VCN feature version: %u, firmware version: 0x%08x\n",
1871		   fw_info.feature, fw_info.ver);
1872
1873	/* DMCU */
1874	query_fw.fw_type = AMDGPU_INFO_FW_DMCU;
1875	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1876	if (ret)
1877		return ret;
1878	seq_printf(m, "DMCU feature version: %u, firmware version: 0x%08x\n",
1879		   fw_info.feature, fw_info.ver);
1880
1881	/* DMCUB */
1882	query_fw.fw_type = AMDGPU_INFO_FW_DMCUB;
1883	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1884	if (ret)
1885		return ret;
1886	seq_printf(m, "DMCUB feature version: %u, firmware version: 0x%08x\n",
1887		   fw_info.feature, fw_info.ver);
1888
1889	/* TOC */
1890	query_fw.fw_type = AMDGPU_INFO_FW_TOC;
1891	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1892	if (ret)
1893		return ret;
1894	seq_printf(m, "TOC feature version: %u, firmware version: 0x%08x\n",
1895		   fw_info.feature, fw_info.ver);
1896
1897	/* CAP */
1898	if (adev->psp.cap_fw) {
1899		query_fw.fw_type = AMDGPU_INFO_FW_CAP;
1900		ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1901		if (ret)
1902			return ret;
1903		seq_printf(m, "CAP feature version: %u, firmware version: 0x%08x\n",
1904				fw_info.feature, fw_info.ver);
1905	}
1906
1907	/* MES_KIQ */
1908	query_fw.fw_type = AMDGPU_INFO_FW_MES_KIQ;
1909	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1910	if (ret)
1911		return ret;
1912	seq_printf(m, "MES_KIQ feature version: %u, firmware version: 0x%08x\n",
1913		   fw_info.feature, fw_info.ver);
1914
1915	/* MES */
1916	query_fw.fw_type = AMDGPU_INFO_FW_MES;
1917	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1918	if (ret)
1919		return ret;
1920	seq_printf(m, "MES feature version: %u, firmware version: 0x%08x\n",
1921		   fw_info.feature, fw_info.ver);
1922
1923	/* VPE */
1924	query_fw.fw_type = AMDGPU_INFO_FW_VPE;
1925	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
1926	if (ret)
1927		return ret;
1928	seq_printf(m, "VPE feature version: %u, firmware version: 0x%08x\n",
1929		   fw_info.feature, fw_info.ver);
1930
1931	seq_printf(m, "VBIOS version: %s\n", ctx->vbios_pn);
1932
1933	return 0;
1934}
1935
1936DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_firmware_info);
1937
1938#endif
1939
1940void amdgpu_debugfs_firmware_init(struct amdgpu_device *adev)
1941{
1942#if defined(CONFIG_DEBUG_FS)
1943	struct drm_minor *minor = adev_to_drm(adev)->primary;
1944	struct dentry *root = minor->debugfs_root;
1945
1946	debugfs_create_file("amdgpu_firmware_info", 0444, root,
1947			    adev, &amdgpu_debugfs_firmware_info_fops);
1948
1949#endif
1950}