si.c

   1/*
   2 * Copyright 2015 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include <drm/amdgpu_drm.h>
  30
  31#include "amdgpu.h"
  32#include "amdgpu_atombios.h"
  33#include "amdgpu_ih.h"
  34#include "amdgpu_uvd.h"
  35#include "amdgpu_vce.h"
  36#include "atom.h"
  37#include "amd_pcie.h"
  38
  39#include "si_dpm.h"
  40#include "sid.h"
  41#include "si_ih.h"
  42#include "gfx_v6_0.h"
  43#include "gmc_v6_0.h"
  44#include "si_dma.h"
  45#include "dce_v6_0.h"
  46#include "si.h"
  47#include "uvd_v3_1.h"
  48#include "vce_v1_0.h"
  49
  50#include "uvd/uvd_4_0_d.h"
  51
  52#include "smu/smu_6_0_d.h"
  53#include "smu/smu_6_0_sh_mask.h"
  54
  55#include "gca/gfx_6_0_d.h"
  56#include "gca/gfx_6_0_sh_mask.h"
  57
  58#include "oss/oss_1_0_d.h"
  59#include "oss/oss_1_0_sh_mask.h"
  60
  61#include "gmc/gmc_6_0_d.h"
  62#include"gmc/gmc_6_0_sh_mask.h"
  63
  64#include "dce/dce_6_0_d.h"
  65#include "dce/dce_6_0_sh_mask.h"
  66
  67#include "bif/bif_3_0_d.h"
  68#include "bif/bif_3_0_sh_mask.h"
  69#include "si_enums.h"
  70
  71#include "amdgpu_dm.h"
  72#include "amdgpu_vkms.h"
  73
  74static const u32 tahiti_golden_registers[] =
  75{
  76	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
  77	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
  78	mmDB_DEBUG, 0xffffffff, 0x00000000,
  79	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
  80	mmDB_DEBUG3, 0x0002021c, 0x00020200,
  81	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
  82	0x340c, 0x000000c0, 0x00800040,
  83	0x360c, 0x000000c0, 0x00800040,
  84	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
  85	mmFBC_MISC, 0x00200000, 0x50100000,
  86	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
  87	mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
  88	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
  89	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
  90	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
  91	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
  92	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
  93	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
  94	0x000c, 0xffffffff, 0x0040,
  95	0x000d, 0x00000040, 0x00004040,
  96	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
  97	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
  98	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
  99	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 100	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 101	mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
 102	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
 103	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
 104	mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
 105	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 106	mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
 107	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 108	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 109	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 110	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 111	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 112};
 113
 114static const u32 tahiti_golden_registers2[] =
 115{
 116	mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
 117};
 118
 119static const u32 tahiti_golden_rlc_registers[] =
 120{
 121	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 122	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 123	0x311f, 0xffffffff, 0x10104040,
 124	0x3122, 0xffffffff, 0x0100000a,
 125	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 126	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 127	mmUVD_CGC_GATE, 0x00000008, 0x00000000,
 128};
 129
 130static const u32 pitcairn_golden_registers[] =
 131{
 132	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 133	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 134	mmDB_DEBUG, 0xffffffff, 0x00000000,
 135	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 136	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 137	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 138	0x340c, 0x000300c0, 0x00800040,
 139	0x360c, 0x000300c0, 0x00800040,
 140	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 141	mmFBC_MISC, 0x00200000, 0x50100000,
 142	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 143	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 144	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 145	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 146	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 147	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 148	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 149	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
 150	0x000c, 0xffffffff, 0x0040,
 151	0x000d, 0x00000040, 0x00004040,
 152	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 153	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 154	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 155	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
 156	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 157	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
 158	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 159	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 160	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 161	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 162	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 163	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 164};
 165
 166static const u32 pitcairn_golden_rlc_registers[] =
 167{
 168	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
 169	mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
 170	0x311f, 0xffffffff, 0x10102020,
 171	0x3122, 0xffffffff, 0x01000020,
 172	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 173	mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
 174};
 175
 176static const u32 verde_pg_init[] =
 177{
 178	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
 179	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
 180	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 181	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 182	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 183	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 184	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 185	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
 186	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
 187	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 188	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 189	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 190	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 191	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 192	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
 193	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
 194	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 195	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 196	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 197	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 198	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 199	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
 200	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
 201	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 202	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 203	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 204	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 205	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 206	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
 207	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
 208	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 209	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 210	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 211	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 212	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 213	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
 214	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
 215	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 216	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 217	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 218	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 219	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 220	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
 221	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
 222	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
 223	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
 224	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 225	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
 226	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
 227	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
 228	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 229	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
 230	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
 231	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
 232	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
 233	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
 234	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
 235	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
 236	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
 237	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
 238	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
 239	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
 240	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
 241	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
 242	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
 243	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
 244	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
 245	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
 246	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
 247	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
 248	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
 249	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
 250	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
 251	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
 252	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
 253	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
 254	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
 255	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
 256	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
 257	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
 258	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
 259	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
 260	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
 261	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
 262	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
 263	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
 264	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
 265	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
 266	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
 267	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
 268	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
 269	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
 270	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
 271	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
 272	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
 273	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
 274	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
 275	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
 276	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
 277	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
 278	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
 279	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
 280	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
 281	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
 282	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
 283	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
 284	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
 285	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
 286	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
 287	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
 288	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
 289	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
 290	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
 291	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
 292	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
 293	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
 294	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
 295	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
 296	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
 297	mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
 298	mmGMCON_MISC2, 0xfc00, 0x2000,
 299	mmGMCON_MISC3, 0xffffffff, 0xfc0,
 300	mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
 301};
 302
 303static const u32 verde_golden_rlc_registers[] =
 304{
 305	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 306	mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
 307	0x311f, 0xffffffff, 0x10808020,
 308	0x3122, 0xffffffff, 0x00800008,
 309	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
 310	mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
 311};
 312
 313static const u32 verde_golden_registers[] =
 314{
 315	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 316	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 317	mmDB_DEBUG, 0xffffffff, 0x00000000,
 318	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 319	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 320	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 321	0x340c, 0x000300c0, 0x00800040,
 322	0x360c, 0x000300c0, 0x00800040,
 323	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 324	mmFBC_MISC, 0x00200000, 0x50100000,
 325	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 326	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 327	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 328	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 329	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 330	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 331	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 332	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
 333	0x000c, 0xffffffff, 0x0040,
 334	0x000d, 0x00000040, 0x00004040,
 335	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 336	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
 337	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
 338	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 339	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 340	mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
 341	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 342	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
 343	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 344	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 345	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 346	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 347	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 348	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 349};
 350
 351static const u32 oland_golden_registers[] =
 352{
 353	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
 354	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 355	mmDB_DEBUG, 0xffffffff, 0x00000000,
 356	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 357	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 358	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
 359	0x340c, 0x000300c0, 0x00800040,
 360	0x360c, 0x000300c0, 0x00800040,
 361	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
 362	mmFBC_MISC, 0x00200000, 0x50100000,
 363	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
 364	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 365	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 366	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 367	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 368	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 369	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 370	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
 371	0x000c, 0xffffffff, 0x0040,
 372	0x000d, 0x00000040, 0x00004040,
 373	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
 374	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 375	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 376	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
 377	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 378	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 379	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 380	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 381	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 382	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 383	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 384	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 385
 386};
 387
 388static const u32 oland_golden_rlc_registers[] =
 389{
 390	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
 391	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
 392	0x311f, 0xffffffff, 0x10104040,
 393	0x3122, 0xffffffff, 0x0100000a,
 394	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
 395	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
 396};
 397
 398static const u32 hainan_golden_registers[] =
 399{
 400	0x17bc, 0x00000030, 0x00000011,
 401	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
 402	mmDB_DEBUG, 0xffffffff, 0x00000000,
 403	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 404	mmDB_DEBUG3, 0x0002021c, 0x00020200,
 405	0x031e, 0x00000080, 0x00000000,
 406	0x3430, 0xff000fff, 0x00000100,
 407	0x340c, 0x000300c0, 0x00800040,
 408	0x3630, 0xff000fff, 0x00000100,
 409	0x360c, 0x000300c0, 0x00800040,
 410	0x16ec, 0x000000f0, 0x00000070,
 411	0x16f0, 0x00200000, 0x50100000,
 412	0x1c0c, 0x31000311, 0x00000011,
 413	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
 414	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
 415	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
 416	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
 417	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 418	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
 419	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
 420	0x000c, 0xffffffff, 0x0040,
 421	0x000d, 0x00000040, 0x00004040,
 422	mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
 423	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
 424	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
 425	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
 426	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
 427	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
 428	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
 429	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
 430	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
 431	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 432	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 433	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
 434};
 435
 436static const u32 hainan_golden_registers2[] =
 437{
 438	mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
 439};
 440
 441static const u32 tahiti_mgcg_cgcg_init[] =
 442{
 443	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 444	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 445	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 446	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 447	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 448	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 449	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 450	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 451	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 452	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 453	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 454	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 455	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 456	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 457	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 458	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 459	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 460	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 461	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 462	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 463	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 464	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 465	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 466	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 467	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 468	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 469	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 470	0x2458, 0xffffffff, 0x00010000,
 471	0x2459, 0xffffffff, 0x00030002,
 472	0x245a, 0xffffffff, 0x00040007,
 473	0x245b, 0xffffffff, 0x00060005,
 474	0x245c, 0xffffffff, 0x00090008,
 475	0x245d, 0xffffffff, 0x00020001,
 476	0x245e, 0xffffffff, 0x00040003,
 477	0x245f, 0xffffffff, 0x00000007,
 478	0x2460, 0xffffffff, 0x00060005,
 479	0x2461, 0xffffffff, 0x00090008,
 480	0x2462, 0xffffffff, 0x00030002,
 481	0x2463, 0xffffffff, 0x00050004,
 482	0x2464, 0xffffffff, 0x00000008,
 483	0x2465, 0xffffffff, 0x00070006,
 484	0x2466, 0xffffffff, 0x000a0009,
 485	0x2467, 0xffffffff, 0x00040003,
 486	0x2468, 0xffffffff, 0x00060005,
 487	0x2469, 0xffffffff, 0x00000009,
 488	0x246a, 0xffffffff, 0x00080007,
 489	0x246b, 0xffffffff, 0x000b000a,
 490	0x246c, 0xffffffff, 0x00050004,
 491	0x246d, 0xffffffff, 0x00070006,
 492	0x246e, 0xffffffff, 0x0008000b,
 493	0x246f, 0xffffffff, 0x000a0009,
 494	0x2470, 0xffffffff, 0x000d000c,
 495	0x2471, 0xffffffff, 0x00060005,
 496	0x2472, 0xffffffff, 0x00080007,
 497	0x2473, 0xffffffff, 0x0000000b,
 498	0x2474, 0xffffffff, 0x000a0009,
 499	0x2475, 0xffffffff, 0x000d000c,
 500	0x2476, 0xffffffff, 0x00070006,
 501	0x2477, 0xffffffff, 0x00090008,
 502	0x2478, 0xffffffff, 0x0000000c,
 503	0x2479, 0xffffffff, 0x000b000a,
 504	0x247a, 0xffffffff, 0x000e000d,
 505	0x247b, 0xffffffff, 0x00080007,
 506	0x247c, 0xffffffff, 0x000a0009,
 507	0x247d, 0xffffffff, 0x0000000d,
 508	0x247e, 0xffffffff, 0x000c000b,
 509	0x247f, 0xffffffff, 0x000f000e,
 510	0x2480, 0xffffffff, 0x00090008,
 511	0x2481, 0xffffffff, 0x000b000a,
 512	0x2482, 0xffffffff, 0x000c000f,
 513	0x2483, 0xffffffff, 0x000e000d,
 514	0x2484, 0xffffffff, 0x00110010,
 515	0x2485, 0xffffffff, 0x000a0009,
 516	0x2486, 0xffffffff, 0x000c000b,
 517	0x2487, 0xffffffff, 0x0000000f,
 518	0x2488, 0xffffffff, 0x000e000d,
 519	0x2489, 0xffffffff, 0x00110010,
 520	0x248a, 0xffffffff, 0x000b000a,
 521	0x248b, 0xffffffff, 0x000d000c,
 522	0x248c, 0xffffffff, 0x00000010,
 523	0x248d, 0xffffffff, 0x000f000e,
 524	0x248e, 0xffffffff, 0x00120011,
 525	0x248f, 0xffffffff, 0x000c000b,
 526	0x2490, 0xffffffff, 0x000e000d,
 527	0x2491, 0xffffffff, 0x00000011,
 528	0x2492, 0xffffffff, 0x0010000f,
 529	0x2493, 0xffffffff, 0x00130012,
 530	0x2494, 0xffffffff, 0x000d000c,
 531	0x2495, 0xffffffff, 0x000f000e,
 532	0x2496, 0xffffffff, 0x00100013,
 533	0x2497, 0xffffffff, 0x00120011,
 534	0x2498, 0xffffffff, 0x00150014,
 535	0x2499, 0xffffffff, 0x000e000d,
 536	0x249a, 0xffffffff, 0x0010000f,
 537	0x249b, 0xffffffff, 0x00000013,
 538	0x249c, 0xffffffff, 0x00120011,
 539	0x249d, 0xffffffff, 0x00150014,
 540	0x249e, 0xffffffff, 0x000f000e,
 541	0x249f, 0xffffffff, 0x00110010,
 542	0x24a0, 0xffffffff, 0x00000014,
 543	0x24a1, 0xffffffff, 0x00130012,
 544	0x24a2, 0xffffffff, 0x00160015,
 545	0x24a3, 0xffffffff, 0x0010000f,
 546	0x24a4, 0xffffffff, 0x00120011,
 547	0x24a5, 0xffffffff, 0x00000015,
 548	0x24a6, 0xffffffff, 0x00140013,
 549	0x24a7, 0xffffffff, 0x00170016,
 550	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 551	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 552	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 553	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 554	0x000c, 0xffffffff, 0x0000001c,
 555	0x000d, 0x000f0000, 0x000f0000,
 556	0x0583, 0xffffffff, 0x00000100,
 557	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 558	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 559	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 560	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 561	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 562	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 563	0x157a, 0x00000001, 0x00000001,
 564	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 565	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 566	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 567	0x3430, 0xfffffff0, 0x00000100,
 568	0x3630, 0xfffffff0, 0x00000100,
 569};
 570static const u32 pitcairn_mgcg_cgcg_init[] =
 571{
 572	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 573	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 574	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 575	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 576	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 577	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 578	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 579	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 580	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 581	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 582	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 583	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 584	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 585	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 586	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 587	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 588	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 589	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 590	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 591	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 592	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 593	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 594	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 595	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 596	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 597	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 598	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 599	0x2458, 0xffffffff, 0x00010000,
 600	0x2459, 0xffffffff, 0x00030002,
 601	0x245a, 0xffffffff, 0x00040007,
 602	0x245b, 0xffffffff, 0x00060005,
 603	0x245c, 0xffffffff, 0x00090008,
 604	0x245d, 0xffffffff, 0x00020001,
 605	0x245e, 0xffffffff, 0x00040003,
 606	0x245f, 0xffffffff, 0x00000007,
 607	0x2460, 0xffffffff, 0x00060005,
 608	0x2461, 0xffffffff, 0x00090008,
 609	0x2462, 0xffffffff, 0x00030002,
 610	0x2463, 0xffffffff, 0x00050004,
 611	0x2464, 0xffffffff, 0x00000008,
 612	0x2465, 0xffffffff, 0x00070006,
 613	0x2466, 0xffffffff, 0x000a0009,
 614	0x2467, 0xffffffff, 0x00040003,
 615	0x2468, 0xffffffff, 0x00060005,
 616	0x2469, 0xffffffff, 0x00000009,
 617	0x246a, 0xffffffff, 0x00080007,
 618	0x246b, 0xffffffff, 0x000b000a,
 619	0x246c, 0xffffffff, 0x00050004,
 620	0x246d, 0xffffffff, 0x00070006,
 621	0x246e, 0xffffffff, 0x0008000b,
 622	0x246f, 0xffffffff, 0x000a0009,
 623	0x2470, 0xffffffff, 0x000d000c,
 624	0x2480, 0xffffffff, 0x00090008,
 625	0x2481, 0xffffffff, 0x000b000a,
 626	0x2482, 0xffffffff, 0x000c000f,
 627	0x2483, 0xffffffff, 0x000e000d,
 628	0x2484, 0xffffffff, 0x00110010,
 629	0x2485, 0xffffffff, 0x000a0009,
 630	0x2486, 0xffffffff, 0x000c000b,
 631	0x2487, 0xffffffff, 0x0000000f,
 632	0x2488, 0xffffffff, 0x000e000d,
 633	0x2489, 0xffffffff, 0x00110010,
 634	0x248a, 0xffffffff, 0x000b000a,
 635	0x248b, 0xffffffff, 0x000d000c,
 636	0x248c, 0xffffffff, 0x00000010,
 637	0x248d, 0xffffffff, 0x000f000e,
 638	0x248e, 0xffffffff, 0x00120011,
 639	0x248f, 0xffffffff, 0x000c000b,
 640	0x2490, 0xffffffff, 0x000e000d,
 641	0x2491, 0xffffffff, 0x00000011,
 642	0x2492, 0xffffffff, 0x0010000f,
 643	0x2493, 0xffffffff, 0x00130012,
 644	0x2494, 0xffffffff, 0x000d000c,
 645	0x2495, 0xffffffff, 0x000f000e,
 646	0x2496, 0xffffffff, 0x00100013,
 647	0x2497, 0xffffffff, 0x00120011,
 648	0x2498, 0xffffffff, 0x00150014,
 649	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 650	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 651	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 652	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 653	0x000c, 0xffffffff, 0x0000001c,
 654	0x000d, 0x000f0000, 0x000f0000,
 655	0x0583, 0xffffffff, 0x00000100,
 656	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 657	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 658	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 659	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 660	0x157a, 0x00000001, 0x00000001,
 661	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 662	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 663	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 664	0x3430, 0xfffffff0, 0x00000100,
 665	0x3630, 0xfffffff0, 0x00000100,
 666};
 667
 668static const u32 verde_mgcg_cgcg_init[] =
 669{
 670	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 671	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 672	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 673	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 674	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 675	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 676	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 677	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 678	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 679	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 680	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 681	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 682	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 683	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 684	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 685	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 686	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 687	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 688	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 689	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 690	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 691	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 692	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 693	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 694	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 695	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 696	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 697	0x2458, 0xffffffff, 0x00010000,
 698	0x2459, 0xffffffff, 0x00030002,
 699	0x245a, 0xffffffff, 0x00040007,
 700	0x245b, 0xffffffff, 0x00060005,
 701	0x245c, 0xffffffff, 0x00090008,
 702	0x245d, 0xffffffff, 0x00020001,
 703	0x245e, 0xffffffff, 0x00040003,
 704	0x245f, 0xffffffff, 0x00000007,
 705	0x2460, 0xffffffff, 0x00060005,
 706	0x2461, 0xffffffff, 0x00090008,
 707	0x2462, 0xffffffff, 0x00030002,
 708	0x2463, 0xffffffff, 0x00050004,
 709	0x2464, 0xffffffff, 0x00000008,
 710	0x2465, 0xffffffff, 0x00070006,
 711	0x2466, 0xffffffff, 0x000a0009,
 712	0x2467, 0xffffffff, 0x00040003,
 713	0x2468, 0xffffffff, 0x00060005,
 714	0x2469, 0xffffffff, 0x00000009,
 715	0x246a, 0xffffffff, 0x00080007,
 716	0x246b, 0xffffffff, 0x000b000a,
 717	0x246c, 0xffffffff, 0x00050004,
 718	0x246d, 0xffffffff, 0x00070006,
 719	0x246e, 0xffffffff, 0x0008000b,
 720	0x246f, 0xffffffff, 0x000a0009,
 721	0x2470, 0xffffffff, 0x000d000c,
 722	0x2480, 0xffffffff, 0x00090008,
 723	0x2481, 0xffffffff, 0x000b000a,
 724	0x2482, 0xffffffff, 0x000c000f,
 725	0x2483, 0xffffffff, 0x000e000d,
 726	0x2484, 0xffffffff, 0x00110010,
 727	0x2485, 0xffffffff, 0x000a0009,
 728	0x2486, 0xffffffff, 0x000c000b,
 729	0x2487, 0xffffffff, 0x0000000f,
 730	0x2488, 0xffffffff, 0x000e000d,
 731	0x2489, 0xffffffff, 0x00110010,
 732	0x248a, 0xffffffff, 0x000b000a,
 733	0x248b, 0xffffffff, 0x000d000c,
 734	0x248c, 0xffffffff, 0x00000010,
 735	0x248d, 0xffffffff, 0x000f000e,
 736	0x248e, 0xffffffff, 0x00120011,
 737	0x248f, 0xffffffff, 0x000c000b,
 738	0x2490, 0xffffffff, 0x000e000d,
 739	0x2491, 0xffffffff, 0x00000011,
 740	0x2492, 0xffffffff, 0x0010000f,
 741	0x2493, 0xffffffff, 0x00130012,
 742	0x2494, 0xffffffff, 0x000d000c,
 743	0x2495, 0xffffffff, 0x000f000e,
 744	0x2496, 0xffffffff, 0x00100013,
 745	0x2497, 0xffffffff, 0x00120011,
 746	0x2498, 0xffffffff, 0x00150014,
 747	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 748	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 749	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 750	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 751	0x000c, 0xffffffff, 0x0000001c,
 752	0x000d, 0x000f0000, 0x000f0000,
 753	0x0583, 0xffffffff, 0x00000100,
 754	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 755	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 756	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 757	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 758	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 759	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 760	0x157a, 0x00000001, 0x00000001,
 761	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 762	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 763	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 764	0x3430, 0xfffffff0, 0x00000100,
 765	0x3630, 0xfffffff0, 0x00000100,
 766};
 767
 768static const u32 oland_mgcg_cgcg_init[] =
 769{
 770	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 771	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 772	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 773	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 774	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 775	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 776	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 777	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 778	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 779	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 780	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 781	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 782	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 783	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 784	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 785	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 786	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 787	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 788	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 789	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 790	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 791	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 792	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 793	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 794	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 795	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 796	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 797	0x2458, 0xffffffff, 0x00010000,
 798	0x2459, 0xffffffff, 0x00030002,
 799	0x245a, 0xffffffff, 0x00040007,
 800	0x245b, 0xffffffff, 0x00060005,
 801	0x245c, 0xffffffff, 0x00090008,
 802	0x245d, 0xffffffff, 0x00020001,
 803	0x245e, 0xffffffff, 0x00040003,
 804	0x245f, 0xffffffff, 0x00000007,
 805	0x2460, 0xffffffff, 0x00060005,
 806	0x2461, 0xffffffff, 0x00090008,
 807	0x2462, 0xffffffff, 0x00030002,
 808	0x2463, 0xffffffff, 0x00050004,
 809	0x2464, 0xffffffff, 0x00000008,
 810	0x2465, 0xffffffff, 0x00070006,
 811	0x2466, 0xffffffff, 0x000a0009,
 812	0x2467, 0xffffffff, 0x00040003,
 813	0x2468, 0xffffffff, 0x00060005,
 814	0x2469, 0xffffffff, 0x00000009,
 815	0x246a, 0xffffffff, 0x00080007,
 816	0x246b, 0xffffffff, 0x000b000a,
 817	0x246c, 0xffffffff, 0x00050004,
 818	0x246d, 0xffffffff, 0x00070006,
 819	0x246e, 0xffffffff, 0x0008000b,
 820	0x246f, 0xffffffff, 0x000a0009,
 821	0x2470, 0xffffffff, 0x000d000c,
 822	0x2471, 0xffffffff, 0x00060005,
 823	0x2472, 0xffffffff, 0x00080007,
 824	0x2473, 0xffffffff, 0x0000000b,
 825	0x2474, 0xffffffff, 0x000a0009,
 826	0x2475, 0xffffffff, 0x000d000c,
 827	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 828	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 829	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 830	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 831	0x000c, 0xffffffff, 0x0000001c,
 832	0x000d, 0x000f0000, 0x000f0000,
 833	0x0583, 0xffffffff, 0x00000100,
 834	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
 835	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
 836	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 837	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 838	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 839	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
 840	0x157a, 0x00000001, 0x00000001,
 841	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 842	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 843	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 844	0x3430, 0xfffffff0, 0x00000100,
 845	0x3630, 0xfffffff0, 0x00000100,
 846};
 847
 848static const u32 hainan_mgcg_cgcg_init[] =
 849{
 850	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
 851	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 852	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 853	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
 854	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
 855	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
 856	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
 857	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
 858	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
 859	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
 860	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
 861	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
 862	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
 863	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
 864	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
 865	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
 866	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
 867	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
 868	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
 869	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
 870	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
 871	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
 872	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
 873	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 874	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
 875	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
 876	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 877	0x2458, 0xffffffff, 0x00010000,
 878	0x2459, 0xffffffff, 0x00030002,
 879	0x245a, 0xffffffff, 0x00040007,
 880	0x245b, 0xffffffff, 0x00060005,
 881	0x245c, 0xffffffff, 0x00090008,
 882	0x245d, 0xffffffff, 0x00020001,
 883	0x245e, 0xffffffff, 0x00040003,
 884	0x245f, 0xffffffff, 0x00000007,
 885	0x2460, 0xffffffff, 0x00060005,
 886	0x2461, 0xffffffff, 0x00090008,
 887	0x2462, 0xffffffff, 0x00030002,
 888	0x2463, 0xffffffff, 0x00050004,
 889	0x2464, 0xffffffff, 0x00000008,
 890	0x2465, 0xffffffff, 0x00070006,
 891	0x2466, 0xffffffff, 0x000a0009,
 892	0x2467, 0xffffffff, 0x00040003,
 893	0x2468, 0xffffffff, 0x00060005,
 894	0x2469, 0xffffffff, 0x00000009,
 895	0x246a, 0xffffffff, 0x00080007,
 896	0x246b, 0xffffffff, 0x000b000a,
 897	0x246c, 0xffffffff, 0x00050004,
 898	0x246d, 0xffffffff, 0x00070006,
 899	0x246e, 0xffffffff, 0x0008000b,
 900	0x246f, 0xffffffff, 0x000a0009,
 901	0x2470, 0xffffffff, 0x000d000c,
 902	0x2471, 0xffffffff, 0x00060005,
 903	0x2472, 0xffffffff, 0x00080007,
 904	0x2473, 0xffffffff, 0x0000000b,
 905	0x2474, 0xffffffff, 0x000a0009,
 906	0x2475, 0xffffffff, 0x000d000c,
 907	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
 908	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
 909	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
 910	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
 911	0x000c, 0xffffffff, 0x0000001c,
 912	0x000d, 0x000f0000, 0x000f0000,
 913	0x0583, 0xffffffff, 0x00000100,
 914	0x0409, 0xffffffff, 0x00000100,
 915	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
 916	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
 917	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
 918	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
 919	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
 920	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
 921	0x3430, 0xfffffff0, 0x00000100,
 922	0x3630, 0xfffffff0, 0x00000100,
 923};
 924
 925/* tahiti, pitcairn, verde */
 926static const struct amdgpu_video_codec_info tahiti_video_codecs_encode_array[] =
 927{
 928	{
 929		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 930		.max_width = 2048,
 931		.max_height = 1152,
 932		.max_pixels_per_frame = 2048 * 1152,
 933		.max_level = 0,
 934	},
 935};
 936
 937static const struct amdgpu_video_codecs tahiti_video_codecs_encode =
 938{
 939	.codec_count = ARRAY_SIZE(tahiti_video_codecs_encode_array),
 940	.codec_array = tahiti_video_codecs_encode_array,
 941};
 942
 943/* oland and hainan don't support encode */
 944static const struct amdgpu_video_codecs hainan_video_codecs_encode =
 945{
 946	.codec_count = 0,
 947	.codec_array = NULL,
 948};
 949
 950/* tahiti, pitcairn, verde, oland */
 951static const struct amdgpu_video_codec_info tahiti_video_codecs_decode_array[] =
 952{
 953	{
 954		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
 955		.max_width = 2048,
 956		.max_height = 1152,
 957		.max_pixels_per_frame = 2048 * 1152,
 958		.max_level = 3,
 959	},
 960	{
 961		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
 962		.max_width = 2048,
 963		.max_height = 1152,
 964		.max_pixels_per_frame = 2048 * 1152,
 965		.max_level = 5,
 966	},
 967	{
 968		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
 969		.max_width = 2048,
 970		.max_height = 1152,
 971		.max_pixels_per_frame = 2048 * 1152,
 972		.max_level = 41,
 973	},
 974	{
 975		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
 976		.max_width = 2048,
 977		.max_height = 1152,
 978		.max_pixels_per_frame = 2048 * 1152,
 979		.max_level = 4,
 980	},
 981};
 982
 983static const struct amdgpu_video_codecs tahiti_video_codecs_decode =
 984{
 985	.codec_count = ARRAY_SIZE(tahiti_video_codecs_decode_array),
 986	.codec_array = tahiti_video_codecs_decode_array,
 987};
 988
 989/* hainan doesn't support decode */
 990static const struct amdgpu_video_codecs hainan_video_codecs_decode =
 991{
 992	.codec_count = 0,
 993	.codec_array = NULL,
 994};
 995
 996static int si_query_video_codecs(struct amdgpu_device *adev, bool encode,
 997				 const struct amdgpu_video_codecs **codecs)
 998{
 999	switch (adev->asic_type) {
1000	case CHIP_VERDE:
1001	case CHIP_TAHITI:
1002	case CHIP_PITCAIRN:
1003		if (encode)
1004			*codecs = &tahiti_video_codecs_encode;
1005		else
1006			*codecs = &tahiti_video_codecs_decode;
1007		return 0;
1008	case CHIP_OLAND:
1009		if (encode)
1010			*codecs = &hainan_video_codecs_encode;
1011		else
1012			*codecs = &tahiti_video_codecs_decode;
1013		return 0;
1014	case CHIP_HAINAN:
1015		if (encode)
1016			*codecs = &hainan_video_codecs_encode;
1017		else
1018			*codecs = &hainan_video_codecs_decode;
1019		return 0;
1020	default:
1021		return -EINVAL;
1022	}
1023}
1024
1025static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
1026{
1027	unsigned long flags;
1028	u32 r;
1029
1030	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1031	WREG32(AMDGPU_PCIE_INDEX, reg);
1032	(void)RREG32(AMDGPU_PCIE_INDEX);
1033	r = RREG32(AMDGPU_PCIE_DATA);
1034	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1035	return r;
1036}
1037
1038static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1039{
1040	unsigned long flags;
1041
1042	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1043	WREG32(AMDGPU_PCIE_INDEX, reg);
1044	(void)RREG32(AMDGPU_PCIE_INDEX);
1045	WREG32(AMDGPU_PCIE_DATA, v);
1046	(void)RREG32(AMDGPU_PCIE_DATA);
1047	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1048}
1049
1050static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
1051{
1052	unsigned long flags;
1053	u32 r;
1054
1055	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1056	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1057	(void)RREG32(PCIE_PORT_INDEX);
1058	r = RREG32(PCIE_PORT_DATA);
1059	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1060	return r;
1061}
1062
1063static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1064{
1065	unsigned long flags;
1066
1067	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1068	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1069	(void)RREG32(PCIE_PORT_INDEX);
1070	WREG32(PCIE_PORT_DATA, (v));
1071	(void)RREG32(PCIE_PORT_DATA);
1072	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1073}
1074
1075static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
1076{
1077	unsigned long flags;
1078	u32 r;
1079
1080	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1081	WREG32(mmSMC_IND_INDEX_0, (reg));
1082	r = RREG32(mmSMC_IND_DATA_0);
1083	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1084	return r;
1085}
1086
1087static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1088{
1089	unsigned long flags;
1090
1091	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1092	WREG32(mmSMC_IND_INDEX_0, (reg));
1093	WREG32(mmSMC_IND_DATA_0, (v));
1094	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1095}
1096
1097static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
1098{
1099	unsigned long flags;
1100	u32 r;
1101
1102	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
1103	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1104	r = RREG32(mmUVD_CTX_DATA);
1105	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
1106	return r;
1107}
1108
1109static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1110{
1111	unsigned long flags;
1112
1113	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
1114	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1115	WREG32(mmUVD_CTX_DATA, (v));
1116	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
1117}
1118
1119static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
1120	{mmGRBM_STATUS},
1121	{mmGRBM_STATUS2},
1122	{mmGRBM_STATUS_SE0},
1123	{mmGRBM_STATUS_SE1},
1124	{mmSRBM_STATUS},
1125	{mmSRBM_STATUS2},
1126	{mmDMA_STATUS_REG + DMA0_REGISTER_OFFSET},
1127	{mmDMA_STATUS_REG + DMA1_REGISTER_OFFSET},
1128	{mmCP_STAT},
1129	{mmCP_STALLED_STAT1},
1130	{mmCP_STALLED_STAT2},
1131	{mmCP_STALLED_STAT3},
1132	{mmGB_ADDR_CONFIG},
1133	{mmMC_ARB_RAMCFG},
1134	{mmGB_TILE_MODE0},
1135	{mmGB_TILE_MODE1},
1136	{mmGB_TILE_MODE2},
1137	{mmGB_TILE_MODE3},
1138	{mmGB_TILE_MODE4},
1139	{mmGB_TILE_MODE5},
1140	{mmGB_TILE_MODE6},
1141	{mmGB_TILE_MODE7},
1142	{mmGB_TILE_MODE8},
1143	{mmGB_TILE_MODE9},
1144	{mmGB_TILE_MODE10},
1145	{mmGB_TILE_MODE11},
1146	{mmGB_TILE_MODE12},
1147	{mmGB_TILE_MODE13},
1148	{mmGB_TILE_MODE14},
1149	{mmGB_TILE_MODE15},
1150	{mmGB_TILE_MODE16},
1151	{mmGB_TILE_MODE17},
1152	{mmGB_TILE_MODE18},
1153	{mmGB_TILE_MODE19},
1154	{mmGB_TILE_MODE20},
1155	{mmGB_TILE_MODE21},
1156	{mmGB_TILE_MODE22},
1157	{mmGB_TILE_MODE23},
1158	{mmGB_TILE_MODE24},
1159	{mmGB_TILE_MODE25},
1160	{mmGB_TILE_MODE26},
1161	{mmGB_TILE_MODE27},
1162	{mmGB_TILE_MODE28},
1163	{mmGB_TILE_MODE29},
1164	{mmGB_TILE_MODE30},
1165	{mmGB_TILE_MODE31},
1166	{mmCC_RB_BACKEND_DISABLE, true},
1167	{mmGC_USER_RB_BACKEND_DISABLE, true},
1168	{mmPA_SC_RASTER_CONFIG, true},
1169};
1170
1171static uint32_t si_get_register_value(struct amdgpu_device *adev,
1172				      bool indexed, u32 se_num,
1173				      u32 sh_num, u32 reg_offset)
1174{
1175	if (indexed) {
1176		uint32_t val;
1177		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1178		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1179
1180		switch (reg_offset) {
1181		case mmCC_RB_BACKEND_DISABLE:
1182			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1183		case mmGC_USER_RB_BACKEND_DISABLE:
1184			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1185		case mmPA_SC_RASTER_CONFIG:
1186			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1187		}
1188
1189		mutex_lock(&adev->grbm_idx_mutex);
1190		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1191			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1192
1193		val = RREG32(reg_offset);
1194
1195		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1196			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1197		mutex_unlock(&adev->grbm_idx_mutex);
1198		return val;
1199	} else {
1200		unsigned idx;
1201
1202		switch (reg_offset) {
1203		case mmGB_ADDR_CONFIG:
1204			return adev->gfx.config.gb_addr_config;
1205		case mmMC_ARB_RAMCFG:
1206			return adev->gfx.config.mc_arb_ramcfg;
1207		case mmGB_TILE_MODE0:
1208		case mmGB_TILE_MODE1:
1209		case mmGB_TILE_MODE2:
1210		case mmGB_TILE_MODE3:
1211		case mmGB_TILE_MODE4:
1212		case mmGB_TILE_MODE5:
1213		case mmGB_TILE_MODE6:
1214		case mmGB_TILE_MODE7:
1215		case mmGB_TILE_MODE8:
1216		case mmGB_TILE_MODE9:
1217		case mmGB_TILE_MODE10:
1218		case mmGB_TILE_MODE11:
1219		case mmGB_TILE_MODE12:
1220		case mmGB_TILE_MODE13:
1221		case mmGB_TILE_MODE14:
1222		case mmGB_TILE_MODE15:
1223		case mmGB_TILE_MODE16:
1224		case mmGB_TILE_MODE17:
1225		case mmGB_TILE_MODE18:
1226		case mmGB_TILE_MODE19:
1227		case mmGB_TILE_MODE20:
1228		case mmGB_TILE_MODE21:
1229		case mmGB_TILE_MODE22:
1230		case mmGB_TILE_MODE23:
1231		case mmGB_TILE_MODE24:
1232		case mmGB_TILE_MODE25:
1233		case mmGB_TILE_MODE26:
1234		case mmGB_TILE_MODE27:
1235		case mmGB_TILE_MODE28:
1236		case mmGB_TILE_MODE29:
1237		case mmGB_TILE_MODE30:
1238		case mmGB_TILE_MODE31:
1239			idx = (reg_offset - mmGB_TILE_MODE0);
1240			return adev->gfx.config.tile_mode_array[idx];
1241		default:
1242			return RREG32(reg_offset);
1243		}
1244	}
1245}
1246static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1247			     u32 sh_num, u32 reg_offset, u32 *value)
1248{
1249	uint32_t i;
1250
1251	*value = 0;
1252	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1253		bool indexed = si_allowed_read_registers[i].grbm_indexed;
1254
1255		if (reg_offset != si_allowed_read_registers[i].reg_offset)
1256			continue;
1257
1258		*value = si_get_register_value(adev, indexed, se_num, sh_num,
1259					       reg_offset);
1260		return 0;
1261	}
1262	return -EINVAL;
1263}
1264
1265static bool si_read_disabled_bios(struct amdgpu_device *adev)
1266{
1267	u32 bus_cntl;
1268	u32 d1vga_control = 0;
1269	u32 d2vga_control = 0;
1270	u32 vga_render_control = 0;
1271	u32 rom_cntl;
1272	bool r;
1273
1274	bus_cntl = RREG32(mmBUS_CNTL);
1275	if (adev->mode_info.num_crtc) {
1276		d1vga_control = RREG32(mmD1VGA_CONTROL);
1277		d2vga_control = RREG32(mmD2VGA_CONTROL);
1278		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
1279	}
1280	rom_cntl = RREG32(R600_ROM_CNTL);
1281
1282	/* enable the rom */
1283	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
1284	if (adev->mode_info.num_crtc) {
1285		/* Disable VGA mode */
1286		WREG32(mmD1VGA_CONTROL,
1287		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
1288					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
1289		WREG32(mmD2VGA_CONTROL,
1290		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
1291					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
1292		WREG32(mmVGA_RENDER_CONTROL,
1293		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
1294	}
1295	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1296
1297	r = amdgpu_read_bios(adev);
1298
1299	/* restore regs */
1300	WREG32(mmBUS_CNTL, bus_cntl);
1301	if (adev->mode_info.num_crtc) {
1302		WREG32(mmD1VGA_CONTROL, d1vga_control);
1303		WREG32(mmD2VGA_CONTROL, d2vga_control);
1304		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1305	}
1306	WREG32(R600_ROM_CNTL, rom_cntl);
1307	return r;
1308}
1309
1310#define mmROM_INDEX 0x2A
1311#define mmROM_DATA  0x2B
1312
1313static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1314				  u8 *bios, u32 length_bytes)
1315{
1316	u32 *dw_ptr;
1317	u32 i, length_dw;
1318
1319	if (bios == NULL)
1320		return false;
1321	if (length_bytes == 0)
1322		return false;
1323	/* APU vbios image is part of sbios image */
1324	if (adev->flags & AMD_IS_APU)
1325		return false;
1326
1327	dw_ptr = (u32 *)bios;
1328	length_dw = ALIGN(length_bytes, 4) / 4;
1329	/* set rom index to 0 */
1330	WREG32(mmROM_INDEX, 0);
1331	for (i = 0; i < length_dw; i++)
1332		dw_ptr[i] = RREG32(mmROM_DATA);
1333
1334	return true;
1335}
1336
1337static void si_set_clk_bypass_mode(struct amdgpu_device *adev)
1338{
1339	u32 tmp, i;
1340
1341	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1342	tmp |= CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN_MASK;
1343	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1344
1345	tmp = RREG32(mmCG_SPLL_FUNC_CNTL_2);
1346	tmp |= CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK;
1347	WREG32(mmCG_SPLL_FUNC_CNTL_2, tmp);
1348
1349	for (i = 0; i < adev->usec_timeout; i++) {
1350		if (RREG32(mmCG_SPLL_STATUS) & CG_SPLL_STATUS__SPLL_CHG_STATUS_MASK)
1351			break;
1352		udelay(1);
1353	}
1354
1355	tmp = RREG32(mmCG_SPLL_FUNC_CNTL_2);
1356	tmp &= ~(CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK |
1357		CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE_MASK);
1358	WREG32(mmCG_SPLL_FUNC_CNTL_2, tmp);
1359
1360	tmp = RREG32(MPLL_CNTL_MODE);
1361	tmp &= ~MPLL_MCLK_SEL;
1362	WREG32(MPLL_CNTL_MODE, tmp);
1363}
1364
1365static void si_spll_powerdown(struct amdgpu_device *adev)
1366{
1367	u32 tmp;
1368
1369	tmp = RREG32(mmSPLL_CNTL_MODE);
1370	tmp |= SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK;
1371	WREG32(mmSPLL_CNTL_MODE, tmp);
1372
1373	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1374	tmp |= CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK;
1375	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1376
1377	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1378	tmp |= CG_SPLL_FUNC_CNTL__SPLL_SLEEP_MASK;
1379	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1380
1381	tmp = RREG32(mmSPLL_CNTL_MODE);
1382	tmp &= ~SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK;
1383	WREG32(mmSPLL_CNTL_MODE, tmp);
1384}
1385
1386static int si_gpu_pci_config_reset(struct amdgpu_device *adev)
1387{
1388	u32 i;
1389	int r = -EINVAL;
1390
1391	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1392
1393	/* set mclk/sclk to bypass */
1394	si_set_clk_bypass_mode(adev);
1395	/* powerdown spll */
1396	si_spll_powerdown(adev);
1397	/* disable BM */
1398	pci_clear_master(adev->pdev);
1399	/* reset */
1400	amdgpu_device_pci_config_reset(adev);
1401
1402	udelay(100);
1403
1404	/* wait for asic to come out of reset */
1405	for (i = 0; i < adev->usec_timeout; i++) {
1406		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1407			/* enable BM */
1408			pci_set_master(adev->pdev);
1409			adev->has_hw_reset = true;
1410			r = 0;
1411			break;
1412		}
1413		udelay(1);
1414	}
1415	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1416
1417	return r;
1418}
1419
1420static int si_asic_supports_baco(struct amdgpu_device *adev)
1421{
1422	return 0;
1423}
1424
1425static enum amd_reset_method
1426si_asic_reset_method(struct amdgpu_device *adev)
1427{
1428	if (amdgpu_reset_method == AMD_RESET_METHOD_PCI)
1429		return amdgpu_reset_method;
1430	else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY &&
1431		 amdgpu_reset_method != -1)
1432		dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
1433			 amdgpu_reset_method);
1434
1435	return AMD_RESET_METHOD_LEGACY;
1436}
1437
1438static int si_asic_reset(struct amdgpu_device *adev)
1439{
1440	int r;
1441
1442	switch (si_asic_reset_method(adev)) {
1443	case AMD_RESET_METHOD_PCI:
1444		dev_info(adev->dev, "PCI reset\n");
1445		r = amdgpu_device_pci_reset(adev);
1446		break;
1447	default:
1448		dev_info(adev->dev, "PCI CONFIG reset\n");
1449		r = si_gpu_pci_config_reset(adev);
1450		break;
1451	}
1452
1453	return r;
1454}
1455
1456static u32 si_get_config_memsize(struct amdgpu_device *adev)
1457{
1458	return RREG32(mmCONFIG_MEMSIZE);
1459}
1460
1461static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1462{
1463	uint32_t temp;
1464
1465	temp = RREG32(mmCONFIG_CNTL);
1466	if (!state) {
1467		temp &= ~(1<<0);
1468		temp |= (1<<1);
1469	} else {
1470		temp &= ~(1<<1);
1471	}
1472	WREG32(mmCONFIG_CNTL, temp);
1473}
1474
1475static u32 si_get_xclk(struct amdgpu_device *adev)
1476{
1477	u32 reference_clock = adev->clock.spll.reference_freq;
1478	u32 tmp;
1479
1480	tmp = RREG32(mmCG_CLKPIN_CNTL_2);
1481	if (tmp & CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK_MASK)
1482		return TCLK;
1483
1484	tmp = RREG32(mmCG_CLKPIN_CNTL);
1485	if (tmp & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
1486		return reference_clock / 4;
1487
1488	return reference_clock;
1489}
1490
1491static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1492{
1493	if (!ring || !ring->funcs->emit_wreg) {
1494		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1495		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1496	} else {
1497		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1498	}
1499}
1500
1501static void si_invalidate_hdp(struct amdgpu_device *adev,
1502			      struct amdgpu_ring *ring)
1503{
1504	if (!ring || !ring->funcs->emit_wreg) {
1505		WREG32(mmHDP_DEBUG0, 1);
1506		RREG32(mmHDP_DEBUG0);
1507	} else {
1508		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1509	}
1510}
1511
1512static bool si_need_full_reset(struct amdgpu_device *adev)
1513{
1514	/* change this when we support soft reset */
1515	return true;
1516}
1517
1518static bool si_need_reset_on_init(struct amdgpu_device *adev)
1519{
1520	return false;
1521}
1522
1523static int si_get_pcie_lanes(struct amdgpu_device *adev)
1524{
1525	u32 link_width_cntl;
1526
1527	if (adev->flags & AMD_IS_APU)
1528		return 0;
1529
1530	link_width_cntl = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
1531
1532	switch ((link_width_cntl & PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT) {
1533	case LC_LINK_WIDTH_X1:
1534		return 1;
1535	case LC_LINK_WIDTH_X2:
1536		return 2;
1537	case LC_LINK_WIDTH_X4:
1538		return 4;
1539	case LC_LINK_WIDTH_X8:
1540		return 8;
1541	case LC_LINK_WIDTH_X0:
1542	case LC_LINK_WIDTH_X16:
1543	default:
1544		return 16;
1545	}
1546}
1547
1548static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1549{
1550	u32 link_width_cntl, mask;
1551
1552	if (adev->flags & AMD_IS_APU)
1553		return;
1554
1555	switch (lanes) {
1556	case 0:
1557		mask = LC_LINK_WIDTH_X0;
1558		break;
1559	case 1:
1560		mask = LC_LINK_WIDTH_X1;
1561		break;
1562	case 2:
1563		mask = LC_LINK_WIDTH_X2;
1564		break;
1565	case 4:
1566		mask = LC_LINK_WIDTH_X4;
1567		break;
1568	case 8:
1569		mask = LC_LINK_WIDTH_X8;
1570		break;
1571	case 16:
1572		mask = LC_LINK_WIDTH_X16;
1573		break;
1574	default:
1575		DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1576		return;
1577	}
1578
1579	link_width_cntl = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
1580	link_width_cntl &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK;
1581	link_width_cntl |= mask << PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT;
1582	link_width_cntl |= (PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK |
1583		PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_ARC_MISSING_ESCAPE_MASK);
1584
1585	WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1586}
1587
1588static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1589			      uint64_t *count1)
1590{
1591	uint32_t perfctr = 0;
1592	uint64_t cnt0_of, cnt1_of;
1593	int tmp;
1594
1595	/* This reports 0 on APUs, so return to avoid writing/reading registers
1596	 * that may or may not be different from their GPU counterparts
1597	 */
1598	if (adev->flags & AMD_IS_APU)
1599		return;
1600
1601	/* Set the 2 events that we wish to watch, defined above */
1602	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1603	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1604	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1605
1606	/* Write to enable desired perf counters */
1607	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1608	/* Zero out and enable the perf counters
1609	 * Write 0x5:
1610	 * Bit 0 = Start all counters(1)
1611	 * Bit 2 = Global counter reset enable(1)
1612	 */
1613	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1614
1615	msleep(1000);
1616
1617	/* Load the shadow and disable the perf counters
1618	 * Write 0x2:
1619	 * Bit 0 = Stop counters(0)
1620	 * Bit 1 = Load the shadow counters(1)
1621	 */
1622	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1623
1624	/* Read register values to get any >32bit overflow */
1625	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1626	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1627	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1628
1629	/* Get the values and add the overflow */
1630	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1631	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1632}
1633
1634static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1635{
1636	uint64_t nak_r, nak_g;
1637
1638	/* Get the number of NAKs received and generated */
1639	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1640	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1641
1642	/* Add the total number of NAKs, i.e the number of replays */
1643	return (nak_r + nak_g);
1644}
1645
1646static int si_uvd_send_upll_ctlreq(struct amdgpu_device *adev,
1647				   unsigned cg_upll_func_cntl)
1648{
1649	unsigned i;
1650
1651	/* Make sure UPLL_CTLREQ is deasserted */
1652	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1653
1654	mdelay(10);
1655
1656	/* Assert UPLL_CTLREQ */
1657	WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1658
1659	/* Wait for CTLACK and CTLACK2 to get asserted */
1660	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1661		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1662
1663		if ((RREG32(cg_upll_func_cntl) & mask) == mask)
1664			break;
1665		mdelay(10);
1666	}
1667
1668	/* Deassert UPLL_CTLREQ */
1669	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1670
1671	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1672		DRM_ERROR("Timeout setting UVD clocks!\n");
1673		return -ETIMEDOUT;
1674	}
1675
1676	return 0;
1677}
1678
1679static unsigned si_uvd_calc_upll_post_div(unsigned vco_freq,
1680					  unsigned target_freq,
1681					  unsigned pd_min,
1682					  unsigned pd_even)
1683{
1684	unsigned post_div = vco_freq / target_freq;
1685
1686	/* Adjust to post divider minimum value */
1687	if (post_div < pd_min)
1688		post_div = pd_min;
1689
1690	/* We alway need a frequency less than or equal the target */
1691	if ((vco_freq / post_div) > target_freq)
1692		post_div += 1;
1693
1694	/* Post dividers above a certain value must be even */
1695	if (post_div > pd_even && post_div % 2)
1696		post_div += 1;
1697
1698	return post_div;
1699}
1700
1701/**
1702 * si_calc_upll_dividers - calc UPLL clock dividers
1703 *
1704 * @adev: amdgpu_device pointer
1705 * @vclk: wanted VCLK
1706 * @dclk: wanted DCLK
1707 * @vco_min: minimum VCO frequency
1708 * @vco_max: maximum VCO frequency
1709 * @fb_factor: factor to multiply vco freq with
1710 * @fb_mask: limit and bitmask for feedback divider
1711 * @pd_min: post divider minimum
1712 * @pd_max: post divider maximum
1713 * @pd_even: post divider must be even above this value
1714 * @optimal_fb_div: resulting feedback divider
1715 * @optimal_vclk_div: resulting vclk post divider
1716 * @optimal_dclk_div: resulting dclk post divider
1717 *
1718 * Calculate dividers for UVDs UPLL (except APUs).
1719 * Returns zero on success; -EINVAL on error.
1720 */
1721static int si_calc_upll_dividers(struct amdgpu_device *adev,
1722				 unsigned vclk, unsigned dclk,
1723				 unsigned vco_min, unsigned vco_max,
1724				 unsigned fb_factor, unsigned fb_mask,
1725				 unsigned pd_min, unsigned pd_max,
1726				 unsigned pd_even,
1727				 unsigned *optimal_fb_div,
1728				 unsigned *optimal_vclk_div,
1729				 unsigned *optimal_dclk_div)
1730{
1731	unsigned vco_freq, ref_freq = adev->clock.spll.reference_freq;
1732
1733	/* Start off with something large */
1734	unsigned optimal_score = ~0;
1735
1736	/* Loop through vco from low to high */
1737	vco_min = max(max(vco_min, vclk), dclk);
1738	for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) {
1739		uint64_t fb_div = (uint64_t)vco_freq * fb_factor;
1740		unsigned vclk_div, dclk_div, score;
1741
1742		do_div(fb_div, ref_freq);
1743
1744		/* fb div out of range ? */
1745		if (fb_div > fb_mask)
1746			break; /* It can oly get worse */
1747
1748		fb_div &= fb_mask;
1749
1750		/* Calc vclk divider with current vco freq */
1751		vclk_div = si_uvd_calc_upll_post_div(vco_freq, vclk,
1752						     pd_min, pd_even);
1753		if (vclk_div > pd_max)
1754			break; /* vco is too big, it has to stop */
1755
1756		/* Calc dclk divider with current vco freq */
1757		dclk_div = si_uvd_calc_upll_post_div(vco_freq, dclk,
1758						     pd_min, pd_even);
1759		if (dclk_div > pd_max)
1760			break; /* vco is too big, it has to stop */
1761
1762		/* Calc score with current vco freq */
1763		score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div);
1764
1765		/* Determine if this vco setting is better than current optimal settings */
1766		if (score < optimal_score) {
1767			*optimal_fb_div = fb_div;
1768			*optimal_vclk_div = vclk_div;
1769			*optimal_dclk_div = dclk_div;
1770			optimal_score = score;
1771			if (optimal_score == 0)
1772				break; /* It can't get better than this */
1773		}
1774	}
1775
1776	/* Did we found a valid setup ? */
1777	if (optimal_score == ~0)
1778		return -EINVAL;
1779
1780	return 0;
1781}
1782
1783static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1784{
1785	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1786	int r;
1787
1788	/* Bypass vclk and dclk with bclk */
1789	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1790		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1791		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1792
1793	/* Put PLL in bypass mode */
1794	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1795
1796	if (!vclk || !dclk) {
1797		/* Keep the Bypass mode */
1798		return 0;
1799	}
1800
1801	r = si_calc_upll_dividers(adev, vclk, dclk, 125000, 250000,
1802				  16384, 0x03FFFFFF, 0, 128, 5,
1803				  &fb_div, &vclk_div, &dclk_div);
1804	if (r)
1805		return r;
1806
1807	/* Set RESET_ANTI_MUX to 0 */
1808	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1809
1810	/* Set VCO_MODE to 1 */
1811	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1812
1813	/* Disable sleep mode */
1814	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1815
1816	/* Deassert UPLL_RESET */
1817	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1818
1819	mdelay(1);
1820
1821	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1822	if (r)
1823		return r;
1824
1825	/* Assert UPLL_RESET again */
1826	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1827
1828	/* Disable spread spectrum. */
1829	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1830
1831	/* Set feedback divider */
1832	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1833
1834	/* Set ref divider to 0 */
1835	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1836
1837	if (fb_div < 307200)
1838		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1839	else
1840		WREG32_P(CG_UPLL_FUNC_CNTL_4,
1841			 UPLL_SPARE_ISPARE9,
1842			 ~UPLL_SPARE_ISPARE9);
1843
1844	/* Set PDIV_A and PDIV_B */
1845	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1846		 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1847		 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1848
1849	/* Give the PLL some time to settle */
1850	mdelay(15);
1851
1852	/* Deassert PLL_RESET */
1853	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1854
1855	mdelay(15);
1856
1857	/* Switch from bypass mode to normal mode */
1858	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1859
1860	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1861	if (r)
1862		return r;
1863
1864	/* Switch VCLK and DCLK selection */
1865	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1866		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1867		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1868
1869	mdelay(100);
1870
1871	return 0;
1872}
1873
1874static int si_vce_send_vcepll_ctlreq(struct amdgpu_device *adev)
1875{
1876	unsigned i;
1877
1878	/* Make sure VCEPLL_CTLREQ is deasserted */
1879	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1880
1881	mdelay(10);
1882
1883	/* Assert UPLL_CTLREQ */
1884	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1885
1886	/* Wait for CTLACK and CTLACK2 to get asserted */
1887	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1888		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1889
1890		if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
1891			break;
1892		mdelay(10);
1893	}
1894
1895	/* Deassert UPLL_CTLREQ */
1896	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1897
1898	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1899		DRM_ERROR("Timeout setting VCE clocks!\n");
1900		return -ETIMEDOUT;
1901	}
1902
1903	return 0;
1904}
1905
1906static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1907{
1908	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
1909	int r;
1910
1911	/* Bypass evclk and ecclk with bclk */
1912	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1913		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
1914		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1915
1916	/* Put PLL in bypass mode */
1917	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
1918		     ~VCEPLL_BYPASS_EN_MASK);
1919
1920	if (!evclk || !ecclk) {
1921		/*
1922		 * On some chips, the PLL takes way too long to get out of
1923		 * sleep mode, causing a timeout waiting on CTLACK/CTLACK2.
1924		 * Leave the PLL running in bypass mode.
1925		 */
1926		if (adev->pdev->device == 0x6780)
1927			return 0;
1928
1929		/* Keep the Bypass mode, put PLL to sleep */
1930		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1931			     ~VCEPLL_SLEEP_MASK);
1932		return 0;
1933	}
1934
1935	r = si_calc_upll_dividers(adev, evclk, ecclk, 125000, 250000,
1936				  16384, 0x03FFFFFF, 0, 128, 5,
1937				  &fb_div, &evclk_div, &ecclk_div);
1938	if (r)
1939		return r;
1940
1941	/* Set RESET_ANTI_MUX to 0 */
1942	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1943
1944	/* Set VCO_MODE to 1 */
1945	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
1946		     ~VCEPLL_VCO_MODE_MASK);
1947
1948	/* Toggle VCEPLL_SLEEP to 1 then back to 0 */
1949	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1950		     ~VCEPLL_SLEEP_MASK);
1951	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
1952
1953	/* Deassert VCEPLL_RESET */
1954	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1955
1956	mdelay(1);
1957
1958	r = si_vce_send_vcepll_ctlreq(adev);
1959	if (r)
1960		return r;
1961
1962	/* Assert VCEPLL_RESET again */
1963	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
1964
1965	/* Disable spread spectrum. */
1966	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1967
1968	/* Set feedback divider */
1969	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3,
1970		     VCEPLL_FB_DIV(fb_div),
1971		     ~VCEPLL_FB_DIV_MASK);
1972
1973	/* Set ref divider to 0 */
1974	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
1975
1976	/* Set PDIV_A and PDIV_B */
1977	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1978		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
1979		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
1980
1981	/* Give the PLL some time to settle */
1982	mdelay(15);
1983
1984	/* Deassert PLL_RESET */
1985	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1986
1987	mdelay(15);
1988
1989	/* Switch from bypass mode to normal mode */
1990	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
1991
1992	r = si_vce_send_vcepll_ctlreq(adev);
1993	if (r)
1994		return r;
1995
1996	/* Switch VCLK and DCLK selection */
1997	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1998		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
1999		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
2000
2001	mdelay(100);
2002
2003	return 0;
2004}
2005
2006static void si_pre_asic_init(struct amdgpu_device *adev)
2007{
2008}
2009
2010static const struct amdgpu_asic_funcs si_asic_funcs =
2011{
2012	.read_disabled_bios = &si_read_disabled_bios,
2013	.read_bios_from_rom = &si_read_bios_from_rom,
2014	.read_register = &si_read_register,
2015	.reset = &si_asic_reset,
2016	.reset_method = &si_asic_reset_method,
2017	.set_vga_state = &si_vga_set_state,
2018	.get_xclk = &si_get_xclk,
2019	.set_uvd_clocks = &si_set_uvd_clocks,
2020	.set_vce_clocks = &si_set_vce_clocks,
2021	.get_pcie_lanes = &si_get_pcie_lanes,
2022	.set_pcie_lanes = &si_set_pcie_lanes,
2023	.get_config_memsize = &si_get_config_memsize,
2024	.flush_hdp = &si_flush_hdp,
2025	.invalidate_hdp = &si_invalidate_hdp,
2026	.need_full_reset = &si_need_full_reset,
2027	.get_pcie_usage = &si_get_pcie_usage,
2028	.need_reset_on_init = &si_need_reset_on_init,
2029	.get_pcie_replay_count = &si_get_pcie_replay_count,
2030	.supports_baco = &si_asic_supports_baco,
2031	.pre_asic_init = &si_pre_asic_init,
2032	.query_video_codecs = &si_query_video_codecs,
2033};
2034
2035static uint32_t si_get_rev_id(struct amdgpu_device *adev)
2036{
2037	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
2038		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
2039}
2040
2041static int si_common_early_init(struct amdgpu_ip_block *ip_block)
2042{
2043	struct amdgpu_device *adev = ip_block->adev;
2044
2045	adev->smc_rreg = &si_smc_rreg;
2046	adev->smc_wreg = &si_smc_wreg;
2047	adev->pcie_rreg = &si_pcie_rreg;
2048	adev->pcie_wreg = &si_pcie_wreg;
2049	adev->pciep_rreg = &si_pciep_rreg;
2050	adev->pciep_wreg = &si_pciep_wreg;
2051	adev->uvd_ctx_rreg = si_uvd_ctx_rreg;
2052	adev->uvd_ctx_wreg = si_uvd_ctx_wreg;
2053	adev->didt_rreg = NULL;
2054	adev->didt_wreg = NULL;
2055
2056	adev->asic_funcs = &si_asic_funcs;
2057
2058	adev->rev_id = si_get_rev_id(adev);
2059	adev->external_rev_id = 0xFF;
2060	switch (adev->asic_type) {
2061	case CHIP_TAHITI:
2062		adev->cg_flags =
2063			AMD_CG_SUPPORT_GFX_MGCG |
2064			AMD_CG_SUPPORT_GFX_MGLS |
2065			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2066			AMD_CG_SUPPORT_GFX_CGLS |
2067			AMD_CG_SUPPORT_GFX_CGTS |
2068			AMD_CG_SUPPORT_GFX_CP_LS |
2069			AMD_CG_SUPPORT_MC_MGCG |
2070			AMD_CG_SUPPORT_SDMA_MGCG |
2071			AMD_CG_SUPPORT_BIF_LS |
2072			AMD_CG_SUPPORT_VCE_MGCG |
2073			AMD_CG_SUPPORT_UVD_MGCG |
2074			AMD_CG_SUPPORT_HDP_LS |
2075			AMD_CG_SUPPORT_HDP_MGCG;
2076		adev->pg_flags = 0;
2077		adev->external_rev_id = (adev->rev_id == 0) ? 1 :
2078					(adev->rev_id == 1) ? 5 : 6;
2079		break;
2080	case CHIP_PITCAIRN:
2081		adev->cg_flags =
2082			AMD_CG_SUPPORT_GFX_MGCG |
2083			AMD_CG_SUPPORT_GFX_MGLS |
2084			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2085			AMD_CG_SUPPORT_GFX_CGLS |
2086			AMD_CG_SUPPORT_GFX_CGTS |
2087			AMD_CG_SUPPORT_GFX_CP_LS |
2088			AMD_CG_SUPPORT_GFX_RLC_LS |
2089			AMD_CG_SUPPORT_MC_LS |
2090			AMD_CG_SUPPORT_MC_MGCG |
2091			AMD_CG_SUPPORT_SDMA_MGCG |
2092			AMD_CG_SUPPORT_BIF_LS |
2093			AMD_CG_SUPPORT_VCE_MGCG |
2094			AMD_CG_SUPPORT_UVD_MGCG |
2095			AMD_CG_SUPPORT_HDP_LS |
2096			AMD_CG_SUPPORT_HDP_MGCG;
2097		adev->pg_flags = 0;
2098		adev->external_rev_id = adev->rev_id + 20;
2099		break;
2100
2101	case CHIP_VERDE:
2102		adev->cg_flags =
2103			AMD_CG_SUPPORT_GFX_MGCG |
2104			AMD_CG_SUPPORT_GFX_MGLS |
2105			AMD_CG_SUPPORT_GFX_CGLS |
2106			AMD_CG_SUPPORT_GFX_CGTS |
2107			AMD_CG_SUPPORT_GFX_CGTS_LS |
2108			AMD_CG_SUPPORT_GFX_CP_LS |
2109			AMD_CG_SUPPORT_MC_LS |
2110			AMD_CG_SUPPORT_MC_MGCG |
2111			AMD_CG_SUPPORT_SDMA_MGCG |
2112			AMD_CG_SUPPORT_SDMA_LS |
2113			AMD_CG_SUPPORT_BIF_LS |
2114			AMD_CG_SUPPORT_VCE_MGCG |
2115			AMD_CG_SUPPORT_UVD_MGCG |
2116			AMD_CG_SUPPORT_HDP_LS |
2117			AMD_CG_SUPPORT_HDP_MGCG;
2118		adev->pg_flags = 0;
2119		//???
2120		adev->external_rev_id = adev->rev_id + 40;
2121		break;
2122	case CHIP_OLAND:
2123		adev->cg_flags =
2124			AMD_CG_SUPPORT_GFX_MGCG |
2125			AMD_CG_SUPPORT_GFX_MGLS |
2126			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2127			AMD_CG_SUPPORT_GFX_CGLS |
2128			AMD_CG_SUPPORT_GFX_CGTS |
2129			AMD_CG_SUPPORT_GFX_CP_LS |
2130			AMD_CG_SUPPORT_GFX_RLC_LS |
2131			AMD_CG_SUPPORT_MC_LS |
2132			AMD_CG_SUPPORT_MC_MGCG |
2133			AMD_CG_SUPPORT_SDMA_MGCG |
2134			AMD_CG_SUPPORT_BIF_LS |
2135			AMD_CG_SUPPORT_UVD_MGCG |
2136			AMD_CG_SUPPORT_HDP_LS |
2137			AMD_CG_SUPPORT_HDP_MGCG;
2138		adev->pg_flags = 0;
2139		adev->external_rev_id = 60;
2140		break;
2141	case CHIP_HAINAN:
2142		adev->cg_flags =
2143			AMD_CG_SUPPORT_GFX_MGCG |
2144			AMD_CG_SUPPORT_GFX_MGLS |
2145			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2146			AMD_CG_SUPPORT_GFX_CGLS |
2147			AMD_CG_SUPPORT_GFX_CGTS |
2148			AMD_CG_SUPPORT_GFX_CP_LS |
2149			AMD_CG_SUPPORT_GFX_RLC_LS |
2150			AMD_CG_SUPPORT_MC_LS |
2151			AMD_CG_SUPPORT_MC_MGCG |
2152			AMD_CG_SUPPORT_SDMA_MGCG |
2153			AMD_CG_SUPPORT_BIF_LS |
2154			AMD_CG_SUPPORT_HDP_LS |
2155			AMD_CG_SUPPORT_HDP_MGCG;
2156		adev->pg_flags = 0;
2157		adev->external_rev_id = 70;
2158		break;
2159
2160	default:
2161		return -EINVAL;
2162	}
2163
2164	return 0;
2165}
2166
2167static void si_init_golden_registers(struct amdgpu_device *adev)
2168{
2169	switch (adev->asic_type) {
2170	case CHIP_TAHITI:
2171		amdgpu_device_program_register_sequence(adev,
2172							tahiti_golden_registers,
2173							ARRAY_SIZE(tahiti_golden_registers));
2174		amdgpu_device_program_register_sequence(adev,
2175							tahiti_golden_rlc_registers,
2176							ARRAY_SIZE(tahiti_golden_rlc_registers));
2177		amdgpu_device_program_register_sequence(adev,
2178							tahiti_mgcg_cgcg_init,
2179							ARRAY_SIZE(tahiti_mgcg_cgcg_init));
2180		amdgpu_device_program_register_sequence(adev,
2181							tahiti_golden_registers2,
2182							ARRAY_SIZE(tahiti_golden_registers2));
2183		break;
2184	case CHIP_PITCAIRN:
2185		amdgpu_device_program_register_sequence(adev,
2186							pitcairn_golden_registers,
2187							ARRAY_SIZE(pitcairn_golden_registers));
2188		amdgpu_device_program_register_sequence(adev,
2189							pitcairn_golden_rlc_registers,
2190							ARRAY_SIZE(pitcairn_golden_rlc_registers));
2191		amdgpu_device_program_register_sequence(adev,
2192							pitcairn_mgcg_cgcg_init,
2193							ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
2194		break;
2195	case CHIP_VERDE:
2196		amdgpu_device_program_register_sequence(adev,
2197							verde_golden_registers,
2198							ARRAY_SIZE(verde_golden_registers));
2199		amdgpu_device_program_register_sequence(adev,
2200							verde_golden_rlc_registers,
2201							ARRAY_SIZE(verde_golden_rlc_registers));
2202		amdgpu_device_program_register_sequence(adev,
2203							verde_mgcg_cgcg_init,
2204							ARRAY_SIZE(verde_mgcg_cgcg_init));
2205		amdgpu_device_program_register_sequence(adev,
2206							verde_pg_init,
2207							ARRAY_SIZE(verde_pg_init));
2208		break;
2209	case CHIP_OLAND:
2210		amdgpu_device_program_register_sequence(adev,
2211							oland_golden_registers,
2212							ARRAY_SIZE(oland_golden_registers));
2213		amdgpu_device_program_register_sequence(adev,
2214							oland_golden_rlc_registers,
2215							ARRAY_SIZE(oland_golden_rlc_registers));
2216		amdgpu_device_program_register_sequence(adev,
2217							oland_mgcg_cgcg_init,
2218							ARRAY_SIZE(oland_mgcg_cgcg_init));
2219		break;
2220	case CHIP_HAINAN:
2221		amdgpu_device_program_register_sequence(adev,
2222							hainan_golden_registers,
2223							ARRAY_SIZE(hainan_golden_registers));
2224		amdgpu_device_program_register_sequence(adev,
2225							hainan_golden_registers2,
2226							ARRAY_SIZE(hainan_golden_registers2));
2227		amdgpu_device_program_register_sequence(adev,
2228							hainan_mgcg_cgcg_init,
2229							ARRAY_SIZE(hainan_mgcg_cgcg_init));
2230		break;
2231
2232
2233	default:
2234		BUG();
2235	}
2236}
2237
2238static void si_pcie_gen3_enable(struct amdgpu_device *adev)
2239{
2240	struct pci_dev *root = adev->pdev->bus->self;
2241	u32 speed_cntl, current_data_rate;
2242	int i;
2243	u16 tmp16;
2244
2245	if (pci_is_root_bus(adev->pdev->bus))
2246		return;
2247
2248	if (amdgpu_pcie_gen2 == 0)
2249		return;
2250
2251	if (adev->flags & AMD_IS_APU)
2252		return;
2253
2254	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2255					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
2256		return;
2257
2258	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2259	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
2260		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
2261	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2262		if (current_data_rate == 2) {
2263			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
2264			return;
2265		}
2266		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
2267	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
2268		if (current_data_rate == 1) {
2269			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
2270			return;
2271		}
2272		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
2273	}
2274
2275	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
2276		return;
2277
2278	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2279		if (current_data_rate != 2) {
2280			u16 bridge_cfg, gpu_cfg;
2281			u16 bridge_cfg2, gpu_cfg2;
2282			u32 max_lw, current_lw, tmp;
2283
2284			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
2285			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
2286
2287			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
2288			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >> PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
2289			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK) >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
2290
2291			if (current_lw < max_lw) {
2292				tmp = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
2293				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
2294					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
2295					tmp |= (max_lw << PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
2296					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
2297					WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
2298				}
2299			}
2300
2301			for (i = 0; i < 10; i++) {
2302				pcie_capability_read_word(adev->pdev,
2303							  PCI_EXP_DEVSTA,
2304							  &tmp16);
2305				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
2306					break;
2307
2308				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2309							  &bridge_cfg);
2310				pcie_capability_read_word(adev->pdev,
2311							  PCI_EXP_LNKCTL,
2312							  &gpu_cfg);
2313
2314				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
2315							  &bridge_cfg2);
2316				pcie_capability_read_word(adev->pdev,
2317							  PCI_EXP_LNKCTL2,
2318							  &gpu_cfg2);
2319
2320				tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2321				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
2322				WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2323
2324				tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2325				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
2326				WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2327
2328				mdelay(100);
2329
2330				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
2331								   PCI_EXP_LNKCTL_HAWD,
2332								   bridge_cfg &
2333								   PCI_EXP_LNKCTL_HAWD);
2334				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
2335								   PCI_EXP_LNKCTL_HAWD,
2336								   gpu_cfg &
2337								   PCI_EXP_LNKCTL_HAWD);
2338
2339				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
2340								   PCI_EXP_LNKCTL2_ENTER_COMP |
2341								   PCI_EXP_LNKCTL2_TX_MARGIN,
2342								   bridge_cfg2 &
2343								   (PCI_EXP_LNKCTL2_ENTER_COMP |
2344								    PCI_EXP_LNKCTL2_TX_MARGIN));
2345				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
2346								   PCI_EXP_LNKCTL2_ENTER_COMP |
2347								   PCI_EXP_LNKCTL2_TX_MARGIN,
2348								   gpu_cfg2 &
2349								   (PCI_EXP_LNKCTL2_ENTER_COMP |
2350								    PCI_EXP_LNKCTL2_TX_MARGIN));
2351
2352				tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2353				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
2354				WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2355			}
2356		}
2357	}
2358
2359	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK | PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
2360	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
2361	WREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL, speed_cntl);
2362
2363	tmp16 = 0;
2364	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
2365		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
2366	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
2367		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
2368	else
2369		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
2370	pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
2371					   PCI_EXP_LNKCTL2_TLS, tmp16);
2372
2373	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2374	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
2375	WREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL, speed_cntl);
2376
2377	for (i = 0; i < adev->usec_timeout; i++) {
2378		speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2379		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
2380			break;
2381		udelay(1);
2382	}
2383}
2384
2385static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
2386{
2387	unsigned long flags;
2388	u32 r;
2389
2390	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2391	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2392	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
2393	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2394	return r;
2395}
2396
2397static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2398{
2399	unsigned long flags;
2400
2401	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2402	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2403	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
2404	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2405}
2406
2407static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
2408{
2409	unsigned long flags;
2410	u32 r;
2411
2412	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2413	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2414	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
2415	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2416	return r;
2417}
2418
2419static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2420{
2421	unsigned long flags;
2422
2423	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2424	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2425	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
2426	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
2427}
2428static void si_program_aspm(struct amdgpu_device *adev)
2429{
2430	u32 data, orig;
2431	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
2432	bool disable_clkreq = false;
2433
2434	if (!amdgpu_device_should_use_aspm(adev))
2435		return;
2436
2437	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL);
2438	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
2439	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) | PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
2440	if (orig != data)
2441		WREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL, data);
2442
2443	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL3);
2444	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
2445	if (orig != data)
2446		WREG32_PCIE_PORT(ixPCIE_LC_CNTL3, data);
2447
2448	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
2449	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
2450	if (orig != data)
2451		WREG32_PCIE(ixPCIE_P_CNTL, data);
2452
2453	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL);
2454	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK | PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
2455	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
2456	if (!disable_l0s)
2457		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
2458
2459	if (!disable_l1) {
2460		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
2461		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
2462		if (orig != data)
2463			WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2464
2465		if (!disable_plloff_in_l1) {
2466			bool clk_req_support;
2467
2468			orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_0);
2469			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
2470			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
2471			if (orig != data)
2472				si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_0, data);
2473
2474			orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_1);
2475			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
2476			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
2477			if (orig != data)
2478				si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_1, data);
2479
2480			orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_0);
2481			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
2482			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
2483			if (orig != data)
2484				si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_0, data);
2485
2486			orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_1);
2487			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
2488			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
2489			if (orig != data)
2490				si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_1, data);
2491
2492			if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
2493				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_0);
2494				data &= ~PB0_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK;
2495				if (orig != data)
2496					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_0, data);
2497
2498				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_1);
2499				data &= ~PB0_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK;
2500				if (orig != data)
2501					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_1, data);
2502
2503				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_2);
2504				data &= ~PB0_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK;
2505				if (orig != data)
2506					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_2, data);
2507
2508				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_3);
2509				data &= ~PB0_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK;
2510				if (orig != data)
2511					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_3, data);
2512
2513				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_0);
2514				data &= ~PB1_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK;
2515				if (orig != data)
2516					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_0, data);
2517
2518				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_1);
2519				data &= ~PB1_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK;
2520				if (orig != data)
2521					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_1, data);
2522
2523				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_2);
2524				data &= ~PB1_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK;
2525				if (orig != data)
2526					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_2, data);
2527
2528				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_3);
2529				data &= ~PB1_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK;
2530				if (orig != data)
2531					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_3, data);
2532			}
2533			orig = data = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
2534			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
2535			data |= (3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
2536			if (orig != data)
2537				WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, data);
2538
2539			orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_CNTL);
2540			data &= ~PB0_PIF_CNTL__LS2_EXIT_TIME_MASK;
2541			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2542				data |= (5 << PB0_PIF_CNTL__LS2_EXIT_TIME__SHIFT);
2543			if (orig != data)
2544				si_pif_phy0_wreg(adev,ixPB0_PIF_CNTL, data);
2545
2546			orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_CNTL);
2547			data &= ~PB1_PIF_CNTL__LS2_EXIT_TIME_MASK;
2548			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2549				data |= (5 << PB1_PIF_CNTL__LS2_EXIT_TIME__SHIFT);
2550			if (orig != data)
2551				si_pif_phy1_wreg(adev,ixPB1_PIF_CNTL, data);
2552
2553			if (!disable_clkreq &&
2554			    !pci_is_root_bus(adev->pdev->bus)) {
2555				struct pci_dev *root = adev->pdev->bus->self;
2556				u32 lnkcap;
2557
2558				clk_req_support = false;
2559				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
2560				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
2561					clk_req_support = true;
2562			} else {
2563				clk_req_support = false;
2564			}
2565
2566			if (clk_req_support) {
2567				orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL2);
2568				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
2569				if (orig != data)
2570					WREG32_PCIE_PORT(ixPCIE_LC_CNTL2, data);
2571
2572				orig = data = RREG32(mmTHM_CLK_CNTL);
2573				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK | THM_CLK_CNTL__TMON_CLK_SEL_MASK);
2574				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) | (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
2575				if (orig != data)
2576					WREG32(mmTHM_CLK_CNTL, data);
2577
2578				orig = data = RREG32(mmMISC_CLK_CNTL);
2579				data &= ~(MISC_CLK_CNTL__DEEP_SLEEP_CLK_SEL_MASK | MISC_CLK_CNTL__ZCLK_SEL_MASK);
2580				data |= (1 << MISC_CLK_CNTL__DEEP_SLEEP_CLK_SEL__SHIFT) | (1 << MISC_CLK_CNTL__ZCLK_SEL__SHIFT);
2581				if (orig != data)
2582					WREG32(mmMISC_CLK_CNTL, data);
2583
2584				orig = data = RREG32(mmCG_CLKPIN_CNTL);
2585				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
2586				if (orig != data)
2587					WREG32(mmCG_CLKPIN_CNTL, data);
2588
2589				orig = data = RREG32(mmCG_CLKPIN_CNTL_2);
2590				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
2591				if (orig != data)
2592					WREG32(mmCG_CLKPIN_CNTL_2, data);
2593
2594				orig = data = RREG32(mmMPLL_BYPASSCLK_SEL);
2595				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
2596				data |= 4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT;
2597				if (orig != data)
2598					WREG32(mmMPLL_BYPASSCLK_SEL, data);
2599
2600				orig = data = RREG32(mmSPLL_CNTL_MODE);
2601				data &= ~SPLL_CNTL_MODE__SPLL_REFCLK_SEL_MASK;
2602				if (orig != data)
2603					WREG32(mmSPLL_CNTL_MODE, data);
2604			}
2605		}
2606	} else {
2607		if (orig != data)
2608			WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2609	}
2610
2611	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
2612	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK | PCIE_CNTL2__MST_MEM_LS_EN_MASK | PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
2613	if (orig != data)
2614		WREG32_PCIE(ixPCIE_CNTL2, data);
2615
2616	if (!disable_l0s) {
2617		data = RREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL);
2618		if((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) == PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
2619			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
2620			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) && (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
2621				orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL);
2622				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
2623				if (orig != data)
2624					WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2625			}
2626		}
2627	}
2628}
2629
2630static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2631{
2632	int readrq;
2633	u16 v;
2634
2635	readrq = pcie_get_readrq(adev->pdev);
2636	v = ffs(readrq) - 8;
2637	if ((v == 0) || (v == 6) || (v == 7))
2638		pcie_set_readrq(adev->pdev, 512);
2639}
2640
2641static int si_common_hw_init(struct amdgpu_ip_block *ip_block)
2642{
2643	struct amdgpu_device *adev = ip_block->adev;
2644
2645	si_fix_pci_max_read_req_size(adev);
2646	si_init_golden_registers(adev);
2647	si_pcie_gen3_enable(adev);
2648	si_program_aspm(adev);
2649
2650	return 0;
2651}
2652
2653static int si_common_hw_fini(struct amdgpu_ip_block *ip_block)
2654{
2655	return 0;
2656}
2657
2658static int si_common_resume(struct amdgpu_ip_block *ip_block)
2659{
2660	return si_common_hw_init(ip_block);
2661}
2662
2663static bool si_common_is_idle(struct amdgpu_ip_block *ip_block)
2664{
2665	return true;
2666}
2667
2668static int si_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2669					    enum amd_clockgating_state state)
2670{
2671	return 0;
2672}
2673
2674static int si_common_set_powergating_state(struct amdgpu_ip_block *ip_block,
2675					    enum amd_powergating_state state)
2676{
2677	return 0;
2678}
2679
2680static const struct amd_ip_funcs si_common_ip_funcs = {
2681	.name = "si_common",
2682	.early_init = si_common_early_init,
2683	.hw_init = si_common_hw_init,
2684	.hw_fini = si_common_hw_fini,
2685	.resume = si_common_resume,
2686	.is_idle = si_common_is_idle,
2687	.set_clockgating_state = si_common_set_clockgating_state,
2688	.set_powergating_state = si_common_set_powergating_state,
2689};
2690
2691static const struct amdgpu_ip_block_version si_common_ip_block =
2692{
2693	.type = AMD_IP_BLOCK_TYPE_COMMON,
2694	.major = 1,
2695	.minor = 0,
2696	.rev = 0,
2697	.funcs = &si_common_ip_funcs,
2698};
2699
2700int si_set_ip_blocks(struct amdgpu_device *adev)
2701{
2702	switch (adev->asic_type) {
2703	case CHIP_VERDE:
2704	case CHIP_TAHITI:
2705	case CHIP_PITCAIRN:
2706		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2707		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2708		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2709		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2710		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2711		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2712		if (adev->enable_virtual_display)
2713			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2714#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2715		else if (amdgpu_device_has_dc_support(adev))
2716			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2717#endif
2718		else
2719			amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
2720		amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2721		amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block);
2722		break;
2723	case CHIP_OLAND:
2724		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2725		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2726		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2727		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2728		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2729		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2730		if (adev->enable_virtual_display)
2731			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2732#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2733		else if (amdgpu_device_has_dc_support(adev))
2734			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2735#endif
2736		else
2737			amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2738		amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2739		break;
2740	case CHIP_HAINAN:
2741		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2742		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2743		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2744		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2745		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2746		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2747		if (adev->enable_virtual_display)
2748			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2749		break;
2750	default:
2751		BUG();
2752	}
2753	return 0;
2754}
2755