gfx_v6_0.c

   1/*
   2 * Copyright 2015 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23#include <linux/firmware.h>
  24#include <linux/module.h>
  25
  26#include "amdgpu.h"
  27#include "amdgpu_ih.h"
  28#include "amdgpu_gfx.h"
  29#include "amdgpu_ucode.h"
  30#include "clearstate_si.h"
  31#include "si.h"
  32#include "sid.h"
  33
  34#include "bif/bif_3_0_d.h"
  35#include "bif/bif_3_0_sh_mask.h"
  36
  37#include "oss/oss_1_0_d.h"
  38#include "oss/oss_1_0_sh_mask.h"
  39
  40#include "gca/gfx_6_0_d.h"
  41#include "gca/gfx_6_0_sh_mask.h"
  42#include "gca/gfx_7_2_enum.h"
  43
  44#include "gmc/gmc_6_0_d.h"
  45#include "gmc/gmc_6_0_sh_mask.h"
  46
  47#include "dce/dce_6_0_d.h"
  48#include "dce/dce_6_0_sh_mask.h"
  49
  50#include "si_enums.h"
  51
  52#define TAHITI_GB_ADDR_CONFIG_GOLDEN        0x12011003
  53#define VERDE_GB_ADDR_CONFIG_GOLDEN         0x12010002
  54#define HAINAN_GB_ADDR_CONFIG_GOLDEN        0x02010001
  55
  56#define GFX6_NUM_GFX_RINGS     1
  57#define GFX6_NUM_COMPUTE_RINGS 2
  58
  59static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev);
  60static void gfx_v6_0_set_irq_funcs(struct amdgpu_device *adev);
  61static void gfx_v6_0_get_cu_info(struct amdgpu_device *adev);
  62
  63MODULE_FIRMWARE("amdgpu/tahiti_pfp.bin");
  64MODULE_FIRMWARE("amdgpu/tahiti_me.bin");
  65MODULE_FIRMWARE("amdgpu/tahiti_ce.bin");
  66MODULE_FIRMWARE("amdgpu/tahiti_rlc.bin");
  67
  68MODULE_FIRMWARE("amdgpu/pitcairn_pfp.bin");
  69MODULE_FIRMWARE("amdgpu/pitcairn_me.bin");
  70MODULE_FIRMWARE("amdgpu/pitcairn_ce.bin");
  71MODULE_FIRMWARE("amdgpu/pitcairn_rlc.bin");
  72
  73MODULE_FIRMWARE("amdgpu/verde_pfp.bin");
  74MODULE_FIRMWARE("amdgpu/verde_me.bin");
  75MODULE_FIRMWARE("amdgpu/verde_ce.bin");
  76MODULE_FIRMWARE("amdgpu/verde_rlc.bin");
  77
  78MODULE_FIRMWARE("amdgpu/oland_pfp.bin");
  79MODULE_FIRMWARE("amdgpu/oland_me.bin");
  80MODULE_FIRMWARE("amdgpu/oland_ce.bin");
  81MODULE_FIRMWARE("amdgpu/oland_rlc.bin");
  82
  83MODULE_FIRMWARE("amdgpu/hainan_pfp.bin");
  84MODULE_FIRMWARE("amdgpu/hainan_me.bin");
  85MODULE_FIRMWARE("amdgpu/hainan_ce.bin");
  86MODULE_FIRMWARE("amdgpu/hainan_rlc.bin");
  87
  88static u32 gfx_v6_0_get_csb_size(struct amdgpu_device *adev);
  89static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer);
  90//static void gfx_v6_0_init_cp_pg_table(struct amdgpu_device *adev);
  91static void gfx_v6_0_init_pg(struct amdgpu_device *adev);
  92
  93#define ARRAY_MODE(x)					((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
  94#define PIPE_CONFIG(x)					((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
  95#define TILE_SPLIT(x)					((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
  96#define MICRO_TILE_MODE(x)				((x) << 0)
  97#define SAMPLE_SPLIT(x)					((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
  98#define BANK_WIDTH(x)					((x) << 14)
  99#define BANK_HEIGHT(x)					((x) << 16)
 100#define MACRO_TILE_ASPECT(x)				((x) << 18)
 101#define NUM_BANKS(x)					((x) << 20)
 102
 103static const u32 verde_rlc_save_restore_register_list[] =
 104{
 105	(0x8000 << 16) | (0x98f4 >> 2),
 106	0x00000000,
 107	(0x8040 << 16) | (0x98f4 >> 2),
 108	0x00000000,
 109	(0x8000 << 16) | (0xe80 >> 2),
 110	0x00000000,
 111	(0x8040 << 16) | (0xe80 >> 2),
 112	0x00000000,
 113	(0x8000 << 16) | (0x89bc >> 2),
 114	0x00000000,
 115	(0x8040 << 16) | (0x89bc >> 2),
 116	0x00000000,
 117	(0x8000 << 16) | (0x8c1c >> 2),
 118	0x00000000,
 119	(0x8040 << 16) | (0x8c1c >> 2),
 120	0x00000000,
 121	(0x9c00 << 16) | (0x98f0 >> 2),
 122	0x00000000,
 123	(0x9c00 << 16) | (0xe7c >> 2),
 124	0x00000000,
 125	(0x8000 << 16) | (0x9148 >> 2),
 126	0x00000000,
 127	(0x8040 << 16) | (0x9148 >> 2),
 128	0x00000000,
 129	(0x9c00 << 16) | (0x9150 >> 2),
 130	0x00000000,
 131	(0x9c00 << 16) | (0x897c >> 2),
 132	0x00000000,
 133	(0x9c00 << 16) | (0x8d8c >> 2),
 134	0x00000000,
 135	(0x9c00 << 16) | (0xac54 >> 2),
 136	0X00000000,
 137	0x3,
 138	(0x9c00 << 16) | (0x98f8 >> 2),
 139	0x00000000,
 140	(0x9c00 << 16) | (0x9910 >> 2),
 141	0x00000000,
 142	(0x9c00 << 16) | (0x9914 >> 2),
 143	0x00000000,
 144	(0x9c00 << 16) | (0x9918 >> 2),
 145	0x00000000,
 146	(0x9c00 << 16) | (0x991c >> 2),
 147	0x00000000,
 148	(0x9c00 << 16) | (0x9920 >> 2),
 149	0x00000000,
 150	(0x9c00 << 16) | (0x9924 >> 2),
 151	0x00000000,
 152	(0x9c00 << 16) | (0x9928 >> 2),
 153	0x00000000,
 154	(0x9c00 << 16) | (0x992c >> 2),
 155	0x00000000,
 156	(0x9c00 << 16) | (0x9930 >> 2),
 157	0x00000000,
 158	(0x9c00 << 16) | (0x9934 >> 2),
 159	0x00000000,
 160	(0x9c00 << 16) | (0x9938 >> 2),
 161	0x00000000,
 162	(0x9c00 << 16) | (0x993c >> 2),
 163	0x00000000,
 164	(0x9c00 << 16) | (0x9940 >> 2),
 165	0x00000000,
 166	(0x9c00 << 16) | (0x9944 >> 2),
 167	0x00000000,
 168	(0x9c00 << 16) | (0x9948 >> 2),
 169	0x00000000,
 170	(0x9c00 << 16) | (0x994c >> 2),
 171	0x00000000,
 172	(0x9c00 << 16) | (0x9950 >> 2),
 173	0x00000000,
 174	(0x9c00 << 16) | (0x9954 >> 2),
 175	0x00000000,
 176	(0x9c00 << 16) | (0x9958 >> 2),
 177	0x00000000,
 178	(0x9c00 << 16) | (0x995c >> 2),
 179	0x00000000,
 180	(0x9c00 << 16) | (0x9960 >> 2),
 181	0x00000000,
 182	(0x9c00 << 16) | (0x9964 >> 2),
 183	0x00000000,
 184	(0x9c00 << 16) | (0x9968 >> 2),
 185	0x00000000,
 186	(0x9c00 << 16) | (0x996c >> 2),
 187	0x00000000,
 188	(0x9c00 << 16) | (0x9970 >> 2),
 189	0x00000000,
 190	(0x9c00 << 16) | (0x9974 >> 2),
 191	0x00000000,
 192	(0x9c00 << 16) | (0x9978 >> 2),
 193	0x00000000,
 194	(0x9c00 << 16) | (0x997c >> 2),
 195	0x00000000,
 196	(0x9c00 << 16) | (0x9980 >> 2),
 197	0x00000000,
 198	(0x9c00 << 16) | (0x9984 >> 2),
 199	0x00000000,
 200	(0x9c00 << 16) | (0x9988 >> 2),
 201	0x00000000,
 202	(0x9c00 << 16) | (0x998c >> 2),
 203	0x00000000,
 204	(0x9c00 << 16) | (0x8c00 >> 2),
 205	0x00000000,
 206	(0x9c00 << 16) | (0x8c14 >> 2),
 207	0x00000000,
 208	(0x9c00 << 16) | (0x8c04 >> 2),
 209	0x00000000,
 210	(0x9c00 << 16) | (0x8c08 >> 2),
 211	0x00000000,
 212	(0x8000 << 16) | (0x9b7c >> 2),
 213	0x00000000,
 214	(0x8040 << 16) | (0x9b7c >> 2),
 215	0x00000000,
 216	(0x8000 << 16) | (0xe84 >> 2),
 217	0x00000000,
 218	(0x8040 << 16) | (0xe84 >> 2),
 219	0x00000000,
 220	(0x8000 << 16) | (0x89c0 >> 2),
 221	0x00000000,
 222	(0x8040 << 16) | (0x89c0 >> 2),
 223	0x00000000,
 224	(0x8000 << 16) | (0x914c >> 2),
 225	0x00000000,
 226	(0x8040 << 16) | (0x914c >> 2),
 227	0x00000000,
 228	(0x8000 << 16) | (0x8c20 >> 2),
 229	0x00000000,
 230	(0x8040 << 16) | (0x8c20 >> 2),
 231	0x00000000,
 232	(0x8000 << 16) | (0x9354 >> 2),
 233	0x00000000,
 234	(0x8040 << 16) | (0x9354 >> 2),
 235	0x00000000,
 236	(0x9c00 << 16) | (0x9060 >> 2),
 237	0x00000000,
 238	(0x9c00 << 16) | (0x9364 >> 2),
 239	0x00000000,
 240	(0x9c00 << 16) | (0x9100 >> 2),
 241	0x00000000,
 242	(0x9c00 << 16) | (0x913c >> 2),
 243	0x00000000,
 244	(0x8000 << 16) | (0x90e0 >> 2),
 245	0x00000000,
 246	(0x8000 << 16) | (0x90e4 >> 2),
 247	0x00000000,
 248	(0x8000 << 16) | (0x90e8 >> 2),
 249	0x00000000,
 250	(0x8040 << 16) | (0x90e0 >> 2),
 251	0x00000000,
 252	(0x8040 << 16) | (0x90e4 >> 2),
 253	0x00000000,
 254	(0x8040 << 16) | (0x90e8 >> 2),
 255	0x00000000,
 256	(0x9c00 << 16) | (0x8bcc >> 2),
 257	0x00000000,
 258	(0x9c00 << 16) | (0x8b24 >> 2),
 259	0x00000000,
 260	(0x9c00 << 16) | (0x88c4 >> 2),
 261	0x00000000,
 262	(0x9c00 << 16) | (0x8e50 >> 2),
 263	0x00000000,
 264	(0x9c00 << 16) | (0x8c0c >> 2),
 265	0x00000000,
 266	(0x9c00 << 16) | (0x8e58 >> 2),
 267	0x00000000,
 268	(0x9c00 << 16) | (0x8e5c >> 2),
 269	0x00000000,
 270	(0x9c00 << 16) | (0x9508 >> 2),
 271	0x00000000,
 272	(0x9c00 << 16) | (0x950c >> 2),
 273	0x00000000,
 274	(0x9c00 << 16) | (0x9494 >> 2),
 275	0x00000000,
 276	(0x9c00 << 16) | (0xac0c >> 2),
 277	0x00000000,
 278	(0x9c00 << 16) | (0xac10 >> 2),
 279	0x00000000,
 280	(0x9c00 << 16) | (0xac14 >> 2),
 281	0x00000000,
 282	(0x9c00 << 16) | (0xae00 >> 2),
 283	0x00000000,
 284	(0x9c00 << 16) | (0xac08 >> 2),
 285	0x00000000,
 286	(0x9c00 << 16) | (0x88d4 >> 2),
 287	0x00000000,
 288	(0x9c00 << 16) | (0x88c8 >> 2),
 289	0x00000000,
 290	(0x9c00 << 16) | (0x88cc >> 2),
 291	0x00000000,
 292	(0x9c00 << 16) | (0x89b0 >> 2),
 293	0x00000000,
 294	(0x9c00 << 16) | (0x8b10 >> 2),
 295	0x00000000,
 296	(0x9c00 << 16) | (0x8a14 >> 2),
 297	0x00000000,
 298	(0x9c00 << 16) | (0x9830 >> 2),
 299	0x00000000,
 300	(0x9c00 << 16) | (0x9834 >> 2),
 301	0x00000000,
 302	(0x9c00 << 16) | (0x9838 >> 2),
 303	0x00000000,
 304	(0x9c00 << 16) | (0x9a10 >> 2),
 305	0x00000000,
 306	(0x8000 << 16) | (0x9870 >> 2),
 307	0x00000000,
 308	(0x8000 << 16) | (0x9874 >> 2),
 309	0x00000000,
 310	(0x8001 << 16) | (0x9870 >> 2),
 311	0x00000000,
 312	(0x8001 << 16) | (0x9874 >> 2),
 313	0x00000000,
 314	(0x8040 << 16) | (0x9870 >> 2),
 315	0x00000000,
 316	(0x8040 << 16) | (0x9874 >> 2),
 317	0x00000000,
 318	(0x8041 << 16) | (0x9870 >> 2),
 319	0x00000000,
 320	(0x8041 << 16) | (0x9874 >> 2),
 321	0x00000000,
 322	0x00000000
 323};
 324
 325static int gfx_v6_0_init_microcode(struct amdgpu_device *adev)
 326{
 327	const char *chip_name;
 328	int err;
 329	const struct gfx_firmware_header_v1_0 *cp_hdr;
 330	const struct rlc_firmware_header_v1_0 *rlc_hdr;
 331
 332	DRM_DEBUG("\n");
 333
 334	switch (adev->asic_type) {
 335	case CHIP_TAHITI:
 336		chip_name = "tahiti";
 337		break;
 338	case CHIP_PITCAIRN:
 339		chip_name = "pitcairn";
 340		break;
 341	case CHIP_VERDE:
 342		chip_name = "verde";
 343		break;
 344	case CHIP_OLAND:
 345		chip_name = "oland";
 346		break;
 347	case CHIP_HAINAN:
 348		chip_name = "hainan";
 349		break;
 350	default: BUG();
 351	}
 352
 353	err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
 354				   AMDGPU_UCODE_REQUIRED,
 355				   "amdgpu/%s_pfp.bin", chip_name);
 356	if (err)
 357		goto out;
 358	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 359	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 360	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 361
 362	err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
 363				   AMDGPU_UCODE_REQUIRED,
 364				   "amdgpu/%s_me.bin", chip_name);
 365	if (err)
 366		goto out;
 367	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 368	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 369	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 370
 371	err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
 372				   AMDGPU_UCODE_REQUIRED,
 373				   "amdgpu/%s_ce.bin", chip_name);
 374	if (err)
 375		goto out;
 376	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
 377	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
 378	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
 379
 380	err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 381				   AMDGPU_UCODE_REQUIRED,
 382				   "amdgpu/%s_rlc.bin", chip_name);
 383	if (err)
 384		goto out;
 385	rlc_hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
 386	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
 387	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
 388
 389out:
 390	if (err) {
 391		pr_err("gfx6: Failed to load firmware %s gfx firmware\n", chip_name);
 392		amdgpu_ucode_release(&adev->gfx.pfp_fw);
 393		amdgpu_ucode_release(&adev->gfx.me_fw);
 394		amdgpu_ucode_release(&adev->gfx.ce_fw);
 395		amdgpu_ucode_release(&adev->gfx.rlc_fw);
 396	}
 397	return err;
 398}
 399
 400static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 401{
 402	const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
 403	u32 reg_offset, split_equal_to_row_size, *tilemode;
 404
 405	memset(adev->gfx.config.tile_mode_array, 0, sizeof(adev->gfx.config.tile_mode_array));
 406	tilemode = adev->gfx.config.tile_mode_array;
 407
 408	switch (adev->gfx.config.mem_row_size_in_kb) {
 409	case 1:
 410		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
 411		break;
 412	case 2:
 413	default:
 414		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
 415		break;
 416	case 4:
 417		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
 418		break;
 419	}
 420
 421	if (adev->asic_type == CHIP_VERDE) {
 422		tilemode[0] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 423				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 424				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 425				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 426				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 427				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 428				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 429				NUM_BANKS(ADDR_SURF_16_BANK);
 430		tilemode[1] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 431				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 432				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 433				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
 434				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 435				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 436				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 437				NUM_BANKS(ADDR_SURF_16_BANK);
 438		tilemode[2] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 439				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 440				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 441				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 442				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 443				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 444				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 445				NUM_BANKS(ADDR_SURF_16_BANK);
 446		tilemode[3] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 447				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 448				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 449				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 450				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 451				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 452				NUM_BANKS(ADDR_SURF_8_BANK) |
 453				TILE_SPLIT(split_equal_to_row_size);
 454		tilemode[4] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 455				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 456				PIPE_CONFIG(ADDR_SURF_P4_8x16);
 457		tilemode[5] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 458				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 459				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 460				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 461				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 462				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 463				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 464				NUM_BANKS(ADDR_SURF_4_BANK);
 465		tilemode[6] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 466				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 467				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 468				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 469				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 470				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 471				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 472				NUM_BANKS(ADDR_SURF_4_BANK);
 473		tilemode[7] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 474				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 475				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 476				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 477				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 478				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 479				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 480				NUM_BANKS(ADDR_SURF_2_BANK);
 481		tilemode[8] =   ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
 482		tilemode[9] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 483				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 484				PIPE_CONFIG(ADDR_SURF_P4_8x16);
 485		tilemode[10] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 486				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 487				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 488				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 489				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 490				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 491				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 492				NUM_BANKS(ADDR_SURF_16_BANK);
 493		tilemode[11] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 494				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 495				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 496				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 497				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 498				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 499				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 500				NUM_BANKS(ADDR_SURF_16_BANK);
 501		tilemode[12] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 502				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 503				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 504				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 505				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 506				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 507				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 508				NUM_BANKS(ADDR_SURF_16_BANK);
 509		tilemode[13] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 510				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 511				PIPE_CONFIG(ADDR_SURF_P4_8x16);
 512		tilemode[14] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 513				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 514				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 515				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 516				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 517				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 518				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 519				NUM_BANKS(ADDR_SURF_16_BANK);
 520		tilemode[15] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 521				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 522				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 523				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 524				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 525				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 526				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 527				NUM_BANKS(ADDR_SURF_16_BANK);
 528		tilemode[16] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 529				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 530				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 531				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 532				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 533				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 534				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 535				NUM_BANKS(ADDR_SURF_16_BANK);
 536		tilemode[17] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 537				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 538				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 539				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 540				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 541				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 542				NUM_BANKS(ADDR_SURF_16_BANK) |
 543				TILE_SPLIT(split_equal_to_row_size);
 544		tilemode[18] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 545				ARRAY_MODE(ARRAY_1D_TILED_THICK) |
 546				PIPE_CONFIG(ADDR_SURF_P4_8x16);
 547		tilemode[19] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 548				ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
 549				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 550				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 551				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 552				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 553				NUM_BANKS(ADDR_SURF_16_BANK) |
 554				TILE_SPLIT(split_equal_to_row_size);
 555		tilemode[20] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 556				ARRAY_MODE(ARRAY_2D_TILED_THICK) |
 557				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 558				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 559				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 560				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 561				NUM_BANKS(ADDR_SURF_16_BANK) |
 562				TILE_SPLIT(split_equal_to_row_size);
 563		tilemode[21] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 564				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 565				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 566				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 567				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 568				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 569				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 570				NUM_BANKS(ADDR_SURF_8_BANK);
 571		tilemode[22] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 572				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 573				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 574				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 575				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 576				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 577				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 578				NUM_BANKS(ADDR_SURF_8_BANK);
 579		tilemode[23] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 580				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 581				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 582				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 583				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 584				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 585				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 586				NUM_BANKS(ADDR_SURF_4_BANK);
 587		tilemode[24] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 588				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 589				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 590				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 591				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 592				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 593				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 594				NUM_BANKS(ADDR_SURF_4_BANK);
 595		tilemode[25] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 596				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 597				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 598				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 599				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 600				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 601				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 602				NUM_BANKS(ADDR_SURF_2_BANK);
 603		tilemode[26] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 604				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 605				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 606				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 607				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 608				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 609				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 610				NUM_BANKS(ADDR_SURF_2_BANK);
 611		tilemode[27] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 612				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 613				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 614				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 615				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 616				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 617				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 618				NUM_BANKS(ADDR_SURF_2_BANK);
 619		tilemode[28] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 620				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 621				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 622				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 623				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 624				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 625				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 626				NUM_BANKS(ADDR_SURF_2_BANK);
 627		tilemode[29] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 628				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 629				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 630				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 631				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 632				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 633				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 634				NUM_BANKS(ADDR_SURF_2_BANK);
 635		tilemode[30] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 636				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 637				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 638				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
 639				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 640				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 641				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 642				NUM_BANKS(ADDR_SURF_2_BANK);
 643		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
 644			WREG32(mmGB_TILE_MODE0 + reg_offset, tilemode[reg_offset]);
 645	} else if (adev->asic_type == CHIP_OLAND) {
 646		tilemode[0] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 647				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 648				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 649				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 650				NUM_BANKS(ADDR_SURF_16_BANK) |
 651				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 652				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 653				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 654		tilemode[1] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 655				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 656				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 657				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
 658				NUM_BANKS(ADDR_SURF_16_BANK) |
 659				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 660				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 661				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 662		tilemode[2] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 663				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 664				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 665				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 666				NUM_BANKS(ADDR_SURF_16_BANK) |
 667				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 668				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 669				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 670		tilemode[3] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 671				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 672				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 673				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
 674				NUM_BANKS(ADDR_SURF_16_BANK) |
 675				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 676				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 677				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 678		tilemode[4] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 679				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 680				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 681				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 682				NUM_BANKS(ADDR_SURF_16_BANK) |
 683				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 684				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 685				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 686		tilemode[5] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 687				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 688				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 689				TILE_SPLIT(split_equal_to_row_size) |
 690				NUM_BANKS(ADDR_SURF_16_BANK) |
 691				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 692				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 693				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 694		tilemode[6] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 695				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 696				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 697				TILE_SPLIT(split_equal_to_row_size) |
 698				NUM_BANKS(ADDR_SURF_16_BANK) |
 699				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 700				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 701				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 702		tilemode[7] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 703				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 704				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 705				TILE_SPLIT(split_equal_to_row_size) |
 706				NUM_BANKS(ADDR_SURF_16_BANK) |
 707				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 708				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 709				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 710		tilemode[8] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 711				ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
 712				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 713				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 714				NUM_BANKS(ADDR_SURF_16_BANK) |
 715				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 716				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 717				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 718		tilemode[9] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 719				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 720				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 721				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 722				NUM_BANKS(ADDR_SURF_16_BANK) |
 723				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 724				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 725				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 726		tilemode[10] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 727				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 728				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 729				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 730				NUM_BANKS(ADDR_SURF_16_BANK) |
 731				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 732				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 733				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 734		tilemode[11] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 735				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 736				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 737				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 738				NUM_BANKS(ADDR_SURF_16_BANK) |
 739				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 740				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 741				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 742		tilemode[12] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 743				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 744				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 745				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 746				NUM_BANKS(ADDR_SURF_16_BANK) |
 747				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 748				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 749				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 750		tilemode[13] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 751				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 752				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 753				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 754				NUM_BANKS(ADDR_SURF_16_BANK) |
 755				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 756				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 757				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 758		tilemode[14] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 759				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 760				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 761				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 762				NUM_BANKS(ADDR_SURF_16_BANK) |
 763				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 764				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 765				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 766		tilemode[15] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 767				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 768				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 769				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 770				NUM_BANKS(ADDR_SURF_16_BANK) |
 771				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 772				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 773				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 774		tilemode[16] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 775				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 776				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 777				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 778				NUM_BANKS(ADDR_SURF_16_BANK) |
 779				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 780				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 781				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 782		tilemode[17] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 783				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 784				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 785				TILE_SPLIT(split_equal_to_row_size) |
 786				NUM_BANKS(ADDR_SURF_16_BANK) |
 787				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 788				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 789				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 790		tilemode[18] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 791				ARRAY_MODE(ARRAY_1D_TILED_THICK) |
 792				PIPE_CONFIG(ADDR_SURF_P4_8x16);
 793		tilemode[19] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 794				ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
 795				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 796				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 797				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 798				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 799				NUM_BANKS(ADDR_SURF_16_BANK) |
 800				TILE_SPLIT(split_equal_to_row_size);
 801		tilemode[20] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 802				ARRAY_MODE(ARRAY_2D_TILED_THICK) |
 803				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
 804				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 805				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 806				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 807				NUM_BANKS(ADDR_SURF_16_BANK) |
 808				TILE_SPLIT(split_equal_to_row_size);
 809		tilemode[21] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 810				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 811				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
 812				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 813				NUM_BANKS(ADDR_SURF_16_BANK) |
 814				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
 815				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 816				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 817		tilemode[22] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 818				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 819				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
 820				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 821				NUM_BANKS(ADDR_SURF_16_BANK) |
 822				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 823				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 824				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4);
 825		tilemode[23] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 826				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 827				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
 828				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 829				NUM_BANKS(ADDR_SURF_16_BANK) |
 830				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 831				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 832				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 833		tilemode[24] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 834				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 835				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
 836				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 837				NUM_BANKS(ADDR_SURF_16_BANK) |
 838				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 839				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 840				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
 841		tilemode[25] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 842				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 843				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
 844				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 845				NUM_BANKS(ADDR_SURF_8_BANK) |
 846				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 847				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 848				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1);
 849		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
 850			WREG32(mmGB_TILE_MODE0 + reg_offset, tilemode[reg_offset]);
 851	} else if (adev->asic_type == CHIP_HAINAN) {
 852		tilemode[0] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 853				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 854				PIPE_CONFIG(ADDR_SURF_P2) |
 855				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
 856				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 857				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 858				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 859				NUM_BANKS(ADDR_SURF_16_BANK);
 860		tilemode[1] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 861				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 862				PIPE_CONFIG(ADDR_SURF_P2) |
 863				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
 864				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 865				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 866				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 867				NUM_BANKS(ADDR_SURF_16_BANK);
 868		tilemode[2] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 869				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 870				PIPE_CONFIG(ADDR_SURF_P2) |
 871				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 872				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 873				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 874				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 875				NUM_BANKS(ADDR_SURF_16_BANK);
 876		tilemode[3] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 877				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 878				PIPE_CONFIG(ADDR_SURF_P2) |
 879				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 880				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 881				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 882				NUM_BANKS(ADDR_SURF_8_BANK) |
 883				TILE_SPLIT(split_equal_to_row_size);
 884		tilemode[4] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 885				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 886				PIPE_CONFIG(ADDR_SURF_P2);
 887		tilemode[5] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 888				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 889				PIPE_CONFIG(ADDR_SURF_P2) |
 890				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 891				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 892				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 893				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 894				NUM_BANKS(ADDR_SURF_8_BANK);
 895		tilemode[6] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 896				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 897				PIPE_CONFIG(ADDR_SURF_P2) |
 898				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 899				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 900				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 901				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 902				NUM_BANKS(ADDR_SURF_8_BANK);
 903		tilemode[7] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 904				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 905				PIPE_CONFIG(ADDR_SURF_P2) |
 906				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
 907				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 908				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 909				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
 910				NUM_BANKS(ADDR_SURF_4_BANK);
 911		tilemode[8] =   ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
 912		tilemode[9] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 913				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 914				PIPE_CONFIG(ADDR_SURF_P2);
 915		tilemode[10] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 916				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 917				PIPE_CONFIG(ADDR_SURF_P2) |
 918				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 919				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 920				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 921				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
 922				NUM_BANKS(ADDR_SURF_16_BANK);
 923		tilemode[11] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 924				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 925				PIPE_CONFIG(ADDR_SURF_P2) |
 926				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 927				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 928				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 929				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 930				NUM_BANKS(ADDR_SURF_16_BANK);
 931		tilemode[12] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 932				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 933				PIPE_CONFIG(ADDR_SURF_P2) |
 934				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 935				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 936				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 937				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 938				NUM_BANKS(ADDR_SURF_16_BANK);
 939		tilemode[13] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 940				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 941				PIPE_CONFIG(ADDR_SURF_P2);
 942		tilemode[14] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 943				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 944				PIPE_CONFIG(ADDR_SURF_P2) |
 945				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 946				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 947				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 948				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 949				NUM_BANKS(ADDR_SURF_16_BANK);
 950		tilemode[15] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 951				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 952				PIPE_CONFIG(ADDR_SURF_P2) |
 953				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 954				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 955				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 956				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 957				NUM_BANKS(ADDR_SURF_16_BANK);
 958		tilemode[16] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 959				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 960				PIPE_CONFIG(ADDR_SURF_P2) |
 961				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
 962				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 963				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 964				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 965				NUM_BANKS(ADDR_SURF_16_BANK);
 966		tilemode[17] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 967				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 968				PIPE_CONFIG(ADDR_SURF_P2) |
 969				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 970				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 971				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 972				NUM_BANKS(ADDR_SURF_16_BANK) |
 973				TILE_SPLIT(split_equal_to_row_size);
 974		tilemode[18] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 975				ARRAY_MODE(ARRAY_1D_TILED_THICK) |
 976				PIPE_CONFIG(ADDR_SURF_P2);
 977		tilemode[19] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 978				ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
 979				PIPE_CONFIG(ADDR_SURF_P2) |
 980				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 981				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 982				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 983				NUM_BANKS(ADDR_SURF_16_BANK) |
 984				TILE_SPLIT(split_equal_to_row_size);
 985		tilemode[20] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 986				ARRAY_MODE(ARRAY_2D_TILED_THICK) |
 987				PIPE_CONFIG(ADDR_SURF_P2) |
 988				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 989				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 990				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
 991				NUM_BANKS(ADDR_SURF_16_BANK) |
 992				TILE_SPLIT(split_equal_to_row_size);
 993		tilemode[21] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 994				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 995				PIPE_CONFIG(ADDR_SURF_P2) |
 996				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
 997				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
 998				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 999				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1000				NUM_BANKS(ADDR_SURF_8_BANK);
1001		tilemode[22] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1002				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1003				PIPE_CONFIG(ADDR_SURF_P2) |
1004				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1005				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1006				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1007				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1008				NUM_BANKS(ADDR_SURF_8_BANK);
1009		tilemode[23] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1010				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1011				PIPE_CONFIG(ADDR_SURF_P2) |
1012				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1013				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1014				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1015				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1016				NUM_BANKS(ADDR_SURF_8_BANK);
1017		tilemode[24] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1018				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1019				PIPE_CONFIG(ADDR_SURF_P2) |
1020				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1021				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1022				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1023				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1024				NUM_BANKS(ADDR_SURF_8_BANK);
1025		tilemode[25] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1026				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1027				PIPE_CONFIG(ADDR_SURF_P2) |
1028				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1029				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1030				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1031				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1032				NUM_BANKS(ADDR_SURF_4_BANK);
1033		tilemode[26] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1034				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1035				PIPE_CONFIG(ADDR_SURF_P2) |
1036				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1037				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1038				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1039				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1040				NUM_BANKS(ADDR_SURF_4_BANK);
1041		tilemode[27] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1042				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1043				PIPE_CONFIG(ADDR_SURF_P2) |
1044				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1045				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1046				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1047				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1048				NUM_BANKS(ADDR_SURF_4_BANK);
1049		tilemode[28] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1050				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1051				PIPE_CONFIG(ADDR_SURF_P2) |
1052				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1053				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1054				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1055				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1056				NUM_BANKS(ADDR_SURF_4_BANK);
1057		tilemode[29] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1058				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1059				PIPE_CONFIG(ADDR_SURF_P2) |
1060				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1061				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1062				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1063				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1064				NUM_BANKS(ADDR_SURF_4_BANK);
1065		tilemode[30] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1066				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1067				PIPE_CONFIG(ADDR_SURF_P2) |
1068				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
1069				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1070				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1071				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1072				NUM_BANKS(ADDR_SURF_4_BANK);
1073		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1074			WREG32(mmGB_TILE_MODE0 + reg_offset, tilemode[reg_offset]);
1075	} else if ((adev->asic_type == CHIP_TAHITI) || (adev->asic_type == CHIP_PITCAIRN)) {
1076		tilemode[0] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1077				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1078				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1079				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1080				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1081				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1082				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1083				NUM_BANKS(ADDR_SURF_16_BANK);
1084		tilemode[1] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1085				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1086				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1087				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1088				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1089				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1090				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1091				NUM_BANKS(ADDR_SURF_16_BANK);
1092		tilemode[2] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1093				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1094				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1095				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1096				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1097				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1098				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1099				NUM_BANKS(ADDR_SURF_16_BANK);
1100		tilemode[3] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1101				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1102				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1103				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1104				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1105				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1106				NUM_BANKS(ADDR_SURF_4_BANK) |
1107				TILE_SPLIT(split_equal_to_row_size);
1108		tilemode[4] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1109				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1110				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16);
1111		tilemode[5] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1112				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1113				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1114				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1115				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1116				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1117				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1118				NUM_BANKS(ADDR_SURF_2_BANK);
1119		tilemode[6] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1120				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1121				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1122				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1123				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1124				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1125				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1126				NUM_BANKS(ADDR_SURF_2_BANK);
1127		tilemode[7] =   MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1128				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1129				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1130				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1131				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1132				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1133				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1134				NUM_BANKS(ADDR_SURF_2_BANK);
1135		tilemode[8] =   ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
1136		tilemode[9] =   MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1137				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1138				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16);
1139		tilemode[10] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1140				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1141				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1142				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1143				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1144				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1145				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1146				NUM_BANKS(ADDR_SURF_16_BANK);
1147		tilemode[11] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1148				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1149				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1150				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1151				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1152				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1153				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1154				NUM_BANKS(ADDR_SURF_16_BANK);
1155		tilemode[12] =  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1156				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1157				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1158				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1159				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1160				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1161				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1162				NUM_BANKS(ADDR_SURF_16_BANK);
1163		tilemode[13] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1164				ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1165				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16);
1166		tilemode[14] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1167				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1168				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1169				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1170				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1171				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1172				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1173				NUM_BANKS(ADDR_SURF_16_BANK);
1174		tilemode[15] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1175				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1176				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1177				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1178				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1179				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1180				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1181				NUM_BANKS(ADDR_SURF_16_BANK);
1182		tilemode[16] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1183				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1184				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1185				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1186				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1187				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1188				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1189				NUM_BANKS(ADDR_SURF_16_BANK);
1190		tilemode[17] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1191				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1192				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1193				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1194				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1195				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1196				NUM_BANKS(ADDR_SURF_16_BANK) |
1197				TILE_SPLIT(split_equal_to_row_size);
1198		tilemode[18] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1199				ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1200				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16);
1201		tilemode[19] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1202				ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1203				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1204				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1205				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1206				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1207				NUM_BANKS(ADDR_SURF_16_BANK) |
1208				TILE_SPLIT(split_equal_to_row_size);
1209		tilemode[20] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1210				ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1211				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1212				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1213				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1214				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1215				NUM_BANKS(ADDR_SURF_16_BANK) |
1216				TILE_SPLIT(split_equal_to_row_size);
1217		tilemode[21] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1218				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1219				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1220				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1221				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1222				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1223				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1224				NUM_BANKS(ADDR_SURF_4_BANK);
1225		tilemode[22] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1226				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1227				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1228				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1229				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1230				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1231				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1232				NUM_BANKS(ADDR_SURF_4_BANK);
1233		tilemode[23] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1234				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1235				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1236				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1237				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1238				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1239				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1240				NUM_BANKS(ADDR_SURF_2_BANK);
1241		tilemode[24] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1242				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1243				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1244				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1245				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1246				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1247				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1248				NUM_BANKS(ADDR_SURF_2_BANK);
1249		tilemode[25] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1250				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1251				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1252				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1253				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1254				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1255				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1256				NUM_BANKS(ADDR_SURF_2_BANK);
1257		tilemode[26] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1258				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1259				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1260				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1261				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1262				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1263				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1264				NUM_BANKS(ADDR_SURF_2_BANK);
1265		tilemode[27] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1266				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1267				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1268				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1269				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1270				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1271				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1272				NUM_BANKS(ADDR_SURF_2_BANK);
1273		tilemode[28] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1274				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1275				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1276				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1277				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1278				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1279				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1280				NUM_BANKS(ADDR_SURF_2_BANK);
1281		tilemode[29] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1282				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1283				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1284				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1285				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1286				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1287				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1288				NUM_BANKS(ADDR_SURF_2_BANK);
1289		tilemode[30] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1290				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1291				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
1292				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
1293				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1294				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1295				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1296				NUM_BANKS(ADDR_SURF_2_BANK);
1297		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1298			WREG32(mmGB_TILE_MODE0 + reg_offset, tilemode[reg_offset]);
1299	} else {
1300		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
1301	}
1302}
1303
1304static void gfx_v6_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1305				  u32 sh_num, u32 instance, int xcc_id)
1306{
1307	u32 data;
1308
1309	if (instance == 0xffffffff)
1310		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
1311	else
1312		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
1313
1314	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
1315		data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1316			GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
1317	else if (se_num == 0xffffffff)
1318		data |= GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK |
1319			(sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT);
1320	else if (sh_num == 0xffffffff)
1321		data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1322			(se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1323	else
1324		data |= (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT) |
1325			(se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1326	WREG32(mmGRBM_GFX_INDEX, data);
1327}
1328
1329static u32 gfx_v6_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1330{
1331	u32 data, mask;
1332
1333	data = RREG32(mmCC_RB_BACKEND_DISABLE) |
1334		RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1335
1336	data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
1337
1338	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se/
1339					 adev->gfx.config.max_sh_per_se);
1340
1341	return ~data & mask;
1342}
1343
1344static void gfx_v6_0_raster_config(struct amdgpu_device *adev, u32 *rconf)
1345{
1346	switch (adev->asic_type) {
1347	case CHIP_TAHITI:
1348	case CHIP_PITCAIRN:
1349		*rconf |=
1350			   (2 << PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT) |
1351			   (1 << PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT) |
1352			   (2 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT) |
1353			   (1 << PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT) |
1354			   (2 << PA_SC_RASTER_CONFIG__SE_MAP__SHIFT) |
1355			   (2 << PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT) |
1356			   (2 << PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT);
1357		break;
1358	case CHIP_VERDE:
1359		*rconf |=
1360			   (1 << PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT) |
1361			   (2 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT) |
1362			   (1 << PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT);
1363		break;
1364	case CHIP_OLAND:
1365		*rconf |= (1 << PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT);
1366		break;
1367	case CHIP_HAINAN:
1368		*rconf |= 0x0;
1369		break;
1370	default:
1371		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
1372		break;
1373	}
1374}
1375
1376static void gfx_v6_0_write_harvested_raster_configs(struct amdgpu_device *adev,
1377						    u32 raster_config, unsigned rb_mask,
1378						    unsigned num_rb)
1379{
1380	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
1381	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
1382	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
1383	unsigned rb_per_se = num_rb / num_se;
1384	unsigned se_mask[4];
1385	unsigned se;
1386
1387	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
1388	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
1389	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
1390	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
1391
1392	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
1393	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
1394	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
1395
1396	for (se = 0; se < num_se; se++) {
1397		unsigned raster_config_se = raster_config;
1398		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
1399		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
1400		int idx = (se / 2) * 2;
1401
1402		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
1403			raster_config_se &= ~PA_SC_RASTER_CONFIG__SE_MAP_MASK;
1404
1405			if (!se_mask[idx])
1406				raster_config_se |= RASTER_CONFIG_SE_MAP_3 << PA_SC_RASTER_CONFIG__SE_MAP__SHIFT;
1407			else
1408				raster_config_se |= RASTER_CONFIG_SE_MAP_0 << PA_SC_RASTER_CONFIG__SE_MAP__SHIFT;
1409		}
1410
1411		pkr0_mask &= rb_mask;
1412		pkr1_mask &= rb_mask;
1413		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
1414			raster_config_se &= ~PA_SC_RASTER_CONFIG__PKR_MAP_MASK;
1415
1416			if (!pkr0_mask)
1417				raster_config_se |= RASTER_CONFIG_PKR_MAP_3 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT;
1418			else
1419				raster_config_se |= RASTER_CONFIG_PKR_MAP_0 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT;
1420		}
1421
1422		if (rb_per_se >= 2) {
1423			unsigned rb0_mask = 1 << (se * rb_per_se);
1424			unsigned rb1_mask = rb0_mask << 1;
1425
1426			rb0_mask &= rb_mask;
1427			rb1_mask &= rb_mask;
1428			if (!rb0_mask || !rb1_mask) {
1429				raster_config_se &= ~PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK;
1430
1431				if (!rb0_mask)
1432					raster_config_se |=
1433						RASTER_CONFIG_RB_MAP_3 << PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT;
1434				else
1435					raster_config_se |=
1436						RASTER_CONFIG_RB_MAP_0 << PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT;
1437			}
1438
1439			if (rb_per_se > 2) {
1440				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
1441				rb1_mask = rb0_mask << 1;
1442				rb0_mask &= rb_mask;
1443				rb1_mask &= rb_mask;
1444				if (!rb0_mask || !rb1_mask) {
1445					raster_config_se &= ~PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK;
1446
1447					if (!rb0_mask)
1448						raster_config_se |=
1449							RASTER_CONFIG_RB_MAP_3 << PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT;
1450					else
1451						raster_config_se |=
1452							RASTER_CONFIG_RB_MAP_0 << PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT;
1453				}
1454			}
1455		}
1456
1457		/* GRBM_GFX_INDEX has a different offset on SI */
1458		gfx_v6_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0);
1459		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
1460	}
1461
1462	/* GRBM_GFX_INDEX has a different offset on SI */
1463	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1464}
1465
1466static void gfx_v6_0_setup_rb(struct amdgpu_device *adev)
1467{
1468	int i, j;
1469	u32 data;
1470	u32 raster_config = 0;
1471	u32 active_rbs = 0;
1472	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1473					adev->gfx.config.max_sh_per_se;
1474	unsigned num_rb_pipes;
1475
1476	mutex_lock(&adev->grbm_idx_mutex);
1477	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1478		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1479			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff, 0);
1480			data = gfx_v6_0_get_rb_active_bitmap(adev);
1481			active_rbs |= data <<
1482				((i * adev->gfx.config.max_sh_per_se + j) *
1483				 rb_bitmap_width_per_sh);
1484		}
1485	}
1486	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1487
1488	adev->gfx.config.backend_enable_mask = active_rbs;
1489	adev->gfx.config.num_rbs = hweight32(active_rbs);
1490
1491	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
1492			     adev->gfx.config.max_shader_engines, 16);
1493
1494	gfx_v6_0_raster_config(adev, &raster_config);
1495
1496	if (!adev->gfx.config.backend_enable_mask ||
1497	     adev->gfx.config.num_rbs >= num_rb_pipes)
1498		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
1499	else
1500		gfx_v6_0_write_harvested_raster_configs(adev, raster_config,
1501							adev->gfx.config.backend_enable_mask,
1502							num_rb_pipes);
1503
1504	/* cache the values for userspace */
1505	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1506		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1507			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff, 0);
1508			adev->gfx.config.rb_config[i][j].rb_backend_disable =
1509				RREG32(mmCC_RB_BACKEND_DISABLE);
1510			adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
1511				RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1512			adev->gfx.config.rb_config[i][j].raster_config =
1513				RREG32(mmPA_SC_RASTER_CONFIG);
1514		}
1515	}
1516	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1517	mutex_unlock(&adev->grbm_idx_mutex);
1518}
1519
1520static void gfx_v6_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
1521						 u32 bitmap)
1522{
1523	u32 data;
1524
1525	if (!bitmap)
1526		return;
1527
1528	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
1529	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
1530
1531	WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
1532}
1533
1534static u32 gfx_v6_0_get_cu_enabled(struct amdgpu_device *adev)
1535{
1536	u32 data, mask;
1537
1538	data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
1539		RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
1540
1541	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
1542	return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
1543}
1544
1545
1546static void gfx_v6_0_setup_spi(struct amdgpu_device *adev)
1547{
1548	int i, j, k;
1549	u32 data, mask;
1550	u32 active_cu = 0;
1551
1552	mutex_lock(&adev->grbm_idx_mutex);
1553	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1554		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1555			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff, 0);
1556			data = RREG32(mmSPI_STATIC_THREAD_MGMT_3);
1557			active_cu = gfx_v6_0_get_cu_enabled(adev);
1558
1559			mask = 1;
1560			for (k = 0; k < 16; k++) {
1561				mask <<= k;
1562				if (active_cu & mask) {
1563					data &= ~mask;
1564					WREG32(mmSPI_STATIC_THREAD_MGMT_3, data);
1565					break;
1566				}
1567			}
1568		}
1569	}
1570	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1571	mutex_unlock(&adev->grbm_idx_mutex);
1572}
1573
1574static void gfx_v6_0_config_init(struct amdgpu_device *adev)
1575{
1576	adev->gfx.config.double_offchip_lds_buf = 0;
1577}
1578
1579static void gfx_v6_0_constants_init(struct amdgpu_device *adev)
1580{
1581	u32 gb_addr_config = 0;
1582	u32 mc_arb_ramcfg;
1583	u32 sx_debug_1;
1584	u32 hdp_host_path_cntl;
1585	u32 tmp;
1586
1587	switch (adev->asic_type) {
1588	case CHIP_TAHITI:
1589		adev->gfx.config.max_shader_engines = 2;
1590		adev->gfx.config.max_tile_pipes = 12;
1591		adev->gfx.config.max_cu_per_sh = 8;
1592		adev->gfx.config.max_sh_per_se = 2;
1593		adev->gfx.config.max_backends_per_se = 4;
1594		adev->gfx.config.max_texture_channel_caches = 12;
1595		adev->gfx.config.max_gprs = 256;
1596		adev->gfx.config.max_gs_threads = 32;
1597		adev->gfx.config.max_hw_contexts = 8;
1598
1599		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1600		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1601		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1602		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1603		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
1604		break;
1605	case CHIP_PITCAIRN:
1606		adev->gfx.config.max_shader_engines = 2;
1607		adev->gfx.config.max_tile_pipes = 8;
1608		adev->gfx.config.max_cu_per_sh = 5;
1609		adev->gfx.config.max_sh_per_se = 2;
1610		adev->gfx.config.max_backends_per_se = 4;
1611		adev->gfx.config.max_texture_channel_caches = 8;
1612		adev->gfx.config.max_gprs = 256;
1613		adev->gfx.config.max_gs_threads = 32;
1614		adev->gfx.config.max_hw_contexts = 8;
1615
1616		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1617		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1618		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1619		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1620		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
1621		break;
1622	case CHIP_VERDE:
1623		adev->gfx.config.max_shader_engines = 1;
1624		adev->gfx.config.max_tile_pipes = 4;
1625		adev->gfx.config.max_cu_per_sh = 5;
1626		adev->gfx.config.max_sh_per_se = 2;
1627		adev->gfx.config.max_backends_per_se = 4;
1628		adev->gfx.config.max_texture_channel_caches = 4;
1629		adev->gfx.config.max_gprs = 256;
1630		adev->gfx.config.max_gs_threads = 32;
1631		adev->gfx.config.max_hw_contexts = 8;
1632
1633		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1634		adev->gfx.config.sc_prim_fifo_size_backend = 0x40;
1635		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1636		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1637		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
1638		break;
1639	case CHIP_OLAND:
1640		adev->gfx.config.max_shader_engines = 1;
1641		adev->gfx.config.max_tile_pipes = 4;
1642		adev->gfx.config.max_cu_per_sh = 6;
1643		adev->gfx.config.max_sh_per_se = 1;
1644		adev->gfx.config.max_backends_per_se = 2;
1645		adev->gfx.config.max_texture_channel_caches = 4;
1646		adev->gfx.config.max_gprs = 256;
1647		adev->gfx.config.max_gs_threads = 16;
1648		adev->gfx.config.max_hw_contexts = 8;
1649
1650		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1651		adev->gfx.config.sc_prim_fifo_size_backend = 0x40;
1652		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1653		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1654		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
1655		break;
1656	case CHIP_HAINAN:
1657		adev->gfx.config.max_shader_engines = 1;
1658		adev->gfx.config.max_tile_pipes = 4;
1659		adev->gfx.config.max_cu_per_sh = 5;
1660		adev->gfx.config.max_sh_per_se = 1;
1661		adev->gfx.config.max_backends_per_se = 1;
1662		adev->gfx.config.max_texture_channel_caches = 2;
1663		adev->gfx.config.max_gprs = 256;
1664		adev->gfx.config.max_gs_threads = 16;
1665		adev->gfx.config.max_hw_contexts = 8;
1666
1667		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1668		adev->gfx.config.sc_prim_fifo_size_backend = 0x40;
1669		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1670		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1671		gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
1672		break;
1673	default:
1674		BUG();
1675		break;
1676	}
1677
1678	WREG32(mmGRBM_CNTL, (0xff << GRBM_CNTL__READ_TIMEOUT__SHIFT));
1679	WREG32(mmSRBM_INT_CNTL, 1);
1680	WREG32(mmSRBM_INT_ACK, 1);
1681
1682	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
1683
1684	adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
1685	mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
1686
1687	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
1688	adev->gfx.config.mem_max_burst_length_bytes = 256;
1689	tmp = (mc_arb_ramcfg & MC_ARB_RAMCFG__NOOFCOLS_MASK) >> MC_ARB_RAMCFG__NOOFCOLS__SHIFT;
1690	adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1691	if (adev->gfx.config.mem_row_size_in_kb > 4)
1692		adev->gfx.config.mem_row_size_in_kb = 4;
1693	adev->gfx.config.shader_engine_tile_size = 32;
1694	adev->gfx.config.num_gpus = 1;
1695	adev->gfx.config.multi_gpu_tile_size = 64;
1696
1697	gb_addr_config &= ~GB_ADDR_CONFIG__ROW_SIZE_MASK;
1698	switch (adev->gfx.config.mem_row_size_in_kb) {
1699	case 1:
1700	default:
1701		gb_addr_config |= 0 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT;
1702		break;
1703	case 2:
1704		gb_addr_config |= 1 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT;
1705		break;
1706	case 4:
1707		gb_addr_config |= 2 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT;
1708		break;
1709	}
1710	gb_addr_config &= ~GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK;
1711	if (adev->gfx.config.max_shader_engines == 2)
1712		gb_addr_config |= 1 << GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT;
1713	adev->gfx.config.gb_addr_config = gb_addr_config;
1714
1715	WREG32(mmGB_ADDR_CONFIG, gb_addr_config);
1716	WREG32(mmDMIF_ADDR_CONFIG, gb_addr_config);
1717	WREG32(mmDMIF_ADDR_CALC, gb_addr_config);
1718	WREG32(mmHDP_ADDR_CONFIG, gb_addr_config);
1719	WREG32(mmDMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1720	WREG32(mmDMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1721
1722#if 0
1723	if (adev->has_uvd) {
1724		WREG32(mmUVD_UDEC_ADDR_CONFIG, gb_addr_config);
1725		WREG32(mmUVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1726		WREG32(mmUVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1727	}
1728#endif
1729	gfx_v6_0_tiling_mode_table_init(adev);
1730
1731	gfx_v6_0_setup_rb(adev);
1732
1733	gfx_v6_0_setup_spi(adev);
1734
1735	gfx_v6_0_get_cu_info(adev);
1736	gfx_v6_0_config_init(adev);
1737
1738	WREG32(mmCP_QUEUE_THRESHOLDS,
1739		((0x16 << CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT) |
1740		(0x2b << CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT)));
1741
1742	/* set HW defaults for 3D engine */
1743	WREG32(mmCP_MEQ_THRESHOLDS,
1744		(0x30 << CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT) |
1745		(0x60 << CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT));
1746
1747	sx_debug_1 = RREG32(mmSX_DEBUG_1);
1748	WREG32(mmSX_DEBUG_1, sx_debug_1);
1749
1750	WREG32(mmSPI_CONFIG_CNTL_1, (4 << SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT));
1751
1752	WREG32(mmPA_SC_FIFO_SIZE, ((adev->gfx.config.sc_prim_fifo_size_frontend << PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1753				   (adev->gfx.config.sc_prim_fifo_size_backend << PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
1754				   (adev->gfx.config.sc_hiz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
1755				   (adev->gfx.config.sc_earlyz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)));
1756
1757	WREG32(mmVGT_NUM_INSTANCES, 1);
1758	WREG32(mmCP_PERFMON_CNTL, 0);
1759	WREG32(mmSQ_CONFIG, 0);
1760	WREG32(mmPA_SC_FORCE_EOV_MAX_CNTS, ((4095 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT) |
1761					  (255 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT)));
1762
1763	WREG32(mmVGT_CACHE_INVALIDATION,
1764		(VC_AND_TC << VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT) |
1765		(ES_AND_GS_AUTO << VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT));
1766
1767	WREG32(mmVGT_GS_VERTEX_REUSE, 16);
1768	WREG32(mmPA_SC_LINE_STIPPLE_STATE, 0);
1769
1770	WREG32(mmCB_PERFCOUNTER0_SELECT0, 0);
1771	WREG32(mmCB_PERFCOUNTER0_SELECT1, 0);
1772	WREG32(mmCB_PERFCOUNTER1_SELECT0, 0);
1773	WREG32(mmCB_PERFCOUNTER1_SELECT1, 0);
1774	WREG32(mmCB_PERFCOUNTER2_SELECT0, 0);
1775	WREG32(mmCB_PERFCOUNTER2_SELECT1, 0);
1776	WREG32(mmCB_PERFCOUNTER3_SELECT0, 0);
1777	WREG32(mmCB_PERFCOUNTER3_SELECT1, 0);
1778
1779	hdp_host_path_cntl = RREG32(mmHDP_HOST_PATH_CNTL);
1780	WREG32(mmHDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1781
1782	WREG32(mmPA_CL_ENHANCE, PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK |
1783				(3 << PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT));
1784
1785	udelay(50);
1786}
1787
1788static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring)
1789{
1790	struct amdgpu_device *adev = ring->adev;
1791	uint32_t tmp = 0;
1792	unsigned i;
1793	int r;
1794
1795	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
1796
1797	r = amdgpu_ring_alloc(ring, 3);
1798	if (r)
1799		return r;
1800
1801	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1802	amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_CONFIG_REG_START);
1803	amdgpu_ring_write(ring, 0xDEADBEEF);
1804	amdgpu_ring_commit(ring);
1805
1806	for (i = 0; i < adev->usec_timeout; i++) {
1807		tmp = RREG32(mmSCRATCH_REG0);
1808		if (tmp == 0xDEADBEEF)
1809			break;
1810		udelay(1);
1811	}
1812
1813	if (i >= adev->usec_timeout)
1814		r = -ETIMEDOUT;
1815	return r;
1816}
1817
1818static void gfx_v6_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
1819{
1820	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
1821	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
1822		EVENT_INDEX(0));
1823}
1824
1825static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1826				     u64 seq, unsigned flags)
1827{
1828	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
1829	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
1830	/* flush read cache over gart */
1831	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1832	amdgpu_ring_write(ring, (mmCP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START));
1833	amdgpu_ring_write(ring, 0);
1834	amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1835	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
1836			  PACKET3_TC_ACTION_ENA |
1837			  PACKET3_SH_KCACHE_ACTION_ENA |
1838			  PACKET3_SH_ICACHE_ACTION_ENA);
1839	amdgpu_ring_write(ring, 0xFFFFFFFF);
1840	amdgpu_ring_write(ring, 0);
1841	amdgpu_ring_write(ring, 10); /* poll interval */
1842	/* EVENT_WRITE_EOP - flush caches, send int */
1843	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1844	amdgpu_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
1845	amdgpu_ring_write(ring, addr & 0xfffffffc);
1846	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
1847				((write64bit ? 2 : 1) << CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT) |
1848				((int_sel ? 2 : 0) << CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT));
1849	amdgpu_ring_write(ring, lower_32_bits(seq));
1850	amdgpu_ring_write(ring, upper_32_bits(seq));
1851}
1852
1853static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
1854				  struct amdgpu_job *job,
1855				  struct amdgpu_ib *ib,
1856				  uint32_t flags)
1857{
1858	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1859	u32 header, control = 0;
1860
1861	/* insert SWITCH_BUFFER packet before first IB in the ring frame */
1862	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
1863		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
1864		amdgpu_ring_write(ring, 0);
1865	}
1866
1867	if (ib->flags & AMDGPU_IB_FLAG_CE)
1868		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
1869	else
1870		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
1871
1872	control |= ib->length_dw | (vmid << 24);
1873
1874	amdgpu_ring_write(ring, header);
1875	amdgpu_ring_write(ring,
1876#ifdef __BIG_ENDIAN
1877			  (2 << 0) |
1878#endif
1879			  (ib->gpu_addr & 0xFFFFFFFC));
1880	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
1881	amdgpu_ring_write(ring, control);
1882}
1883
1884/**
1885 * gfx_v6_0_ring_test_ib - basic ring IB test
1886 *
1887 * @ring: amdgpu_ring structure holding ring information
1888 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1889 *
1890 * Allocate an IB and execute it on the gfx ring (SI).
1891 * Provides a basic gfx ring test to verify that IBs are working.
1892 * Returns 0 on success, error on failure.
1893 */
1894static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1895{
1896	struct amdgpu_device *adev = ring->adev;
1897	struct dma_fence *f = NULL;
1898	struct amdgpu_ib ib;
1899	uint32_t tmp = 0;
1900	long r;
1901
1902	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
1903	memset(&ib, 0, sizeof(ib));
1904	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
1905	if (r)
1906		return r;
1907
1908	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
1909	ib.ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_CONFIG_REG_START;
1910	ib.ptr[2] = 0xDEADBEEF;
1911	ib.length_dw = 3;
1912
1913	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1914	if (r)
1915		goto error;
1916
1917	r = dma_fence_wait_timeout(f, false, timeout);
1918	if (r == 0) {
1919		r = -ETIMEDOUT;
1920		goto error;
1921	} else if (r < 0) {
1922		goto error;
1923	}
1924	tmp = RREG32(mmSCRATCH_REG0);
1925	if (tmp == 0xDEADBEEF)
1926		r = 0;
1927	else
1928		r = -EINVAL;
1929
1930error:
1931	amdgpu_ib_free(&ib, NULL);
1932	dma_fence_put(f);
1933	return r;
1934}
1935
1936static void gfx_v6_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
1937{
1938	if (enable) {
1939		WREG32(mmCP_ME_CNTL, 0);
1940	} else {
1941		WREG32(mmCP_ME_CNTL, (CP_ME_CNTL__ME_HALT_MASK |
1942				      CP_ME_CNTL__PFP_HALT_MASK |
1943				      CP_ME_CNTL__CE_HALT_MASK));
1944		WREG32(mmSCRATCH_UMSK, 0);
1945	}
1946	udelay(50);
1947}
1948
1949static int gfx_v6_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
1950{
1951	unsigned i;
1952	const struct gfx_firmware_header_v1_0 *pfp_hdr;
1953	const struct gfx_firmware_header_v1_0 *ce_hdr;
1954	const struct gfx_firmware_header_v1_0 *me_hdr;
1955	const __le32 *fw_data;
1956	u32 fw_size;
1957
1958	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
1959		return -EINVAL;
1960
1961	gfx_v6_0_cp_gfx_enable(adev, false);
1962	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1963	ce_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1964	me_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1965
1966	amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
1967	amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
1968	amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
1969
1970	/* PFP */
1971	fw_data = (const __le32 *)
1972		(adev->gfx.pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
1973	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
1974	WREG32(mmCP_PFP_UCODE_ADDR, 0);
1975	for (i = 0; i < fw_size; i++)
1976		WREG32(mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
1977	WREG32(mmCP_PFP_UCODE_ADDR, 0);
1978
1979	/* CE */
1980	fw_data = (const __le32 *)
1981		(adev->gfx.ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
1982	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
1983	WREG32(mmCP_CE_UCODE_ADDR, 0);
1984	for (i = 0; i < fw_size; i++)
1985		WREG32(mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
1986	WREG32(mmCP_CE_UCODE_ADDR, 0);
1987
1988	/* ME */
1989	fw_data = (const __be32 *)
1990		(adev->gfx.me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
1991	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
1992	WREG32(mmCP_ME_RAM_WADDR, 0);
1993	for (i = 0; i < fw_size; i++)
1994		WREG32(mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
1995	WREG32(mmCP_ME_RAM_WADDR, 0);
1996
1997	WREG32(mmCP_PFP_UCODE_ADDR, 0);
1998	WREG32(mmCP_CE_UCODE_ADDR, 0);
1999	WREG32(mmCP_ME_RAM_WADDR, 0);
2000	WREG32(mmCP_ME_RAM_RADDR, 0);
2001	return 0;
2002}
2003
2004static int gfx_v6_0_cp_gfx_start(struct amdgpu_device *adev)
2005{
2006	const struct cs_section_def *sect = NULL;
2007	const struct cs_extent_def *ext = NULL;
2008	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
2009	int r, i;
2010
2011	r = amdgpu_ring_alloc(ring, 7 + 4);
2012	if (r) {
2013		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2014		return r;
2015	}
2016	amdgpu_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2017	amdgpu_ring_write(ring, 0x1);
2018	amdgpu_ring_write(ring, 0x0);
2019	amdgpu_ring_write(ring, adev->gfx.config.max_hw_contexts - 1);
2020	amdgpu_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2021	amdgpu_ring_write(ring, 0);
2022	amdgpu_ring_write(ring, 0);
2023
2024	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2025	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2026	amdgpu_ring_write(ring, 0xc000);
2027	amdgpu_ring_write(ring, 0xe000);
2028	amdgpu_ring_commit(ring);
2029
2030	gfx_v6_0_cp_gfx_enable(adev, true);
2031
2032	r = amdgpu_ring_alloc(ring, gfx_v6_0_get_csb_size(adev) + 10);
2033	if (r) {
2034		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2035		return r;
2036	}
2037
2038	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2039	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2040
2041	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2042		for (ext = sect->section; ext->extent != NULL; ++ext) {
2043			if (sect->id == SECT_CONTEXT) {
2044				amdgpu_ring_write(ring,
2045						  PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2046				amdgpu_ring_write(ring, ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2047				for (i = 0; i < ext->reg_count; i++)
2048					amdgpu_ring_write(ring, ext->extent[i]);
2049			}
2050		}
2051	}
2052
2053	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2054	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2055
2056	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2057	amdgpu_ring_write(ring, 0);
2058
2059	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2060	amdgpu_ring_write(ring, 0x00000316);
2061	amdgpu_ring_write(ring, 0x0000000e);
2062	amdgpu_ring_write(ring, 0x00000010);
2063
2064	amdgpu_ring_commit(ring);
2065
2066	return 0;
2067}
2068
2069static int gfx_v6_0_cp_gfx_resume(struct amdgpu_device *adev)
2070{
2071	struct amdgpu_ring *ring;
2072	u32 tmp;
2073	u32 rb_bufsz;
2074	int r;
2075	u64 rptr_addr;
2076
2077	WREG32(mmCP_SEM_WAIT_TIMER, 0x0);
2078	WREG32(mmCP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2079
2080	/* Set the write pointer delay */
2081	WREG32(mmCP_RB_WPTR_DELAY, 0);
2082
2083	WREG32(mmCP_DEBUG, 0);
2084	WREG32(mmSCRATCH_ADDR, 0);
2085
2086	/* ring 0 - compute and gfx */
2087	/* Set ring buffer size */
2088	ring = &adev->gfx.gfx_ring[0];
2089	rb_bufsz = order_base_2(ring->ring_size / 8);
2090	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2091
2092#ifdef __BIG_ENDIAN
2093	tmp |= BUF_SWAP_32BIT;
2094#endif
2095	WREG32(mmCP_RB0_CNTL, tmp);
2096
2097	/* Initialize the ring buffer's read and write pointers */
2098	WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
2099	ring->wptr = 0;
2100	WREG32(mmCP_RB0_WPTR, ring->wptr);
2101
2102	/* set the wb address whether it's enabled or not */
2103	rptr_addr = ring->rptr_gpu_addr;
2104	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2105	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
2106
2107	WREG32(mmSCRATCH_UMSK, 0);
2108
2109	mdelay(1);
2110	WREG32(mmCP_RB0_CNTL, tmp);
2111
2112	WREG32(mmCP_RB0_BASE, ring->gpu_addr >> 8);
2113
2114	/* start the rings */
2115	gfx_v6_0_cp_gfx_start(adev);
2116	r = amdgpu_ring_test_helper(ring);
2117	if (r)
2118		return r;
2119
2120	return 0;
2121}
2122
2123static u64 gfx_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
2124{
2125	return *ring->rptr_cpu_addr;
2126}
2127
2128static u64 gfx_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
2129{
2130	struct amdgpu_device *adev = ring->adev;
2131
2132	if (ring == &adev->gfx.gfx_ring[0])
2133		return RREG32(mmCP_RB0_WPTR);
2134	else if (ring == &adev->gfx.compute_ring[0])
2135		return RREG32(mmCP_RB1_WPTR);
2136	else if (ring == &adev->gfx.compute_ring[1])
2137		return RREG32(mmCP_RB2_WPTR);
2138	else
2139		BUG();
2140}
2141
2142static void gfx_v6_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
2143{
2144	struct amdgpu_device *adev = ring->adev;
2145
2146	WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2147	(void)RREG32(mmCP_RB0_WPTR);
2148}
2149
2150static void gfx_v6_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
2151{
2152	struct amdgpu_device *adev = ring->adev;
2153
2154	if (ring == &adev->gfx.compute_ring[0]) {
2155		WREG32(mmCP_RB1_WPTR, lower_32_bits(ring->wptr));
2156		(void)RREG32(mmCP_RB1_WPTR);
2157	} else if (ring == &adev->gfx.compute_ring[1]) {
2158		WREG32(mmCP_RB2_WPTR, lower_32_bits(ring->wptr));
2159		(void)RREG32(mmCP_RB2_WPTR);
2160	} else {
2161		BUG();
2162	}
2163
2164}
2165
2166static int gfx_v6_0_cp_compute_resume(struct amdgpu_device *adev)
2167{
2168	struct amdgpu_ring *ring;
2169	u32 tmp;
2170	u32 rb_bufsz;
2171	int i, r;
2172	u64 rptr_addr;
2173
2174	/* ring1  - compute only */
2175	/* Set ring buffer size */
2176
2177	ring = &adev->gfx.compute_ring[0];
2178	rb_bufsz = order_base_2(ring->ring_size / 8);
2179	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2180#ifdef __BIG_ENDIAN
2181	tmp |= BUF_SWAP_32BIT;
2182#endif
2183	WREG32(mmCP_RB1_CNTL, tmp);
2184
2185	WREG32(mmCP_RB1_CNTL, tmp | CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK);
2186	ring->wptr = 0;
2187	WREG32(mmCP_RB1_WPTR, ring->wptr);
2188
2189	rptr_addr = ring->rptr_gpu_addr;
2190	WREG32(mmCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
2191	WREG32(mmCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
2192
2193	mdelay(1);
2194	WREG32(mmCP_RB1_CNTL, tmp);
2195	WREG32(mmCP_RB1_BASE, ring->gpu_addr >> 8);
2196
2197	ring = &adev->gfx.compute_ring[1];
2198	rb_bufsz = order_base_2(ring->ring_size / 8);
2199	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2200#ifdef __BIG_ENDIAN
2201	tmp |= BUF_SWAP_32BIT;
2202#endif
2203	WREG32(mmCP_RB2_CNTL, tmp);
2204
2205	WREG32(mmCP_RB2_CNTL, tmp | CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK);
2206	ring->wptr = 0;
2207	WREG32(mmCP_RB2_WPTR, ring->wptr);
2208	rptr_addr = ring->rptr_gpu_addr;
2209	WREG32(mmCP_RB2_RPTR_ADDR, lower_32_bits(rptr_addr));
2210	WREG32(mmCP_RB2_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
2211
2212	mdelay(1);
2213	WREG32(mmCP_RB2_CNTL, tmp);
2214	WREG32(mmCP_RB2_BASE, ring->gpu_addr >> 8);
2215
2216
2217	for (i = 0; i < 2; i++) {
2218		r = amdgpu_ring_test_helper(&adev->gfx.compute_ring[i]);
2219		if (r)
2220			return r;
2221	}
2222
2223	return 0;
2224}
2225
2226static void gfx_v6_0_cp_enable(struct amdgpu_device *adev, bool enable)
2227{
2228	gfx_v6_0_cp_gfx_enable(adev, enable);
2229}
2230
2231static int gfx_v6_0_cp_load_microcode(struct amdgpu_device *adev)
2232{
2233	return gfx_v6_0_cp_gfx_load_microcode(adev);
2234}
2235
2236static void gfx_v6_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2237					       bool enable)
2238{
2239	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
2240	u32 mask;
2241	int i;
2242
2243	if (enable)
2244		tmp |= (CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK |
2245			CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK);
2246	else
2247		tmp &= ~(CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK |
2248			 CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK);
2249	WREG32(mmCP_INT_CNTL_RING0, tmp);
2250
2251	if (!enable) {
2252		/* read a gfx register */
2253		tmp = RREG32(mmDB_DEPTH_INFO);
2254
2255		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
2256		for (i = 0; i < adev->usec_timeout; i++) {
2257			if ((RREG32(mmRLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
2258				break;
2259			udelay(1);
2260		}
2261	}
2262}
2263
2264static int gfx_v6_0_cp_resume(struct amdgpu_device *adev)
2265{
2266	int r;
2267
2268	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
2269
2270	r = gfx_v6_0_cp_load_microcode(adev);
2271	if (r)
2272		return r;
2273
2274	r = gfx_v6_0_cp_gfx_resume(adev);
2275	if (r)
2276		return r;
2277	r = gfx_v6_0_cp_compute_resume(adev);
2278	if (r)
2279		return r;
2280
2281	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
2282
2283	return 0;
2284}
2285
2286static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
2287{
2288	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
2289	uint32_t seq = ring->fence_drv.sync_seq;
2290	uint64_t addr = ring->fence_drv.gpu_addr;
2291
2292	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2293	amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
2294				 WAIT_REG_MEM_FUNCTION(3) | /* equal */
2295				 WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
2296	amdgpu_ring_write(ring, addr & 0xfffffffc);
2297	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
2298	amdgpu_ring_write(ring, seq);
2299	amdgpu_ring_write(ring, 0xffffffff);
2300	amdgpu_ring_write(ring, 4); /* poll interval */
2301
2302	if (usepfp) {
2303		/* synce CE with ME to prevent CE fetch CEIB before context switch done */
2304		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2305		amdgpu_ring_write(ring, 0);
2306		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2307		amdgpu_ring_write(ring, 0);
2308	}
2309}
2310
2311static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
2312					unsigned vmid, uint64_t pd_addr)
2313{
2314	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
2315
2316	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
2317
2318	/* wait for the invalidate to complete */
2319	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2320	amdgpu_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2321				 WAIT_REG_MEM_ENGINE(0))); /* me */
2322	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
2323	amdgpu_ring_write(ring, 0);
2324	amdgpu_ring_write(ring, 0); /* ref */
2325	amdgpu_ring_write(ring, 0); /* mask */
2326	amdgpu_ring_write(ring, 0x20); /* poll interval */
2327
2328	if (usepfp) {
2329		/* sync PFP to ME, otherwise we might get invalid PFP reads */
2330		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2331		amdgpu_ring_write(ring, 0x0);
2332
2333		/* synce CE with ME to prevent CE fetch CEIB before context switch done */
2334		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2335		amdgpu_ring_write(ring, 0);
2336		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2337		amdgpu_ring_write(ring, 0);
2338	}
2339}
2340
2341static void gfx_v6_0_ring_emit_wreg(struct amdgpu_ring *ring,
2342				    uint32_t reg, uint32_t val)
2343{
2344	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
2345
2346	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2347	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
2348				 WRITE_DATA_DST_SEL(0)));
2349	amdgpu_ring_write(ring, reg);
2350	amdgpu_ring_write(ring, 0);
2351	amdgpu_ring_write(ring, val);
2352}
2353
2354static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
2355{
2356	const u32 *src_ptr;
2357	u32 *dst_ptr;
2358	u32 dws;
2359	u64 reg_list_mc_addr;
2360	const struct cs_section_def *cs_data;
2361	int r;
2362
2363	adev->gfx.rlc.reg_list = verde_rlc_save_restore_register_list;
2364	adev->gfx.rlc.reg_list_size =
2365			(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
2366
2367	adev->gfx.rlc.cs_data = si_cs_data;
2368	src_ptr = adev->gfx.rlc.reg_list;
2369	dws = adev->gfx.rlc.reg_list_size;
2370	cs_data = adev->gfx.rlc.cs_data;
2371
2372	if (src_ptr) {
2373		/* init save restore block */
2374		r = amdgpu_gfx_rlc_init_sr(adev, dws);
2375		if (r)
2376			return r;
2377	}
2378
2379	if (cs_data) {
2380		/* clear state block */
2381		adev->gfx.rlc.clear_state_size = gfx_v6_0_get_csb_size(adev);
2382		dws = adev->gfx.rlc.clear_state_size + (256 / 4);
2383
2384		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
2385					      AMDGPU_GEM_DOMAIN_VRAM |
2386					      AMDGPU_GEM_DOMAIN_GTT,
2387					      &adev->gfx.rlc.clear_state_obj,
2388					      &adev->gfx.rlc.clear_state_gpu_addr,
2389					      (void **)&adev->gfx.rlc.cs_ptr);
2390		if (r) {
2391			dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
2392			amdgpu_gfx_rlc_fini(adev);
2393			return r;
2394		}
2395
2396		/* set up the cs buffer */
2397		dst_ptr = adev->gfx.rlc.cs_ptr;
2398		reg_list_mc_addr = adev->gfx.rlc.clear_state_gpu_addr + 256;
2399		dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
2400		dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
2401		dst_ptr[2] = cpu_to_le32(adev->gfx.rlc.clear_state_size);
2402		gfx_v6_0_get_csb_buffer(adev, &dst_ptr[(256/4)]);
2403		amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
2404		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
2405	}
2406
2407	return 0;
2408}
2409
2410static void gfx_v6_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
2411{
2412	WREG32_FIELD(RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
2413
2414	if (!enable) {
2415		gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2416		WREG32(mmSPI_LB_CU_MASK, 0x00ff);
2417	}
2418}
2419
2420static void gfx_v6_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2421{
2422	int i;
2423
2424	for (i = 0; i < adev->usec_timeout; i++) {
2425		if (RREG32(mmRLC_SERDES_MASTER_BUSY_0) == 0)
2426			break;
2427		udelay(1);
2428	}
2429
2430	for (i = 0; i < adev->usec_timeout; i++) {
2431		if (RREG32(mmRLC_SERDES_MASTER_BUSY_1) == 0)
2432			break;
2433		udelay(1);
2434	}
2435}
2436
2437static void gfx_v6_0_update_rlc(struct amdgpu_device *adev, u32 rlc)
2438{
2439	u32 tmp;
2440
2441	tmp = RREG32(mmRLC_CNTL);
2442	if (tmp != rlc)
2443		WREG32(mmRLC_CNTL, rlc);
2444}
2445
2446static u32 gfx_v6_0_halt_rlc(struct amdgpu_device *adev)
2447{
2448	u32 data, orig;
2449
2450	orig = data = RREG32(mmRLC_CNTL);
2451
2452	if (data & RLC_CNTL__RLC_ENABLE_F32_MASK) {
2453		data &= ~RLC_CNTL__RLC_ENABLE_F32_MASK;
2454		WREG32(mmRLC_CNTL, data);
2455
2456		gfx_v6_0_wait_for_rlc_serdes(adev);
2457	}
2458
2459	return orig;
2460}
2461
2462static void gfx_v6_0_rlc_stop(struct amdgpu_device *adev)
2463{
2464	WREG32(mmRLC_CNTL, 0);
2465
2466	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
2467	gfx_v6_0_wait_for_rlc_serdes(adev);
2468}
2469
2470static void gfx_v6_0_rlc_start(struct amdgpu_device *adev)
2471{
2472	WREG32(mmRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
2473
2474	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
2475
2476	udelay(50);
2477}
2478
2479static void gfx_v6_0_rlc_reset(struct amdgpu_device *adev)
2480{
2481	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2482	udelay(50);
2483	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2484	udelay(50);
2485}
2486
2487static bool gfx_v6_0_lbpw_supported(struct amdgpu_device *adev)
2488{
2489	u32 tmp;
2490
2491	/* Enable LBPW only for DDR3 */
2492	tmp = RREG32(mmMC_SEQ_MISC0);
2493	if ((tmp & 0xF0000000) == 0xB0000000)
2494		return true;
2495	return false;
2496}
2497
2498static void gfx_v6_0_init_cg(struct amdgpu_device *adev)
2499{
2500}
2501
2502static int gfx_v6_0_rlc_resume(struct amdgpu_device *adev)
2503{
2504	u32 i;
2505	const struct rlc_firmware_header_v1_0 *hdr;
2506	const __le32 *fw_data;
2507	u32 fw_size;
2508
2509
2510	if (!adev->gfx.rlc_fw)
2511		return -EINVAL;
2512
2513	adev->gfx.rlc.funcs->stop(adev);
2514	adev->gfx.rlc.funcs->reset(adev);
2515	gfx_v6_0_init_pg(adev);
2516	gfx_v6_0_init_cg(adev);
2517
2518	WREG32(mmRLC_RL_BASE, 0);
2519	WREG32(mmRLC_RL_SIZE, 0);
2520	WREG32(mmRLC_LB_CNTL, 0);
2521	WREG32(mmRLC_LB_CNTR_MAX, 0xffffffff);
2522	WREG32(mmRLC_LB_CNTR_INIT, 0);
2523	WREG32(mmRLC_LB_INIT_CU_MASK, 0xffffffff);
2524
2525	WREG32(mmRLC_MC_CNTL, 0);
2526	WREG32(mmRLC_UCODE_CNTL, 0);
2527
2528	hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
2529	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2530	fw_data = (const __le32 *)
2531		(adev->gfx.rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2532
2533	amdgpu_ucode_print_rlc_hdr(&hdr->header);
2534
2535	for (i = 0; i < fw_size; i++) {
2536		WREG32(mmRLC_UCODE_ADDR, i);
2537		WREG32(mmRLC_UCODE_DATA, le32_to_cpup(fw_data++));
2538	}
2539	WREG32(mmRLC_UCODE_ADDR, 0);
2540
2541	gfx_v6_0_enable_lbpw(adev, gfx_v6_0_lbpw_supported(adev));
2542	adev->gfx.rlc.funcs->start(adev);
2543
2544	return 0;
2545}
2546
2547static void gfx_v6_0_enable_cgcg(struct amdgpu_device *adev, bool enable)
2548{
2549	u32 data, orig, tmp;
2550
2551	orig = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
2552
2553	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
2554		gfx_v6_0_enable_gui_idle_interrupt(adev, true);
2555
2556		WREG32(mmRLC_GCPM_GENERAL_3, 0x00000080);
2557
2558		tmp = gfx_v6_0_halt_rlc(adev);
2559
2560		WREG32(mmRLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
2561		WREG32(mmRLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
2562		WREG32(mmRLC_SERDES_WR_CTRL, 0x00b000ff);
2563
2564		gfx_v6_0_wait_for_rlc_serdes(adev);
2565		gfx_v6_0_update_rlc(adev, tmp);
2566
2567		WREG32(mmRLC_SERDES_WR_CTRL, 0x007000ff);
2568
2569		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2570	} else {
2571		gfx_v6_0_enable_gui_idle_interrupt(adev, false);
2572
2573		RREG32(mmCB_CGTT_SCLK_CTRL);
2574		RREG32(mmCB_CGTT_SCLK_CTRL);
2575		RREG32(mmCB_CGTT_SCLK_CTRL);
2576		RREG32(mmCB_CGTT_SCLK_CTRL);
2577
2578		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
2579	}
2580
2581	if (orig != data)
2582		WREG32(mmRLC_CGCG_CGLS_CTRL, data);
2583
2584}
2585
2586static void gfx_v6_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
2587{
2588
2589	u32 data, orig, tmp = 0;
2590
2591	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
2592		orig = data = RREG32(mmCGTS_SM_CTRL_REG);
2593		data = 0x96940200;
2594		if (orig != data)
2595			WREG32(mmCGTS_SM_CTRL_REG, data);
2596
2597		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
2598			orig = data = RREG32(mmCP_MEM_SLP_CNTL);
2599			data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2600			if (orig != data)
2601				WREG32(mmCP_MEM_SLP_CNTL, data);
2602		}
2603
2604		orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
2605		data &= 0xffffffc0;
2606		if (orig != data)
2607			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
2608
2609		tmp = gfx_v6_0_halt_rlc(adev);
2610
2611		WREG32(mmRLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
2612		WREG32(mmRLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
2613		WREG32(mmRLC_SERDES_WR_CTRL, 0x00d000ff);
2614
2615		gfx_v6_0_update_rlc(adev, tmp);
2616	} else {
2617		orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
2618		data |= 0x00000003;
2619		if (orig != data)
2620			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
2621
2622		data = RREG32(mmCP_MEM_SLP_CNTL);
2623		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
2624			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2625			WREG32(mmCP_MEM_SLP_CNTL, data);
2626		}
2627		orig = data = RREG32(mmCGTS_SM_CTRL_REG);
2628		data |= CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK | CGTS_SM_CTRL_REG__OVERRIDE_MASK;
2629		if (orig != data)
2630			WREG32(mmCGTS_SM_CTRL_REG, data);
2631
2632		tmp = gfx_v6_0_halt_rlc(adev);
2633
2634		WREG32(mmRLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
2635		WREG32(mmRLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
2636		WREG32(mmRLC_SERDES_WR_CTRL, 0x00e000ff);
2637
2638		gfx_v6_0_update_rlc(adev, tmp);
2639	}
2640}
2641/*
2642static void gfx_v6_0_update_cg(struct amdgpu_device *adev,
2643			       bool enable)
2644{
2645	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
2646	if (enable) {
2647		gfx_v6_0_enable_mgcg(adev, true);
2648		gfx_v6_0_enable_cgcg(adev, true);
2649	} else {
2650		gfx_v6_0_enable_cgcg(adev, false);
2651		gfx_v6_0_enable_mgcg(adev, false);
2652	}
2653	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
2654}
2655*/
2656
2657static void gfx_v6_0_enable_sclk_slowdown_on_pu(struct amdgpu_device *adev,
2658						bool enable)
2659{
2660}
2661
2662static void gfx_v6_0_enable_sclk_slowdown_on_pd(struct amdgpu_device *adev,
2663						bool enable)
2664{
2665}
2666
2667static void gfx_v6_0_enable_cp_pg(struct amdgpu_device *adev, bool enable)
2668{
2669	u32 data, orig;
2670
2671	orig = data = RREG32(mmRLC_PG_CNTL);
2672	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_CP))
2673		data &= ~0x8000;
2674	else
2675		data |= 0x8000;
2676	if (orig != data)
2677		WREG32(mmRLC_PG_CNTL, data);
2678}
2679
2680static void gfx_v6_0_enable_gds_pg(struct amdgpu_device *adev, bool enable)
2681{
2682}
2683/*
2684static void gfx_v6_0_init_cp_pg_table(struct amdgpu_device *adev)
2685{
2686	const __le32 *fw_data;
2687	volatile u32 *dst_ptr;
2688	int me, i, max_me = 4;
2689	u32 bo_offset = 0;
2690	u32 table_offset, table_size;
2691
2692	if (adev->asic_type == CHIP_KAVERI)
2693		max_me = 5;
2694
2695	if (adev->gfx.rlc.cp_table_ptr == NULL)
2696		return;
2697
2698	dst_ptr = adev->gfx.rlc.cp_table_ptr;
2699	for (me = 0; me < max_me; me++) {
2700		if (me == 0) {
2701			const struct gfx_firmware_header_v1_0 *hdr =
2702				(const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
2703			fw_data = (const __le32 *)
2704				(adev->gfx.ce_fw->data +
2705				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2706			table_offset = le32_to_cpu(hdr->jt_offset);
2707			table_size = le32_to_cpu(hdr->jt_size);
2708		} else if (me == 1) {
2709			const struct gfx_firmware_header_v1_0 *hdr =
2710				(const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
2711			fw_data = (const __le32 *)
2712				(adev->gfx.pfp_fw->data +
2713				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2714			table_offset = le32_to_cpu(hdr->jt_offset);
2715			table_size = le32_to_cpu(hdr->jt_size);
2716		} else if (me == 2) {
2717			const struct gfx_firmware_header_v1_0 *hdr =
2718				(const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
2719			fw_data = (const __le32 *)
2720				(adev->gfx.me_fw->data +
2721				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2722			table_offset = le32_to_cpu(hdr->jt_offset);
2723			table_size = le32_to_cpu(hdr->jt_size);
2724		} else if (me == 3) {
2725			const struct gfx_firmware_header_v1_0 *hdr =
2726				(const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2727			fw_data = (const __le32 *)
2728				(adev->gfx.mec_fw->data +
2729				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2730			table_offset = le32_to_cpu(hdr->jt_offset);
2731			table_size = le32_to_cpu(hdr->jt_size);
2732		} else {
2733			const struct gfx_firmware_header_v1_0 *hdr =
2734				(const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
2735			fw_data = (const __le32 *)
2736				(adev->gfx.mec2_fw->data +
2737				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2738			table_offset = le32_to_cpu(hdr->jt_offset);
2739			table_size = le32_to_cpu(hdr->jt_size);
2740		}
2741
2742		for (i = 0; i < table_size; i ++) {
2743			dst_ptr[bo_offset + i] =
2744				cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
2745		}
2746
2747		bo_offset += table_size;
2748	}
2749}
2750*/
2751static void gfx_v6_0_enable_gfx_cgpg(struct amdgpu_device *adev,
2752				     bool enable)
2753{
2754	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
2755		WREG32(mmRLC_TTOP_D, RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10));
2756		WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, 1);
2757		WREG32_FIELD(RLC_AUTO_PG_CTRL, AUTO_PG_EN, 1);
2758	} else {
2759		WREG32_FIELD(RLC_AUTO_PG_CTRL, AUTO_PG_EN, 0);
2760		(void)RREG32(mmDB_RENDER_CONTROL);
2761	}
2762}
2763
2764static void gfx_v6_0_init_ao_cu_mask(struct amdgpu_device *adev)
2765{
2766	u32 tmp;
2767
2768	WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
2769
2770	tmp = RREG32(mmRLC_MAX_PG_CU);
2771	tmp &= ~RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK;
2772	tmp |= (adev->gfx.cu_info.number << RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT);
2773	WREG32(mmRLC_MAX_PG_CU, tmp);
2774}
2775
2776static void gfx_v6_0_enable_gfx_static_mgpg(struct amdgpu_device *adev,
2777					    bool enable)
2778{
2779	u32 data, orig;
2780
2781	orig = data = RREG32(mmRLC_PG_CNTL);
2782	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG))
2783		data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
2784	else
2785		data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
2786	if (orig != data)
2787		WREG32(mmRLC_PG_CNTL, data);
2788}
2789
2790static void gfx_v6_0_enable_gfx_dynamic_mgpg(struct amdgpu_device *adev,
2791					     bool enable)
2792{
2793	u32 data, orig;
2794
2795	orig = data = RREG32(mmRLC_PG_CNTL);
2796	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG))
2797		data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
2798	else
2799		data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
2800	if (orig != data)
2801		WREG32(mmRLC_PG_CNTL, data);
2802}
2803
2804static void gfx_v6_0_init_gfx_cgpg(struct amdgpu_device *adev)
2805{
2806	u32 tmp;
2807
2808	WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
2809	WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_SRC, 1);
2810	WREG32(mmRLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
2811
2812	tmp = RREG32(mmRLC_AUTO_PG_CTRL);
2813	tmp &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2814	tmp |= (0x700 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2815	tmp &= ~RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK;
2816	WREG32(mmRLC_AUTO_PG_CTRL, tmp);
2817}
2818
2819static void gfx_v6_0_update_gfx_pg(struct amdgpu_device *adev, bool enable)
2820{
2821	gfx_v6_0_enable_gfx_cgpg(adev, enable);
2822	gfx_v6_0_enable_gfx_static_mgpg(adev, enable);
2823	gfx_v6_0_enable_gfx_dynamic_mgpg(adev, enable);
2824}
2825
2826static u32 gfx_v6_0_get_csb_size(struct amdgpu_device *adev)
2827{
2828	u32 count = 0;
2829	const struct cs_section_def *sect = NULL;
2830	const struct cs_extent_def *ext = NULL;
2831
2832	if (adev->gfx.rlc.cs_data == NULL)
2833		return 0;
2834
2835	/* begin clear state */
2836	count += 2;
2837	/* context control state */
2838	count += 3;
2839
2840	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2841		for (ext = sect->section; ext->extent != NULL; ++ext) {
2842			if (sect->id == SECT_CONTEXT)
2843				count += 2 + ext->reg_count;
2844			else
2845				return 0;
2846		}
2847	}
2848	/* pa_sc_raster_config */
2849	count += 3;
2850	/* end clear state */
2851	count += 2;
2852	/* clear state */
2853	count += 2;
2854
2855	return count;
2856}
2857
2858static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
2859{
2860	u32 count = 0;
2861
2862	if (adev->gfx.rlc.cs_data == NULL)
2863		return;
2864	if (buffer == NULL)
2865		return;
2866
2867	count = amdgpu_gfx_csb_preamble_start(buffer);
2868	count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
2869
2870	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
2871	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
2872	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
2873
2874	amdgpu_gfx_csb_preamble_end(buffer, count);
2875}
2876
2877static void gfx_v6_0_init_pg(struct amdgpu_device *adev)
2878{
2879	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2880			      AMD_PG_SUPPORT_GFX_SMG |
2881			      AMD_PG_SUPPORT_GFX_DMG |
2882			      AMD_PG_SUPPORT_CP |
2883			      AMD_PG_SUPPORT_GDS |
2884			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
2885		gfx_v6_0_enable_sclk_slowdown_on_pu(adev, true);
2886		gfx_v6_0_enable_sclk_slowdown_on_pd(adev, true);
2887		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
2888			gfx_v6_0_init_gfx_cgpg(adev);
2889			gfx_v6_0_enable_cp_pg(adev, true);
2890			gfx_v6_0_enable_gds_pg(adev, true);
2891		} else {
2892			WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
2893			WREG32(mmRLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
2894
2895		}
2896		gfx_v6_0_init_ao_cu_mask(adev);
2897		gfx_v6_0_update_gfx_pg(adev, true);
2898	} else {
2899
2900		WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
2901		WREG32(mmRLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
2902	}
2903}
2904
2905static void gfx_v6_0_fini_pg(struct amdgpu_device *adev)
2906{
2907	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2908			      AMD_PG_SUPPORT_GFX_SMG |
2909			      AMD_PG_SUPPORT_GFX_DMG |
2910			      AMD_PG_SUPPORT_CP |
2911			      AMD_PG_SUPPORT_GDS |
2912			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
2913		gfx_v6_0_update_gfx_pg(adev, false);
2914		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
2915			gfx_v6_0_enable_cp_pg(adev, false);
2916			gfx_v6_0_enable_gds_pg(adev, false);
2917		}
2918	}
2919}
2920
2921static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev)
2922{
2923	uint64_t clock;
2924
2925	mutex_lock(&adev->gfx.gpu_clock_mutex);
2926	WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
2927	clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
2928	        ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
2929	mutex_unlock(&adev->gfx.gpu_clock_mutex);
2930	return clock;
2931}
2932
2933static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
2934{
2935	if (flags & AMDGPU_HAVE_CTX_SWITCH)
2936		gfx_v6_0_ring_emit_vgt_flush(ring);
2937	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2938	amdgpu_ring_write(ring, 0x80000000);
2939	amdgpu_ring_write(ring, 0);
2940}
2941
2942
2943static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
2944{
2945	WREG32(mmSQ_IND_INDEX,
2946		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2947		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2948		(address << SQ_IND_INDEX__INDEX__SHIFT) |
2949		(SQ_IND_INDEX__FORCE_READ_MASK));
2950	return RREG32(mmSQ_IND_DATA);
2951}
2952
2953static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
2954			   uint32_t wave, uint32_t thread,
2955			   uint32_t regno, uint32_t num, uint32_t *out)
2956{
2957	WREG32(mmSQ_IND_INDEX,
2958		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2959		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2960		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
2961		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2962		(SQ_IND_INDEX__FORCE_READ_MASK) |
2963		(SQ_IND_INDEX__AUTO_INCR_MASK));
2964	while (num--)
2965		*(out++) = RREG32(mmSQ_IND_DATA);
2966}
2967
2968static void gfx_v6_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2969{
2970	/* type 0 wave data */
2971	dst[(*no_fields)++] = 0;
2972	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2973	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2974	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2975	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2976	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2977	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2978	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2979	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2980	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2981	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2982	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2983	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2984	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
2985	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
2986	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
2987	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
2988	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2989	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2990	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
2991}
2992
2993static void gfx_v6_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
2994				     uint32_t wave, uint32_t start,
2995				     uint32_t size, uint32_t *dst)
2996{
2997	wave_read_regs(
2998		adev, simd, wave, 0,
2999		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
3000}
3001
3002static void gfx_v6_0_select_me_pipe_q(struct amdgpu_device *adev,
3003				  u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
3004{
3005	DRM_INFO("Not implemented\n");
3006}
3007
3008static const struct amdgpu_gfx_funcs gfx_v6_0_gfx_funcs = {
3009	.get_gpu_clock_counter = &gfx_v6_0_get_gpu_clock_counter,
3010	.select_se_sh = &gfx_v6_0_select_se_sh,
3011	.read_wave_data = &gfx_v6_0_read_wave_data,
3012	.read_wave_sgprs = &gfx_v6_0_read_wave_sgprs,
3013	.select_me_pipe_q = &gfx_v6_0_select_me_pipe_q
3014};
3015
3016static const struct amdgpu_rlc_funcs gfx_v6_0_rlc_funcs = {
3017	.init = gfx_v6_0_rlc_init,
3018	.resume = gfx_v6_0_rlc_resume,
3019	.stop = gfx_v6_0_rlc_stop,
3020	.reset = gfx_v6_0_rlc_reset,
3021	.start = gfx_v6_0_rlc_start
3022};
3023
3024static int gfx_v6_0_early_init(struct amdgpu_ip_block *ip_block)
3025{
3026	struct amdgpu_device *adev = ip_block->adev;
3027
3028	adev->gfx.xcc_mask = 1;
3029	adev->gfx.num_gfx_rings = GFX6_NUM_GFX_RINGS;
3030	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
3031					  GFX6_NUM_COMPUTE_RINGS);
3032	adev->gfx.funcs = &gfx_v6_0_gfx_funcs;
3033	adev->gfx.rlc.funcs = &gfx_v6_0_rlc_funcs;
3034	gfx_v6_0_set_ring_funcs(adev);
3035	gfx_v6_0_set_irq_funcs(adev);
3036
3037	return 0;
3038}
3039
3040static int gfx_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
3041{
3042	struct amdgpu_ring *ring;
3043	struct amdgpu_device *adev = ip_block->adev;
3044	int i, r;
3045
3046	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 181, &adev->gfx.eop_irq);
3047	if (r)
3048		return r;
3049
3050	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 184, &adev->gfx.priv_reg_irq);
3051	if (r)
3052		return r;
3053
3054	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 185, &adev->gfx.priv_inst_irq);
3055	if (r)
3056		return r;
3057
3058	r = gfx_v6_0_init_microcode(adev);
3059	if (r) {
3060		DRM_ERROR("Failed to load gfx firmware!\n");
3061		return r;
3062	}
3063
3064	r = adev->gfx.rlc.funcs->init(adev);
3065	if (r) {
3066		DRM_ERROR("Failed to init rlc BOs!\n");
3067		return r;
3068	}
3069
3070	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3071		ring = &adev->gfx.gfx_ring[i];
3072		ring->ring_obj = NULL;
3073		sprintf(ring->name, "gfx");
3074		r = amdgpu_ring_init(adev, ring, 2048,
3075				     &adev->gfx.eop_irq,
3076				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
3077				     AMDGPU_RING_PRIO_DEFAULT, NULL);
3078		if (r)
3079			return r;
3080	}
3081
3082	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3083		unsigned irq_type;
3084
3085		if ((i >= 32) || (i >= AMDGPU_MAX_COMPUTE_RINGS)) {
3086			DRM_ERROR("Too many (%d) compute rings!\n", i);
3087			break;
3088		}
3089		ring = &adev->gfx.compute_ring[i];
3090		ring->ring_obj = NULL;
3091		ring->use_doorbell = false;
3092		ring->doorbell_index = 0;
3093		ring->me = 1;
3094		ring->pipe = i;
3095		ring->queue = i;
3096		sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
3097		irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
3098		r = amdgpu_ring_init(adev, ring, 1024,
3099				     &adev->gfx.eop_irq, irq_type,
3100				     AMDGPU_RING_PRIO_DEFAULT, NULL);
3101		if (r)
3102			return r;
3103	}
3104
3105	adev->gfx.gfx_supported_reset =
3106		amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
3107	adev->gfx.compute_supported_reset =
3108		amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
3109
3110	return r;
3111}
3112
3113static int gfx_v6_0_sw_fini(struct amdgpu_ip_block *ip_block)
3114{
3115	int i;
3116	struct amdgpu_device *adev = ip_block->adev;
3117
3118	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3119		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
3120	for (i = 0; i < adev->gfx.num_compute_rings; i++)
3121		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
3122
3123	amdgpu_gfx_rlc_fini(adev);
3124
3125	return 0;
3126}
3127
3128static int gfx_v6_0_hw_init(struct amdgpu_ip_block *ip_block)
3129{
3130	int r;
3131	struct amdgpu_device *adev = ip_block->adev;
3132
3133	gfx_v6_0_constants_init(adev);
3134
3135	r = adev->gfx.rlc.funcs->resume(adev);
3136	if (r)
3137		return r;
3138
3139	r = gfx_v6_0_cp_resume(adev);
3140	if (r)
3141		return r;
3142
3143	adev->gfx.ce_ram_size = 0x8000;
3144
3145	return r;
3146}
3147
3148static int gfx_v6_0_hw_fini(struct amdgpu_ip_block *ip_block)
3149{
3150	struct amdgpu_device *adev = ip_block->adev;
3151
3152	gfx_v6_0_cp_enable(adev, false);
3153	adev->gfx.rlc.funcs->stop(adev);
3154	gfx_v6_0_fini_pg(adev);
3155
3156	return 0;
3157}
3158
3159static int gfx_v6_0_suspend(struct amdgpu_ip_block *ip_block)
3160{
3161	return gfx_v6_0_hw_fini(ip_block);
3162}
3163
3164static int gfx_v6_0_resume(struct amdgpu_ip_block *ip_block)
3165{
3166	return gfx_v6_0_hw_init(ip_block);
3167}
3168
3169static bool gfx_v6_0_is_idle(struct amdgpu_ip_block *ip_block)
3170{
3171	struct amdgpu_device *adev = ip_block->adev;
3172
3173	if (RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
3174		return false;
3175	else
3176		return true;
3177}
3178
3179static int gfx_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
3180{
3181	unsigned i;
3182	struct amdgpu_device *adev = ip_block->adev;
3183
3184	for (i = 0; i < adev->usec_timeout; i++) {
3185		if (gfx_v6_0_is_idle(ip_block))
3186			return 0;
3187		udelay(1);
3188	}
3189	return -ETIMEDOUT;
3190}
3191
3192static void gfx_v6_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
3193						 enum amdgpu_interrupt_state state)
3194{
3195	u32 cp_int_cntl;
3196
3197	switch (state) {
3198	case AMDGPU_IRQ_STATE_DISABLE:
3199		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
3200		cp_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
3201		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
3202		break;
3203	case AMDGPU_IRQ_STATE_ENABLE:
3204		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
3205		cp_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
3206		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
3207		break;
3208	default:
3209		break;
3210	}
3211}
3212
3213static void gfx_v6_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
3214						     int ring,
3215						     enum amdgpu_interrupt_state state)
3216{
3217	u32 cp_int_cntl;
3218	switch (state){
3219	case AMDGPU_IRQ_STATE_DISABLE:
3220		if (ring == 0) {
3221			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING1);
3222			cp_int_cntl &= ~CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK;
3223			WREG32(mmCP_INT_CNTL_RING1, cp_int_cntl);
3224			break;
3225		} else {
3226			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING2);
3227			cp_int_cntl &= ~CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK;
3228			WREG32(mmCP_INT_CNTL_RING2, cp_int_cntl);
3229			break;
3230
3231		}
3232	case AMDGPU_IRQ_STATE_ENABLE:
3233		if (ring == 0) {
3234			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING1);
3235			cp_int_cntl |= CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK;
3236			WREG32(mmCP_INT_CNTL_RING1, cp_int_cntl);
3237			break;
3238		} else {
3239			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING2);
3240			cp_int_cntl |= CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK;
3241			WREG32(mmCP_INT_CNTL_RING2, cp_int_cntl);
3242			break;
3243
3244		}
3245
3246	default:
3247		BUG();
3248		break;
3249
3250	}
3251}
3252
3253static int gfx_v6_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
3254					     struct amdgpu_irq_src *src,
3255					     unsigned type,
3256					     enum amdgpu_interrupt_state state)
3257{
3258	u32 cp_int_cntl;
3259
3260	switch (state) {
3261	case AMDGPU_IRQ_STATE_DISABLE:
3262		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
3263		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
3264		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
3265		break;
3266	case AMDGPU_IRQ_STATE_ENABLE:
3267		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
3268		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
3269		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
3270		break;
3271	default:
3272		break;
3273	}
3274
3275	return 0;
3276}
3277
3278static int gfx_v6_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
3279					      struct amdgpu_irq_src *src,
3280					      unsigned type,
3281					      enum amdgpu_interrupt_state state)
3282{
3283	u32 cp_int_cntl;
3284
3285	switch (state) {
3286	case AMDGPU_IRQ_STATE_DISABLE:
3287		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
3288		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
3289		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
3290		break;
3291	case AMDGPU_IRQ_STATE_ENABLE:
3292		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
3293		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
3294		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
3295		break;
3296	default:
3297		break;
3298	}
3299
3300	return 0;
3301}
3302
3303static int gfx_v6_0_set_eop_interrupt_state(struct amdgpu_device *adev,
3304					    struct amdgpu_irq_src *src,
3305					    unsigned type,
3306					    enum amdgpu_interrupt_state state)
3307{
3308	switch (type) {
3309	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
3310		gfx_v6_0_set_gfx_eop_interrupt_state(adev, state);
3311		break;
3312	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
3313		gfx_v6_0_set_compute_eop_interrupt_state(adev, 0, state);
3314		break;
3315	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
3316		gfx_v6_0_set_compute_eop_interrupt_state(adev, 1, state);
3317		break;
3318	default:
3319		break;
3320	}
3321	return 0;
3322}
3323
3324static int gfx_v6_0_eop_irq(struct amdgpu_device *adev,
3325			    struct amdgpu_irq_src *source,
3326			    struct amdgpu_iv_entry *entry)
3327{
3328	switch (entry->ring_id) {
3329	case 0:
3330		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
3331		break;
3332	case 1:
3333	case 2:
3334		amdgpu_fence_process(&adev->gfx.compute_ring[entry->ring_id - 1]);
3335		break;
3336	default:
3337		break;
3338	}
3339	return 0;
3340}
3341
3342static void gfx_v6_0_fault(struct amdgpu_device *adev,
3343			   struct amdgpu_iv_entry *entry)
3344{
3345	struct amdgpu_ring *ring;
3346
3347	switch (entry->ring_id) {
3348	case 0:
3349		ring = &adev->gfx.gfx_ring[0];
3350		break;
3351	case 1:
3352	case 2:
3353		ring = &adev->gfx.compute_ring[entry->ring_id - 1];
3354		break;
3355	default:
3356		return;
3357	}
3358	drm_sched_fault(&ring->sched);
3359}
3360
3361static int gfx_v6_0_priv_reg_irq(struct amdgpu_device *adev,
3362				 struct amdgpu_irq_src *source,
3363				 struct amdgpu_iv_entry *entry)
3364{
3365	DRM_ERROR("Illegal register access in command stream\n");
3366	gfx_v6_0_fault(adev, entry);
3367	return 0;
3368}
3369
3370static int gfx_v6_0_priv_inst_irq(struct amdgpu_device *adev,
3371				  struct amdgpu_irq_src *source,
3372				  struct amdgpu_iv_entry *entry)
3373{
3374	DRM_ERROR("Illegal instruction in command stream\n");
3375	gfx_v6_0_fault(adev, entry);
3376	return 0;
3377}
3378
3379static int gfx_v6_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
3380					  enum amd_clockgating_state state)
3381{
3382	bool gate = false;
3383	struct amdgpu_device *adev = ip_block->adev;
3384
3385	if (state == AMD_CG_STATE_GATE)
3386		gate = true;
3387
3388	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
3389	if (gate) {
3390		gfx_v6_0_enable_mgcg(adev, true);
3391		gfx_v6_0_enable_cgcg(adev, true);
3392	} else {
3393		gfx_v6_0_enable_cgcg(adev, false);
3394		gfx_v6_0_enable_mgcg(adev, false);
3395	}
3396	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
3397
3398	return 0;
3399}
3400
3401static int gfx_v6_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
3402					  enum amd_powergating_state state)
3403{
3404	bool gate = false;
3405	struct amdgpu_device *adev = ip_block->adev;
3406
3407	if (state == AMD_PG_STATE_GATE)
3408		gate = true;
3409
3410	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3411			      AMD_PG_SUPPORT_GFX_SMG |
3412			      AMD_PG_SUPPORT_GFX_DMG |
3413			      AMD_PG_SUPPORT_CP |
3414			      AMD_PG_SUPPORT_GDS |
3415			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
3416		gfx_v6_0_update_gfx_pg(adev, gate);
3417		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
3418			gfx_v6_0_enable_cp_pg(adev, gate);
3419			gfx_v6_0_enable_gds_pg(adev, gate);
3420		}
3421	}
3422
3423	return 0;
3424}
3425
3426static void gfx_v6_0_emit_mem_sync(struct amdgpu_ring *ring)
3427{
3428	amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3429	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3430			  PACKET3_TC_ACTION_ENA |
3431			  PACKET3_SH_KCACHE_ACTION_ENA |
3432			  PACKET3_SH_ICACHE_ACTION_ENA);  /* CP_COHER_CNTL */
3433	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
3434	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE */
3435	amdgpu_ring_write(ring, 0x0000000A); /* poll interval */
3436}
3437
3438static const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
3439	.name = "gfx_v6_0",
3440	.early_init = gfx_v6_0_early_init,
3441	.sw_init = gfx_v6_0_sw_init,
3442	.sw_fini = gfx_v6_0_sw_fini,
3443	.hw_init = gfx_v6_0_hw_init,
3444	.hw_fini = gfx_v6_0_hw_fini,
3445	.suspend = gfx_v6_0_suspend,
3446	.resume = gfx_v6_0_resume,
3447	.is_idle = gfx_v6_0_is_idle,
3448	.wait_for_idle = gfx_v6_0_wait_for_idle,
3449	.set_clockgating_state = gfx_v6_0_set_clockgating_state,
3450	.set_powergating_state = gfx_v6_0_set_powergating_state,
3451};
3452
3453static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = {
3454	.type = AMDGPU_RING_TYPE_GFX,
3455	.align_mask = 0xff,
3456	.nop = 0x80000000,
3457	.support_64bit_ptrs = false,
3458	.get_rptr = gfx_v6_0_ring_get_rptr,
3459	.get_wptr = gfx_v6_0_ring_get_wptr,
3460	.set_wptr = gfx_v6_0_ring_set_wptr_gfx,
3461	.emit_frame_size =
3462		5 + 5 + /* hdp flush / invalidate */
3463		14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
3464		7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */
3465		SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 6 + /* gfx_v6_0_ring_emit_vm_flush */
3466		3 + 2 + /* gfx_v6_ring_emit_cntxcntl including vgt flush */
3467		5, /* SURFACE_SYNC */
3468	.emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */
3469	.emit_ib = gfx_v6_0_ring_emit_ib,
3470	.emit_fence = gfx_v6_0_ring_emit_fence,
3471	.emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync,
3472	.emit_vm_flush = gfx_v6_0_ring_emit_vm_flush,
3473	.test_ring = gfx_v6_0_ring_test_ring,
3474	.test_ib = gfx_v6_0_ring_test_ib,
3475	.insert_nop = amdgpu_ring_insert_nop,
3476	.emit_cntxcntl = gfx_v6_ring_emit_cntxcntl,
3477	.emit_wreg = gfx_v6_0_ring_emit_wreg,
3478	.emit_mem_sync = gfx_v6_0_emit_mem_sync,
3479};
3480
3481static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = {
3482	.type = AMDGPU_RING_TYPE_COMPUTE,
3483	.align_mask = 0xff,
3484	.nop = 0x80000000,
3485	.get_rptr = gfx_v6_0_ring_get_rptr,
3486	.get_wptr = gfx_v6_0_ring_get_wptr,
3487	.set_wptr = gfx_v6_0_ring_set_wptr_compute,
3488	.emit_frame_size =
3489		5 + 5 + /* hdp flush / invalidate */
3490		7 + /* gfx_v6_0_ring_emit_pipeline_sync */
3491		SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v6_0_ring_emit_vm_flush */
3492		14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
3493		5, /* SURFACE_SYNC */
3494	.emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */
3495	.emit_ib = gfx_v6_0_ring_emit_ib,
3496	.emit_fence = gfx_v6_0_ring_emit_fence,
3497	.emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync,
3498	.emit_vm_flush = gfx_v6_0_ring_emit_vm_flush,
3499	.test_ring = gfx_v6_0_ring_test_ring,
3500	.test_ib = gfx_v6_0_ring_test_ib,
3501	.insert_nop = amdgpu_ring_insert_nop,
3502	.emit_wreg = gfx_v6_0_ring_emit_wreg,
3503	.emit_mem_sync = gfx_v6_0_emit_mem_sync,
3504};
3505
3506static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev)
3507{
3508	int i;
3509
3510	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3511		adev->gfx.gfx_ring[i].funcs = &gfx_v6_0_ring_funcs_gfx;
3512	for (i = 0; i < adev->gfx.num_compute_rings; i++)
3513		adev->gfx.compute_ring[i].funcs = &gfx_v6_0_ring_funcs_compute;
3514}
3515
3516static const struct amdgpu_irq_src_funcs gfx_v6_0_eop_irq_funcs = {
3517	.set = gfx_v6_0_set_eop_interrupt_state,
3518	.process = gfx_v6_0_eop_irq,
3519};
3520
3521static const struct amdgpu_irq_src_funcs gfx_v6_0_priv_reg_irq_funcs = {
3522	.set = gfx_v6_0_set_priv_reg_fault_state,
3523	.process = gfx_v6_0_priv_reg_irq,
3524};
3525
3526static const struct amdgpu_irq_src_funcs gfx_v6_0_priv_inst_irq_funcs = {
3527	.set = gfx_v6_0_set_priv_inst_fault_state,
3528	.process = gfx_v6_0_priv_inst_irq,
3529};
3530
3531static void gfx_v6_0_set_irq_funcs(struct amdgpu_device *adev)
3532{
3533	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
3534	adev->gfx.eop_irq.funcs = &gfx_v6_0_eop_irq_funcs;
3535
3536	adev->gfx.priv_reg_irq.num_types = 1;
3537	adev->gfx.priv_reg_irq.funcs = &gfx_v6_0_priv_reg_irq_funcs;
3538
3539	adev->gfx.priv_inst_irq.num_types = 1;
3540	adev->gfx.priv_inst_irq.funcs = &gfx_v6_0_priv_inst_irq_funcs;
3541}
3542
3543static void gfx_v6_0_get_cu_info(struct amdgpu_device *adev)
3544{
3545	int i, j, k, counter, active_cu_number = 0;
3546	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
3547	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
3548	unsigned disable_masks[4 * 2];
3549	u32 ao_cu_num;
3550
3551	if (adev->flags & AMD_IS_APU)
3552		ao_cu_num = 2;
3553	else
3554		ao_cu_num = adev->gfx.config.max_cu_per_sh;
3555
3556	memset(cu_info, 0, sizeof(*cu_info));
3557
3558	amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
3559
3560	mutex_lock(&adev->grbm_idx_mutex);
3561	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3562		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3563			mask = 1;
3564			ao_bitmap = 0;
3565			counter = 0;
3566			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff, 0);
3567			if (i < 4 && j < 2)
3568				gfx_v6_0_set_user_cu_inactive_bitmap(
3569					adev, disable_masks[i * 2 + j]);
3570			bitmap = gfx_v6_0_get_cu_enabled(adev);
3571			cu_info->bitmap[0][i][j] = bitmap;
3572
3573			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
3574				if (bitmap & mask) {
3575					if (counter < ao_cu_num)
3576						ao_bitmap |= mask;
3577					counter ++;
3578				}
3579				mask <<= 1;
3580			}
3581			active_cu_number += counter;
3582			if (i < 2 && j < 2)
3583				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
3584			cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
3585		}
3586	}
3587
3588	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3589	mutex_unlock(&adev->grbm_idx_mutex);
3590
3591	cu_info->number = active_cu_number;
3592	cu_info->ao_cu_mask = ao_cu_mask;
3593}
3594
3595const struct amdgpu_ip_block_version gfx_v6_0_ip_block =
3596{
3597	.type = AMD_IP_BLOCK_TYPE_GFX,
3598	.major = 6,
3599	.minor = 0,
3600	.rev = 0,
3601	.funcs = &gfx_v6_0_ip_funcs,
3602};