mmhub_v3_3.c

  1/*
  2 * Copyright 2023 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include "amdgpu.h"
 25#include "mmhub_v3_3.h"
 26
 27#include "mmhub/mmhub_3_3_0_offset.h"
 28#include "mmhub/mmhub_3_3_0_sh_mask.h"
 29
 30#include "navi10_enum.h"
 31#include "soc15_common.h"
 32
 33#define regMMVM_L2_CNTL3_DEFAULT				0x80100007
 34#define regMMVM_L2_CNTL4_DEFAULT				0x000000c1
 35#define regMMVM_L2_CNTL5_DEFAULT				0x00003fe0
 36#define regDAGB0_L1TLB_REG_RW_3_3                   0x00a4
 37#define regDAGB0_L1TLB_REG_RW_3_3_BASE_IDX          1
 38#define regDAGB1_L1TLB_REG_RW_3_3                   0x0163
 39#define regDAGB1_L1TLB_REG_RW_3_3_BASE_IDX          1
 40
 41static const char *mmhub_client_ids_v3_3[][2] = {
 42	[0][0] = "VMC",
 43	[1][0] = "ISPXT",
 44	[2][0] = "ISPIXT",
 45	[4][0] = "DCEDMC",
 46	[6][0] = "MP0",
 47	[7][0] = "MP1",
 48	[8][0] = "MPM",
 49	[9][0] = "ISPPDPRD",
 50	[10][0] = "ISPCSTATRD",
 51	[11][0] = "ISPBYRPRD",
 52	[12][0] = "ISPRGBPRD",
 53	[13][0] = "ISPMCFPRD",
 54	[14][0] = "ISPMCFPRD1",
 55	[15][0] = "ISPYUVPRD",
 56	[16][0] = "ISPMCSCRD",
 57	[17][0] = "ISPGDCRD",
 58	[18][0] = "ISPLMERD",
 59	[22][0] = "ISPXT1",
 60	[23][0] = "ISPIXT1",
 61	[24][0] = "HDP",
 62	[25][0] = "LSDMA",
 63	[26][0] = "JPEG",
 64	[27][0] = "VPE",
 65	[28][0] = "VSCH",
 66	[29][0] = "VCNU",
 67	[30][0] = "VCN",
 68	[1][1] = "ISPXT",
 69	[2][1] = "ISPIXT",
 70	[3][1] = "DCEDWB",
 71	[4][1] = "DCEDMC",
 72	[5][1] = "ISPCSISWR",
 73	[6][1] = "MP0",
 74	[7][1] = "MP1",
 75	[8][1] = "MPM",
 76	[9][1] = "ISPPDPWR",
 77	[10][1] = "ISPCSTATWR",
 78	[11][1] = "ISPBYRPWR",
 79	[12][1] = "ISPRGBPWR",
 80	[13][1] = "ISPMCFPWR",
 81	[14][1] = "ISPMWR0",
 82	[15][1] = "ISPYUVPWR",
 83	[16][1] = "ISPMCSCWR",
 84	[17][1] = "ISPGDCWR",
 85	[18][1] = "ISPLMEWR",
 86	[20][1] = "ISPMWR2",
 87	[21][1] = "OSSSYS",
 88	[22][1] = "ISPXT1",
 89	[23][1] = "ISPIXT1",
 90	[24][1] = "HDP",
 91	[25][1] = "LSDMA",
 92	[26][1] = "JPEG",
 93	[27][1] = "VPE",
 94	[28][1] = "VSCH",
 95	[29][1] = "VCNU",
 96	[30][1] = "VCN",
 97};
 98
 99static const char *mmhub_client_ids_v3_3_1[][2] = {
100	[0][0] = "VMC",
101	[4][0] = "DCEDMC",
102	[6][0] = "MP0",
103	[7][0] = "MP1",
104	[8][0] = "MPM",
105	[24][0] = "HDP",
106	[25][0] = "LSDMA",
107	[26][0] = "JPEG0",
108	[27][0] = "VPE0",
109	[28][0] = "VSCH",
110	[29][0] = "VCNU0",
111	[30][0] = "VCN0",
112	[32+1][0] = "ISPXT",
113	[32+2][0] = "ISPIXT",
114	[32+9][0] = "ISPPDPRD",
115	[32+10][0] = "ISPCSTATRD",
116	[32+11][0] = "ISPBYRPRD",
117	[32+12][0] = "ISPRGBPRD",
118	[32+13][0] = "ISPMCFPRD",
119	[32+14][0] = "ISPMCFPRD1",
120	[32+15][0] = "ISPYUVPRD",
121	[32+16][0] = "ISPMCSCRD",
122	[32+17][0] = "ISPGDCRD",
123	[32+18][0] = "ISPLMERD",
124	[32+22][0] = "ISPXT1",
125	[32+23][0] = "ISPIXT1",
126	[32+26][0] = "JPEG1",
127	[32+27][0] = "VPE1",
128	[32+29][0] = "VCNU1",
129	[32+30][0] = "VCN1",
130	[3][1] = "DCEDWB",
131	[4][1] = "DCEDMC",
132	[6][1] = "MP0",
133	[7][1] = "MP1",
134	[8][1] = "MPM",
135	[21][1] = "OSSSYS",
136	[24][1] = "HDP",
137	[25][1] = "LSDMA",
138	[26][1] = "JPEG0",
139	[27][1] = "VPE0",
140	[28][1] = "VSCH",
141	[29][1] = "VCNU0",
142	[30][1] = "VCN0",
143	[32+1][1] = "ISPXT",
144	[32+2][1] = "ISPIXT",
145	[32+5][1] = "ISPCSISWR",
146	[32+9][1] = "ISPPDPWR",
147	[32+10][1] = "ISPCSTATWR",
148	[32+11][1] = "ISPBYRPWR",
149	[32+12][1] = "ISPRGBPWR",
150	[32+13][1] = "ISPMCFPWR",
151	[32+14][1] = "ISPMWR0",
152	[32+15][1] = "ISPYUVPWR",
153	[32+16][1] = "ISPMCSCWR",
154	[32+17][1] = "ISPGDCWR",
155	[32+18][1] = "ISPLMEWR",
156	[32+19][1] = "ISPMWR1",
157	[32+20][1] = "ISPMWR2",
158	[32+22][1] = "ISPXT1",
159	[32+23][1] = "ISPIXT1",
160	[32+26][1] = "JPEG1",
161	[32+27][1] = "VPE1",
162	[32+29][1] = "VCNU1",
163	[32+30][1] = "VCN1",
164};
165
166static uint32_t mmhub_v3_3_get_invalidate_req(unsigned int vmid,
167						uint32_t flush_type)
168{
169	u32 req = 0;
170
171	/* invalidate using legacy mode on vmid*/
172	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ,
173			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
174	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type ? : 1);
175	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
176	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
177	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
178	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
179	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
180	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ,
181			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
182
183	return req;
184}
185
186static void
187mmhub_v3_3_print_l2_protection_fault_status(struct amdgpu_device *adev,
188					      uint32_t status)
189{
190	uint32_t cid, rw;
191	const char *mmhub_cid = NULL;
192
193	cid = REG_GET_FIELD(status,
194			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
195	rw = REG_GET_FIELD(status,
196			   MMVM_L2_PROTECTION_FAULT_STATUS, RW);
197
198	dev_err(adev->dev,
199		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
200		status);
201
202	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
203	case IP_VERSION(3, 3, 0):
204	case IP_VERSION(3, 3, 2):
205		mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_3) ?
206			    mmhub_client_ids_v3_3[cid][rw] :
207			    cid == 0x140 ? "UMSCH" : NULL;
208		break;
209	case IP_VERSION(3, 3, 1):
210		mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_3_1) ?
211			    mmhub_client_ids_v3_3_1[cid][rw] :
212			    cid == 0x140 ? "UMSCH" : NULL;
213		break;
214	default:
215		mmhub_cid = NULL;
216		break;
217	}
218
219	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
220		mmhub_cid ? mmhub_cid : "unknown", cid);
221	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
222		REG_GET_FIELD(status,
223		MMVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
224	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
225		REG_GET_FIELD(status,
226		MMVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
227	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
228		REG_GET_FIELD(status,
229		MMVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
230	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
231		REG_GET_FIELD(status,
232		MMVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
233	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
234}
235
236static void mmhub_v3_3_setup_vm_pt_regs(struct amdgpu_device *adev,
237					  uint32_t vmid,
238					  uint64_t page_table_base)
239{
240	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
241
242	WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
243			    hub->ctx_addr_distance * vmid,
244			    lower_32_bits(page_table_base));
245
246	WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
247			    hub->ctx_addr_distance * vmid,
248			    upper_32_bits(page_table_base));
249
250}
251
252static void mmhub_v3_3_init_gart_aperture_regs(struct amdgpu_device *adev)
253{
254	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
255
256	mmhub_v3_3_setup_vm_pt_regs(adev, 0, pt_base);
257
258	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
259		     (u32)(adev->gmc.gart_start >> 12));
260	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
261		     (u32)(adev->gmc.gart_start >> 44));
262
263	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
264		     (u32)(adev->gmc.gart_end >> 12));
265	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
266		     (u32)(adev->gmc.gart_end >> 44));
267}
268
269static void mmhub_v3_3_init_system_aperture_regs(struct amdgpu_device *adev)
270{
271	uint64_t value;
272	uint32_t tmp;
273
274	/* Program the AGP BAR */
275	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BASE, 0);
276	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
277	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
278
279	/*
280	 * the new L1 policy will block SRIOV guest from writing
281	 * these regs, and they will be programed at host.
282	 * so skip programing these regs.
283	 */
284	/* Program the system aperture low logical page number. */
285	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR,
286		     min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
287	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
288		     max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
289
290	/* Set default page address. */
291	value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
292	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
293		     (u32)(value >> 12));
294	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
295		     (u32)(value >> 44));
296
297	/* Program "protection fault". */
298	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
299		     (u32)(adev->dummy_page_addr >> 12));
300	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
301		     (u32)((u64)adev->dummy_page_addr >> 44));
302
303	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL2);
304	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
305			    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
306	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL2, tmp);
307}
308
309static void mmhub_v3_3_init_tlb_regs(struct amdgpu_device *adev)
310{
311	uint32_t tmp;
312
313	/* Setup TLB control */
314	tmp = RREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL);
315
316	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
317	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
318	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
319			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
320	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
321			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
322	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
323	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
324			    MTYPE, MTYPE_UC); /* UC, uncached */
325
326	WREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL, tmp);
327}
328
329static void mmhub_v3_3_init_cache_regs(struct amdgpu_device *adev)
330{
331	uint32_t tmp;
332
333	/* Setup L2 cache */
334	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL);
335	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1);
336	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
337	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
338			    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
339	/* XXX for emulation, Refer to closed source code.*/
340	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
341			    0);
342	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
343	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
344	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
345	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL, tmp);
346
347	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL2);
348	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
349	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
350	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL2, tmp);
351
352	tmp = regMMVM_L2_CNTL3_DEFAULT;
353	if (adev->gmc.translate_further) {
354		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 12);
355		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
356				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
357	} else {
358		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 9);
359		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
360				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
361	}
362	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL3, tmp);
363
364	tmp = regMMVM_L2_CNTL4_DEFAULT;
365	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
366	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
367	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL4, tmp);
368
369	tmp = regMMVM_L2_CNTL5_DEFAULT;
370	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
371	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL5, tmp);
372}
373
374static void mmhub_v3_3_enable_system_domain(struct amdgpu_device *adev)
375{
376	uint32_t tmp;
377
378	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_CNTL);
379	tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
380	tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
381	tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
382			    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
383
384	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_CNTL, tmp);
385}
386
387static void mmhub_v3_3_disable_identity_aperture(struct amdgpu_device *adev)
388{
389	WREG32_SOC15(MMHUB, 0,
390		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
391		     0xFFFFFFFF);
392	WREG32_SOC15(MMHUB, 0,
393		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
394		     0x0000000F);
395
396	WREG32_SOC15(MMHUB, 0,
397		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
398	WREG32_SOC15(MMHUB, 0,
399		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
400
401	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
402		     0);
403	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
404		     0);
405}
406
407static void mmhub_v3_3_setup_vmid_config(struct amdgpu_device *adev)
408{
409	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
410	int i;
411	uint32_t tmp;
412
413	for (i = 0; i <= 14; i++) {
414		tmp = RREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance);
415		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
416		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
417				    adev->vm_manager.num_level);
418		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
419				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
420		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
421				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
422				    1);
423		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
424				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
425		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
426				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
427		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
428				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
429		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
430				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
431		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
432				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
433		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
434				    PAGE_TABLE_BLOCK_SIZE,
435				    adev->vm_manager.block_size - 9);
436		/* Send no-retry XNACK on fault to suppress VM fault storm. */
437		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
438				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
439				    !amdgpu_noretry);
440		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL,
441				    i * hub->ctx_distance, tmp);
442		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
443				    i * hub->ctx_addr_distance, 0);
444		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
445				    i * hub->ctx_addr_distance, 0);
446		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
447				    i * hub->ctx_addr_distance,
448				    lower_32_bits(adev->vm_manager.max_pfn - 1));
449		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
450				    i * hub->ctx_addr_distance,
451				    upper_32_bits(adev->vm_manager.max_pfn - 1));
452	}
453
454	hub->vm_cntx_cntl = tmp;
455}
456
457static void mmhub_v3_3_program_invalidation(struct amdgpu_device *adev)
458{
459	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
460	unsigned int i;
461
462	for (i = 0; i < 18; ++i) {
463		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
464				    i * hub->eng_addr_distance, 0xffffffff);
465		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
466				    i * hub->eng_addr_distance, 0x1f);
467	}
468}
469
470static void mmhub_v3_3_init_saw_regs(struct amdgpu_device *adev)
471{
472	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
473	uint32_t tmp;
474
475	/* Program page table base, gart start, gart end */
476	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
477			lower_32_bits(pt_base >> 12));
478	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
479			upper_32_bits(pt_base >> 12));
480
481	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
482		     (u32)(adev->gmc.gart_start >> 12));
483	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
484		     (u32)(adev->gmc.gart_start >> 44));
485
486	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
487		     (u32)(adev->gmc.gart_end >> 12));
488	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
489		     (u32)(adev->gmc.gart_end >> 44));
490
491	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_CNTL);
492	tmp = REG_SET_FIELD(tmp, MMVM_L2_SAW_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
493	tmp = REG_SET_FIELD(tmp, MMVM_L2_SAW_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
494	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXT0_CNTL, tmp);
495
496	/* Disable all contexts except context 0 */
497	tmp = 0xfffe;
498	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CONTEXTS_DISABLE, tmp);
499
500	/* Program saw cntl4 */
501	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CNTL4);
502	tmp = REG_SET_FIELD(tmp, MMVM_L2_SAW_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 1);
503	tmp = REG_SET_FIELD(tmp, MMVM_L2_SAW_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 1);
504	WREG32_SOC15(MMHUB, 0, regMMVM_L2_SAW_CNTL4, tmp);
505}
506
507static void mmhub_v3_3_enable_tls(struct amdgpu_device *adev)
508{
509	WREG32_SOC15(MMHUB, 0, regDAGB0_L1TLB_REG_RW_3_3, 0);
510	WREG32_SOC15(MMHUB, 0, regDAGB1_L1TLB_REG_RW_3_3, 3);
511}
512
513static int mmhub_v3_3_gart_enable(struct amdgpu_device *adev)
514{
515	/* GART Enable. */
516	mmhub_v3_3_init_gart_aperture_regs(adev);
517	mmhub_v3_3_init_system_aperture_regs(adev);
518	mmhub_v3_3_init_tlb_regs(adev);
519	mmhub_v3_3_init_cache_regs(adev);
520
521	mmhub_v3_3_enable_system_domain(adev);
522	mmhub_v3_3_disable_identity_aperture(adev);
523	mmhub_v3_3_setup_vmid_config(adev);
524	mmhub_v3_3_program_invalidation(adev);
525
526	/* standalone alone walker init */
527	mmhub_v3_3_init_saw_regs(adev);
528
529	/* enable mmhub tls */
530	mmhub_v3_3_enable_tls(adev);
531
532	return 0;
533}
534
535static void mmhub_v3_3_gart_disable(struct amdgpu_device *adev)
536{
537	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
538	u32 tmp;
539	u32 i;
540
541	/* Disable all tables */
542	for (i = 0; i < 16; i++)
543		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_CNTL,
544				    i * hub->ctx_distance, 0);
545
546	/* Setup TLB control */
547	tmp = RREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL);
548	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
549	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
550			    ENABLE_ADVANCED_DRIVER_MODEL, 0);
551	WREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL, tmp);
552
553	/* Setup L2 cache */
554	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL);
555	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0);
556	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL, tmp);
557	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL3, 0);
558}
559
560/**
561 * mmhub_v3_3_set_fault_enable_default - update GART/VM fault handling
562 *
563 * @adev: amdgpu_device pointer
564 * @value: true redirects VM faults to the default page
565 */
566static void mmhub_v3_3_set_fault_enable_default(struct amdgpu_device *adev,
567						  bool value)
568{
569	u32 tmp;
570
571	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL);
572	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
573			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
574	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
575			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
576	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
577			    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
578	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
579			    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
580	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
581			    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
582			    value);
583	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
584			    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
585	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
586			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
587	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
588			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
589	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
590			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
591	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
592			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
593	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
594			    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
595	if (!value) {
596		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
597				CRASH_ON_NO_RETRY_FAULT, 1);
598		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
599				CRASH_ON_RETRY_FAULT, 1);
600	}
601	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL, tmp);
602}
603
604static const struct amdgpu_vmhub_funcs mmhub_v3_3_vmhub_funcs = {
605	.print_l2_protection_fault_status = mmhub_v3_3_print_l2_protection_fault_status,
606	.get_invalidate_req = mmhub_v3_3_get_invalidate_req,
607};
608
609static void mmhub_v3_3_init(struct amdgpu_device *adev)
610{
611	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
612
613	hub->ctx0_ptb_addr_lo32 =
614		SOC15_REG_OFFSET(MMHUB, 0,
615				 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
616	hub->ctx0_ptb_addr_hi32 =
617		SOC15_REG_OFFSET(MMHUB, 0,
618				 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
619	hub->vm_inv_eng0_sem =
620		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_SEM);
621	hub->vm_inv_eng0_req =
622		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_REQ);
623	hub->vm_inv_eng0_ack =
624		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ACK);
625	hub->vm_context0_cntl =
626		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_CNTL);
627	hub->vm_l2_pro_fault_status =
628		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_STATUS);
629	hub->vm_l2_pro_fault_cntl =
630		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL);
631
632	hub->ctx_distance = regMMVM_CONTEXT1_CNTL - regMMVM_CONTEXT0_CNTL;
633	hub->ctx_addr_distance = regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
634		regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
635	hub->eng_distance = regMMVM_INVALIDATE_ENG1_REQ -
636		regMMVM_INVALIDATE_ENG0_REQ;
637	hub->eng_addr_distance = regMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
638		regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
639
640	hub->vm_cntx_cntl_vm_fault = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
641		MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
642		MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
643		MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
644		MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
645		MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
646		MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
647
648	hub->vmhub_funcs = &mmhub_v3_3_vmhub_funcs;
649}
650
651static u64 mmhub_v3_3_get_fb_location(struct amdgpu_device *adev)
652{
653	u64 base;
654
655	base = RREG32_SOC15(MMHUB, 0, regMMMC_VM_FB_LOCATION_BASE);
656	base &= MMMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
657	base <<= 24;
658
659	return base;
660}
661
662static u64 mmhub_v3_3_get_mc_fb_offset(struct amdgpu_device *adev)
663{
664	u64 offset;
665
666	offset = RREG32_SOC15(MMHUB, 0, regMMMC_VM_FB_OFFSET);
667	offset &= MMMC_VM_FB_OFFSET__FB_OFFSET_MASK;
668	offset <<= 24;
669
670	return offset;
671}
672
673static void mmhub_v3_3_update_medium_grain_clock_gating(struct amdgpu_device *adev,
674							  bool enable)
675{
676	uint32_t def, data;
677
678	def = data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG);
679
680	if (enable)
681		data |= MM_ATC_L2_MISC_CG__ENABLE_MASK;
682	else
683		data &= ~MM_ATC_L2_MISC_CG__ENABLE_MASK;
684
685	if (def != data)
686		WREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG, data);
687}
688
689static void mmhub_v3_3_update_medium_grain_light_sleep(struct amdgpu_device *adev,
690							 bool enable)
691{
692	uint32_t def, data;
693
694	def = data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG);
695
696	if (enable)
697		data |= MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
698	else
699		data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
700
701	if (def != data)
702		WREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG, data);
703}
704
705static int mmhub_v3_3_set_clockgating(struct amdgpu_device *adev,
706					enum amd_clockgating_state state)
707{
708	if (amdgpu_sriov_vf(adev))
709		return 0;
710
711	mmhub_v3_3_update_medium_grain_clock_gating(adev,
712			state == AMD_CG_STATE_GATE);
713	mmhub_v3_3_update_medium_grain_light_sleep(adev,
714			state == AMD_CG_STATE_GATE);
715	return 0;
716}
717
718static void mmhub_v3_3_get_clockgating(struct amdgpu_device *adev, u64 *flags)
719{
720	u32 data;
721
722	if (amdgpu_sriov_vf(adev))
723		*flags = 0;
724
725	data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG);
726
727	/* AMD_CG_SUPPORT_MC_MGCG */
728	if (data & MM_ATC_L2_MISC_CG__ENABLE_MASK)
729		*flags |= AMD_CG_SUPPORT_MC_MGCG;
730
731	/* AMD_CG_SUPPORT_MC_LS */
732	if (data & MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
733		*flags |= AMD_CG_SUPPORT_MC_LS;
734}
735
736const struct amdgpu_mmhub_funcs mmhub_v3_3_funcs = {
737	.init = mmhub_v3_3_init,
738	.get_fb_location = mmhub_v3_3_get_fb_location,
739	.get_mc_fb_offset = mmhub_v3_3_get_mc_fb_offset,
740	.gart_enable = mmhub_v3_3_gart_enable,
741	.set_fault_enable_default = mmhub_v3_3_set_fault_enable_default,
742	.gart_disable = mmhub_v3_3_gart_disable,
743	.set_clockgating = mmhub_v3_3_set_clockgating,
744	.get_clockgating = mmhub_v3_3_get_clockgating,
745	.setup_vm_pt_regs = mmhub_v3_3_setup_vm_pt_regs,
746};