File Index
Files and directories
Sign in
FILE

mmhub_v3_0_1.c

File preview with syntax highlighting.

linux-6.19.14/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c C 
  1/*
  2 * Copyright 2022 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include "amdgpu.h"
 25#include "mmhub_v3_0_1.h"
 26
 27#include "mmhub/mmhub_3_0_1_offset.h"
 28#include "mmhub/mmhub_3_0_1_sh_mask.h"
 29#include "navi10_enum.h"
 30
 31#include "soc15_common.h"
 32
 33#define regMMVM_L2_CNTL3_DEFAULT				0x80100007
 34#define regMMVM_L2_CNTL4_DEFAULT				0x000000c1
 35#define regMMVM_L2_CNTL5_DEFAULT				0x00003fe0
 36
 37static const char *mmhub_client_ids_v3_0_1[][2] = {
 38	[0][0] = "VMC",
 39	[1][0] = "ISPXT",
 40	[2][0] = "ISPIXT",
 41	[4][0] = "DCEDMC",
 42	[5][0] = "DCEVGA",
 43	[6][0] = "MP0",
 44	[7][0] = "MP1",
 45	[8][0] = "MPM",
 46	[12][0] = "ISPTNR",
 47	[14][0] = "ISPCRD0",
 48	[15][0] = "ISPCRD1",
 49	[16][0] = "ISPCRD2",
 50	[22][0] = "HDP",
 51	[23][0] = "LSDMA",
 52	[24][0] = "JPEG",
 53	[27][0] = "VSCH",
 54	[28][0] = "VCNU",
 55	[29][0] = "VCN",
 56	[1][1] = "ISPXT",
 57	[2][1] = "ISPIXT",
 58	[3][1] = "DCEDWB",
 59	[4][1] = "DCEDMC",
 60	[5][1] = "DCEVGA",
 61	[6][1] = "MP0",
 62	[7][1] = "MP1",
 63	[8][1] = "MPM",
 64	[10][1] = "ISPMWR0",
 65	[11][1] = "ISPMWR1",
 66	[12][1] = "ISPTNR",
 67	[13][1] = "ISPSWR",
 68	[14][1] = "ISPCWR0",
 69	[15][1] = "ISPCWR1",
 70	[16][1] = "ISPCWR2",
 71	[17][1] = "ISPCWR3",
 72	[18][1] = "XDP",
 73	[21][1] = "OSSSYS",
 74	[22][1] = "HDP",
 75	[23][1] = "LSDMA",
 76	[24][1] = "JPEG",
 77	[27][1] = "VSCH",
 78	[28][1] = "VCNU",
 79	[29][1] = "VCN",
 80};
 81
 82static uint32_t mmhub_v3_0_1_get_invalidate_req(unsigned int vmid,
 83						uint32_t flush_type)
 84{
 85	u32 req = 0;
 86
 87	/* invalidate using legacy mode on vmid*/
 88	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ,
 89			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
 90	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
 91	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
 92	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
 93	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
 94	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
 95	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
 96	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ,
 97			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
 98
 99	return req;
100}
101
102static void
103mmhub_v3_0_1_print_l2_protection_fault_status(struct amdgpu_device *adev,
104					      uint32_t status)
105{
106	uint32_t cid, rw;
107	const char *mmhub_cid = NULL;
108
109	cid = REG_GET_FIELD(status,
110			    MMVM_L2_PROTECTION_FAULT_STATUS, CID);
111	rw = REG_GET_FIELD(status,
112			   MMVM_L2_PROTECTION_FAULT_STATUS, RW);
113
114	dev_err(adev->dev,
115		"MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
116		status);
117
118	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
119	case IP_VERSION(3, 0, 1):
120		mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_0_1) ?
121			mmhub_client_ids_v3_0_1[cid][rw] : NULL;
122		break;
123	default:
124		mmhub_cid = NULL;
125		break;
126	}
127
128	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
129		mmhub_cid ? mmhub_cid : "unknown", cid);
130	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
131		REG_GET_FIELD(status,
132		MMVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
133	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
134		REG_GET_FIELD(status,
135		MMVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
136	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
137		REG_GET_FIELD(status,
138		MMVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
139	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
140		REG_GET_FIELD(status,
141		MMVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
142	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
143}
144
145static void mmhub_v3_0_1_setup_vm_pt_regs(struct amdgpu_device *adev,
146					  uint32_t vmid,
147					  uint64_t page_table_base)
148{
149	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
150
151	WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
152			    hub->ctx_addr_distance * vmid,
153			    lower_32_bits(page_table_base));
154
155	WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
156			    hub->ctx_addr_distance * vmid,
157			    upper_32_bits(page_table_base));
158}
159
160static void mmhub_v3_0_1_init_gart_aperture_regs(struct amdgpu_device *adev)
161{
162	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
163
164	mmhub_v3_0_1_setup_vm_pt_regs(adev, 0, pt_base);
165
166	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
167		     (u32)(adev->gmc.gart_start >> 12));
168	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
169		     (u32)(adev->gmc.gart_start >> 44));
170
171	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
172		     (u32)(adev->gmc.gart_end >> 12));
173	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
174		     (u32)(adev->gmc.gart_end >> 44));
175}
176
177static void mmhub_v3_0_1_init_system_aperture_regs(struct amdgpu_device *adev)
178{
179	uint64_t value;
180	uint32_t tmp;
181
182	/* Program the AGP BAR */
183	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BASE, 0);
184	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
185	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
186
187	/*
188	 * the new L1 policy will block SRIOV guest from writing
189	 * these regs, and they will be programed at host.
190	 * so skip programing these regs.
191	 */
192	/* Program the system aperture low logical page number. */
193	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR,
194		     min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
195	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
196		     max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
197
198	/* Set default page address. */
199	value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
200	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
201		     (u32)(value >> 12));
202	WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
203		     (u32)(value >> 44));
204
205	/* Program "protection fault". */
206	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
207		     (u32)(adev->dummy_page_addr >> 12));
208	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
209		     (u32)((u64)adev->dummy_page_addr >> 44));
210
211	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL2);
212	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
213			    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
214	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL2, tmp);
215}
216
217static void mmhub_v3_0_1_init_tlb_regs(struct amdgpu_device *adev)
218{
219	uint32_t tmp;
220
221	/* Setup TLB control */
222	tmp = RREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL);
223
224	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
225	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
226	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
227			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
228	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
229			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
230	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
231	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
232			    MTYPE, MTYPE_UC); /* UC, uncached */
233
234	WREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL, tmp);
235}
236
237static void mmhub_v3_0_1_init_cache_regs(struct amdgpu_device *adev)
238{
239	uint32_t tmp;
240
241	/* Setup L2 cache */
242	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL);
243	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1);
244	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
245	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
246			    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
247	/* XXX for emulation, Refer to closed source code.*/
248	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
249			    0);
250	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
251	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
252	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
253	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL, tmp);
254
255	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL2);
256	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
257	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
258	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL2, tmp);
259
260	tmp = regMMVM_L2_CNTL3_DEFAULT;
261	if (adev->gmc.translate_further) {
262		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 12);
263		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
264				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
265	} else {
266		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 9);
267		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
268				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
269	}
270	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL3, tmp);
271
272	tmp = regMMVM_L2_CNTL4_DEFAULT;
273	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
274	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
275	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL4, tmp);
276
277	tmp = regMMVM_L2_CNTL5_DEFAULT;
278	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
279	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL5, tmp);
280}
281
282static void mmhub_v3_0_1_enable_system_domain(struct amdgpu_device *adev)
283{
284	uint32_t tmp;
285
286	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_CNTL);
287	tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
288	tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
289	tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
290			    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
291	WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_CNTL, tmp);
292}
293
294static void mmhub_v3_0_1_disable_identity_aperture(struct amdgpu_device *adev)
295{
296	WREG32_SOC15(MMHUB, 0,
297		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
298		     0xFFFFFFFF);
299	WREG32_SOC15(MMHUB, 0,
300		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
301		     0x0000000F);
302
303	WREG32_SOC15(MMHUB, 0,
304		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
305	WREG32_SOC15(MMHUB, 0,
306		     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
307
308	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
309		     0);
310	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
311		     0);
312}
313
314static void mmhub_v3_0_1_setup_vmid_config(struct amdgpu_device *adev)
315{
316	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
317	int i;
318	uint32_t tmp;
319
320	for (i = 0; i <= 14; i++) {
321		tmp = RREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance);
322		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
323		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
324				    adev->vm_manager.num_level);
325		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
326				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
327		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
328				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
329				    1);
330		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
331				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
332		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
333				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
334		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
335				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
336		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
337				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
338		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
339				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
340		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
341				    PAGE_TABLE_BLOCK_SIZE,
342				    adev->vm_manager.block_size - 9);
343		/* Send no-retry XNACK on fault to suppress VM fault storm. */
344		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
345				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
346				    !amdgpu_noretry);
347		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL,
348				    i * hub->ctx_distance, tmp);
349		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
350				    i * hub->ctx_addr_distance, 0);
351		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
352				    i * hub->ctx_addr_distance, 0);
353		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
354				    i * hub->ctx_addr_distance,
355				    lower_32_bits(adev->vm_manager.max_pfn - 1));
356		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
357				    i * hub->ctx_addr_distance,
358				    upper_32_bits(adev->vm_manager.max_pfn - 1));
359	}
360
361	hub->vm_cntx_cntl = tmp;
362}
363
364static void mmhub_v3_0_1_program_invalidation(struct amdgpu_device *adev)
365{
366	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
367	unsigned i;
368
369	for (i = 0; i < 18; ++i) {
370		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
371				    i * hub->eng_addr_distance, 0xffffffff);
372		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
373				    i * hub->eng_addr_distance, 0x1f);
374	}
375}
376
377static int mmhub_v3_0_1_gart_enable(struct amdgpu_device *adev)
378{
379	/* GART Enable. */
380	mmhub_v3_0_1_init_gart_aperture_regs(adev);
381	mmhub_v3_0_1_init_system_aperture_regs(adev);
382	mmhub_v3_0_1_init_tlb_regs(adev);
383	mmhub_v3_0_1_init_cache_regs(adev);
384
385	mmhub_v3_0_1_enable_system_domain(adev);
386	mmhub_v3_0_1_disable_identity_aperture(adev);
387	mmhub_v3_0_1_setup_vmid_config(adev);
388	mmhub_v3_0_1_program_invalidation(adev);
389
390	return 0;
391}
392
393static void mmhub_v3_0_1_gart_disable(struct amdgpu_device *adev)
394{
395	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
396	u32 tmp;
397	u32 i;
398
399	/* Disable all tables */
400	for (i = 0; i < 16; i++)
401		WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_CNTL,
402				    i * hub->ctx_distance, 0);
403
404	/* Setup TLB control */
405	tmp = RREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL);
406	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
407	tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
408			    ENABLE_ADVANCED_DRIVER_MODEL, 0);
409	WREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL, tmp);
410
411	/* Setup L2 cache */
412	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL);
413	tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0);
414	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL, tmp);
415	WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL3, 0);
416}
417
418/**
419 * mmhub_v3_0_1_set_fault_enable_default - update GART/VM fault handling
420 *
421 * @adev: amdgpu_device pointer
422 * @value: true redirects VM faults to the default page
423 */
424static void mmhub_v3_0_1_set_fault_enable_default(struct amdgpu_device *adev,
425						  bool value)
426{
427	u32 tmp;
428
429	tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL);
430	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
431			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
432	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
433			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
434	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
435			    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
436	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
437			    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
438	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
439			    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
440			    value);
441	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
442			    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
443	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
444			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
445	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
446			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
447	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
448			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
449	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
450			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
451	tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
452			    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
453	if (!value) {
454		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
455				CRASH_ON_NO_RETRY_FAULT, 1);
456		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL,
457				CRASH_ON_RETRY_FAULT, 1);
458	}
459	WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL, tmp);
460}
461
462static const struct amdgpu_vmhub_funcs mmhub_v3_0_1_vmhub_funcs = {
463	.print_l2_protection_fault_status = mmhub_v3_0_1_print_l2_protection_fault_status,
464	.get_invalidate_req = mmhub_v3_0_1_get_invalidate_req,
465};
466
467static void mmhub_v3_0_1_init(struct amdgpu_device *adev)
468{
469	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
470
471	hub->ctx0_ptb_addr_lo32 =
472		SOC15_REG_OFFSET(MMHUB, 0,
473				 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
474	hub->ctx0_ptb_addr_hi32 =
475		SOC15_REG_OFFSET(MMHUB, 0,
476				 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
477	hub->vm_inv_eng0_sem =
478		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_SEM);
479	hub->vm_inv_eng0_req =
480		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_REQ);
481	hub->vm_inv_eng0_ack =
482		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ACK);
483	hub->vm_context0_cntl =
484		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_CNTL);
485	hub->vm_l2_pro_fault_status =
486		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_STATUS);
487	hub->vm_l2_pro_fault_cntl =
488		SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL);
489
490	hub->ctx_distance = regMMVM_CONTEXT1_CNTL - regMMVM_CONTEXT0_CNTL;
491	hub->ctx_addr_distance = regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
492		regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
493	hub->eng_distance = regMMVM_INVALIDATE_ENG1_REQ -
494		regMMVM_INVALIDATE_ENG0_REQ;
495	hub->eng_addr_distance = regMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
496		regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
497
498	hub->vm_cntx_cntl_vm_fault = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
499		MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
500		MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
501		MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
502		MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
503		MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
504		MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
505
506	hub->vmhub_funcs = &mmhub_v3_0_1_vmhub_funcs;
507}
508
509static u64 mmhub_v3_0_1_get_fb_location(struct amdgpu_device *adev)
510{
511	u64 base;
512
513	base = RREG32_SOC15(MMHUB, 0, regMMMC_VM_FB_LOCATION_BASE);
514	base &= MMMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
515	base <<= 24;
516
517	return base;
518}
519
520static u64 mmhub_v3_0_1_get_mc_fb_offset(struct amdgpu_device *adev)
521{
522	return (u64)RREG32_SOC15(MMHUB, 0, regMMMC_VM_FB_OFFSET) << 24;
523}
524
525static void mmhub_v3_0_1_update_medium_grain_clock_gating(struct amdgpu_device *adev,
526							  bool enable)
527{
528	uint32_t def, data;
529
530	def = data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG);
531
532	if (enable)
533		data |= MM_ATC_L2_MISC_CG__ENABLE_MASK;
534	else
535		data &= ~MM_ATC_L2_MISC_CG__ENABLE_MASK;
536
537	if (def != data)
538		WREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG, data);
539}
540
541static void mmhub_v3_0_1_update_medium_grain_light_sleep(struct amdgpu_device *adev,
542							 bool enable)
543{
544	uint32_t def, data;
545
546	def = data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG);
547
548	if (enable)
549		data |= MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
550	else
551		data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
552
553	if (def != data)
554		WREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG, data);
555}
556
557static int mmhub_v3_0_1_set_clockgating(struct amdgpu_device *adev,
558					enum amd_clockgating_state state)
559{
560	if (amdgpu_sriov_vf(adev))
561		return 0;
562
563	mmhub_v3_0_1_update_medium_grain_clock_gating(adev,
564			state == AMD_CG_STATE_GATE);
565	mmhub_v3_0_1_update_medium_grain_light_sleep(adev,
566			state == AMD_CG_STATE_GATE);
567	return 0;
568}
569
570static void mmhub_v3_0_1_get_clockgating(struct amdgpu_device *adev, u64 *flags)
571{
572	int data;
573
574	if (amdgpu_sriov_vf(adev))
575		*flags = 0;
576
577	data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG);
578
579	/* AMD_CG_SUPPORT_MC_MGCG */
580	if (data & MM_ATC_L2_MISC_CG__ENABLE_MASK)
581		*flags |= AMD_CG_SUPPORT_MC_MGCG;
582
583	/* AMD_CG_SUPPORT_MC_LS */
584	if (data & MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
585		*flags |= AMD_CG_SUPPORT_MC_LS;
586}
587
588const struct amdgpu_mmhub_funcs mmhub_v3_0_1_funcs = {
589	.init = mmhub_v3_0_1_init,
590	.get_fb_location = mmhub_v3_0_1_get_fb_location,
591	.get_mc_fb_offset = mmhub_v3_0_1_get_mc_fb_offset,
592	.gart_enable = mmhub_v3_0_1_gart_enable,
593	.set_fault_enable_default = mmhub_v3_0_1_set_fault_enable_default,
594	.gart_disable = mmhub_v3_0_1_gart_disable,
595	.set_clockgating = mmhub_v3_0_1_set_clockgating,
596	.get_clockgating = mmhub_v3_0_1_get_clockgating,
597	.setup_vm_pt_regs = mmhub_v3_0_1_setup_vm_pt_regs,
598};
~/ Read-only file preview.