vmmgh100.c

  1/* SPDX-License-Identifier: MIT
  2 *
  3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
  4 */
  5#include "vmm.h"
  6
  7#include <subdev/fb.h>
  8
  9#include <nvhw/drf.h>
 10#include <nvhw/ref/gh100/dev_mmu.h>
 11
 12static inline void
 13gh100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
 14		  struct nvkm_vmm_map *map, u64 addr)
 15{
 16	u64 data = addr | map->type;
 17
 18	while (ptes--) {
 19		VMM_WO064(pt, vmm, ptei++ * NV_MMU_VER3_PTE__SIZE, data);
 20		data += map->next;
 21	}
 22}
 23
 24static void
 25gh100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
 26		  struct nvkm_vmm_map *map)
 27{
 28	VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte);
 29}
 30
 31static void
 32gh100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
 33		  struct nvkm_vmm_map *map)
 34{
 35	if (map->page->shift == PAGE_SHIFT) {
 36		VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
 37
 38		nvkm_kmap(pt->memory);
 39		while (ptes--) {
 40			const u64 data = *map->dma++ | map->type;
 41
 42			VMM_WO064(pt, vmm, ptei++ * NV_MMU_VER3_PTE__SIZE, data);
 43		}
 44		nvkm_done(pt->memory);
 45		return;
 46	}
 47
 48	VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte);
 49}
 50
 51static void
 52gh100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
 53		  struct nvkm_vmm_map *map)
 54{
 55	VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte);
 56}
 57
 58static void
 59gh100_vmm_pgt_sparse(struct nvkm_vmm *vmm,
 60		     struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
 61{
 62	const u64 data = NVDEF(NV_MMU, VER3_PTE, PCF, SPARSE);
 63
 64	VMM_FO064(pt, vmm, ptei * NV_MMU_VER3_PTE__SIZE, data, ptes);
 65}
 66
 67static const struct nvkm_vmm_desc_func
 68gh100_vmm_desc_spt = {
 69	.unmap = gf100_vmm_pgt_unmap,
 70	.sparse = gh100_vmm_pgt_sparse,
 71	.mem = gh100_vmm_pgt_mem,
 72	.dma = gh100_vmm_pgt_dma,
 73	.sgl = gh100_vmm_pgt_sgl,
 74};
 75
 76static void
 77gh100_vmm_lpt_invalid(struct nvkm_vmm *vmm,
 78		      struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
 79{
 80	const u64 data = NVDEF(NV_MMU, VER3_PTE, PCF, NO_VALID_4KB_PAGE);
 81
 82	VMM_FO064(pt, vmm, ptei * NV_MMU_VER3_PTE__SIZE, data, ptes);
 83}
 84
 85static const struct nvkm_vmm_desc_func
 86gh100_vmm_desc_lpt = {
 87	.invalid = gh100_vmm_lpt_invalid,
 88	.unmap = gf100_vmm_pgt_unmap,
 89	.sparse = gh100_vmm_pgt_sparse,
 90	.mem = gh100_vmm_pgt_mem,
 91};
 92
 93static inline void
 94gh100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
 95		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
 96{
 97	u64 data = addr | map->type;
 98
 99	while (ptes--) {
100		VMM_WO128(pt, vmm, ptei++ * NV_MMU_VER3_DUAL_PDE__SIZE, data, 0ULL);
101		data += map->next;
102	}
103}
104
105static void
106gh100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
107		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
108{
109	VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gh100_vmm_pd0_pte);
110}
111
112static inline bool
113gh100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data)
114{
115	switch (nvkm_memory_target(pt->memory)) {
116	case NVKM_MEM_TARGET_VRAM:
117		*data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, VIDEO_MEMORY);
118		*data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_CACHED_ATS_NOT_ALLOWED);
119		break;
120	case NVKM_MEM_TARGET_HOST:
121		*data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, SYSTEM_COHERENT_MEMORY);
122		*data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_UNCACHED_ATS_ALLOWED);
123		break;
124	case NVKM_MEM_TARGET_NCOH:
125		*data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, SYSTEM_NON_COHERENT_MEMORY);
126		*data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_CACHED_ATS_ALLOWED);
127		break;
128	default:
129		WARN_ON(1);
130		return false;
131	}
132
133	*data |= pt->addr;
134	return true;
135}
136
137static void
138gh100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
139{
140	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
141	struct nvkm_mmu_pt *pd = pgd->pt[0];
142	u64 data[2] = {};
143
144	if (pgt->pt[0] && !gh100_vmm_pde(pgt->pt[0], &data[0]))
145		return;
146	if (pgt->pt[1] && !gh100_vmm_pde(pgt->pt[1], &data[1]))
147		return;
148
149	nvkm_kmap(pd->memory);
150	VMM_WO128(pd, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, data[0], data[1]);
151	nvkm_done(pd->memory);
152}
153
154static void
155gh100_vmm_pd0_sparse(struct nvkm_vmm *vmm,
156		     struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
157{
158	const u64 data = NVDEF(NV_MMU, VER3_DUAL_PDE, PCF_BIG, SPARSE_ATS_ALLOWED);
159
160	VMM_FO128(pt, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, data, 0ULL, pdes);
161}
162
163static void
164gh100_vmm_pd0_unmap(struct nvkm_vmm *vmm,
165		    struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
166{
167	VMM_FO128(pt, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, 0ULL, 0ULL, pdes);
168}
169
170static const struct nvkm_vmm_desc_func
171gh100_vmm_desc_pd0 = {
172	.unmap = gh100_vmm_pd0_unmap,
173	.sparse = gh100_vmm_pd0_sparse,
174	.pde = gh100_vmm_pd0_pde,
175	.mem = gh100_vmm_pd0_mem,
176};
177
178static void
179gh100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
180{
181	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
182	struct nvkm_mmu_pt *pd = pgd->pt[0];
183	u64 data = 0;
184
185	if (!gh100_vmm_pde(pgt->pt[0], &data))
186		return;
187
188	nvkm_kmap(pd->memory);
189	VMM_WO064(pd, vmm, pdei * NV_MMU_VER3_PDE__SIZE, data);
190	nvkm_done(pd->memory);
191}
192
193static const struct nvkm_vmm_desc_func
194gh100_vmm_desc_pd1 = {
195	.unmap = gf100_vmm_pgt_unmap,
196	.sparse = gh100_vmm_pgt_sparse,
197	.pde = gh100_vmm_pd1_pde,
198};
199
200static const struct nvkm_vmm_desc
201gh100_vmm_desc_16[] = {
202	{ LPT, 5,  8, 0x0100, &gh100_vmm_desc_lpt },
203	{ PGD, 8, 16, 0x1000, &gh100_vmm_desc_pd0 },
204	{ PGD, 9,  8, 0x1000, &gh100_vmm_desc_pd1 },
205	{ PGD, 9,  8, 0x1000, &gh100_vmm_desc_pd1 },
206	{ PGD, 9,  8, 0x1000, &gh100_vmm_desc_pd1 },
207	{ PGD, 1,  8, 0x1000, &gh100_vmm_desc_pd1 },
208	{}
209};
210
211static const struct nvkm_vmm_desc
212gh100_vmm_desc_12[] = {
213	{ SPT, 9,  8, 0x1000, &gh100_vmm_desc_spt },
214	{ PGD, 8, 16, 0x1000, &gh100_vmm_desc_pd0 },
215	{ PGD, 9,  8, 0x1000, &gh100_vmm_desc_pd1 },
216	{ PGD, 9,  8, 0x1000, &gh100_vmm_desc_pd1 },
217	{ PGD, 9,  8, 0x1000, &gh100_vmm_desc_pd1 },
218	{ PGD, 1,  8, 0x1000, &gh100_vmm_desc_pd1 },
219	{}
220};
221
222static int
223gh100_vmm_valid(struct nvkm_vmm *vmm, bool ro, bool priv, u8 kind, u8 comp,
224		struct nvkm_vmm_map *map)
225{
226	const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
227	const bool vol = target == NVKM_MEM_TARGET_HOST;
228	const struct nvkm_vmm_page *page = map->page;
229	u8 kind_inv, pcf;
230	int kindn, aper;
231	const u8 *kindm;
232
233	map->next = 1ULL << page->shift;
234	map->type = 0;
235
236	aper = vmm->func->aper(target);
237	if (WARN_ON(aper < 0))
238		return aper;
239
240	kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);
241	if (kind >= kindn || kindm[kind] == kind_inv) {
242		VMM_DEBUG(vmm, "kind %02x", kind);
243		return -EINVAL;
244	}
245
246	if (priv) {
247		if (ro) {
248			if (vol)
249				pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_UNCACHED_ACD;
250			else
251				pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_CACHED_ACD;
252		} else {
253			if (vol)
254				pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_UNCACHED_ACD;
255			else
256				pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_CACHED_ACD;
257		}
258	} else {
259		if (ro) {
260			if (vol)
261				pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_UNCACHED_ACD;
262			else
263				pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_CACHED_ACD;
264		} else {
265			if (vol)
266				pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACD;
267			else
268				pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACD;
269		}
270	}
271
272	map->type |= NVDEF(NV_MMU, VER3_PTE, VALID, TRUE);
273	map->type |= NVVAL(NV_MMU, VER3_PTE, APERTURE, aper);
274	map->type |= NVVAL(NV_MMU, VER3_PTE, PCF, pcf);
275	map->type |= NVVAL(NV_MMU, VER3_PTE, KIND, kind);
276	return 0;
277}
278
279static const struct nvkm_vmm_func
280gh100_vmm = {
281	.join = gv100_vmm_join,
282	.part = gf100_vmm_part,
283	.aper = gf100_vmm_aper,
284	.valid = gp100_vmm_valid,
285	.valid2 = gh100_vmm_valid,
286	.flush = tu102_vmm_flush,
287	.page = {
288		{ 56, &gh100_vmm_desc_16[5], NVKM_VMM_PAGE_Sxxx },
289		{ 47, &gh100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
290		{ 38, &gh100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
291		{ 29, &gh100_vmm_desc_16[2], NVKM_VMM_PAGE_SVxC },
292		{ 21, &gh100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC },
293		{ 16, &gh100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC },
294		{ 12, &gh100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx },
295		{}
296	}
297};
298
299int
300gh100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
301	      void *argv, u32 argc, struct lock_class_key *key,
302	      const char *name, struct nvkm_vmm **pvmm)
303{
304	return gp100_vmm_new_(&gh100_vmm, mmu, managed, addr, size,
305			      argv, argc, key, name, pvmm);
306}