xe_lmtt.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2023 Intel Corporation
  4 */
  5
  6#include <linux/align.h>
  7
  8#include <drm/drm_managed.h>
  9
 10#include "regs/xe_gt_regs.h"
 11
 12#include "xe_assert.h"
 13#include "xe_bo.h"
 14#include "xe_tlb_inval.h"
 15#include "xe_lmtt.h"
 16#include "xe_map.h"
 17#include "xe_mmio.h"
 18#include "xe_res_cursor.h"
 19#include "xe_sriov.h"
 20#include "xe_tile_sriov_printk.h"
 21
 22/**
 23 * DOC: Local Memory Translation Table
 24 *
 25 * The Local Memory Translation Table (LMTT) provides additional abstraction
 26 * when Virtual Function (VF) is accessing device Local Memory (VRAM).
 27 *
 28 * The Root LMTT Page Directory contains one entry for each VF. Entries are
 29 * indexed by the function number (1-based, index 0 is unused).
 30 *
 31 * See `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_.
 32 */
 33
 34#define lmtt_assert(lmtt, condition)	xe_tile_assert(lmtt_to_tile(lmtt), condition)
 35#define lmtt_debug(lmtt, msg...)	xe_tile_sriov_dbg_verbose(lmtt_to_tile(lmtt), "LMTT: " msg)
 36
 37static bool xe_has_multi_level_lmtt(struct xe_device *xe)
 38{
 39	return GRAPHICS_VERx100(xe) >= 1260;
 40}
 41
 42static struct xe_tile *lmtt_to_tile(struct xe_lmtt *lmtt)
 43{
 44	return container_of(lmtt, struct xe_tile, sriov.pf.lmtt);
 45}
 46
 47static struct xe_device *lmtt_to_xe(struct xe_lmtt *lmtt)
 48{
 49	return tile_to_xe(lmtt_to_tile(lmtt));
 50}
 51
 52static u64 lmtt_page_size(struct xe_lmtt *lmtt)
 53{
 54	return BIT_ULL(lmtt->ops->lmtt_pte_shift(0));
 55}
 56
 57static struct xe_lmtt_pt *lmtt_pt_alloc(struct xe_lmtt *lmtt, unsigned int level)
 58{
 59	unsigned int num_entries = level ? lmtt->ops->lmtt_pte_num(level) : 0;
 60	struct xe_lmtt_pt *pt;
 61	struct xe_bo *bo;
 62	int err;
 63
 64	pt = kzalloc(struct_size(pt, entries, num_entries), GFP_KERNEL);
 65	if (!pt) {
 66		err = -ENOMEM;
 67		goto out;
 68	}
 69
 70	bo = xe_bo_create_pin_map_novm(lmtt_to_xe(lmtt), lmtt_to_tile(lmtt),
 71				       PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
 72						  lmtt->ops->lmtt_pte_num(level)),
 73				       ttm_bo_type_kernel,
 74				       XE_BO_FLAG_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) |
 75				       XE_BO_FLAG_NEEDS_64K, false);
 76	if (IS_ERR(bo)) {
 77		err = PTR_ERR(bo);
 78		goto out_free_pt;
 79	}
 80
 81	lmtt_assert(lmtt, xe_bo_is_vram(bo));
 82	lmtt_debug(lmtt, "level=%u addr=%#llx\n", level, (u64)xe_bo_main_addr(bo, XE_PAGE_SIZE));
 83
 84	xe_map_memset(lmtt_to_xe(lmtt), &bo->vmap, 0, 0, xe_bo_size(bo));
 85
 86	pt->level = level;
 87	pt->bo = bo;
 88	return pt;
 89
 90out_free_pt:
 91	kfree(pt);
 92out:
 93	return ERR_PTR(err);
 94}
 95
 96static void lmtt_pt_free(struct xe_lmtt_pt *pt)
 97{
 98	lmtt_debug(&pt->bo->tile->sriov.pf.lmtt, "level=%u addr=%llx\n",
 99		   pt->level, (u64)xe_bo_main_addr(pt->bo, XE_PAGE_SIZE));
100
101	xe_bo_unpin_map_no_vm(pt->bo);
102	kfree(pt);
103}
104
105static int lmtt_init_pd(struct xe_lmtt *lmtt)
106{
107	struct xe_lmtt_pt *pd;
108
109	lmtt_assert(lmtt, !lmtt->pd);
110	lmtt_assert(lmtt, lmtt->ops->lmtt_root_pd_level());
111
112	pd = lmtt_pt_alloc(lmtt, lmtt->ops->lmtt_root_pd_level());
113	if (IS_ERR(pd))
114		return PTR_ERR(pd);
115
116	lmtt->pd = pd;
117	return 0;
118}
119
120static void lmtt_fini_pd(struct xe_lmtt *lmtt)
121{
122	struct xe_lmtt_pt *pd = lmtt->pd;
123	unsigned int num_entries = lmtt->ops->lmtt_pte_num(pd->level);
124	unsigned int n = 0;
125
126	/* make sure we don't leak */
127	for (n = 0; n < num_entries; n++)
128		lmtt_assert(lmtt, !pd->entries[n]);
129
130	lmtt->pd = NULL;
131	lmtt_pt_free(pd);
132}
133
134static void fini_lmtt(struct drm_device *drm, void *arg)
135{
136	struct xe_lmtt *lmtt = arg;
137
138	lmtt_assert(lmtt, !(!!lmtt->ops ^ !!lmtt->pd));
139
140	if (!lmtt->pd)
141		return;
142
143	lmtt_fini_pd(lmtt);
144	lmtt->ops = NULL;
145}
146
147/**
148 * xe_lmtt_init - LMTT software initialization.
149 * @lmtt: the &xe_lmtt to initialize
150 *
151 * The LMTT initialization requires two steps.
152 *
153 * The xe_lmtt_init() checks if LMTT is required on current device and selects
154 * and initialize proper variant of the LMTT Root Directory. Currently supported
155 * variants are `Two-Level LMTT Structure`_ and `Multi-Level LMTT Structure`_.
156 *
157 * In next step xe_lmtt_init_hw() will register this directory on the hardware.
158 *
159 * Notes:
160 * The LMTT allocations are managed and will be implicitly released on driver unload.
161 * This function shall be called only once and only when running as a PF driver.
162 * Any LMTT initialization failure should block VFs enabling.
163 *
164 * Return: 0 on success or a negative error code on failure.
165 */
166int xe_lmtt_init(struct xe_lmtt *lmtt)
167{
168	struct xe_device *xe = lmtt_to_xe(lmtt);
169	int err;
170
171	lmtt_assert(lmtt, IS_SRIOV_PF(xe));
172	lmtt_assert(lmtt, !lmtt->ops);
173
174	if (!xe_device_has_lmtt(xe))
175		return 0;
176
177	if (xe_has_multi_level_lmtt(xe))
178		lmtt->ops = &lmtt_ml_ops;
179	else
180		lmtt->ops = &lmtt_2l_ops;
181
182	err = lmtt_init_pd(lmtt);
183	if (unlikely(err))
184		goto fail;
185
186	return drmm_add_action_or_reset(&xe->drm, fini_lmtt, lmtt);
187
188fail:
189	lmtt->ops = NULL;
190	return err;
191}
192
193static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt)
194{
195	struct xe_tile *tile = lmtt_to_tile(lmtt);
196	struct xe_device *xe = tile_to_xe(tile);
197	dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE);
198	struct xe_gt *gt;
199	u8 id;
200
201	lmtt_debug(lmtt, "DIR offset %pad\n", &offset);
202	lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo));
203	lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K));
204
205	for_each_gt_on_tile(gt, tile, id)
206		xe_mmio_write32(&gt->mmio,
207				GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG,
208				LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K));
209}
210
211/**
212 * xe_lmtt_init_hw - Perform LMTT hardware initialization.
213 * @lmtt: the &xe_lmtt to initialize
214 *
215 * This function is a second step of the LMTT initialization.
216 * This function registers LMTT Root Directory prepared in xe_lmtt_init().
217 *
218 * This function shall be called after every hardware reset.
219 * This function shall be called only when running as a PF driver.
220 */
221void xe_lmtt_init_hw(struct xe_lmtt *lmtt)
222{
223	if (!lmtt->pd)
224		return;
225
226	lmtt_setup_dir_ptr(lmtt);
227}
228
229static int lmtt_invalidate_hw(struct xe_lmtt *lmtt)
230{
231	struct xe_tlb_inval_fence fences[XE_MAX_GT_PER_TILE];
232	struct xe_tlb_inval_fence *fence = fences;
233	struct xe_tile *tile = lmtt_to_tile(lmtt);
234	struct xe_gt *gt;
235	int result = 0;
236	int err;
237	u8 id;
238
239	for_each_gt_on_tile(gt, tile, id) {
240		xe_tlb_inval_fence_init(&gt->tlb_inval, fence, true);
241		err = xe_tlb_inval_all(&gt->tlb_inval, fence);
242		result = result ?: err;
243		fence++;
244	}
245
246	lmtt_debug(lmtt, "num_fences=%d err=%d\n", (int)(fence - fences), result);
247
248	/*
249	 * It is fine to wait for all fences, even for those which covers the
250	 * invalidation request that failed, as such fence should be already
251	 * marked as signaled.
252	 */
253	fence = fences;
254	for_each_gt_on_tile(gt, tile, id)
255		xe_tlb_inval_fence_wait(fence++);
256
257	return result;
258}
259
260/**
261 * xe_lmtt_invalidate_hw - Invalidate LMTT hardware.
262 * @lmtt: the &xe_lmtt to invalidate
263 *
264 * Send requests to all GuCs on this tile to invalidate all TLBs.
265 *
266 * This function should be called only when running as a PF driver.
267 */
268void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt)
269{
270	int err;
271
272	lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
273
274	err = lmtt_invalidate_hw(lmtt);
275	if (err)
276		xe_tile_sriov_err(lmtt_to_tile(lmtt), "LMTT invalidation failed (%pe)",
277				  ERR_PTR(err));
278}
279
280static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt,
281			   u64 pte, unsigned int idx)
282{
283	unsigned int level = pt->level;
284
285	lmtt_assert(lmtt, idx <= lmtt->ops->lmtt_pte_num(level));
286	lmtt_debug(lmtt, "WRITE level=%u index=%u pte=%#llx\n", level, idx, pte);
287
288	switch (lmtt->ops->lmtt_pte_size(level)) {
289	case sizeof(u32):
290		lmtt_assert(lmtt, !overflows_type(pte, u32));
291		lmtt_assert(lmtt, !pte || !iosys_map_rd(&pt->bo->vmap, idx * sizeof(u32), u32));
292
293		xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u32), u32, pte);
294		break;
295	case sizeof(u64):
296		lmtt_assert(lmtt, !pte || !iosys_map_rd(&pt->bo->vmap, idx * sizeof(u64), u64));
297
298		xe_map_wr(lmtt_to_xe(lmtt), &pt->bo->vmap, idx * sizeof(u64), u64, pte);
299		break;
300	default:
301		lmtt_assert(lmtt, !!!"invalid pte size");
302	}
303}
304
305static void lmtt_destroy_pt(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd)
306{
307	unsigned int num_entries = pd->level ? lmtt->ops->lmtt_pte_num(pd->level) : 0;
308	struct xe_lmtt_pt *pt;
309	unsigned int i;
310
311	for (i = 0; i < num_entries; i++) {
312		pt = pd->entries[i];
313		pd->entries[i] = NULL;
314		if (!pt)
315			continue;
316
317		lmtt_destroy_pt(lmtt, pt);
318	}
319
320	lmtt_pt_free(pd);
321}
322
323static void lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid)
324{
325	struct xe_lmtt_pt *pd = lmtt->pd;
326	struct xe_lmtt_pt *pt;
327
328	pt = pd->entries[vfid];
329	pd->entries[vfid] = NULL;
330	if (!pt)
331		return;
332
333	lmtt_write_pte(lmtt, pd, LMTT_PTE_INVALID, vfid);
334	lmtt_invalidate_hw(lmtt);
335
336	lmtt_assert(lmtt, pd->level > 0);
337	lmtt_assert(lmtt, pt->level == pd->level - 1);
338	lmtt_destroy_pt(lmtt, pt);
339}
340
341static int __lmtt_alloc_range(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pd,
342			      u64 start, u64 end)
343{
344	u64 pte_addr_shift = BIT_ULL(lmtt->ops->lmtt_pte_shift(pd->level));
345	u64 offset;
346	int err;
347
348	lmtt_assert(lmtt, pd->level > 0);
349
350	offset = start;
351	while (offset < end) {
352		struct xe_lmtt_pt *pt;
353		u64 next, pde, pt_addr;
354		unsigned int idx;
355
356		pt = lmtt_pt_alloc(lmtt, pd->level - 1);
357		if (IS_ERR(pt))
358			return PTR_ERR(pt);
359
360		pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE);
361
362		idx = lmtt->ops->lmtt_pte_index(offset, pd->level);
363		pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level);
364
365		lmtt_write_pte(lmtt, pd, pde, idx);
366
367		pd->entries[idx] = pt;
368
369		next = min(end, round_up(offset + 1, pte_addr_shift));
370
371		if (pt->level != 0) {
372			err = __lmtt_alloc_range(lmtt, pt, offset, next);
373			if (err)
374				return err;
375		}
376
377		offset = next;
378	}
379
380	return 0;
381}
382
383static int lmtt_alloc_range(struct xe_lmtt *lmtt, unsigned int vfid, u64 start, u64 end)
384{
385	struct xe_lmtt_pt *pd = lmtt->pd;
386	struct xe_lmtt_pt *pt;
387	u64 pt_addr;
388	u64 pde;
389	int err;
390
391	lmtt_assert(lmtt, pd->level > 0);
392	lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level));
393	lmtt_assert(lmtt, IS_ALIGNED(start, lmtt_page_size(lmtt)));
394	lmtt_assert(lmtt, IS_ALIGNED(end, lmtt_page_size(lmtt)));
395
396	if (pd->entries[vfid])
397		return -ENOTEMPTY;
398
399	pt = lmtt_pt_alloc(lmtt, pd->level - 1);
400	if (IS_ERR(pt))
401		return PTR_ERR(pt);
402
403	pt_addr = xe_bo_main_addr(pt->bo, XE_PAGE_SIZE);
404
405	pde = lmtt->ops->lmtt_pte_encode(pt_addr, pd->level);
406
407	lmtt_write_pte(lmtt, pd, pde, vfid);
408
409	pd->entries[vfid] = pt;
410
411	if (pt->level != 0) {
412		err = __lmtt_alloc_range(lmtt, pt, start, end);
413		if (err)
414			goto out_free_pt;
415	}
416
417	return 0;
418
419out_free_pt:
420	lmtt_pt_free(pt);
421	return err;
422}
423
424static struct xe_lmtt_pt *lmtt_leaf_pt(struct xe_lmtt *lmtt, unsigned int vfid, u64 addr)
425{
426	struct xe_lmtt_pt *pd = lmtt->pd;
427	struct xe_lmtt_pt *pt;
428
429	lmtt_assert(lmtt, vfid <= lmtt->ops->lmtt_pte_num(pd->level));
430	pt = pd->entries[vfid];
431
432	while (pt->level) {
433		lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <=
434			    lmtt->ops->lmtt_pte_num(pt->level));
435
436		pt = pt->entries[lmtt->ops->lmtt_pte_index(addr, pt->level)];
437
438		addr >>= lmtt->ops->lmtt_pte_shift(pt->level);
439	}
440
441	lmtt_assert(lmtt, lmtt->ops->lmtt_pte_index(addr, pt->level) <=
442		    lmtt->ops->lmtt_pte_num(pt->level));
443	lmtt_assert(lmtt, pt->level != pd->level);
444	lmtt_assert(lmtt, pt->level == 0);
445	return pt;
446}
447
448static void lmtt_insert_bo(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 start)
449{
450	u64 page_size = lmtt_page_size(lmtt);
451	struct xe_res_cursor cur;
452	struct xe_lmtt_pt *pt;
453	u64 addr, vram_offset;
454
455	lmtt_assert(lmtt, IS_ALIGNED(start, page_size));
456	lmtt_assert(lmtt, IS_ALIGNED(xe_bo_size(bo), page_size));
457	lmtt_assert(lmtt, xe_bo_is_vram(bo));
458
459	vram_offset = vram_region_gpu_offset(bo->ttm.resource);
460	xe_res_first(bo->ttm.resource, 0, xe_bo_size(bo), &cur);
461	while (cur.remaining) {
462		addr = xe_res_dma(&cur);
463		addr += vram_offset; /* XXX */
464
465		pt = lmtt_leaf_pt(lmtt, vfid, start);
466
467		lmtt_write_pte(lmtt, pt, lmtt->ops->lmtt_pte_encode(addr, 0),
468					 lmtt->ops->lmtt_pte_index(start, 0));
469
470		xe_res_next(&cur, page_size);
471		start += page_size;
472	}
473}
474
475/**
476 * xe_lmtt_prepare_pages - Create VF's LMTT Page Tables.
477 * @lmtt: the &xe_lmtt to update
478 * @vfid: the VF identifier (1-based)
479 * @range: top range of LMEM offset to be supported
480 *
481 * This function creates empty LMTT page tables for given VF to support
482 * up to maximum #range LMEM offset. The LMTT page tables created by this
483 * function must be released using xe_lmtt_drop_pages() function.
484 *
485 * Notes:
486 * This function shall be called only after successful LMTT initialization.
487 * See xe_lmtt_init().
488 *
489 * Return: 0 on success or a negative error code on failure.
490 */
491int xe_lmtt_prepare_pages(struct xe_lmtt *lmtt, unsigned int vfid, u64 range)
492{
493	lmtt_assert(lmtt, lmtt->pd);
494	lmtt_assert(lmtt, vfid);
495
496	return lmtt_alloc_range(lmtt, vfid, 0, range);
497}
498
499/**
500 * xe_lmtt_populate_pages - Update VF's LMTT Page Table Entries.
501 * @lmtt: the &xe_lmtt to update
502 * @vfid: the VF identifier (1-based)
503 * @bo: the buffer object with LMEM allocation to be mapped
504 * @offset: the offset at which #bo should be mapped
505 *
506 * This function updates VF's LMTT entries to use given buffer object as a backstore.
507 *
508 * Notes:
509 * This function shall be called only after successful preparation of the
510 * VF's LMTT Page Tables. See xe_lmtt_prepare().
511 *
512 * Return: 0 on success or a negative error code on failure.
513 */
514int xe_lmtt_populate_pages(struct xe_lmtt *lmtt, unsigned int vfid, struct xe_bo *bo, u64 offset)
515{
516	lmtt_assert(lmtt, lmtt->pd);
517	lmtt_assert(lmtt, vfid);
518
519	lmtt_insert_bo(lmtt, vfid, bo, offset);
520	return 0;
521}
522
523/**
524 * xe_lmtt_drop_pages - Remove VF's LMTT Pages.
525 * @lmtt: the &xe_lmtt to update
526 * @vfid: the VF identifier (1-based)
527 *
528 * This function removes all LMTT Page Tables prepared by xe_lmtt_prepare_pages().
529 *
530 * This function shall be called only after successful LMTT initialization.
531 * See xe_lmtt_init().
532 */
533void xe_lmtt_drop_pages(struct xe_lmtt *lmtt, unsigned int vfid)
534{
535	lmtt_assert(lmtt, lmtt->pd);
536	lmtt_assert(lmtt, vfid);
537
538	lmtt_drop_pages(lmtt, vfid);
539}
540
541/**
542 * xe_lmtt_estimate_pt_size - Estimate size of LMTT PT allocations.
543 * @lmtt: the &xe_lmtt
544 * @size: the size of the LMEM to be mapped over LMTT (including any offset)
545 *
546 * This function shall be called only by PF.
547 *
548 * Return: size of the PT allocation(s) needed to support given LMEM size.
549 */
550u64 xe_lmtt_estimate_pt_size(struct xe_lmtt *lmtt, u64 size)
551{
552	unsigned int level = 0;
553	u64 pt_size;
554
555	lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
556	lmtt_assert(lmtt, xe_device_has_lmtt(lmtt_to_xe(lmtt)));
557	lmtt_assert(lmtt, lmtt->ops);
558
559	pt_size = PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
560			     lmtt->ops->lmtt_pte_num(level));
561
562	while (++level < lmtt->ops->lmtt_root_pd_level()) {
563		pt_size *= lmtt->ops->lmtt_pte_index(size, level) + 1;
564		pt_size += PAGE_ALIGN(lmtt->ops->lmtt_pte_size(level) *
565				      lmtt->ops->lmtt_pte_num(level));
566	}
567
568	return pt_size;
569}
570
571#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
572#include "tests/xe_lmtt_test.c"
573#endif