xe_fb_pin.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021 Intel Corporation
  4 */
  5
  6#include <drm/ttm/ttm_bo.h>
  7
  8#include "i915_vma.h"
  9#include "intel_display_core.h"
 10#include "intel_display_types.h"
 11#include "intel_dpt.h"
 12#include "intel_fb.h"
 13#include "intel_fb_pin.h"
 14#include "intel_fbdev.h"
 15#include "xe_bo.h"
 16#include "xe_device.h"
 17#include "xe_ggtt.h"
 18#include "xe_pm.h"
 19#include "xe_vram_types.h"
 20
 21static void
 22write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs,
 23		  u32 width, u32 height, u32 src_stride, u32 dst_stride)
 24{
 25	struct xe_device *xe = xe_bo_device(bo);
 26	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
 27	u32 column, row;
 28	u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
 29
 30	/* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,
 31	 * by writing dpt/ggtt in a different order?
 32	 */
 33
 34	for (column = 0; column < width; column++) {
 35		u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
 36
 37		for (row = 0; row < height; row++) {
 38			u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
 39
 40			iosys_map_wr(map, *dpt_ofs, u64, pte | addr);
 41			*dpt_ofs += 8;
 42			src_idx -= src_stride;
 43		}
 44
 45		/* The DE ignores the PTEs for the padding tiles */
 46		*dpt_ofs += (dst_stride - height) * 8;
 47	}
 48
 49	/* Align to next page */
 50	*dpt_ofs = ALIGN(*dpt_ofs, 4096);
 51}
 52
 53static void
 54write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
 55		   u32 bo_ofs, u32 width, u32 height, u32 src_stride,
 56		   u32 dst_stride)
 57{
 58	struct xe_device *xe = xe_bo_device(bo);
 59	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
 60	u32 column, row;
 61	u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
 62
 63	for (row = 0; row < height; row++) {
 64		u32 src_idx = src_stride * row + bo_ofs;
 65
 66		for (column = 0; column < width; column++) {
 67			u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
 68			iosys_map_wr(map, *dpt_ofs, u64, pte | addr);
 69
 70			*dpt_ofs += 8;
 71			src_idx++;
 72		}
 73
 74		/* The DE ignores the PTEs for the padding tiles */
 75		*dpt_ofs += (dst_stride - width) * 8;
 76	}
 77
 78	/* Align to next page */
 79	*dpt_ofs = ALIGN(*dpt_ofs, 4096);
 80}
 81
 82static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
 83			       const struct i915_gtt_view *view,
 84			       struct i915_vma *vma,
 85			       unsigned int alignment)
 86{
 87	struct xe_device *xe = to_xe_device(fb->base.dev);
 88	struct xe_tile *tile0 = xe_device_get_root_tile(xe);
 89	struct xe_ggtt *ggtt = tile0->mem.ggtt;
 90	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
 91	struct xe_bo *bo = gem_to_xe_bo(obj), *dpt;
 92	u32 dpt_size, size = bo->ttm.base.size;
 93
 94	if (view->type == I915_GTT_VIEW_NORMAL)
 95		dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);
 96	else if (view->type == I915_GTT_VIEW_REMAPPED)
 97		dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,
 98				 XE_PAGE_SIZE);
 99	else
100		/* display uses 4K tiles instead of bytes here, convert to entries.. */
101		dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,
102				 XE_PAGE_SIZE);
103
104	if (IS_DGFX(xe))
105		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
106						   dpt_size, ~0ull,
107						   ttm_bo_type_kernel,
108						   XE_BO_FLAG_VRAM0 |
109						   XE_BO_FLAG_GGTT |
110						   XE_BO_FLAG_PAGETABLE,
111						   alignment, false);
112	else
113		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
114						   dpt_size,  ~0ull,
115						   ttm_bo_type_kernel,
116						   XE_BO_FLAG_STOLEN |
117						   XE_BO_FLAG_GGTT |
118						   XE_BO_FLAG_PAGETABLE,
119						   alignment, false);
120	if (IS_ERR(dpt))
121		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
122						   dpt_size,  ~0ull,
123						   ttm_bo_type_kernel,
124						   XE_BO_FLAG_SYSTEM |
125						   XE_BO_FLAG_GGTT |
126						   XE_BO_FLAG_PAGETABLE,
127						   alignment, false);
128	if (IS_ERR(dpt))
129		return PTR_ERR(dpt);
130
131	if (view->type == I915_GTT_VIEW_NORMAL) {
132		u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
133		u32 x;
134
135		for (x = 0; x < size / XE_PAGE_SIZE; x++) {
136			u64 addr = xe_bo_addr(bo, x * XE_PAGE_SIZE, XE_PAGE_SIZE);
137
138			iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr);
139		}
140	} else if (view->type == I915_GTT_VIEW_REMAPPED) {
141		const struct intel_remapped_info *remap_info = &view->remapped;
142		u32 i, dpt_ofs = 0;
143
144		for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)
145			write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,
146					   remap_info->plane[i].offset,
147					   remap_info->plane[i].width,
148					   remap_info->plane[i].height,
149					   remap_info->plane[i].src_stride,
150					   remap_info->plane[i].dst_stride);
151
152	} else {
153		const struct intel_rotation_info *rot_info = &view->rotated;
154		u32 i, dpt_ofs = 0;
155
156		for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
157			write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs,
158					  rot_info->plane[i].offset,
159					  rot_info->plane[i].width,
160					  rot_info->plane[i].height,
161					  rot_info->plane[i].src_stride,
162					  rot_info->plane[i].dst_stride);
163	}
164
165	vma->dpt = dpt;
166	vma->node = dpt->ggtt_node[tile0->id];
167
168	/* Ensure DPT writes are flushed */
169	xe_device_l2_flush(xe);
170	return 0;
171}
172
173static void
174write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs,
175		   u32 width, u32 height, u32 src_stride, u32 dst_stride)
176{
177	struct xe_device *xe = xe_bo_device(bo);
178	u32 column, row;
179	u64 pte = ggtt->pt_ops->pte_encode_flags(bo, xe->pat.idx[XE_CACHE_NONE]);
180
181	for (column = 0; column < width; column++) {
182		u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
183
184		for (row = 0; row < height; row++) {
185			u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
186
187			ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte | addr);
188			*ggtt_ofs += XE_PAGE_SIZE;
189			src_idx -= src_stride;
190		}
191
192		/* The DE ignores the PTEs for the padding tiles */
193		*ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE;
194	}
195}
196
197static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
198				const struct i915_gtt_view *view,
199				struct i915_vma *vma,
200				unsigned int alignment)
201{
202	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
203	struct xe_bo *bo = gem_to_xe_bo(obj);
204	struct xe_device *xe = to_xe_device(fb->base.dev);
205	struct xe_tile *tile0 = xe_device_get_root_tile(xe);
206	struct xe_ggtt *ggtt = tile0->mem.ggtt;
207	u32 align;
208	int ret;
209
210	/* TODO: Consider sharing framebuffer mapping?
211	 * embed i915_vma inside intel_framebuffer
212	 */
213	xe_pm_runtime_get_noresume(xe);
214	ret = mutex_lock_interruptible(&ggtt->lock);
215	if (ret)
216		goto out;
217
218	align = XE_PAGE_SIZE;
219	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
220		align = max_t(u32, align, SZ_64K);
221
222	if (bo->ggtt_node[tile0->id] && view->type == I915_GTT_VIEW_NORMAL) {
223		vma->node = bo->ggtt_node[tile0->id];
224	} else if (view->type == I915_GTT_VIEW_NORMAL) {
225		vma->node = xe_ggtt_node_init(ggtt);
226		if (IS_ERR(vma->node)) {
227			ret = PTR_ERR(vma->node);
228			goto out_unlock;
229		}
230
231		ret = xe_ggtt_node_insert_locked(vma->node, xe_bo_size(bo), align, 0);
232		if (ret) {
233			xe_ggtt_node_fini(vma->node);
234			goto out_unlock;
235		}
236
237		xe_ggtt_map_bo(ggtt, vma->node, bo, xe->pat.idx[XE_CACHE_NONE]);
238	} else {
239		u32 i, ggtt_ofs;
240		const struct intel_rotation_info *rot_info = &view->rotated;
241
242		/* display seems to use tiles instead of bytes here, so convert it back.. */
243		u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE;
244
245		vma->node = xe_ggtt_node_init(ggtt);
246		if (IS_ERR(vma->node)) {
247			ret = PTR_ERR(vma->node);
248			goto out_unlock;
249		}
250
251		ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
252		if (ret) {
253			xe_ggtt_node_fini(vma->node);
254			goto out_unlock;
255		}
256
257		ggtt_ofs = vma->node->base.start;
258
259		for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
260			write_ggtt_rotated(bo, ggtt, &ggtt_ofs,
261					   rot_info->plane[i].offset,
262					   rot_info->plane[i].width,
263					   rot_info->plane[i].height,
264					   rot_info->plane[i].src_stride,
265					   rot_info->plane[i].dst_stride);
266	}
267
268out_unlock:
269	mutex_unlock(&ggtt->lock);
270out:
271	xe_pm_runtime_put(xe);
272	return ret;
273}
274
275static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
276					const struct i915_gtt_view *view,
277					unsigned int alignment)
278{
279	struct drm_device *dev = fb->base.dev;
280	struct xe_device *xe = to_xe_device(dev);
281	struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
282	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
283	struct xe_bo *bo = gem_to_xe_bo(obj);
284	struct xe_validation_ctx ctx;
285	struct drm_exec exec;
286	int ret = 0;
287
288	if (!vma)
289		return ERR_PTR(-ENODEV);
290
291	refcount_set(&vma->ref, 1);
292	if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
293	    intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
294	    !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {
295		struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram;
296
297		/*
298		 * If we need to able to access the clear-color value stored in
299		 * the buffer, then we require that such buffers are also CPU
300		 * accessible.  This is important on small-bar systems where
301		 * only some subset of VRAM is CPU accessible.
302		 */
303		if (xe_vram_region_io_size(vram) < xe_vram_region_usable_size(vram)) {
304			ret = -EINVAL;
305			goto err;
306		}
307	}
308
309	/*
310	 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the
311	 * assumptions are incorrect for framebuffers
312	 */
313	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true},
314			    ret) {
315		ret = drm_exec_lock_obj(&exec, &bo->ttm.base);
316		drm_exec_retry_on_contention(&exec);
317		if (ret)
318			break;
319
320		if (IS_DGFX(xe))
321			ret = xe_bo_migrate(bo, XE_PL_VRAM0, NULL, &exec);
322		else
323			ret = xe_bo_validate(bo, NULL, true, &exec);
324		drm_exec_retry_on_contention(&exec);
325		xe_validation_retry_on_oom(&ctx, &ret);
326		if (!ret)
327			ttm_bo_pin(&bo->ttm);
328	}
329	if (ret)
330		goto err;
331
332	vma->bo = bo;
333	if (intel_fb_uses_dpt(&fb->base))
334		ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment);
335	else
336		ret = __xe_pin_fb_vma_ggtt(fb, view, vma,  alignment);
337	if (ret)
338		goto err_unpin;
339
340	return vma;
341
342err_unpin:
343	ttm_bo_reserve(&bo->ttm, false, false, NULL);
344	ttm_bo_unpin(&bo->ttm);
345	ttm_bo_unreserve(&bo->ttm);
346err:
347	kfree(vma);
348	return ERR_PTR(ret);
349}
350
351static void __xe_unpin_fb_vma(struct i915_vma *vma)
352{
353	u8 tile_id = xe_device_get_root_tile(xe_bo_device(vma->bo))->id;
354
355	if (!refcount_dec_and_test(&vma->ref))
356		return;
357
358	if (vma->dpt)
359		xe_bo_unpin_map_no_vm(vma->dpt);
360	else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node[tile_id]) ||
361		 vma->bo->ggtt_node[tile_id]->base.start != vma->node->base.start)
362		xe_ggtt_node_remove(vma->node, false);
363
364	ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
365	ttm_bo_unpin(&vma->bo->ttm);
366	ttm_bo_unreserve(&vma->bo->ttm);
367	kfree(vma);
368}
369
370struct i915_vma *
371intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
372		     const struct i915_gtt_view *view,
373		     unsigned int alignment,
374		     unsigned int phys_alignment,
375		     unsigned int vtd_guard,
376		     bool uses_fence,
377		     unsigned long *out_flags)
378{
379	*out_flags = 0;
380
381	return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment);
382}
383
384void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
385{
386	__xe_unpin_fb_vma(vma);
387}
388
389static bool reuse_vma(struct intel_plane_state *new_plane_state,
390		      const struct intel_plane_state *old_plane_state)
391{
392	struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb);
393	struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
394	struct xe_device *xe = to_xe_device(fb->base.dev);
395	struct intel_display *display = xe->display;
396	struct i915_vma *vma;
397
398	if (old_plane_state->hw.fb == new_plane_state->hw.fb &&
399	    !memcmp(&old_plane_state->view.gtt,
400		    &new_plane_state->view.gtt,
401		    sizeof(new_plane_state->view.gtt))) {
402		vma = old_plane_state->ggtt_vma;
403		goto found;
404	}
405
406	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
407		vma = intel_fbdev_vma_pointer(display->fbdev.fbdev);
408		if (vma)
409			goto found;
410	}
411
412	return false;
413
414found:
415	refcount_inc(&vma->ref);
416	new_plane_state->ggtt_vma = vma;
417
418	new_plane_state->surf = i915_ggtt_offset(new_plane_state->ggtt_vma) +
419		plane->surf_offset(new_plane_state);
420
421	return true;
422}
423
424int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
425		       const struct intel_plane_state *old_plane_state)
426{
427	struct drm_framebuffer *fb = new_plane_state->hw.fb;
428	struct drm_gem_object *obj = intel_fb_bo(fb);
429	struct xe_bo *bo = gem_to_xe_bo(obj);
430	struct i915_vma *vma;
431	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
432	struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
433	unsigned int alignment = plane->min_alignment(plane, fb, 0);
434
435	if (reuse_vma(new_plane_state, old_plane_state))
436		return 0;
437
438	/* We reject creating !SCANOUT fb's, so this is weird.. */
439	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));
440
441	vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment);
442
443	if (IS_ERR(vma))
444		return PTR_ERR(vma);
445
446	new_plane_state->ggtt_vma = vma;
447
448	new_plane_state->surf = i915_ggtt_offset(new_plane_state->ggtt_vma) +
449		plane->surf_offset(new_plane_state);
450
451	return 0;
452}
453
454void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
455{
456	__xe_unpin_fb_vma(old_plane_state->ggtt_vma);
457	old_plane_state->ggtt_vma = NULL;
458}
459
460/*
461 * For Xe introduce dummy intel_dpt_create which just return NULL,
462 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0;
463 */
464struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
465{
466	return NULL;
467}
468
469void intel_dpt_destroy(struct i915_address_space *vm)
470{
471	return;
472}
473
474u64 intel_dpt_offset(struct i915_vma *dpt_vma)
475{
476	return 0;
477}
478
479void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map)
480{
481	*map = vma->bo->vmap;
482}