intel_dpt.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021 Intel Corporation
  4 */
  5
  6#include <drm/drm_print.h>
  7
  8#include "gem/i915_gem_domain.h"
  9#include "gem/i915_gem_internal.h"
 10#include "gem/i915_gem_lmem.h"
 11#include "gt/gen8_ppgtt.h"
 12
 13#include "i915_drv.h"
 14#include "intel_display_core.h"
 15#include "intel_display_rpm.h"
 16#include "intel_display_types.h"
 17#include "intel_dpt.h"
 18#include "intel_fb.h"
 19
 20struct i915_dpt {
 21	struct i915_address_space vm;
 22
 23	struct drm_i915_gem_object *obj;
 24	struct i915_vma *vma;
 25	void __iomem *iomem;
 26};
 27
 28#define i915_is_dpt(vm) ((vm)->is_dpt)
 29
 30static inline struct i915_dpt *
 31i915_vm_to_dpt(struct i915_address_space *vm)
 32{
 33	BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
 34	drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm));
 35	return container_of(vm, struct i915_dpt, vm);
 36}
 37
 38static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 39{
 40	writeq(pte, addr);
 41}
 42
 43static void dpt_insert_page(struct i915_address_space *vm,
 44			    dma_addr_t addr,
 45			    u64 offset,
 46			    unsigned int pat_index,
 47			    u32 flags)
 48{
 49	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
 50	gen8_pte_t __iomem *base = dpt->iomem;
 51
 52	gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
 53		     vm->pte_encode(addr, pat_index, flags));
 54}
 55
 56static void dpt_insert_entries(struct i915_address_space *vm,
 57			       struct i915_vma_resource *vma_res,
 58			       unsigned int pat_index,
 59			       u32 flags)
 60{
 61	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
 62	gen8_pte_t __iomem *base = dpt->iomem;
 63	const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
 64	struct sgt_iter sgt_iter;
 65	dma_addr_t addr;
 66	int i;
 67
 68	/*
 69	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
 70	 * not to allow the user to override access to a read only page.
 71	 */
 72
 73	i = vma_res->start / I915_GTT_PAGE_SIZE;
 74	for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)
 75		gen8_set_pte(&base[i++], pte_encode | addr);
 76}
 77
 78static void dpt_clear_range(struct i915_address_space *vm,
 79			    u64 start, u64 length)
 80{
 81}
 82
 83static void dpt_bind_vma(struct i915_address_space *vm,
 84			 struct i915_vm_pt_stash *stash,
 85			 struct i915_vma_resource *vma_res,
 86			 unsigned int pat_index,
 87			 u32 flags)
 88{
 89	u32 pte_flags;
 90
 91	if (vma_res->bound_flags)
 92		return;
 93
 94	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 95	pte_flags = 0;
 96	if (vm->has_read_only && vma_res->bi.readonly)
 97		pte_flags |= PTE_READ_ONLY;
 98	if (vma_res->bi.lmem)
 99		pte_flags |= PTE_LM;
100
101	vm->insert_entries(vm, vma_res, pat_index, pte_flags);
102
103	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
104
105	/*
106	 * Without aliasing PPGTT there's no difference between
107	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
108	 * upgrade to both bound if we bind either to avoid double-binding.
109	 */
110	vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
111}
112
113static void dpt_unbind_vma(struct i915_address_space *vm,
114			   struct i915_vma_resource *vma_res)
115{
116	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
117}
118
119static void dpt_cleanup(struct i915_address_space *vm)
120{
121	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
122
123	i915_gem_object_put(dpt->obj);
124}
125
126struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm,
127				       unsigned int alignment)
128{
129	struct drm_i915_private *i915 = vm->i915;
130	struct intel_display *display = i915->display;
131	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
132	struct ref_tracker *wakeref;
133	struct i915_vma *vma;
134	void __iomem *iomem;
135	struct i915_gem_ww_ctx ww;
136	u64 pin_flags = 0;
137	int err;
138
139	if (i915_gem_object_is_stolen(dpt->obj))
140		pin_flags |= PIN_MAPPABLE;
141
142	wakeref = intel_display_rpm_get(display);
143	atomic_inc(&display->restore.pending_fb_pin);
144
145	for_i915_gem_ww(&ww, err, true) {
146		err = i915_gem_object_lock(dpt->obj, &ww);
147		if (err)
148			continue;
149
150		vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0,
151						  alignment, pin_flags);
152		if (IS_ERR(vma)) {
153			err = PTR_ERR(vma);
154			continue;
155		}
156
157		iomem = i915_vma_pin_iomap(vma);
158		i915_vma_unpin(vma);
159
160		if (IS_ERR(iomem)) {
161			err = PTR_ERR(iomem);
162			continue;
163		}
164
165		dpt->vma = vma;
166		dpt->iomem = iomem;
167
168		i915_vma_get(vma);
169	}
170
171	dpt->obj->mm.dirty = true;
172
173	atomic_dec(&display->restore.pending_fb_pin);
174	intel_display_rpm_put(display, wakeref);
175
176	return err ? ERR_PTR(err) : vma;
177}
178
179void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm)
180{
181	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
182
183	i915_vma_unpin_iomap(dpt->vma);
184	i915_vma_put(dpt->vma);
185}
186
187/**
188 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
189 * @display: display device instance
190 *
191 * Restore the memory mapping during system resume for all framebuffers which
192 * are mapped to HW via a GGTT->DPT page table. The content of these page
193 * tables are not stored in the hibernation image during S4 and S3RST->S4
194 * transitions, so here we reprogram the PTE entries in those tables.
195 *
196 * This function must be called after the mappings in GGTT have been restored calling
197 * i915_ggtt_resume().
198 */
199void intel_dpt_resume(struct intel_display *display)
200{
201	struct drm_framebuffer *drm_fb;
202
203	if (!HAS_DISPLAY(display))
204		return;
205
206	mutex_lock(&display->drm->mode_config.fb_lock);
207	drm_for_each_fb(drm_fb, display->drm) {
208		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
209
210		if (fb->dpt_vm)
211			i915_ggtt_resume_vm(fb->dpt_vm, true);
212	}
213	mutex_unlock(&display->drm->mode_config.fb_lock);
214}
215
216/**
217 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
218 * @display: display device instance
219 *
220 * Suspend the memory mapping during system suspend for all framebuffers which
221 * are mapped to HW via a GGTT->DPT page table.
222 *
223 * This function must be called before the mappings in GGTT are suspended calling
224 * i915_ggtt_suspend().
225 */
226void intel_dpt_suspend(struct intel_display *display)
227{
228	struct drm_framebuffer *drm_fb;
229
230	if (!HAS_DISPLAY(display))
231		return;
232
233	mutex_lock(&display->drm->mode_config.fb_lock);
234
235	drm_for_each_fb(drm_fb, display->drm) {
236		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
237
238		if (fb->dpt_vm)
239			i915_ggtt_suspend_vm(fb->dpt_vm, true);
240	}
241
242	mutex_unlock(&display->drm->mode_config.fb_lock);
243}
244
245struct i915_address_space *
246intel_dpt_create(struct intel_framebuffer *fb)
247{
248	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
249	struct drm_i915_private *i915 = to_i915(obj->dev);
250	struct drm_i915_gem_object *dpt_obj;
251	struct i915_address_space *vm;
252	struct i915_dpt *dpt;
253	size_t size;
254	int ret;
255
256	if (intel_fb_needs_pot_stride_remap(fb))
257		size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
258	else
259		size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
260
261	size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
262
263	dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);
264	if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))
265		dpt_obj = i915_gem_object_create_stolen(i915, size);
266	if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {
267		drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");
268		dpt_obj = i915_gem_object_create_shmem(i915, size);
269	}
270	if (IS_ERR(dpt_obj))
271		return ERR_CAST(dpt_obj);
272
273	ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);
274	if (!ret) {
275		ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
276		i915_gem_object_unlock(dpt_obj);
277	}
278	if (ret) {
279		i915_gem_object_put(dpt_obj);
280		return ERR_PTR(ret);
281	}
282
283	dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
284	if (!dpt) {
285		i915_gem_object_put(dpt_obj);
286		return ERR_PTR(-ENOMEM);
287	}
288
289	vm = &dpt->vm;
290
291	vm->gt = to_gt(i915);
292	vm->i915 = i915;
293	vm->dma = i915->drm.dev;
294	vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
295	vm->is_dpt = true;
296
297	i915_address_space_init(vm, VM_CLASS_DPT);
298
299	vm->insert_page = dpt_insert_page;
300	vm->clear_range = dpt_clear_range;
301	vm->insert_entries = dpt_insert_entries;
302	vm->cleanup = dpt_cleanup;
303
304	vm->vma_ops.bind_vma    = dpt_bind_vma;
305	vm->vma_ops.unbind_vma  = dpt_unbind_vma;
306
307	vm->pte_encode = vm->gt->ggtt->vm.pte_encode;
308
309	dpt->obj = dpt_obj;
310	dpt->obj->is_dpt = true;
311
312	return &dpt->vm;
313}
314
315void intel_dpt_destroy(struct i915_address_space *vm)
316{
317	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
318
319	dpt->obj->is_dpt = false;
320	i915_vm_put(&dpt->vm);
321}
322
323u64 intel_dpt_offset(struct i915_vma *dpt_vma)
324{
325	return i915_vma_offset(dpt_vma);
326}