panfrost_gem.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
  3
  4#include <linux/cleanup.h>
  5#include <linux/err.h>
  6#include <linux/slab.h>
  7#include <linux/dma-buf.h>
  8#include <linux/dma-mapping.h>
  9
 10#include <drm/panfrost_drm.h>
 11#include <drm/drm_print.h>
 12#include "panfrost_device.h"
 13#include "panfrost_gem.h"
 14#include "panfrost_mmu.h"
 15
 16#ifdef CONFIG_DEBUG_FS
 17static void panfrost_gem_debugfs_bo_add(struct panfrost_device *pfdev,
 18					struct panfrost_gem_object *bo)
 19{
 20	bo->debugfs.creator.tgid = current->group_leader->pid;
 21	get_task_comm(bo->debugfs.creator.process_name, current->group_leader);
 22
 23	mutex_lock(&pfdev->debugfs.gems_lock);
 24	list_add_tail(&bo->debugfs.node, &pfdev->debugfs.gems_list);
 25	mutex_unlock(&pfdev->debugfs.gems_lock);
 26}
 27
 28static void panfrost_gem_debugfs_bo_rm(struct panfrost_gem_object *bo)
 29{
 30	struct panfrost_device *pfdev = to_panfrost_device(bo->base.base.dev);
 31
 32	if (list_empty(&bo->debugfs.node))
 33		return;
 34
 35	mutex_lock(&pfdev->debugfs.gems_lock);
 36	list_del_init(&bo->debugfs.node);
 37	mutex_unlock(&pfdev->debugfs.gems_lock);
 38}
 39#else
 40static void panfrost_gem_debugfs_bo_add(struct panfrost_device *pfdev,
 41					struct panfrost_gem_object *bo)
 42{}
 43static void panfrost_gem_debugfs_bo_rm(struct panfrost_gem_object *bo) {}
 44#endif
 45
 46/* Called DRM core on the last userspace/kernel unreference of the
 47 * BO.
 48 */
 49static void panfrost_gem_free_object(struct drm_gem_object *obj)
 50{
 51	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 52	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
 53
 54	/*
 55	 * Make sure the BO is no longer inserted in the shrinker list before
 56	 * taking care of the destruction itself. If we don't do that we have a
 57	 * race condition between this function and what's done in
 58	 * panfrost_gem_shrinker_scan().
 59	 */
 60	mutex_lock(&pfdev->shrinker_lock);
 61	list_del_init(&bo->base.madv_list);
 62	mutex_unlock(&pfdev->shrinker_lock);
 63
 64	/*
 65	 * If we still have mappings attached to the BO, there's a problem in
 66	 * our refcounting.
 67	 */
 68	WARN_ON_ONCE(!list_empty(&bo->mappings.list));
 69
 70	kfree_const(bo->label.str);
 71	panfrost_gem_debugfs_bo_rm(bo);
 72	mutex_destroy(&bo->label.lock);
 73
 74	if (bo->sgts) {
 75		int i;
 76		int n_sgt = bo->base.base.size / SZ_2M;
 77
 78		for (i = 0; i < n_sgt; i++) {
 79			if (bo->sgts[i].sgl) {
 80				dma_unmap_sgtable(pfdev->base.dev, &bo->sgts[i],
 81						  DMA_BIDIRECTIONAL, 0);
 82				sg_free_table(&bo->sgts[i]);
 83			}
 84		}
 85		kvfree(bo->sgts);
 86	}
 87
 88	drm_gem_shmem_free(&bo->base);
 89}
 90
 91struct panfrost_gem_mapping *
 92panfrost_gem_mapping_get(struct panfrost_gem_object *bo,
 93			 struct panfrost_file_priv *priv)
 94{
 95	struct panfrost_gem_mapping *iter, *mapping = NULL;
 96
 97	mutex_lock(&bo->mappings.lock);
 98	list_for_each_entry(iter, &bo->mappings.list, node) {
 99		if (iter->mmu == priv->mmu) {
100			kref_get(&iter->refcount);
101			mapping = iter;
102			break;
103		}
104	}
105	mutex_unlock(&bo->mappings.lock);
106
107	return mapping;
108}
109
110static void
111panfrost_gem_teardown_mapping(struct panfrost_gem_mapping *mapping)
112{
113	if (mapping->active)
114		panfrost_mmu_unmap(mapping);
115
116	spin_lock(&mapping->mmu->mm_lock);
117	if (drm_mm_node_allocated(&mapping->mmnode))
118		drm_mm_remove_node(&mapping->mmnode);
119	spin_unlock(&mapping->mmu->mm_lock);
120}
121
122static void panfrost_gem_mapping_release(struct kref *kref)
123{
124	struct panfrost_gem_mapping *mapping;
125
126	mapping = container_of(kref, struct panfrost_gem_mapping, refcount);
127
128	panfrost_gem_teardown_mapping(mapping);
129	drm_gem_object_put(&mapping->obj->base.base);
130	panfrost_mmu_ctx_put(mapping->mmu);
131	kfree(mapping);
132}
133
134void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping)
135{
136	if (!mapping)
137		return;
138
139	kref_put(&mapping->refcount, panfrost_gem_mapping_release);
140}
141
142void panfrost_gem_teardown_mappings_locked(struct panfrost_gem_object *bo)
143{
144	struct panfrost_gem_mapping *mapping;
145
146	list_for_each_entry(mapping, &bo->mappings.list, node)
147		panfrost_gem_teardown_mapping(mapping);
148}
149
150int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_priv)
151{
152	int ret;
153	size_t size = obj->size;
154	u64 align;
155	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
156	unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0;
157	struct panfrost_file_priv *priv = file_priv->driver_priv;
158	struct panfrost_gem_mapping *mapping;
159
160	mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
161	if (!mapping)
162		return -ENOMEM;
163
164	INIT_LIST_HEAD(&mapping->node);
165	kref_init(&mapping->refcount);
166	drm_gem_object_get(obj);
167	mapping->obj = bo;
168
169	/*
170	 * Executable buffers cannot cross a 16MB boundary as the program
171	 * counter is 24-bits. We assume executable buffers will be less than
172	 * 16MB and aligning executable buffers to their size will avoid
173	 * crossing a 16MB boundary.
174	 */
175	if (!bo->noexec)
176		align = size >> PAGE_SHIFT;
177	else
178		align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0;
179
180	mapping->mmu = panfrost_mmu_ctx_get(priv->mmu);
181	spin_lock(&mapping->mmu->mm_lock);
182	ret = drm_mm_insert_node_generic(&mapping->mmu->mm, &mapping->mmnode,
183					 size >> PAGE_SHIFT, align, color, 0);
184	spin_unlock(&mapping->mmu->mm_lock);
185	if (ret)
186		goto err;
187
188	if (!bo->is_heap) {
189		ret = panfrost_mmu_map(mapping);
190		if (ret)
191			goto err;
192	}
193
194	mutex_lock(&bo->mappings.lock);
195	WARN_ON(bo->base.madv != PANFROST_MADV_WILLNEED);
196	list_add_tail(&mapping->node, &bo->mappings.list);
197	mutex_unlock(&bo->mappings.lock);
198
199err:
200	if (ret)
201		panfrost_gem_mapping_put(mapping);
202	return ret;
203}
204
205void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv)
206{
207	struct panfrost_file_priv *priv = file_priv->driver_priv;
208	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
209	struct panfrost_gem_mapping *mapping = NULL, *iter;
210
211	mutex_lock(&bo->mappings.lock);
212	list_for_each_entry(iter, &bo->mappings.list, node) {
213		if (iter->mmu == priv->mmu) {
214			mapping = iter;
215			list_del(&iter->node);
216			break;
217		}
218	}
219	mutex_unlock(&bo->mappings.lock);
220
221	panfrost_gem_mapping_put(mapping);
222}
223
224static int panfrost_gem_pin(struct drm_gem_object *obj)
225{
226	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
227
228	if (bo->is_heap)
229		return -EINVAL;
230
231	return drm_gem_shmem_pin_locked(&bo->base);
232}
233
234static enum drm_gem_object_status panfrost_gem_status(struct drm_gem_object *obj)
235{
236	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
237	enum drm_gem_object_status res = 0;
238
239	if (drm_gem_is_imported(&bo->base.base) || bo->base.pages)
240		res |= DRM_GEM_OBJECT_RESIDENT;
241
242	if (bo->base.madv == PANFROST_MADV_DONTNEED)
243		res |= DRM_GEM_OBJECT_PURGEABLE;
244
245	return res;
246}
247
248static size_t panfrost_gem_rss(struct drm_gem_object *obj)
249{
250	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
251
252	if (bo->is_heap) {
253		return bo->heap_rss_size;
254	} else if (bo->base.pages) {
255		WARN_ON(bo->heap_rss_size);
256		return bo->base.base.size;
257	}
258
259	return 0;
260}
261
262static const struct drm_gem_object_funcs panfrost_gem_funcs = {
263	.free = panfrost_gem_free_object,
264	.open = panfrost_gem_open,
265	.close = panfrost_gem_close,
266	.print_info = drm_gem_shmem_object_print_info,
267	.pin = panfrost_gem_pin,
268	.unpin = drm_gem_shmem_object_unpin,
269	.get_sg_table = drm_gem_shmem_object_get_sg_table,
270	.vmap = drm_gem_shmem_object_vmap,
271	.vunmap = drm_gem_shmem_object_vunmap,
272	.mmap = drm_gem_shmem_object_mmap,
273	.status = panfrost_gem_status,
274	.rss = panfrost_gem_rss,
275	.vm_ops = &drm_gem_shmem_vm_ops,
276};
277
278/**
279 * panfrost_gem_create_object - Implementation of driver->gem_create_object.
280 * @dev: DRM device
281 * @size: Size in bytes of the memory the object will reference
282 *
283 * This lets the GEM helpers allocate object structs for us, and keep
284 * our BO stats correct.
285 */
286struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size)
287{
288	struct panfrost_device *pfdev = to_panfrost_device(dev);
289	struct panfrost_gem_object *obj;
290
291	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
292	if (!obj)
293		return ERR_PTR(-ENOMEM);
294
295	INIT_LIST_HEAD(&obj->mappings.list);
296	mutex_init(&obj->mappings.lock);
297	obj->base.base.funcs = &panfrost_gem_funcs;
298	obj->base.map_wc = !pfdev->coherent;
299	mutex_init(&obj->label.lock);
300
301	panfrost_gem_debugfs_bo_add(pfdev, obj);
302
303	return &obj->base.base;
304}
305
306struct panfrost_gem_object *
307panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
308{
309	struct drm_gem_shmem_object *shmem;
310	struct panfrost_gem_object *bo;
311
312	/* Round up heap allocations to 2MB to keep fault handling simple */
313	if (flags & PANFROST_BO_HEAP)
314		size = roundup(size, SZ_2M);
315
316	shmem = drm_gem_shmem_create(dev, size);
317	if (IS_ERR(shmem))
318		return ERR_CAST(shmem);
319
320	bo = to_panfrost_bo(&shmem->base);
321	bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
322	bo->is_heap = !!(flags & PANFROST_BO_HEAP);
323
324	return bo;
325}
326
327struct drm_gem_object *
328panfrost_gem_prime_import_sg_table(struct drm_device *dev,
329				   struct dma_buf_attachment *attach,
330				   struct sg_table *sgt)
331{
332	struct drm_gem_object *obj;
333	struct panfrost_gem_object *bo;
334
335	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
336	if (IS_ERR(obj))
337		return ERR_CAST(obj);
338
339	bo = to_panfrost_bo(obj);
340	bo->noexec = true;
341
342	/*
343	 * We assign this generic label because this function cannot
344	 * be reached through any of the Panfrost UM driver-specific
345	 * code paths, unless one is given by explicitly calling the
346	 * SET_LABEL_BO ioctl. It is therefore preferable to have a
347	 * blanket BO tag that tells us the object was imported from
348	 * another driver than nothing at all.
349	 */
350	panfrost_gem_internal_set_label(obj, "GEM PRIME buffer");
351
352	return obj;
353}
354
355void
356panfrost_gem_set_label(struct drm_gem_object *obj, const char *label)
357{
358	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
359	const char *old_label;
360
361	scoped_guard(mutex, &bo->label.lock) {
362		old_label = bo->label.str;
363		bo->label.str = label;
364	}
365
366	kfree_const(old_label);
367}
368
369void
370panfrost_gem_internal_set_label(struct drm_gem_object *obj, const char *label)
371{
372	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
373	const char *str;
374
375	/* We should never attempt labelling a UM-exposed GEM object */
376	if (drm_WARN_ON(bo->base.base.dev, bo->base.base.handle_count > 0))
377		return;
378
379	if (!label)
380		return;
381
382	str = kstrdup_const(label, GFP_KERNEL);
383	if (!str) {
384		/* Failing to allocate memory for a label isn't a fatal condition */
385		drm_warn(bo->base.base.dev, "Not enough memory to allocate BO label");
386		return;
387	}
388
389	panfrost_gem_set_label(obj, str);
390}
391
392#ifdef CONFIG_DEBUG_FS
393struct gem_size_totals {
394	size_t size;
395	size_t resident;
396	size_t reclaimable;
397};
398
399struct flag_def {
400	u32 flag;
401	const char *name;
402};
403
404static void panfrost_gem_debugfs_print_flag_names(struct seq_file *m)
405{
406	int len;
407	int i;
408
409	static const struct flag_def gem_state_flags_names[] = {
410		{PANFROST_DEBUGFS_GEM_STATE_FLAG_IMPORTED, "imported"},
411		{PANFROST_DEBUGFS_GEM_STATE_FLAG_EXPORTED, "exported"},
412		{PANFROST_DEBUGFS_GEM_STATE_FLAG_PURGED, "purged"},
413		{PANFROST_DEBUGFS_GEM_STATE_FLAG_PURGEABLE, "purgeable"},
414	};
415
416	seq_puts(m, "GEM state flags: ");
417	for (i = 0, len = ARRAY_SIZE(gem_state_flags_names); i < len; i++) {
418		seq_printf(m, "%s (0x%x)%s", gem_state_flags_names[i].name,
419			   gem_state_flags_names[i].flag, (i < len - 1) ? ", " : "\n\n");
420	}
421}
422
423static void panfrost_gem_debugfs_bo_print(struct panfrost_gem_object *bo,
424					  struct seq_file *m,
425					  struct gem_size_totals *totals)
426{
427	unsigned int refcount = kref_read(&bo->base.base.refcount);
428	char creator_info[32] = {};
429	size_t resident_size;
430	u32 gem_state_flags = 0;
431
432	/* Skip BOs being destroyed. */
433	if (!refcount)
434		return;
435
436	resident_size = panfrost_gem_rss(&bo->base.base);
437
438	snprintf(creator_info, sizeof(creator_info),
439		 "%s/%d", bo->debugfs.creator.process_name, bo->debugfs.creator.tgid);
440	seq_printf(m, "%-32s%-16d%-16d%-16zd%-16zd0x%-16lx",
441		   creator_info,
442		   bo->base.base.name,
443		   refcount,
444		   bo->base.base.size,
445		   resident_size,
446		   drm_vma_node_start(&bo->base.base.vma_node));
447
448	if (bo->base.base.import_attach)
449		gem_state_flags |= PANFROST_DEBUGFS_GEM_STATE_FLAG_IMPORTED;
450	if (bo->base.base.dma_buf)
451		gem_state_flags |= PANFROST_DEBUGFS_GEM_STATE_FLAG_EXPORTED;
452
453	if (bo->base.madv < 0)
454		gem_state_flags |= PANFROST_DEBUGFS_GEM_STATE_FLAG_PURGED;
455	else if (bo->base.madv > 0)
456		gem_state_flags |= PANFROST_DEBUGFS_GEM_STATE_FLAG_PURGEABLE;
457
458	seq_printf(m, "0x%-10x", gem_state_flags);
459
460	scoped_guard(mutex, &bo->label.lock) {
461		seq_printf(m, "%s\n", bo->label.str ? : "");
462	}
463
464	totals->size += bo->base.base.size;
465	totals->resident += resident_size;
466	if (bo->base.madv > 0)
467		totals->reclaimable += resident_size;
468}
469
470void panfrost_gem_debugfs_print_bos(struct panfrost_device *pfdev,
471				    struct seq_file *m)
472{
473	struct gem_size_totals totals = {0};
474	struct panfrost_gem_object *bo;
475
476	panfrost_gem_debugfs_print_flag_names(m);
477
478	seq_puts(m, "created-by                      global-name     refcount        size            resident-size   file-offset       state       label\n");
479	seq_puts(m, "-----------------------------------------------------------------------------------------------------------------------------------\n");
480
481	scoped_guard(mutex, &pfdev->debugfs.gems_lock) {
482		list_for_each_entry(bo, &pfdev->debugfs.gems_list, debugfs.node) {
483			panfrost_gem_debugfs_bo_print(bo, m, &totals);
484		}
485	}
486
487	seq_puts(m, "===================================================================================================================================\n");
488	seq_printf(m, "Total size: %zd, Total resident: %zd, Total reclaimable: %zd\n",
489		   totals.size, totals.resident, totals.reclaimable);
490}
491#endif