dmabuf.c

  1/*
  2 * Copyright 2017 Intel Corporation. All rights reserved.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 21 * DEALINGS IN THE SOFTWARE.
 22 *
 23 * Authors:
 24 *    Zhiyuan Lv <zhiyuan.lv@intel.com>
 25 *
 26 * Contributors:
 27 *    Xiaoguang Chen
 28 *    Tina Zhang <tina.zhang@intel.com>
 29 */
 30
 31#include <linux/dma-buf.h>
 32#include <linux/mdev.h>
 33
 34#include <drm/drm_fourcc.h>
 35#include <drm/drm_plane.h>
 36#include <drm/drm_print.h>
 37
 38#include "gem/i915_gem_dmabuf.h"
 39
 40#include "i915_drv.h"
 41#include "gvt.h"
 42
 43#include "display/skl_universal_plane_regs.h"
 44
 45#define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12))
 46
 47static int vgpu_gem_get_pages(struct drm_i915_gem_object *obj)
 48{
 49	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 50	struct intel_vgpu *vgpu;
 51	struct sg_table *st;
 52	struct scatterlist *sg;
 53	int i, j, ret;
 54	gen8_pte_t __iomem *gtt_entries;
 55	struct intel_vgpu_fb_info *fb_info;
 56	unsigned int page_num; /* limited by sg_alloc_table */
 57
 58	if (overflows_type(obj->base.size >> PAGE_SHIFT, page_num))
 59		return -E2BIG;
 60
 61	page_num = obj->base.size >> PAGE_SHIFT;
 62	fb_info = (struct intel_vgpu_fb_info *)obj->gvt_info;
 63	if (drm_WARN_ON(&dev_priv->drm, !fb_info))
 64		return -ENODEV;
 65
 66	vgpu = fb_info->obj->vgpu;
 67	if (drm_WARN_ON(&dev_priv->drm, !vgpu))
 68		return -ENODEV;
 69
 70	st = kmalloc(sizeof(*st), GFP_KERNEL);
 71	if (unlikely(!st))
 72		return -ENOMEM;
 73
 74	ret = sg_alloc_table(st, page_num, GFP_KERNEL);
 75	if (ret) {
 76		kfree(st);
 77		return ret;
 78	}
 79	gtt_entries = (gen8_pte_t __iomem *)to_gt(dev_priv)->ggtt->gsm +
 80		(fb_info->start >> PAGE_SHIFT);
 81	for_each_sg(st->sgl, sg, page_num, i) {
 82		dma_addr_t dma_addr =
 83			GEN8_DECODE_PTE(readq(&gtt_entries[i]));
 84		if (intel_gvt_dma_pin_guest_page(vgpu, dma_addr)) {
 85			ret = -EINVAL;
 86			goto out;
 87		}
 88
 89		sg->offset = 0;
 90		sg->length = PAGE_SIZE;
 91		sg_dma_len(sg) = PAGE_SIZE;
 92		sg_dma_address(sg) = dma_addr;
 93	}
 94
 95	__i915_gem_object_set_pages(obj, st);
 96out:
 97	if (ret) {
 98		dma_addr_t dma_addr;
 99
100		for_each_sg(st->sgl, sg, i, j) {
101			dma_addr = sg_dma_address(sg);
102			if (dma_addr)
103				intel_gvt_dma_unmap_guest_page(vgpu, dma_addr);
104		}
105		sg_free_table(st);
106		kfree(st);
107	}
108
109	return ret;
110
111}
112
113static void vgpu_gem_put_pages(struct drm_i915_gem_object *obj,
114		struct sg_table *pages)
115{
116	struct scatterlist *sg;
117
118	if (obj->base.dma_buf) {
119		struct intel_vgpu_fb_info *fb_info = obj->gvt_info;
120		struct intel_vgpu_dmabuf_obj *obj = fb_info->obj;
121		struct intel_vgpu *vgpu = obj->vgpu;
122		int i;
123
124		for_each_sg(pages->sgl, sg, fb_info->size, i)
125			intel_gvt_dma_unmap_guest_page(vgpu,
126					       sg_dma_address(sg));
127	}
128
129	sg_free_table(pages);
130	kfree(pages);
131}
132
133static void dmabuf_gem_object_free(struct kref *kref)
134{
135	struct intel_vgpu_dmabuf_obj *obj =
136		container_of(kref, struct intel_vgpu_dmabuf_obj, kref);
137	struct intel_vgpu *vgpu = obj->vgpu;
138	struct list_head *pos;
139	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
140
141	if (vgpu && test_bit(INTEL_VGPU_STATUS_ACTIVE, vgpu->status) &&
142	    !list_empty(&vgpu->dmabuf_obj_list_head)) {
143		list_for_each(pos, &vgpu->dmabuf_obj_list_head) {
144			dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list);
145			if (dmabuf_obj == obj) {
146				list_del(pos);
147				idr_remove(&vgpu->object_idr,
148					   dmabuf_obj->dmabuf_id);
149				kfree(dmabuf_obj->info);
150				kfree(dmabuf_obj);
151				break;
152			}
153		}
154	} else {
155		/* Free the orphan dmabuf_objs here */
156		kfree(obj->info);
157		kfree(obj);
158	}
159}
160
161
162static inline void dmabuf_obj_get(struct intel_vgpu_dmabuf_obj *obj)
163{
164	kref_get(&obj->kref);
165}
166
167static inline void dmabuf_obj_put(struct intel_vgpu_dmabuf_obj *obj)
168{
169	kref_put(&obj->kref, dmabuf_gem_object_free);
170}
171
172static void vgpu_gem_release(struct drm_i915_gem_object *gem_obj)
173{
174
175	struct intel_vgpu_fb_info *fb_info = gem_obj->gvt_info;
176	struct intel_vgpu_dmabuf_obj *obj = fb_info->obj;
177	struct intel_vgpu *vgpu = obj->vgpu;
178
179	if (vgpu) {
180		mutex_lock(&vgpu->dmabuf_lock);
181		gem_obj->base.dma_buf = NULL;
182		dmabuf_obj_put(obj);
183		mutex_unlock(&vgpu->dmabuf_lock);
184	} else {
185		/* vgpu is NULL, as it has been removed already */
186		gem_obj->base.dma_buf = NULL;
187		dmabuf_obj_put(obj);
188	}
189}
190
191static const struct drm_i915_gem_object_ops intel_vgpu_gem_ops = {
192	.name = "i915_gem_object_vgpu",
193	.flags = I915_GEM_OBJECT_IS_PROXY,
194	.get_pages = vgpu_gem_get_pages,
195	.put_pages = vgpu_gem_put_pages,
196	.release = vgpu_gem_release,
197};
198
199static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev,
200		struct intel_vgpu_fb_info *info)
201{
202	static struct lock_class_key lock_class;
203	struct drm_i915_private *dev_priv = to_i915(dev);
204	struct drm_i915_gem_object *obj;
205
206	obj = i915_gem_object_alloc();
207	if (obj == NULL)
208		return NULL;
209
210	drm_gem_private_object_init(dev, &obj->base,
211		roundup(info->size, PAGE_SIZE));
212	i915_gem_object_init(obj, &intel_vgpu_gem_ops, &lock_class, 0);
213	i915_gem_object_set_readonly(obj);
214
215	obj->read_domains = I915_GEM_DOMAIN_GTT;
216	obj->write_domain = 0;
217	if (GRAPHICS_VER(dev_priv) >= 9) {
218		unsigned int tiling_mode = 0;
219		unsigned int stride = 0;
220
221		switch (info->drm_format_mod) {
222		case DRM_FORMAT_MOD_LINEAR:
223			tiling_mode = I915_TILING_NONE;
224			break;
225		case I915_FORMAT_MOD_X_TILED:
226			tiling_mode = I915_TILING_X;
227			stride = info->stride;
228			break;
229		case I915_FORMAT_MOD_Y_TILED:
230		case I915_FORMAT_MOD_Yf_TILED:
231			tiling_mode = I915_TILING_Y;
232			stride = info->stride;
233			break;
234		default:
235			gvt_dbg_core("invalid drm_format_mod %llx for tiling\n",
236				     info->drm_format_mod);
237		}
238		obj->tiling_and_stride = tiling_mode | stride;
239	} else {
240		obj->tiling_and_stride = info->drm_format_mod ?
241					I915_TILING_X : 0;
242	}
243
244	return obj;
245}
246
247static bool validate_hotspot(struct intel_vgpu_cursor_plane_format *c)
248{
249	if (c && c->x_hot <= c->width && c->y_hot <= c->height)
250		return true;
251	else
252		return false;
253}
254
255static int vgpu_get_plane_info(struct drm_device *dev,
256		struct intel_vgpu *vgpu,
257		struct intel_vgpu_fb_info *info,
258		int plane_id)
259{
260	struct intel_vgpu_primary_plane_format p;
261	struct intel_vgpu_cursor_plane_format c;
262	int ret, tile_height = 1;
263
264	memset(info, 0, sizeof(*info));
265
266	if (plane_id == DRM_PLANE_TYPE_PRIMARY) {
267		ret = intel_vgpu_decode_primary_plane(vgpu, &p);
268		if (ret)
269			return ret;
270		info->start = p.base;
271		info->start_gpa = p.base_gpa;
272		info->width = p.width;
273		info->height = p.height;
274		info->stride = p.stride;
275		info->drm_format = p.drm_format;
276
277		switch (p.tiled) {
278		case PLANE_CTL_TILED_LINEAR:
279			info->drm_format_mod = DRM_FORMAT_MOD_LINEAR;
280			break;
281		case PLANE_CTL_TILED_X:
282			info->drm_format_mod = I915_FORMAT_MOD_X_TILED;
283			tile_height = 8;
284			break;
285		case PLANE_CTL_TILED_Y:
286			info->drm_format_mod = I915_FORMAT_MOD_Y_TILED;
287			tile_height = 32;
288			break;
289		case PLANE_CTL_TILED_YF:
290			info->drm_format_mod = I915_FORMAT_MOD_Yf_TILED;
291			tile_height = 32;
292			break;
293		default:
294			gvt_vgpu_err("invalid tiling mode: %x\n", p.tiled);
295		}
296	} else if (plane_id == DRM_PLANE_TYPE_CURSOR) {
297		ret = intel_vgpu_decode_cursor_plane(vgpu, &c);
298		if (ret)
299			return ret;
300		info->start = c.base;
301		info->start_gpa = c.base_gpa;
302		info->width = c.width;
303		info->height = c.height;
304		info->stride = c.width * (c.bpp / 8);
305		info->drm_format = c.drm_format;
306		info->drm_format_mod = 0;
307		info->x_pos = c.x_pos;
308		info->y_pos = c.y_pos;
309
310		if (validate_hotspot(&c)) {
311			info->x_hot = c.x_hot;
312			info->y_hot = c.y_hot;
313		} else {
314			info->x_hot = UINT_MAX;
315			info->y_hot = UINT_MAX;
316		}
317	} else {
318		gvt_vgpu_err("invalid plane id:%d\n", plane_id);
319		return -EINVAL;
320	}
321
322	info->size = info->stride * roundup(info->height, tile_height);
323	if (info->size == 0) {
324		gvt_vgpu_err("fb size is zero\n");
325		return -EINVAL;
326	}
327
328	if (info->start & (PAGE_SIZE - 1)) {
329		gvt_vgpu_err("Not aligned fb address:0x%llx\n", info->start);
330		return -EFAULT;
331	}
332
333	if (!intel_gvt_ggtt_validate_range(vgpu, info->start, info->size)) {
334		gvt_vgpu_err("invalid gma addr\n");
335		return -EFAULT;
336	}
337
338	return 0;
339}
340
341static struct intel_vgpu_dmabuf_obj *
342pick_dmabuf_by_info(struct intel_vgpu *vgpu,
343		    struct intel_vgpu_fb_info *latest_info)
344{
345	struct list_head *pos;
346	struct intel_vgpu_fb_info *fb_info;
347	struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL;
348	struct intel_vgpu_dmabuf_obj *ret = NULL;
349
350	list_for_each(pos, &vgpu->dmabuf_obj_list_head) {
351		dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list);
352		if (!dmabuf_obj->info)
353			continue;
354
355		fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info;
356		if ((fb_info->start == latest_info->start) &&
357		    (fb_info->start_gpa == latest_info->start_gpa) &&
358		    (fb_info->size == latest_info->size) &&
359		    (fb_info->drm_format_mod == latest_info->drm_format_mod) &&
360		    (fb_info->drm_format == latest_info->drm_format) &&
361		    (fb_info->width == latest_info->width) &&
362		    (fb_info->height == latest_info->height)) {
363			ret = dmabuf_obj;
364			break;
365		}
366	}
367
368	return ret;
369}
370
371static struct intel_vgpu_dmabuf_obj *
372pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id)
373{
374	struct list_head *pos;
375	struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL;
376	struct intel_vgpu_dmabuf_obj *ret = NULL;
377
378	list_for_each(pos, &vgpu->dmabuf_obj_list_head) {
379		dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list);
380		if (dmabuf_obj->dmabuf_id == id) {
381			ret = dmabuf_obj;
382			break;
383		}
384	}
385
386	return ret;
387}
388
389static void update_fb_info(struct vfio_device_gfx_plane_info *gvt_dmabuf,
390		      struct intel_vgpu_fb_info *fb_info)
391{
392	gvt_dmabuf->drm_format = fb_info->drm_format;
393	gvt_dmabuf->drm_format_mod = fb_info->drm_format_mod;
394	gvt_dmabuf->width = fb_info->width;
395	gvt_dmabuf->height = fb_info->height;
396	gvt_dmabuf->stride = fb_info->stride;
397	gvt_dmabuf->size = fb_info->size;
398	gvt_dmabuf->x_pos = fb_info->x_pos;
399	gvt_dmabuf->y_pos = fb_info->y_pos;
400	gvt_dmabuf->x_hot = fb_info->x_hot;
401	gvt_dmabuf->y_hot = fb_info->y_hot;
402}
403
404int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args)
405{
406	struct drm_device *dev = &vgpu->gvt->gt->i915->drm;
407	struct vfio_device_gfx_plane_info *gfx_plane_info = args;
408	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
409	struct intel_vgpu_fb_info fb_info;
410	int ret = 0;
411
412	if (gfx_plane_info->flags == (VFIO_GFX_PLANE_TYPE_DMABUF |
413				       VFIO_GFX_PLANE_TYPE_PROBE))
414		return ret;
415	else if ((gfx_plane_info->flags & ~VFIO_GFX_PLANE_TYPE_DMABUF) ||
416			(!gfx_plane_info->flags))
417		return -EINVAL;
418
419	ret = vgpu_get_plane_info(dev, vgpu, &fb_info,
420					gfx_plane_info->drm_plane_type);
421	if (ret != 0)
422		goto out;
423
424	mutex_lock(&vgpu->dmabuf_lock);
425	/* If exists, pick up the exposed dmabuf_obj */
426	dmabuf_obj = pick_dmabuf_by_info(vgpu, &fb_info);
427	if (dmabuf_obj) {
428		update_fb_info(gfx_plane_info, &fb_info);
429		gfx_plane_info->dmabuf_id = dmabuf_obj->dmabuf_id;
430
431		/* This buffer may be released between query_plane ioctl and
432		 * get_dmabuf ioctl. Add the refcount to make sure it won't
433		 * be released between the two ioctls.
434		 */
435		if (!dmabuf_obj->initref) {
436			dmabuf_obj->initref = true;
437			dmabuf_obj_get(dmabuf_obj);
438		}
439		ret = 0;
440		gvt_dbg_dpy("vgpu%d: reuse dmabuf_obj ref %d, id %d\n",
441			    vgpu->id, kref_read(&dmabuf_obj->kref),
442			    gfx_plane_info->dmabuf_id);
443		mutex_unlock(&vgpu->dmabuf_lock);
444		goto out;
445	}
446
447	mutex_unlock(&vgpu->dmabuf_lock);
448
449	/* Need to allocate a new one*/
450	dmabuf_obj = kmalloc(sizeof(struct intel_vgpu_dmabuf_obj), GFP_KERNEL);
451	if (unlikely(!dmabuf_obj)) {
452		gvt_vgpu_err("alloc dmabuf_obj failed\n");
453		ret = -ENOMEM;
454		goto out;
455	}
456
457	dmabuf_obj->info = kmalloc(sizeof(struct intel_vgpu_fb_info),
458				   GFP_KERNEL);
459	if (unlikely(!dmabuf_obj->info)) {
460		gvt_vgpu_err("allocate intel vgpu fb info failed\n");
461		ret = -ENOMEM;
462		goto out_free_dmabuf;
463	}
464	memcpy(dmabuf_obj->info, &fb_info, sizeof(struct intel_vgpu_fb_info));
465
466	((struct intel_vgpu_fb_info *)dmabuf_obj->info)->obj = dmabuf_obj;
467
468	dmabuf_obj->vgpu = vgpu;
469
470	ret = idr_alloc(&vgpu->object_idr, dmabuf_obj, 1, 0, GFP_NOWAIT);
471	if (ret < 0)
472		goto out_free_info;
473	gfx_plane_info->dmabuf_id = ret;
474	dmabuf_obj->dmabuf_id = ret;
475
476	dmabuf_obj->initref = true;
477
478	kref_init(&dmabuf_obj->kref);
479
480	update_fb_info(gfx_plane_info, &fb_info);
481
482	INIT_LIST_HEAD(&dmabuf_obj->list);
483	mutex_lock(&vgpu->dmabuf_lock);
484	list_add_tail(&dmabuf_obj->list, &vgpu->dmabuf_obj_list_head);
485	mutex_unlock(&vgpu->dmabuf_lock);
486
487	gvt_dbg_dpy("vgpu%d: %s new dmabuf_obj ref %d, id %d\n", vgpu->id,
488		    __func__, kref_read(&dmabuf_obj->kref), ret);
489
490	return 0;
491
492out_free_info:
493	kfree(dmabuf_obj->info);
494out_free_dmabuf:
495	kfree(dmabuf_obj);
496out:
497	/* ENODEV means plane isn't ready, which might be a normal case. */
498	return (ret == -ENODEV) ? 0 : ret;
499}
500
501/* To associate an exposed dmabuf with the dmabuf_obj */
502int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id)
503{
504	struct drm_device *dev = &vgpu->gvt->gt->i915->drm;
505	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
506	struct drm_i915_gem_object *obj;
507	struct dma_buf *dmabuf;
508	int dmabuf_fd;
509	int ret = 0;
510
511	mutex_lock(&vgpu->dmabuf_lock);
512
513	dmabuf_obj = pick_dmabuf_by_num(vgpu, dmabuf_id);
514	if (dmabuf_obj == NULL) {
515		gvt_vgpu_err("invalid dmabuf id:%d\n", dmabuf_id);
516		ret = -EINVAL;
517		goto out;
518	}
519
520	obj = vgpu_create_gem(dev, dmabuf_obj->info);
521	if (obj == NULL) {
522		gvt_vgpu_err("create gvt gem obj failed\n");
523		ret = -ENOMEM;
524		goto out;
525	}
526
527	obj->gvt_info = dmabuf_obj->info;
528
529	dmabuf = i915_gem_prime_export(&obj->base, DRM_CLOEXEC | DRM_RDWR);
530	if (IS_ERR(dmabuf)) {
531		gvt_vgpu_err("export dma-buf failed\n");
532		ret = PTR_ERR(dmabuf);
533		goto out_free_gem;
534	}
535
536	ret = dma_buf_fd(dmabuf, DRM_CLOEXEC | DRM_RDWR);
537	if (ret < 0) {
538		gvt_vgpu_err("create dma-buf fd failed ret:%d\n", ret);
539		goto out_free_dmabuf;
540	}
541	dmabuf_fd = ret;
542
543	dmabuf_obj_get(dmabuf_obj);
544
545	if (dmabuf_obj->initref) {
546		dmabuf_obj->initref = false;
547		dmabuf_obj_put(dmabuf_obj);
548	}
549
550	mutex_unlock(&vgpu->dmabuf_lock);
551
552	gvt_dbg_dpy("vgpu%d: dmabuf:%d, dmabuf ref %d, fd:%d\n"
553		    "        file count: %ld, GEM ref: %d\n",
554		    vgpu->id, dmabuf_obj->dmabuf_id,
555		    kref_read(&dmabuf_obj->kref),
556		    dmabuf_fd,
557		    file_count(dmabuf->file),
558		    kref_read(&obj->base.refcount));
559
560	i915_gem_object_put(obj);
561
562	return dmabuf_fd;
563
564out_free_dmabuf:
565	dma_buf_put(dmabuf);
566out_free_gem:
567	i915_gem_object_put(obj);
568out:
569	mutex_unlock(&vgpu->dmabuf_lock);
570	return ret;
571}
572
573void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu)
574{
575	struct list_head *pos, *n;
576	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
577
578	mutex_lock(&vgpu->dmabuf_lock);
579	list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) {
580		dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list);
581		dmabuf_obj->vgpu = NULL;
582
583		idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id);
584		list_del(pos);
585
586		/* dmabuf_obj might be freed in dmabuf_obj_put */
587		if (dmabuf_obj->initref) {
588			dmabuf_obj->initref = false;
589			dmabuf_obj_put(dmabuf_obj);
590		}
591
592	}
593	mutex_unlock(&vgpu->dmabuf_lock);
594}