exynos_drm_g2d.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2012 Samsung Electronics Co.Ltd
   4 * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
   5 */
   6
   7#include <linux/refcount.h>
   8#include <linux/clk.h>
   9#include <linux/component.h>
  10#include <linux/delay.h>
  11#include <linux/dma-mapping.h>
  12#include <linux/err.h>
  13#include <linux/interrupt.h>
  14#include <linux/io.h>
  15#include <linux/kernel.h>
  16#include <linux/of.h>
  17#include <linux/platform_device.h>
  18#include <linux/pm_runtime.h>
  19#include <linux/slab.h>
  20#include <linux/uaccess.h>
  21#include <linux/workqueue.h>
  22
  23#include <drm/drm_file.h>
  24#include <drm/drm_print.h>
  25#include <drm/exynos_drm.h>
  26
  27#include "exynos_drm_drv.h"
  28#include "exynos_drm_g2d.h"
  29#include "exynos_drm_gem.h"
  30
  31#define G2D_HW_MAJOR_VER		4
  32#define G2D_HW_MINOR_VER		1
  33
  34/* vaild register range set from user: 0x0104 ~ 0x0880 */
  35#define G2D_VALID_START			0x0104
  36#define G2D_VALID_END			0x0880
  37
  38/* general registers */
  39#define G2D_SOFT_RESET			0x0000
  40#define G2D_INTEN			0x0004
  41#define G2D_INTC_PEND			0x000C
  42#define G2D_DMA_SFR_BASE_ADDR		0x0080
  43#define G2D_DMA_COMMAND			0x0084
  44#define G2D_DMA_STATUS			0x008C
  45#define G2D_DMA_HOLD_CMD		0x0090
  46
  47/* command registers */
  48#define G2D_BITBLT_START		0x0100
  49
  50/* registers for base address */
  51#define G2D_SRC_BASE_ADDR		0x0304
  52#define G2D_SRC_STRIDE			0x0308
  53#define G2D_SRC_COLOR_MODE		0x030C
  54#define G2D_SRC_LEFT_TOP		0x0310
  55#define G2D_SRC_RIGHT_BOTTOM		0x0314
  56#define G2D_SRC_PLANE2_BASE_ADDR	0x0318
  57#define G2D_DST_BASE_ADDR		0x0404
  58#define G2D_DST_STRIDE			0x0408
  59#define G2D_DST_COLOR_MODE		0x040C
  60#define G2D_DST_LEFT_TOP		0x0410
  61#define G2D_DST_RIGHT_BOTTOM		0x0414
  62#define G2D_DST_PLANE2_BASE_ADDR	0x0418
  63#define G2D_PAT_BASE_ADDR		0x0500
  64#define G2D_MSK_BASE_ADDR		0x0520
  65
  66/* G2D_SOFT_RESET */
  67#define G2D_SFRCLEAR			(1 << 1)
  68#define G2D_R				(1 << 0)
  69
  70/* G2D_INTEN */
  71#define G2D_INTEN_ACF			(1 << 3)
  72#define G2D_INTEN_UCF			(1 << 2)
  73#define G2D_INTEN_GCF			(1 << 1)
  74#define G2D_INTEN_SCF			(1 << 0)
  75
  76/* G2D_INTC_PEND */
  77#define G2D_INTP_ACMD_FIN		(1 << 3)
  78#define G2D_INTP_UCMD_FIN		(1 << 2)
  79#define G2D_INTP_GCMD_FIN		(1 << 1)
  80#define G2D_INTP_SCMD_FIN		(1 << 0)
  81
  82/* G2D_DMA_COMMAND */
  83#define G2D_DMA_HALT			(1 << 2)
  84#define G2D_DMA_CONTINUE		(1 << 1)
  85#define G2D_DMA_START			(1 << 0)
  86
  87/* G2D_DMA_STATUS */
  88#define G2D_DMA_LIST_DONE_COUNT		(0xFF << 17)
  89#define G2D_DMA_BITBLT_DONE_COUNT	(0xFFFF << 1)
  90#define G2D_DMA_DONE			(1 << 0)
  91#define G2D_DMA_LIST_DONE_COUNT_OFFSET	17
  92
  93/* G2D_DMA_HOLD_CMD */
  94#define G2D_USER_HOLD			(1 << 2)
  95#define G2D_LIST_HOLD			(1 << 1)
  96#define G2D_BITBLT_HOLD			(1 << 0)
  97
  98/* G2D_BITBLT_START */
  99#define G2D_START_CASESEL		(1 << 2)
 100#define G2D_START_NHOLT			(1 << 1)
 101#define G2D_START_BITBLT		(1 << 0)
 102
 103/* buffer color format */
 104#define G2D_FMT_XRGB8888		0
 105#define G2D_FMT_ARGB8888		1
 106#define G2D_FMT_RGB565			2
 107#define G2D_FMT_XRGB1555		3
 108#define G2D_FMT_ARGB1555		4
 109#define G2D_FMT_XRGB4444		5
 110#define G2D_FMT_ARGB4444		6
 111#define G2D_FMT_PACKED_RGB888		7
 112#define G2D_FMT_A8			11
 113#define G2D_FMT_L8			12
 114
 115/* buffer valid length */
 116#define G2D_LEN_MIN			1
 117#define G2D_LEN_MAX			8000
 118
 119#define G2D_CMDLIST_SIZE		(PAGE_SIZE / 4)
 120#define G2D_CMDLIST_NUM			64
 121#define G2D_CMDLIST_POOL_SIZE		(G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
 122#define G2D_CMDLIST_DATA_NUM		(G2D_CMDLIST_SIZE / sizeof(u32) - 2)
 123
 124/* maximum buffer pool size of userptr is 64MB as default */
 125#define MAX_POOL		(64 * 1024 * 1024)
 126
 127enum {
 128	BUF_TYPE_GEM = 1,
 129	BUF_TYPE_USERPTR,
 130};
 131
 132enum g2d_reg_type {
 133	REG_TYPE_NONE = -1,
 134	REG_TYPE_SRC,
 135	REG_TYPE_SRC_PLANE2,
 136	REG_TYPE_DST,
 137	REG_TYPE_DST_PLANE2,
 138	REG_TYPE_PAT,
 139	REG_TYPE_MSK,
 140	MAX_REG_TYPE_NR
 141};
 142
 143enum g2d_flag_bits {
 144	/*
 145	 * If set, suspends the runqueue worker after the currently
 146	 * processed node is finished.
 147	 */
 148	G2D_BIT_SUSPEND_RUNQUEUE,
 149	/*
 150	 * If set, indicates that the engine is currently busy.
 151	 */
 152	G2D_BIT_ENGINE_BUSY,
 153};
 154
 155/* cmdlist data structure */
 156struct g2d_cmdlist {
 157	u32		head;
 158	unsigned long	data[G2D_CMDLIST_DATA_NUM];
 159	u32		last;	/* last data offset */
 160};
 161
 162/*
 163 * A structure of buffer description
 164 *
 165 * @format: color format
 166 * @stride: buffer stride/pitch in bytes
 167 * @left_x: the x coordinates of left top corner
 168 * @top_y: the y coordinates of left top corner
 169 * @right_x: the x coordinates of right bottom corner
 170 * @bottom_y: the y coordinates of right bottom corner
 171 *
 172 */
 173struct g2d_buf_desc {
 174	unsigned int	format;
 175	unsigned int	stride;
 176	unsigned int	left_x;
 177	unsigned int	top_y;
 178	unsigned int	right_x;
 179	unsigned int	bottom_y;
 180};
 181
 182/*
 183 * A structure of buffer information
 184 *
 185 * @map_nr: manages the number of mapped buffers
 186 * @reg_types: stores regitster type in the order of requested command
 187 * @handles: stores buffer handle in its reg_type position
 188 * @types: stores buffer type in its reg_type position
 189 * @descs: stores buffer description in its reg_type position
 190 *
 191 */
 192struct g2d_buf_info {
 193	unsigned int		map_nr;
 194	enum g2d_reg_type	reg_types[MAX_REG_TYPE_NR];
 195	void			*obj[MAX_REG_TYPE_NR];
 196	unsigned int		types[MAX_REG_TYPE_NR];
 197	struct g2d_buf_desc	descs[MAX_REG_TYPE_NR];
 198};
 199
 200struct drm_exynos_pending_g2d_event {
 201	struct drm_pending_event	base;
 202	struct drm_exynos_g2d_event	event;
 203};
 204
 205struct g2d_cmdlist_userptr {
 206	struct list_head	list;
 207	dma_addr_t		dma_addr;
 208	unsigned long		userptr;
 209	unsigned long		size;
 210	struct page		**pages;
 211	unsigned int		npages;
 212	struct sg_table		*sgt;
 213	refcount_t		refcount;
 214	bool			in_pool;
 215	bool			out_of_list;
 216};
 217struct g2d_cmdlist_node {
 218	struct list_head	list;
 219	struct g2d_cmdlist	*cmdlist;
 220	dma_addr_t		dma_addr;
 221	struct g2d_buf_info	buf_info;
 222
 223	struct drm_exynos_pending_g2d_event	*event;
 224};
 225
 226struct g2d_runqueue_node {
 227	struct list_head	list;
 228	struct list_head	run_cmdlist;
 229	struct list_head	event_list;
 230	struct drm_file		*filp;
 231	pid_t			pid;
 232	struct completion	complete;
 233	int			async;
 234};
 235
 236struct g2d_data {
 237	struct device			*dev;
 238	void				*dma_priv;
 239	struct clk			*gate_clk;
 240	void __iomem			*regs;
 241	int				irq;
 242	struct workqueue_struct		*g2d_workq;
 243	struct work_struct		runqueue_work;
 244	struct drm_device		*drm_dev;
 245	unsigned long			flags;
 246
 247	/* cmdlist */
 248	struct g2d_cmdlist_node		*cmdlist_node;
 249	struct list_head		free_cmdlist;
 250	struct mutex			cmdlist_mutex;
 251	dma_addr_t			cmdlist_pool;
 252	void				*cmdlist_pool_virt;
 253	unsigned long			cmdlist_dma_attrs;
 254
 255	/* runqueue*/
 256	struct g2d_runqueue_node	*runqueue_node;
 257	struct list_head		runqueue;
 258	struct mutex			runqueue_mutex;
 259	struct kmem_cache		*runqueue_slab;
 260
 261	unsigned long			current_pool;
 262	unsigned long			max_pool;
 263};
 264
 265static inline void g2d_hw_reset(struct g2d_data *g2d)
 266{
 267	writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);
 268	clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 269}
 270
 271static int g2d_init_cmdlist(struct g2d_data *g2d)
 272{
 273	struct device *dev = g2d->dev;
 274	struct g2d_cmdlist_node *node;
 275	int nr;
 276	int ret;
 277	struct g2d_buf_info *buf_info;
 278
 279	g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
 280
 281	g2d->cmdlist_pool_virt = dma_alloc_attrs(to_dma_dev(g2d->drm_dev),
 282						G2D_CMDLIST_POOL_SIZE,
 283						&g2d->cmdlist_pool, GFP_KERNEL,
 284						g2d->cmdlist_dma_attrs);
 285	if (!g2d->cmdlist_pool_virt) {
 286		dev_err(dev, "failed to allocate dma memory\n");
 287		return -ENOMEM;
 288	}
 289
 290	node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
 291	if (!node) {
 292		ret = -ENOMEM;
 293		goto err;
 294	}
 295
 296	for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
 297		unsigned int i;
 298
 299		node[nr].cmdlist =
 300			g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
 301		node[nr].dma_addr =
 302			g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
 303
 304		buf_info = &node[nr].buf_info;
 305		for (i = 0; i < MAX_REG_TYPE_NR; i++)
 306			buf_info->reg_types[i] = REG_TYPE_NONE;
 307
 308		list_add_tail(&node[nr].list, &g2d->free_cmdlist);
 309	}
 310
 311	return 0;
 312
 313err:
 314	dma_free_attrs(to_dma_dev(g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
 315			g2d->cmdlist_pool_virt,
 316			g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
 317	return ret;
 318}
 319
 320static void g2d_fini_cmdlist(struct g2d_data *g2d)
 321{
 322	kfree(g2d->cmdlist_node);
 323
 324	if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
 325		dma_free_attrs(to_dma_dev(g2d->drm_dev),
 326				G2D_CMDLIST_POOL_SIZE,
 327				g2d->cmdlist_pool_virt,
 328				g2d->cmdlist_pool, g2d->cmdlist_dma_attrs);
 329	}
 330}
 331
 332static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
 333{
 334	struct device *dev = g2d->dev;
 335	struct g2d_cmdlist_node *node;
 336
 337	mutex_lock(&g2d->cmdlist_mutex);
 338	if (list_empty(&g2d->free_cmdlist)) {
 339		dev_err(dev, "there is no free cmdlist\n");
 340		mutex_unlock(&g2d->cmdlist_mutex);
 341		return NULL;
 342	}
 343
 344	node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
 345				list);
 346	list_del_init(&node->list);
 347	mutex_unlock(&g2d->cmdlist_mutex);
 348
 349	return node;
 350}
 351
 352static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
 353{
 354	mutex_lock(&g2d->cmdlist_mutex);
 355	list_move_tail(&node->list, &g2d->free_cmdlist);
 356	mutex_unlock(&g2d->cmdlist_mutex);
 357}
 358
 359static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
 360				     struct g2d_cmdlist_node *node)
 361{
 362	struct g2d_cmdlist_node *lnode;
 363
 364	if (list_empty(&file_priv->inuse_cmdlist))
 365		goto add_to_list;
 366
 367	/* this links to base address of new cmdlist */
 368	lnode = list_entry(file_priv->inuse_cmdlist.prev,
 369				struct g2d_cmdlist_node, list);
 370	lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
 371
 372add_to_list:
 373	list_add_tail(&node->list, &file_priv->inuse_cmdlist);
 374
 375	if (node->event)
 376		list_add_tail(&node->event->base.link, &file_priv->event_list);
 377}
 378
 379static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
 380					void *obj,
 381					bool force)
 382{
 383	struct g2d_cmdlist_userptr *g2d_userptr = obj;
 384
 385	if (!obj)
 386		return;
 387
 388	if (force)
 389		goto out;
 390
 391	refcount_dec(&g2d_userptr->refcount);
 392
 393	if (refcount_read(&g2d_userptr->refcount) > 0)
 394		return;
 395
 396	if (g2d_userptr->in_pool)
 397		return;
 398
 399out:
 400	dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt,
 401			  DMA_BIDIRECTIONAL, 0);
 402
 403	unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages,
 404				    true);
 405	kvfree(g2d_userptr->pages);
 406
 407	if (!g2d_userptr->out_of_list)
 408		list_del_init(&g2d_userptr->list);
 409
 410	sg_free_table(g2d_userptr->sgt);
 411	kfree(g2d_userptr->sgt);
 412	kfree(g2d_userptr);
 413}
 414
 415static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 416					unsigned long userptr,
 417					unsigned long size,
 418					struct drm_file *filp,
 419					void **obj)
 420{
 421	struct drm_exynos_file_private *file_priv = filp->driver_priv;
 422	struct g2d_cmdlist_userptr *g2d_userptr;
 423	struct sg_table	*sgt;
 424	unsigned long start, end;
 425	unsigned int npages, offset;
 426	int ret;
 427
 428	if (!size) {
 429		DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
 430		return ERR_PTR(-EINVAL);
 431	}
 432
 433	/* check if userptr already exists in userptr_list. */
 434	list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
 435		if (g2d_userptr->userptr == userptr) {
 436			/*
 437			 * also check size because there could be same address
 438			 * and different size.
 439			 */
 440			if (g2d_userptr->size == size) {
 441				refcount_inc(&g2d_userptr->refcount);
 442				*obj = g2d_userptr;
 443
 444				return &g2d_userptr->dma_addr;
 445			}
 446
 447			/*
 448			 * at this moment, maybe g2d dma is accessing this
 449			 * g2d_userptr memory region so just remove this
 450			 * g2d_userptr object from userptr_list not to be
 451			 * referred again and also except it the userptr
 452			 * pool to be released after the dma access completion.
 453			 */
 454			g2d_userptr->out_of_list = true;
 455			g2d_userptr->in_pool = false;
 456			list_del_init(&g2d_userptr->list);
 457
 458			break;
 459		}
 460	}
 461
 462	g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
 463	if (!g2d_userptr)
 464		return ERR_PTR(-ENOMEM);
 465
 466	refcount_set(&g2d_userptr->refcount, 1);
 467	g2d_userptr->size = size;
 468
 469	start = userptr & PAGE_MASK;
 470	offset = userptr & ~PAGE_MASK;
 471	end = PAGE_ALIGN(userptr + size);
 472	npages = (end - start) >> PAGE_SHIFT;
 473	g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages),
 474					    GFP_KERNEL);
 475	if (!g2d_userptr->pages) {
 476		ret = -ENOMEM;
 477		goto err_free;
 478	}
 479
 480	ret = pin_user_pages_fast(start, npages,
 481				  FOLL_WRITE | FOLL_LONGTERM,
 482				  g2d_userptr->pages);
 483	if (ret != npages) {
 484		DRM_DEV_ERROR(g2d->dev,
 485			      "failed to get user pages from userptr.\n");
 486		if (ret < 0)
 487			goto err_destroy_pages;
 488		npages = ret;
 489		ret = -EFAULT;
 490		goto err_unpin_pages;
 491	}
 492	g2d_userptr->npages = npages;
 493
 494	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
 495	if (!sgt) {
 496		ret = -ENOMEM;
 497		goto err_unpin_pages;
 498	}
 499
 500	ret = sg_alloc_table_from_pages(sgt,
 501					g2d_userptr->pages,
 502					npages, offset, size, GFP_KERNEL);
 503	if (ret < 0) {
 504		DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
 505		goto err_free_sgt;
 506	}
 507
 508	g2d_userptr->sgt = sgt;
 509
 510	ret = dma_map_sgtable(to_dma_dev(g2d->drm_dev), sgt,
 511			      DMA_BIDIRECTIONAL, 0);
 512	if (ret) {
 513		DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
 514		goto err_sg_free_table;
 515	}
 516
 517	g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
 518	g2d_userptr->userptr = userptr;
 519
 520	list_add_tail(&g2d_userptr->list, &file_priv->userptr_list);
 521
 522	if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
 523		g2d->current_pool += npages << PAGE_SHIFT;
 524		g2d_userptr->in_pool = true;
 525	}
 526
 527	*obj = g2d_userptr;
 528
 529	return &g2d_userptr->dma_addr;
 530
 531err_sg_free_table:
 532	sg_free_table(sgt);
 533
 534err_free_sgt:
 535	kfree(sgt);
 536
 537err_unpin_pages:
 538	unpin_user_pages(g2d_userptr->pages, npages);
 539
 540err_destroy_pages:
 541	kvfree(g2d_userptr->pages);
 542
 543err_free:
 544	kfree(g2d_userptr);
 545
 546	return ERR_PTR(ret);
 547}
 548
 549static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
 550{
 551	struct drm_exynos_file_private *file_priv = filp->driver_priv;
 552	struct g2d_cmdlist_userptr *g2d_userptr, *n;
 553
 554	list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
 555		if (g2d_userptr->in_pool)
 556			g2d_userptr_put_dma_addr(g2d, g2d_userptr, true);
 557
 558	g2d->current_pool = 0;
 559}
 560
 561static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
 562{
 563	enum g2d_reg_type reg_type;
 564
 565	switch (reg_offset) {
 566	case G2D_SRC_BASE_ADDR:
 567	case G2D_SRC_STRIDE:
 568	case G2D_SRC_COLOR_MODE:
 569	case G2D_SRC_LEFT_TOP:
 570	case G2D_SRC_RIGHT_BOTTOM:
 571		reg_type = REG_TYPE_SRC;
 572		break;
 573	case G2D_SRC_PLANE2_BASE_ADDR:
 574		reg_type = REG_TYPE_SRC_PLANE2;
 575		break;
 576	case G2D_DST_BASE_ADDR:
 577	case G2D_DST_STRIDE:
 578	case G2D_DST_COLOR_MODE:
 579	case G2D_DST_LEFT_TOP:
 580	case G2D_DST_RIGHT_BOTTOM:
 581		reg_type = REG_TYPE_DST;
 582		break;
 583	case G2D_DST_PLANE2_BASE_ADDR:
 584		reg_type = REG_TYPE_DST_PLANE2;
 585		break;
 586	case G2D_PAT_BASE_ADDR:
 587		reg_type = REG_TYPE_PAT;
 588		break;
 589	case G2D_MSK_BASE_ADDR:
 590		reg_type = REG_TYPE_MSK;
 591		break;
 592	default:
 593		reg_type = REG_TYPE_NONE;
 594		DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
 595			      reg_offset);
 596		break;
 597	}
 598
 599	return reg_type;
 600}
 601
 602static unsigned long g2d_get_buf_bpp(unsigned int format)
 603{
 604	unsigned long bpp;
 605
 606	switch (format) {
 607	case G2D_FMT_XRGB8888:
 608	case G2D_FMT_ARGB8888:
 609		bpp = 4;
 610		break;
 611	case G2D_FMT_RGB565:
 612	case G2D_FMT_XRGB1555:
 613	case G2D_FMT_ARGB1555:
 614	case G2D_FMT_XRGB4444:
 615	case G2D_FMT_ARGB4444:
 616		bpp = 2;
 617		break;
 618	case G2D_FMT_PACKED_RGB888:
 619		bpp = 3;
 620		break;
 621	default:
 622		bpp = 1;
 623		break;
 624	}
 625
 626	return bpp;
 627}
 628
 629static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
 630					struct g2d_buf_desc *buf_desc,
 631					enum g2d_reg_type reg_type,
 632					unsigned long size)
 633{
 634	int width, height;
 635	unsigned long bpp, last_pos;
 636
 637	/*
 638	 * check source and destination buffers only.
 639	 * so the others are always valid.
 640	 */
 641	if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)
 642		return true;
 643
 644	/* This check also makes sure that right_x > left_x. */
 645	width = (int)buf_desc->right_x - (int)buf_desc->left_x;
 646	if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
 647		DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
 648		return false;
 649	}
 650
 651	/* This check also makes sure that bottom_y > top_y. */
 652	height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
 653	if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
 654		DRM_DEV_ERROR(g2d->dev,
 655			      "height[%d] is out of range!\n", height);
 656		return false;
 657	}
 658
 659	bpp = g2d_get_buf_bpp(buf_desc->format);
 660
 661	/* Compute the position of the last byte that the engine accesses. */
 662	last_pos = ((unsigned long)buf_desc->bottom_y - 1) *
 663		(unsigned long)buf_desc->stride +
 664		(unsigned long)buf_desc->right_x * bpp - 1;
 665
 666	/*
 667	 * Since right_x > left_x and bottom_y > top_y we already know
 668	 * that the first_pos < last_pos (first_pos being the position
 669	 * of the first byte the engine accesses), it just remains to
 670	 * check if last_pos is smaller then the buffer size.
 671	 */
 672
 673	if (last_pos >= size) {
 674		DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
 675			      "is out of range [%lu]!\n", last_pos, size);
 676		return false;
 677	}
 678
 679	return true;
 680}
 681
 682static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 683				struct g2d_cmdlist_node *node,
 684				struct drm_device *drm_dev,
 685				struct drm_file *file)
 686{
 687	struct g2d_cmdlist *cmdlist = node->cmdlist;
 688	struct g2d_buf_info *buf_info = &node->buf_info;
 689	int offset;
 690	int ret;
 691	int i;
 692
 693	for (i = 0; i < buf_info->map_nr; i++) {
 694		struct g2d_buf_desc *buf_desc;
 695		enum g2d_reg_type reg_type;
 696		int reg_pos;
 697		unsigned long handle;
 698		dma_addr_t *addr;
 699
 700		reg_pos = cmdlist->last - 2 * (i + 1);
 701
 702		offset = cmdlist->data[reg_pos];
 703		handle = cmdlist->data[reg_pos + 1];
 704
 705		reg_type = g2d_get_reg_type(g2d, offset);
 706		if (reg_type == REG_TYPE_NONE) {
 707			ret = -EFAULT;
 708			goto err;
 709		}
 710
 711		buf_desc = &buf_info->descs[reg_type];
 712
 713		if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
 714			struct exynos_drm_gem *exynos_gem;
 715
 716			exynos_gem = exynos_drm_gem_get(file, handle);
 717			if (!exynos_gem) {
 718				ret = -EFAULT;
 719				goto err;
 720			}
 721
 722			if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
 723							 reg_type, exynos_gem->size)) {
 724				exynos_drm_gem_put(exynos_gem);
 725				ret = -EFAULT;
 726				goto err;
 727			}
 728
 729			addr = &exynos_gem->dma_addr;
 730			buf_info->obj[reg_type] = exynos_gem;
 731		} else {
 732			struct drm_exynos_g2d_userptr g2d_userptr;
 733
 734			if (copy_from_user(&g2d_userptr, (void __user *)handle,
 735				sizeof(struct drm_exynos_g2d_userptr))) {
 736				ret = -EFAULT;
 737				goto err;
 738			}
 739
 740			if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
 741							 reg_type,
 742							 g2d_userptr.size)) {
 743				ret = -EFAULT;
 744				goto err;
 745			}
 746
 747			addr = g2d_userptr_get_dma_addr(g2d,
 748							g2d_userptr.userptr,
 749							g2d_userptr.size,
 750							file,
 751							&buf_info->obj[reg_type]);
 752			if (IS_ERR(addr)) {
 753				ret = -EFAULT;
 754				goto err;
 755			}
 756		}
 757
 758		cmdlist->data[reg_pos + 1] = *addr;
 759		buf_info->reg_types[i] = reg_type;
 760	}
 761
 762	return 0;
 763
 764err:
 765	buf_info->map_nr = i;
 766	return ret;
 767}
 768
 769static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
 770				  struct g2d_cmdlist_node *node,
 771				  struct drm_file *filp)
 772{
 773	struct g2d_buf_info *buf_info = &node->buf_info;
 774	int i;
 775
 776	for (i = 0; i < buf_info->map_nr; i++) {
 777		struct g2d_buf_desc *buf_desc;
 778		enum g2d_reg_type reg_type;
 779		void *obj;
 780
 781		reg_type = buf_info->reg_types[i];
 782
 783		buf_desc = &buf_info->descs[reg_type];
 784		obj = buf_info->obj[reg_type];
 785
 786		if (buf_info->types[reg_type] == BUF_TYPE_GEM)
 787			exynos_drm_gem_put(obj);
 788		else
 789			g2d_userptr_put_dma_addr(g2d, obj, false);
 790
 791		buf_info->reg_types[i] = REG_TYPE_NONE;
 792		buf_info->obj[reg_type] = NULL;
 793		buf_info->types[reg_type] = 0;
 794		memset(buf_desc, 0x00, sizeof(*buf_desc));
 795	}
 796
 797	buf_info->map_nr = 0;
 798}
 799
 800static void g2d_dma_start(struct g2d_data *g2d,
 801			  struct g2d_runqueue_node *runqueue_node)
 802{
 803	struct g2d_cmdlist_node *node =
 804				list_first_entry(&runqueue_node->run_cmdlist,
 805						struct g2d_cmdlist_node, list);
 806
 807	set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 808	writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
 809	writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
 810}
 811
 812static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
 813{
 814	struct g2d_runqueue_node *runqueue_node;
 815
 816	if (list_empty(&g2d->runqueue))
 817		return NULL;
 818
 819	runqueue_node = list_first_entry(&g2d->runqueue,
 820					 struct g2d_runqueue_node, list);
 821	list_del_init(&runqueue_node->list);
 822	return runqueue_node;
 823}
 824
 825static void g2d_free_runqueue_node(struct g2d_data *g2d,
 826				   struct g2d_runqueue_node *runqueue_node)
 827{
 828	struct g2d_cmdlist_node *node;
 829
 830	mutex_lock(&g2d->cmdlist_mutex);
 831	/*
 832	 * commands in run_cmdlist have been completed so unmap all gem
 833	 * objects in each command node so that they are unreferenced.
 834	 */
 835	list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
 836		g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
 837	list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
 838	mutex_unlock(&g2d->cmdlist_mutex);
 839
 840	kmem_cache_free(g2d->runqueue_slab, runqueue_node);
 841}
 842
 843/**
 844 * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
 845 * @g2d: G2D state object
 846 * @file: if not zero, only remove items with this DRM file
 847 *
 848 * Has to be called under runqueue lock.
 849 */
 850static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
 851{
 852	struct g2d_runqueue_node *node, *n;
 853
 854	if (list_empty(&g2d->runqueue))
 855		return;
 856
 857	list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
 858		if (file && node->filp != file)
 859			continue;
 860
 861		list_del_init(&node->list);
 862		g2d_free_runqueue_node(g2d, node);
 863	}
 864}
 865
 866static void g2d_runqueue_worker(struct work_struct *work)
 867{
 868	struct g2d_data *g2d = container_of(work, struct g2d_data,
 869					    runqueue_work);
 870	struct g2d_runqueue_node *runqueue_node;
 871
 872	/*
 873	 * The engine is busy and the completion of the current node is going
 874	 * to poke the runqueue worker, so nothing to do here.
 875	 */
 876	if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
 877		return;
 878
 879	mutex_lock(&g2d->runqueue_mutex);
 880
 881	runqueue_node = g2d->runqueue_node;
 882	g2d->runqueue_node = NULL;
 883
 884	if (runqueue_node) {
 885		pm_runtime_mark_last_busy(g2d->dev);
 886		pm_runtime_put_autosuspend(g2d->dev);
 887
 888		complete(&runqueue_node->complete);
 889		if (runqueue_node->async)
 890			g2d_free_runqueue_node(g2d, runqueue_node);
 891	}
 892
 893	if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
 894		g2d->runqueue_node = g2d_get_runqueue_node(g2d);
 895
 896		if (g2d->runqueue_node) {
 897			int ret;
 898
 899			ret = pm_runtime_resume_and_get(g2d->dev);
 900			if (ret < 0) {
 901				dev_err(g2d->dev, "failed to enable G2D device.\n");
 902				goto out;
 903			}
 904
 905			g2d_dma_start(g2d, g2d->runqueue_node);
 906		}
 907	}
 908
 909out:
 910	mutex_unlock(&g2d->runqueue_mutex);
 911}
 912
 913static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
 914{
 915	struct drm_device *drm_dev = g2d->drm_dev;
 916	struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
 917	struct drm_exynos_pending_g2d_event *e;
 918	struct timespec64 now;
 919
 920	if (list_empty(&runqueue_node->event_list))
 921		return;
 922
 923	e = list_first_entry(&runqueue_node->event_list,
 924			     struct drm_exynos_pending_g2d_event, base.link);
 925
 926	ktime_get_ts64(&now);
 927	e->event.tv_sec = now.tv_sec;
 928	e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
 929	e->event.cmdlist_no = cmdlist_no;
 930
 931	drm_send_event(drm_dev, &e->base);
 932}
 933
 934static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
 935{
 936	struct g2d_data *g2d = dev_id;
 937	u32 pending;
 938
 939	pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
 940	if (pending)
 941		writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
 942
 943	if (pending & G2D_INTP_GCMD_FIN) {
 944		u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
 945
 946		cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
 947						G2D_DMA_LIST_DONE_COUNT_OFFSET;
 948
 949		g2d_finish_event(g2d, cmdlist_no);
 950
 951		writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
 952		if (!(pending & G2D_INTP_ACMD_FIN)) {
 953			writel_relaxed(G2D_DMA_CONTINUE,
 954					g2d->regs + G2D_DMA_COMMAND);
 955		}
 956	}
 957
 958	if (pending & G2D_INTP_ACMD_FIN) {
 959		clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 960		queue_work(g2d->g2d_workq, &g2d->runqueue_work);
 961	}
 962
 963	return IRQ_HANDLED;
 964}
 965
 966/**
 967 * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
 968 * @g2d: G2D state object
 969 * @file: if not zero, only wait if the current runqueue node belongs
 970 *        to the DRM file
 971 *
 972 * Should the engine not become idle after a 100ms timeout, a hardware
 973 * reset is issued.
 974 */
 975static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
 976{
 977	struct device *dev = g2d->dev;
 978
 979	struct g2d_runqueue_node *runqueue_node = NULL;
 980	unsigned int tries = 10;
 981
 982	mutex_lock(&g2d->runqueue_mutex);
 983
 984	/* If no node is currently processed, we have nothing to do. */
 985	if (!g2d->runqueue_node)
 986		goto out;
 987
 988	runqueue_node = g2d->runqueue_node;
 989
 990	/* Check if the currently processed item belongs to us. */
 991	if (file && runqueue_node->filp != file)
 992		goto out;
 993
 994	mutex_unlock(&g2d->runqueue_mutex);
 995
 996	/* Wait for the G2D engine to finish. */
 997	while (tries-- && (g2d->runqueue_node == runqueue_node))
 998		mdelay(10);
 999
1000	mutex_lock(&g2d->runqueue_mutex);
1001
1002	if (g2d->runqueue_node != runqueue_node)
1003		goto out;
1004
1005	dev_err(dev, "wait timed out, resetting engine...\n");
1006	g2d_hw_reset(g2d);
1007
1008	/*
1009	 * After the hardware reset of the engine we are going to loose
1010	 * the IRQ which triggers the PM runtime put().
1011	 * So do this manually here.
1012	 */
1013	pm_runtime_mark_last_busy(dev);
1014	pm_runtime_put_autosuspend(dev);
1015
1016	complete(&runqueue_node->complete);
1017	if (runqueue_node->async)
1018		g2d_free_runqueue_node(g2d, runqueue_node);
1019
1020out:
1021	mutex_unlock(&g2d->runqueue_mutex);
1022}
1023
1024static int g2d_check_reg_offset(struct g2d_data *g2d,
1025				struct g2d_cmdlist_node *node,
1026				int nr, bool for_addr)
1027{
1028	struct g2d_cmdlist *cmdlist = node->cmdlist;
1029	int reg_offset;
1030	int index;
1031	int i;
1032
1033	for (i = 0; i < nr; i++) {
1034		struct g2d_buf_info *buf_info = &node->buf_info;
1035		struct g2d_buf_desc *buf_desc;
1036		enum g2d_reg_type reg_type;
1037		unsigned long value;
1038
1039		index = cmdlist->last - 2 * (i + 1);
1040
1041		reg_offset = cmdlist->data[index] & ~0xfffff000;
1042		if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
1043			goto err;
1044		if (reg_offset % 4)
1045			goto err;
1046
1047		switch (reg_offset) {
1048		case G2D_SRC_BASE_ADDR:
1049		case G2D_SRC_PLANE2_BASE_ADDR:
1050		case G2D_DST_BASE_ADDR:
1051		case G2D_DST_PLANE2_BASE_ADDR:
1052		case G2D_PAT_BASE_ADDR:
1053		case G2D_MSK_BASE_ADDR:
1054			if (!for_addr)
1055				goto err;
1056
1057			reg_type = g2d_get_reg_type(g2d, reg_offset);
1058
1059			/* check userptr buffer type. */
1060			if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
1061				buf_info->types[reg_type] = BUF_TYPE_USERPTR;
1062				cmdlist->data[index] &= ~G2D_BUF_USERPTR;
1063			} else
1064				buf_info->types[reg_type] = BUF_TYPE_GEM;
1065			break;
1066		case G2D_SRC_STRIDE:
1067		case G2D_DST_STRIDE:
1068			if (for_addr)
1069				goto err;
1070
1071			reg_type = g2d_get_reg_type(g2d, reg_offset);
1072
1073			buf_desc = &buf_info->descs[reg_type];
1074			buf_desc->stride = cmdlist->data[index + 1];
1075			break;
1076		case G2D_SRC_COLOR_MODE:
1077		case G2D_DST_COLOR_MODE:
1078			if (for_addr)
1079				goto err;
1080
1081			reg_type = g2d_get_reg_type(g2d, reg_offset);
1082
1083			buf_desc = &buf_info->descs[reg_type];
1084			value = cmdlist->data[index + 1];
1085
1086			buf_desc->format = value & 0xf;
1087			break;
1088		case G2D_SRC_LEFT_TOP:
1089		case G2D_DST_LEFT_TOP:
1090			if (for_addr)
1091				goto err;
1092
1093			reg_type = g2d_get_reg_type(g2d, reg_offset);
1094
1095			buf_desc = &buf_info->descs[reg_type];
1096			value = cmdlist->data[index + 1];
1097
1098			buf_desc->left_x = value & 0x1fff;
1099			buf_desc->top_y = (value & 0x1fff0000) >> 16;
1100			break;
1101		case G2D_SRC_RIGHT_BOTTOM:
1102		case G2D_DST_RIGHT_BOTTOM:
1103			if (for_addr)
1104				goto err;
1105
1106			reg_type = g2d_get_reg_type(g2d, reg_offset);
1107
1108			buf_desc = &buf_info->descs[reg_type];
1109			value = cmdlist->data[index + 1];
1110
1111			buf_desc->right_x = value & 0x1fff;
1112			buf_desc->bottom_y = (value & 0x1fff0000) >> 16;
1113			break;
1114		default:
1115			if (for_addr)
1116				goto err;
1117			break;
1118		}
1119	}
1120
1121	return 0;
1122
1123err:
1124	dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
1125	return -EINVAL;
1126}
1127
1128/* ioctl functions */
1129int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
1130			     struct drm_file *file)
1131{
1132	struct drm_exynos_g2d_get_ver *ver = data;
1133
1134	ver->major = G2D_HW_MAJOR_VER;
1135	ver->minor = G2D_HW_MINOR_VER;
1136
1137	return 0;
1138}
1139
1140int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
1141				 struct drm_file *file)
1142{
1143	struct drm_exynos_file_private *file_priv = file->driver_priv;
1144	struct exynos_drm_private *priv = drm_dev->dev_private;
1145	struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1146	struct drm_exynos_g2d_set_cmdlist *req = data;
1147	struct drm_exynos_g2d_cmd *cmd;
1148	struct drm_exynos_pending_g2d_event *e;
1149	struct g2d_cmdlist_node *node;
1150	struct g2d_cmdlist *cmdlist;
1151	int size;
1152	int ret;
1153
1154	node = g2d_get_cmdlist(g2d);
1155	if (!node)
1156		return -ENOMEM;
1157
1158	/*
1159	 * To avoid an integer overflow for the later size computations, we
1160	 * enforce a maximum number of submitted commands here. This limit is
1161	 * sufficient for all conceivable usage cases of the G2D.
1162	 */
1163	if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
1164	    req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
1165		dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
1166		return -EINVAL;
1167	}
1168
1169	node->event = NULL;
1170
1171	if (req->event_type != G2D_EVENT_NOT) {
1172		e = kzalloc(sizeof(*node->event), GFP_KERNEL);
1173		if (!e) {
1174			ret = -ENOMEM;
1175			goto err;
1176		}
1177
1178		e->event.base.type = DRM_EXYNOS_G2D_EVENT;
1179		e->event.base.length = sizeof(e->event);
1180		e->event.user_data = req->user_data;
1181
1182		ret = drm_event_reserve_init(drm_dev, file, &e->base, &e->event.base);
1183		if (ret) {
1184			kfree(e);
1185			goto err;
1186		}
1187
1188		node->event = e;
1189	}
1190
1191	cmdlist = node->cmdlist;
1192
1193	cmdlist->last = 0;
1194
1195	/*
1196	 * If don't clear SFR registers, the cmdlist is affected by register
1197	 * values of previous cmdlist. G2D hw executes SFR clear command and
1198	 * a next command at the same time then the next command is ignored and
1199	 * is executed rightly from next next command, so needs a dummy command
1200	 * to next command of SFR clear command.
1201	 */
1202	cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
1203	cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
1204	cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
1205	cmdlist->data[cmdlist->last++] = 0;
1206
1207	/*
1208	 * 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
1209	 * and GCF bit should be set to INTEN register if user wants
1210	 * G2D interrupt event once current command list execution is
1211	 * finished.
1212	 * Otherwise only ACF bit should be set to INTEN register so
1213	 * that one interrupt is occurred after all command lists
1214	 * have been completed.
1215	 */
1216	if (node->event) {
1217		cmdlist->data[cmdlist->last++] = G2D_INTEN;
1218		cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
1219		cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
1220		cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
1221	} else {
1222		cmdlist->data[cmdlist->last++] = G2D_INTEN;
1223		cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
1224	}
1225
1226	/*
1227	 * Check the size of cmdlist. The 2 that is added last comes from
1228	 * the implicit G2D_BITBLT_START that is appended once we have
1229	 * checked all the submitted commands.
1230	 */
1231	size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
1232	if (size > G2D_CMDLIST_DATA_NUM) {
1233		dev_err(g2d->dev, "cmdlist size is too big\n");
1234		ret = -EINVAL;
1235		goto err_free_event;
1236	}
1237
1238	cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;
1239
1240	if (copy_from_user(cmdlist->data + cmdlist->last,
1241				(void __user *)cmd,
1242				sizeof(*cmd) * req->cmd_nr)) {
1243		ret = -EFAULT;
1244		goto err_free_event;
1245	}
1246	cmdlist->last += req->cmd_nr * 2;
1247
1248	ret = g2d_check_reg_offset(g2d, node, req->cmd_nr, false);
1249	if (ret < 0)
1250		goto err_free_event;
1251
1252	node->buf_info.map_nr = req->cmd_buf_nr;
1253	if (req->cmd_buf_nr) {
1254		struct drm_exynos_g2d_cmd *cmd_buf;
1255
1256		cmd_buf = (struct drm_exynos_g2d_cmd *)
1257				(unsigned long)req->cmd_buf;
1258
1259		if (copy_from_user(cmdlist->data + cmdlist->last,
1260					(void __user *)cmd_buf,
1261					sizeof(*cmd_buf) * req->cmd_buf_nr)) {
1262			ret = -EFAULT;
1263			goto err_free_event;
1264		}
1265		cmdlist->last += req->cmd_buf_nr * 2;
1266
1267		ret = g2d_check_reg_offset(g2d, node, req->cmd_buf_nr, true);
1268		if (ret < 0)
1269			goto err_free_event;
1270
1271		ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
1272		if (ret < 0)
1273			goto err_unmap;
1274	}
1275
1276	cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
1277	cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
1278
1279	/* head */
1280	cmdlist->head = cmdlist->last / 2;
1281
1282	/* tail */
1283	cmdlist->data[cmdlist->last] = 0;
1284
1285	g2d_add_cmdlist_to_inuse(file_priv, node);
1286
1287	return 0;
1288
1289err_unmap:
1290	g2d_unmap_cmdlist_gem(g2d, node, file);
1291err_free_event:
1292	if (node->event)
1293		drm_event_cancel_free(drm_dev, &node->event->base);
1294err:
1295	g2d_put_cmdlist(g2d, node);
1296	return ret;
1297}
1298
1299int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
1300			  struct drm_file *file)
1301{
1302	struct drm_exynos_file_private *file_priv = file->driver_priv;
1303	struct exynos_drm_private *priv = drm_dev->dev_private;
1304	struct g2d_data *g2d = dev_get_drvdata(priv->g2d_dev);
1305	struct drm_exynos_g2d_exec *req = data;
1306	struct g2d_runqueue_node *runqueue_node;
1307	struct list_head *run_cmdlist;
1308	struct list_head *event_list;
1309
1310	runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
1311	if (!runqueue_node)
1312		return -ENOMEM;
1313
1314	run_cmdlist = &runqueue_node->run_cmdlist;
1315	event_list = &runqueue_node->event_list;
1316	INIT_LIST_HEAD(run_cmdlist);
1317	INIT_LIST_HEAD(event_list);
1318	init_completion(&runqueue_node->complete);
1319	runqueue_node->async = req->async;
1320
1321	list_splice_init(&file_priv->inuse_cmdlist, run_cmdlist);
1322	list_splice_init(&file_priv->event_list, event_list);
1323
1324	if (list_empty(run_cmdlist)) {
1325		dev_err(g2d->dev, "there is no inuse cmdlist\n");
1326		kmem_cache_free(g2d->runqueue_slab, runqueue_node);
1327		return -EPERM;
1328	}
1329
1330	mutex_lock(&g2d->runqueue_mutex);
1331	runqueue_node->pid = current->pid;
1332	runqueue_node->filp = file;
1333	list_add_tail(&runqueue_node->list, &g2d->runqueue);
1334	mutex_unlock(&g2d->runqueue_mutex);
1335
1336	/* Let the runqueue know that there is work to do. */
1337	queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1338
1339	if (req->async)
1340		goto out;
1341
1342	wait_for_completion(&runqueue_node->complete);
1343	g2d_free_runqueue_node(g2d, runqueue_node);
1344
1345out:
1346	return 0;
1347}
1348
1349int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
1350{
1351	struct drm_exynos_file_private *file_priv = file->driver_priv;
1352
1353	INIT_LIST_HEAD(&file_priv->inuse_cmdlist);
1354	INIT_LIST_HEAD(&file_priv->event_list);
1355	INIT_LIST_HEAD(&file_priv->userptr_list);
1356
1357	return 0;
1358}
1359
1360void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
1361{
1362	struct drm_exynos_file_private *file_priv = file->driver_priv;
1363	struct exynos_drm_private *priv = drm_dev->dev_private;
1364	struct g2d_data *g2d;
1365	struct g2d_cmdlist_node *node, *n;
1366
1367	if (!priv->g2d_dev)
1368		return;
1369
1370	g2d = dev_get_drvdata(priv->g2d_dev);
1371
1372	/* Remove the runqueue nodes that belong to us. */
1373	mutex_lock(&g2d->runqueue_mutex);
1374	g2d_remove_runqueue_nodes(g2d, file);
1375	mutex_unlock(&g2d->runqueue_mutex);
1376
1377	/*
1378	 * Wait for the runqueue worker to finish its current node.
1379	 * After this the engine should no longer be accessing any
1380	 * memory belonging to us.
1381	 */
1382	g2d_wait_finish(g2d, file);
1383
1384	/*
1385	 * Even after the engine is idle, there might still be stale cmdlists
1386	 * (i.e. cmdlisst which we submitted but never executed) around, with
1387	 * their corresponding GEM/userptr buffers.
1388	 * Properly unmap these buffers here.
1389	 */
1390	mutex_lock(&g2d->cmdlist_mutex);
1391	list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
1392		g2d_unmap_cmdlist_gem(g2d, node, file);
1393		list_move_tail(&node->list, &g2d->free_cmdlist);
1394	}
1395	mutex_unlock(&g2d->cmdlist_mutex);
1396
1397	/* release all g2d_userptr in pool. */
1398	g2d_userptr_free_all(g2d, file);
1399}
1400
1401static int g2d_bind(struct device *dev, struct device *master, void *data)
1402{
1403	struct g2d_data *g2d = dev_get_drvdata(dev);
1404	struct drm_device *drm_dev = data;
1405	struct exynos_drm_private *priv = drm_dev->dev_private;
1406	int ret;
1407
1408	g2d->drm_dev = drm_dev;
1409
1410	/* allocate dma-aware cmdlist buffer. */
1411	ret = g2d_init_cmdlist(g2d);
1412	if (ret < 0) {
1413		dev_err(dev, "cmdlist init failed\n");
1414		return ret;
1415	}
1416
1417	ret = exynos_drm_register_dma(drm_dev, dev, &g2d->dma_priv);
1418	if (ret < 0) {
1419		dev_err(dev, "failed to enable iommu.\n");
1420		g2d_fini_cmdlist(g2d);
1421		return ret;
1422	}
1423	priv->g2d_dev = dev;
1424
1425	dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
1426			G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1427	return 0;
1428}
1429
1430static void g2d_unbind(struct device *dev, struct device *master, void *data)
1431{
1432	struct g2d_data *g2d = dev_get_drvdata(dev);
1433	struct drm_device *drm_dev = data;
1434	struct exynos_drm_private *priv = drm_dev->dev_private;
1435
1436	/* Suspend operation and wait for engine idle. */
1437	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1438	g2d_wait_finish(g2d, NULL);
1439	priv->g2d_dev = NULL;
1440
1441	cancel_work_sync(&g2d->runqueue_work);
1442	exynos_drm_unregister_dma(g2d->drm_dev, dev, &g2d->dma_priv);
1443}
1444
1445static const struct component_ops g2d_component_ops = {
1446	.bind	= g2d_bind,
1447	.unbind = g2d_unbind,
1448};
1449
1450static int g2d_probe(struct platform_device *pdev)
1451{
1452	struct device *dev = &pdev->dev;
1453	struct g2d_data *g2d;
1454	int ret;
1455
1456	g2d = devm_kzalloc(dev, sizeof(*g2d), GFP_KERNEL);
1457	if (!g2d)
1458		return -ENOMEM;
1459
1460	g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1461			sizeof(struct g2d_runqueue_node), 0, 0, NULL);
1462	if (!g2d->runqueue_slab)
1463		return -ENOMEM;
1464
1465	g2d->dev = dev;
1466
1467	g2d->g2d_workq = create_singlethread_workqueue("g2d");
1468	if (!g2d->g2d_workq) {
1469		dev_err(dev, "failed to create workqueue\n");
1470		ret = -EINVAL;
1471		goto err_destroy_slab;
1472	}
1473
1474	INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1475	INIT_LIST_HEAD(&g2d->free_cmdlist);
1476	INIT_LIST_HEAD(&g2d->runqueue);
1477
1478	mutex_init(&g2d->cmdlist_mutex);
1479	mutex_init(&g2d->runqueue_mutex);
1480
1481	g2d->gate_clk = devm_clk_get(dev, "fimg2d");
1482	if (IS_ERR(g2d->gate_clk)) {
1483		dev_err(dev, "failed to get gate clock\n");
1484		ret = PTR_ERR(g2d->gate_clk);
1485		goto err_destroy_workqueue;
1486	}
1487
1488	pm_runtime_use_autosuspend(dev);
1489	pm_runtime_set_autosuspend_delay(dev, 2000);
1490	pm_runtime_enable(dev);
1491	clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1492	clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
1493
1494	g2d->regs = devm_platform_ioremap_resource(pdev, 0);
1495	if (IS_ERR(g2d->regs)) {
1496		ret = PTR_ERR(g2d->regs);
1497		goto err_put_clk;
1498	}
1499
1500	g2d->irq = platform_get_irq(pdev, 0);
1501	if (g2d->irq < 0) {
1502		ret = g2d->irq;
1503		goto err_put_clk;
1504	}
1505
1506	ret = devm_request_irq(dev, g2d->irq, g2d_irq_handler, 0,
1507								"drm_g2d", g2d);
1508	if (ret < 0) {
1509		dev_err(dev, "irq request failed\n");
1510		goto err_put_clk;
1511	}
1512
1513	g2d->max_pool = MAX_POOL;
1514
1515	platform_set_drvdata(pdev, g2d);
1516
1517	ret = component_add(dev, &g2d_component_ops);
1518	if (ret < 0) {
1519		dev_err(dev, "failed to register drm g2d device\n");
1520		goto err_put_clk;
1521	}
1522
1523	return 0;
1524
1525err_put_clk:
1526	pm_runtime_disable(dev);
1527err_destroy_workqueue:
1528	destroy_workqueue(g2d->g2d_workq);
1529err_destroy_slab:
1530	kmem_cache_destroy(g2d->runqueue_slab);
1531	return ret;
1532}
1533
1534static void g2d_remove(struct platform_device *pdev)
1535{
1536	struct g2d_data *g2d = platform_get_drvdata(pdev);
1537
1538	component_del(&pdev->dev, &g2d_component_ops);
1539
1540	/* There should be no locking needed here. */
1541	g2d_remove_runqueue_nodes(g2d, NULL);
1542
1543	pm_runtime_dont_use_autosuspend(&pdev->dev);
1544	pm_runtime_disable(&pdev->dev);
1545
1546	g2d_fini_cmdlist(g2d);
1547	destroy_workqueue(g2d->g2d_workq);
1548	kmem_cache_destroy(g2d->runqueue_slab);
1549}
1550
1551static int g2d_suspend(struct device *dev)
1552{
1553	struct g2d_data *g2d = dev_get_drvdata(dev);
1554
1555	/*
1556	 * Suspend the runqueue worker operation and wait until the G2D
1557	 * engine is idle.
1558	 */
1559	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1560	g2d_wait_finish(g2d, NULL);
1561	flush_work(&g2d->runqueue_work);
1562
1563	return 0;
1564}
1565
1566static int g2d_resume(struct device *dev)
1567{
1568	struct g2d_data *g2d = dev_get_drvdata(dev);
1569
1570	clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
1571	queue_work(g2d->g2d_workq, &g2d->runqueue_work);
1572
1573	return 0;
1574}
1575
1576static int g2d_runtime_suspend(struct device *dev)
1577{
1578	struct g2d_data *g2d = dev_get_drvdata(dev);
1579
1580	clk_disable_unprepare(g2d->gate_clk);
1581
1582	return 0;
1583}
1584
1585static int g2d_runtime_resume(struct device *dev)
1586{
1587	struct g2d_data *g2d = dev_get_drvdata(dev);
1588	int ret;
1589
1590	ret = clk_prepare_enable(g2d->gate_clk);
1591	if (ret < 0)
1592		dev_warn(dev, "failed to enable clock.\n");
1593
1594	return ret;
1595}
1596
1597static const struct dev_pm_ops g2d_pm_ops = {
1598	SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)
1599	RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)
1600};
1601
1602static const struct of_device_id exynos_g2d_match[] = {
1603	{ .compatible = "samsung,exynos5250-g2d" },
1604	{ .compatible = "samsung,exynos4212-g2d" },
1605	{},
1606};
1607MODULE_DEVICE_TABLE(of, exynos_g2d_match);
1608
1609struct platform_driver g2d_driver = {
1610	.probe		= g2d_probe,
1611	.remove		= g2d_remove,
1612	.driver		= {
1613		.name	= "exynos-drm-g2d",
1614		.pm	= pm_ptr(&g2d_pm_ops),
1615		.of_match_table = exynos_g2d_match,
1616	},
1617};