omap_gem.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2011 Texas Instruments Incorporated - https://www.ti.com/
   4 * Author: Rob Clark <rob.clark@linaro.org>
   5 */
   6
   7#include <linux/dma-mapping.h>
   8#include <linux/seq_file.h>
   9#include <linux/shmem_fs.h>
  10#include <linux/spinlock.h>
  11#include <linux/vmalloc.h>
  12
  13#include <drm/drm_dumb_buffers.h>
  14#include <drm/drm_prime.h>
  15#include <drm/drm_print.h>
  16#include <drm/drm_vma_manager.h>
  17
  18#include "omap_drv.h"
  19#include "omap_dmm_tiler.h"
  20
  21/*
  22 * GEM buffer object implementation.
  23 */
  24
  25/* note: we use upper 8 bits of flags for driver-internal flags: */
  26#define OMAP_BO_MEM_DMA_API	0x01000000	/* memory allocated with the dma_alloc_* API */
  27#define OMAP_BO_MEM_SHMEM	0x02000000	/* memory allocated through shmem backing */
  28#define OMAP_BO_MEM_DMABUF	0x08000000	/* memory imported from a dmabuf */
  29
  30struct omap_gem_object {
  31	struct drm_gem_object base;
  32
  33	struct list_head mm_list;
  34
  35	u32 flags;
  36
  37	/** width/height for tiled formats (rounded up to slot boundaries) */
  38	u16 width, height;
  39
  40	/** roll applied when mapping to DMM */
  41	u32 roll;
  42
  43	/** protects pin_cnt, block, pages, dma_addrs and vaddr */
  44	struct mutex lock;
  45
  46	/**
  47	 * dma_addr contains the buffer DMA address. It is valid for
  48	 *
  49	 * - buffers allocated through the DMA mapping API (with the
  50	 *   OMAP_BO_MEM_DMA_API flag set)
  51	 *
  52	 * - buffers imported from dmabuf (with the OMAP_BO_MEM_DMABUF flag set)
  53	 *   if they are physically contiguous
  54	 *
  55	 * - buffers mapped through the TILER when pin_cnt is not zero, in which
  56	 *   case the DMA address points to the TILER aperture
  57	 *
  58	 * Physically contiguous buffers have their DMA address equal to the
  59	 * physical address as we don't remap those buffers through the TILER.
  60	 *
  61	 * Buffers mapped to the TILER have their DMA address pointing to the
  62	 * TILER aperture. As TILER mappings are refcounted (through pin_cnt)
  63	 * the DMA address must be accessed through omap_gem_pin() to ensure
  64	 * that the mapping won't disappear unexpectedly. References must be
  65	 * released with omap_gem_unpin().
  66	 */
  67	dma_addr_t dma_addr;
  68
  69	/**
  70	 * # of users
  71	 */
  72	refcount_t pin_cnt;
  73
  74	/**
  75	 * If the buffer has been imported from a dmabuf the OMAP_DB_DMABUF flag
  76	 * is set and the sgt field is valid.
  77	 */
  78	struct sg_table *sgt;
  79
  80	/**
  81	 * tiler block used when buffer is remapped in DMM/TILER.
  82	 */
  83	struct tiler_block *block;
  84
  85	/**
  86	 * Array of backing pages, if allocated.  Note that pages are never
  87	 * allocated for buffers originally allocated from contiguous memory
  88	 */
  89	struct page **pages;
  90
  91	/** addresses corresponding to pages in above array */
  92	dma_addr_t *dma_addrs;
  93
  94	/**
  95	 * Virtual address, if mapped.
  96	 */
  97	void *vaddr;
  98};
  99
 100#define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
 101
 102/* To deal with userspace mmap'ings of 2d tiled buffers, which (a) are
 103 * not necessarily pinned in TILER all the time, and (b) when they are
 104 * they are not necessarily page aligned, we reserve one or more small
 105 * regions in each of the 2d containers to use as a user-GART where we
 106 * can create a second page-aligned mapping of parts of the buffer
 107 * being accessed from userspace.
 108 *
 109 * Note that we could optimize slightly when we know that multiple
 110 * tiler containers are backed by the same PAT.. but I'll leave that
 111 * for later..
 112 */
 113#define NUM_USERGART_ENTRIES 2
 114struct omap_drm_usergart_entry {
 115	struct tiler_block *block;	/* the reserved tiler block */
 116	dma_addr_t dma_addr;
 117	struct drm_gem_object *obj;	/* the current pinned obj */
 118	pgoff_t obj_pgoff;		/* page offset of obj currently
 119					   mapped in */
 120};
 121
 122struct omap_drm_usergart {
 123	struct omap_drm_usergart_entry entry[NUM_USERGART_ENTRIES];
 124	int height;				/* height in rows */
 125	int height_shift;		/* ilog2(height in rows) */
 126	int slot_shift;			/* ilog2(width per slot) */
 127	int stride_pfn;			/* stride in pages */
 128	int last;				/* index of last used entry */
 129};
 130
 131/* -----------------------------------------------------------------------------
 132 * Helpers
 133 */
 134
 135/** get mmap offset */
 136u64 omap_gem_mmap_offset(struct drm_gem_object *obj)
 137{
 138	struct drm_device *dev = obj->dev;
 139	int ret;
 140	size_t size;
 141
 142	/* Make it mmapable */
 143	size = omap_gem_mmap_size(obj);
 144	ret = drm_gem_create_mmap_offset_size(obj, size);
 145	if (ret) {
 146		dev_err(dev->dev, "could not allocate mmap offset\n");
 147		return 0;
 148	}
 149
 150	return drm_vma_node_offset_addr(&obj->vma_node);
 151}
 152
 153static bool omap_gem_sgt_is_contiguous(struct sg_table *sgt, size_t size)
 154{
 155	return !(drm_prime_get_contiguous_size(sgt) < size);
 156}
 157
 158static bool omap_gem_is_contiguous(struct omap_gem_object *omap_obj)
 159{
 160	if (omap_obj->flags & OMAP_BO_MEM_DMA_API)
 161		return true;
 162
 163	if ((omap_obj->flags & OMAP_BO_MEM_DMABUF) &&
 164	    omap_gem_sgt_is_contiguous(omap_obj->sgt, omap_obj->base.size))
 165		return true;
 166
 167	return false;
 168}
 169
 170/* -----------------------------------------------------------------------------
 171 * Eviction
 172 */
 173
 174static void omap_gem_evict_entry(struct drm_gem_object *obj,
 175		enum tiler_fmt fmt, struct omap_drm_usergart_entry *entry)
 176{
 177	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 178	struct omap_drm_private *priv = obj->dev->dev_private;
 179	int n = priv->usergart[fmt].height;
 180	size_t size = PAGE_SIZE * n;
 181	loff_t off = omap_gem_mmap_offset(obj) +
 182			(entry->obj_pgoff << PAGE_SHIFT);
 183	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);
 184
 185	if (m > 1) {
 186		int i;
 187		/* if stride > than PAGE_SIZE then sparse mapping: */
 188		for (i = n; i > 0; i--) {
 189			unmap_mapping_range(obj->dev->anon_inode->i_mapping,
 190					    off, PAGE_SIZE, 1);
 191			off += PAGE_SIZE * m;
 192		}
 193	} else {
 194		unmap_mapping_range(obj->dev->anon_inode->i_mapping,
 195				    off, size, 1);
 196	}
 197
 198	entry->obj = NULL;
 199}
 200
 201/* Evict a buffer from usergart, if it is mapped there */
 202static void omap_gem_evict(struct drm_gem_object *obj)
 203{
 204	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 205	struct omap_drm_private *priv = obj->dev->dev_private;
 206
 207	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 208		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
 209		int i;
 210
 211		for (i = 0; i < NUM_USERGART_ENTRIES; i++) {
 212			struct omap_drm_usergart_entry *entry =
 213				&priv->usergart[fmt].entry[i];
 214
 215			if (entry->obj == obj)
 216				omap_gem_evict_entry(obj, fmt, entry);
 217		}
 218	}
 219}
 220
 221/* -----------------------------------------------------------------------------
 222 * Page Management
 223 */
 224
 225/*
 226 * Ensure backing pages are allocated. Must be called with the omap_obj.lock
 227 * held.
 228 */
 229static int omap_gem_attach_pages(struct drm_gem_object *obj)
 230{
 231	struct drm_device *dev = obj->dev;
 232	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 233	struct page **pages;
 234	int npages = obj->size >> PAGE_SHIFT;
 235	int i, ret;
 236	dma_addr_t *addrs;
 237
 238	lockdep_assert_held(&omap_obj->lock);
 239
 240	/*
 241	 * If not using shmem (in which case backing pages don't need to be
 242	 * allocated) or if pages are already allocated we're done.
 243	 */
 244	if (!(omap_obj->flags & OMAP_BO_MEM_SHMEM) || omap_obj->pages)
 245		return 0;
 246
 247	pages = drm_gem_get_pages(obj);
 248	if (IS_ERR(pages)) {
 249		dev_err(obj->dev->dev, "could not get pages: %ld\n", PTR_ERR(pages));
 250		return PTR_ERR(pages);
 251	}
 252
 253	/* for non-cached buffers, ensure the new pages are clean because
 254	 * DSS, GPU, etc. are not cache coherent:
 255	 */
 256	if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) {
 257		addrs = kmalloc_array(npages, sizeof(*addrs), GFP_KERNEL);
 258		if (!addrs) {
 259			ret = -ENOMEM;
 260			goto free_pages;
 261		}
 262
 263		for (i = 0; i < npages; i++) {
 264			addrs[i] = dma_map_page(dev->dev, pages[i],
 265					0, PAGE_SIZE, DMA_TO_DEVICE);
 266
 267			if (dma_mapping_error(dev->dev, addrs[i])) {
 268				dev_warn(dev->dev,
 269					"%s: failed to map page\n", __func__);
 270
 271				for (i = i - 1; i >= 0; --i) {
 272					dma_unmap_page(dev->dev, addrs[i],
 273						PAGE_SIZE, DMA_TO_DEVICE);
 274				}
 275
 276				ret = -ENOMEM;
 277				goto free_addrs;
 278			}
 279		}
 280	} else {
 281		addrs = kcalloc(npages, sizeof(*addrs), GFP_KERNEL);
 282		if (!addrs) {
 283			ret = -ENOMEM;
 284			goto free_pages;
 285		}
 286	}
 287
 288	omap_obj->dma_addrs = addrs;
 289	omap_obj->pages = pages;
 290
 291	return 0;
 292
 293free_addrs:
 294	kfree(addrs);
 295free_pages:
 296	drm_gem_put_pages(obj, pages, true, false);
 297
 298	return ret;
 299}
 300
 301/* Release backing pages. Must be called with the omap_obj.lock held. */
 302static void omap_gem_detach_pages(struct drm_gem_object *obj)
 303{
 304	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 305	unsigned int npages = obj->size >> PAGE_SHIFT;
 306	unsigned int i;
 307
 308	lockdep_assert_held(&omap_obj->lock);
 309
 310	for (i = 0; i < npages; i++) {
 311		if (omap_obj->dma_addrs[i])
 312			dma_unmap_page(obj->dev->dev, omap_obj->dma_addrs[i],
 313				       PAGE_SIZE, DMA_TO_DEVICE);
 314	}
 315
 316	kfree(omap_obj->dma_addrs);
 317	omap_obj->dma_addrs = NULL;
 318
 319	drm_gem_put_pages(obj, omap_obj->pages, true, false);
 320	omap_obj->pages = NULL;
 321}
 322
 323/* get buffer flags */
 324u32 omap_gem_flags(struct drm_gem_object *obj)
 325{
 326	return to_omap_bo(obj)->flags;
 327}
 328
 329/** get mmap size */
 330size_t omap_gem_mmap_size(struct drm_gem_object *obj)
 331{
 332	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 333	size_t size = obj->size;
 334
 335	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 336		/* for tiled buffers, the virtual size has stride rounded up
 337		 * to 4kb.. (to hide the fact that row n+1 might start 16kb or
 338		 * 32kb later!).  But we don't back the entire buffer with
 339		 * pages, only the valid picture part.. so need to adjust for
 340		 * this in the size used to mmap and generate mmap offset
 341		 */
 342		size = tiler_vsize(gem2fmt(omap_obj->flags),
 343				omap_obj->width, omap_obj->height);
 344	}
 345
 346	return size;
 347}
 348
 349/* -----------------------------------------------------------------------------
 350 * Fault Handling
 351 */
 352
 353/* Normal handling for the case of faulting in non-tiled buffers */
 354static vm_fault_t omap_gem_fault_1d(struct drm_gem_object *obj,
 355		struct vm_area_struct *vma, struct vm_fault *vmf)
 356{
 357	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 358	unsigned long pfn;
 359	pgoff_t pgoff;
 360
 361	/* We don't use vmf->pgoff since that has the fake offset: */
 362	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 363
 364	if (omap_obj->pages) {
 365		omap_gem_cpu_sync_page(obj, pgoff);
 366		pfn = page_to_pfn(omap_obj->pages[pgoff]);
 367	} else {
 368		BUG_ON(!omap_gem_is_contiguous(omap_obj));
 369		pfn = (omap_obj->dma_addr >> PAGE_SHIFT) + pgoff;
 370	}
 371
 372	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 373			pfn, pfn << PAGE_SHIFT);
 374
 375	return vmf_insert_mixed(vma, vmf->address, pfn);
 376}
 377
 378/* Special handling for the case of faulting in 2d tiled buffers */
 379static vm_fault_t omap_gem_fault_2d(struct drm_gem_object *obj,
 380		struct vm_area_struct *vma, struct vm_fault *vmf)
 381{
 382	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 383	struct omap_drm_private *priv = obj->dev->dev_private;
 384	struct omap_drm_usergart_entry *entry;
 385	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
 386	struct page *pages[64];  /* XXX is this too much to have on stack? */
 387	unsigned long pfn;
 388	pgoff_t pgoff, base_pgoff;
 389	unsigned long vaddr;
 390	int i, err, slots;
 391	vm_fault_t ret = VM_FAULT_NOPAGE;
 392
 393	/*
 394	 * Note the height of the slot is also equal to the number of pages
 395	 * that need to be mapped in to fill 4kb wide CPU page.  If the slot
 396	 * height is 64, then 64 pages fill a 4kb wide by 64 row region.
 397	 */
 398	const int n = priv->usergart[fmt].height;
 399	const int n_shift = priv->usergart[fmt].height_shift;
 400
 401	/*
 402	 * If buffer width in bytes > PAGE_SIZE then the virtual stride is
 403	 * rounded up to next multiple of PAGE_SIZE.. this need to be taken
 404	 * into account in some of the math, so figure out virtual stride
 405	 * in pages
 406	 */
 407	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);
 408
 409	/* We don't use vmf->pgoff since that has the fake offset: */
 410	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 411
 412	/*
 413	 * Actual address we start mapping at is rounded down to previous slot
 414	 * boundary in the y direction:
 415	 */
 416	base_pgoff = round_down(pgoff, m << n_shift);
 417
 418	/* figure out buffer width in slots */
 419	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;
 420
 421	vaddr = vmf->address - ((pgoff - base_pgoff) << PAGE_SHIFT);
 422
 423	entry = &priv->usergart[fmt].entry[priv->usergart[fmt].last];
 424
 425	/* evict previous buffer using this usergart entry, if any: */
 426	if (entry->obj)
 427		omap_gem_evict_entry(entry->obj, fmt, entry);
 428
 429	entry->obj = obj;
 430	entry->obj_pgoff = base_pgoff;
 431
 432	/* now convert base_pgoff to phys offset from virt offset: */
 433	base_pgoff = (base_pgoff >> n_shift) * slots;
 434
 435	/* for wider-than 4k.. figure out which part of the slot-row we want: */
 436	if (m > 1) {
 437		int off = pgoff % m;
 438		entry->obj_pgoff += off;
 439		base_pgoff /= m;
 440		slots = min(slots - (off << n_shift), n);
 441		base_pgoff += off << n_shift;
 442		vaddr += off << PAGE_SHIFT;
 443	}
 444
 445	/*
 446	 * Map in pages. Beyond the valid pixel part of the buffer, we set
 447	 * pages[i] to NULL to get a dummy page mapped in.. if someone
 448	 * reads/writes it they will get random/undefined content, but at
 449	 * least it won't be corrupting whatever other random page used to
 450	 * be mapped in, or other undefined behavior.
 451	 */
 452	memcpy(pages, &omap_obj->pages[base_pgoff],
 453			sizeof(struct page *) * slots);
 454	memset(pages + slots, 0,
 455			sizeof(struct page *) * (n - slots));
 456
 457	err = tiler_pin(entry->block, pages, ARRAY_SIZE(pages), 0, true);
 458	if (err) {
 459		ret = vmf_error(err);
 460		dev_err(obj->dev->dev, "failed to pin: %d\n", err);
 461		return ret;
 462	}
 463
 464	pfn = entry->dma_addr >> PAGE_SHIFT;
 465
 466	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 467			pfn, pfn << PAGE_SHIFT);
 468
 469	for (i = n; i > 0; i--) {
 470		ret = vmf_insert_mixed(vma, vaddr, pfn);
 471		if (ret & VM_FAULT_ERROR)
 472			break;
 473		pfn += priv->usergart[fmt].stride_pfn;
 474		vaddr += PAGE_SIZE * m;
 475	}
 476
 477	/* simple round-robin: */
 478	priv->usergart[fmt].last = (priv->usergart[fmt].last + 1)
 479				 % NUM_USERGART_ENTRIES;
 480
 481	return ret;
 482}
 483
 484/**
 485 * omap_gem_fault		-	pagefault handler for GEM objects
 486 * @vmf: fault detail
 487 *
 488 * Invoked when a fault occurs on an mmap of a GEM managed area. GEM
 489 * does most of the work for us including the actual map/unmap calls
 490 * but we need to do the actual page work.
 491 *
 492 * The VMA was set up by GEM. In doing so it also ensured that the
 493 * vma->vm_private_data points to the GEM object that is backing this
 494 * mapping.
 495 */
 496static vm_fault_t omap_gem_fault(struct vm_fault *vmf)
 497{
 498	struct vm_area_struct *vma = vmf->vma;
 499	struct drm_gem_object *obj = vma->vm_private_data;
 500	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 501	int err;
 502	vm_fault_t ret;
 503
 504	/* Make sure we don't parallel update on a fault, nor move or remove
 505	 * something from beneath our feet
 506	 */
 507	mutex_lock(&omap_obj->lock);
 508
 509	/* if a shmem backed object, make sure we have pages attached now */
 510	err = omap_gem_attach_pages(obj);
 511	if (err) {
 512		ret = vmf_error(err);
 513		goto fail;
 514	}
 515
 516	/* where should we do corresponding put_pages().. we are mapping
 517	 * the original page, rather than thru a GART, so we can't rely
 518	 * on eviction to trigger this.  But munmap() or all mappings should
 519	 * probably trigger put_pages()?
 520	 */
 521
 522	if (omap_obj->flags & OMAP_BO_TILED_MASK)
 523		ret = omap_gem_fault_2d(obj, vma, vmf);
 524	else
 525		ret = omap_gem_fault_1d(obj, vma, vmf);
 526
 527
 528fail:
 529	mutex_unlock(&omap_obj->lock);
 530	return ret;
 531}
 532
 533static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 534{
 535	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 536
 537	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_IO | VM_MIXEDMAP);
 538
 539	if (omap_obj->flags & OMAP_BO_WC) {
 540		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
 541	} else if (omap_obj->flags & OMAP_BO_UNCACHED) {
 542		vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
 543	} else {
 544		/*
 545		 * We do have some private objects, at least for scanout buffers
 546		 * on hardware without DMM/TILER.  But these are allocated write-
 547		 * combine
 548		 */
 549		if (WARN_ON(!obj->filp))
 550			return -EINVAL;
 551
 552		/*
 553		 * Shunt off cached objs to shmem file so they have their own
 554		 * address_space (so unmap_mapping_range does what we want,
 555		 * in particular in the case of mmap'd dmabufs)
 556		 */
 557		vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node);
 558		vma_set_file(vma, obj->filp);
 559
 560		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
 561	}
 562
 563	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 564
 565	return 0;
 566}
 567
 568/* -----------------------------------------------------------------------------
 569 * Dumb Buffers
 570 */
 571
 572/**
 573 * omap_gem_dumb_create	-	create a dumb buffer
 574 * @file: our client file
 575 * @dev: our device
 576 * @args: the requested arguments copied from userspace
 577 *
 578 * Allocate a buffer suitable for use for a frame buffer of the
 579 * form described by user space. Give userspace a handle by which
 580 * to reference it.
 581 */
 582int omap_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 583		struct drm_mode_create_dumb *args)
 584{
 585	union omap_gem_size gsize = { };
 586	int ret;
 587
 588	ret = drm_mode_size_dumb(dev, args, SZ_8, 0);
 589	if (ret)
 590		return ret;
 591	gsize.bytes = args->size;
 592
 593	return omap_gem_new_handle(dev, file, gsize,
 594			OMAP_BO_SCANOUT | OMAP_BO_WC, &args->handle);
 595}
 596
 597/**
 598 * omap_gem_dumb_map_offset - create an offset for a dumb buffer
 599 * @file: our drm client file
 600 * @dev: drm device
 601 * @handle: GEM handle to the object (from dumb_create)
 602 * @offset: memory map offset placeholder
 603 *
 604 * Do the necessary setup to allow the mapping of the frame buffer
 605 * into user memory. We don't have to do much here at the moment.
 606 */
 607int omap_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 608		u32 handle, u64 *offset)
 609{
 610	struct drm_gem_object *obj;
 611	int ret = 0;
 612
 613	/* GEM does all our handle to object mapping */
 614	obj = drm_gem_object_lookup(file, handle);
 615	if (obj == NULL) {
 616		ret = -ENOENT;
 617		goto fail;
 618	}
 619
 620	*offset = omap_gem_mmap_offset(obj);
 621
 622	drm_gem_object_put(obj);
 623
 624fail:
 625	return ret;
 626}
 627
 628#ifdef CONFIG_DRM_FBDEV_EMULATION
 629/* Set scrolling position.  This allows us to implement fast scrolling
 630 * for console.
 631 *
 632 * Call only from non-atomic contexts.
 633 */
 634int omap_gem_roll(struct drm_gem_object *obj, u32 roll)
 635{
 636	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 637	u32 npages = obj->size >> PAGE_SHIFT;
 638	int ret = 0;
 639
 640	if (roll > npages) {
 641		dev_err(obj->dev->dev, "invalid roll: %d\n", roll);
 642		return -EINVAL;
 643	}
 644
 645	omap_obj->roll = roll;
 646
 647	mutex_lock(&omap_obj->lock);
 648
 649	/* if we aren't mapped yet, we don't need to do anything */
 650	if (omap_obj->block) {
 651		ret = omap_gem_attach_pages(obj);
 652		if (ret)
 653			goto fail;
 654
 655		ret = tiler_pin(omap_obj->block, omap_obj->pages, npages,
 656				roll, true);
 657		if (ret)
 658			dev_err(obj->dev->dev, "could not repin: %d\n", ret);
 659	}
 660
 661fail:
 662	mutex_unlock(&omap_obj->lock);
 663
 664	return ret;
 665}
 666#endif
 667
 668/* -----------------------------------------------------------------------------
 669 * Memory Management & DMA Sync
 670 */
 671
 672/*
 673 * shmem buffers that are mapped cached are not coherent.
 674 *
 675 * We keep track of dirty pages using page faulting to perform cache management.
 676 * When a page is mapped to the CPU in read/write mode the device can't access
 677 * it and omap_obj->dma_addrs[i] is NULL. When a page is mapped to the device
 678 * the omap_obj->dma_addrs[i] is set to the DMA address, and the page is
 679 * unmapped from the CPU.
 680 */
 681static inline bool omap_gem_is_cached_coherent(struct drm_gem_object *obj)
 682{
 683	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 684
 685	return !((omap_obj->flags & OMAP_BO_MEM_SHMEM) &&
 686		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED));
 687}
 688
 689/* Sync the buffer for CPU access.. note pages should already be
 690 * attached, ie. omap_gem_get_pages()
 691 */
 692void omap_gem_cpu_sync_page(struct drm_gem_object *obj, int pgoff)
 693{
 694	struct drm_device *dev = obj->dev;
 695	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 696
 697	if (omap_gem_is_cached_coherent(obj))
 698		return;
 699
 700	if (omap_obj->dma_addrs[pgoff]) {
 701		dma_unmap_page(dev->dev, omap_obj->dma_addrs[pgoff],
 702				PAGE_SIZE, DMA_TO_DEVICE);
 703		omap_obj->dma_addrs[pgoff] = 0;
 704	}
 705}
 706
 707/* sync the buffer for DMA access */
 708void omap_gem_dma_sync_buffer(struct drm_gem_object *obj,
 709		enum dma_data_direction dir)
 710{
 711	struct drm_device *dev = obj->dev;
 712	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 713	int i, npages = obj->size >> PAGE_SHIFT;
 714	struct page **pages = omap_obj->pages;
 715	bool dirty = false;
 716
 717	if (omap_gem_is_cached_coherent(obj))
 718		return;
 719
 720	for (i = 0; i < npages; i++) {
 721		if (!omap_obj->dma_addrs[i]) {
 722			dma_addr_t addr;
 723
 724			addr = dma_map_page(dev->dev, pages[i], 0,
 725					    PAGE_SIZE, dir);
 726			if (dma_mapping_error(dev->dev, addr)) {
 727				dev_warn(dev->dev, "%s: failed to map page\n",
 728					__func__);
 729				break;
 730			}
 731
 732			dirty = true;
 733			omap_obj->dma_addrs[i] = addr;
 734		}
 735	}
 736
 737	if (dirty) {
 738		unmap_mapping_range(obj->filp->f_mapping, 0,
 739				    omap_gem_mmap_size(obj), 1);
 740	}
 741}
 742
 743static int omap_gem_pin_tiler(struct drm_gem_object *obj)
 744{
 745	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 746	u32 npages = obj->size >> PAGE_SHIFT;
 747	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
 748	struct tiler_block *block;
 749	int ret;
 750
 751	BUG_ON(omap_obj->block);
 752
 753	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
 754		block = tiler_reserve_2d(fmt, omap_obj->width, omap_obj->height,
 755					 PAGE_SIZE);
 756	} else {
 757		block = tiler_reserve_1d(obj->size);
 758	}
 759
 760	if (IS_ERR(block)) {
 761		ret = PTR_ERR(block);
 762		dev_err(obj->dev->dev, "could not remap: %d (%d)\n", ret, fmt);
 763		goto fail;
 764	}
 765
 766	/* TODO: enable async refill.. */
 767	ret = tiler_pin(block, omap_obj->pages, npages, omap_obj->roll, true);
 768	if (ret) {
 769		tiler_release(block);
 770		dev_err(obj->dev->dev, "could not pin: %d\n", ret);
 771		goto fail;
 772	}
 773
 774	omap_obj->dma_addr = tiler_ssptr(block);
 775	omap_obj->block = block;
 776
 777	DBG("got dma address: %pad", &omap_obj->dma_addr);
 778
 779fail:
 780	return ret;
 781}
 782
 783/**
 784 * omap_gem_pin() - Pin a GEM object in memory
 785 * @obj: the GEM object
 786 * @dma_addr: the DMA address
 787 *
 788 * Pin the given GEM object in memory and fill the dma_addr pointer with the
 789 * object's DMA address. If the buffer is not physically contiguous it will be
 790 * remapped through the TILER to provide a contiguous view.
 791 *
 792 * Pins are reference-counted, calling this function multiple times is allowed
 793 * as long the corresponding omap_gem_unpin() calls are balanced.
 794 *
 795 * Return 0 on success or a negative error code otherwise.
 796 */
 797int omap_gem_pin(struct drm_gem_object *obj, dma_addr_t *dma_addr)
 798{
 799	struct omap_drm_private *priv = obj->dev->dev_private;
 800	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 801	int ret = 0;
 802
 803	mutex_lock(&omap_obj->lock);
 804
 805	if (!omap_gem_is_contiguous(omap_obj)) {
 806		if (refcount_read(&omap_obj->pin_cnt) == 0) {
 807
 808			refcount_set(&omap_obj->pin_cnt, 1);
 809
 810			ret = omap_gem_attach_pages(obj);
 811			if (ret)
 812				goto fail;
 813
 814			if (omap_obj->flags & OMAP_BO_SCANOUT) {
 815				if (priv->has_dmm) {
 816					ret = omap_gem_pin_tiler(obj);
 817					if (ret)
 818						goto fail;
 819				}
 820			}
 821		} else {
 822			refcount_inc(&omap_obj->pin_cnt);
 823		}
 824	}
 825
 826	if (dma_addr)
 827		*dma_addr = omap_obj->dma_addr;
 828
 829fail:
 830	mutex_unlock(&omap_obj->lock);
 831
 832	return ret;
 833}
 834
 835/**
 836 * omap_gem_unpin_locked() - Unpin a GEM object from memory
 837 * @obj: the GEM object
 838 *
 839 * omap_gem_unpin() without locking.
 840 */
 841static void omap_gem_unpin_locked(struct drm_gem_object *obj)
 842{
 843	struct omap_drm_private *priv = obj->dev->dev_private;
 844	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 845	int ret;
 846
 847	if (omap_gem_is_contiguous(omap_obj))
 848		return;
 849
 850	if (refcount_dec_and_test(&omap_obj->pin_cnt)) {
 851		if (omap_obj->sgt) {
 852			sg_free_table(omap_obj->sgt);
 853			kfree(omap_obj->sgt);
 854			omap_obj->sgt = NULL;
 855		}
 856		if (!(omap_obj->flags & OMAP_BO_SCANOUT))
 857			return;
 858		if (priv->has_dmm) {
 859			ret = tiler_unpin(omap_obj->block);
 860			if (ret) {
 861				dev_err(obj->dev->dev,
 862					"could not unpin pages: %d\n", ret);
 863			}
 864			ret = tiler_release(omap_obj->block);
 865			if (ret) {
 866				dev_err(obj->dev->dev,
 867					"could not release unmap: %d\n", ret);
 868			}
 869			omap_obj->dma_addr = 0;
 870			omap_obj->block = NULL;
 871		}
 872	}
 873}
 874
 875/**
 876 * omap_gem_unpin() - Unpin a GEM object from memory
 877 * @obj: the GEM object
 878 *
 879 * Unpin the given GEM object previously pinned with omap_gem_pin(). Pins are
 880 * reference-counted, the actual unpin will only be performed when the number
 881 * of calls to this function matches the number of calls to omap_gem_pin().
 882 */
 883void omap_gem_unpin(struct drm_gem_object *obj)
 884{
 885	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 886
 887	mutex_lock(&omap_obj->lock);
 888	omap_gem_unpin_locked(obj);
 889	mutex_unlock(&omap_obj->lock);
 890}
 891
 892/* Get rotated scanout address (only valid if already pinned), at the
 893 * specified orientation and x,y offset from top-left corner of buffer
 894 * (only valid for tiled 2d buffers)
 895 */
 896int omap_gem_rotated_dma_addr(struct drm_gem_object *obj, u32 orient,
 897		int x, int y, dma_addr_t *dma_addr)
 898{
 899	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 900	int ret = -EINVAL;
 901
 902	mutex_lock(&omap_obj->lock);
 903
 904	if ((refcount_read(&omap_obj->pin_cnt) > 0) && omap_obj->block &&
 905			(omap_obj->flags & OMAP_BO_TILED_MASK)) {
 906		*dma_addr = tiler_tsptr(omap_obj->block, orient, x, y);
 907		ret = 0;
 908	}
 909
 910	mutex_unlock(&omap_obj->lock);
 911
 912	return ret;
 913}
 914
 915/* Get tiler stride for the buffer (only valid for 2d tiled buffers) */
 916int omap_gem_tiled_stride(struct drm_gem_object *obj, u32 orient)
 917{
 918	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 919	int ret = -EINVAL;
 920	if (omap_obj->flags & OMAP_BO_TILED_MASK)
 921		ret = tiler_stride(gem2fmt(omap_obj->flags), orient);
 922	return ret;
 923}
 924
 925/* if !remap, and we don't have pages backing, then fail, rather than
 926 * increasing the pin count (which we don't really do yet anyways,
 927 * because we don't support swapping pages back out).  And 'remap'
 928 * might not be quite the right name, but I wanted to keep it working
 929 * similarly to omap_gem_pin().  Note though that mutex is not
 930 * aquired if !remap (because this can be called in atomic ctxt),
 931 * but probably omap_gem_unpin() should be changed to work in the
 932 * same way.  If !remap, a matching omap_gem_put_pages() call is not
 933 * required (and should not be made).
 934 */
 935int omap_gem_get_pages(struct drm_gem_object *obj, struct page ***pages,
 936		bool remap)
 937{
 938	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 939	int ret = 0;
 940
 941	mutex_lock(&omap_obj->lock);
 942
 943	if (remap) {
 944		ret = omap_gem_attach_pages(obj);
 945		if (ret)
 946			goto unlock;
 947	}
 948
 949	if (!omap_obj->pages) {
 950		ret = -ENOMEM;
 951		goto unlock;
 952	}
 953
 954	*pages = omap_obj->pages;
 955
 956unlock:
 957	mutex_unlock(&omap_obj->lock);
 958
 959	return ret;
 960}
 961
 962/* release pages when DMA no longer being performed */
 963int omap_gem_put_pages(struct drm_gem_object *obj)
 964{
 965	/* do something here if we dynamically attach/detach pages.. at
 966	 * least they would no longer need to be pinned if everyone has
 967	 * released the pages..
 968	 */
 969	return 0;
 970}
 971
 972struct sg_table *omap_gem_get_sg(struct drm_gem_object *obj,
 973		enum dma_data_direction dir)
 974{
 975	struct omap_gem_object *omap_obj = to_omap_bo(obj);
 976	dma_addr_t addr;
 977	struct sg_table *sgt;
 978	struct scatterlist *sg;
 979	unsigned int count, len, stride, i;
 980	int ret;
 981
 982	ret = omap_gem_pin(obj, &addr);
 983	if (ret)
 984		return ERR_PTR(ret);
 985
 986	mutex_lock(&omap_obj->lock);
 987
 988	sgt = omap_obj->sgt;
 989	if (sgt)
 990		goto out;
 991
 992	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
 993	if (!sgt) {
 994		ret = -ENOMEM;
 995		goto err_unpin;
 996	}
 997
 998	if (addr) {
 999		if (omap_obj->flags & OMAP_BO_TILED_MASK) {
1000			enum tiler_fmt fmt = gem2fmt(omap_obj->flags);
1001
1002			len = omap_obj->width << (int)fmt;
1003			count = omap_obj->height;
1004			stride = tiler_stride(fmt, 0);
1005		} else {
1006			len = obj->size;
1007			count = 1;
1008			stride = 0;
1009		}
1010	} else {
1011		count = obj->size >> PAGE_SHIFT;
1012	}
1013
1014	ret = sg_alloc_table(sgt, count, GFP_KERNEL);
1015	if (ret)
1016		goto err_free;
1017
1018	/* this must be after omap_gem_pin() to ensure we have pages attached */
1019	omap_gem_dma_sync_buffer(obj, dir);
1020
1021	if (addr) {
1022		for_each_sg(sgt->sgl, sg, count, i) {
1023			sg_set_page(sg, pfn_to_page(__phys_to_pfn(addr)),
1024				    len, offset_in_page(addr));
1025			sg_dma_address(sg) = addr;
1026			sg_dma_len(sg) = len;
1027
1028			addr += stride;
1029		}
1030	} else {
1031		for_each_sg(sgt->sgl, sg, count, i) {
1032			sg_set_page(sg, omap_obj->pages[i], PAGE_SIZE, 0);
1033			sg_dma_address(sg) = omap_obj->dma_addrs[i];
1034			sg_dma_len(sg) =  PAGE_SIZE;
1035		}
1036	}
1037
1038	omap_obj->sgt = sgt;
1039out:
1040	mutex_unlock(&omap_obj->lock);
1041	return sgt;
1042
1043err_free:
1044	kfree(sgt);
1045err_unpin:
1046	mutex_unlock(&omap_obj->lock);
1047	omap_gem_unpin(obj);
1048	return ERR_PTR(ret);
1049}
1050
1051void omap_gem_put_sg(struct drm_gem_object *obj, struct sg_table *sgt)
1052{
1053	struct omap_gem_object *omap_obj = to_omap_bo(obj);
1054
1055	if (WARN_ON(omap_obj->sgt != sgt))
1056		return;
1057
1058	omap_gem_unpin(obj);
1059}
1060
1061#ifdef CONFIG_DRM_FBDEV_EMULATION
1062/*
1063 * Get kernel virtual address for CPU access.. this more or less only
1064 * exists for omap_fbdev.
1065 */
1066void *omap_gem_vaddr(struct drm_gem_object *obj)
1067{
1068	struct omap_gem_object *omap_obj = to_omap_bo(obj);
1069	void *vaddr;
1070	int ret;
1071
1072	mutex_lock(&omap_obj->lock);
1073
1074	if (!omap_obj->vaddr) {
1075		ret = omap_gem_attach_pages(obj);
1076		if (ret) {
1077			vaddr = ERR_PTR(ret);
1078			goto unlock;
1079		}
1080
1081		omap_obj->vaddr = vmap(omap_obj->pages, obj->size >> PAGE_SHIFT,
1082				VM_MAP, pgprot_writecombine(PAGE_KERNEL));
1083	}
1084
1085	vaddr = omap_obj->vaddr;
1086
1087unlock:
1088	mutex_unlock(&omap_obj->lock);
1089	return vaddr;
1090}
1091#endif
1092
1093/* -----------------------------------------------------------------------------
1094 * Power Management
1095 */
1096
1097#ifdef CONFIG_PM
1098/* re-pin objects in DMM in resume path: */
1099int omap_gem_resume(struct drm_device *dev)
1100{
1101	struct omap_drm_private *priv = dev->dev_private;
1102	struct omap_gem_object *omap_obj;
1103	int ret = 0;
1104
1105	mutex_lock(&priv->list_lock);
1106	list_for_each_entry(omap_obj, &priv->obj_list, mm_list) {
1107		if (omap_obj->block) {
1108			struct drm_gem_object *obj = &omap_obj->base;
1109			u32 npages = obj->size >> PAGE_SHIFT;
1110
1111			WARN_ON(!omap_obj->pages);  /* this can't happen */
1112			ret = tiler_pin(omap_obj->block,
1113					omap_obj->pages, npages,
1114					omap_obj->roll, true);
1115			if (ret) {
1116				dev_err(dev->dev, "could not repin: %d\n", ret);
1117				goto done;
1118			}
1119		}
1120	}
1121
1122done:
1123	mutex_unlock(&priv->list_lock);
1124	return ret;
1125}
1126#endif
1127
1128/* -----------------------------------------------------------------------------
1129 * DebugFS
1130 */
1131
1132#ifdef CONFIG_DEBUG_FS
1133void omap_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
1134{
1135	struct omap_gem_object *omap_obj = to_omap_bo(obj);
1136	u64 off;
1137
1138	off = drm_vma_node_start(&obj->vma_node);
1139
1140	mutex_lock(&omap_obj->lock);
1141
1142	seq_printf(m, "%08x: %2d (%2d) %08llx %pad (%2d) %p %4d",
1143			omap_obj->flags, obj->name, kref_read(&obj->refcount),
1144			off, &omap_obj->dma_addr,
1145			refcount_read(&omap_obj->pin_cnt),
1146			omap_obj->vaddr, omap_obj->roll);
1147
1148	if (omap_obj->flags & OMAP_BO_TILED_MASK) {
1149		seq_printf(m, " %dx%d", omap_obj->width, omap_obj->height);
1150		if (omap_obj->block) {
1151			struct tcm_area *area = &omap_obj->block->area;
1152			seq_printf(m, " (%dx%d, %dx%d)",
1153					area->p0.x, area->p0.y,
1154					area->p1.x, area->p1.y);
1155		}
1156	} else {
1157		seq_printf(m, " %zu", obj->size);
1158	}
1159
1160	mutex_unlock(&omap_obj->lock);
1161
1162	seq_printf(m, "\n");
1163}
1164
1165void omap_gem_describe_objects(struct list_head *list, struct seq_file *m)
1166{
1167	struct omap_gem_object *omap_obj;
1168	int count = 0;
1169	size_t size = 0;
1170
1171	list_for_each_entry(omap_obj, list, mm_list) {
1172		struct drm_gem_object *obj = &omap_obj->base;
1173		seq_printf(m, "   ");
1174		omap_gem_describe(obj, m);
1175		count++;
1176		size += obj->size;
1177	}
1178
1179	seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
1180}
1181#endif
1182
1183/* -----------------------------------------------------------------------------
1184 * Constructor & Destructor
1185 */
1186
1187static void omap_gem_free_object(struct drm_gem_object *obj)
1188{
1189	struct drm_device *dev = obj->dev;
1190	struct omap_drm_private *priv = dev->dev_private;
1191	struct omap_gem_object *omap_obj = to_omap_bo(obj);
1192
1193	omap_gem_evict(obj);
1194
1195	mutex_lock(&priv->list_lock);
1196	list_del(&omap_obj->mm_list);
1197	mutex_unlock(&priv->list_lock);
1198
1199	/*
1200	 * We own the sole reference to the object at this point, but to keep
1201	 * lockdep happy, we must still take the omap_obj_lock to call
1202	 * omap_gem_detach_pages(). This should hardly make any difference as
1203	 * there can't be any lock contention.
1204	 */
1205	mutex_lock(&omap_obj->lock);
1206
1207	/* The object should not be pinned. */
1208	WARN_ON(refcount_read(&omap_obj->pin_cnt) > 0);
1209
1210	if (omap_obj->pages) {
1211		if (omap_obj->flags & OMAP_BO_MEM_DMABUF)
1212			kfree(omap_obj->pages);
1213		else
1214			omap_gem_detach_pages(obj);
1215	}
1216
1217	if (omap_obj->flags & OMAP_BO_MEM_DMA_API) {
1218		dma_free_wc(dev->dev, obj->size, omap_obj->vaddr,
1219			    omap_obj->dma_addr);
1220	} else if (omap_obj->vaddr) {
1221		vunmap(omap_obj->vaddr);
1222	} else if (obj->import_attach) {
1223		drm_prime_gem_destroy(obj, omap_obj->sgt);
1224	}
1225
1226	mutex_unlock(&omap_obj->lock);
1227
1228	drm_gem_object_release(obj);
1229
1230	mutex_destroy(&omap_obj->lock);
1231
1232	kfree(omap_obj);
1233}
1234
1235static bool omap_gem_validate_flags(struct drm_device *dev, u32 flags)
1236{
1237	struct omap_drm_private *priv = dev->dev_private;
1238
1239	switch (flags & OMAP_BO_CACHE_MASK) {
1240	case OMAP_BO_CACHED:
1241	case OMAP_BO_WC:
1242	case OMAP_BO_CACHE_MASK:
1243		break;
1244
1245	default:
1246		return false;
1247	}
1248
1249	if (flags & OMAP_BO_TILED_MASK) {
1250		if (!priv->usergart)
1251			return false;
1252
1253		switch (flags & OMAP_BO_TILED_MASK) {
1254		case OMAP_BO_TILED_8:
1255		case OMAP_BO_TILED_16:
1256		case OMAP_BO_TILED_32:
1257			break;
1258
1259		default:
1260			return false;
1261		}
1262	}
1263
1264	return true;
1265}
1266
1267static const struct vm_operations_struct omap_gem_vm_ops = {
1268	.fault = omap_gem_fault,
1269	.open = drm_gem_vm_open,
1270	.close = drm_gem_vm_close,
1271};
1272
1273static const struct drm_gem_object_funcs omap_gem_object_funcs = {
1274	.free = omap_gem_free_object,
1275	.export = omap_gem_prime_export,
1276	.mmap = omap_gem_object_mmap,
1277	.vm_ops = &omap_gem_vm_ops,
1278};
1279
1280/* GEM buffer object constructor */
1281struct drm_gem_object *omap_gem_new(struct drm_device *dev,
1282		union omap_gem_size gsize, u32 flags)
1283{
1284	struct omap_drm_private *priv = dev->dev_private;
1285	struct omap_gem_object *omap_obj;
1286	struct drm_gem_object *obj;
1287	struct address_space *mapping;
1288	size_t size;
1289	int ret;
1290
1291	if (!omap_gem_validate_flags(dev, flags))
1292		return NULL;
1293
1294	/* Validate the flags and compute the memory and cache flags. */
1295	if (flags & OMAP_BO_TILED_MASK) {
1296		/*
1297		 * Tiled buffers are always shmem paged backed. When they are
1298		 * scanned out, they are remapped into DMM/TILER.
1299		 */
1300		flags |= OMAP_BO_MEM_SHMEM;
1301
1302		/*
1303		 * Currently don't allow cached buffers. There is some caching
1304		 * stuff that needs to be handled better.
1305		 */
1306		flags &= ~(OMAP_BO_CACHED|OMAP_BO_WC|OMAP_BO_UNCACHED);
1307		flags |= tiler_get_cpu_cache_flags();
1308	} else if ((flags & OMAP_BO_SCANOUT) && !priv->has_dmm) {
1309		/*
1310		 * If we don't have DMM, we must allocate scanout buffers
1311		 * from contiguous DMA memory.
1312		 */
1313		flags |= OMAP_BO_MEM_DMA_API;
1314	} else if (!(flags & OMAP_BO_MEM_DMABUF)) {
1315		/*
1316		 * All other buffers not backed by dma_buf are shmem-backed.
1317		 */
1318		flags |= OMAP_BO_MEM_SHMEM;
1319	}
1320
1321	/* Allocate the initialize the OMAP GEM object. */
1322	omap_obj = kzalloc(sizeof(*omap_obj), GFP_KERNEL);
1323	if (!omap_obj)
1324		return NULL;
1325
1326	obj = &omap_obj->base;
1327	omap_obj->flags = flags;
1328	mutex_init(&omap_obj->lock);
1329
1330	if (flags & OMAP_BO_TILED_MASK) {
1331		/*
1332		 * For tiled buffers align dimensions to slot boundaries and
1333		 * calculate size based on aligned dimensions.
1334		 */
1335		tiler_align(gem2fmt(flags), &gsize.tiled.width,
1336			    &gsize.tiled.height);
1337
1338		size = tiler_size(gem2fmt(flags), gsize.tiled.width,
1339				  gsize.tiled.height);
1340
1341		omap_obj->width = gsize.tiled.width;
1342		omap_obj->height = gsize.tiled.height;
1343	} else {
1344		size = PAGE_ALIGN(gsize.bytes);
1345	}
1346
1347	obj->funcs = &omap_gem_object_funcs;
1348
1349	/* Initialize the GEM object. */
1350	if (!(flags & OMAP_BO_MEM_SHMEM)) {
1351		drm_gem_private_object_init(dev, obj, size);
1352	} else {
1353		ret = drm_gem_object_init(dev, obj, size);
1354		if (ret)
1355			goto err_free;
1356
1357		mapping = obj->filp->f_mapping;
1358		mapping_set_gfp_mask(mapping, GFP_USER | __GFP_DMA32);
1359	}
1360
1361	/* Allocate memory if needed. */
1362	if (flags & OMAP_BO_MEM_DMA_API) {
1363		omap_obj->vaddr = dma_alloc_wc(dev->dev, size,
1364					       &omap_obj->dma_addr,
1365					       GFP_KERNEL);
1366		if (!omap_obj->vaddr)
1367			goto err_release;
1368	}
1369
1370	mutex_lock(&priv->list_lock);
1371	list_add(&omap_obj->mm_list, &priv->obj_list);
1372	mutex_unlock(&priv->list_lock);
1373
1374	return obj;
1375
1376err_release:
1377	drm_gem_object_release(obj);
1378err_free:
1379	kfree(omap_obj);
1380	return NULL;
1381}
1382
1383struct drm_gem_object *omap_gem_new_dmabuf(struct drm_device *dev, size_t size,
1384					   struct sg_table *sgt)
1385{
1386	struct omap_drm_private *priv = dev->dev_private;
1387	struct omap_gem_object *omap_obj;
1388	struct drm_gem_object *obj;
1389	union omap_gem_size gsize;
1390
1391	/* Without a DMM only physically contiguous buffers can be supported. */
1392	if (!omap_gem_sgt_is_contiguous(sgt, size) && !priv->has_dmm)
1393		return ERR_PTR(-EINVAL);
1394
1395	gsize.bytes = PAGE_ALIGN(size);
1396	obj = omap_gem_new(dev, gsize, OMAP_BO_MEM_DMABUF | OMAP_BO_WC);
1397	if (!obj)
1398		return ERR_PTR(-ENOMEM);
1399
1400	omap_obj = to_omap_bo(obj);
1401
1402	omap_obj->sgt = sgt;
1403
1404	if (omap_gem_sgt_is_contiguous(sgt, size)) {
1405		omap_obj->dma_addr = sg_dma_address(sgt->sgl);
1406	} else {
1407		/* Create pages list from sgt */
1408		struct page **pages;
1409		unsigned int npages;
1410		unsigned int ret;
1411
1412		npages = DIV_ROUND_UP(size, PAGE_SIZE);
1413		pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
1414		if (!pages) {
1415			omap_gem_free_object(obj);
1416			return ERR_PTR(-ENOMEM);
1417		}
1418
1419		omap_obj->pages = pages;
1420		ret = drm_prime_sg_to_page_array(sgt, pages, npages);
1421		if (ret) {
1422			omap_gem_free_object(obj);
1423			return ERR_PTR(-ENOMEM);
1424		}
1425	}
1426
1427	return obj;
1428}
1429
1430/* convenience method to construct a GEM buffer object, and userspace handle */
1431int omap_gem_new_handle(struct drm_device *dev, struct drm_file *file,
1432		union omap_gem_size gsize, u32 flags, u32 *handle)
1433{
1434	struct drm_gem_object *obj;
1435	int ret;
1436
1437	obj = omap_gem_new(dev, gsize, flags);
1438	if (!obj)
1439		return -ENOMEM;
1440
1441	ret = drm_gem_handle_create(file, obj, handle);
1442	if (ret) {
1443		omap_gem_free_object(obj);
1444		return ret;
1445	}
1446
1447	/* drop reference from allocate - handle holds it now */
1448	drm_gem_object_put(obj);
1449
1450	return 0;
1451}
1452
1453/* -----------------------------------------------------------------------------
1454 * Init & Cleanup
1455 */
1456
1457/* If DMM is used, we need to set some stuff up.. */
1458void omap_gem_init(struct drm_device *dev)
1459{
1460	struct omap_drm_private *priv = dev->dev_private;
1461	struct omap_drm_usergart *usergart;
1462	const enum tiler_fmt fmts[] = {
1463			TILFMT_8BIT, TILFMT_16BIT, TILFMT_32BIT
1464	};
1465	int i, j;
1466
1467	if (!dmm_is_available()) {
1468		/* DMM only supported on OMAP4 and later, so this isn't fatal */
1469		dev_warn(dev->dev, "DMM not available, disable DMM support\n");
1470		return;
1471	}
1472
1473	usergart = kcalloc(3, sizeof(*usergart), GFP_KERNEL);
1474	if (!usergart)
1475		return;
1476
1477	/* reserve 4k aligned/wide regions for userspace mappings: */
1478	for (i = 0; i < ARRAY_SIZE(fmts); i++) {
1479		u16 h = 1, w = PAGE_SIZE >> i;
1480
1481		tiler_align(fmts[i], &w, &h);
1482		/* note: since each region is 1 4kb page wide, and minimum
1483		 * number of rows, the height ends up being the same as the
1484		 * # of pages in the region
1485		 */
1486		usergart[i].height = h;
1487		usergart[i].height_shift = ilog2(h);
1488		usergart[i].stride_pfn = tiler_stride(fmts[i], 0) >> PAGE_SHIFT;
1489		usergart[i].slot_shift = ilog2((PAGE_SIZE / h) >> i);
1490		for (j = 0; j < NUM_USERGART_ENTRIES; j++) {
1491			struct omap_drm_usergart_entry *entry;
1492			struct tiler_block *block;
1493
1494			entry = &usergart[i].entry[j];
1495			block = tiler_reserve_2d(fmts[i], w, h, PAGE_SIZE);
1496			if (IS_ERR(block)) {
1497				dev_err(dev->dev,
1498						"reserve failed: %d, %d, %ld\n",
1499						i, j, PTR_ERR(block));
1500				return;
1501			}
1502			entry->dma_addr = tiler_ssptr(block);
1503			entry->block = block;
1504
1505			DBG("%d:%d: %dx%d: dma_addr=%pad stride=%d", i, j, w, h,
1506					&entry->dma_addr,
1507					usergart[i].stride_pfn << PAGE_SHIFT);
1508		}
1509	}
1510
1511	priv->usergart = usergart;
1512	priv->has_dmm = true;
1513}
1514
1515void omap_gem_deinit(struct drm_device *dev)
1516{
1517	struct omap_drm_private *priv = dev->dev_private;
1518
1519	/* I believe we can rely on there being no more outstanding GEM
1520	 * objects which could depend on usergart/dmm at this point.
1521	 */
1522	kfree(priv->usergart);
1523}