drm_gem.c

   1/*
   2 * Copyright © 2008 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * Authors:
  24 *    Eric Anholt <eric@anholt.net>
  25 *
  26 */
  27
  28#include <linux/dma-buf.h>
  29#include <linux/export.h>
  30#include <linux/file.h>
  31#include <linux/fs.h>
  32#include <linux/iosys-map.h>
  33#include <linux/mem_encrypt.h>
  34#include <linux/mm.h>
  35#include <linux/mman.h>
  36#include <linux/module.h>
  37#include <linux/pagemap.h>
  38#include <linux/pagevec.h>
  39#include <linux/shmem_fs.h>
  40#include <linux/slab.h>
  41#include <linux/string_helpers.h>
  42#include <linux/types.h>
  43#include <linux/uaccess.h>
  44
  45#include <drm/drm.h>
  46#include <drm/drm_device.h>
  47#include <drm/drm_drv.h>
  48#include <drm/drm_file.h>
  49#include <drm/drm_gem.h>
  50#include <drm/drm_managed.h>
  51#include <drm/drm_print.h>
  52#include <drm/drm_vma_manager.h>
  53
  54#include "drm_internal.h"
  55
  56/** @file drm_gem.c
  57 *
  58 * This file provides some of the base ioctls and library routines for
  59 * the graphics memory manager implemented by each device driver.
  60 *
  61 * Because various devices have different requirements in terms of
  62 * synchronization and migration strategies, implementing that is left up to
  63 * the driver, and all that the general API provides should be generic --
  64 * allocating objects, reading/writing data with the cpu, freeing objects.
  65 * Even there, platform-dependent optimizations for reading/writing data with
  66 * the CPU mean we'll likely hook those out to driver-specific calls.  However,
  67 * the DRI2 implementation wants to have at least allocate/mmap be generic.
  68 *
  69 * The goal was to have swap-backed object allocation managed through
  70 * struct file.  However, file descriptors as handles to a struct file have
  71 * two major failings:
  72 * - Process limits prevent more than 1024 or so being used at a time by
  73 *   default.
  74 * - Inability to allocate high fds will aggravate the X Server's select()
  75 *   handling, and likely that of many GL client applications as well.
  76 *
  77 * This led to a plan of using our own integer IDs (called handles, following
  78 * DRM terminology) to mimic fds, and implement the fd syscalls we need as
  79 * ioctls.  The objects themselves will still include the struct file so
  80 * that we can transition to fds if the required kernel infrastructure shows
  81 * up at a later date, and as our interface with shmfs for memory allocation.
  82 */
  83
  84static void
  85drm_gem_init_release(struct drm_device *dev, void *ptr)
  86{
  87	drm_vma_offset_manager_destroy(dev->vma_offset_manager);
  88}
  89
  90/**
  91 * drm_gem_init - Initialize the GEM device fields
  92 * @dev: drm_devic structure to initialize
  93 */
  94int
  95drm_gem_init(struct drm_device *dev)
  96{
  97	struct drm_vma_offset_manager *vma_offset_manager;
  98
  99	mutex_init(&dev->object_name_lock);
 100	idr_init_base(&dev->object_name_idr, 1);
 101
 102	vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager),
 103					  GFP_KERNEL);
 104	if (!vma_offset_manager)
 105		return -ENOMEM;
 106
 107	dev->vma_offset_manager = vma_offset_manager;
 108	drm_vma_offset_manager_init(vma_offset_manager,
 109				    DRM_FILE_PAGE_OFFSET_START,
 110				    DRM_FILE_PAGE_OFFSET_SIZE);
 111
 112	return drmm_add_action(dev, drm_gem_init_release, NULL);
 113}
 114
 115/**
 116 * drm_gem_object_init_with_mnt - initialize an allocated shmem-backed GEM
 117 * object in a given shmfs mountpoint
 118 *
 119 * @dev: drm_device the object should be initialized for
 120 * @obj: drm_gem_object to initialize
 121 * @size: object size
 122 * @gemfs: tmpfs mount where the GEM object will be created. If NULL, use
 123 * the usual tmpfs mountpoint (`shm_mnt`).
 124 *
 125 * Initialize an already allocated GEM object of the specified size with
 126 * shmfs backing store.
 127 */
 128int drm_gem_object_init_with_mnt(struct drm_device *dev,
 129				 struct drm_gem_object *obj, size_t size,
 130				 struct vfsmount *gemfs)
 131{
 132	struct file *filp;
 133
 134	drm_gem_private_object_init(dev, obj, size);
 135
 136	if (gemfs)
 137		filp = shmem_file_setup_with_mnt(gemfs, "drm mm object", size,
 138						 VM_NORESERVE);
 139	else
 140		filp = shmem_file_setup("drm mm object", size, VM_NORESERVE);
 141
 142	if (IS_ERR(filp))
 143		return PTR_ERR(filp);
 144
 145	obj->filp = filp;
 146
 147	return 0;
 148}
 149EXPORT_SYMBOL(drm_gem_object_init_with_mnt);
 150
 151/**
 152 * drm_gem_object_init - initialize an allocated shmem-backed GEM object
 153 * @dev: drm_device the object should be initialized for
 154 * @obj: drm_gem_object to initialize
 155 * @size: object size
 156 *
 157 * Initialize an already allocated GEM object of the specified size with
 158 * shmfs backing store.
 159 */
 160int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj,
 161			size_t size)
 162{
 163	return drm_gem_object_init_with_mnt(dev, obj, size, NULL);
 164}
 165EXPORT_SYMBOL(drm_gem_object_init);
 166
 167/**
 168 * drm_gem_private_object_init - initialize an allocated private GEM object
 169 * @dev: drm_device the object should be initialized for
 170 * @obj: drm_gem_object to initialize
 171 * @size: object size
 172 *
 173 * Initialize an already allocated GEM object of the specified size with
 174 * no GEM provided backing store. Instead the caller is responsible for
 175 * backing the object and handling it.
 176 */
 177void drm_gem_private_object_init(struct drm_device *dev,
 178				 struct drm_gem_object *obj, size_t size)
 179{
 180	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
 181
 182	obj->dev = dev;
 183	obj->filp = NULL;
 184
 185	kref_init(&obj->refcount);
 186	obj->handle_count = 0;
 187	obj->size = size;
 188	mutex_init(&obj->gpuva.lock);
 189	dma_resv_init(&obj->_resv);
 190	if (!obj->resv)
 191		obj->resv = &obj->_resv;
 192
 193	if (drm_core_check_feature(dev, DRIVER_GEM_GPUVA))
 194		drm_gem_gpuva_init(obj);
 195
 196	drm_vma_node_reset(&obj->vma_node);
 197	INIT_LIST_HEAD(&obj->lru_node);
 198}
 199EXPORT_SYMBOL(drm_gem_private_object_init);
 200
 201/**
 202 * drm_gem_private_object_fini - Finalize a failed drm_gem_object
 203 * @obj: drm_gem_object
 204 *
 205 * Uninitialize an already allocated GEM object when it initialized failed
 206 */
 207void drm_gem_private_object_fini(struct drm_gem_object *obj)
 208{
 209	WARN_ON(obj->dma_buf);
 210
 211	dma_resv_fini(&obj->_resv);
 212	mutex_destroy(&obj->gpuva.lock);
 213}
 214EXPORT_SYMBOL(drm_gem_private_object_fini);
 215
 216static void drm_gem_object_handle_get(struct drm_gem_object *obj)
 217{
 218	struct drm_device *dev = obj->dev;
 219
 220	drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock));
 221
 222	if (obj->handle_count++ == 0)
 223		drm_gem_object_get(obj);
 224}
 225
 226/**
 227 * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any
 228 * @obj: GEM object
 229 *
 230 * Acquires a reference on the GEM buffer object's handle. Required to keep
 231 * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked()
 232 * to release the reference. Does nothing if the buffer object has no handle.
 233 *
 234 * Returns:
 235 * True if a handle exists, or false otherwise
 236 */
 237bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj)
 238{
 239	struct drm_device *dev = obj->dev;
 240
 241	guard(mutex)(&dev->object_name_lock);
 242
 243	/*
 244	 * First ref taken during GEM object creation, if any. Some
 245	 * drivers set up internal framebuffers with GEM objects that
 246	 * do not have a GEM handle. Hence, this counter can be zero.
 247	 */
 248	if (!obj->handle_count)
 249		return false;
 250
 251	drm_gem_object_handle_get(obj);
 252
 253	return true;
 254}
 255
 256/**
 257 * drm_gem_object_handle_free - release resources bound to userspace handles
 258 * @obj: GEM object to clean up.
 259 *
 260 * Called after the last handle to the object has been closed
 261 *
 262 * Removes any name for the object. Note that this must be
 263 * called before drm_gem_object_free or we'll be touching
 264 * freed memory
 265 */
 266static void drm_gem_object_handle_free(struct drm_gem_object *obj)
 267{
 268	struct drm_device *dev = obj->dev;
 269
 270	/* Remove any name for this object */
 271	if (obj->name) {
 272		idr_remove(&dev->object_name_idr, obj->name);
 273		obj->name = 0;
 274	}
 275}
 276
 277static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
 278{
 279	/* Unbreak the reference cycle if we have an exported dma_buf. */
 280	if (obj->dma_buf) {
 281		dma_buf_put(obj->dma_buf);
 282		obj->dma_buf = NULL;
 283	}
 284}
 285
 286/**
 287 * drm_gem_object_handle_put_unlocked - releases reference on user-space handle
 288 * @obj: GEM object
 289 *
 290 * Releases a reference on the GEM buffer object's handle. Possibly releases
 291 * the GEM buffer object and associated dma-buf objects.
 292 */
 293void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
 294{
 295	struct drm_device *dev = obj->dev;
 296	bool final = false;
 297
 298	if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0))
 299		return;
 300
 301	/*
 302	 * Must bump handle count first as this may be the last
 303	 * ref, in which case the object would disappear before
 304	 * we checked for a name.
 305	 */
 306
 307	mutex_lock(&dev->object_name_lock);
 308	if (--obj->handle_count == 0) {
 309		drm_gem_object_handle_free(obj);
 310		drm_gem_object_exported_dma_buf_free(obj);
 311		final = true;
 312	}
 313	mutex_unlock(&dev->object_name_lock);
 314
 315	if (final)
 316		drm_gem_object_put(obj);
 317}
 318
 319/*
 320 * Called at device or object close to release the file's
 321 * handle references on objects.
 322 */
 323static int
 324drm_gem_object_release_handle(int id, void *ptr, void *data)
 325{
 326	struct drm_file *file_priv = data;
 327	struct drm_gem_object *obj = ptr;
 328
 329	if (drm_WARN_ON(obj->dev, !data))
 330		return 0;
 331
 332	if (obj->funcs->close)
 333		obj->funcs->close(obj, file_priv);
 334
 335	mutex_lock(&file_priv->prime.lock);
 336
 337	drm_prime_remove_buf_handle(&file_priv->prime, id);
 338
 339	mutex_unlock(&file_priv->prime.lock);
 340
 341	drm_vma_node_revoke(&obj->vma_node, file_priv);
 342
 343	drm_gem_object_handle_put_unlocked(obj);
 344
 345	return 0;
 346}
 347
 348/**
 349 * drm_gem_handle_delete - deletes the given file-private handle
 350 * @filp: drm file-private structure to use for the handle look up
 351 * @handle: userspace handle to delete
 352 *
 353 * Removes the GEM handle from the @filp lookup table which has been added with
 354 * drm_gem_handle_create(). If this is the last handle also cleans up linked
 355 * resources like GEM names.
 356 */
 357int
 358drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 359{
 360	struct drm_gem_object *obj;
 361
 362	spin_lock(&filp->table_lock);
 363
 364	/* Check if we currently have a reference on the object */
 365	obj = idr_replace(&filp->object_idr, NULL, handle);
 366	spin_unlock(&filp->table_lock);
 367	if (IS_ERR_OR_NULL(obj))
 368		return -EINVAL;
 369
 370	/* Release driver's reference and decrement refcount. */
 371	drm_gem_object_release_handle(handle, obj, filp);
 372
 373	/* And finally make the handle available for future allocations. */
 374	spin_lock(&filp->table_lock);
 375	idr_remove(&filp->object_idr, handle);
 376	spin_unlock(&filp->table_lock);
 377
 378	return 0;
 379}
 380EXPORT_SYMBOL(drm_gem_handle_delete);
 381
 382/**
 383 * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object
 384 * @file: drm file-private structure containing the gem object
 385 * @dev: corresponding drm_device
 386 * @handle: gem object handle
 387 * @offset: return location for the fake mmap offset
 388 *
 389 * This implements the &drm_driver.dumb_map_offset kms driver callback for
 390 * drivers which use gem to manage their backing storage.
 391 *
 392 * Returns:
 393 * 0 on success or a negative error code on failure.
 394 */
 395int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 396			    u32 handle, u64 *offset)
 397{
 398	struct drm_gem_object *obj;
 399	int ret;
 400
 401	obj = drm_gem_object_lookup(file, handle);
 402	if (!obj)
 403		return -ENOENT;
 404
 405	/* Don't allow imported objects to be mapped */
 406	if (drm_gem_is_imported(obj)) {
 407		ret = -EINVAL;
 408		goto out;
 409	}
 410
 411	ret = drm_gem_create_mmap_offset(obj);
 412	if (ret)
 413		goto out;
 414
 415	*offset = drm_vma_node_offset_addr(&obj->vma_node);
 416out:
 417	drm_gem_object_put(obj);
 418
 419	return ret;
 420}
 421EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
 422
 423/**
 424 * drm_gem_handle_create_tail - internal functions to create a handle
 425 * @file_priv: drm file-private structure to register the handle for
 426 * @obj: object to register
 427 * @handlep: pointer to return the created handle to the caller
 428 *
 429 * This expects the &drm_device.object_name_lock to be held already and will
 430 * drop it before returning. Used to avoid races in establishing new handles
 431 * when importing an object from either an flink name or a dma-buf.
 432 *
 433 * Handles must be release again through drm_gem_handle_delete(). This is done
 434 * when userspace closes @file_priv for all attached handles, or through the
 435 * GEM_CLOSE ioctl for individual handles.
 436 */
 437int
 438drm_gem_handle_create_tail(struct drm_file *file_priv,
 439			   struct drm_gem_object *obj,
 440			   u32 *handlep)
 441{
 442	struct drm_device *dev = obj->dev;
 443	u32 handle;
 444	int ret;
 445
 446	WARN_ON(!mutex_is_locked(&dev->object_name_lock));
 447
 448	drm_gem_object_handle_get(obj);
 449
 450	/*
 451	 * Get the user-visible handle using idr.  Preload and perform
 452	 * allocation under our spinlock.
 453	 */
 454	idr_preload(GFP_KERNEL);
 455	spin_lock(&file_priv->table_lock);
 456
 457	ret = idr_alloc(&file_priv->object_idr, NULL, 1, 0, GFP_NOWAIT);
 458
 459	spin_unlock(&file_priv->table_lock);
 460	idr_preload_end();
 461
 462	mutex_unlock(&dev->object_name_lock);
 463	if (ret < 0)
 464		goto err_unref;
 465
 466	handle = ret;
 467
 468	ret = drm_vma_node_allow(&obj->vma_node, file_priv);
 469	if (ret)
 470		goto err_remove;
 471
 472	if (obj->funcs->open) {
 473		ret = obj->funcs->open(obj, file_priv);
 474		if (ret)
 475			goto err_revoke;
 476	}
 477
 478	/* mirrors drm_gem_handle_delete to avoid races */
 479	spin_lock(&file_priv->table_lock);
 480	obj = idr_replace(&file_priv->object_idr, obj, handle);
 481	WARN_ON(obj != NULL);
 482	spin_unlock(&file_priv->table_lock);
 483	*handlep = handle;
 484	return 0;
 485
 486err_revoke:
 487	drm_vma_node_revoke(&obj->vma_node, file_priv);
 488err_remove:
 489	spin_lock(&file_priv->table_lock);
 490	idr_remove(&file_priv->object_idr, handle);
 491	spin_unlock(&file_priv->table_lock);
 492err_unref:
 493	drm_gem_object_handle_put_unlocked(obj);
 494	return ret;
 495}
 496
 497/**
 498 * drm_gem_handle_create - create a gem handle for an object
 499 * @file_priv: drm file-private structure to register the handle for
 500 * @obj: object to register
 501 * @handlep: pointer to return the created handle to the caller
 502 *
 503 * Create a handle for this object. This adds a handle reference to the object,
 504 * which includes a regular reference count. Callers will likely want to
 505 * dereference the object afterwards.
 506 *
 507 * Since this publishes @obj to userspace it must be fully set up by this point,
 508 * drivers must call this last in their buffer object creation callbacks.
 509 */
 510int drm_gem_handle_create(struct drm_file *file_priv,
 511			  struct drm_gem_object *obj,
 512			  u32 *handlep)
 513{
 514	mutex_lock(&obj->dev->object_name_lock);
 515
 516	return drm_gem_handle_create_tail(file_priv, obj, handlep);
 517}
 518EXPORT_SYMBOL(drm_gem_handle_create);
 519
 520
 521/**
 522 * drm_gem_free_mmap_offset - release a fake mmap offset for an object
 523 * @obj: obj in question
 524 *
 525 * This routine frees fake offsets allocated by drm_gem_create_mmap_offset().
 526 *
 527 * Note that drm_gem_object_release() already calls this function, so drivers
 528 * don't have to take care of releasing the mmap offset themselves when freeing
 529 * the GEM object.
 530 */
 531void
 532drm_gem_free_mmap_offset(struct drm_gem_object *obj)
 533{
 534	struct drm_device *dev = obj->dev;
 535
 536	drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node);
 537}
 538EXPORT_SYMBOL(drm_gem_free_mmap_offset);
 539
 540/**
 541 * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object
 542 * @obj: obj in question
 543 * @size: the virtual size
 544 *
 545 * GEM memory mapping works by handing back to userspace a fake mmap offset
 546 * it can use in a subsequent mmap(2) call.  The DRM core code then looks
 547 * up the object based on the offset and sets up the various memory mapping
 548 * structures.
 549 *
 550 * This routine allocates and attaches a fake offset for @obj, in cases where
 551 * the virtual size differs from the physical size (ie. &drm_gem_object.size).
 552 * Otherwise just use drm_gem_create_mmap_offset().
 553 *
 554 * This function is idempotent and handles an already allocated mmap offset
 555 * transparently. Drivers do not need to check for this case.
 556 */
 557int
 558drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)
 559{
 560	struct drm_device *dev = obj->dev;
 561
 562	return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node,
 563				  size / PAGE_SIZE);
 564}
 565EXPORT_SYMBOL(drm_gem_create_mmap_offset_size);
 566
 567/**
 568 * drm_gem_create_mmap_offset - create a fake mmap offset for an object
 569 * @obj: obj in question
 570 *
 571 * GEM memory mapping works by handing back to userspace a fake mmap offset
 572 * it can use in a subsequent mmap(2) call.  The DRM core code then looks
 573 * up the object based on the offset and sets up the various memory mapping
 574 * structures.
 575 *
 576 * This routine allocates and attaches a fake offset for @obj.
 577 *
 578 * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release
 579 * the fake offset again.
 580 */
 581int drm_gem_create_mmap_offset(struct drm_gem_object *obj)
 582{
 583	return drm_gem_create_mmap_offset_size(obj, obj->size);
 584}
 585EXPORT_SYMBOL(drm_gem_create_mmap_offset);
 586
 587/*
 588 * Move folios to appropriate lru and release the folios, decrementing the
 589 * ref count of those folios.
 590 */
 591static void drm_gem_check_release_batch(struct folio_batch *fbatch)
 592{
 593	check_move_unevictable_folios(fbatch);
 594	__folio_batch_release(fbatch);
 595	cond_resched();
 596}
 597
 598/**
 599 * drm_gem_get_pages - helper to allocate backing pages for a GEM object
 600 * from shmem
 601 * @obj: obj in question
 602 *
 603 * This reads the page-array of the shmem-backing storage of the given gem
 604 * object. An array of pages is returned. If a page is not allocated or
 605 * swapped-out, this will allocate/swap-in the required pages. Note that the
 606 * whole object is covered by the page-array and pinned in memory.
 607 *
 608 * Use drm_gem_put_pages() to release the array and unpin all pages.
 609 *
 610 * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()).
 611 * If you require other GFP-masks, you have to do those allocations yourself.
 612 *
 613 * Note that you are not allowed to change gfp-zones during runtime. That is,
 614 * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as
 615 * set during initialization. If you have special zone constraints, set them
 616 * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care
 617 * to keep pages in the required zone during swap-in.
 618 *
 619 * This function is only valid on objects initialized with
 620 * drm_gem_object_init(), but not for those initialized with
 621 * drm_gem_private_object_init() only.
 622 */
 623struct page **drm_gem_get_pages(struct drm_gem_object *obj)
 624{
 625	struct address_space *mapping;
 626	struct page **pages;
 627	struct folio *folio;
 628	struct folio_batch fbatch;
 629	unsigned long i, j, npages;
 630
 631	if (WARN_ON(!obj->filp))
 632		return ERR_PTR(-EINVAL);
 633
 634	/* This is the shared memory object that backs the GEM resource */
 635	mapping = obj->filp->f_mapping;
 636
 637	/* We already BUG_ON() for non-page-aligned sizes in
 638	 * drm_gem_object_init(), so we should never hit this unless
 639	 * driver author is doing something really wrong:
 640	 */
 641	WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
 642
 643	npages = obj->size >> PAGE_SHIFT;
 644
 645	pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
 646	if (pages == NULL)
 647		return ERR_PTR(-ENOMEM);
 648
 649	mapping_set_unevictable(mapping);
 650
 651	i = 0;
 652	while (i < npages) {
 653		unsigned long nr;
 654		folio = shmem_read_folio_gfp(mapping, i,
 655				mapping_gfp_mask(mapping));
 656		if (IS_ERR(folio))
 657			goto fail;
 658		nr = min(npages - i, folio_nr_pages(folio));
 659		for (j = 0; j < nr; j++, i++)
 660			pages[i] = folio_file_page(folio, i);
 661
 662		/* Make sure shmem keeps __GFP_DMA32 allocated pages in the
 663		 * correct region during swapin. Note that this requires
 664		 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping)
 665		 * so shmem can relocate pages during swapin if required.
 666		 */
 667		BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) &&
 668				(folio_pfn(folio) >= 0x00100000UL));
 669	}
 670
 671	return pages;
 672
 673fail:
 674	mapping_clear_unevictable(mapping);
 675	folio_batch_init(&fbatch);
 676	j = 0;
 677	while (j < i) {
 678		struct folio *f = page_folio(pages[j]);
 679		if (!folio_batch_add(&fbatch, f))
 680			drm_gem_check_release_batch(&fbatch);
 681		j += folio_nr_pages(f);
 682	}
 683	if (fbatch.nr)
 684		drm_gem_check_release_batch(&fbatch);
 685
 686	kvfree(pages);
 687	return ERR_CAST(folio);
 688}
 689EXPORT_SYMBOL(drm_gem_get_pages);
 690
 691/**
 692 * drm_gem_put_pages - helper to free backing pages for a GEM object
 693 * @obj: obj in question
 694 * @pages: pages to free
 695 * @dirty: if true, pages will be marked as dirty
 696 * @accessed: if true, the pages will be marked as accessed
 697 */
 698void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
 699		bool dirty, bool accessed)
 700{
 701	int i, npages;
 702	struct address_space *mapping;
 703	struct folio_batch fbatch;
 704
 705	mapping = file_inode(obj->filp)->i_mapping;
 706	mapping_clear_unevictable(mapping);
 707
 708	/* We already BUG_ON() for non-page-aligned sizes in
 709	 * drm_gem_object_init(), so we should never hit this unless
 710	 * driver author is doing something really wrong:
 711	 */
 712	WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
 713
 714	npages = obj->size >> PAGE_SHIFT;
 715
 716	folio_batch_init(&fbatch);
 717	for (i = 0; i < npages; i++) {
 718		struct folio *folio;
 719
 720		if (!pages[i])
 721			continue;
 722		folio = page_folio(pages[i]);
 723
 724		if (dirty)
 725			folio_mark_dirty(folio);
 726
 727		if (accessed)
 728			folio_mark_accessed(folio);
 729
 730		/* Undo the reference we took when populating the table */
 731		if (!folio_batch_add(&fbatch, folio))
 732			drm_gem_check_release_batch(&fbatch);
 733		i += folio_nr_pages(folio) - 1;
 734	}
 735	if (folio_batch_count(&fbatch))
 736		drm_gem_check_release_batch(&fbatch);
 737
 738	kvfree(pages);
 739}
 740EXPORT_SYMBOL(drm_gem_put_pages);
 741
 742static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
 743			  struct drm_gem_object **objs)
 744{
 745	int i, ret = 0;
 746	struct drm_gem_object *obj;
 747
 748	spin_lock(&filp->table_lock);
 749
 750	for (i = 0; i < count; i++) {
 751		/* Check if we currently have a reference on the object */
 752		obj = idr_find(&filp->object_idr, handle[i]);
 753		if (!obj) {
 754			ret = -ENOENT;
 755			break;
 756		}
 757		drm_gem_object_get(obj);
 758		objs[i] = obj;
 759	}
 760	spin_unlock(&filp->table_lock);
 761
 762	return ret;
 763}
 764
 765/**
 766 * drm_gem_objects_lookup - look up GEM objects from an array of handles
 767 * @filp: DRM file private date
 768 * @bo_handles: user pointer to array of userspace handle
 769 * @count: size of handle array
 770 * @objs_out: returned pointer to array of drm_gem_object pointers
 771 *
 772 * Takes an array of userspace handles and returns a newly allocated array of
 773 * GEM objects.
 774 *
 775 * For a single handle lookup, use drm_gem_object_lookup().
 776 *
 777 * Returns:
 778 * @objs filled in with GEM object pointers. Returned GEM objects need to be
 779 * released with drm_gem_object_put(). -ENOENT is returned on a lookup
 780 * failure. 0 is returned on success.
 781 *
 782 */
 783int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
 784			   int count, struct drm_gem_object ***objs_out)
 785{
 786	struct drm_gem_object **objs;
 787	u32 *handles;
 788	int ret;
 789
 790	if (!count)
 791		return 0;
 792
 793	objs = kvmalloc_array(count, sizeof(struct drm_gem_object *),
 794			     GFP_KERNEL | __GFP_ZERO);
 795	if (!objs)
 796		return -ENOMEM;
 797
 798	*objs_out = objs;
 799
 800	handles = vmemdup_array_user(bo_handles, count, sizeof(u32));
 801	if (IS_ERR(handles))
 802		return PTR_ERR(handles);
 803
 804	ret = objects_lookup(filp, handles, count, objs);
 805	kvfree(handles);
 806	return ret;
 807
 808}
 809EXPORT_SYMBOL(drm_gem_objects_lookup);
 810
 811/**
 812 * drm_gem_object_lookup - look up a GEM object from its handle
 813 * @filp: DRM file private date
 814 * @handle: userspace handle
 815 *
 816 * If looking up an array of handles, use drm_gem_objects_lookup().
 817 *
 818 * Returns:
 819 * A reference to the object named by the handle if such exists on @filp, NULL
 820 * otherwise.
 821 */
 822struct drm_gem_object *
 823drm_gem_object_lookup(struct drm_file *filp, u32 handle)
 824{
 825	struct drm_gem_object *obj = NULL;
 826
 827	objects_lookup(filp, &handle, 1, &obj);
 828	return obj;
 829}
 830EXPORT_SYMBOL(drm_gem_object_lookup);
 831
 832/**
 833 * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects
 834 * shared and/or exclusive fences.
 835 * @filep: DRM file private date
 836 * @handle: userspace handle
 837 * @wait_all: if true, wait on all fences, else wait on just exclusive fence
 838 * @timeout: timeout value in jiffies or zero to return immediately
 839 *
 840 * Returns:
 841 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
 842 * greater than 0 on success.
 843 */
 844long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle,
 845				    bool wait_all, unsigned long timeout)
 846{
 847	struct drm_device *dev = filep->minor->dev;
 848	struct drm_gem_object *obj;
 849	long ret;
 850
 851	obj = drm_gem_object_lookup(filep, handle);
 852	if (!obj) {
 853		drm_dbg_core(dev, "Failed to look up GEM BO %d\n", handle);
 854		return -EINVAL;
 855	}
 856
 857	ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all),
 858				    true, timeout);
 859	if (ret == 0)
 860		ret = -ETIME;
 861	else if (ret > 0)
 862		ret = 0;
 863
 864	drm_gem_object_put(obj);
 865
 866	return ret;
 867}
 868EXPORT_SYMBOL(drm_gem_dma_resv_wait);
 869
 870int
 871drm_gem_close_ioctl(struct drm_device *dev, void *data,
 872		    struct drm_file *file_priv)
 873{
 874	struct drm_gem_close *args = data;
 875	int ret;
 876
 877	if (!drm_core_check_feature(dev, DRIVER_GEM))
 878		return -EOPNOTSUPP;
 879
 880	ret = drm_gem_handle_delete(file_priv, args->handle);
 881
 882	return ret;
 883}
 884
 885int
 886drm_gem_flink_ioctl(struct drm_device *dev, void *data,
 887		    struct drm_file *file_priv)
 888{
 889	struct drm_gem_flink *args = data;
 890	struct drm_gem_object *obj;
 891	int ret;
 892
 893	if (!drm_core_check_feature(dev, DRIVER_GEM))
 894		return -EOPNOTSUPP;
 895
 896	obj = drm_gem_object_lookup(file_priv, args->handle);
 897	if (obj == NULL)
 898		return -ENOENT;
 899
 900	mutex_lock(&dev->object_name_lock);
 901	/* prevent races with concurrent gem_close. */
 902	if (obj->handle_count == 0) {
 903		ret = -ENOENT;
 904		goto err;
 905	}
 906
 907	if (!obj->name) {
 908		ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
 909		if (ret < 0)
 910			goto err;
 911
 912		obj->name = ret;
 913	}
 914
 915	args->name = (uint64_t) obj->name;
 916	ret = 0;
 917
 918err:
 919	mutex_unlock(&dev->object_name_lock);
 920	drm_gem_object_put(obj);
 921	return ret;
 922}
 923
 924int
 925drm_gem_open_ioctl(struct drm_device *dev, void *data,
 926		   struct drm_file *file_priv)
 927{
 928	struct drm_gem_open *args = data;
 929	struct drm_gem_object *obj;
 930	int ret;
 931	u32 handle;
 932
 933	if (!drm_core_check_feature(dev, DRIVER_GEM))
 934		return -EOPNOTSUPP;
 935
 936	mutex_lock(&dev->object_name_lock);
 937	obj = idr_find(&dev->object_name_idr, (int) args->name);
 938	if (obj) {
 939		drm_gem_object_get(obj);
 940	} else {
 941		mutex_unlock(&dev->object_name_lock);
 942		return -ENOENT;
 943	}
 944
 945	/* drm_gem_handle_create_tail unlocks dev->object_name_lock. */
 946	ret = drm_gem_handle_create_tail(file_priv, obj, &handle);
 947	if (ret)
 948		goto err;
 949
 950	args->handle = handle;
 951	args->size = obj->size;
 952
 953err:
 954	drm_gem_object_put(obj);
 955	return ret;
 956}
 957
 958int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
 959				struct drm_file *file_priv)
 960{
 961	struct drm_gem_change_handle *args = data;
 962	struct drm_gem_object *obj;
 963	int handle, ret;
 964
 965	if (!drm_core_check_feature(dev, DRIVER_GEM))
 966		return -EOPNOTSUPP;
 967
 968	/* idr_alloc() limitation. */
 969	if (args->new_handle > INT_MAX)
 970		return -EINVAL;
 971	handle = args->new_handle;
 972
 973	obj = drm_gem_object_lookup(file_priv, args->handle);
 974	if (!obj)
 975		return -ENOENT;
 976
 977	if (args->handle == handle) {
 978		ret = 0;
 979		goto out;
 980	}
 981
 982	mutex_lock(&file_priv->prime.lock);
 983
 984	spin_lock(&file_priv->table_lock);
 985	ret = idr_alloc(&file_priv->object_idr, obj, handle, handle + 1,
 986			GFP_NOWAIT);
 987	spin_unlock(&file_priv->table_lock);
 988
 989	if (ret < 0)
 990		goto out_unlock;
 991
 992	if (obj->dma_buf) {
 993		ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf,
 994					       handle);
 995		if (ret < 0) {
 996			spin_lock(&file_priv->table_lock);
 997			idr_remove(&file_priv->object_idr, handle);
 998			spin_unlock(&file_priv->table_lock);
 999			goto out_unlock;
1000		}
1001
1002		drm_prime_remove_buf_handle(&file_priv->prime, args->handle);
1003	}
1004
1005	ret = 0;
1006
1007	spin_lock(&file_priv->table_lock);
1008	idr_remove(&file_priv->object_idr, args->handle);
1009	spin_unlock(&file_priv->table_lock);
1010
1011out_unlock:
1012	mutex_unlock(&file_priv->prime.lock);
1013out:
1014	drm_gem_object_put(obj);
1015
1016	return ret;
1017}
1018
1019/**
1020 * drm_gem_open - initializes GEM file-private structures at devnode open time
1021 * @dev: drm_device which is being opened by userspace
1022 * @file_private: drm file-private structure to set up
1023 *
1024 * Called at device open time, sets up the structure for handling refcounting
1025 * of mm objects.
1026 */
1027void
1028drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
1029{
1030	idr_init_base(&file_private->object_idr, 1);
1031	spin_lock_init(&file_private->table_lock);
1032}
1033
1034/**
1035 * drm_gem_release - release file-private GEM resources
1036 * @dev: drm_device which is being closed by userspace
1037 * @file_private: drm file-private structure to clean up
1038 *
1039 * Called at close time when the filp is going away.
1040 *
1041 * Releases any remaining references on objects by this filp.
1042 */
1043void
1044drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
1045{
1046	idr_for_each(&file_private->object_idr,
1047		     &drm_gem_object_release_handle, file_private);
1048	idr_destroy(&file_private->object_idr);
1049}
1050
1051/**
1052 * drm_gem_object_release - release GEM buffer object resources
1053 * @obj: GEM buffer object
1054 *
1055 * This releases any structures and resources used by @obj and is the inverse of
1056 * drm_gem_object_init().
1057 */
1058void
1059drm_gem_object_release(struct drm_gem_object *obj)
1060{
1061	if (obj->filp)
1062		fput(obj->filp);
1063
1064	drm_gem_private_object_fini(obj);
1065
1066	drm_gem_free_mmap_offset(obj);
1067	drm_gem_lru_remove(obj);
1068}
1069EXPORT_SYMBOL(drm_gem_object_release);
1070
1071/**
1072 * drm_gem_object_free - free a GEM object
1073 * @kref: kref of the object to free
1074 *
1075 * Called after the last reference to the object has been lost.
1076 *
1077 * Frees the object
1078 */
1079void
1080drm_gem_object_free(struct kref *kref)
1081{
1082	struct drm_gem_object *obj =
1083		container_of(kref, struct drm_gem_object, refcount);
1084
1085	if (WARN_ON(!obj->funcs->free))
1086		return;
1087
1088	obj->funcs->free(obj);
1089}
1090EXPORT_SYMBOL(drm_gem_object_free);
1091
1092/**
1093 * drm_gem_vm_open - vma->ops->open implementation for GEM
1094 * @vma: VM area structure
1095 *
1096 * This function implements the #vm_operations_struct open() callback for GEM
1097 * drivers. This must be used together with drm_gem_vm_close().
1098 */
1099void drm_gem_vm_open(struct vm_area_struct *vma)
1100{
1101	struct drm_gem_object *obj = vma->vm_private_data;
1102
1103	drm_gem_object_get(obj);
1104}
1105EXPORT_SYMBOL(drm_gem_vm_open);
1106
1107/**
1108 * drm_gem_vm_close - vma->ops->close implementation for GEM
1109 * @vma: VM area structure
1110 *
1111 * This function implements the #vm_operations_struct close() callback for GEM
1112 * drivers. This must be used together with drm_gem_vm_open().
1113 */
1114void drm_gem_vm_close(struct vm_area_struct *vma)
1115{
1116	struct drm_gem_object *obj = vma->vm_private_data;
1117
1118	drm_gem_object_put(obj);
1119}
1120EXPORT_SYMBOL(drm_gem_vm_close);
1121
1122/**
1123 * drm_gem_mmap_obj - memory map a GEM object
1124 * @obj: the GEM object to map
1125 * @obj_size: the object size to be mapped, in bytes
1126 * @vma: VMA for the area to be mapped
1127 *
1128 * Set up the VMA to prepare mapping of the GEM object using the GEM object's
1129 * vm_ops. Depending on their requirements, GEM objects can either
1130 * provide a fault handler in their vm_ops (in which case any accesses to
1131 * the object will be trapped, to perform migration, GTT binding, surface
1132 * register allocation, or performance monitoring), or mmap the buffer memory
1133 * synchronously after calling drm_gem_mmap_obj.
1134 *
1135 * This function is mainly intended to implement the DMABUF mmap operation, when
1136 * the GEM object is not looked up based on its fake offset. To implement the
1137 * DRM mmap operation, drivers should use the drm_gem_mmap() function.
1138 *
1139 * drm_gem_mmap_obj() assumes the user is granted access to the buffer while
1140 * drm_gem_mmap() prevents unprivileged users from mapping random objects. So
1141 * callers must verify access restrictions before calling this helper.
1142 *
1143 * Return 0 or success or -EINVAL if the object size is smaller than the VMA
1144 * size, or if no vm_ops are provided.
1145 */
1146int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
1147		     struct vm_area_struct *vma)
1148{
1149	int ret;
1150
1151	/* Check for valid size. */
1152	if (obj_size < vma->vm_end - vma->vm_start)
1153		return -EINVAL;
1154
1155	/* Take a ref for this mapping of the object, so that the fault
1156	 * handler can dereference the mmap offset's pointer to the object.
1157	 * This reference is cleaned up by the corresponding vm_close
1158	 * (which should happen whether the vma was created by this call, or
1159	 * by a vm_open due to mremap or partial unmap or whatever).
1160	 */
1161	drm_gem_object_get(obj);
1162
1163	vma->vm_private_data = obj;
1164	vma->vm_ops = obj->funcs->vm_ops;
1165
1166	if (obj->funcs->mmap) {
1167		ret = obj->funcs->mmap(obj, vma);
1168		if (ret)
1169			goto err_drm_gem_object_put;
1170		WARN_ON(!(vma->vm_flags & VM_DONTEXPAND));
1171	} else {
1172		if (!vma->vm_ops) {
1173			ret = -EINVAL;
1174			goto err_drm_gem_object_put;
1175		}
1176
1177		vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1178		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1179		vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1180	}
1181
1182	return 0;
1183
1184err_drm_gem_object_put:
1185	drm_gem_object_put(obj);
1186	return ret;
1187}
1188EXPORT_SYMBOL(drm_gem_mmap_obj);
1189
1190/**
1191 * drm_gem_mmap - memory map routine for GEM objects
1192 * @filp: DRM file pointer
1193 * @vma: VMA for the area to be mapped
1194 *
1195 * If a driver supports GEM object mapping, mmap calls on the DRM file
1196 * descriptor will end up here.
1197 *
1198 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
1199 * contain the fake offset we created when the GTT map ioctl was called on
1200 * the object) and map it with a call to drm_gem_mmap_obj().
1201 *
1202 * If the caller is not granted access to the buffer object, the mmap will fail
1203 * with EACCES. Please see the vma manager for more information.
1204 */
1205int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1206{
1207	struct drm_file *priv = filp->private_data;
1208	struct drm_device *dev = priv->minor->dev;
1209	struct drm_gem_object *obj = NULL;
1210	struct drm_vma_offset_node *node;
1211	int ret;
1212
1213	if (drm_dev_is_unplugged(dev))
1214		return -ENODEV;
1215
1216	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1217	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
1218						  vma->vm_pgoff,
1219						  vma_pages(vma));
1220	if (likely(node)) {
1221		obj = container_of(node, struct drm_gem_object, vma_node);
1222		/*
1223		 * When the object is being freed, after it hits 0-refcnt it
1224		 * proceeds to tear down the object. In the process it will
1225		 * attempt to remove the VMA offset and so acquire this
1226		 * mgr->vm_lock.  Therefore if we find an object with a 0-refcnt
1227		 * that matches our range, we know it is in the process of being
1228		 * destroyed and will be freed as soon as we release the lock -
1229		 * so we have to check for the 0-refcnted object and treat it as
1230		 * invalid.
1231		 */
1232		if (!kref_get_unless_zero(&obj->refcount))
1233			obj = NULL;
1234	}
1235	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1236
1237	if (!obj)
1238		return -EINVAL;
1239
1240	if (!drm_vma_node_is_allowed(node, priv)) {
1241		drm_gem_object_put(obj);
1242		return -EACCES;
1243	}
1244
1245	ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
1246			       vma);
1247
1248	drm_gem_object_put(obj);
1249
1250	return ret;
1251}
1252EXPORT_SYMBOL(drm_gem_mmap);
1253
1254void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
1255			const struct drm_gem_object *obj)
1256{
1257	drm_printf_indent(p, indent, "name=%d\n", obj->name);
1258	drm_printf_indent(p, indent, "refcount=%u\n",
1259			  kref_read(&obj->refcount));
1260	drm_printf_indent(p, indent, "start=%08lx\n",
1261			  drm_vma_node_start(&obj->vma_node));
1262	drm_printf_indent(p, indent, "size=%zu\n", obj->size);
1263	drm_printf_indent(p, indent, "imported=%s\n",
1264			  str_yes_no(drm_gem_is_imported(obj)));
1265
1266	if (obj->funcs->print_info)
1267		obj->funcs->print_info(p, indent, obj);
1268}
1269
1270int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map)
1271{
1272	int ret;
1273
1274	dma_resv_assert_held(obj->resv);
1275
1276	if (!obj->funcs->vmap)
1277		return -EOPNOTSUPP;
1278
1279	ret = obj->funcs->vmap(obj, map);
1280	if (ret)
1281		return ret;
1282	else if (iosys_map_is_null(map))
1283		return -ENOMEM;
1284
1285	return 0;
1286}
1287EXPORT_SYMBOL(drm_gem_vmap_locked);
1288
1289void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map)
1290{
1291	dma_resv_assert_held(obj->resv);
1292
1293	if (iosys_map_is_null(map))
1294		return;
1295
1296	if (obj->funcs->vunmap)
1297		obj->funcs->vunmap(obj, map);
1298
1299	/* Always set the mapping to NULL. Callers may rely on this. */
1300	iosys_map_clear(map);
1301}
1302EXPORT_SYMBOL(drm_gem_vunmap_locked);
1303
1304void drm_gem_lock(struct drm_gem_object *obj)
1305{
1306	dma_resv_lock(obj->resv, NULL);
1307}
1308EXPORT_SYMBOL(drm_gem_lock);
1309
1310void drm_gem_unlock(struct drm_gem_object *obj)
1311{
1312	dma_resv_unlock(obj->resv);
1313}
1314EXPORT_SYMBOL(drm_gem_unlock);
1315
1316int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1317{
1318	int ret;
1319
1320	dma_resv_lock(obj->resv, NULL);
1321	ret = drm_gem_vmap_locked(obj, map);
1322	dma_resv_unlock(obj->resv);
1323
1324	return ret;
1325}
1326EXPORT_SYMBOL(drm_gem_vmap);
1327
1328void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1329{
1330	dma_resv_lock(obj->resv, NULL);
1331	drm_gem_vunmap_locked(obj, map);
1332	dma_resv_unlock(obj->resv);
1333}
1334EXPORT_SYMBOL(drm_gem_vunmap);
1335
1336/**
1337 * drm_gem_lock_reservations - Sets up the ww context and acquires
1338 * the lock on an array of GEM objects.
1339 *
1340 * Once you've locked your reservations, you'll want to set up space
1341 * for your shared fences (if applicable), submit your job, then
1342 * drm_gem_unlock_reservations().
1343 *
1344 * @objs: drm_gem_objects to lock
1345 * @count: Number of objects in @objs
1346 * @acquire_ctx: struct ww_acquire_ctx that will be initialized as
1347 * part of tracking this set of locked reservations.
1348 */
1349int
1350drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
1351			  struct ww_acquire_ctx *acquire_ctx)
1352{
1353	int contended = -1;
1354	int i, ret;
1355
1356	ww_acquire_init(acquire_ctx, &reservation_ww_class);
1357
1358retry:
1359	if (contended != -1) {
1360		struct drm_gem_object *obj = objs[contended];
1361
1362		ret = dma_resv_lock_slow_interruptible(obj->resv,
1363								 acquire_ctx);
1364		if (ret) {
1365			ww_acquire_fini(acquire_ctx);
1366			return ret;
1367		}
1368	}
1369
1370	for (i = 0; i < count; i++) {
1371		if (i == contended)
1372			continue;
1373
1374		ret = dma_resv_lock_interruptible(objs[i]->resv,
1375							    acquire_ctx);
1376		if (ret) {
1377			int j;
1378
1379			for (j = 0; j < i; j++)
1380				dma_resv_unlock(objs[j]->resv);
1381
1382			if (contended != -1 && contended >= i)
1383				dma_resv_unlock(objs[contended]->resv);
1384
1385			if (ret == -EDEADLK) {
1386				contended = i;
1387				goto retry;
1388			}
1389
1390			ww_acquire_fini(acquire_ctx);
1391			return ret;
1392		}
1393	}
1394
1395	ww_acquire_done(acquire_ctx);
1396
1397	return 0;
1398}
1399EXPORT_SYMBOL(drm_gem_lock_reservations);
1400
1401void
1402drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
1403			    struct ww_acquire_ctx *acquire_ctx)
1404{
1405	int i;
1406
1407	for (i = 0; i < count; i++)
1408		dma_resv_unlock(objs[i]->resv);
1409
1410	ww_acquire_fini(acquire_ctx);
1411}
1412EXPORT_SYMBOL(drm_gem_unlock_reservations);
1413
1414/**
1415 * drm_gem_lru_init - initialize a LRU
1416 *
1417 * @lru: The LRU to initialize
1418 * @lock: The lock protecting the LRU
1419 */
1420void
1421drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock)
1422{
1423	lru->lock = lock;
1424	lru->count = 0;
1425	INIT_LIST_HEAD(&lru->list);
1426}
1427EXPORT_SYMBOL(drm_gem_lru_init);
1428
1429static void
1430drm_gem_lru_remove_locked(struct drm_gem_object *obj)
1431{
1432	obj->lru->count -= obj->size >> PAGE_SHIFT;
1433	WARN_ON(obj->lru->count < 0);
1434	list_del(&obj->lru_node);
1435	obj->lru = NULL;
1436}
1437
1438/**
1439 * drm_gem_lru_remove - remove object from whatever LRU it is in
1440 *
1441 * If the object is currently in any LRU, remove it.
1442 *
1443 * @obj: The GEM object to remove from current LRU
1444 */
1445void
1446drm_gem_lru_remove(struct drm_gem_object *obj)
1447{
1448	struct drm_gem_lru *lru = obj->lru;
1449
1450	if (!lru)
1451		return;
1452
1453	mutex_lock(lru->lock);
1454	drm_gem_lru_remove_locked(obj);
1455	mutex_unlock(lru->lock);
1456}
1457EXPORT_SYMBOL(drm_gem_lru_remove);
1458
1459/**
1460 * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU
1461 *
1462 * Like &drm_gem_lru_move_tail but lru lock must be held
1463 *
1464 * @lru: The LRU to move the object into.
1465 * @obj: The GEM object to move into this LRU
1466 */
1467void
1468drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1469{
1470	lockdep_assert_held_once(lru->lock);
1471
1472	if (obj->lru)
1473		drm_gem_lru_remove_locked(obj);
1474
1475	lru->count += obj->size >> PAGE_SHIFT;
1476	list_add_tail(&obj->lru_node, &lru->list);
1477	obj->lru = lru;
1478}
1479EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
1480
1481/**
1482 * drm_gem_lru_move_tail - move the object to the tail of the LRU
1483 *
1484 * If the object is already in this LRU it will be moved to the
1485 * tail.  Otherwise it will be removed from whichever other LRU
1486 * it is in (if any) and moved into this LRU.
1487 *
1488 * @lru: The LRU to move the object into.
1489 * @obj: The GEM object to move into this LRU
1490 */
1491void
1492drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1493{
1494	mutex_lock(lru->lock);
1495	drm_gem_lru_move_tail_locked(lru, obj);
1496	mutex_unlock(lru->lock);
1497}
1498EXPORT_SYMBOL(drm_gem_lru_move_tail);
1499
1500/**
1501 * drm_gem_lru_scan - helper to implement shrinker.scan_objects
1502 *
1503 * If the shrink callback succeeds, it is expected that the driver
1504 * move the object out of this LRU.
1505 *
1506 * If the LRU possibly contain active buffers, it is the responsibility
1507 * of the shrink callback to check for this (ie. dma_resv_test_signaled())
1508 * or if necessary block until the buffer becomes idle.
1509 *
1510 * @lru: The LRU to scan
1511 * @nr_to_scan: The number of pages to try to reclaim
1512 * @remaining: The number of pages left to reclaim, should be initialized by caller
1513 * @shrink: Callback to try to shrink/reclaim the object.
1514 * @ticket: Optional ww_acquire_ctx context to use for locking
1515 */
1516unsigned long
1517drm_gem_lru_scan(struct drm_gem_lru *lru,
1518		 unsigned int nr_to_scan,
1519		 unsigned long *remaining,
1520		 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
1521		 struct ww_acquire_ctx *ticket)
1522{
1523	struct drm_gem_lru still_in_lru;
1524	struct drm_gem_object *obj;
1525	unsigned freed = 0;
1526
1527	drm_gem_lru_init(&still_in_lru, lru->lock);
1528
1529	mutex_lock(lru->lock);
1530
1531	while (freed < nr_to_scan) {
1532		obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
1533
1534		if (!obj)
1535			break;
1536
1537		drm_gem_lru_move_tail_locked(&still_in_lru, obj);
1538
1539		/*
1540		 * If it's in the process of being freed, gem_object->free()
1541		 * may be blocked on lock waiting to remove it.  So just
1542		 * skip it.
1543		 */
1544		if (!kref_get_unless_zero(&obj->refcount))
1545			continue;
1546
1547		/*
1548		 * Now that we own a reference, we can drop the lock for the
1549		 * rest of the loop body, to reduce contention with other
1550		 * code paths that need the LRU lock
1551		 */
1552		mutex_unlock(lru->lock);
1553
1554		if (ticket)
1555			ww_acquire_init(ticket, &reservation_ww_class);
1556
1557		/*
1558		 * Note that this still needs to be trylock, since we can
1559		 * hit shrinker in response to trying to get backing pages
1560		 * for this obj (ie. while it's lock is already held)
1561		 */
1562		if (!ww_mutex_trylock(&obj->resv->lock, ticket)) {
1563			*remaining += obj->size >> PAGE_SHIFT;
1564			goto tail;
1565		}
1566
1567		if (shrink(obj, ticket)) {
1568			freed += obj->size >> PAGE_SHIFT;
1569
1570			/*
1571			 * If we succeeded in releasing the object's backing
1572			 * pages, we expect the driver to have moved the object
1573			 * out of this LRU
1574			 */
1575			WARN_ON(obj->lru == &still_in_lru);
1576			WARN_ON(obj->lru == lru);
1577		}
1578
1579		dma_resv_unlock(obj->resv);
1580
1581		if (ticket)
1582			ww_acquire_fini(ticket);
1583
1584tail:
1585		drm_gem_object_put(obj);
1586		mutex_lock(lru->lock);
1587	}
1588
1589	/*
1590	 * Move objects we've skipped over out of the temporary still_in_lru
1591	 * back into this LRU
1592	 */
1593	list_for_each_entry (obj, &still_in_lru.list, lru_node)
1594		obj->lru = lru;
1595	list_splice_tail(&still_in_lru.list, &lru->list);
1596	lru->count += still_in_lru.count;
1597
1598	mutex_unlock(lru->lock);
1599
1600	return freed;
1601}
1602EXPORT_SYMBOL(drm_gem_lru_scan);
1603
1604/**
1605 * drm_gem_evict_locked - helper to evict backing pages for a GEM object
1606 * @obj: obj in question
1607 */
1608int drm_gem_evict_locked(struct drm_gem_object *obj)
1609{
1610	dma_resv_assert_held(obj->resv);
1611
1612	if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ))
1613		return -EBUSY;
1614
1615	if (obj->funcs->evict)
1616		return obj->funcs->evict(obj);
1617
1618	return 0;
1619}
1620EXPORT_SYMBOL(drm_gem_evict_locked);