amdgpu_gem.c

   1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28#include <linux/ktime.h>
  29#include <linux/module.h>
  30#include <linux/pagemap.h>
  31#include <linux/pci.h>
  32#include <linux/dma-buf.h>
  33
  34#include <drm/amdgpu_drm.h>
  35#include <drm/drm_drv.h>
  36#include <drm/drm_exec.h>
  37#include <drm/drm_gem_ttm_helper.h>
  38#include <drm/ttm/ttm_tt.h>
  39#include <drm/drm_syncobj.h>
  40
  41#include "amdgpu.h"
  42#include "amdgpu_display.h"
  43#include "amdgpu_dma_buf.h"
  44#include "amdgpu_hmm.h"
  45#include "amdgpu_xgmi.h"
  46#include "amdgpu_vm.h"
  47
  48static int
  49amdgpu_gem_add_input_fence(struct drm_file *filp,
  50			   uint64_t syncobj_handles_array,
  51			   uint32_t num_syncobj_handles)
  52{
  53	struct dma_fence *fence;
  54	uint32_t *syncobj_handles;
  55	int ret, i;
  56
  57	if (!num_syncobj_handles)
  58		return 0;
  59
  60	syncobj_handles = memdup_user(u64_to_user_ptr(syncobj_handles_array),
  61				      size_mul(sizeof(uint32_t), num_syncobj_handles));
  62	if (IS_ERR(syncobj_handles))
  63		return PTR_ERR(syncobj_handles);
  64
  65	for (i = 0; i < num_syncobj_handles; i++) {
  66
  67		if (!syncobj_handles[i]) {
  68			ret = -EINVAL;
  69			goto free_memdup;
  70		}
  71
  72		ret = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 0, &fence);
  73		if (ret)
  74			goto free_memdup;
  75
  76		dma_fence_wait(fence, false);
  77
  78		/* TODO: optimize async handling */
  79		dma_fence_put(fence);
  80	}
  81
  82free_memdup:
  83	kfree(syncobj_handles);
  84	return ret;
  85}
  86
  87static int
  88amdgpu_gem_update_timeline_node(struct drm_file *filp,
  89				uint32_t syncobj_handle,
  90				uint64_t point,
  91				struct drm_syncobj **syncobj,
  92				struct dma_fence_chain **chain)
  93{
  94	if (!syncobj_handle)
  95		return 0;
  96
  97	/* Find the sync object */
  98	*syncobj = drm_syncobj_find(filp, syncobj_handle);
  99	if (!*syncobj)
 100		return -ENOENT;
 101
 102	if (!point)
 103		return 0;
 104
 105	/* Allocate the chain node */
 106	*chain = dma_fence_chain_alloc();
 107	if (!*chain) {
 108		drm_syncobj_put(*syncobj);
 109		return -ENOMEM;
 110	}
 111
 112	return 0;
 113}
 114
 115static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
 116{
 117	struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
 118	struct drm_device *ddev = bo->base.dev;
 119	vm_fault_t ret;
 120	int idx;
 121
 122	ret = ttm_bo_vm_reserve(bo, vmf);
 123	if (ret)
 124		return ret;
 125
 126	if (drm_dev_enter(ddev, &idx)) {
 127		ret = amdgpu_bo_fault_reserve_notify(bo);
 128		if (ret) {
 129			drm_dev_exit(idx);
 130			goto unlock;
 131		}
 132
 133		ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
 134					       TTM_BO_VM_NUM_PREFAULT);
 135
 136		drm_dev_exit(idx);
 137	} else {
 138		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
 139	}
 140	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
 141		return ret;
 142
 143unlock:
 144	dma_resv_unlock(bo->base.resv);
 145	return ret;
 146}
 147
 148static const struct vm_operations_struct amdgpu_gem_vm_ops = {
 149	.fault = amdgpu_gem_fault,
 150	.open = ttm_bo_vm_open,
 151	.close = ttm_bo_vm_close,
 152	.access = ttm_bo_vm_access
 153};
 154
 155static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
 156{
 157	struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
 158
 159	amdgpu_hmm_unregister(aobj);
 160	ttm_bo_fini(&aobj->tbo);
 161}
 162
 163int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 164			     int alignment, u32 initial_domain,
 165			     u64 flags, enum ttm_bo_type type,
 166			     struct dma_resv *resv,
 167			     struct drm_gem_object **obj, int8_t xcp_id_plus1)
 168{
 169	struct amdgpu_bo *bo;
 170	struct amdgpu_bo_user *ubo;
 171	struct amdgpu_bo_param bp;
 172	int r;
 173
 174	memset(&bp, 0, sizeof(bp));
 175	*obj = NULL;
 176	flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
 177
 178	bp.size = size;
 179	bp.byte_align = alignment;
 180	bp.type = type;
 181	bp.resv = resv;
 182	bp.preferred_domain = initial_domain;
 183	bp.flags = flags;
 184	bp.domain = initial_domain;
 185	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
 186	bp.xcp_id_plus1 = xcp_id_plus1;
 187
 188	r = amdgpu_bo_create_user(adev, &bp, &ubo);
 189	if (r)
 190		return r;
 191
 192	bo = &ubo->bo;
 193	*obj = &bo->tbo.base;
 194
 195	return 0;
 196}
 197
 198void amdgpu_gem_force_release(struct amdgpu_device *adev)
 199{
 200	struct drm_device *ddev = adev_to_drm(adev);
 201	struct drm_file *file;
 202
 203	mutex_lock(&ddev->filelist_mutex);
 204
 205	list_for_each_entry(file, &ddev->filelist, lhead) {
 206		struct drm_gem_object *gobj;
 207		int handle;
 208
 209		WARN_ONCE(1, "Still active user space clients!\n");
 210		spin_lock(&file->table_lock);
 211		idr_for_each_entry(&file->object_idr, gobj, handle) {
 212			WARN_ONCE(1, "And also active allocations!\n");
 213			drm_gem_object_put(gobj);
 214		}
 215		idr_destroy(&file->object_idr);
 216		spin_unlock(&file->table_lock);
 217	}
 218
 219	mutex_unlock(&ddev->filelist_mutex);
 220}
 221
 222/*
 223 * Call from drm_gem_handle_create which appear in both new and open ioctl
 224 * case.
 225 */
 226static int amdgpu_gem_object_open(struct drm_gem_object *obj,
 227				  struct drm_file *file_priv)
 228{
 229	struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
 230	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
 231	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 232	struct amdgpu_vm *vm = &fpriv->vm;
 233	struct amdgpu_bo_va *bo_va;
 234	struct mm_struct *mm;
 235	int r;
 236
 237	mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
 238	if (mm && mm != current->mm)
 239		return -EPERM;
 240
 241	if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
 242	    !amdgpu_vm_is_bo_always_valid(vm, abo))
 243		return -EPERM;
 244
 245	r = amdgpu_bo_reserve(abo, false);
 246	if (r)
 247		return r;
 248
 249	amdgpu_vm_bo_update_shared(abo);
 250	bo_va = amdgpu_vm_bo_find(vm, abo);
 251	if (!bo_va)
 252		bo_va = amdgpu_vm_bo_add(adev, vm, abo);
 253	else
 254		++bo_va->ref_count;
 255
 256	/* attach gfx eviction fence */
 257	r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo);
 258	if (r) {
 259		DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n");
 260		amdgpu_bo_unreserve(abo);
 261		return r;
 262	}
 263
 264	amdgpu_bo_unreserve(abo);
 265
 266	/* Validate and add eviction fence to DMABuf imports with dynamic
 267	 * attachment in compute VMs. Re-validation will be done by
 268	 * amdgpu_vm_validate. Fences are on the reservation shared with the
 269	 * export, which is currently required to be validated and fenced
 270	 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.
 271	 *
 272	 * Nested locking below for the case that a GEM object is opened in
 273	 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,
 274	 * but not for export, this is a different lock class that cannot lead to
 275	 * circular lock dependencies.
 276	 */
 277	if (!vm->is_compute_context || !vm->process_info)
 278		return 0;
 279	if (!drm_gem_is_imported(obj) ||
 280	    !dma_buf_is_dynamic(obj->import_attach->dmabuf))
 281		return 0;
 282	mutex_lock_nested(&vm->process_info->lock, 1);
 283	if (!WARN_ON(!vm->process_info->eviction_fence)) {
 284		r = amdgpu_amdkfd_bo_validate_and_fence(abo, AMDGPU_GEM_DOMAIN_GTT,
 285							&vm->process_info->eviction_fence->base);
 286		if (r) {
 287			struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);
 288
 289			dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);
 290			if (ti) {
 291				dev_warn(adev->dev, "pid %d\n", ti->task.pid);
 292				amdgpu_vm_put_task_info(ti);
 293			}
 294		}
 295	}
 296	mutex_unlock(&vm->process_info->lock);
 297
 298	return r;
 299}
 300
 301static void amdgpu_gem_object_close(struct drm_gem_object *obj,
 302				    struct drm_file *file_priv)
 303{
 304	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
 305	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 306	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 307	struct amdgpu_vm *vm = &fpriv->vm;
 308
 309	struct dma_fence *fence = NULL;
 310	struct amdgpu_bo_va *bo_va;
 311	struct drm_exec exec;
 312	long r;
 313
 314	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
 315	drm_exec_until_all_locked(&exec) {
 316		r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);
 317		drm_exec_retry_on_contention(&exec);
 318		if (unlikely(r))
 319			goto out_unlock;
 320
 321		r = amdgpu_vm_lock_pd(vm, &exec, 0);
 322		drm_exec_retry_on_contention(&exec);
 323		if (unlikely(r))
 324			goto out_unlock;
 325	}
 326
 327	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
 328		amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo);
 329
 330	bo_va = amdgpu_vm_bo_find(vm, bo);
 331	if (!bo_va || --bo_va->ref_count)
 332		goto out_unlock;
 333
 334	amdgpu_vm_bo_del(adev, bo_va);
 335	amdgpu_vm_bo_update_shared(bo);
 336	if (!amdgpu_vm_ready(vm))
 337		goto out_unlock;
 338
 339	r = amdgpu_vm_clear_freed(adev, vm, &fence);
 340	if (unlikely(r < 0))
 341		dev_err(adev->dev, "failed to clear page "
 342			"tables on GEM object close (%ld)\n", r);
 343	if (r || !fence)
 344		goto out_unlock;
 345
 346	amdgpu_bo_fence(bo, fence, true);
 347	dma_fence_put(fence);
 348
 349out_unlock:
 350	if (r)
 351		dev_err(adev->dev, "leaking bo va (%ld)\n", r);
 352	drm_exec_fini(&exec);
 353}
 354
 355static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 356{
 357	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
 358
 359	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
 360		return -EPERM;
 361	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
 362		return -EPERM;
 363
 364	/* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
 365	 * for debugger access to invisible VRAM. Should have used MAP_SHARED
 366	 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
 367	 * becoming writable and makes is_cow_mapping(vm_flags) false.
 368	 */
 369	if (is_cow_mapping(vma->vm_flags) &&
 370	    !(vma->vm_flags & VM_ACCESS_FLAGS))
 371		vm_flags_clear(vma, VM_MAYWRITE);
 372
 373	return drm_gem_ttm_mmap(obj, vma);
 374}
 375
 376const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
 377	.free = amdgpu_gem_object_free,
 378	.open = amdgpu_gem_object_open,
 379	.close = amdgpu_gem_object_close,
 380	.export = amdgpu_gem_prime_export,
 381	.vmap = drm_gem_ttm_vmap,
 382	.vunmap = drm_gem_ttm_vunmap,
 383	.mmap = amdgpu_gem_object_mmap,
 384	.vm_ops = &amdgpu_gem_vm_ops,
 385};
 386
 387/*
 388 * GEM ioctls.
 389 */
 390int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 391			    struct drm_file *filp)
 392{
 393	struct amdgpu_device *adev = drm_to_adev(dev);
 394	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 395	struct amdgpu_vm *vm = &fpriv->vm;
 396	union drm_amdgpu_gem_create *args = data;
 397	uint64_t flags = args->in.domain_flags;
 398	uint64_t size = args->in.bo_size;
 399	struct dma_resv *resv = NULL;
 400	struct drm_gem_object *gobj;
 401	uint32_t handle, initial_domain;
 402	int r;
 403
 404	/* reject invalid gem flags */
 405	if (flags & ~AMDGPU_GEM_CREATE_SETTABLE_MASK)
 406		return -EINVAL;
 407
 408	/* reject invalid gem domains */
 409	if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
 410		return -EINVAL;
 411
 412	if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
 413		DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
 414		return -EINVAL;
 415	}
 416
 417	/* always clear VRAM */
 418	flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
 419
 420	/* create a gem object to contain this object in */
 421	if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
 422	    AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
 423		if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
 424			/* if gds bo is created from user space, it must be
 425			 * passed to bo list
 426			 */
 427			DRM_ERROR("GDS bo cannot be per-vm-bo\n");
 428			return -EINVAL;
 429		}
 430		flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
 431	}
 432
 433	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
 434		r = amdgpu_bo_reserve(vm->root.bo, false);
 435		if (r)
 436			return r;
 437
 438		resv = vm->root.bo->tbo.base.resv;
 439	}
 440
 441	initial_domain = (u32)(0xffffffff & args->in.domains);
 442retry:
 443	r = amdgpu_gem_object_create(adev, size, args->in.alignment,
 444				     initial_domain,
 445				     flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
 446	if (r && r != -ERESTARTSYS) {
 447		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
 448			flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 449			goto retry;
 450		}
 451
 452		if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
 453			initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
 454			goto retry;
 455		}
 456		DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
 457				size, initial_domain, args->in.alignment, r);
 458	}
 459
 460	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
 461		if (!r) {
 462			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
 463
 464			abo->parent = amdgpu_bo_ref(vm->root.bo);
 465		}
 466		amdgpu_bo_unreserve(vm->root.bo);
 467	}
 468	if (r)
 469		return r;
 470
 471	r = drm_gem_handle_create(filp, gobj, &handle);
 472	/* drop reference from allocate - handle holds it now */
 473	drm_gem_object_put(gobj);
 474	if (r)
 475		return r;
 476
 477	memset(args, 0, sizeof(*args));
 478	args->out.handle = handle;
 479	return 0;
 480}
 481
 482int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 483			     struct drm_file *filp)
 484{
 485	struct ttm_operation_ctx ctx = { true, false };
 486	struct amdgpu_device *adev = drm_to_adev(dev);
 487	struct drm_amdgpu_gem_userptr *args = data;
 488	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 489	struct drm_gem_object *gobj;
 490	struct amdgpu_hmm_range *range;
 491	struct amdgpu_bo *bo;
 492	uint32_t handle;
 493	int r;
 494
 495	args->addr = untagged_addr(args->addr);
 496
 497	if (offset_in_page(args->addr | args->size))
 498		return -EINVAL;
 499
 500	/* reject unknown flag values */
 501	if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
 502	    AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
 503	    AMDGPU_GEM_USERPTR_REGISTER))
 504		return -EINVAL;
 505
 506	if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
 507	     !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
 508
 509		/* if we want to write to it we must install a MMU notifier */
 510		return -EACCES;
 511	}
 512
 513	/* create a gem object to contain this object in */
 514	r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
 515				     0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
 516	if (r)
 517		return r;
 518
 519	bo = gem_to_amdgpu_bo(gobj);
 520	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
 521	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
 522	r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
 523	if (r)
 524		goto release_object;
 525
 526	r = amdgpu_hmm_register(bo, args->addr);
 527	if (r)
 528		goto release_object;
 529
 530	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
 531		range = amdgpu_hmm_range_alloc(NULL);
 532		if (unlikely(!range))
 533			return -ENOMEM;
 534		r = amdgpu_ttm_tt_get_user_pages(bo, range);
 535		if (r) {
 536			amdgpu_hmm_range_free(range);
 537			goto release_object;
 538		}
 539		r = amdgpu_bo_reserve(bo, true);
 540		if (r)
 541			goto user_pages_done;
 542
 543		amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
 544
 545		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
 546		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 547		amdgpu_bo_unreserve(bo);
 548		if (r)
 549			goto user_pages_done;
 550	}
 551
 552	r = drm_gem_handle_create(filp, gobj, &handle);
 553	if (r)
 554		goto user_pages_done;
 555
 556	args->handle = handle;
 557
 558user_pages_done:
 559	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
 560		amdgpu_hmm_range_free(range);
 561release_object:
 562	drm_gem_object_put(gobj);
 563
 564	return r;
 565}
 566
 567int amdgpu_mode_dumb_mmap(struct drm_file *filp,
 568			  struct drm_device *dev,
 569			  uint32_t handle, uint64_t *offset_p)
 570{
 571	struct drm_gem_object *gobj;
 572	struct amdgpu_bo *robj;
 573
 574	gobj = drm_gem_object_lookup(filp, handle);
 575	if (!gobj)
 576		return -ENOENT;
 577
 578	robj = gem_to_amdgpu_bo(gobj);
 579	if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
 580	    (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
 581		drm_gem_object_put(gobj);
 582		return -EPERM;
 583	}
 584	*offset_p = amdgpu_bo_mmap_offset(robj);
 585	drm_gem_object_put(gobj);
 586	return 0;
 587}
 588
 589int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
 590			  struct drm_file *filp)
 591{
 592	union drm_amdgpu_gem_mmap *args = data;
 593	uint32_t handle = args->in.handle;
 594
 595	memset(args, 0, sizeof(*args));
 596	return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
 597}
 598
 599/**
 600 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
 601 *
 602 * @timeout_ns: timeout in ns
 603 *
 604 * Calculate the timeout in jiffies from an absolute timeout in ns.
 605 */
 606unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
 607{
 608	unsigned long timeout_jiffies;
 609	ktime_t timeout;
 610
 611	/* clamp timeout if it's to large */
 612	if (((int64_t)timeout_ns) < 0)
 613		return MAX_SCHEDULE_TIMEOUT;
 614
 615	timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
 616	if (ktime_to_ns(timeout) < 0)
 617		return 0;
 618
 619	timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
 620	/*  clamp timeout to avoid unsigned-> signed overflow */
 621	if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT)
 622		return MAX_SCHEDULE_TIMEOUT - 1;
 623
 624	return timeout_jiffies;
 625}
 626
 627int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
 628			      struct drm_file *filp)
 629{
 630	union drm_amdgpu_gem_wait_idle *args = data;
 631	struct drm_gem_object *gobj;
 632	struct amdgpu_bo *robj;
 633	uint32_t handle = args->in.handle;
 634	unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
 635	int r = 0;
 636	long ret;
 637
 638	gobj = drm_gem_object_lookup(filp, handle);
 639	if (!gobj)
 640		return -ENOENT;
 641
 642	robj = gem_to_amdgpu_bo(gobj);
 643	ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
 644				    true, timeout);
 645
 646	/* ret == 0 means not signaled,
 647	 * ret > 0 means signaled
 648	 * ret < 0 means interrupted before timeout
 649	 */
 650	if (ret >= 0) {
 651		memset(args, 0, sizeof(*args));
 652		args->out.status = (ret == 0);
 653	} else
 654		r = ret;
 655
 656	drm_gem_object_put(gobj);
 657	return r;
 658}
 659
 660int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
 661				struct drm_file *filp)
 662{
 663	struct drm_amdgpu_gem_metadata *args = data;
 664	struct drm_gem_object *gobj;
 665	struct amdgpu_bo *robj;
 666	int r = -1;
 667
 668	DRM_DEBUG("%d\n", args->handle);
 669	gobj = drm_gem_object_lookup(filp, args->handle);
 670	if (gobj == NULL)
 671		return -ENOENT;
 672	robj = gem_to_amdgpu_bo(gobj);
 673
 674	r = amdgpu_bo_reserve(robj, false);
 675	if (unlikely(r != 0))
 676		goto out;
 677
 678	if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
 679		amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
 680		r = amdgpu_bo_get_metadata(robj, args->data.data,
 681					   sizeof(args->data.data),
 682					   &args->data.data_size_bytes,
 683					   &args->data.flags);
 684	} else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
 685		if (args->data.data_size_bytes > sizeof(args->data.data)) {
 686			r = -EINVAL;
 687			goto unreserve;
 688		}
 689		r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
 690		if (!r)
 691			r = amdgpu_bo_set_metadata(robj, args->data.data,
 692						   args->data.data_size_bytes,
 693						   args->data.flags);
 694	}
 695
 696unreserve:
 697	amdgpu_bo_unreserve(robj);
 698out:
 699	drm_gem_object_put(gobj);
 700	return r;
 701}
 702
 703/**
 704 * amdgpu_gem_va_update_vm -update the bo_va in its VM
 705 *
 706 * @adev: amdgpu_device pointer
 707 * @vm: vm to update
 708 * @bo_va: bo_va to update
 709 * @operation: map, unmap or clear
 710 *
 711 * Update the bo_va directly after setting its address. Errors are not
 712 * vital here, so they are not reported back to userspace.
 713 *
 714 * Returns resulting fence if freed BO(s) got cleared from the PT.
 715 * otherwise stub fence in case of error.
 716 */
 717static struct dma_fence *
 718amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 719			struct amdgpu_vm *vm,
 720			struct amdgpu_bo_va *bo_va,
 721			uint32_t operation)
 722{
 723	struct dma_fence *fence;
 724	int r = 0;
 725
 726	/* Always start from the VM's existing last update fence. */
 727	fence = dma_fence_get(vm->last_update);
 728
 729	if (!amdgpu_vm_ready(vm))
 730		return fence;
 731
 732	/*
 733	 * First clean up any freed mappings in the VM.
 734	 *
 735	 * amdgpu_vm_clear_freed() may replace @fence with a new fence if it
 736	 * schedules GPU work. If nothing needs clearing, @fence can remain as
 737	 * the original vm->last_update.
 738	 */
 739	r = amdgpu_vm_clear_freed(adev, vm, &fence);
 740	if (r)
 741		goto error;
 742
 743	/* For MAP/REPLACE we also need to update the BO mappings. */
 744	if (operation == AMDGPU_VA_OP_MAP ||
 745	    operation == AMDGPU_VA_OP_REPLACE) {
 746		r = amdgpu_vm_bo_update(adev, bo_va, false);
 747		if (r)
 748			goto error;
 749	}
 750
 751	/* Always update PDEs after we touched the mappings. */
 752	r = amdgpu_vm_update_pdes(adev, vm, false);
 753	if (r)
 754		goto error;
 755
 756	/*
 757	 * Decide which fence best represents the last update:
 758	 *
 759	 * MAP/REPLACE:
 760	 *   - For always-valid mappings, use vm->last_update.
 761	 *   - Otherwise, export bo_va->last_pt_update.
 762	 *
 763	 * UNMAP/CLEAR:
 764	 *   Keep the fence returned by amdgpu_vm_clear_freed(). If no work was
 765	 *   needed, it can remain as vm->last_pt_update.
 766	 *
 767	 * The VM and BO update fences are always initialized to a valid value.
 768	 * vm->last_update and bo_va->last_pt_update always start as valid fences.
 769	 * and are never expected to be NULL.
 770	 */
 771	switch (operation) {
 772	case AMDGPU_VA_OP_MAP:
 773	case AMDGPU_VA_OP_REPLACE:
 774		/*
 775		 * For MAP/REPLACE, return the page table update fence for the
 776		 * mapping we just modified. bo_va is expected to be valid here.
 777		 */
 778		dma_fence_put(fence);
 779
 780		if (amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo))
 781			fence = dma_fence_get(vm->last_update);
 782		else
 783			fence = dma_fence_get(bo_va->last_pt_update);
 784		break;
 785	case AMDGPU_VA_OP_UNMAP:
 786	case AMDGPU_VA_OP_CLEAR:
 787	default:
 788		/* keep @fence as returned by amdgpu_vm_clear_freed() */
 789		break;
 790	}
 791
 792error:
 793	if (r && r != -ERESTARTSYS)
 794		DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
 795
 796	return fence;
 797}
 798
 799int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 800			  struct drm_file *filp)
 801{
 802	const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
 803		AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
 804		AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
 805		AMDGPU_VM_PAGE_NOALLOC;
 806	const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
 807		AMDGPU_VM_PAGE_PRT;
 808
 809	struct drm_amdgpu_gem_va *args = data;
 810	struct drm_gem_object *gobj;
 811	struct amdgpu_device *adev = drm_to_adev(dev);
 812	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 813	struct amdgpu_bo *abo;
 814	struct amdgpu_bo_va *bo_va;
 815	struct drm_syncobj *timeline_syncobj = NULL;
 816	struct dma_fence_chain *timeline_chain = NULL;
 817	struct dma_fence *fence;
 818	struct drm_exec exec;
 819	uint64_t vm_size;
 820	int r = 0;
 821
 822	/* Validate virtual address range against reserved regions. */
 823	if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
 824		dev_dbg(dev->dev,
 825			"va_address 0x%llx is in reserved area 0x%llx\n",
 826			args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
 827		return -EINVAL;
 828	}
 829
 830	if (args->va_address >= AMDGPU_GMC_HOLE_START &&
 831	    args->va_address < AMDGPU_GMC_HOLE_END) {
 832		dev_dbg(dev->dev,
 833			"va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
 834			args->va_address, AMDGPU_GMC_HOLE_START,
 835			AMDGPU_GMC_HOLE_END);
 836		return -EINVAL;
 837	}
 838
 839	args->va_address &= AMDGPU_GMC_HOLE_MASK;
 840
 841	vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
 842	vm_size -= AMDGPU_VA_RESERVED_TOP;
 843	if (args->va_address + args->map_size > vm_size) {
 844		dev_dbg(dev->dev,
 845			"va_address 0x%llx is in top reserved area 0x%llx\n",
 846			args->va_address + args->map_size, vm_size);
 847		return -EINVAL;
 848	}
 849
 850	if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
 851		dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
 852			args->flags);
 853		return -EINVAL;
 854	}
 855
 856	/* Validate operation type. */
 857	switch (args->operation) {
 858	case AMDGPU_VA_OP_MAP:
 859	case AMDGPU_VA_OP_UNMAP:
 860	case AMDGPU_VA_OP_CLEAR:
 861	case AMDGPU_VA_OP_REPLACE:
 862		break;
 863	default:
 864		dev_dbg(dev->dev, "unsupported operation %d\n",
 865			args->operation);
 866		return -EINVAL;
 867	}
 868
 869	if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
 870	    !(args->flags & AMDGPU_VM_PAGE_PRT)) {
 871		gobj = drm_gem_object_lookup(filp, args->handle);
 872		if (gobj == NULL)
 873			return -ENOENT;
 874		abo = gem_to_amdgpu_bo(gobj);
 875	} else {
 876		gobj = NULL;
 877		abo = NULL;
 878	}
 879
 880	/* Add input syncobj fences (if any) for synchronization. */
 881	r = amdgpu_gem_add_input_fence(filp,
 882				       args->input_fence_syncobj_handles,
 883				       args->num_syncobj_handles);
 884	if (r)
 885		goto error_put_gobj;
 886
 887	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
 888		      DRM_EXEC_IGNORE_DUPLICATES, 0);
 889	drm_exec_until_all_locked(&exec) {
 890		if (gobj) {
 891			r = drm_exec_lock_obj(&exec, gobj);
 892			drm_exec_retry_on_contention(&exec);
 893			if (unlikely(r))
 894				goto error;
 895		}
 896
 897		r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
 898		drm_exec_retry_on_contention(&exec);
 899		if (unlikely(r))
 900			goto error;
 901	}
 902
 903	/* Resolve the BO-VA mapping for this VM/BO combination. */
 904	if (abo) {
 905		bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
 906		if (!bo_va) {
 907			r = -ENOENT;
 908			goto error;
 909		}
 910	} else if (args->operation != AMDGPU_VA_OP_CLEAR) {
 911		bo_va = fpriv->prt_va;
 912	} else {
 913		bo_va = NULL;
 914	}
 915
 916	/*
 917	 * Prepare the timeline syncobj node if the user requested a VM
 918	 * timeline update. This only allocates/looks up the syncobj and
 919	 * chain node; the actual fence is attached later.
 920	 */
 921	r = amdgpu_gem_update_timeline_node(filp,
 922					    args->vm_timeline_syncobj_out,
 923					    args->vm_timeline_point,
 924					    &timeline_syncobj,
 925					    &timeline_chain);
 926	if (r)
 927		goto error;
 928
 929	switch (args->operation) {
 930	case AMDGPU_VA_OP_MAP:
 931		r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
 932				     args->offset_in_bo, args->map_size,
 933				     args->flags);
 934		break;
 935	case AMDGPU_VA_OP_UNMAP:
 936		r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
 937		break;
 938
 939	case AMDGPU_VA_OP_CLEAR:
 940		r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
 941						args->va_address,
 942						args->map_size);
 943		break;
 944	case AMDGPU_VA_OP_REPLACE:
 945		r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
 946					     args->offset_in_bo, args->map_size,
 947					     args->flags);
 948		break;
 949	default:
 950		break;
 951	}
 952
 953	/*
 954	 * Once the VA operation is done, update the VM and obtain the fence
 955	 * that represents the last relevant update for this mapping. This
 956	 * fence can then be exported to the user-visible VM timeline.
 957	 */
 958	if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) {
 959		fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
 960						args->operation);
 961
 962		if (timeline_syncobj && fence) {
 963			if (!args->vm_timeline_point) {
 964				/* Replace the existing fence when no point is given. */
 965				drm_syncobj_replace_fence(timeline_syncobj,
 966							  fence);
 967			} else {
 968				/* Attach the last-update fence at a specific point. */
 969				drm_syncobj_add_point(timeline_syncobj,
 970						      timeline_chain,
 971						      fence,
 972						      args->vm_timeline_point);
 973			}
 974		}
 975		dma_fence_put(fence);
 976
 977	}
 978
 979error:
 980	drm_exec_fini(&exec);
 981error_put_gobj:
 982	drm_gem_object_put(gobj);
 983	return r;
 984}
 985
 986int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 987			struct drm_file *filp)
 988{
 989	struct drm_amdgpu_gem_op *args = data;
 990	struct drm_gem_object *gobj;
 991	struct amdgpu_vm_bo_base *base;
 992	struct amdgpu_bo *robj;
 993	struct drm_exec exec;
 994	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 995	int r;
 996
 997	if (args->padding)
 998		return -EINVAL;
 999
1000	gobj = drm_gem_object_lookup(filp, args->handle);
1001	if (!gobj)
1002		return -ENOENT;
1003
1004	robj = gem_to_amdgpu_bo(gobj);
1005
1006	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
1007			  DRM_EXEC_IGNORE_DUPLICATES, 0);
1008	drm_exec_until_all_locked(&exec) {
1009		r = drm_exec_lock_obj(&exec, gobj);
1010		drm_exec_retry_on_contention(&exec);
1011		if (r)
1012			goto out_exec;
1013
1014		if (args->op == AMDGPU_GEM_OP_GET_MAPPING_INFO) {
1015			r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
1016			drm_exec_retry_on_contention(&exec);
1017			if (r)
1018				goto out_exec;
1019		}
1020	}
1021
1022	switch (args->op) {
1023	case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
1024		struct drm_amdgpu_gem_create_in info;
1025		void __user *out = u64_to_user_ptr(args->value);
1026
1027		info.bo_size = robj->tbo.base.size;
1028		info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
1029		info.domains = robj->preferred_domains;
1030		info.domain_flags = robj->flags;
1031		drm_exec_fini(&exec);
1032		if (copy_to_user(out, &info, sizeof(info)))
1033			r = -EFAULT;
1034		break;
1035	}
1036	case AMDGPU_GEM_OP_SET_PLACEMENT:
1037		if (drm_gem_is_imported(&robj->tbo.base) &&
1038		    args->value & AMDGPU_GEM_DOMAIN_VRAM) {
1039			r = -EINVAL;
1040			goto out_exec;
1041		}
1042		if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
1043			r = -EPERM;
1044			goto out_exec;
1045		}
1046		for (base = robj->vm_bo; base; base = base->next)
1047			if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
1048				amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
1049				r = -EINVAL;
1050				goto out_exec;
1051			}
1052
1053
1054		robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
1055							AMDGPU_GEM_DOMAIN_GTT |
1056							AMDGPU_GEM_DOMAIN_CPU);
1057		robj->allowed_domains = robj->preferred_domains;
1058		if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
1059			robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
1060
1061		if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
1062			amdgpu_vm_bo_invalidate(robj, true);
1063		drm_exec_fini(&exec);
1064		break;
1065	case AMDGPU_GEM_OP_GET_MAPPING_INFO: {
1066		struct amdgpu_bo_va *bo_va = amdgpu_vm_bo_find(&fpriv->vm, robj);
1067		struct drm_amdgpu_gem_vm_entry *vm_entries;
1068		struct amdgpu_bo_va_mapping *mapping;
1069		int num_mappings = 0;
1070		/*
1071		 * num_entries is set as an input to the size of the user-allocated array of
1072		 * drm_amdgpu_gem_vm_entry stored at args->value.
1073		 * num_entries is sent back as output as the number of mappings the bo has.
1074		 * If that number is larger than the size of the array, the ioctl must
1075		 * be retried.
1076		 */
1077		vm_entries = kvcalloc(args->num_entries, sizeof(*vm_entries), GFP_KERNEL);
1078		if (!vm_entries)
1079			return -ENOMEM;
1080
1081		amdgpu_vm_bo_va_for_each_valid_mapping(bo_va, mapping) {
1082			if (num_mappings < args->num_entries) {
1083				vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE;
1084				vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE;
1085				vm_entries[num_mappings].offset = mapping->offset;
1086				vm_entries[num_mappings].flags = mapping->flags;
1087			}
1088			num_mappings += 1;
1089		}
1090
1091		amdgpu_vm_bo_va_for_each_invalid_mapping(bo_va, mapping) {
1092			if (num_mappings < args->num_entries) {
1093				vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE;
1094				vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE;
1095				vm_entries[num_mappings].offset = mapping->offset;
1096				vm_entries[num_mappings].flags = mapping->flags;
1097			}
1098			num_mappings += 1;
1099		}
1100
1101		drm_exec_fini(&exec);
1102
1103		if (num_mappings > 0 && num_mappings <= args->num_entries)
1104			if (copy_to_user(u64_to_user_ptr(args->value), vm_entries, num_mappings * sizeof(*vm_entries)))
1105				r = -EFAULT;
1106
1107		args->num_entries = num_mappings;
1108
1109		kvfree(vm_entries);
1110		break;
1111	}
1112	default:
1113		drm_exec_fini(&exec);
1114		r = -EINVAL;
1115	}
1116
1117	drm_gem_object_put(gobj);
1118	return r;
1119out_exec:
1120	drm_exec_fini(&exec);
1121	drm_gem_object_put(gobj);
1122	return r;
1123}
1124
1125/**
1126 * amdgpu_gem_list_handles_ioctl - get information about a process' buffer objects
1127 *
1128 * @dev: drm device pointer
1129 * @data: drm_amdgpu_gem_list_handles
1130 * @filp: drm file pointer
1131 *
1132 * num_entries is set as an input to the size of the entries array.
1133 * num_entries is sent back as output as the number of bos in the process.
1134 * If that number is larger than the size of the array, the ioctl must
1135 * be retried.
1136 *
1137 * Returns:
1138 * 0 for success, -errno for errors.
1139 */
1140int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data,
1141				  struct drm_file *filp)
1142{
1143	struct drm_amdgpu_gem_list_handles *args = data;
1144	struct drm_amdgpu_gem_list_handles_entry *bo_entries;
1145	struct drm_gem_object *gobj;
1146	int id, ret = 0;
1147	int bo_index = 0;
1148	int num_bos = 0;
1149
1150	spin_lock(&filp->table_lock);
1151	idr_for_each_entry(&filp->object_idr, gobj, id)
1152		num_bos += 1;
1153	spin_unlock(&filp->table_lock);
1154
1155	if (args->num_entries < num_bos) {
1156		args->num_entries = num_bos;
1157		return 0;
1158	}
1159
1160	if (num_bos == 0) {
1161		args->num_entries = 0;
1162		return 0;
1163	}
1164
1165	bo_entries = kvcalloc(num_bos, sizeof(*bo_entries), GFP_KERNEL);
1166	if (!bo_entries)
1167		return -ENOMEM;
1168
1169	spin_lock(&filp->table_lock);
1170	idr_for_each_entry(&filp->object_idr, gobj, id) {
1171		struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1172		struct drm_amdgpu_gem_list_handles_entry *bo_entry;
1173
1174		if (bo_index >= num_bos) {
1175			ret = -EAGAIN;
1176			break;
1177		}
1178
1179		bo_entry = &bo_entries[bo_index];
1180
1181		bo_entry->size = amdgpu_bo_size(bo);
1182		bo_entry->alloc_flags = bo->flags & AMDGPU_GEM_CREATE_SETTABLE_MASK;
1183		bo_entry->preferred_domains = bo->preferred_domains;
1184		bo_entry->gem_handle = id;
1185		bo_entry->alignment = bo->tbo.page_alignment;
1186
1187		if (bo->tbo.base.import_attach)
1188			bo_entry->flags |= AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT;
1189
1190		bo_index += 1;
1191	}
1192	spin_unlock(&filp->table_lock);
1193
1194	args->num_entries = bo_index;
1195
1196	if (!ret)
1197		if (copy_to_user(u64_to_user_ptr(args->entries), bo_entries, num_bos * sizeof(*bo_entries)))
1198			ret = -EFAULT;
1199
1200	kvfree(bo_entries);
1201
1202	return ret;
1203}
1204
1205static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
1206				  int width,
1207				  int cpp,
1208				  bool tiled)
1209{
1210	int aligned = width;
1211	int pitch_mask = 0;
1212
1213	switch (cpp) {
1214	case 1:
1215		pitch_mask = 255;
1216		break;
1217	case 2:
1218		pitch_mask = 127;
1219		break;
1220	case 3:
1221	case 4:
1222		pitch_mask = 63;
1223		break;
1224	}
1225
1226	aligned += pitch_mask;
1227	aligned &= ~pitch_mask;
1228	return aligned * cpp;
1229}
1230
1231int amdgpu_mode_dumb_create(struct drm_file *file_priv,
1232			    struct drm_device *dev,
1233			    struct drm_mode_create_dumb *args)
1234{
1235	struct amdgpu_device *adev = drm_to_adev(dev);
1236	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1237	struct drm_gem_object *gobj;
1238	uint32_t handle;
1239	u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1240		    AMDGPU_GEM_CREATE_CPU_GTT_USWC |
1241		    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1242	u32 domain;
1243	int r;
1244
1245	/*
1246	 * The buffer returned from this function should be cleared, but
1247	 * it can only be done if the ring is enabled or we'll fail to
1248	 * create the buffer.
1249	 */
1250	if (adev->mman.buffer_funcs_enabled)
1251		flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
1252
1253	args->pitch = amdgpu_gem_align_pitch(adev, args->width,
1254					     DIV_ROUND_UP(args->bpp, 8), 0);
1255	args->size = (u64)args->pitch * args->height;
1256	args->size = ALIGN(args->size, PAGE_SIZE);
1257	domain = amdgpu_bo_get_preferred_domain(adev,
1258				amdgpu_display_supported_domains(adev, flags));
1259	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
1260				     ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
1261	if (r)
1262		return -ENOMEM;
1263
1264	r = drm_gem_handle_create(file_priv, gobj, &handle);
1265	/* drop reference from allocate - handle holds it now */
1266	drm_gem_object_put(gobj);
1267	if (r)
1268		return r;
1269
1270	args->handle = handle;
1271	return 0;
1272}
1273
1274#if defined(CONFIG_DEBUG_FS)
1275static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
1276{
1277	struct amdgpu_device *adev = m->private;
1278	struct drm_device *dev = adev_to_drm(adev);
1279	struct drm_file *file;
1280	int r;
1281
1282	r = mutex_lock_interruptible(&dev->filelist_mutex);
1283	if (r)
1284		return r;
1285
1286	list_for_each_entry(file, &dev->filelist, lhead) {
1287		struct task_struct *task;
1288		struct drm_gem_object *gobj;
1289		struct pid *pid;
1290		int id;
1291
1292		/*
1293		 * Although we have a valid reference on file->pid, that does
1294		 * not guarantee that the task_struct who called get_pid() is
1295		 * still alive (e.g. get_pid(current) => fork() => exit()).
1296		 * Therefore, we need to protect this ->comm access using RCU.
1297		 */
1298		rcu_read_lock();
1299		pid = rcu_dereference(file->pid);
1300		task = pid_task(pid, PIDTYPE_TGID);
1301		seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),
1302			   task ? task->comm : "<unknown>");
1303		rcu_read_unlock();
1304
1305		spin_lock(&file->table_lock);
1306		idr_for_each_entry(&file->object_idr, gobj, id) {
1307			struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1308
1309			amdgpu_bo_print_info(id, bo, m);
1310		}
1311		spin_unlock(&file->table_lock);
1312	}
1313
1314	mutex_unlock(&dev->filelist_mutex);
1315	return 0;
1316}
1317
1318DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
1319
1320#endif
1321
1322void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
1323{
1324#if defined(CONFIG_DEBUG_FS)
1325	struct drm_minor *minor = adev_to_drm(adev)->primary;
1326	struct dentry *root = minor->debugfs_root;
1327
1328	debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1329			    &amdgpu_debugfs_gem_info_fops);
1330#endif
1331}