xe_device.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2021 Intel Corporation
   4 */
   5
   6#include "xe_device.h"
   7
   8#include <linux/aperture.h>
   9#include <linux/delay.h>
  10#include <linux/fault-inject.h>
  11#include <linux/iopoll.h>
  12#include <linux/units.h>
  13
  14#include <drm/drm_atomic_helper.h>
  15#include <drm/drm_client.h>
  16#include <drm/drm_gem_ttm_helper.h>
  17#include <drm/drm_ioctl.h>
  18#include <drm/drm_managed.h>
  19#include <drm/drm_print.h>
  20#include <uapi/drm/xe_drm.h>
  21
  22#include "display/xe_display.h"
  23#include "instructions/xe_gpu_commands.h"
  24#include "regs/xe_gt_regs.h"
  25#include "regs/xe_regs.h"
  26#include "xe_bo.h"
  27#include "xe_bo_evict.h"
  28#include "xe_debugfs.h"
  29#include "xe_devcoredump.h"
  30#include "xe_device_sysfs.h"
  31#include "xe_dma_buf.h"
  32#include "xe_drm_client.h"
  33#include "xe_drv.h"
  34#include "xe_exec.h"
  35#include "xe_exec_queue.h"
  36#include "xe_force_wake.h"
  37#include "xe_ggtt.h"
  38#include "xe_gsc_proxy.h"
  39#include "xe_gt.h"
  40#include "xe_gt_mcr.h"
  41#include "xe_gt_printk.h"
  42#include "xe_gt_sriov_vf.h"
  43#include "xe_guc.h"
  44#include "xe_guc_pc.h"
  45#include "xe_hw_engine_group.h"
  46#include "xe_hwmon.h"
  47#include "xe_i2c.h"
  48#include "xe_irq.h"
  49#include "xe_late_bind_fw.h"
  50#include "xe_mmio.h"
  51#include "xe_module.h"
  52#include "xe_nvm.h"
  53#include "xe_oa.h"
  54#include "xe_observation.h"
  55#include "xe_pagefault.h"
  56#include "xe_pat.h"
  57#include "xe_pcode.h"
  58#include "xe_pm.h"
  59#include "xe_pmu.h"
  60#include "xe_psmi.h"
  61#include "xe_pxp.h"
  62#include "xe_query.h"
  63#include "xe_shrinker.h"
  64#include "xe_survivability_mode.h"
  65#include "xe_sriov.h"
  66#include "xe_tile.h"
  67#include "xe_ttm_stolen_mgr.h"
  68#include "xe_ttm_sys_mgr.h"
  69#include "xe_vm.h"
  70#include "xe_vm_madvise.h"
  71#include "xe_vram.h"
  72#include "xe_vram_types.h"
  73#include "xe_vsec.h"
  74#include "xe_wait_user_fence.h"
  75#include "xe_wa.h"
  76
  77#include <generated/xe_device_wa_oob.h>
  78#include <generated/xe_wa_oob.h>
  79
  80static int xe_file_open(struct drm_device *dev, struct drm_file *file)
  81{
  82	struct xe_device *xe = to_xe_device(dev);
  83	struct xe_drm_client *client;
  84	struct xe_file *xef;
  85	int ret = -ENOMEM;
  86	struct task_struct *task = NULL;
  87
  88	xef = kzalloc(sizeof(*xef), GFP_KERNEL);
  89	if (!xef)
  90		return ret;
  91
  92	client = xe_drm_client_alloc();
  93	if (!client) {
  94		kfree(xef);
  95		return ret;
  96	}
  97
  98	xef->drm = file;
  99	xef->client = client;
 100	xef->xe = xe;
 101
 102	mutex_init(&xef->vm.lock);
 103	xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
 104
 105	mutex_init(&xef->exec_queue.lock);
 106	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
 107
 108	file->driver_priv = xef;
 109	kref_init(&xef->refcount);
 110
 111	task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
 112	if (task) {
 113		xef->process_name = kstrdup(task->comm, GFP_KERNEL);
 114		xef->pid = task->pid;
 115		put_task_struct(task);
 116	}
 117
 118	return 0;
 119}
 120
 121static void xe_file_destroy(struct kref *ref)
 122{
 123	struct xe_file *xef = container_of(ref, struct xe_file, refcount);
 124
 125	xa_destroy(&xef->exec_queue.xa);
 126	mutex_destroy(&xef->exec_queue.lock);
 127	xa_destroy(&xef->vm.xa);
 128	mutex_destroy(&xef->vm.lock);
 129
 130	xe_drm_client_put(xef->client);
 131	kfree(xef->process_name);
 132	kfree(xef);
 133}
 134
 135/**
 136 * xe_file_get() - Take a reference to the xe file object
 137 * @xef: Pointer to the xe file
 138 *
 139 * Anyone with a pointer to xef must take a reference to the xe file
 140 * object using this call.
 141 *
 142 * Return: xe file pointer
 143 */
 144struct xe_file *xe_file_get(struct xe_file *xef)
 145{
 146	kref_get(&xef->refcount);
 147	return xef;
 148}
 149
 150/**
 151 * xe_file_put() - Drop a reference to the xe file object
 152 * @xef: Pointer to the xe file
 153 *
 154 * Used to drop reference to the xef object
 155 */
 156void xe_file_put(struct xe_file *xef)
 157{
 158	kref_put(&xef->refcount, xe_file_destroy);
 159}
 160
 161static void xe_file_close(struct drm_device *dev, struct drm_file *file)
 162{
 163	struct xe_device *xe = to_xe_device(dev);
 164	struct xe_file *xef = file->driver_priv;
 165	struct xe_vm *vm;
 166	struct xe_exec_queue *q;
 167	unsigned long idx;
 168
 169	xe_pm_runtime_get(xe);
 170
 171	/*
 172	 * No need for exec_queue.lock here as there is no contention for it
 173	 * when FD is closing as IOCTLs presumably can't be modifying the
 174	 * xarray. Taking exec_queue.lock here causes undue dependency on
 175	 * vm->lock taken during xe_exec_queue_kill().
 176	 */
 177	xa_for_each(&xef->exec_queue.xa, idx, q) {
 178		if (q->vm && q->hwe->hw_engine_group)
 179			xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
 180		xe_exec_queue_kill(q);
 181		xe_exec_queue_put(q);
 182	}
 183	xa_for_each(&xef->vm.xa, idx, vm)
 184		xe_vm_close_and_put(vm);
 185
 186	xe_file_put(xef);
 187
 188	xe_pm_runtime_put(xe);
 189}
 190
 191static const struct drm_ioctl_desc xe_ioctls[] = {
 192	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
 193	DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
 194	DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
 195			  DRM_RENDER_ALLOW),
 196	DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
 197	DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
 198	DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
 199	DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
 200	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
 201			  DRM_RENDER_ALLOW),
 202	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
 203			  DRM_RENDER_ALLOW),
 204	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
 205			  DRM_RENDER_ALLOW),
 206	DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
 207			  DRM_RENDER_ALLOW),
 208	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
 209	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
 210	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
 211			  DRM_RENDER_ALLOW),
 212};
 213
 214static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 215{
 216	struct drm_file *file_priv = file->private_data;
 217	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
 218	long ret;
 219
 220	if (xe_device_wedged(xe))
 221		return -ECANCELED;
 222
 223	ret = xe_pm_runtime_get_ioctl(xe);
 224	if (ret >= 0)
 225		ret = drm_ioctl(file, cmd, arg);
 226	xe_pm_runtime_put(xe);
 227
 228	return ret;
 229}
 230
 231#ifdef CONFIG_COMPAT
 232static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 233{
 234	struct drm_file *file_priv = file->private_data;
 235	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
 236	long ret;
 237
 238	if (xe_device_wedged(xe))
 239		return -ECANCELED;
 240
 241	ret = xe_pm_runtime_get_ioctl(xe);
 242	if (ret >= 0)
 243		ret = drm_compat_ioctl(file, cmd, arg);
 244	xe_pm_runtime_put(xe);
 245
 246	return ret;
 247}
 248#else
 249/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
 250#define xe_drm_compat_ioctl NULL
 251#endif
 252
 253static void barrier_open(struct vm_area_struct *vma)
 254{
 255	drm_dev_get(vma->vm_private_data);
 256}
 257
 258static void barrier_close(struct vm_area_struct *vma)
 259{
 260	drm_dev_put(vma->vm_private_data);
 261}
 262
 263static void barrier_release_dummy_page(struct drm_device *dev, void *res)
 264{
 265	struct page *dummy_page = (struct page *)res;
 266
 267	__free_page(dummy_page);
 268}
 269
 270static vm_fault_t barrier_fault(struct vm_fault *vmf)
 271{
 272	struct drm_device *dev = vmf->vma->vm_private_data;
 273	struct vm_area_struct *vma = vmf->vma;
 274	vm_fault_t ret = VM_FAULT_NOPAGE;
 275	pgprot_t prot;
 276	int idx;
 277
 278	prot = vm_get_page_prot(vma->vm_flags);
 279
 280	if (drm_dev_enter(dev, &idx)) {
 281		unsigned long pfn;
 282
 283#define LAST_DB_PAGE_OFFSET 0x7ff001
 284		pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
 285				LAST_DB_PAGE_OFFSET);
 286		ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
 287					  pgprot_noncached(prot));
 288		drm_dev_exit(idx);
 289	} else {
 290		struct page *page;
 291
 292		/* Allocate new dummy page to map all the VA range in this VMA to it*/
 293		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 294		if (!page)
 295			return VM_FAULT_OOM;
 296
 297		/* Set the page to be freed using drmm release action */
 298		if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
 299			return VM_FAULT_OOM;
 300
 301		ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
 302					  prot);
 303	}
 304
 305	return ret;
 306}
 307
 308static const struct vm_operations_struct vm_ops_barrier = {
 309	.open = barrier_open,
 310	.close = barrier_close,
 311	.fault = barrier_fault,
 312};
 313
 314static int xe_pci_barrier_mmap(struct file *filp,
 315			       struct vm_area_struct *vma)
 316{
 317	struct drm_file *priv = filp->private_data;
 318	struct drm_device *dev = priv->minor->dev;
 319	struct xe_device *xe = to_xe_device(dev);
 320
 321	if (!IS_DGFX(xe))
 322		return -EINVAL;
 323
 324	if (vma->vm_end - vma->vm_start > SZ_4K)
 325		return -EINVAL;
 326
 327	if (is_cow_mapping(vma->vm_flags))
 328		return -EINVAL;
 329
 330	if (vma->vm_flags & (VM_READ | VM_EXEC))
 331		return -EINVAL;
 332
 333	vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
 334	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
 335	vma->vm_ops = &vm_ops_barrier;
 336	vma->vm_private_data = dev;
 337	drm_dev_get(vma->vm_private_data);
 338
 339	return 0;
 340}
 341
 342static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
 343{
 344	struct drm_file *priv = filp->private_data;
 345	struct drm_device *dev = priv->minor->dev;
 346
 347	if (drm_dev_is_unplugged(dev))
 348		return -ENODEV;
 349
 350	switch (vma->vm_pgoff) {
 351	case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
 352		return xe_pci_barrier_mmap(filp, vma);
 353	}
 354
 355	return drm_gem_mmap(filp, vma);
 356}
 357
 358static const struct file_operations xe_driver_fops = {
 359	.owner = THIS_MODULE,
 360	.open = drm_open,
 361	.release = drm_release_noglobal,
 362	.unlocked_ioctl = xe_drm_ioctl,
 363	.mmap = xe_mmap,
 364	.poll = drm_poll,
 365	.read = drm_read,
 366	.compat_ioctl = xe_drm_compat_ioctl,
 367	.llseek = noop_llseek,
 368#ifdef CONFIG_PROC_FS
 369	.show_fdinfo = drm_show_fdinfo,
 370#endif
 371	.fop_flags = FOP_UNSIGNED_OFFSET,
 372};
 373
 374static struct drm_driver driver = {
 375	/* Don't use MTRRs here; the Xserver or userspace app should
 376	 * deal with them for Intel hardware.
 377	 */
 378	.driver_features =
 379	    DRIVER_GEM |
 380	    DRIVER_RENDER | DRIVER_SYNCOBJ |
 381	    DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
 382	.open = xe_file_open,
 383	.postclose = xe_file_close,
 384
 385	.gem_prime_import = xe_gem_prime_import,
 386
 387	.dumb_create = xe_bo_dumb_create,
 388	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
 389#ifdef CONFIG_PROC_FS
 390	.show_fdinfo = xe_drm_client_fdinfo,
 391#endif
 392	.ioctls = xe_ioctls,
 393	.num_ioctls = ARRAY_SIZE(xe_ioctls),
 394	.fops = &xe_driver_fops,
 395	.name = DRIVER_NAME,
 396	.desc = DRIVER_DESC,
 397	.major = DRIVER_MAJOR,
 398	.minor = DRIVER_MINOR,
 399	.patchlevel = DRIVER_PATCHLEVEL,
 400};
 401
 402static void xe_device_destroy(struct drm_device *dev, void *dummy)
 403{
 404	struct xe_device *xe = to_xe_device(dev);
 405
 406	xe_bo_dev_fini(&xe->bo_device);
 407
 408	if (xe->preempt_fence_wq)
 409		destroy_workqueue(xe->preempt_fence_wq);
 410
 411	if (xe->ordered_wq)
 412		destroy_workqueue(xe->ordered_wq);
 413
 414	if (xe->unordered_wq)
 415		destroy_workqueue(xe->unordered_wq);
 416
 417	if (xe->destroy_wq)
 418		destroy_workqueue(xe->destroy_wq);
 419
 420	ttm_device_fini(&xe->ttm);
 421}
 422
 423struct xe_device *xe_device_create(struct pci_dev *pdev,
 424				   const struct pci_device_id *ent)
 425{
 426	struct xe_device *xe;
 427	int err;
 428
 429	xe_display_driver_set_hooks(&driver);
 430
 431	err = aperture_remove_conflicting_pci_devices(pdev, driver.name);
 432	if (err)
 433		return ERR_PTR(err);
 434
 435	xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
 436	if (IS_ERR(xe))
 437		return xe;
 438
 439	err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
 440			      xe->drm.anon_inode->i_mapping,
 441			      xe->drm.vma_offset_manager, 0);
 442	if (WARN_ON(err))
 443		goto err;
 444
 445	xe_bo_dev_init(&xe->bo_device);
 446	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
 447	if (err)
 448		goto err;
 449
 450	err = xe_shrinker_create(xe);
 451	if (err)
 452		goto err;
 453
 454	xe->info.devid = pdev->device;
 455	xe->info.revid = pdev->revision;
 456	xe->info.force_execlist = xe_modparam.force_execlist;
 457	xe->atomic_svm_timeslice_ms = 5;
 458
 459	err = xe_irq_init(xe);
 460	if (err)
 461		goto err;
 462
 463	xe_validation_device_init(&xe->val);
 464
 465	init_waitqueue_head(&xe->ufence_wq);
 466
 467	init_rwsem(&xe->usm.lock);
 468
 469	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
 470
 471	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
 472		/* Trigger a large asid and an early asid wrap. */
 473		u32 asid;
 474
 475		BUILD_BUG_ON(XE_MAX_ASID < 2);
 476		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
 477				      XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
 478				      &xe->usm.next_asid, GFP_KERNEL);
 479		drm_WARN_ON(&xe->drm, err);
 480		if (err >= 0)
 481			xa_erase(&xe->usm.asid_to_vm, asid);
 482	}
 483
 484	err = xe_bo_pinned_init(xe);
 485	if (err)
 486		goto err;
 487
 488	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
 489						       WQ_MEM_RECLAIM);
 490	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
 491	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
 492	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);
 493	if (!xe->ordered_wq || !xe->unordered_wq ||
 494	    !xe->preempt_fence_wq || !xe->destroy_wq) {
 495		/*
 496		 * Cleanup done in xe_device_destroy via
 497		 * drmm_add_action_or_reset register above
 498		 */
 499		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
 500		err = -ENOMEM;
 501		goto err;
 502	}
 503
 504	err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
 505	if (err)
 506		goto err;
 507
 508	return xe;
 509
 510err:
 511	return ERR_PTR(err);
 512}
 513ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
 514
 515static bool xe_driver_flr_disabled(struct xe_device *xe)
 516{
 517	if (IS_SRIOV_VF(xe))
 518		return true;
 519
 520	if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
 521		drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
 522		return true;
 523	}
 524
 525	return false;
 526}
 527
 528/*
 529 * The driver-initiated FLR is the highest level of reset that we can trigger
 530 * from within the driver. It is different from the PCI FLR in that it doesn't
 531 * fully reset the SGUnit and doesn't modify the PCI config space and therefore
 532 * it doesn't require a re-enumeration of the PCI BARs. However, the
 533 * driver-initiated FLR does still cause a reset of both GT and display and a
 534 * memory wipe of local and stolen memory, so recovery would require a full HW
 535 * re-init and saving/restoring (or re-populating) the wiped memory. Since we
 536 * perform the FLR as the very last action before releasing access to the HW
 537 * during the driver release flow, we don't attempt recovery at all, because
 538 * if/when a new instance of Xe is bound to the device it will do a full
 539 * re-init anyway.
 540 */
 541static void __xe_driver_flr(struct xe_device *xe)
 542{
 543	const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
 544	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
 545	int ret;
 546
 547	drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
 548
 549	/*
 550	 * Make sure any pending FLR requests have cleared by waiting for the
 551	 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
 552	 * to make sure it's not still set from a prior attempt (it's a write to
 553	 * clear bit).
 554	 * Note that we should never be in a situation where a previous attempt
 555	 * is still pending (unless the HW is totally dead), but better to be
 556	 * safe in case something unexpected happens
 557	 */
 558	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
 559	if (ret) {
 560		drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
 561		return;
 562	}
 563	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
 564
 565	/* Trigger the actual Driver-FLR */
 566	xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
 567
 568	/* Wait for hardware teardown to complete */
 569	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
 570	if (ret) {
 571		drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
 572		return;
 573	}
 574
 575	/* Wait for hardware/firmware re-init to complete */
 576	ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
 577			     flr_timeout, NULL, false);
 578	if (ret) {
 579		drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
 580		return;
 581	}
 582
 583	/* Clear sticky completion status */
 584	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
 585}
 586
 587static void xe_driver_flr(struct xe_device *xe)
 588{
 589	if (xe_driver_flr_disabled(xe))
 590		return;
 591
 592	__xe_driver_flr(xe);
 593}
 594
 595static void xe_driver_flr_fini(void *arg)
 596{
 597	struct xe_device *xe = arg;
 598
 599	if (xe->needs_flr_on_fini)
 600		xe_driver_flr(xe);
 601}
 602
 603static void xe_device_sanitize(void *arg)
 604{
 605	struct xe_device *xe = arg;
 606	struct xe_gt *gt;
 607	u8 id;
 608
 609	for_each_gt(gt, xe, id)
 610		xe_gt_sanitize(gt);
 611}
 612
 613static int xe_set_dma_info(struct xe_device *xe)
 614{
 615	unsigned int mask_size = xe->info.dma_mask_size;
 616	int err;
 617
 618	dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
 619
 620	err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
 621	if (err)
 622		goto mask_err;
 623
 624	err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
 625	if (err)
 626		goto mask_err;
 627
 628	return 0;
 629
 630mask_err:
 631	drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
 632	return err;
 633}
 634
 635static int lmem_initializing(struct xe_device *xe)
 636{
 637	if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT)
 638		return 0;
 639
 640	if (signal_pending(current))
 641		return -EINTR;
 642
 643	return 1;
 644}
 645
 646static int wait_for_lmem_ready(struct xe_device *xe)
 647{
 648	const unsigned long TIMEOUT_SEC = 60;
 649	unsigned long prev_jiffies;
 650	int initializing;
 651
 652	if (!IS_DGFX(xe))
 653		return 0;
 654
 655	if (IS_SRIOV_VF(xe))
 656		return 0;
 657
 658	if (!lmem_initializing(xe))
 659		return 0;
 660
 661	drm_dbg(&xe->drm, "Waiting for lmem initialization\n");
 662	prev_jiffies = jiffies;
 663
 664	/*
 665	 * The boot firmware initializes local memory and
 666	 * assesses its health. If memory training fails,
 667	 * the punit will have been instructed to keep the GT powered
 668	 * down.we won't be able to communicate with it
 669	 *
 670	 * If the status check is done before punit updates the register,
 671	 * it can lead to the system being unusable.
 672	 * use a timeout and defer the probe to prevent this.
 673	 */
 674	poll_timeout_us(initializing = lmem_initializing(xe),
 675			initializing <= 0,
 676			20 * USEC_PER_MSEC, TIMEOUT_SEC * USEC_PER_SEC, true);
 677	if (initializing < 0)
 678		return initializing;
 679
 680	if (initializing) {
 681		drm_dbg(&xe->drm, "lmem not initialized by firmware\n");
 682		return -EPROBE_DEFER;
 683	}
 684
 685	drm_dbg(&xe->drm, "lmem ready after %ums",
 686		jiffies_to_msecs(jiffies - prev_jiffies));
 687
 688	return 0;
 689}
 690ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */
 691
 692static void vf_update_device_info(struct xe_device *xe)
 693{
 694	xe_assert(xe, IS_SRIOV_VF(xe));
 695	/* disable features that are not available/applicable to VFs */
 696	xe->info.probe_display = 0;
 697	xe->info.has_heci_cscfi = 0;
 698	xe->info.has_heci_gscfi = 0;
 699	xe->info.has_late_bind = 0;
 700	xe->info.skip_guc_pc = 1;
 701	xe->info.skip_pcode = 1;
 702}
 703
 704static int xe_device_vram_alloc(struct xe_device *xe)
 705{
 706	struct xe_vram_region *vram;
 707
 708	if (!IS_DGFX(xe))
 709		return 0;
 710
 711	vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL);
 712	if (!vram)
 713		return -ENOMEM;
 714
 715	xe->mem.vram = vram;
 716	return 0;
 717}
 718
 719/**
 720 * xe_device_probe_early: Device early probe
 721 * @xe: xe device instance
 722 *
 723 * Initialize MMIO resources that don't require any
 724 * knowledge about tile count. Also initialize pcode and
 725 * check vram initialization on root tile.
 726 *
 727 * Return: 0 on success, error code on failure
 728 */
 729int xe_device_probe_early(struct xe_device *xe)
 730{
 731	int err;
 732
 733	xe_wa_device_init(xe);
 734	xe_wa_process_device_oob(xe);
 735
 736	err = xe_mmio_probe_early(xe);
 737	if (err)
 738		return err;
 739
 740	xe_sriov_probe_early(xe);
 741
 742	if (IS_SRIOV_VF(xe))
 743		vf_update_device_info(xe);
 744
 745	err = xe_pcode_probe_early(xe);
 746	if (err || xe_survivability_mode_is_requested(xe)) {
 747		int save_err = err;
 748
 749		/*
 750		 * Try to leave device in survivability mode if device is
 751		 * possible, but still return the previous error for error
 752		 * propagation
 753		 */
 754		err = xe_survivability_mode_boot_enable(xe);
 755		if (err)
 756			return err;
 757
 758		return save_err;
 759	}
 760
 761	err = wait_for_lmem_ready(xe);
 762	if (err)
 763		return err;
 764
 765	xe->wedged.mode = xe_modparam.wedged_mode;
 766
 767	err = xe_device_vram_alloc(xe);
 768	if (err)
 769		return err;
 770
 771	return 0;
 772}
 773ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
 774
 775static int probe_has_flat_ccs(struct xe_device *xe)
 776{
 777	struct xe_gt *gt;
 778	unsigned int fw_ref;
 779	u32 reg;
 780
 781	/* Always enabled/disabled, no runtime check to do */
 782	if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
 783		return 0;
 784
 785	gt = xe_root_mmio_gt(xe);
 786	if (!gt)
 787		return 0;
 788
 789	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 790	if (!fw_ref)
 791		return -ETIMEDOUT;
 792
 793	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
 794	xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
 795
 796	if (!xe->info.has_flat_ccs)
 797		drm_dbg(&xe->drm,
 798			"Flat CCS has been disabled in bios, May lead to performance impact");
 799
 800	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 801
 802	return 0;
 803}
 804
 805int xe_device_probe(struct xe_device *xe)
 806{
 807	struct xe_tile *tile;
 808	struct xe_gt *gt;
 809	int err;
 810	u8 id;
 811
 812	xe_pat_init_early(xe);
 813
 814	err = xe_sriov_init(xe);
 815	if (err)
 816		return err;
 817
 818	xe->info.mem_region_mask = 1;
 819
 820	err = xe_set_dma_info(xe);
 821	if (err)
 822		return err;
 823
 824	err = xe_mmio_probe_tiles(xe);
 825	if (err)
 826		return err;
 827
 828	for_each_gt(gt, xe, id) {
 829		err = xe_gt_init_early(gt);
 830		if (err)
 831			return err;
 832	}
 833
 834	for_each_tile(tile, xe, id) {
 835		err = xe_ggtt_init_early(tile->mem.ggtt);
 836		if (err)
 837			return err;
 838	}
 839
 840	/*
 841	 * From here on, if a step fails, make sure a Driver-FLR is triggereed
 842	 */
 843	err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
 844	if (err)
 845		return err;
 846
 847	err = probe_has_flat_ccs(xe);
 848	if (err)
 849		return err;
 850
 851	err = xe_vram_probe(xe);
 852	if (err)
 853		return err;
 854
 855	for_each_tile(tile, xe, id) {
 856		err = xe_tile_init_noalloc(tile);
 857		if (err)
 858			return err;
 859	}
 860
 861	/*
 862	 * Allow allocations only now to ensure xe_display_init_early()
 863	 * is the first to allocate, always.
 864	 */
 865	err = xe_ttm_sys_mgr_init(xe);
 866	if (err)
 867		return err;
 868
 869	/* Allocate and map stolen after potential VRAM resize */
 870	err = xe_ttm_stolen_mgr_init(xe);
 871	if (err)
 872		return err;
 873
 874	/*
 875	 * Now that GT is initialized (TTM in particular),
 876	 * we can try to init display, and inherit the initial fb.
 877	 * This is the reason the first allocation needs to be done
 878	 * inside display.
 879	 */
 880	err = xe_display_init_early(xe);
 881	if (err)
 882		return err;
 883
 884	for_each_tile(tile, xe, id) {
 885		err = xe_tile_init(tile);
 886		if (err)
 887			return err;
 888	}
 889
 890	err = xe_irq_install(xe);
 891	if (err)
 892		return err;
 893
 894	for_each_gt(gt, xe, id) {
 895		err = xe_gt_init(gt);
 896		if (err)
 897			return err;
 898	}
 899
 900	err = xe_pagefault_init(xe);
 901	if (err)
 902		return err;
 903
 904	if (xe->tiles->media_gt &&
 905	    XE_GT_WA(xe->tiles->media_gt, 15015404425_disable))
 906		XE_DEVICE_WA_DISABLE(xe, 15015404425);
 907
 908	err = xe_devcoredump_init(xe);
 909	if (err)
 910		return err;
 911
 912	xe_nvm_init(xe);
 913
 914	err = xe_heci_gsc_init(xe);
 915	if (err)
 916		return err;
 917
 918	err = xe_late_bind_init(&xe->late_bind);
 919	if (err)
 920		return err;
 921
 922	err = xe_oa_init(xe);
 923	if (err)
 924		return err;
 925
 926	err = xe_display_init(xe);
 927	if (err)
 928		return err;
 929
 930	err = xe_pxp_init(xe);
 931	if (err)
 932		return err;
 933
 934	err = xe_psmi_init(xe);
 935	if (err)
 936		return err;
 937
 938	err = drm_dev_register(&xe->drm, 0);
 939	if (err)
 940		return err;
 941
 942	xe_display_register(xe);
 943
 944	err = xe_oa_register(xe);
 945	if (err)
 946		goto err_unregister_display;
 947
 948	err = xe_pmu_register(&xe->pmu);
 949	if (err)
 950		goto err_unregister_display;
 951
 952	err = xe_device_sysfs_init(xe);
 953	if (err)
 954		goto err_unregister_display;
 955
 956	xe_debugfs_register(xe);
 957
 958	err = xe_hwmon_register(xe);
 959	if (err)
 960		goto err_unregister_display;
 961
 962	err = xe_i2c_probe(xe);
 963	if (err)
 964		goto err_unregister_display;
 965
 966	for_each_gt(gt, xe, id)
 967		xe_gt_sanitize_freq(gt);
 968
 969	xe_vsec_init(xe);
 970
 971	err = xe_sriov_init_late(xe);
 972	if (err)
 973		goto err_unregister_display;
 974
 975	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
 976
 977err_unregister_display:
 978	xe_display_unregister(xe);
 979	drm_dev_unregister(&xe->drm);
 980
 981	return err;
 982}
 983
 984void xe_device_remove(struct xe_device *xe)
 985{
 986	xe_display_unregister(xe);
 987
 988	drm_dev_unplug(&xe->drm);
 989
 990	xe_bo_pci_dev_remove_all(xe);
 991}
 992
 993void xe_device_shutdown(struct xe_device *xe)
 994{
 995	struct xe_gt *gt;
 996	u8 id;
 997
 998	drm_dbg(&xe->drm, "Shutting down device\n");
 999
1000	xe_display_pm_shutdown(xe);
1001
1002	xe_irq_suspend(xe);
1003
1004	for_each_gt(gt, xe, id)
1005		xe_gt_shutdown(gt);
1006
1007	xe_display_pm_shutdown_late(xe);
1008
1009	if (!xe_driver_flr_disabled(xe)) {
1010		/* BOOM! */
1011		__xe_driver_flr(xe);
1012	}
1013}
1014
1015/**
1016 * xe_device_wmb() - Device specific write memory barrier
1017 * @xe: the &xe_device
1018 *
1019 * While wmb() is sufficient for a barrier if we use system memory, on discrete
1020 * platforms with device memory we additionally need to issue a register write.
1021 * Since it doesn't matter which register we write to, use the read-only VF_CAP
1022 * register that is also marked as accessible by the VFs.
1023 */
1024void xe_device_wmb(struct xe_device *xe)
1025{
1026	wmb();
1027	if (IS_DGFX(xe))
1028		xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
1029}
1030
1031/*
1032 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
1033 */
1034static void tdf_request_sync(struct xe_device *xe)
1035{
1036	unsigned int fw_ref;
1037	struct xe_gt *gt;
1038	u8 id;
1039
1040	for_each_gt(gt, xe, id) {
1041		if (xe_gt_is_media_type(gt))
1042			continue;
1043
1044		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1045		if (!fw_ref)
1046			return;
1047
1048		xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
1049
1050		/*
1051		 * FIXME: We can likely do better here with our choice of
1052		 * timeout. Currently we just assume the worst case, i.e. 150us,
1053		 * which is believed to be sufficient to cover the worst case
1054		 * scenario on current platforms if all cache entries are
1055		 * transient and need to be flushed..
1056		 */
1057		if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1058				   300, NULL, false))
1059			xe_gt_err_once(gt, "TD flush timeout\n");
1060
1061		xe_force_wake_put(gt_to_fw(gt), fw_ref);
1062	}
1063}
1064
1065void xe_device_l2_flush(struct xe_device *xe)
1066{
1067	struct xe_gt *gt;
1068	unsigned int fw_ref;
1069
1070	gt = xe_root_mmio_gt(xe);
1071	if (!gt)
1072		return;
1073
1074	if (!XE_GT_WA(gt, 16023588340))
1075		return;
1076
1077	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1078	if (!fw_ref)
1079		return;
1080
1081	spin_lock(&gt->global_invl_lock);
1082
1083	xe_mmio_write32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1);
1084	if (xe_mmio_wait32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true))
1085		xe_gt_err_once(gt, "Global invalidation timeout\n");
1086
1087	spin_unlock(&gt->global_invl_lock);
1088
1089	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1090}
1091
1092/**
1093 * xe_device_td_flush() - Flush transient L3 cache entries
1094 * @xe: The device
1095 *
1096 * Display engine has direct access to memory and is never coherent with L3/L4
1097 * caches (or CPU caches), however KMD is responsible for specifically flushing
1098 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
1099 * can happen from such a surface without seeing corruption.
1100 *
1101 * Display surfaces can be tagged as transient by mapping it using one of the
1102 * various L3:XD PAT index modes on Xe2.
1103 *
1104 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
1105 * at the end of each submission via PIPE_CONTROL for compute/render, since SA
1106 * Media is not coherent with L3 and we want to support render-vs-media
1107 * usescases. For other engines like copy/blt the HW internally forces uncached
1108 * behaviour, hence why we can skip the TDF on such platforms.
1109 */
1110void xe_device_td_flush(struct xe_device *xe)
1111{
1112	struct xe_gt *root_gt;
1113
1114	if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1115		return;
1116
1117	root_gt = xe_root_mmio_gt(xe);
1118	if (!root_gt)
1119		return;
1120
1121	if (XE_GT_WA(root_gt, 16023588340)) {
1122		/* A transient flush is not sufficient: flush the L2 */
1123		xe_device_l2_flush(xe);
1124	} else {
1125		xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
1126		tdf_request_sync(xe);
1127		xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc);
1128	}
1129}
1130
1131u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1132{
1133	return xe_device_has_flat_ccs(xe) ?
1134		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1135}
1136
1137/**
1138 * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1139 * @xe: xe device instance
1140 *
1141 * To be used before any kind of memory access. It will splat a debug warning
1142 * if the device is currently sleeping. But it doesn't guarantee in any way
1143 * that the device is going to remain awake. Xe PM runtime get and put
1144 * functions might be added to the outer bound of the memory access, while
1145 * this check is intended for inner usage to splat some warning if the worst
1146 * case has just happened.
1147 */
1148void xe_device_assert_mem_access(struct xe_device *xe)
1149{
1150	xe_assert(xe, !xe_pm_runtime_suspended(xe));
1151}
1152
1153void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1154{
1155	struct xe_gt *gt;
1156	u8 id;
1157
1158	drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
1159	drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
1160
1161	for_each_gt(gt, xe, id) {
1162		drm_printf(p, "GT id: %u\n", id);
1163		drm_printf(p, "\tTile: %u\n", gt->tile->id);
1164		drm_printf(p, "\tType: %s\n",
1165			   gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1166		drm_printf(p, "\tIP ver: %u.%u.%u\n",
1167			   REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1168			   REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1169			   REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1170		drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
1171	}
1172}
1173
1174u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1175{
1176	return sign_extend64(address, xe->info.va_bits - 1);
1177}
1178
1179u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1180{
1181	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1182}
1183
1184static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
1185{
1186	struct xe_device *xe = arg;
1187
1188	xe_pm_runtime_put(xe);
1189}
1190
1191/**
1192 * DOC: Xe Device Wedging
1193 *
1194 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst.
1195 * When device is in wedged state, every IOCTL will be blocked and GT cannot be
1196 * used. Certain critical errors like gt reset failure, firmware failures can cause
1197 * the device to be wedged. The default recovery method for a wedged state
1198 * is rebind/bus-reset.
1199 *
1200 * Another recovery method is vendor-specific. Below are the cases that send
1201 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent.
1202 *
1203 * Case: Firmware Flash
1204 * --------------------
1205 *
1206 * Identification Hint
1207 * +++++++++++++++++++
1208 *
1209 * ``WEDGED=vendor-specific`` drm device wedged uevent with
1210 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify
1211 * admin/userspace consumer about the need for a firmware flash.
1212 *
1213 * Recovery Procedure
1214 * ++++++++++++++++++
1215 *
1216 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow
1217 * the below steps
1218 *
1219 * - Check Runtime Survivability mode sysfs.
1220 *   If enabled, firmware flash is required to recover the device.
1221 *
1222 *   /sys/bus/pci/devices/<device>/survivability_mode
1223 *
1224 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash
1225 *   firmware and restore device to normal operation.
1226 */
1227
1228/**
1229 * xe_device_set_wedged_method - Set wedged recovery method
1230 * @xe: xe device instance
1231 * @method: recovery method to set
1232 *
1233 * Set wedged recovery method to be sent in drm wedged uevent.
1234 */
1235void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
1236{
1237	xe->wedged.method = method;
1238}
1239
1240/**
1241 * xe_device_declare_wedged - Declare device wedged
1242 * @xe: xe device instance
1243 *
1244 * This is a final state that can only be cleared with the recovery method
1245 * specified in the drm wedged uevent. The method can be set using
1246 * xe_device_set_wedged_method before declaring the device as wedged. If no method
1247 * is set, reprobe (unbind/re-bind) will be sent by default.
1248 *
1249 * In this state every IOCTL will be blocked so the GT cannot be used.
1250 * In general it will be called upon any critical error such as gt reset
1251 * failure or guc loading failure. Userspace will be notified of this state
1252 * through device wedged uevent.
1253 * If xe.wedged module parameter is set to 2, this function will be called
1254 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1255 * snapshot capture. In this mode, GT reset won't be attempted so the state of
1256 * the issue is preserved for further debugging.
1257 */
1258void xe_device_declare_wedged(struct xe_device *xe)
1259{
1260	struct xe_gt *gt;
1261	u8 id;
1262
1263	if (xe->wedged.mode == 0) {
1264		drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1265		return;
1266	}
1267
1268	xe_pm_runtime_get_noresume(xe);
1269
1270	if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) {
1271		drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n");
1272		return;
1273	}
1274
1275	if (!atomic_xchg(&xe->wedged.flag, 1)) {
1276		xe->needs_flr_on_fini = true;
1277		drm_err(&xe->drm,
1278			"CRITICAL: Xe has declared device %s as wedged.\n"
1279			"IOCTLs and executions are blocked. Only a rebind may clear the failure\n"
1280			"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
1281			dev_name(xe->drm.dev));
1282	}
1283
1284	for_each_gt(gt, xe, id)
1285		xe_gt_declare_wedged(gt);
1286
1287	if (xe_device_wedged(xe)) {
1288		/* If no wedge recovery method is set, use default */
1289		if (!xe->wedged.method)
1290			xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND |
1291						    DRM_WEDGE_RECOVERY_BUS_RESET);
1292
1293		/* Notify userspace of wedged device */
1294		drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL);
1295	}
1296}