vmwgfx_drv.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/**************************************************************************
   3 *
   4 * Copyright (c) 2009-2025 Broadcom. All Rights Reserved. The term
   5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
   6 *
   7 **************************************************************************/
   8
   9#include "vmwgfx_drv.h"
  10
  11#include "vmwgfx_bo.h"
  12#include "vmwgfx_binding.h"
  13#include "vmwgfx_devcaps.h"
  14#include "vmwgfx_mksstat.h"
  15#include "vmwgfx_vkms.h"
  16#include "ttm_object.h"
  17
  18#include <drm/clients/drm_client_setup.h>
  19#include <drm/drm_drv.h>
  20#include <drm/drm_fbdev_ttm.h>
  21#include <drm/drm_gem_ttm_helper.h>
  22#include <drm/drm_ioctl.h>
  23#include <drm/drm_module.h>
  24#include <drm/drm_sysfs.h>
  25#include <drm/ttm/ttm_range_manager.h>
  26#include <drm/ttm/ttm_placement.h>
  27#include <generated/utsrelease.h>
  28
  29#ifdef CONFIG_X86
  30#include <asm/hypervisor.h>
  31#endif
  32
  33#include <linux/aperture.h>
  34#include <linux/cc_platform.h>
  35#include <linux/dma-mapping.h>
  36#include <linux/module.h>
  37#include <linux/pci.h>
  38#include <linux/version.h>
  39#include <linux/vmalloc.h>
  40
  41#define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices"
  42
  43/*
  44 * Fully encoded drm commands. Might move to vmw_drm.h
  45 */
  46
  47#define DRM_IOCTL_VMW_GET_PARAM					\
  48	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GET_PARAM,		\
  49		 struct drm_vmw_getparam_arg)
  50#define DRM_IOCTL_VMW_ALLOC_DMABUF				\
  51	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_ALLOC_DMABUF,	\
  52		union drm_vmw_alloc_dmabuf_arg)
  53#define DRM_IOCTL_VMW_UNREF_DMABUF				\
  54	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_UNREF_DMABUF,	\
  55		struct drm_vmw_unref_dmabuf_arg)
  56#define DRM_IOCTL_VMW_CURSOR_BYPASS				\
  57	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_CURSOR_BYPASS,	\
  58		 struct drm_vmw_cursor_bypass_arg)
  59
  60#define DRM_IOCTL_VMW_CONTROL_STREAM				\
  61	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_CONTROL_STREAM,	\
  62		 struct drm_vmw_control_stream_arg)
  63#define DRM_IOCTL_VMW_CLAIM_STREAM				\
  64	DRM_IOR(DRM_COMMAND_BASE + DRM_VMW_CLAIM_STREAM,	\
  65		 struct drm_vmw_stream_arg)
  66#define DRM_IOCTL_VMW_UNREF_STREAM				\
  67	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_UNREF_STREAM,	\
  68		 struct drm_vmw_stream_arg)
  69
  70#define DRM_IOCTL_VMW_CREATE_CONTEXT				\
  71	DRM_IOR(DRM_COMMAND_BASE + DRM_VMW_CREATE_CONTEXT,	\
  72		struct drm_vmw_context_arg)
  73#define DRM_IOCTL_VMW_UNREF_CONTEXT				\
  74	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_UNREF_CONTEXT,	\
  75		struct drm_vmw_context_arg)
  76#define DRM_IOCTL_VMW_CREATE_SURFACE				\
  77	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_CREATE_SURFACE,	\
  78		 union drm_vmw_surface_create_arg)
  79#define DRM_IOCTL_VMW_UNREF_SURFACE				\
  80	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_UNREF_SURFACE,	\
  81		 struct drm_vmw_surface_arg)
  82#define DRM_IOCTL_VMW_REF_SURFACE				\
  83	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_REF_SURFACE,	\
  84		 union drm_vmw_surface_reference_arg)
  85#define DRM_IOCTL_VMW_EXECBUF					\
  86	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_EXECBUF,		\
  87		struct drm_vmw_execbuf_arg)
  88#define DRM_IOCTL_VMW_GET_3D_CAP				\
  89	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_GET_3D_CAP,		\
  90		 struct drm_vmw_get_3d_cap_arg)
  91#define DRM_IOCTL_VMW_FENCE_WAIT				\
  92	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_FENCE_WAIT,		\
  93		 struct drm_vmw_fence_wait_arg)
  94#define DRM_IOCTL_VMW_FENCE_SIGNALED				\
  95	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_FENCE_SIGNALED,	\
  96		 struct drm_vmw_fence_signaled_arg)
  97#define DRM_IOCTL_VMW_FENCE_UNREF				\
  98	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_FENCE_UNREF,		\
  99		 struct drm_vmw_fence_arg)
 100#define DRM_IOCTL_VMW_FENCE_EVENT				\
 101	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_FENCE_EVENT,		\
 102		 struct drm_vmw_fence_event_arg)
 103#define DRM_IOCTL_VMW_PRESENT					\
 104	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_PRESENT,		\
 105		 struct drm_vmw_present_arg)
 106#define DRM_IOCTL_VMW_PRESENT_READBACK				\
 107	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_PRESENT_READBACK,	\
 108		 struct drm_vmw_present_readback_arg)
 109#define DRM_IOCTL_VMW_UPDATE_LAYOUT				\
 110	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_UPDATE_LAYOUT,	\
 111		 struct drm_vmw_update_layout_arg)
 112#define DRM_IOCTL_VMW_CREATE_SHADER				\
 113	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_CREATE_SHADER,	\
 114		 struct drm_vmw_shader_create_arg)
 115#define DRM_IOCTL_VMW_UNREF_SHADER				\
 116	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_UNREF_SHADER,	\
 117		 struct drm_vmw_shader_arg)
 118#define DRM_IOCTL_VMW_GB_SURFACE_CREATE				\
 119	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GB_SURFACE_CREATE,	\
 120		 union drm_vmw_gb_surface_create_arg)
 121#define DRM_IOCTL_VMW_GB_SURFACE_REF				\
 122	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GB_SURFACE_REF,	\
 123		 union drm_vmw_gb_surface_reference_arg)
 124#define DRM_IOCTL_VMW_SYNCCPU					\
 125	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_SYNCCPU,		\
 126		 struct drm_vmw_synccpu_arg)
 127#define DRM_IOCTL_VMW_CREATE_EXTENDED_CONTEXT			\
 128	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_CREATE_EXTENDED_CONTEXT,	\
 129		struct drm_vmw_context_arg)
 130#define DRM_IOCTL_VMW_GB_SURFACE_CREATE_EXT				\
 131	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GB_SURFACE_CREATE_EXT,	\
 132		union drm_vmw_gb_surface_create_ext_arg)
 133#define DRM_IOCTL_VMW_GB_SURFACE_REF_EXT				\
 134	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_GB_SURFACE_REF_EXT,		\
 135		union drm_vmw_gb_surface_reference_ext_arg)
 136#define DRM_IOCTL_VMW_MSG						\
 137	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_MSG,			\
 138		struct drm_vmw_msg_arg)
 139#define DRM_IOCTL_VMW_MKSSTAT_RESET				\
 140	DRM_IO(DRM_COMMAND_BASE + DRM_VMW_MKSSTAT_RESET)
 141#define DRM_IOCTL_VMW_MKSSTAT_ADD				\
 142	DRM_IOWR(DRM_COMMAND_BASE + DRM_VMW_MKSSTAT_ADD,	\
 143		struct drm_vmw_mksstat_add_arg)
 144#define DRM_IOCTL_VMW_MKSSTAT_REMOVE				\
 145	DRM_IOW(DRM_COMMAND_BASE + DRM_VMW_MKSSTAT_REMOVE,	\
 146		struct drm_vmw_mksstat_remove_arg)
 147
 148/*
 149 * Ioctl definitions.
 150 */
 151
 152static const struct drm_ioctl_desc vmw_ioctls[] = {
 153	DRM_IOCTL_DEF_DRV(VMW_GET_PARAM, vmw_getparam_ioctl,
 154			  DRM_RENDER_ALLOW),
 155	DRM_IOCTL_DEF_DRV(VMW_ALLOC_DMABUF, vmw_gem_object_create_ioctl,
 156			  DRM_RENDER_ALLOW),
 157	DRM_IOCTL_DEF_DRV(VMW_UNREF_DMABUF, vmw_bo_unref_ioctl,
 158			  DRM_RENDER_ALLOW),
 159	DRM_IOCTL_DEF_DRV(VMW_CURSOR_BYPASS,
 160			  vmw_kms_cursor_bypass_ioctl,
 161			  DRM_MASTER),
 162
 163	DRM_IOCTL_DEF_DRV(VMW_CONTROL_STREAM, vmw_overlay_ioctl,
 164			  DRM_MASTER),
 165	DRM_IOCTL_DEF_DRV(VMW_CLAIM_STREAM, vmw_stream_claim_ioctl,
 166			  DRM_MASTER),
 167	DRM_IOCTL_DEF_DRV(VMW_UNREF_STREAM, vmw_stream_unref_ioctl,
 168			  DRM_MASTER),
 169
 170	DRM_IOCTL_DEF_DRV(VMW_CREATE_CONTEXT, vmw_context_define_ioctl,
 171			  DRM_RENDER_ALLOW),
 172	DRM_IOCTL_DEF_DRV(VMW_UNREF_CONTEXT, vmw_context_destroy_ioctl,
 173			  DRM_RENDER_ALLOW),
 174	DRM_IOCTL_DEF_DRV(VMW_CREATE_SURFACE, vmw_surface_define_ioctl,
 175			  DRM_RENDER_ALLOW),
 176	DRM_IOCTL_DEF_DRV(VMW_UNREF_SURFACE, vmw_surface_destroy_ioctl,
 177			  DRM_RENDER_ALLOW),
 178	DRM_IOCTL_DEF_DRV(VMW_REF_SURFACE, vmw_surface_reference_ioctl,
 179			  DRM_RENDER_ALLOW),
 180	DRM_IOCTL_DEF_DRV(VMW_EXECBUF, vmw_execbuf_ioctl,
 181			  DRM_RENDER_ALLOW),
 182	DRM_IOCTL_DEF_DRV(VMW_FENCE_WAIT, vmw_fence_obj_wait_ioctl,
 183			  DRM_RENDER_ALLOW),
 184	DRM_IOCTL_DEF_DRV(VMW_FENCE_SIGNALED,
 185			  vmw_fence_obj_signaled_ioctl,
 186			  DRM_RENDER_ALLOW),
 187	DRM_IOCTL_DEF_DRV(VMW_FENCE_UNREF, vmw_fence_obj_unref_ioctl,
 188			  DRM_RENDER_ALLOW),
 189	DRM_IOCTL_DEF_DRV(VMW_FENCE_EVENT, vmw_fence_event_ioctl,
 190			  DRM_RENDER_ALLOW),
 191	DRM_IOCTL_DEF_DRV(VMW_GET_3D_CAP, vmw_get_cap_3d_ioctl,
 192			  DRM_RENDER_ALLOW),
 193
 194	/* these allow direct access to the framebuffers mark as master only */
 195	DRM_IOCTL_DEF_DRV(VMW_PRESENT, vmw_present_ioctl,
 196			  DRM_MASTER | DRM_AUTH),
 197	DRM_IOCTL_DEF_DRV(VMW_PRESENT_READBACK,
 198			  vmw_present_readback_ioctl,
 199			  DRM_MASTER | DRM_AUTH),
 200	/*
 201	 * The permissions of the below ioctl are overridden in
 202	 * vmw_generic_ioctl(). We require either
 203	 * DRM_MASTER or capable(CAP_SYS_ADMIN).
 204	 */
 205	DRM_IOCTL_DEF_DRV(VMW_UPDATE_LAYOUT,
 206			  vmw_kms_update_layout_ioctl,
 207			  DRM_RENDER_ALLOW),
 208	DRM_IOCTL_DEF_DRV(VMW_CREATE_SHADER,
 209			  vmw_shader_define_ioctl,
 210			  DRM_RENDER_ALLOW),
 211	DRM_IOCTL_DEF_DRV(VMW_UNREF_SHADER,
 212			  vmw_shader_destroy_ioctl,
 213			  DRM_RENDER_ALLOW),
 214	DRM_IOCTL_DEF_DRV(VMW_GB_SURFACE_CREATE,
 215			  vmw_gb_surface_define_ioctl,
 216			  DRM_RENDER_ALLOW),
 217	DRM_IOCTL_DEF_DRV(VMW_GB_SURFACE_REF,
 218			  vmw_gb_surface_reference_ioctl,
 219			  DRM_RENDER_ALLOW),
 220	DRM_IOCTL_DEF_DRV(VMW_SYNCCPU,
 221			  vmw_user_bo_synccpu_ioctl,
 222			  DRM_RENDER_ALLOW),
 223	DRM_IOCTL_DEF_DRV(VMW_CREATE_EXTENDED_CONTEXT,
 224			  vmw_extended_context_define_ioctl,
 225			  DRM_RENDER_ALLOW),
 226	DRM_IOCTL_DEF_DRV(VMW_GB_SURFACE_CREATE_EXT,
 227			  vmw_gb_surface_define_ext_ioctl,
 228			  DRM_RENDER_ALLOW),
 229	DRM_IOCTL_DEF_DRV(VMW_GB_SURFACE_REF_EXT,
 230			  vmw_gb_surface_reference_ext_ioctl,
 231			  DRM_RENDER_ALLOW),
 232	DRM_IOCTL_DEF_DRV(VMW_MSG,
 233			  vmw_msg_ioctl,
 234			  DRM_RENDER_ALLOW),
 235	DRM_IOCTL_DEF_DRV(VMW_MKSSTAT_RESET,
 236			  vmw_mksstat_reset_ioctl,
 237			  DRM_RENDER_ALLOW),
 238	DRM_IOCTL_DEF_DRV(VMW_MKSSTAT_ADD,
 239			  vmw_mksstat_add_ioctl,
 240			  DRM_RENDER_ALLOW),
 241	DRM_IOCTL_DEF_DRV(VMW_MKSSTAT_REMOVE,
 242			  vmw_mksstat_remove_ioctl,
 243			  DRM_RENDER_ALLOW),
 244};
 245
 246static const struct pci_device_id vmw_pci_id_list[] = {
 247	{ PCI_DEVICE(PCI_VENDOR_ID_VMWARE, VMWGFX_PCI_ID_SVGA2) },
 248	{ PCI_DEVICE(PCI_VENDOR_ID_VMWARE, VMWGFX_PCI_ID_SVGA3) },
 249	{ }
 250};
 251MODULE_DEVICE_TABLE(pci, vmw_pci_id_list);
 252
 253static int vmw_restrict_iommu;
 254static int vmw_force_coherent;
 255static int vmw_restrict_dma_mask;
 256static int vmw_assume_16bpp;
 257
 258static int vmw_probe(struct pci_dev *, const struct pci_device_id *);
 259static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
 260			      void *ptr);
 261
 262MODULE_PARM_DESC(restrict_iommu, "Try to limit IOMMU usage for TTM pages");
 263module_param_named(restrict_iommu, vmw_restrict_iommu, int, 0600);
 264MODULE_PARM_DESC(force_coherent, "Force coherent TTM pages");
 265module_param_named(force_coherent, vmw_force_coherent, int, 0600);
 266MODULE_PARM_DESC(restrict_dma_mask, "Restrict DMA mask to 44 bits with IOMMU");
 267module_param_named(restrict_dma_mask, vmw_restrict_dma_mask, int, 0600);
 268MODULE_PARM_DESC(assume_16bpp, "Assume 16-bpp when filtering modes");
 269module_param_named(assume_16bpp, vmw_assume_16bpp, int, 0600);
 270
 271
 272struct bitmap_name {
 273	uint32 value;
 274	const char *name;
 275};
 276
 277static const struct bitmap_name cap1_names[] = {
 278	{ SVGA_CAP_RECT_COPY, "rect copy" },
 279	{ SVGA_CAP_CURSOR, "cursor" },
 280	{ SVGA_CAP_CURSOR_BYPASS, "cursor bypass" },
 281	{ SVGA_CAP_CURSOR_BYPASS_2, "cursor bypass 2" },
 282	{ SVGA_CAP_8BIT_EMULATION, "8bit emulation" },
 283	{ SVGA_CAP_ALPHA_CURSOR, "alpha cursor" },
 284	{ SVGA_CAP_3D, "3D" },
 285	{ SVGA_CAP_EXTENDED_FIFO, "extended fifo" },
 286	{ SVGA_CAP_MULTIMON, "multimon" },
 287	{ SVGA_CAP_PITCHLOCK, "pitchlock" },
 288	{ SVGA_CAP_IRQMASK, "irq mask" },
 289	{ SVGA_CAP_DISPLAY_TOPOLOGY, "display topology" },
 290	{ SVGA_CAP_GMR, "gmr" },
 291	{ SVGA_CAP_TRACES, "traces" },
 292	{ SVGA_CAP_GMR2, "gmr2" },
 293	{ SVGA_CAP_SCREEN_OBJECT_2, "screen object 2" },
 294	{ SVGA_CAP_COMMAND_BUFFERS, "command buffers" },
 295	{ SVGA_CAP_CMD_BUFFERS_2, "command buffers 2" },
 296	{ SVGA_CAP_GBOBJECTS, "gbobject" },
 297	{ SVGA_CAP_DX, "dx" },
 298	{ SVGA_CAP_HP_CMD_QUEUE, "hp cmd queue" },
 299	{ SVGA_CAP_NO_BB_RESTRICTION, "no bb restriction" },
 300	{ SVGA_CAP_CAP2_REGISTER, "cap2 register" },
 301};
 302
 303
 304static const struct bitmap_name cap2_names[] = {
 305	{ SVGA_CAP2_GROW_OTABLE, "grow otable" },
 306	{ SVGA_CAP2_INTRA_SURFACE_COPY, "intra surface copy" },
 307	{ SVGA_CAP2_DX2, "dx2" },
 308	{ SVGA_CAP2_GB_MEMSIZE_2, "gb memsize 2" },
 309	{ SVGA_CAP2_SCREENDMA_REG, "screendma reg" },
 310	{ SVGA_CAP2_OTABLE_PTDEPTH_2, "otable ptdepth2" },
 311	{ SVGA_CAP2_NON_MS_TO_MS_STRETCHBLT, "non ms to ms stretchblt" },
 312	{ SVGA_CAP2_CURSOR_MOB, "cursor mob" },
 313	{ SVGA_CAP2_MSHINT, "mshint" },
 314	{ SVGA_CAP2_CB_MAX_SIZE_4MB, "cb max size 4mb" },
 315	{ SVGA_CAP2_DX3, "dx3" },
 316	{ SVGA_CAP2_FRAME_TYPE, "frame type" },
 317	{ SVGA_CAP2_COTABLE_COPY, "cotable copy" },
 318	{ SVGA_CAP2_TRACE_FULL_FB, "trace full fb" },
 319	{ SVGA_CAP2_EXTRA_REGS, "extra regs" },
 320	{ SVGA_CAP2_LO_STAGING, "lo staging" },
 321};
 322
 323static void vmw_print_bitmap(struct drm_device *drm,
 324			     const char *prefix, uint32_t bitmap,
 325			     const struct bitmap_name *bnames,
 326			     uint32_t num_names)
 327{
 328	char buf[512];
 329	uint32_t i;
 330	uint32_t offset = 0;
 331	for (i = 0; i < num_names; ++i) {
 332		if ((bitmap & bnames[i].value) != 0) {
 333			offset += snprintf(buf + offset,
 334					   ARRAY_SIZE(buf) - offset,
 335					   "%s, ", bnames[i].name);
 336			bitmap &= ~bnames[i].value;
 337		}
 338	}
 339
 340	drm_info(drm, "%s: %s\n", prefix, buf);
 341	if (bitmap != 0)
 342		drm_dbg(drm, "%s: unknown enums: %x\n", prefix, bitmap);
 343}
 344
 345
 346static void vmw_print_sm_type(struct vmw_private *dev_priv)
 347{
 348	static const char *names[] = {
 349		[VMW_SM_LEGACY] = "Legacy",
 350		[VMW_SM_4] = "SM4",
 351		[VMW_SM_4_1] = "SM4_1",
 352		[VMW_SM_5] = "SM_5",
 353		[VMW_SM_5_1X] = "SM_5_1X",
 354		[VMW_SM_MAX] = "Invalid"
 355	};
 356	BUILD_BUG_ON(ARRAY_SIZE(names) != (VMW_SM_MAX + 1));
 357	drm_info(&dev_priv->drm, "Available shader model: %s.\n",
 358		 names[dev_priv->sm_type]);
 359}
 360
 361/**
 362 * vmw_dummy_query_bo_create - create a bo to hold a dummy query result
 363 *
 364 * @dev_priv: A device private structure.
 365 *
 366 * This function creates a small buffer object that holds the query
 367 * result for dummy queries emitted as query barriers.
 368 * The function will then map the first page and initialize a pending
 369 * occlusion query result structure, Finally it will unmap the buffer.
 370 * No interruptible waits are done within this function.
 371 *
 372 * Returns an error if bo creation or initialization fails.
 373 */
 374static int vmw_dummy_query_bo_create(struct vmw_private *dev_priv)
 375{
 376	int ret;
 377	struct vmw_bo *vbo;
 378	struct ttm_bo_kmap_obj map;
 379	volatile SVGA3dQueryResult *result;
 380	bool dummy;
 381	struct vmw_bo_params bo_params = {
 382		.domain = VMW_BO_DOMAIN_SYS,
 383		.busy_domain = VMW_BO_DOMAIN_SYS,
 384		.bo_type = ttm_bo_type_kernel,
 385		.size = PAGE_SIZE,
 386		.pin = true,
 387		.keep_resv = true,
 388	};
 389
 390	/*
 391	 * Create the vbo as pinned, so that a tryreserve will
 392	 * immediately succeed. This is because we're the only
 393	 * user of the bo currently.
 394	 */
 395	ret = vmw_bo_create(dev_priv, &bo_params, &vbo);
 396	if (unlikely(ret != 0))
 397		return ret;
 398
 399	ret = ttm_bo_kmap(&vbo->tbo, 0, 1, &map);
 400	if (likely(ret == 0)) {
 401		result = ttm_kmap_obj_virtual(&map, &dummy);
 402		result->totalSize = sizeof(*result);
 403		result->state = SVGA3D_QUERYSTATE_PENDING;
 404		result->result32 = 0xff;
 405		ttm_bo_kunmap(&map);
 406	}
 407	vmw_bo_pin_reserved(vbo, false);
 408	ttm_bo_unreserve(&vbo->tbo);
 409
 410	if (unlikely(ret != 0)) {
 411		DRM_ERROR("Dummy query buffer map failed.\n");
 412		vmw_bo_unreference(&vbo);
 413	} else
 414		dev_priv->dummy_query_bo = vbo;
 415
 416	return ret;
 417}
 418
 419static int vmw_device_init(struct vmw_private *dev_priv)
 420{
 421	bool uses_fb_traces = false;
 422
 423	dev_priv->enable_state = vmw_read(dev_priv, SVGA_REG_ENABLE);
 424	dev_priv->config_done_state = vmw_read(dev_priv, SVGA_REG_CONFIG_DONE);
 425	dev_priv->traces_state = vmw_read(dev_priv, SVGA_REG_TRACES);
 426
 427	vmw_write(dev_priv, SVGA_REG_ENABLE, SVGA_REG_ENABLE_ENABLE |
 428		  SVGA_REG_ENABLE_HIDE);
 429
 430	uses_fb_traces = !vmw_cmd_supported(dev_priv) &&
 431			 (dev_priv->capabilities & SVGA_CAP_TRACES) != 0;
 432
 433	vmw_write(dev_priv, SVGA_REG_TRACES, uses_fb_traces);
 434	dev_priv->fifo = vmw_fifo_create(dev_priv);
 435	if (IS_ERR(dev_priv->fifo)) {
 436		int err = PTR_ERR(dev_priv->fifo);
 437		dev_priv->fifo = NULL;
 438		return err;
 439	} else if (!dev_priv->fifo) {
 440		vmw_write(dev_priv, SVGA_REG_CONFIG_DONE, 1);
 441	}
 442
 443	u32 seqno = vmw_fence_read(dev_priv);
 444
 445	atomic_set(&dev_priv->last_read_seqno, seqno);
 446	atomic_set(&dev_priv->marker_seq, seqno);
 447	return 0;
 448}
 449
 450static void vmw_device_fini(struct vmw_private *vmw)
 451{
 452	/*
 453	 * Legacy sync
 454	 */
 455	vmw_write(vmw, SVGA_REG_SYNC, SVGA_SYNC_GENERIC);
 456	while (vmw_read(vmw, SVGA_REG_BUSY) != 0)
 457		;
 458
 459	atomic_set(&vmw->last_read_seqno, vmw_fence_read(vmw));
 460
 461	vmw_write(vmw, SVGA_REG_CONFIG_DONE,
 462		  vmw->config_done_state);
 463	vmw_write(vmw, SVGA_REG_ENABLE,
 464		  vmw->enable_state);
 465	vmw_write(vmw, SVGA_REG_TRACES,
 466		  vmw->traces_state);
 467
 468	vmw_fifo_destroy(vmw);
 469}
 470
 471/**
 472 * vmw_request_device_late - Perform late device setup
 473 *
 474 * @dev_priv: Pointer to device private.
 475 *
 476 * This function performs setup of otables and enables large command
 477 * buffer submission. These tasks are split out to a separate function
 478 * because it reverts vmw_release_device_early and is intended to be used
 479 * by an error path in the hibernation code.
 480 */
 481static int vmw_request_device_late(struct vmw_private *dev_priv)
 482{
 483	int ret;
 484
 485	if (dev_priv->has_mob) {
 486		ret = vmw_otables_setup(dev_priv);
 487		if (unlikely(ret != 0)) {
 488			DRM_ERROR("Unable to initialize "
 489				  "guest Memory OBjects.\n");
 490			return ret;
 491		}
 492	}
 493
 494	if (dev_priv->cman) {
 495		ret = vmw_cmdbuf_set_pool_size(dev_priv->cman, 256*4096);
 496		if (ret) {
 497			struct vmw_cmdbuf_man *man = dev_priv->cman;
 498
 499			dev_priv->cman = NULL;
 500			vmw_cmdbuf_man_destroy(man);
 501		}
 502	}
 503
 504	return 0;
 505}
 506
 507static int vmw_request_device(struct vmw_private *dev_priv)
 508{
 509	int ret;
 510
 511	ret = vmw_device_init(dev_priv);
 512	if (unlikely(ret != 0)) {
 513		DRM_ERROR("Unable to initialize the device.\n");
 514		return ret;
 515	}
 516	vmw_fence_fifo_up(dev_priv->fman);
 517	dev_priv->cman = vmw_cmdbuf_man_create(dev_priv);
 518	if (IS_ERR(dev_priv->cman)) {
 519		dev_priv->cman = NULL;
 520		dev_priv->sm_type = VMW_SM_LEGACY;
 521	}
 522
 523	ret = vmw_request_device_late(dev_priv);
 524	if (ret)
 525		goto out_no_mob;
 526
 527	ret = vmw_dummy_query_bo_create(dev_priv);
 528	if (unlikely(ret != 0))
 529		goto out_no_query_bo;
 530
 531	return 0;
 532
 533out_no_query_bo:
 534	if (dev_priv->cman)
 535		vmw_cmdbuf_remove_pool(dev_priv->cman);
 536	if (dev_priv->has_mob) {
 537		struct ttm_resource_manager *man;
 538
 539		man = ttm_manager_type(&dev_priv->bdev, VMW_PL_MOB);
 540		ttm_resource_manager_evict_all(&dev_priv->bdev, man);
 541		vmw_otables_takedown(dev_priv);
 542	}
 543	if (dev_priv->cman)
 544		vmw_cmdbuf_man_destroy(dev_priv->cman);
 545out_no_mob:
 546	vmw_fence_fifo_down(dev_priv->fman);
 547	vmw_device_fini(dev_priv);
 548	return ret;
 549}
 550
 551/**
 552 * vmw_release_device_early - Early part of fifo takedown.
 553 *
 554 * @dev_priv: Pointer to device private struct.
 555 *
 556 * This is the first part of command submission takedown, to be called before
 557 * buffer management is taken down.
 558 */
 559static void vmw_release_device_early(struct vmw_private *dev_priv)
 560{
 561	/*
 562	 * Previous destructions should've released
 563	 * the pinned bo.
 564	 */
 565
 566	BUG_ON(dev_priv->pinned_bo != NULL);
 567
 568	vmw_bo_unreference(&dev_priv->dummy_query_bo);
 569	if (dev_priv->cman)
 570		vmw_cmdbuf_remove_pool(dev_priv->cman);
 571
 572	if (dev_priv->has_mob) {
 573		struct ttm_resource_manager *man;
 574
 575		man = ttm_manager_type(&dev_priv->bdev, VMW_PL_MOB);
 576		ttm_resource_manager_evict_all(&dev_priv->bdev, man);
 577		vmw_otables_takedown(dev_priv);
 578	}
 579}
 580
 581/**
 582 * vmw_release_device_late - Late part of fifo takedown.
 583 *
 584 * @dev_priv: Pointer to device private struct.
 585 *
 586 * This is the last part of the command submission takedown, to be called when
 587 * command submission is no longer needed. It may wait on pending fences.
 588 */
 589static void vmw_release_device_late(struct vmw_private *dev_priv)
 590{
 591	vmw_fence_fifo_down(dev_priv->fman);
 592	if (dev_priv->cman)
 593		vmw_cmdbuf_man_destroy(dev_priv->cman);
 594
 595	vmw_device_fini(dev_priv);
 596}
 597
 598/*
 599 * Sets the initial_[width|height] fields on the given vmw_private.
 600 *
 601 * It does so by reading SVGA_REG_[WIDTH|HEIGHT] regs and then
 602 * clamping the value to fb_max_[width|height] fields and the
 603 * VMW_MIN_INITIAL_[WIDTH|HEIGHT].
 604 * If the values appear to be invalid, set them to
 605 * VMW_MIN_INITIAL_[WIDTH|HEIGHT].
 606 */
 607static void vmw_get_initial_size(struct vmw_private *dev_priv)
 608{
 609	uint32_t width;
 610	uint32_t height;
 611
 612	width = vmw_read(dev_priv, SVGA_REG_WIDTH);
 613	height = vmw_read(dev_priv, SVGA_REG_HEIGHT);
 614
 615	width = max_t(uint32_t, width, VMWGFX_MIN_INITIAL_WIDTH);
 616	height = max_t(uint32_t, height, VMWGFX_MIN_INITIAL_HEIGHT);
 617
 618	if (width > dev_priv->fb_max_width ||
 619	    height > dev_priv->fb_max_height) {
 620
 621		/*
 622		 * This is a host error and shouldn't occur.
 623		 */
 624
 625		width  = VMWGFX_MIN_INITIAL_WIDTH;
 626		height = VMWGFX_MIN_INITIAL_HEIGHT;
 627	}
 628
 629	dev_priv->initial_width = width;
 630	dev_priv->initial_height = height;
 631}
 632
 633/**
 634 * vmw_dma_select_mode - Determine how DMA mappings should be set up for this
 635 * system.
 636 *
 637 * @dev_priv: Pointer to a struct vmw_private
 638 *
 639 * This functions tries to determine what actions need to be taken by the
 640 * driver to make system pages visible to the device.
 641 * If this function decides that DMA is not possible, it returns -EINVAL.
 642 * The driver may then try to disable features of the device that require
 643 * DMA.
 644 */
 645static int vmw_dma_select_mode(struct vmw_private *dev_priv)
 646{
 647	static const char *names[vmw_dma_map_max] = {
 648		[vmw_dma_alloc_coherent] = "Using coherent TTM pages.",
 649		[vmw_dma_map_populate] = "Caching DMA mappings.",
 650		[vmw_dma_map_bind] = "Giving up DMA mappings early."};
 651
 652	/*
 653	 * When running with SEV we always want dma mappings, because
 654	 * otherwise ttm tt pool pages will bounce through swiotlb running
 655	 * out of available space.
 656	 */
 657	if (vmw_force_coherent || cc_platform_has(CC_ATTR_MEM_ENCRYPT))
 658		dev_priv->map_mode = vmw_dma_alloc_coherent;
 659	else if (vmw_restrict_iommu)
 660		dev_priv->map_mode = vmw_dma_map_bind;
 661	else
 662		dev_priv->map_mode = vmw_dma_map_populate;
 663
 664	drm_info(&dev_priv->drm,
 665		 "DMA map mode: %s\n", names[dev_priv->map_mode]);
 666	return 0;
 667}
 668
 669/**
 670 * vmw_dma_masks - set required page- and dma masks
 671 *
 672 * @dev_priv: Pointer to struct drm-device
 673 *
 674 * With 32-bit we can only handle 32 bit PFNs. Optionally set that
 675 * restriction also for 64-bit systems.
 676 */
 677static int vmw_dma_masks(struct vmw_private *dev_priv)
 678{
 679	struct drm_device *dev = &dev_priv->drm;
 680	int ret = 0;
 681
 682	ret = dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64));
 683	if (sizeof(unsigned long) == 4 || vmw_restrict_dma_mask) {
 684		drm_info(&dev_priv->drm,
 685			 "Restricting DMA addresses to 44 bits.\n");
 686		return dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(44));
 687	}
 688
 689	return ret;
 690}
 691
 692static int vmw_vram_manager_init(struct vmw_private *dev_priv)
 693{
 694	int ret;
 695	ret = ttm_range_man_init(&dev_priv->bdev, TTM_PL_VRAM, false,
 696				 dev_priv->vram_size >> PAGE_SHIFT);
 697	ttm_resource_manager_set_used(ttm_manager_type(&dev_priv->bdev, TTM_PL_VRAM), false);
 698	return ret;
 699}
 700
 701static void vmw_vram_manager_fini(struct vmw_private *dev_priv)
 702{
 703	ttm_range_man_fini(&dev_priv->bdev, TTM_PL_VRAM);
 704}
 705
 706static int vmw_setup_pci_resources(struct vmw_private *dev,
 707				   u32 pci_id)
 708{
 709	resource_size_t rmmio_start;
 710	resource_size_t rmmio_size;
 711	resource_size_t fifo_start;
 712	resource_size_t fifo_size;
 713	int ret;
 714	struct pci_dev *pdev = to_pci_dev(dev->drm.dev);
 715
 716	pci_set_master(pdev);
 717
 718	ret = pcim_request_all_regions(pdev, "vmwgfx probe");
 719	if (ret)
 720		return ret;
 721
 722	dev->pci_id = pci_id;
 723	if (pci_id == VMWGFX_PCI_ID_SVGA3) {
 724		rmmio_start = pci_resource_start(pdev, 0);
 725		rmmio_size = pci_resource_len(pdev, 0);
 726		dev->vram_start = pci_resource_start(pdev, 2);
 727		dev->vram_size = pci_resource_len(pdev, 2);
 728
 729		drm_info(&dev->drm,
 730			"Register MMIO at 0x%pa size is %llu KiB\n",
 731			 &rmmio_start, (uint64_t)rmmio_size / 1024);
 732		dev->rmmio = devm_ioremap(dev->drm.dev,
 733					  rmmio_start,
 734					  rmmio_size);
 735		if (!dev->rmmio) {
 736			drm_err(&dev->drm,
 737				"Failed mapping registers mmio memory.\n");
 738			return -ENOMEM;
 739		}
 740	} else if (pci_id == VMWGFX_PCI_ID_SVGA2) {
 741		dev->io_start = pci_resource_start(pdev, 0);
 742		dev->vram_start = pci_resource_start(pdev, 1);
 743		dev->vram_size = pci_resource_len(pdev, 1);
 744		fifo_start = pci_resource_start(pdev, 2);
 745		fifo_size = pci_resource_len(pdev, 2);
 746
 747		drm_info(&dev->drm,
 748			 "FIFO at %pa size is %llu KiB\n",
 749			 &fifo_start, (uint64_t)fifo_size / 1024);
 750		dev->fifo_mem = devm_memremap(dev->drm.dev,
 751					      fifo_start,
 752					      fifo_size,
 753					      MEMREMAP_WB | MEMREMAP_DEC);
 754
 755		if (IS_ERR(dev->fifo_mem)) {
 756			drm_err(&dev->drm,
 757				  "Failed mapping FIFO memory.\n");
 758			return PTR_ERR(dev->fifo_mem);
 759		}
 760	} else {
 761		return -EINVAL;
 762	}
 763
 764	/*
 765	 * This is approximate size of the vram, the exact size will only
 766	 * be known after we read SVGA_REG_VRAM_SIZE. The PCI resource
 767	 * size will be equal to or bigger than the size reported by
 768	 * SVGA_REG_VRAM_SIZE.
 769	 */
 770	drm_info(&dev->drm,
 771		 "VRAM at %pa size is %llu KiB\n",
 772		 &dev->vram_start, (uint64_t)dev->vram_size / 1024);
 773
 774	return 0;
 775}
 776
 777static int vmw_detect_version(struct vmw_private *dev)
 778{
 779	uint32_t svga_id;
 780
 781	vmw_write(dev, SVGA_REG_ID, vmw_is_svga_v3(dev) ?
 782			  SVGA_ID_3 : SVGA_ID_2);
 783	svga_id = vmw_read(dev, SVGA_REG_ID);
 784	if (svga_id != SVGA_ID_2 && svga_id != SVGA_ID_3) {
 785		drm_err(&dev->drm,
 786			"Unsupported SVGA ID 0x%x on chipset 0x%x\n",
 787			svga_id, dev->pci_id);
 788		return -ENOSYS;
 789	}
 790	BUG_ON(vmw_is_svga_v3(dev) && (svga_id != SVGA_ID_3));
 791	drm_info(&dev->drm,
 792		 "Running on SVGA version %d.\n", (svga_id & 0xff));
 793	return 0;
 794}
 795
 796static void vmw_write_driver_id(struct vmw_private *dev)
 797{
 798	if ((dev->capabilities2 & SVGA_CAP2_DX2) != 0) {
 799		vmw_write(dev,  SVGA_REG_GUEST_DRIVER_ID,
 800			  SVGA_REG_GUEST_DRIVER_ID_LINUX);
 801
 802		vmw_write(dev, SVGA_REG_GUEST_DRIVER_VERSION1,
 803			  LINUX_VERSION_MAJOR << 24 |
 804			  LINUX_VERSION_PATCHLEVEL << 16 |
 805			  LINUX_VERSION_SUBLEVEL);
 806		vmw_write(dev, SVGA_REG_GUEST_DRIVER_VERSION2,
 807			  VMWGFX_DRIVER_MAJOR << 24 |
 808			  VMWGFX_DRIVER_MINOR << 16 |
 809			  VMWGFX_DRIVER_PATCHLEVEL);
 810		vmw_write(dev, SVGA_REG_GUEST_DRIVER_VERSION3, 0);
 811
 812		vmw_write(dev, SVGA_REG_GUEST_DRIVER_ID,
 813			  SVGA_REG_GUEST_DRIVER_ID_SUBMIT);
 814	}
 815}
 816
 817static void vmw_sw_context_init(struct vmw_private *dev_priv)
 818{
 819	struct vmw_sw_context *sw_context = &dev_priv->ctx;
 820
 821	hash_init(sw_context->res_ht);
 822}
 823
 824static void vmw_sw_context_fini(struct vmw_private *dev_priv)
 825{
 826	struct vmw_sw_context *sw_context = &dev_priv->ctx;
 827
 828	vfree(sw_context->cmd_bounce);
 829	if (sw_context->staged_bindings)
 830		vmw_binding_state_free(sw_context->staged_bindings);
 831}
 832
 833static int vmw_driver_load(struct vmw_private *dev_priv, u32 pci_id)
 834{
 835	int ret;
 836	enum vmw_res_type i;
 837	bool refuse_dma = false;
 838
 839	vmw_sw_context_init(dev_priv);
 840
 841	mutex_init(&dev_priv->cmdbuf_mutex);
 842	mutex_init(&dev_priv->binding_mutex);
 843	spin_lock_init(&dev_priv->resource_lock);
 844	spin_lock_init(&dev_priv->hw_lock);
 845	spin_lock_init(&dev_priv->waiter_lock);
 846	spin_lock_init(&dev_priv->cursor_lock);
 847
 848	ret = vmw_setup_pci_resources(dev_priv, pci_id);
 849	if (ret)
 850		return ret;
 851	ret = vmw_detect_version(dev_priv);
 852	if (ret)
 853		return ret;
 854
 855
 856	for (i = vmw_res_context; i < vmw_res_max; ++i) {
 857		idr_init_base(&dev_priv->res_idr[i], 1);
 858		INIT_LIST_HEAD(&dev_priv->res_lru[i]);
 859	}
 860
 861	init_waitqueue_head(&dev_priv->fence_queue);
 862	init_waitqueue_head(&dev_priv->fifo_queue);
 863	dev_priv->fence_queue_waiters = 0;
 864	dev_priv->fifo_queue_waiters = 0;
 865
 866	dev_priv->used_memory_size = 0;
 867
 868	dev_priv->assume_16bpp = !!vmw_assume_16bpp;
 869
 870	dev_priv->capabilities = vmw_read(dev_priv, SVGA_REG_CAPABILITIES);
 871	vmw_print_bitmap(&dev_priv->drm, "Capabilities",
 872			 dev_priv->capabilities,
 873			 cap1_names, ARRAY_SIZE(cap1_names));
 874	if (dev_priv->capabilities & SVGA_CAP_CAP2_REGISTER) {
 875		dev_priv->capabilities2 = vmw_read(dev_priv, SVGA_REG_CAP2);
 876		vmw_print_bitmap(&dev_priv->drm, "Capabilities2",
 877				 dev_priv->capabilities2,
 878				 cap2_names, ARRAY_SIZE(cap2_names));
 879	}
 880
 881	if (!vmwgfx_supported(dev_priv)) {
 882		vmw_disable_backdoor();
 883		drm_err_once(&dev_priv->drm,
 884			     "vmwgfx seems to be running on an unsupported hypervisor.");
 885		drm_err_once(&dev_priv->drm,
 886			     "This configuration is likely broken.");
 887		drm_err_once(&dev_priv->drm,
 888			     "Please switch to a supported graphics device to avoid problems.");
 889	}
 890
 891	vmw_vkms_init(dev_priv);
 892
 893	ret = vmw_dma_select_mode(dev_priv);
 894	if (unlikely(ret != 0)) {
 895		drm_info(&dev_priv->drm,
 896			 "Restricting capabilities since DMA not available.\n");
 897		refuse_dma = true;
 898		if (dev_priv->capabilities & SVGA_CAP_GBOBJECTS)
 899			drm_info(&dev_priv->drm,
 900				 "Disabling 3D acceleration.\n");
 901	}
 902
 903	dev_priv->vram_size = vmw_read(dev_priv, SVGA_REG_VRAM_SIZE);
 904	dev_priv->fifo_mem_size = vmw_read(dev_priv, SVGA_REG_MEM_SIZE);
 905	dev_priv->fb_max_width = vmw_read(dev_priv, SVGA_REG_MAX_WIDTH);
 906	dev_priv->fb_max_height = vmw_read(dev_priv, SVGA_REG_MAX_HEIGHT);
 907
 908	vmw_get_initial_size(dev_priv);
 909
 910	if (dev_priv->capabilities & SVGA_CAP_GMR2) {
 911		dev_priv->max_gmr_ids =
 912			vmw_read(dev_priv, SVGA_REG_GMR_MAX_IDS);
 913		dev_priv->max_gmr_pages =
 914			vmw_read(dev_priv, SVGA_REG_GMRS_MAX_PAGES);
 915		dev_priv->memory_size =
 916			vmw_read(dev_priv, SVGA_REG_MEMORY_SIZE);
 917		dev_priv->memory_size -= dev_priv->vram_size;
 918	} else {
 919		/*
 920		 * An arbitrary limit of 512MiB on surface
 921		 * memory. But all HWV8 hardware supports GMR2.
 922		 */
 923		dev_priv->memory_size = 512*1024*1024;
 924	}
 925	dev_priv->max_mob_pages = 0;
 926	dev_priv->max_mob_size = 0;
 927	if (dev_priv->capabilities & SVGA_CAP_GBOBJECTS) {
 928		uint64_t mem_size;
 929
 930		if (dev_priv->capabilities2 & SVGA_CAP2_GB_MEMSIZE_2)
 931			mem_size = vmw_read(dev_priv,
 932					    SVGA_REG_GBOBJECT_MEM_SIZE_KB);
 933		else
 934			mem_size =
 935				vmw_read(dev_priv,
 936					 SVGA_REG_SUGGESTED_GBOBJECT_MEM_SIZE_KB);
 937
 938		dev_priv->max_mob_pages = mem_size * 1024 / PAGE_SIZE;
 939		dev_priv->max_primary_mem =
 940			vmw_read(dev_priv, SVGA_REG_MAX_PRIMARY_MEM);
 941		dev_priv->max_mob_size =
 942			vmw_read(dev_priv, SVGA_REG_MOB_MAX_SIZE);
 943		dev_priv->stdu_max_width =
 944			vmw_read(dev_priv, SVGA_REG_SCREENTARGET_MAX_WIDTH);
 945		dev_priv->stdu_max_height =
 946			vmw_read(dev_priv, SVGA_REG_SCREENTARGET_MAX_HEIGHT);
 947
 948		vmw_write(dev_priv, SVGA_REG_DEV_CAP,
 949			  SVGA3D_DEVCAP_MAX_TEXTURE_WIDTH);
 950		dev_priv->texture_max_width = vmw_read(dev_priv,
 951						       SVGA_REG_DEV_CAP);
 952		vmw_write(dev_priv, SVGA_REG_DEV_CAP,
 953			  SVGA3D_DEVCAP_MAX_TEXTURE_HEIGHT);
 954		dev_priv->texture_max_height = vmw_read(dev_priv,
 955							SVGA_REG_DEV_CAP);
 956	} else {
 957		dev_priv->texture_max_width = 8192;
 958		dev_priv->texture_max_height = 8192;
 959		dev_priv->max_primary_mem = dev_priv->vram_size;
 960	}
 961	drm_info(&dev_priv->drm,
 962		 "Legacy memory limits: VRAM = %llu KiB, FIFO = %llu KiB, surface = %u KiB\n",
 963		 (u64)dev_priv->vram_size / 1024,
 964		 (u64)dev_priv->fifo_mem_size / 1024,
 965		 dev_priv->memory_size / 1024);
 966
 967	drm_info(&dev_priv->drm,
 968		 "MOB limits: max mob size = %u KiB, max mob pages = %u\n",
 969		 dev_priv->max_mob_size / 1024, dev_priv->max_mob_pages);
 970
 971	ret = vmw_dma_masks(dev_priv);
 972	if (unlikely(ret != 0))
 973		goto out_err0;
 974
 975	dma_set_max_seg_size(dev_priv->drm.dev, U32_MAX);
 976
 977	if (dev_priv->capabilities & SVGA_CAP_GMR2) {
 978		drm_info(&dev_priv->drm,
 979			 "Max GMR ids is %u\n",
 980			 (unsigned)dev_priv->max_gmr_ids);
 981		drm_info(&dev_priv->drm,
 982			 "Max number of GMR pages is %u\n",
 983			 (unsigned)dev_priv->max_gmr_pages);
 984	}
 985	drm_info(&dev_priv->drm,
 986		 "Maximum display memory size is %llu KiB\n",
 987		 (uint64_t)dev_priv->max_primary_mem / 1024);
 988
 989	/* Need mmio memory to check for fifo pitchlock cap. */
 990	if (!(dev_priv->capabilities & SVGA_CAP_DISPLAY_TOPOLOGY) &&
 991	    !(dev_priv->capabilities & SVGA_CAP_PITCHLOCK) &&
 992	    !vmw_fifo_have_pitchlock(dev_priv)) {
 993		ret = -ENOSYS;
 994		DRM_ERROR("Hardware has no pitchlock\n");
 995		goto out_err0;
 996	}
 997
 998	dev_priv->tdev = ttm_object_device_init(&vmw_prime_dmabuf_ops);
 999
1000	if (unlikely(dev_priv->tdev == NULL)) {
1001		drm_err(&dev_priv->drm,
1002			"Unable to initialize TTM object management.\n");
1003		ret = -ENOMEM;
1004		goto out_err0;
1005	}
1006
1007	if (dev_priv->capabilities & SVGA_CAP_IRQMASK) {
1008		ret = vmw_irq_install(dev_priv);
1009		if (ret != 0) {
1010			drm_err(&dev_priv->drm,
1011				"Failed installing irq: %d\n", ret);
1012			goto out_no_irq;
1013		}
1014	}
1015
1016	dev_priv->fman = vmw_fence_manager_init(dev_priv);
1017	if (unlikely(dev_priv->fman == NULL)) {
1018		ret = -ENOMEM;
1019		goto out_no_fman;
1020	}
1021
1022	ret = ttm_device_init(&dev_priv->bdev, &vmw_bo_driver,
1023			      dev_priv->drm.dev,
1024			      dev_priv->drm.anon_inode->i_mapping,
1025			      dev_priv->drm.vma_offset_manager,
1026			      (dev_priv->map_mode == vmw_dma_alloc_coherent) ?
1027			      TTM_ALLOCATION_POOL_USE_DMA_ALLOC : 0);
1028	if (unlikely(ret != 0)) {
1029		drm_err(&dev_priv->drm,
1030			"Failed initializing TTM buffer object driver.\n");
1031		goto out_no_bdev;
1032	}
1033
1034	/*
1035	 * Enable VRAM, but initially don't use it until SVGA is enabled and
1036	 * unhidden.
1037	 */
1038
1039	ret = vmw_vram_manager_init(dev_priv);
1040	if (unlikely(ret != 0)) {
1041		drm_err(&dev_priv->drm,
1042			"Failed initializing memory manager for VRAM.\n");
1043		goto out_no_vram;
1044	}
1045
1046	ret = vmw_devcaps_create(dev_priv);
1047	if (unlikely(ret != 0)) {
1048		drm_err(&dev_priv->drm,
1049			"Failed initializing device caps.\n");
1050		goto out_no_vram;
1051	}
1052
1053	/*
1054	 * "Guest Memory Regions" is an aperture like feature with
1055	 *  one slot per bo. There is an upper limit of the number of
1056	 *  slots as well as the bo size.
1057	 */
1058	dev_priv->has_gmr = true;
1059	/* TODO: This is most likely not correct */
1060	if (((dev_priv->capabilities & (SVGA_CAP_GMR | SVGA_CAP_GMR2)) == 0) ||
1061	    refuse_dma ||
1062	    vmw_gmrid_man_init(dev_priv, VMW_PL_GMR) != 0) {
1063		drm_info(&dev_priv->drm,
1064			  "No GMR memory available. "
1065			 "Graphics memory resources are very limited.\n");
1066		dev_priv->has_gmr = false;
1067	}
1068
1069	if (dev_priv->capabilities & SVGA_CAP_GBOBJECTS && !refuse_dma) {
1070		dev_priv->has_mob = true;
1071
1072		if (vmw_gmrid_man_init(dev_priv, VMW_PL_MOB) != 0) {
1073			drm_info(&dev_priv->drm,
1074				 "No MOB memory available. "
1075				 "3D will be disabled.\n");
1076			dev_priv->has_mob = false;
1077		}
1078		if (vmw_sys_man_init(dev_priv) != 0) {
1079			drm_info(&dev_priv->drm,
1080				 "No MOB page table memory available. "
1081				 "3D will be disabled.\n");
1082			dev_priv->has_mob = false;
1083		}
1084	}
1085
1086	if (dev_priv->has_mob && (dev_priv->capabilities & SVGA_CAP_DX)) {
1087		if (vmw_devcap_get(dev_priv, SVGA3D_DEVCAP_DXCONTEXT))
1088			dev_priv->sm_type = VMW_SM_4;
1089	}
1090
1091	/* SVGA_CAP2_DX2 (DefineGBSurface_v3) is needed for SM4_1 support */
1092	if (has_sm4_context(dev_priv) &&
1093	    (dev_priv->capabilities2 & SVGA_CAP2_DX2)) {
1094		if (vmw_devcap_get(dev_priv, SVGA3D_DEVCAP_SM41))
1095			dev_priv->sm_type = VMW_SM_4_1;
1096		if (has_sm4_1_context(dev_priv) &&
1097				(dev_priv->capabilities2 & SVGA_CAP2_DX3)) {
1098			if (vmw_devcap_get(dev_priv, SVGA3D_DEVCAP_SM5)) {
1099				dev_priv->sm_type = VMW_SM_5;
1100				if (vmw_devcap_get(dev_priv, SVGA3D_DEVCAP_GL43))
1101					dev_priv->sm_type = VMW_SM_5_1X;
1102			}
1103		}
1104	}
1105
1106	ret = vmw_kms_init(dev_priv);
1107	if (unlikely(ret != 0))
1108		goto out_no_kms;
1109	vmw_overlay_init(dev_priv);
1110
1111	ret = vmw_request_device(dev_priv);
1112	if (ret)
1113		goto out_no_fifo;
1114
1115	vmw_print_sm_type(dev_priv);
1116	vmw_host_printf("vmwgfx: Module Version: %d.%d.%d (kernel: %s)",
1117			VMWGFX_DRIVER_MAJOR, VMWGFX_DRIVER_MINOR,
1118			VMWGFX_DRIVER_PATCHLEVEL, UTS_RELEASE);
1119	vmw_write_driver_id(dev_priv);
1120
1121	dev_priv->pm_nb.notifier_call = vmwgfx_pm_notifier;
1122	register_pm_notifier(&dev_priv->pm_nb);
1123
1124	return 0;
1125
1126out_no_fifo:
1127	vmw_overlay_close(dev_priv);
1128	vmw_kms_close(dev_priv);
1129out_no_kms:
1130	if (dev_priv->has_mob) {
1131		vmw_gmrid_man_fini(dev_priv, VMW_PL_MOB);
1132		vmw_sys_man_fini(dev_priv);
1133	}
1134	if (dev_priv->has_gmr)
1135		vmw_gmrid_man_fini(dev_priv, VMW_PL_GMR);
1136	vmw_devcaps_destroy(dev_priv);
1137	vmw_vram_manager_fini(dev_priv);
1138out_no_vram:
1139	ttm_device_fini(&dev_priv->bdev);
1140out_no_bdev:
1141	vmw_fence_manager_takedown(dev_priv->fman);
1142out_no_fman:
1143	if (dev_priv->capabilities & SVGA_CAP_IRQMASK)
1144		vmw_irq_uninstall(&dev_priv->drm);
1145out_no_irq:
1146	ttm_object_device_release(&dev_priv->tdev);
1147out_err0:
1148	for (i = vmw_res_context; i < vmw_res_max; ++i)
1149		idr_destroy(&dev_priv->res_idr[i]);
1150
1151	if (dev_priv->ctx.staged_bindings)
1152		vmw_binding_state_free(dev_priv->ctx.staged_bindings);
1153
1154	return ret;
1155}
1156
1157static void vmw_driver_unload(struct drm_device *dev)
1158{
1159	struct vmw_private *dev_priv = vmw_priv(dev);
1160	enum vmw_res_type i;
1161
1162	unregister_pm_notifier(&dev_priv->pm_nb);
1163
1164	vmw_sw_context_fini(dev_priv);
1165	vmw_fifo_resource_dec(dev_priv);
1166
1167	vmw_svga_disable(dev_priv);
1168
1169	vmw_vkms_cleanup(dev_priv);
1170	vmw_kms_close(dev_priv);
1171	vmw_overlay_close(dev_priv);
1172
1173	if (dev_priv->has_gmr)
1174		vmw_gmrid_man_fini(dev_priv, VMW_PL_GMR);
1175
1176	vmw_release_device_early(dev_priv);
1177	if (dev_priv->has_mob) {
1178		vmw_gmrid_man_fini(dev_priv, VMW_PL_MOB);
1179		vmw_sys_man_fini(dev_priv);
1180	}
1181	vmw_devcaps_destroy(dev_priv);
1182	vmw_vram_manager_fini(dev_priv);
1183	ttm_device_fini(&dev_priv->bdev);
1184	vmw_release_device_late(dev_priv);
1185	vmw_fence_manager_takedown(dev_priv->fman);
1186	if (dev_priv->capabilities & SVGA_CAP_IRQMASK)
1187		vmw_irq_uninstall(&dev_priv->drm);
1188
1189	ttm_object_device_release(&dev_priv->tdev);
1190
1191	for (i = vmw_res_context; i < vmw_res_max; ++i)
1192		idr_destroy(&dev_priv->res_idr[i]);
1193
1194	vmw_mksstat_remove_all(dev_priv);
1195}
1196
1197static void vmw_postclose(struct drm_device *dev,
1198			 struct drm_file *file_priv)
1199{
1200	struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
1201
1202	ttm_object_file_release(&vmw_fp->tfile);
1203	kfree(vmw_fp);
1204}
1205
1206static int vmw_driver_open(struct drm_device *dev, struct drm_file *file_priv)
1207{
1208	struct vmw_private *dev_priv = vmw_priv(dev);
1209	struct vmw_fpriv *vmw_fp;
1210	int ret = -ENOMEM;
1211
1212	vmw_fp = kzalloc(sizeof(*vmw_fp), GFP_KERNEL);
1213	if (unlikely(!vmw_fp))
1214		return ret;
1215
1216	vmw_fp->tfile = ttm_object_file_init(dev_priv->tdev);
1217	if (unlikely(vmw_fp->tfile == NULL))
1218		goto out_no_tfile;
1219
1220	file_priv->driver_priv = vmw_fp;
1221
1222	return 0;
1223
1224out_no_tfile:
1225	kfree(vmw_fp);
1226	return ret;
1227}
1228
1229static long vmw_generic_ioctl(struct file *filp, unsigned int cmd,
1230			      unsigned long arg,
1231			      long (*ioctl_func)(struct file *, unsigned int,
1232						 unsigned long))
1233{
1234	struct drm_file *file_priv = filp->private_data;
1235	struct drm_device *dev = file_priv->minor->dev;
1236	unsigned int nr = DRM_IOCTL_NR(cmd);
1237	unsigned int flags;
1238
1239	/*
1240	 * Do extra checking on driver private ioctls.
1241	 */
1242
1243	if ((nr >= DRM_COMMAND_BASE) && (nr < DRM_COMMAND_END)
1244	    && (nr < DRM_COMMAND_BASE + dev->driver->num_ioctls)) {
1245		const struct drm_ioctl_desc *ioctl =
1246			&vmw_ioctls[nr - DRM_COMMAND_BASE];
1247
1248		if (nr == DRM_COMMAND_BASE + DRM_VMW_EXECBUF) {
1249			return ioctl_func(filp, cmd, arg);
1250		} else if (nr == DRM_COMMAND_BASE + DRM_VMW_UPDATE_LAYOUT) {
1251			if (!drm_is_current_master(file_priv) &&
1252			    !capable(CAP_SYS_ADMIN))
1253				return -EACCES;
1254		}
1255
1256		if (unlikely(ioctl->cmd != cmd))
1257			goto out_io_encoding;
1258
1259		flags = ioctl->flags;
1260	} else if (!drm_ioctl_flags(nr, &flags))
1261		return -EINVAL;
1262
1263	return ioctl_func(filp, cmd, arg);
1264
1265out_io_encoding:
1266	DRM_ERROR("Invalid command format, ioctl %d\n",
1267		  nr - DRM_COMMAND_BASE);
1268
1269	return -EINVAL;
1270}
1271
1272static long vmw_unlocked_ioctl(struct file *filp, unsigned int cmd,
1273			       unsigned long arg)
1274{
1275	return vmw_generic_ioctl(filp, cmd, arg, &drm_ioctl);
1276}
1277
1278#ifdef CONFIG_COMPAT
1279static long vmw_compat_ioctl(struct file *filp, unsigned int cmd,
1280			     unsigned long arg)
1281{
1282	return vmw_generic_ioctl(filp, cmd, arg, &drm_compat_ioctl);
1283}
1284#endif
1285
1286static void vmw_master_set(struct drm_device *dev,
1287			   struct drm_file *file_priv,
1288			   bool from_open)
1289{
1290	/*
1291	 * Inform a new master that the layout may have changed while
1292	 * it was gone.
1293	 */
1294	if (!from_open)
1295		drm_sysfs_hotplug_event(dev);
1296}
1297
1298static void vmw_master_drop(struct drm_device *dev,
1299			    struct drm_file *file_priv)
1300{
1301}
1302
1303bool vmwgfx_supported(struct vmw_private *vmw)
1304{
1305#if defined(CONFIG_X86)
1306	return hypervisor_is_type(X86_HYPER_VMWARE);
1307#elif defined(CONFIG_ARM64)
1308	/*
1309	 * On aarch64 only svga3 is supported
1310	 */
1311	return vmw->pci_id == VMWGFX_PCI_ID_SVGA3;
1312#else
1313	drm_warn_once(&vmw->drm,
1314		      "vmwgfx is running on an unknown architecture.");
1315	return false;
1316#endif
1317}
1318
1319/**
1320 * __vmw_svga_enable - Enable SVGA mode, FIFO and use of VRAM.
1321 *
1322 * @dev_priv: Pointer to device private struct.
1323 * Needs the reservation sem to be held in non-exclusive mode.
1324 */
1325static void __vmw_svga_enable(struct vmw_private *dev_priv)
1326{
1327	struct ttm_resource_manager *man = ttm_manager_type(&dev_priv->bdev, TTM_PL_VRAM);
1328
1329	if (!ttm_resource_manager_used(man)) {
1330		vmw_write(dev_priv, SVGA_REG_ENABLE, SVGA_REG_ENABLE_ENABLE);
1331		ttm_resource_manager_set_used(man, true);
1332	}
1333}
1334
1335/**
1336 * vmw_svga_enable - Enable SVGA mode, FIFO and use of VRAM.
1337 *
1338 * @dev_priv: Pointer to device private struct.
1339 */
1340void vmw_svga_enable(struct vmw_private *dev_priv)
1341{
1342	__vmw_svga_enable(dev_priv);
1343}
1344
1345/**
1346 * __vmw_svga_disable - Disable SVGA mode and use of VRAM.
1347 *
1348 * @dev_priv: Pointer to device private struct.
1349 * Needs the reservation sem to be held in exclusive mode.
1350 * Will not empty VRAM. VRAM must be emptied by caller.
1351 */
1352static void __vmw_svga_disable(struct vmw_private *dev_priv)
1353{
1354	struct ttm_resource_manager *man = ttm_manager_type(&dev_priv->bdev, TTM_PL_VRAM);
1355
1356	if (ttm_resource_manager_used(man)) {
1357		ttm_resource_manager_set_used(man, false);
1358		vmw_write(dev_priv, SVGA_REG_ENABLE,
1359			  SVGA_REG_ENABLE_HIDE |
1360			  SVGA_REG_ENABLE_ENABLE);
1361	}
1362}
1363
1364/**
1365 * vmw_svga_disable - Disable SVGA_MODE, and use of VRAM. Keep the fifo
1366 * running.
1367 *
1368 * @dev_priv: Pointer to device private struct.
1369 * Will empty VRAM.
1370 */
1371void vmw_svga_disable(struct vmw_private *dev_priv)
1372{
1373	struct ttm_resource_manager *man = ttm_manager_type(&dev_priv->bdev, TTM_PL_VRAM);
1374	/*
1375	 * Disabling SVGA will turn off device modesetting capabilities, so
1376	 * notify KMS about that so that it doesn't cache atomic state that
1377	 * isn't valid anymore, for example crtcs turned on.
1378	 * Strictly we'd want to do this under the SVGA lock (or an SVGA mutex),
1379	 * but vmw_kms_lost_device() takes the reservation sem and thus we'll
1380	 * end up with lock order reversal. Thus, a master may actually perform
1381	 * a new modeset just after we call vmw_kms_lost_device() and race with
1382	 * vmw_svga_disable(), but that should at worst cause atomic KMS state
1383	 * to be inconsistent with the device, causing modesetting problems.
1384	 *
1385	 */
1386	vmw_kms_lost_device(&dev_priv->drm);
1387	if (ttm_resource_manager_used(man)) {
1388		if (ttm_resource_manager_evict_all(&dev_priv->bdev, man))
1389			DRM_ERROR("Failed evicting VRAM buffers.\n");
1390		ttm_resource_manager_set_used(man, false);
1391		vmw_write(dev_priv, SVGA_REG_ENABLE,
1392			  SVGA_REG_ENABLE_HIDE |
1393			  SVGA_REG_ENABLE_ENABLE);
1394	}
1395}
1396
1397static void vmw_remove(struct pci_dev *pdev)
1398{
1399	struct drm_device *dev = pci_get_drvdata(pdev);
1400
1401	drm_dev_unregister(dev);
1402	vmw_driver_unload(dev);
1403}
1404
1405static void vmw_debugfs_resource_managers_init(struct vmw_private *vmw)
1406{
1407	struct drm_minor *minor = vmw->drm.primary;
1408	struct dentry *root = minor->debugfs_root;
1409
1410	ttm_resource_manager_create_debugfs(ttm_manager_type(&vmw->bdev, TTM_PL_SYSTEM),
1411					    root, "system_ttm");
1412	ttm_resource_manager_create_debugfs(ttm_manager_type(&vmw->bdev, TTM_PL_VRAM),
1413					    root, "vram_ttm");
1414	if (vmw->has_gmr)
1415		ttm_resource_manager_create_debugfs(ttm_manager_type(&vmw->bdev, VMW_PL_GMR),
1416						    root, "gmr_ttm");
1417	if (vmw->has_mob) {
1418		ttm_resource_manager_create_debugfs(ttm_manager_type(&vmw->bdev, VMW_PL_MOB),
1419						    root, "mob_ttm");
1420		ttm_resource_manager_create_debugfs(ttm_manager_type(&vmw->bdev, VMW_PL_SYSTEM),
1421						    root, "system_mob_ttm");
1422	}
1423}
1424
1425static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
1426			      void *ptr)
1427{
1428	struct vmw_private *dev_priv =
1429		container_of(nb, struct vmw_private, pm_nb);
1430
1431	switch (val) {
1432	case PM_HIBERNATION_PREPARE:
1433		/*
1434		 * Take the reservation sem in write mode, which will make sure
1435		 * there are no other processes holding a buffer object
1436		 * reservation, meaning we should be able to evict all buffer
1437		 * objects if needed.
1438		 * Once user-space processes have been frozen, we can release
1439		 * the lock again.
1440		 */
1441		dev_priv->suspend_locked = true;
1442		break;
1443	case PM_POST_HIBERNATION:
1444	case PM_POST_RESTORE:
1445		if (READ_ONCE(dev_priv->suspend_locked)) {
1446			dev_priv->suspend_locked = false;
1447		}
1448		break;
1449	default:
1450		break;
1451	}
1452	return 0;
1453}
1454
1455static int vmw_pci_suspend(struct pci_dev *pdev, pm_message_t state)
1456{
1457	struct drm_device *dev = pci_get_drvdata(pdev);
1458	struct vmw_private *dev_priv = vmw_priv(dev);
1459
1460	if (dev_priv->refuse_hibernation)
1461		return -EBUSY;
1462
1463	pci_save_state(pdev);
1464	pci_disable_device(pdev);
1465	pci_set_power_state(pdev, PCI_D3hot);
1466	return 0;
1467}
1468
1469static int vmw_pci_resume(struct pci_dev *pdev)
1470{
1471	pci_set_power_state(pdev, PCI_D0);
1472	pci_restore_state(pdev);
1473	return pci_enable_device(pdev);
1474}
1475
1476static int vmw_pm_suspend(struct device *kdev)
1477{
1478	struct pci_dev *pdev = to_pci_dev(kdev);
1479	struct pm_message dummy;
1480
1481	dummy.event = 0;
1482
1483	return vmw_pci_suspend(pdev, dummy);
1484}
1485
1486static int vmw_pm_resume(struct device *kdev)
1487{
1488	struct pci_dev *pdev = to_pci_dev(kdev);
1489
1490	return vmw_pci_resume(pdev);
1491}
1492
1493static int vmw_pm_freeze(struct device *kdev)
1494{
1495	struct pci_dev *pdev = to_pci_dev(kdev);
1496	struct drm_device *dev = pci_get_drvdata(pdev);
1497	struct vmw_private *dev_priv = vmw_priv(dev);
1498	struct ttm_operation_ctx ctx = {
1499		.interruptible = false,
1500		.no_wait_gpu = false
1501	};
1502	int ret;
1503
1504	/*
1505	 * No user-space processes should be running now.
1506	 */
1507	ret = vmw_kms_suspend(&dev_priv->drm);
1508	if (ret) {
1509		DRM_ERROR("Failed to freeze modesetting.\n");
1510		return ret;
1511	}
1512
1513	vmw_execbuf_release_pinned_bo(dev_priv);
1514	vmw_resource_evict_all(dev_priv);
1515	vmw_release_device_early(dev_priv);
1516	while (ttm_device_swapout(&dev_priv->bdev, &ctx, GFP_KERNEL) > 0);
1517	vmw_fifo_resource_dec(dev_priv);
1518	if (atomic_read(&dev_priv->num_fifo_resources) != 0) {
1519		DRM_ERROR("Can't hibernate while 3D resources are active.\n");
1520		vmw_fifo_resource_inc(dev_priv);
1521		WARN_ON(vmw_request_device_late(dev_priv));
1522		dev_priv->suspend_locked = false;
1523		if (dev_priv->suspend_state)
1524			vmw_kms_resume(dev);
1525		return -EBUSY;
1526	}
1527
1528	vmw_fence_fifo_down(dev_priv->fman);
1529	__vmw_svga_disable(dev_priv);
1530
1531	vmw_release_device_late(dev_priv);
1532	return 0;
1533}
1534
1535static int vmw_pm_restore(struct device *kdev)
1536{
1537	struct pci_dev *pdev = to_pci_dev(kdev);
1538	struct drm_device *dev = pci_get_drvdata(pdev);
1539	struct vmw_private *dev_priv = vmw_priv(dev);
1540	int ret;
1541
1542	vmw_detect_version(dev_priv);
1543
1544	vmw_fifo_resource_inc(dev_priv);
1545
1546	ret = vmw_request_device(dev_priv);
1547	if (ret)
1548		return ret;
1549
1550	__vmw_svga_enable(dev_priv);
1551
1552	vmw_fence_fifo_up(dev_priv->fman);
1553	dev_priv->suspend_locked = false;
1554	if (dev_priv->suspend_state)
1555		vmw_kms_resume(&dev_priv->drm);
1556
1557	return 0;
1558}
1559
1560static const struct dev_pm_ops vmw_pm_ops = {
1561	.freeze = vmw_pm_freeze,
1562	.thaw = vmw_pm_restore,
1563	.restore = vmw_pm_restore,
1564	.suspend = vmw_pm_suspend,
1565	.resume = vmw_pm_resume,
1566};
1567
1568static const struct file_operations vmwgfx_driver_fops = {
1569	.owner = THIS_MODULE,
1570	.open = drm_open,
1571	.release = drm_release,
1572	.unlocked_ioctl = vmw_unlocked_ioctl,
1573	.mmap = drm_gem_mmap,
1574	.poll = drm_poll,
1575	.read = drm_read,
1576#if defined(CONFIG_COMPAT)
1577	.compat_ioctl = vmw_compat_ioctl,
1578#endif
1579	.llseek = noop_llseek,
1580	.fop_flags = FOP_UNSIGNED_OFFSET,
1581};
1582
1583static const struct drm_driver driver = {
1584	.driver_features =
1585	DRIVER_MODESET | DRIVER_RENDER | DRIVER_ATOMIC | DRIVER_GEM | DRIVER_CURSOR_HOTSPOT,
1586	.ioctls = vmw_ioctls,
1587	.num_ioctls = ARRAY_SIZE(vmw_ioctls),
1588	.master_set = vmw_master_set,
1589	.master_drop = vmw_master_drop,
1590	.open = vmw_driver_open,
1591	.postclose = vmw_postclose,
1592
1593	.dumb_create = vmw_dumb_create,
1594	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
1595
1596	.prime_fd_to_handle = vmw_prime_fd_to_handle,
1597	.prime_handle_to_fd = vmw_prime_handle_to_fd,
1598	.gem_prime_import_sg_table = vmw_prime_import_sg_table,
1599
1600	DRM_FBDEV_TTM_DRIVER_OPS,
1601
1602	.fops = &vmwgfx_driver_fops,
1603	.name = VMWGFX_DRIVER_NAME,
1604	.desc = VMWGFX_DRIVER_DESC,
1605	.major = VMWGFX_DRIVER_MAJOR,
1606	.minor = VMWGFX_DRIVER_MINOR,
1607	.patchlevel = VMWGFX_DRIVER_PATCHLEVEL
1608};
1609
1610static struct pci_driver vmw_pci_driver = {
1611	.name = VMWGFX_DRIVER_NAME,
1612	.id_table = vmw_pci_id_list,
1613	.probe = vmw_probe,
1614	.remove = vmw_remove,
1615	.driver = {
1616		.pm = &vmw_pm_ops
1617	}
1618};
1619
1620static int vmw_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1621{
1622	struct vmw_private *vmw;
1623	int ret;
1624
1625	ret = aperture_remove_conflicting_pci_devices(pdev, driver.name);
1626	if (ret)
1627		goto out_error;
1628
1629	ret = pcim_enable_device(pdev);
1630	if (ret)
1631		goto out_error;
1632
1633	vmw = devm_drm_dev_alloc(&pdev->dev, &driver,
1634				 struct vmw_private, drm);
1635	if (IS_ERR(vmw)) {
1636		ret = PTR_ERR(vmw);
1637		goto out_error;
1638	}
1639
1640	pci_set_drvdata(pdev, &vmw->drm);
1641
1642	ret = vmw_driver_load(vmw, ent->device);
1643	if (ret)
1644		goto out_error;
1645
1646	ret = drm_dev_register(&vmw->drm, 0);
1647	if (ret)
1648		goto out_unload;
1649
1650	vmw_fifo_resource_inc(vmw);
1651	vmw_svga_enable(vmw);
1652	drm_client_setup(&vmw->drm, NULL);
1653
1654	vmw_debugfs_gem_init(vmw);
1655	vmw_debugfs_resource_managers_init(vmw);
1656
1657	return 0;
1658out_unload:
1659	vmw_driver_unload(&vmw->drm);
1660out_error:
1661	return ret;
1662}
1663
1664drm_module_pci_driver(vmw_pci_driver);
1665
1666MODULE_AUTHOR("VMware Inc. and others");
1667MODULE_DESCRIPTION("Standalone drm driver for the VMware SVGA device");
1668MODULE_LICENSE("GPL and additional rights");
1669MODULE_VERSION(__stringify(VMWGFX_DRIVER_MAJOR) "."
1670	       __stringify(VMWGFX_DRIVER_MINOR) "."
1671	       __stringify(VMWGFX_DRIVER_PATCHLEVEL) "."
1672	       "0");