sev-dev.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * AMD Secure Encrypted Virtualization (SEV) interface
   4 *
   5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
   6 *
   7 * Author: Brijesh Singh <brijesh.singh@amd.com>
   8 */
   9
  10#include <linux/bitfield.h>
  11#include <linux/module.h>
  12#include <linux/kernel.h>
  13#include <linux/kthread.h>
  14#include <linux/sched.h>
  15#include <linux/interrupt.h>
  16#include <linux/spinlock.h>
  17#include <linux/spinlock_types.h>
  18#include <linux/types.h>
  19#include <linux/mutex.h>
  20#include <linux/delay.h>
  21#include <linux/hw_random.h>
  22#include <linux/ccp.h>
  23#include <linux/firmware.h>
  24#include <linux/panic_notifier.h>
  25#include <linux/gfp.h>
  26#include <linux/cpufeature.h>
  27#include <linux/fs.h>
  28#include <linux/fs_struct.h>
  29#include <linux/psp.h>
  30#include <linux/amd-iommu.h>
  31#include <linux/crash_dump.h>
  32
  33#include <asm/smp.h>
  34#include <asm/cacheflush.h>
  35#include <asm/e820/types.h>
  36#include <asm/sev.h>
  37#include <asm/msr.h>
  38
  39#include "psp-dev.h"
  40#include "sev-dev.h"
  41
  42#define DEVICE_NAME		"sev"
  43#define SEV_FW_FILE		"amd/sev.fw"
  44#define SEV_FW_NAME_SIZE	64
  45
  46/* Minimum firmware version required for the SEV-SNP support */
  47#define SNP_MIN_API_MAJOR	1
  48#define SNP_MIN_API_MINOR	51
  49
  50/*
  51 * Maximum number of firmware-writable buffers that might be specified
  52 * in the parameters of a legacy SEV command buffer.
  53 */
  54#define CMD_BUF_FW_WRITABLE_MAX 2
  55
  56/* Leave room in the descriptor array for an end-of-list indicator. */
  57#define CMD_BUF_DESC_MAX (CMD_BUF_FW_WRITABLE_MAX + 1)
  58
  59static DEFINE_MUTEX(sev_cmd_mutex);
  60static struct sev_misc_dev *misc_dev;
  61
  62static int psp_cmd_timeout = 100;
  63module_param(psp_cmd_timeout, int, 0644);
  64MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands");
  65
  66static int psp_probe_timeout = 5;
  67module_param(psp_probe_timeout, int, 0644);
  68MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
  69
  70static char *init_ex_path;
  71module_param(init_ex_path, charp, 0444);
  72MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
  73
  74static bool psp_init_on_probe = true;
  75module_param(psp_init_on_probe, bool, 0444);
  76MODULE_PARM_DESC(psp_init_on_probe, "  if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
  77
  78#if IS_ENABLED(CONFIG_PCI_TSM)
  79static bool sev_tio_enabled = true;
  80module_param_named(tio, sev_tio_enabled, bool, 0444);
  81MODULE_PARM_DESC(tio, "Enables TIO in SNP_INIT_EX");
  82#else
  83static const bool sev_tio_enabled = false;
  84#endif
  85
  86MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
  87MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
  88MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
  89MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */
  90
  91static bool psp_dead;
  92static int psp_timeout;
  93
  94enum snp_hv_fixed_pages_state {
  95	ALLOCATED,
  96	HV_FIXED,
  97};
  98
  99struct snp_hv_fixed_pages_entry {
 100	struct list_head list;
 101	struct page *page;
 102	unsigned int order;
 103	bool free;
 104	enum snp_hv_fixed_pages_state page_state;
 105};
 106
 107static LIST_HEAD(snp_hv_fixed_pages);
 108
 109/* Trusted Memory Region (TMR):
 110 *   The TMR is a 1MB area that must be 1MB aligned.  Use the page allocator
 111 *   to allocate the memory, which will return aligned memory for the specified
 112 *   allocation order.
 113 *
 114 * When SEV-SNP is enabled the TMR needs to be 2MB aligned and 2MB sized.
 115 */
 116#define SEV_TMR_SIZE		(1024 * 1024)
 117#define SNP_TMR_SIZE		(2 * 1024 * 1024)
 118
 119static void *sev_es_tmr;
 120static size_t sev_es_tmr_size = SEV_TMR_SIZE;
 121
 122/* INIT_EX NV Storage:
 123 *   The NV Storage is a 32Kb area and must be 4Kb page aligned.  Use the page
 124 *   allocator to allocate the memory, which will return aligned memory for the
 125 *   specified allocation order.
 126 */
 127#define NV_LENGTH (32 * 1024)
 128static void *sev_init_ex_buffer;
 129
 130static void __sev_firmware_shutdown(struct sev_device *sev, bool panic);
 131
 132static int snp_shutdown_on_panic(struct notifier_block *nb,
 133				 unsigned long reason, void *arg);
 134
 135static struct notifier_block snp_panic_notifier = {
 136	.notifier_call = snp_shutdown_on_panic,
 137};
 138
 139static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
 140{
 141	struct sev_device *sev = psp_master->sev_data;
 142
 143	if (sev->api_major > maj)
 144		return true;
 145
 146	if (sev->api_major == maj && sev->api_minor >= min)
 147		return true;
 148
 149	return false;
 150}
 151
 152static void sev_irq_handler(int irq, void *data, unsigned int status)
 153{
 154	struct sev_device *sev = data;
 155	int reg;
 156
 157	/* Check if it is command completion: */
 158	if (!(status & SEV_CMD_COMPLETE))
 159		return;
 160
 161	/* Check if it is SEV command completion: */
 162	reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
 163	if (FIELD_GET(PSP_CMDRESP_RESP, reg)) {
 164		sev->int_rcvd = 1;
 165		wake_up(&sev->int_queue);
 166	}
 167}
 168
 169static int sev_wait_cmd_ioc(struct sev_device *sev,
 170			    unsigned int *reg, unsigned int timeout)
 171{
 172	int ret;
 173
 174	/*
 175	 * If invoked during panic handling, local interrupts are disabled,
 176	 * so the PSP command completion interrupt can't be used. Poll for
 177	 * PSP command completion instead.
 178	 */
 179	if (irqs_disabled()) {
 180		unsigned long timeout_usecs = (timeout * USEC_PER_SEC) / 10;
 181
 182		/* Poll for SEV command completion: */
 183		while (timeout_usecs--) {
 184			*reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
 185			if (*reg & PSP_CMDRESP_RESP)
 186				return 0;
 187
 188			udelay(10);
 189		}
 190		return -ETIMEDOUT;
 191	}
 192
 193	ret = wait_event_timeout(sev->int_queue,
 194			sev->int_rcvd, timeout * HZ);
 195	if (!ret)
 196		return -ETIMEDOUT;
 197
 198	*reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
 199
 200	return 0;
 201}
 202
 203static int sev_cmd_buffer_len(int cmd)
 204{
 205	switch (cmd) {
 206	case SEV_CMD_INIT:			return sizeof(struct sev_data_init);
 207	case SEV_CMD_INIT_EX:                   return sizeof(struct sev_data_init_ex);
 208	case SEV_CMD_SNP_SHUTDOWN_EX:		return sizeof(struct sev_data_snp_shutdown_ex);
 209	case SEV_CMD_SNP_INIT_EX:		return sizeof(struct sev_data_snp_init_ex);
 210	case SEV_CMD_PLATFORM_STATUS:		return sizeof(struct sev_user_data_status);
 211	case SEV_CMD_PEK_CSR:			return sizeof(struct sev_data_pek_csr);
 212	case SEV_CMD_PEK_CERT_IMPORT:		return sizeof(struct sev_data_pek_cert_import);
 213	case SEV_CMD_PDH_CERT_EXPORT:		return sizeof(struct sev_data_pdh_cert_export);
 214	case SEV_CMD_LAUNCH_START:		return sizeof(struct sev_data_launch_start);
 215	case SEV_CMD_LAUNCH_UPDATE_DATA:	return sizeof(struct sev_data_launch_update_data);
 216	case SEV_CMD_LAUNCH_UPDATE_VMSA:	return sizeof(struct sev_data_launch_update_vmsa);
 217	case SEV_CMD_LAUNCH_FINISH:		return sizeof(struct sev_data_launch_finish);
 218	case SEV_CMD_LAUNCH_MEASURE:		return sizeof(struct sev_data_launch_measure);
 219	case SEV_CMD_ACTIVATE:			return sizeof(struct sev_data_activate);
 220	case SEV_CMD_DEACTIVATE:		return sizeof(struct sev_data_deactivate);
 221	case SEV_CMD_DECOMMISSION:		return sizeof(struct sev_data_decommission);
 222	case SEV_CMD_GUEST_STATUS:		return sizeof(struct sev_data_guest_status);
 223	case SEV_CMD_DBG_DECRYPT:		return sizeof(struct sev_data_dbg);
 224	case SEV_CMD_DBG_ENCRYPT:		return sizeof(struct sev_data_dbg);
 225	case SEV_CMD_SEND_START:		return sizeof(struct sev_data_send_start);
 226	case SEV_CMD_SEND_UPDATE_DATA:		return sizeof(struct sev_data_send_update_data);
 227	case SEV_CMD_SEND_UPDATE_VMSA:		return sizeof(struct sev_data_send_update_vmsa);
 228	case SEV_CMD_SEND_FINISH:		return sizeof(struct sev_data_send_finish);
 229	case SEV_CMD_RECEIVE_START:		return sizeof(struct sev_data_receive_start);
 230	case SEV_CMD_RECEIVE_FINISH:		return sizeof(struct sev_data_receive_finish);
 231	case SEV_CMD_RECEIVE_UPDATE_DATA:	return sizeof(struct sev_data_receive_update_data);
 232	case SEV_CMD_RECEIVE_UPDATE_VMSA:	return sizeof(struct sev_data_receive_update_vmsa);
 233	case SEV_CMD_LAUNCH_UPDATE_SECRET:	return sizeof(struct sev_data_launch_secret);
 234	case SEV_CMD_DOWNLOAD_FIRMWARE:		return sizeof(struct sev_data_download_firmware);
 235	case SEV_CMD_GET_ID:			return sizeof(struct sev_data_get_id);
 236	case SEV_CMD_ATTESTATION_REPORT:	return sizeof(struct sev_data_attestation_report);
 237	case SEV_CMD_SEND_CANCEL:		return sizeof(struct sev_data_send_cancel);
 238	case SEV_CMD_SNP_GCTX_CREATE:		return sizeof(struct sev_data_snp_addr);
 239	case SEV_CMD_SNP_LAUNCH_START:		return sizeof(struct sev_data_snp_launch_start);
 240	case SEV_CMD_SNP_LAUNCH_UPDATE:		return sizeof(struct sev_data_snp_launch_update);
 241	case SEV_CMD_SNP_ACTIVATE:		return sizeof(struct sev_data_snp_activate);
 242	case SEV_CMD_SNP_DECOMMISSION:		return sizeof(struct sev_data_snp_addr);
 243	case SEV_CMD_SNP_PAGE_RECLAIM:		return sizeof(struct sev_data_snp_page_reclaim);
 244	case SEV_CMD_SNP_GUEST_STATUS:		return sizeof(struct sev_data_snp_guest_status);
 245	case SEV_CMD_SNP_LAUNCH_FINISH:		return sizeof(struct sev_data_snp_launch_finish);
 246	case SEV_CMD_SNP_DBG_DECRYPT:		return sizeof(struct sev_data_snp_dbg);
 247	case SEV_CMD_SNP_DBG_ENCRYPT:		return sizeof(struct sev_data_snp_dbg);
 248	case SEV_CMD_SNP_PAGE_UNSMASH:		return sizeof(struct sev_data_snp_page_unsmash);
 249	case SEV_CMD_SNP_PLATFORM_STATUS:	return sizeof(struct sev_data_snp_addr);
 250	case SEV_CMD_SNP_GUEST_REQUEST:		return sizeof(struct sev_data_snp_guest_request);
 251	case SEV_CMD_SNP_CONFIG:		return sizeof(struct sev_user_data_snp_config);
 252	case SEV_CMD_SNP_COMMIT:		return sizeof(struct sev_data_snp_commit);
 253	case SEV_CMD_SNP_FEATURE_INFO:		return sizeof(struct sev_data_snp_feature_info);
 254	case SEV_CMD_SNP_VLEK_LOAD:		return sizeof(struct sev_user_data_snp_vlek_load);
 255	default:				return sev_tio_cmd_buffer_len(cmd);
 256	}
 257
 258	return 0;
 259}
 260
 261static struct file *open_file_as_root(const char *filename, int flags, umode_t mode)
 262{
 263	struct path root __free(path_put) = {};
 264
 265	task_lock(&init_task);
 266	get_fs_root(init_task.fs, &root);
 267	task_unlock(&init_task);
 268
 269	CLASS(prepare_creds, cred)();
 270	if (!cred)
 271		return ERR_PTR(-ENOMEM);
 272
 273	cred->fsuid = GLOBAL_ROOT_UID;
 274
 275	scoped_with_creds(cred)
 276		return file_open_root(&root, filename, flags, mode);
 277}
 278
 279static int sev_read_init_ex_file(void)
 280{
 281	struct sev_device *sev = psp_master->sev_data;
 282	struct file *fp;
 283	ssize_t nread;
 284
 285	lockdep_assert_held(&sev_cmd_mutex);
 286
 287	if (!sev_init_ex_buffer)
 288		return -EOPNOTSUPP;
 289
 290	fp = open_file_as_root(init_ex_path, O_RDONLY, 0);
 291	if (IS_ERR(fp)) {
 292		int ret = PTR_ERR(fp);
 293
 294		if (ret == -ENOENT) {
 295			dev_info(sev->dev,
 296				"SEV: %s does not exist and will be created later.\n",
 297				init_ex_path);
 298			ret = 0;
 299		} else {
 300			dev_err(sev->dev,
 301				"SEV: could not open %s for read, error %d\n",
 302				init_ex_path, ret);
 303		}
 304		return ret;
 305	}
 306
 307	nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
 308	if (nread != NV_LENGTH) {
 309		dev_info(sev->dev,
 310			"SEV: could not read %u bytes to non volatile memory area, ret %ld\n",
 311			NV_LENGTH, nread);
 312	}
 313
 314	dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
 315	filp_close(fp, NULL);
 316
 317	return 0;
 318}
 319
 320static int sev_write_init_ex_file(void)
 321{
 322	struct sev_device *sev = psp_master->sev_data;
 323	struct file *fp;
 324	loff_t offset = 0;
 325	ssize_t nwrite;
 326
 327	lockdep_assert_held(&sev_cmd_mutex);
 328
 329	if (!sev_init_ex_buffer)
 330		return 0;
 331
 332	fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600);
 333	if (IS_ERR(fp)) {
 334		int ret = PTR_ERR(fp);
 335
 336		dev_err(sev->dev,
 337			"SEV: could not open file for write, error %d\n",
 338			ret);
 339		return ret;
 340	}
 341
 342	nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
 343	vfs_fsync(fp, 0);
 344	filp_close(fp, NULL);
 345
 346	if (nwrite != NV_LENGTH) {
 347		dev_err(sev->dev,
 348			"SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
 349			NV_LENGTH, nwrite);
 350		return -EIO;
 351	}
 352
 353	dev_dbg(sev->dev, "SEV: write successful to NV file\n");
 354
 355	return 0;
 356}
 357
 358static int sev_write_init_ex_file_if_required(int cmd_id)
 359{
 360	lockdep_assert_held(&sev_cmd_mutex);
 361
 362	if (!sev_init_ex_buffer)
 363		return 0;
 364
 365	/*
 366	 * Only a few platform commands modify the SPI/NV area, but none of the
 367	 * non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
 368	 * PEK_CERT_IMPORT, and PDH_GEN do.
 369	 */
 370	switch (cmd_id) {
 371	case SEV_CMD_FACTORY_RESET:
 372	case SEV_CMD_INIT_EX:
 373	case SEV_CMD_PDH_GEN:
 374	case SEV_CMD_PEK_CERT_IMPORT:
 375	case SEV_CMD_PEK_GEN:
 376		break;
 377	default:
 378		return 0;
 379	}
 380
 381	return sev_write_init_ex_file();
 382}
 383
 384int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked)
 385{
 386	int ret, err, i;
 387
 388	paddr = __sme_clr(ALIGN_DOWN(paddr, PAGE_SIZE));
 389
 390	for (i = 0; i < npages; i++, paddr += PAGE_SIZE) {
 391		struct sev_data_snp_page_reclaim data = {0};
 392
 393		data.paddr = paddr;
 394
 395		if (locked)
 396			ret = __sev_do_cmd_locked(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
 397		else
 398			ret = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
 399
 400		if (ret)
 401			goto cleanup;
 402
 403		ret = rmp_make_shared(__phys_to_pfn(paddr), PG_LEVEL_4K);
 404		if (ret)
 405			goto cleanup;
 406	}
 407
 408	return 0;
 409
 410cleanup:
 411	/*
 412	 * If there was a failure reclaiming the page then it is no longer safe
 413	 * to release it back to the system; leak it instead.
 414	 */
 415	snp_leak_pages(__phys_to_pfn(paddr), npages - i);
 416	return ret;
 417}
 418EXPORT_SYMBOL_GPL(snp_reclaim_pages);
 419
 420static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked)
 421{
 422	unsigned long pfn = __sme_clr(paddr) >> PAGE_SHIFT;
 423	int rc, i;
 424
 425	for (i = 0; i < npages; i++, pfn++) {
 426		rc = rmp_make_private(pfn, 0, PG_LEVEL_4K, 0, true);
 427		if (rc)
 428			goto cleanup;
 429	}
 430
 431	return 0;
 432
 433cleanup:
 434	/*
 435	 * Try unrolling the firmware state changes by
 436	 * reclaiming the pages which were already changed to the
 437	 * firmware state.
 438	 */
 439	snp_reclaim_pages(paddr, i, locked);
 440
 441	return rc;
 442}
 443
 444static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked)
 445{
 446	unsigned long npages = 1ul << order, paddr;
 447	struct sev_device *sev;
 448	struct page *page;
 449
 450	if (!psp_master || !psp_master->sev_data)
 451		return NULL;
 452
 453	page = alloc_pages(gfp_mask, order);
 454	if (!page)
 455		return NULL;
 456
 457	/* If SEV-SNP is initialized then add the page in RMP table. */
 458	sev = psp_master->sev_data;
 459	if (!sev->snp_initialized)
 460		return page;
 461
 462	paddr = __pa((unsigned long)page_address(page));
 463	if (rmp_mark_pages_firmware(paddr, npages, locked))
 464		return NULL;
 465
 466	return page;
 467}
 468
 469void *snp_alloc_firmware_page(gfp_t gfp_mask)
 470{
 471	struct page *page;
 472
 473	page = __snp_alloc_firmware_pages(gfp_mask, 0, false);
 474
 475	return page ? page_address(page) : NULL;
 476}
 477EXPORT_SYMBOL_GPL(snp_alloc_firmware_page);
 478
 479static void __snp_free_firmware_pages(struct page *page, int order, bool locked)
 480{
 481	struct sev_device *sev = psp_master->sev_data;
 482	unsigned long paddr, npages = 1ul << order;
 483
 484	if (!page)
 485		return;
 486
 487	paddr = __pa((unsigned long)page_address(page));
 488	if (sev->snp_initialized &&
 489	    snp_reclaim_pages(paddr, npages, locked))
 490		return;
 491
 492	__free_pages(page, order);
 493}
 494
 495void snp_free_firmware_page(void *addr)
 496{
 497	if (!addr)
 498		return;
 499
 500	__snp_free_firmware_pages(virt_to_page(addr), 0, false);
 501}
 502EXPORT_SYMBOL_GPL(snp_free_firmware_page);
 503
 504static void *sev_fw_alloc(unsigned long len)
 505{
 506	struct page *page;
 507
 508	page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true);
 509	if (!page)
 510		return NULL;
 511
 512	return page_address(page);
 513}
 514
 515/**
 516 * struct cmd_buf_desc - descriptors for managing legacy SEV command address
 517 * parameters corresponding to buffers that may be written to by firmware.
 518 *
 519 * @paddr_ptr:  pointer to the address parameter in the command buffer which may
 520 *              need to be saved/restored depending on whether a bounce buffer
 521 *              is used. In the case of a bounce buffer, the command buffer
 522 *              needs to be updated with the address of the new bounce buffer
 523 *              snp_map_cmd_buf_desc() has allocated specifically for it. Must
 524 *              be NULL if this descriptor is only an end-of-list indicator.
 525 *
 526 * @paddr_orig: storage for the original address parameter, which can be used to
 527 *              restore the original value in @paddr_ptr in cases where it is
 528 *              replaced with the address of a bounce buffer.
 529 *
 530 * @len: length of buffer located at the address originally stored at @paddr_ptr
 531 *
 532 * @guest_owned: true if the address corresponds to guest-owned pages, in which
 533 *               case bounce buffers are not needed.
 534 */
 535struct cmd_buf_desc {
 536	u64 *paddr_ptr;
 537	u64 paddr_orig;
 538	u32 len;
 539	bool guest_owned;
 540};
 541
 542/*
 543 * If a legacy SEV command parameter is a memory address, those pages in
 544 * turn need to be transitioned to/from firmware-owned before/after
 545 * executing the firmware command.
 546 *
 547 * Additionally, in cases where those pages are not guest-owned, a bounce
 548 * buffer is needed in place of the original memory address parameter.
 549 *
 550 * A set of descriptors are used to keep track of this handling, and
 551 * initialized here based on the specific commands being executed.
 552 */
 553static void snp_populate_cmd_buf_desc_list(int cmd, void *cmd_buf,
 554					   struct cmd_buf_desc *desc_list)
 555{
 556	switch (cmd) {
 557	case SEV_CMD_PDH_CERT_EXPORT: {
 558		struct sev_data_pdh_cert_export *data = cmd_buf;
 559
 560		desc_list[0].paddr_ptr = &data->pdh_cert_address;
 561		desc_list[0].len = data->pdh_cert_len;
 562		desc_list[1].paddr_ptr = &data->cert_chain_address;
 563		desc_list[1].len = data->cert_chain_len;
 564		break;
 565	}
 566	case SEV_CMD_GET_ID: {
 567		struct sev_data_get_id *data = cmd_buf;
 568
 569		desc_list[0].paddr_ptr = &data->address;
 570		desc_list[0].len = data->len;
 571		break;
 572	}
 573	case SEV_CMD_PEK_CSR: {
 574		struct sev_data_pek_csr *data = cmd_buf;
 575
 576		desc_list[0].paddr_ptr = &data->address;
 577		desc_list[0].len = data->len;
 578		break;
 579	}
 580	case SEV_CMD_LAUNCH_UPDATE_DATA: {
 581		struct sev_data_launch_update_data *data = cmd_buf;
 582
 583		desc_list[0].paddr_ptr = &data->address;
 584		desc_list[0].len = data->len;
 585		desc_list[0].guest_owned = true;
 586		break;
 587	}
 588	case SEV_CMD_LAUNCH_UPDATE_VMSA: {
 589		struct sev_data_launch_update_vmsa *data = cmd_buf;
 590
 591		desc_list[0].paddr_ptr = &data->address;
 592		desc_list[0].len = data->len;
 593		desc_list[0].guest_owned = true;
 594		break;
 595	}
 596	case SEV_CMD_LAUNCH_MEASURE: {
 597		struct sev_data_launch_measure *data = cmd_buf;
 598
 599		desc_list[0].paddr_ptr = &data->address;
 600		desc_list[0].len = data->len;
 601		break;
 602	}
 603	case SEV_CMD_LAUNCH_UPDATE_SECRET: {
 604		struct sev_data_launch_secret *data = cmd_buf;
 605
 606		desc_list[0].paddr_ptr = &data->guest_address;
 607		desc_list[0].len = data->guest_len;
 608		desc_list[0].guest_owned = true;
 609		break;
 610	}
 611	case SEV_CMD_DBG_DECRYPT: {
 612		struct sev_data_dbg *data = cmd_buf;
 613
 614		desc_list[0].paddr_ptr = &data->dst_addr;
 615		desc_list[0].len = data->len;
 616		desc_list[0].guest_owned = true;
 617		break;
 618	}
 619	case SEV_CMD_DBG_ENCRYPT: {
 620		struct sev_data_dbg *data = cmd_buf;
 621
 622		desc_list[0].paddr_ptr = &data->dst_addr;
 623		desc_list[0].len = data->len;
 624		desc_list[0].guest_owned = true;
 625		break;
 626	}
 627	case SEV_CMD_ATTESTATION_REPORT: {
 628		struct sev_data_attestation_report *data = cmd_buf;
 629
 630		desc_list[0].paddr_ptr = &data->address;
 631		desc_list[0].len = data->len;
 632		break;
 633	}
 634	case SEV_CMD_SEND_START: {
 635		struct sev_data_send_start *data = cmd_buf;
 636
 637		desc_list[0].paddr_ptr = &data->session_address;
 638		desc_list[0].len = data->session_len;
 639		break;
 640	}
 641	case SEV_CMD_SEND_UPDATE_DATA: {
 642		struct sev_data_send_update_data *data = cmd_buf;
 643
 644		desc_list[0].paddr_ptr = &data->hdr_address;
 645		desc_list[0].len = data->hdr_len;
 646		desc_list[1].paddr_ptr = &data->trans_address;
 647		desc_list[1].len = data->trans_len;
 648		break;
 649	}
 650	case SEV_CMD_SEND_UPDATE_VMSA: {
 651		struct sev_data_send_update_vmsa *data = cmd_buf;
 652
 653		desc_list[0].paddr_ptr = &data->hdr_address;
 654		desc_list[0].len = data->hdr_len;
 655		desc_list[1].paddr_ptr = &data->trans_address;
 656		desc_list[1].len = data->trans_len;
 657		break;
 658	}
 659	case SEV_CMD_RECEIVE_UPDATE_DATA: {
 660		struct sev_data_receive_update_data *data = cmd_buf;
 661
 662		desc_list[0].paddr_ptr = &data->guest_address;
 663		desc_list[0].len = data->guest_len;
 664		desc_list[0].guest_owned = true;
 665		break;
 666	}
 667	case SEV_CMD_RECEIVE_UPDATE_VMSA: {
 668		struct sev_data_receive_update_vmsa *data = cmd_buf;
 669
 670		desc_list[0].paddr_ptr = &data->guest_address;
 671		desc_list[0].len = data->guest_len;
 672		desc_list[0].guest_owned = true;
 673		break;
 674	}
 675	default:
 676		break;
 677	}
 678}
 679
 680static int snp_map_cmd_buf_desc(struct cmd_buf_desc *desc)
 681{
 682	unsigned int npages;
 683
 684	if (!desc->len)
 685		return 0;
 686
 687	/* Allocate a bounce buffer if this isn't a guest owned page. */
 688	if (!desc->guest_owned) {
 689		struct page *page;
 690
 691		page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(desc->len));
 692		if (!page) {
 693			pr_warn("Failed to allocate bounce buffer for SEV legacy command.\n");
 694			return -ENOMEM;
 695		}
 696
 697		desc->paddr_orig = *desc->paddr_ptr;
 698		*desc->paddr_ptr = __psp_pa(page_to_virt(page));
 699	}
 700
 701	npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
 702
 703	/* Transition the buffer to firmware-owned. */
 704	if (rmp_mark_pages_firmware(*desc->paddr_ptr, npages, true)) {
 705		pr_warn("Error moving pages to firmware-owned state for SEV legacy command.\n");
 706		return -EFAULT;
 707	}
 708
 709	return 0;
 710}
 711
 712static int snp_unmap_cmd_buf_desc(struct cmd_buf_desc *desc)
 713{
 714	unsigned int npages;
 715
 716	if (!desc->len)
 717		return 0;
 718
 719	npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
 720
 721	/* Transition the buffers back to hypervisor-owned. */
 722	if (snp_reclaim_pages(*desc->paddr_ptr, npages, true)) {
 723		pr_warn("Failed to reclaim firmware-owned pages while issuing SEV legacy command.\n");
 724		return -EFAULT;
 725	}
 726
 727	/* Copy data from bounce buffer and then free it. */
 728	if (!desc->guest_owned) {
 729		void *bounce_buf = __va(__sme_clr(*desc->paddr_ptr));
 730		void *dst_buf = __va(__sme_clr(desc->paddr_orig));
 731
 732		memcpy(dst_buf, bounce_buf, desc->len);
 733		__free_pages(virt_to_page(bounce_buf), get_order(desc->len));
 734
 735		/* Restore the original address in the command buffer. */
 736		*desc->paddr_ptr = desc->paddr_orig;
 737	}
 738
 739	return 0;
 740}
 741
 742static int snp_map_cmd_buf_desc_list(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
 743{
 744	int i;
 745
 746	snp_populate_cmd_buf_desc_list(cmd, cmd_buf, desc_list);
 747
 748	for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
 749		struct cmd_buf_desc *desc = &desc_list[i];
 750
 751		if (!desc->paddr_ptr)
 752			break;
 753
 754		if (snp_map_cmd_buf_desc(desc))
 755			goto err_unmap;
 756	}
 757
 758	return 0;
 759
 760err_unmap:
 761	for (i--; i >= 0; i--)
 762		snp_unmap_cmd_buf_desc(&desc_list[i]);
 763
 764	return -EFAULT;
 765}
 766
 767static int snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc *desc_list)
 768{
 769	int i, ret = 0;
 770
 771	for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
 772		struct cmd_buf_desc *desc = &desc_list[i];
 773
 774		if (!desc->paddr_ptr)
 775			break;
 776
 777		if (snp_unmap_cmd_buf_desc(&desc_list[i]))
 778			ret = -EFAULT;
 779	}
 780
 781	return ret;
 782}
 783
 784static bool sev_cmd_buf_writable(int cmd)
 785{
 786	switch (cmd) {
 787	case SEV_CMD_PLATFORM_STATUS:
 788	case SEV_CMD_GUEST_STATUS:
 789	case SEV_CMD_LAUNCH_START:
 790	case SEV_CMD_RECEIVE_START:
 791	case SEV_CMD_LAUNCH_MEASURE:
 792	case SEV_CMD_SEND_START:
 793	case SEV_CMD_SEND_UPDATE_DATA:
 794	case SEV_CMD_SEND_UPDATE_VMSA:
 795	case SEV_CMD_PEK_CSR:
 796	case SEV_CMD_PDH_CERT_EXPORT:
 797	case SEV_CMD_GET_ID:
 798	case SEV_CMD_ATTESTATION_REPORT:
 799		return true;
 800	default:
 801		return false;
 802	}
 803}
 804
 805/* After SNP is INIT'ed, the behavior of legacy SEV commands is changed. */
 806static bool snp_legacy_handling_needed(int cmd)
 807{
 808	struct sev_device *sev = psp_master->sev_data;
 809
 810	return cmd < SEV_CMD_SNP_INIT && sev->snp_initialized;
 811}
 812
 813static int snp_prep_cmd_buf(int cmd, void *cmd_buf, struct cmd_buf_desc *desc_list)
 814{
 815	if (!snp_legacy_handling_needed(cmd))
 816		return 0;
 817
 818	if (snp_map_cmd_buf_desc_list(cmd, cmd_buf, desc_list))
 819		return -EFAULT;
 820
 821	/*
 822	 * Before command execution, the command buffer needs to be put into
 823	 * the firmware-owned state.
 824	 */
 825	if (sev_cmd_buf_writable(cmd)) {
 826		if (rmp_mark_pages_firmware(__pa(cmd_buf), 1, true))
 827			return -EFAULT;
 828	}
 829
 830	return 0;
 831}
 832
 833static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf)
 834{
 835	if (!snp_legacy_handling_needed(cmd))
 836		return 0;
 837
 838	/*
 839	 * After command completion, the command buffer needs to be put back
 840	 * into the hypervisor-owned state.
 841	 */
 842	if (sev_cmd_buf_writable(cmd))
 843		if (snp_reclaim_pages(__pa(cmd_buf), 1, true))
 844			return -EFAULT;
 845
 846	return 0;
 847}
 848
 849int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
 850{
 851	struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0};
 852	struct psp_device *psp = psp_master;
 853	struct sev_device *sev;
 854	unsigned int cmdbuff_hi, cmdbuff_lo;
 855	unsigned int phys_lsb, phys_msb;
 856	unsigned int reg;
 857	void *cmd_buf;
 858	int buf_len;
 859	int ret = 0;
 860
 861	if (!psp || !psp->sev_data)
 862		return -ENODEV;
 863
 864	if (psp_dead)
 865		return -EBUSY;
 866
 867	sev = psp->sev_data;
 868
 869	buf_len = sev_cmd_buffer_len(cmd);
 870	if (WARN_ON_ONCE(!data != !buf_len))
 871		return -EINVAL;
 872
 873	/*
 874	 * Copy the incoming data to driver's scratch buffer as __pa() will not
 875	 * work for some memory, e.g. vmalloc'd addresses, and @data may not be
 876	 * physically contiguous.
 877	 */
 878	if (data) {
 879		/*
 880		 * Commands are generally issued one at a time and require the
 881		 * sev_cmd_mutex, but there could be recursive firmware requests
 882		 * due to SEV_CMD_SNP_PAGE_RECLAIM needing to be issued while
 883		 * preparing buffers for another command. This is the only known
 884		 * case of nesting in the current code, so exactly one
 885		 * additional command buffer is available for that purpose.
 886		 */
 887		if (!sev->cmd_buf_active) {
 888			cmd_buf = sev->cmd_buf;
 889			sev->cmd_buf_active = true;
 890		} else if (!sev->cmd_buf_backup_active) {
 891			cmd_buf = sev->cmd_buf_backup;
 892			sev->cmd_buf_backup_active = true;
 893		} else {
 894			dev_err(sev->dev,
 895				"SEV: too many firmware commands in progress, no command buffers available.\n");
 896			return -EBUSY;
 897		}
 898
 899		memcpy(cmd_buf, data, buf_len);
 900
 901		/*
 902		 * The behavior of the SEV-legacy commands is altered when the
 903		 * SNP firmware is in the INIT state.
 904		 */
 905		ret = snp_prep_cmd_buf(cmd, cmd_buf, desc_list);
 906		if (ret) {
 907			dev_err(sev->dev,
 908				"SEV: failed to prepare buffer for legacy command 0x%x. Error: %d\n",
 909				cmd, ret);
 910			return ret;
 911		}
 912	} else {
 913		cmd_buf = sev->cmd_buf;
 914	}
 915
 916	/* Get the physical address of the command buffer */
 917	phys_lsb = data ? lower_32_bits(__psp_pa(cmd_buf)) : 0;
 918	phys_msb = data ? upper_32_bits(__psp_pa(cmd_buf)) : 0;
 919
 920	dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n",
 921		cmd, phys_msb, phys_lsb, psp_timeout);
 922
 923	print_hex_dump_debug("(in):  ", DUMP_PREFIX_OFFSET, 16, 2, data,
 924			     buf_len, false);
 925
 926	iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
 927	iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
 928
 929	sev->int_rcvd = 0;
 930
 931	reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd);
 932
 933	/*
 934	 * If invoked during panic handling, local interrupts are disabled so
 935	 * the PSP command completion interrupt can't be used.
 936	 * sev_wait_cmd_ioc() already checks for interrupts disabled and
 937	 * polls for PSP command completion.  Ensure we do not request an
 938	 * interrupt from the PSP if irqs disabled.
 939	 */
 940	if (!irqs_disabled())
 941		reg |= SEV_CMDRESP_IOC;
 942
 943	iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg);
 944
 945	/* wait for command completion */
 946	ret = sev_wait_cmd_ioc(sev, &reg, psp_timeout);
 947	if (ret) {
 948		if (psp_ret)
 949			*psp_ret = 0;
 950
 951		dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd);
 952		psp_dead = true;
 953
 954		return ret;
 955	}
 956
 957	psp_timeout = psp_cmd_timeout;
 958
 959	if (psp_ret)
 960		*psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg);
 961
 962	if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
 963		dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n",
 964			cmd, FIELD_GET(PSP_CMDRESP_STS, reg));
 965
 966		/*
 967		 * PSP firmware may report additional error information in the
 968		 * command buffer registers on error. Print contents of command
 969		 * buffer registers if they changed.
 970		 */
 971		cmdbuff_hi = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
 972		cmdbuff_lo = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
 973		if (cmdbuff_hi != phys_msb || cmdbuff_lo != phys_lsb) {
 974			dev_dbg(sev->dev, "Additional error information reported in cmdbuff:");
 975			dev_dbg(sev->dev, "  cmdbuff hi: %#010x\n", cmdbuff_hi);
 976			dev_dbg(sev->dev, "  cmdbuff lo: %#010x\n", cmdbuff_lo);
 977		}
 978		ret = -EIO;
 979	} else {
 980		ret = sev_write_init_ex_file_if_required(cmd);
 981	}
 982
 983	/*
 984	 * Copy potential output from the PSP back to data.  Do this even on
 985	 * failure in case the caller wants to glean something from the error.
 986	 */
 987	if (data) {
 988		int ret_reclaim;
 989		/*
 990		 * Restore the page state after the command completes.
 991		 */
 992		ret_reclaim = snp_reclaim_cmd_buf(cmd, cmd_buf);
 993		if (ret_reclaim) {
 994			dev_err(sev->dev,
 995				"SEV: failed to reclaim buffer for legacy command %#x. Error: %d\n",
 996				cmd, ret_reclaim);
 997			return ret_reclaim;
 998		}
 999
1000		memcpy(data, cmd_buf, buf_len);
1001
1002		if (sev->cmd_buf_backup_active)
1003			sev->cmd_buf_backup_active = false;
1004		else
1005			sev->cmd_buf_active = false;
1006
1007		if (snp_unmap_cmd_buf_desc_list(desc_list))
1008			return -EFAULT;
1009	}
1010
1011	print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
1012			     buf_len, false);
1013
1014	return ret;
1015}
1016
1017int sev_do_cmd(int cmd, void *data, int *psp_ret)
1018{
1019	int rc;
1020
1021	mutex_lock(&sev_cmd_mutex);
1022	rc = __sev_do_cmd_locked(cmd, data, psp_ret);
1023	mutex_unlock(&sev_cmd_mutex);
1024
1025	return rc;
1026}
1027EXPORT_SYMBOL_GPL(sev_do_cmd);
1028
1029static int __sev_init_locked(int *error)
1030{
1031	struct sev_data_init data;
1032
1033	memset(&data, 0, sizeof(data));
1034	if (sev_es_tmr) {
1035		/*
1036		 * Do not include the encryption mask on the physical
1037		 * address of the TMR (firmware should clear it anyway).
1038		 */
1039		data.tmr_address = __pa(sev_es_tmr);
1040
1041		data.flags |= SEV_INIT_FLAGS_SEV_ES;
1042		data.tmr_len = sev_es_tmr_size;
1043	}
1044
1045	return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
1046}
1047
1048static int __sev_init_ex_locked(int *error)
1049{
1050	struct sev_data_init_ex data;
1051
1052	memset(&data, 0, sizeof(data));
1053	data.length = sizeof(data);
1054	data.nv_address = __psp_pa(sev_init_ex_buffer);
1055	data.nv_len = NV_LENGTH;
1056
1057	if (sev_es_tmr) {
1058		/*
1059		 * Do not include the encryption mask on the physical
1060		 * address of the TMR (firmware should clear it anyway).
1061		 */
1062		data.tmr_address = __pa(sev_es_tmr);
1063
1064		data.flags |= SEV_INIT_FLAGS_SEV_ES;
1065		data.tmr_len = sev_es_tmr_size;
1066	}
1067
1068	return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
1069}
1070
1071static inline int __sev_do_init_locked(int *psp_ret)
1072{
1073	if (sev_init_ex_buffer)
1074		return __sev_init_ex_locked(psp_ret);
1075	else
1076		return __sev_init_locked(psp_ret);
1077}
1078
1079static void snp_set_hsave_pa(void *arg)
1080{
1081	wrmsrq(MSR_VM_HSAVE_PA, 0);
1082}
1083
1084/* Hypervisor Fixed pages API interface */
1085static void snp_hv_fixed_pages_state_update(struct sev_device *sev,
1086					    enum snp_hv_fixed_pages_state page_state)
1087{
1088	struct snp_hv_fixed_pages_entry *entry;
1089
1090	/* List is protected by sev_cmd_mutex */
1091	lockdep_assert_held(&sev_cmd_mutex);
1092
1093	if (list_empty(&snp_hv_fixed_pages))
1094		return;
1095
1096	list_for_each_entry(entry, &snp_hv_fixed_pages, list)
1097		entry->page_state = page_state;
1098}
1099
1100/*
1101 * Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole
1102 * 2MB pages are marked as HV_FIXED.
1103 */
1104struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages)
1105{
1106	struct psp_device *psp_master = psp_get_master_device();
1107	struct snp_hv_fixed_pages_entry *entry;
1108	unsigned int order;
1109	struct page *page;
1110
1111	if (!psp_master)
1112		return NULL;
1113
1114	order = get_order(PMD_SIZE * num_2mb_pages);
1115
1116	/*
1117	 * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1118	 * also needs to be protected using the same mutex.
1119	 */
1120	guard(mutex)(&sev_cmd_mutex);
1121
1122	/*
1123	 * This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed
1124	 * page state, fail if SNP is already initialized.
1125	 */
1126	if (psp_master->sev_data &&
1127	    ((struct sev_device *)psp_master->sev_data)->snp_initialized)
1128		return NULL;
1129
1130	/* Re-use freed pages that match the request */
1131	list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1132		/* Hypervisor fixed page allocator implements exact fit policy */
1133		if (entry->order == order && entry->free) {
1134			entry->free = false;
1135			memset(page_address(entry->page), 0,
1136			       (1 << entry->order) * PAGE_SIZE);
1137			return entry->page;
1138		}
1139	}
1140
1141	page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
1142	if (!page)
1143		return NULL;
1144
1145	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1146	if (!entry) {
1147		__free_pages(page, order);
1148		return NULL;
1149	}
1150
1151	entry->page = page;
1152	entry->order = order;
1153	list_add_tail(&entry->list, &snp_hv_fixed_pages);
1154
1155	return page;
1156}
1157
1158void snp_free_hv_fixed_pages(struct page *page)
1159{
1160	struct psp_device *psp_master = psp_get_master_device();
1161	struct snp_hv_fixed_pages_entry *entry, *nentry;
1162
1163	if (!psp_master)
1164		return;
1165
1166	/*
1167	 * SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1168	 * also needs to be protected using the same mutex.
1169	 */
1170	guard(mutex)(&sev_cmd_mutex);
1171
1172	list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) {
1173		if (entry->page != page)
1174			continue;
1175
1176		/*
1177		 * HV_FIXED page state cannot be changed until reboot
1178		 * and they cannot be used by an SNP guest, so they cannot
1179		 * be returned back to the page allocator.
1180		 * Mark the pages as free internally to allow possible re-use.
1181		 */
1182		if (entry->page_state == HV_FIXED) {
1183			entry->free = true;
1184		} else {
1185			__free_pages(page, entry->order);
1186			list_del(&entry->list);
1187			kfree(entry);
1188		}
1189		return;
1190	}
1191}
1192
1193static void snp_add_hv_fixed_pages(struct sev_device *sev, struct sev_data_range_list *range_list)
1194{
1195	struct snp_hv_fixed_pages_entry *entry;
1196	struct sev_data_range *range;
1197	int num_elements;
1198
1199	lockdep_assert_held(&sev_cmd_mutex);
1200
1201	if (list_empty(&snp_hv_fixed_pages))
1202		return;
1203
1204	num_elements = list_count_nodes(&snp_hv_fixed_pages) +
1205		       range_list->num_elements;
1206
1207	/*
1208	 * Ensure the list of HV_FIXED pages that will be passed to firmware
1209	 * do not exceed the page-sized argument buffer.
1210	 */
1211	if (num_elements * sizeof(*range) + sizeof(*range_list) > PAGE_SIZE) {
1212		dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n");
1213		return;
1214	}
1215
1216	range = &range_list->ranges[range_list->num_elements];
1217	list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1218		range->base = page_to_pfn(entry->page) << PAGE_SHIFT;
1219		range->page_count = 1 << entry->order;
1220		range++;
1221	}
1222	range_list->num_elements = num_elements;
1223}
1224
1225static void snp_leak_hv_fixed_pages(void)
1226{
1227	struct snp_hv_fixed_pages_entry *entry;
1228
1229	/* List is protected by sev_cmd_mutex */
1230	lockdep_assert_held(&sev_cmd_mutex);
1231
1232	if (list_empty(&snp_hv_fixed_pages))
1233		return;
1234
1235	list_for_each_entry(entry, &snp_hv_fixed_pages, list)
1236		if (entry->page_state == HV_FIXED)
1237			__snp_leak_pages(page_to_pfn(entry->page),
1238					 1 << entry->order, false);
1239}
1240
1241bool sev_is_snp_ciphertext_hiding_supported(void)
1242{
1243	struct psp_device *psp = psp_master;
1244	struct sev_device *sev;
1245
1246	if (!psp || !psp->sev_data)
1247		return false;
1248
1249	sev = psp->sev_data;
1250
1251	/*
1252	 * Feature information indicates if CipherTextHiding feature is
1253	 * supported by the SEV firmware and additionally platform status
1254	 * indicates if CipherTextHiding feature is enabled in the
1255	 * Platform BIOS.
1256	 */
1257	return ((sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) &&
1258		 sev->snp_plat_status.ciphertext_hiding_cap);
1259}
1260EXPORT_SYMBOL_GPL(sev_is_snp_ciphertext_hiding_supported);
1261
1262static int snp_get_platform_data(struct sev_device *sev, int *error)
1263{
1264	struct sev_data_snp_feature_info snp_feat_info;
1265	struct snp_feature_info *feat_info;
1266	struct sev_data_snp_addr buf;
1267	struct page *page;
1268	int rc;
1269
1270	/*
1271	 * This function is expected to be called before SNP is
1272	 * initialized.
1273	 */
1274	if (sev->snp_initialized)
1275		return -EINVAL;
1276
1277	buf.address = __psp_pa(&sev->snp_plat_status);
1278	rc = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
1279	if (rc) {
1280		dev_err(sev->dev, "SNP PLATFORM_STATUS command failed, ret = %d, error = %#x\n",
1281			rc, *error);
1282		return rc;
1283	}
1284
1285	sev->api_major = sev->snp_plat_status.api_major;
1286	sev->api_minor = sev->snp_plat_status.api_minor;
1287	sev->build = sev->snp_plat_status.build_id;
1288
1289	/*
1290	 * Do feature discovery of the currently loaded firmware,
1291	 * and cache feature information from CPUID 0x8000_0024,
1292	 * sub-function 0.
1293	 */
1294	if (!sev->snp_plat_status.feature_info)
1295		return 0;
1296
1297	/*
1298	 * Use dynamically allocated structure for the SNP_FEATURE_INFO
1299	 * command to ensure structure is 8-byte aligned, and does not
1300	 * cross a page boundary.
1301	 */
1302	page = alloc_page(GFP_KERNEL);
1303	if (!page)
1304		return -ENOMEM;
1305
1306	feat_info = page_address(page);
1307	snp_feat_info.length = sizeof(snp_feat_info);
1308	snp_feat_info.ecx_in = 0;
1309	snp_feat_info.feature_info_paddr = __psp_pa(feat_info);
1310
1311	rc = sev_do_cmd(SEV_CMD_SNP_FEATURE_INFO, &snp_feat_info, error);
1312	if (!rc)
1313		sev->snp_feat_info_0 = *feat_info;
1314	else
1315		dev_err(sev->dev, "SNP FEATURE_INFO command failed, ret = %d, error = %#x\n",
1316			rc, *error);
1317
1318	__free_page(page);
1319
1320	return rc;
1321}
1322
1323static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
1324{
1325	struct sev_data_range_list *range_list = arg;
1326	struct sev_data_range *range = &range_list->ranges[range_list->num_elements];
1327	size_t size;
1328
1329	/*
1330	 * Ensure the list of HV_FIXED pages that will be passed to firmware
1331	 * do not exceed the page-sized argument buffer.
1332	 */
1333	if ((range_list->num_elements * sizeof(struct sev_data_range) +
1334	     sizeof(struct sev_data_range_list)) > PAGE_SIZE)
1335		return -E2BIG;
1336
1337	switch (rs->desc) {
1338	case E820_TYPE_RESERVED:
1339	case E820_TYPE_PMEM:
1340	case E820_TYPE_ACPI:
1341		range->base = rs->start & PAGE_MASK;
1342		size = PAGE_ALIGN((rs->end + 1) - rs->start);
1343		range->page_count = size >> PAGE_SHIFT;
1344		range_list->num_elements++;
1345		break;
1346	default:
1347		break;
1348	}
1349
1350	return 0;
1351}
1352
1353static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
1354{
1355	struct sev_data_range_list *snp_range_list __free(kfree) = NULL;
1356	struct psp_device *psp = psp_master;
1357	struct sev_data_snp_init_ex data;
1358	struct sev_device *sev;
1359	void *arg = &data;
1360	int cmd, rc = 0;
1361
1362	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1363		return -ENODEV;
1364
1365	sev = psp->sev_data;
1366
1367	if (sev->snp_initialized)
1368		return 0;
1369
1370	if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) {
1371		dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n",
1372			SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR);
1373		return -EOPNOTSUPP;
1374	}
1375
1376	/* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
1377	on_each_cpu(snp_set_hsave_pa, NULL, 1);
1378
1379	/*
1380	 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
1381	 * of system physical address ranges to convert into HV-fixed page
1382	 * states during the RMP initialization.  For instance, the memory that
1383	 * UEFI reserves should be included in the that list. This allows system
1384	 * components that occasionally write to memory (e.g. logging to UEFI
1385	 * reserved regions) to not fail due to RMP initialization and SNP
1386	 * enablement.
1387	 *
1388	 */
1389	if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) {
1390		bool tio_supp = !!(sev->snp_feat_info_0.ebx & SNP_SEV_TIO_SUPPORTED);
1391
1392		/*
1393		 * Firmware checks that the pages containing the ranges enumerated
1394		 * in the RANGES structure are either in the default page state or in the
1395		 * firmware page state.
1396		 */
1397		snp_range_list = kzalloc(PAGE_SIZE, GFP_KERNEL);
1398		if (!snp_range_list) {
1399			dev_err(sev->dev,
1400				"SEV: SNP_INIT_EX range list memory allocation failed\n");
1401			return -ENOMEM;
1402		}
1403
1404		/*
1405		 * Retrieve all reserved memory regions from the e820 memory map
1406		 * to be setup as HV-fixed pages.
1407		 */
1408		rc = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM, 0, ~0,
1409					 snp_range_list, snp_filter_reserved_mem_regions);
1410		if (rc) {
1411			dev_err(sev->dev,
1412				"SEV: SNP_INIT_EX walk_iomem_res_desc failed rc = %d\n", rc);
1413			return rc;
1414		}
1415
1416		/*
1417		 * Add HV_Fixed pages from other PSP sub-devices, such as SFS to the
1418		 * HV_Fixed page list.
1419		 */
1420		snp_add_hv_fixed_pages(sev, snp_range_list);
1421
1422		memset(&data, 0, sizeof(data));
1423
1424		if (max_snp_asid) {
1425			data.ciphertext_hiding_en = 1;
1426			data.max_snp_asid = max_snp_asid;
1427		}
1428
1429		data.init_rmp = 1;
1430		data.list_paddr_en = 1;
1431		data.list_paddr = __psp_pa(snp_range_list);
1432
1433		data.tio_en = tio_supp && sev_tio_enabled && amd_iommu_sev_tio_supported();
1434
1435		/*
1436		 * When psp_init_on_probe is disabled, the userspace calling
1437		 * SEV ioctl can inadvertently shut down SNP and SEV-TIO causing
1438		 * unexpected state loss.
1439		 */
1440		if (data.tio_en && !psp_init_on_probe)
1441			dev_warn(sev->dev, "SEV-TIO as incompatible with psp_init_on_probe=0\n");
1442
1443		cmd = SEV_CMD_SNP_INIT_EX;
1444	} else {
1445		cmd = SEV_CMD_SNP_INIT;
1446		arg = NULL;
1447	}
1448
1449	/*
1450	 * The following sequence must be issued before launching the first SNP
1451	 * guest to ensure all dirty cache lines are flushed, including from
1452	 * updates to the RMP table itself via the RMPUPDATE instruction:
1453	 *
1454	 * - WBINVD on all running CPUs
1455	 * - SEV_CMD_SNP_INIT[_EX] firmware command
1456	 * - WBINVD on all running CPUs
1457	 * - SEV_CMD_SNP_DF_FLUSH firmware command
1458	 */
1459	wbinvd_on_all_cpus();
1460
1461	rc = __sev_do_cmd_locked(cmd, arg, error);
1462	if (rc) {
1463		dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n",
1464			cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT",
1465			rc, *error);
1466		return rc;
1467	}
1468
1469	/* Prepare for first SNP guest launch after INIT. */
1470	wbinvd_on_all_cpus();
1471	rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error);
1472	if (rc) {
1473		dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n",
1474			rc, *error);
1475		return rc;
1476	}
1477
1478	snp_hv_fixed_pages_state_update(sev, HV_FIXED);
1479	sev->snp_initialized = true;
1480	dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n",
1481		data.tio_en ? "enabled" : "disabled");
1482
1483	dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major,
1484		 sev->api_minor, sev->build);
1485
1486	atomic_notifier_chain_register(&panic_notifier_list,
1487				       &snp_panic_notifier);
1488
1489	if (data.tio_en) {
1490		/*
1491		 * This executes with the sev_cmd_mutex held so down the stack
1492		 * snp_reclaim_pages(locked=false) might be needed (which is extremely
1493		 * unlikely) but will cause a deadlock.
1494		 * Instead of exporting __snp_alloc_firmware_pages(), allocate a page
1495		 * for this one call here.
1496		 */
1497		void *tio_status = page_address(__snp_alloc_firmware_pages(
1498			GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0, true));
1499
1500		if (tio_status) {
1501			sev_tsm_init_locked(sev, tio_status);
1502			__snp_free_firmware_pages(virt_to_page(tio_status), 0, true);
1503		}
1504	}
1505
1506	sev_es_tmr_size = SNP_TMR_SIZE;
1507
1508	return 0;
1509}
1510
1511static void __sev_platform_init_handle_tmr(struct sev_device *sev)
1512{
1513	if (sev_es_tmr)
1514		return;
1515
1516	/* Obtain the TMR memory area for SEV-ES use */
1517	sev_es_tmr = sev_fw_alloc(sev_es_tmr_size);
1518	if (sev_es_tmr) {
1519		/* Must flush the cache before giving it to the firmware */
1520		if (!sev->snp_initialized)
1521			clflush_cache_range(sev_es_tmr, sev_es_tmr_size);
1522	} else {
1523			dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n");
1524	}
1525}
1526
1527/*
1528 * If an init_ex_path is provided allocate a buffer for the file and
1529 * read in the contents. Additionally, if SNP is initialized, convert
1530 * the buffer pages to firmware pages.
1531 */
1532static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
1533{
1534	struct page *page;
1535	int rc;
1536
1537	if (!init_ex_path)
1538		return 0;
1539
1540	if (sev_init_ex_buffer)
1541		return 0;
1542
1543	page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH));
1544	if (!page) {
1545		dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n");
1546		return -ENOMEM;
1547	}
1548
1549	sev_init_ex_buffer = page_address(page);
1550
1551	rc = sev_read_init_ex_file();
1552	if (rc)
1553		return rc;
1554
1555	/* If SEV-SNP is initialized, transition to firmware page. */
1556	if (sev->snp_initialized) {
1557		unsigned long npages;
1558
1559		npages = 1UL << get_order(NV_LENGTH);
1560		if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) {
1561			dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n");
1562			return -ENOMEM;
1563		}
1564	}
1565
1566	return 0;
1567}
1568
1569static int __sev_platform_init_locked(int *error)
1570{
1571	int rc, psp_ret, dfflush_error;
1572	struct sev_device *sev;
1573
1574	psp_ret = dfflush_error = SEV_RET_NO_FW_CALL;
1575
1576	if (!psp_master || !psp_master->sev_data)
1577		return -ENODEV;
1578
1579	sev = psp_master->sev_data;
1580
1581	if (sev->sev_plat_status.state == SEV_STATE_INIT)
1582		return 0;
1583
1584	__sev_platform_init_handle_tmr(sev);
1585
1586	rc = __sev_platform_init_handle_init_ex_path(sev);
1587	if (rc)
1588		return rc;
1589
1590	rc = __sev_do_init_locked(&psp_ret);
1591	if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
1592		/*
1593		 * Initialization command returned an integrity check failure
1594		 * status code, meaning that firmware load and validation of SEV
1595		 * related persistent data has failed. Retrying the
1596		 * initialization function should succeed by replacing the state
1597		 * with a reset state.
1598		 */
1599		dev_err(sev->dev,
1600"SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
1601		rc = __sev_do_init_locked(&psp_ret);
1602	}
1603
1604	if (error)
1605		*error = psp_ret;
1606
1607	if (rc) {
1608		dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n",
1609			sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc);
1610		return rc;
1611	}
1612
1613	sev->sev_plat_status.state = SEV_STATE_INIT;
1614
1615	/* Prepare for first SEV guest launch after INIT */
1616	wbinvd_on_all_cpus();
1617	rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error);
1618	if (rc) {
1619		dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n",
1620			dfflush_error, rc);
1621		return rc;
1622	}
1623
1624	dev_dbg(sev->dev, "SEV firmware initialized\n");
1625
1626	dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
1627		 sev->api_minor, sev->build);
1628
1629	return 0;
1630}
1631
1632static int _sev_platform_init_locked(struct sev_platform_init_args *args)
1633{
1634	struct sev_device *sev;
1635	int rc;
1636
1637	if (!psp_master || !psp_master->sev_data)
1638		return -ENODEV;
1639
1640	/*
1641	 * Skip SNP/SEV initialization under a kdump kernel as SEV/SNP
1642	 * may already be initialized in the previous kernel. Since no
1643	 * SNP/SEV guests are run under a kdump kernel, there is no
1644	 * need to initialize SNP or SEV during kdump boot.
1645	 */
1646	if (is_kdump_kernel())
1647		return 0;
1648
1649	sev = psp_master->sev_data;
1650
1651	if (sev->sev_plat_status.state == SEV_STATE_INIT)
1652		return 0;
1653
1654	rc = __sev_snp_init_locked(&args->error, args->max_snp_asid);
1655	if (rc && rc != -ENODEV)
1656		return rc;
1657
1658	/* Defer legacy SEV/SEV-ES support if allowed by caller/module. */
1659	if (args->probe && !psp_init_on_probe)
1660		return 0;
1661
1662	return __sev_platform_init_locked(&args->error);
1663}
1664
1665int sev_platform_init(struct sev_platform_init_args *args)
1666{
1667	int rc;
1668
1669	mutex_lock(&sev_cmd_mutex);
1670	rc = _sev_platform_init_locked(args);
1671	mutex_unlock(&sev_cmd_mutex);
1672
1673	return rc;
1674}
1675EXPORT_SYMBOL_GPL(sev_platform_init);
1676
1677static int __sev_platform_shutdown_locked(int *error)
1678{
1679	struct psp_device *psp = psp_master;
1680	struct sev_device *sev;
1681	int ret;
1682
1683	if (!psp || !psp->sev_data)
1684		return 0;
1685
1686	sev = psp->sev_data;
1687
1688	if (sev->sev_plat_status.state == SEV_STATE_UNINIT)
1689		return 0;
1690
1691	ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error);
1692	if (ret) {
1693		dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n",
1694			*error, ret);
1695		return ret;
1696	}
1697
1698	sev->sev_plat_status.state = SEV_STATE_UNINIT;
1699	dev_dbg(sev->dev, "SEV firmware shutdown\n");
1700
1701	return ret;
1702}
1703
1704static int sev_get_platform_state(int *state, int *error)
1705{
1706	struct sev_user_data_status data;
1707	int rc;
1708
1709	rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error);
1710	if (rc)
1711		return rc;
1712
1713	*state = data.state;
1714	return rc;
1715}
1716
1717static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
1718{
1719	struct sev_platform_init_args init_args = {0};
1720	int rc;
1721
1722	rc = _sev_platform_init_locked(&init_args);
1723	if (rc) {
1724		argp->error = SEV_RET_INVALID_PLATFORM_STATE;
1725		return rc;
1726	}
1727
1728	*shutdown_required = true;
1729
1730	return 0;
1731}
1732
1733static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
1734{
1735	int error, rc;
1736
1737	rc = __sev_snp_init_locked(&error, 0);
1738	if (rc) {
1739		argp->error = SEV_RET_INVALID_PLATFORM_STATE;
1740		return rc;
1741	}
1742
1743	*shutdown_required = true;
1744
1745	return 0;
1746}
1747
1748static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable)
1749{
1750	int state, rc;
1751
1752	if (!writable)
1753		return -EPERM;
1754
1755	/*
1756	 * The SEV spec requires that FACTORY_RESET must be issued in
1757	 * UNINIT state. Before we go further lets check if any guest is
1758	 * active.
1759	 *
1760	 * If FW is in WORKING state then deny the request otherwise issue
1761	 * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET.
1762	 *
1763	 */
1764	rc = sev_get_platform_state(&state, &argp->error);
1765	if (rc)
1766		return rc;
1767
1768	if (state == SEV_STATE_WORKING)
1769		return -EBUSY;
1770
1771	if (state == SEV_STATE_INIT) {
1772		rc = __sev_platform_shutdown_locked(&argp->error);
1773		if (rc)
1774			return rc;
1775	}
1776
1777	return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error);
1778}
1779
1780static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp)
1781{
1782	struct sev_user_data_status data;
1783	int ret;
1784
1785	memset(&data, 0, sizeof(data));
1786
1787	ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error);
1788	if (ret)
1789		return ret;
1790
1791	if (copy_to_user((void __user *)argp->data, &data, sizeof(data)))
1792		ret = -EFAULT;
1793
1794	return ret;
1795}
1796
1797static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable)
1798{
1799	struct sev_device *sev = psp_master->sev_data;
1800	bool shutdown_required = false;
1801	int rc;
1802
1803	if (!writable)
1804		return -EPERM;
1805
1806	if (sev->sev_plat_status.state == SEV_STATE_UNINIT) {
1807		rc = sev_move_to_init_state(argp, &shutdown_required);
1808		if (rc)
1809			return rc;
1810	}
1811
1812	rc = __sev_do_cmd_locked(cmd, NULL, &argp->error);
1813
1814	if (shutdown_required)
1815		__sev_firmware_shutdown(sev, false);
1816
1817	return rc;
1818}
1819
1820static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable)
1821{
1822	struct sev_device *sev = psp_master->sev_data;
1823	struct sev_user_data_pek_csr input;
1824	bool shutdown_required = false;
1825	struct sev_data_pek_csr data;
1826	void __user *input_address;
1827	void *blob = NULL;
1828	int ret;
1829
1830	if (!writable)
1831		return -EPERM;
1832
1833	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1834		return -EFAULT;
1835
1836	memset(&data, 0, sizeof(data));
1837
1838	/* userspace wants to query CSR length */
1839	if (!input.address || !input.length)
1840		goto cmd;
1841
1842	/* allocate a physically contiguous buffer to store the CSR blob */
1843	input_address = (void __user *)input.address;
1844	if (input.length > SEV_FW_BLOB_MAX_SIZE)
1845		return -EFAULT;
1846
1847	blob = kzalloc(input.length, GFP_KERNEL);
1848	if (!blob)
1849		return -ENOMEM;
1850
1851	data.address = __psp_pa(blob);
1852	data.len = input.length;
1853
1854cmd:
1855	if (sev->sev_plat_status.state == SEV_STATE_UNINIT) {
1856		ret = sev_move_to_init_state(argp, &shutdown_required);
1857		if (ret)
1858			goto e_free_blob;
1859	}
1860
1861	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error);
1862
1863	 /* If we query the CSR length, FW responded with expected data. */
1864	input.length = data.len;
1865
1866	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
1867		ret = -EFAULT;
1868		goto e_free_blob;
1869	}
1870
1871	if (blob) {
1872		if (copy_to_user(input_address, blob, input.length))
1873			ret = -EFAULT;
1874	}
1875
1876e_free_blob:
1877	if (shutdown_required)
1878		__sev_firmware_shutdown(sev, false);
1879
1880	kfree(blob);
1881	return ret;
1882}
1883
1884void *psp_copy_user_blob(u64 uaddr, u32 len)
1885{
1886	if (!uaddr || !len)
1887		return ERR_PTR(-EINVAL);
1888
1889	/* verify that blob length does not exceed our limit */
1890	if (len > SEV_FW_BLOB_MAX_SIZE)
1891		return ERR_PTR(-EINVAL);
1892
1893	return memdup_user((void __user *)uaddr, len);
1894}
1895EXPORT_SYMBOL_GPL(psp_copy_user_blob);
1896
1897static int sev_get_api_version(void)
1898{
1899	struct sev_device *sev = psp_master->sev_data;
1900	struct sev_user_data_status status;
1901	int error = 0, ret;
1902
1903	/*
1904	 * Cache SNP platform status and SNP feature information
1905	 * if SNP is available.
1906	 */
1907	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) {
1908		ret = snp_get_platform_data(sev, &error);
1909		if (ret)
1910			return 1;
1911	}
1912
1913	ret = sev_platform_status(&status, &error);
1914	if (ret) {
1915		dev_err(sev->dev,
1916			"SEV: failed to get status. Error: %#x\n", error);
1917		return 1;
1918	}
1919
1920	/* Cache SEV platform status */
1921	sev->sev_plat_status = status;
1922
1923	sev->api_major = status.api_major;
1924	sev->api_minor = status.api_minor;
1925	sev->build = status.build;
1926
1927	return 0;
1928}
1929
1930static int sev_get_firmware(struct device *dev,
1931			    const struct firmware **firmware)
1932{
1933	char fw_name_specific[SEV_FW_NAME_SIZE];
1934	char fw_name_subset[SEV_FW_NAME_SIZE];
1935
1936	snprintf(fw_name_specific, sizeof(fw_name_specific),
1937		 "amd/amd_sev_fam%.2xh_model%.2xh.sbin",
1938		 boot_cpu_data.x86, boot_cpu_data.x86_model);
1939
1940	snprintf(fw_name_subset, sizeof(fw_name_subset),
1941		 "amd/amd_sev_fam%.2xh_model%.1xxh.sbin",
1942		 boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4);
1943
1944	/* Check for SEV FW for a particular model.
1945	 * Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h
1946	 *
1947	 * or
1948	 *
1949	 * Check for SEV FW common to a subset of models.
1950	 * Ex. amd_sev_fam17h_model0xh.sbin for
1951	 *     Family 17h Model 00h -- Family 17h Model 0Fh
1952	 *
1953	 * or
1954	 *
1955	 * Fall-back to using generic name: sev.fw
1956	 */
1957	if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) ||
1958	    (firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) ||
1959	    (firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0))
1960		return 0;
1961
1962	return -ENOENT;
1963}
1964
1965/* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
1966static int sev_update_firmware(struct device *dev)
1967{
1968	struct sev_data_download_firmware *data;
1969	const struct firmware *firmware;
1970	int ret, error, order;
1971	struct page *p;
1972	u64 data_size;
1973
1974	if (!sev_version_greater_or_equal(0, 15)) {
1975		dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n");
1976		return -1;
1977	}
1978
1979	if (sev_get_firmware(dev, &firmware) == -ENOENT) {
1980		dev_dbg(dev, "No SEV firmware file present\n");
1981		return -1;
1982	}
1983
1984	/*
1985	 * SEV FW expects the physical address given to it to be 32
1986	 * byte aligned. Memory allocated has structure placed at the
1987	 * beginning followed by the firmware being passed to the SEV
1988	 * FW. Allocate enough memory for data structure + alignment
1989	 * padding + SEV FW.
1990	 */
1991	data_size = ALIGN(sizeof(struct sev_data_download_firmware), 32);
1992
1993	order = get_order(firmware->size + data_size);
1994	p = alloc_pages(GFP_KERNEL, order);
1995	if (!p) {
1996		ret = -1;
1997		goto fw_err;
1998	}
1999
2000	/*
2001	 * Copy firmware data to a kernel allocated contiguous
2002	 * memory region.
2003	 */
2004	data = page_address(p);
2005	memcpy(page_address(p) + data_size, firmware->data, firmware->size);
2006
2007	data->address = __psp_pa(page_address(p) + data_size);
2008	data->len = firmware->size;
2009
2010	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
2011
2012	/*
2013	 * A quirk for fixing the committed TCB version, when upgrading from
2014	 * earlier firmware version than 1.50.
2015	 */
2016	if (!ret && !sev_version_greater_or_equal(1, 50))
2017		ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error);
2018
2019	if (ret)
2020		dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
2021
2022	__free_pages(p, order);
2023
2024fw_err:
2025	release_firmware(firmware);
2026
2027	return ret;
2028}
2029
2030static int __sev_snp_shutdown_locked(int *error, bool panic)
2031{
2032	struct psp_device *psp = psp_master;
2033	struct sev_device *sev;
2034	struct sev_data_snp_shutdown_ex data;
2035	int ret;
2036
2037	if (!psp || !psp->sev_data)
2038		return 0;
2039
2040	sev = psp->sev_data;
2041
2042	if (!sev->snp_initialized)
2043		return 0;
2044
2045	memset(&data, 0, sizeof(data));
2046	data.len = sizeof(data);
2047	data.iommu_snp_shutdown = 1;
2048
2049	/*
2050	 * If invoked during panic handling, local interrupts are disabled
2051	 * and all CPUs are stopped, so wbinvd_on_all_cpus() can't be called.
2052	 * In that case, a wbinvd() is done on remote CPUs via the NMI
2053	 * callback, so only a local wbinvd() is needed here.
2054	 */
2055	if (!panic)
2056		wbinvd_on_all_cpus();
2057	else
2058		wbinvd();
2059
2060	ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error);
2061	/* SHUTDOWN may require DF_FLUSH */
2062	if (*error == SEV_RET_DFFLUSH_REQUIRED) {
2063		int dfflush_error = SEV_RET_NO_FW_CALL;
2064
2065		ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error);
2066		if (ret) {
2067			dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n",
2068				ret, dfflush_error);
2069			return ret;
2070		}
2071		/* reissue the shutdown command */
2072		ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data,
2073					  error);
2074	}
2075	if (ret) {
2076		dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n",
2077			ret, *error);
2078		return ret;
2079	}
2080
2081	/*
2082	 * SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP
2083	 * enforcement by the IOMMU and also transitions all pages
2084	 * associated with the IOMMU to the Reclaim state.
2085	 * Firmware was transitioning the IOMMU pages to Hypervisor state
2086	 * before version 1.53. But, accounting for the number of assigned
2087	 * 4kB pages in a 2M page was done incorrectly by not transitioning
2088	 * to the Reclaim state. This resulted in RMP #PF when later accessing
2089	 * the 2M page containing those pages during kexec boot. Hence, the
2090	 * firmware now transitions these pages to Reclaim state and hypervisor
2091	 * needs to transition these pages to shared state. SNP Firmware
2092	 * version 1.53 and above are needed for kexec boot.
2093	 */
2094	ret = amd_iommu_snp_disable();
2095	if (ret) {
2096		dev_err(sev->dev, "SNP IOMMU shutdown failed\n");
2097		return ret;
2098	}
2099
2100	snp_leak_hv_fixed_pages();
2101	sev->snp_initialized = false;
2102	dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n");
2103
2104	/*
2105	 * __sev_snp_shutdown_locked() deadlocks when it tries to unregister
2106	 * itself during panic as the panic notifier is called with RCU read
2107	 * lock held and notifier unregistration does RCU synchronization.
2108	 */
2109	if (!panic)
2110		atomic_notifier_chain_unregister(&panic_notifier_list,
2111						 &snp_panic_notifier);
2112
2113	/* Reset TMR size back to default */
2114	sev_es_tmr_size = SEV_TMR_SIZE;
2115
2116	return ret;
2117}
2118
2119static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable)
2120{
2121	struct sev_device *sev = psp_master->sev_data;
2122	struct sev_user_data_pek_cert_import input;
2123	struct sev_data_pek_cert_import data;
2124	bool shutdown_required = false;
2125	void *pek_blob, *oca_blob;
2126	int ret;
2127
2128	if (!writable)
2129		return -EPERM;
2130
2131	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2132		return -EFAULT;
2133
2134	/* copy PEK certificate blobs from userspace */
2135	pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len);
2136	if (IS_ERR(pek_blob))
2137		return PTR_ERR(pek_blob);
2138
2139	data.reserved = 0;
2140	data.pek_cert_address = __psp_pa(pek_blob);
2141	data.pek_cert_len = input.pek_cert_len;
2142
2143	/* copy PEK certificate blobs from userspace */
2144	oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len);
2145	if (IS_ERR(oca_blob)) {
2146		ret = PTR_ERR(oca_blob);
2147		goto e_free_pek;
2148	}
2149
2150	data.oca_cert_address = __psp_pa(oca_blob);
2151	data.oca_cert_len = input.oca_cert_len;
2152
2153	/* If platform is not in INIT state then transition it to INIT */
2154	if (sev->sev_plat_status.state != SEV_STATE_INIT) {
2155		ret = sev_move_to_init_state(argp, &shutdown_required);
2156		if (ret)
2157			goto e_free_oca;
2158	}
2159
2160	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error);
2161
2162e_free_oca:
2163	if (shutdown_required)
2164		__sev_firmware_shutdown(sev, false);
2165
2166	kfree(oca_blob);
2167e_free_pek:
2168	kfree(pek_blob);
2169	return ret;
2170}
2171
2172static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp)
2173{
2174	struct sev_user_data_get_id2 input;
2175	struct sev_data_get_id data;
2176	void __user *input_address;
2177	void *id_blob = NULL;
2178	int ret;
2179
2180	/* SEV GET_ID is available from SEV API v0.16 and up */
2181	if (!sev_version_greater_or_equal(0, 16))
2182		return -ENOTSUPP;
2183
2184	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2185		return -EFAULT;
2186
2187	input_address = (void __user *)input.address;
2188
2189	if (input.address && input.length) {
2190		/*
2191		 * The length of the ID shouldn't be assumed by software since
2192		 * it may change in the future.  The allocation size is limited
2193		 * to 1 << (PAGE_SHIFT + MAX_PAGE_ORDER) by the page allocator.
2194		 * If the allocation fails, simply return ENOMEM rather than
2195		 * warning in the kernel log.
2196		 */
2197		id_blob = kzalloc(input.length, GFP_KERNEL | __GFP_NOWARN);
2198		if (!id_blob)
2199			return -ENOMEM;
2200
2201		data.address = __psp_pa(id_blob);
2202		data.len = input.length;
2203	} else {
2204		data.address = 0;
2205		data.len = 0;
2206	}
2207
2208	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error);
2209
2210	/*
2211	 * Firmware will return the length of the ID value (either the minimum
2212	 * required length or the actual length written), return it to the user.
2213	 */
2214	input.length = data.len;
2215
2216	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2217		ret = -EFAULT;
2218		goto e_free;
2219	}
2220
2221	if (id_blob) {
2222		if (copy_to_user(input_address, id_blob, data.len)) {
2223			ret = -EFAULT;
2224			goto e_free;
2225		}
2226	}
2227
2228e_free:
2229	kfree(id_blob);
2230
2231	return ret;
2232}
2233
2234static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
2235{
2236	struct sev_data_get_id *data;
2237	u64 data_size, user_size;
2238	void *id_blob, *mem;
2239	int ret;
2240
2241	/* SEV GET_ID available from SEV API v0.16 and up */
2242	if (!sev_version_greater_or_equal(0, 16))
2243		return -ENOTSUPP;
2244
2245	/* SEV FW expects the buffer it fills with the ID to be
2246	 * 8-byte aligned. Memory allocated should be enough to
2247	 * hold data structure + alignment padding + memory
2248	 * where SEV FW writes the ID.
2249	 */
2250	data_size = ALIGN(sizeof(struct sev_data_get_id), 8);
2251	user_size = sizeof(struct sev_user_data_get_id);
2252
2253	mem = kzalloc(data_size + user_size, GFP_KERNEL);
2254	if (!mem)
2255		return -ENOMEM;
2256
2257	data = mem;
2258	id_blob = mem + data_size;
2259
2260	data->address = __psp_pa(id_blob);
2261	data->len = user_size;
2262
2263	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error);
2264	if (!ret) {
2265		if (copy_to_user((void __user *)argp->data, id_blob, data->len))
2266			ret = -EFAULT;
2267	}
2268
2269	kfree(mem);
2270
2271	return ret;
2272}
2273
2274static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
2275{
2276	struct sev_device *sev = psp_master->sev_data;
2277	struct sev_user_data_pdh_cert_export input;
2278	void *pdh_blob = NULL, *cert_blob = NULL;
2279	struct sev_data_pdh_cert_export data;
2280	void __user *input_cert_chain_address;
2281	void __user *input_pdh_cert_address;
2282	bool shutdown_required = false;
2283	int ret;
2284
2285	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2286		return -EFAULT;
2287
2288	memset(&data, 0, sizeof(data));
2289
2290	input_pdh_cert_address = (void __user *)input.pdh_cert_address;
2291	input_cert_chain_address = (void __user *)input.cert_chain_address;
2292
2293	/* Userspace wants to query the certificate length. */
2294	if (!input.pdh_cert_address ||
2295	    !input.pdh_cert_len ||
2296	    !input.cert_chain_address)
2297		goto cmd;
2298
2299	/* Allocate a physically contiguous buffer to store the PDH blob. */
2300	if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE)
2301		return -EFAULT;
2302
2303	/* Allocate a physically contiguous buffer to store the cert chain blob. */
2304	if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE)
2305		return -EFAULT;
2306
2307	pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL);
2308	if (!pdh_blob)
2309		return -ENOMEM;
2310
2311	data.pdh_cert_address = __psp_pa(pdh_blob);
2312	data.pdh_cert_len = input.pdh_cert_len;
2313
2314	cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL);
2315	if (!cert_blob) {
2316		ret = -ENOMEM;
2317		goto e_free_pdh;
2318	}
2319
2320	data.cert_chain_address = __psp_pa(cert_blob);
2321	data.cert_chain_len = input.cert_chain_len;
2322
2323cmd:
2324	/* If platform is not in INIT state then transition it to INIT. */
2325	if (sev->sev_plat_status.state != SEV_STATE_INIT) {
2326		if (!writable) {
2327			ret = -EPERM;
2328			goto e_free_cert;
2329		}
2330		ret = sev_move_to_init_state(argp, &shutdown_required);
2331		if (ret)
2332			goto e_free_cert;
2333	}
2334
2335	ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error);
2336
2337	/* If we query the length, FW responded with expected data. */
2338	input.cert_chain_len = data.cert_chain_len;
2339	input.pdh_cert_len = data.pdh_cert_len;
2340
2341	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2342		ret = -EFAULT;
2343		goto e_free_cert;
2344	}
2345
2346	if (pdh_blob) {
2347		if (copy_to_user(input_pdh_cert_address,
2348				 pdh_blob, input.pdh_cert_len)) {
2349			ret = -EFAULT;
2350			goto e_free_cert;
2351		}
2352	}
2353
2354	if (cert_blob) {
2355		if (copy_to_user(input_cert_chain_address,
2356				 cert_blob, input.cert_chain_len))
2357			ret = -EFAULT;
2358	}
2359
2360e_free_cert:
2361	if (shutdown_required)
2362		__sev_firmware_shutdown(sev, false);
2363
2364	kfree(cert_blob);
2365e_free_pdh:
2366	kfree(pdh_blob);
2367	return ret;
2368}
2369
2370static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
2371{
2372	struct sev_device *sev = psp_master->sev_data;
2373	struct sev_data_snp_addr buf;
2374	struct page *status_page;
2375	void *data;
2376	int ret;
2377
2378	if (!argp->data)
2379		return -EINVAL;
2380
2381	status_page = alloc_page(GFP_KERNEL_ACCOUNT);
2382	if (!status_page)
2383		return -ENOMEM;
2384
2385	data = page_address(status_page);
2386
2387	/*
2388	 * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed
2389	 * when SNP has been initialized, the status page must be firmware-owned.
2390	 */
2391	if (sev->snp_initialized) {
2392		/*
2393		 * Firmware expects the status page to be in Firmware state,
2394		 * otherwise it will report an error INVALID_PAGE_STATE.
2395		 */
2396		if (rmp_mark_pages_firmware(__pa(data), 1, true)) {
2397			ret = -EFAULT;
2398			goto cleanup;
2399		}
2400	}
2401
2402	buf.address = __psp_pa(data);
2403	ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error);
2404
2405	if (sev->snp_initialized) {
2406		/*
2407		 * The status page will be in Reclaim state on success, or left
2408		 * in Firmware state on failure. Use snp_reclaim_pages() to
2409		 * transition either case back to Hypervisor-owned state.
2410		 */
2411		if (snp_reclaim_pages(__pa(data), 1, true))
2412			return -EFAULT;
2413	}
2414
2415	if (ret)
2416		goto cleanup;
2417
2418	if (copy_to_user((void __user *)argp->data, data,
2419			 sizeof(struct sev_user_data_snp_status)))
2420		ret = -EFAULT;
2421
2422cleanup:
2423	__free_pages(status_page, 0);
2424	return ret;
2425}
2426
2427static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
2428{
2429	struct sev_device *sev = psp_master->sev_data;
2430	struct sev_data_snp_commit buf;
2431	bool shutdown_required = false;
2432	int ret, error;
2433
2434	if (!sev->snp_initialized) {
2435		ret = snp_move_to_init_state(argp, &shutdown_required);
2436		if (ret)
2437			return ret;
2438	}
2439
2440	buf.len = sizeof(buf);
2441
2442	ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error);
2443
2444	if (shutdown_required)
2445		__sev_snp_shutdown_locked(&error, false);
2446
2447	return ret;
2448}
2449
2450static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable)
2451{
2452	struct sev_device *sev = psp_master->sev_data;
2453	struct sev_user_data_snp_config config;
2454	bool shutdown_required = false;
2455	int ret, error;
2456
2457	if (!argp->data)
2458		return -EINVAL;
2459
2460	if (!writable)
2461		return -EPERM;
2462
2463	if (copy_from_user(&config, (void __user *)argp->data, sizeof(config)))
2464		return -EFAULT;
2465
2466	if (!sev->snp_initialized) {
2467		ret = snp_move_to_init_state(argp, &shutdown_required);
2468		if (ret)
2469			return ret;
2470	}
2471
2472	ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
2473
2474	if (shutdown_required)
2475		__sev_snp_shutdown_locked(&error, false);
2476
2477	return ret;
2478}
2479
2480static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
2481{
2482	struct sev_device *sev = psp_master->sev_data;
2483	struct sev_user_data_snp_vlek_load input;
2484	bool shutdown_required = false;
2485	int ret, error;
2486	void *blob;
2487
2488	if (!argp->data)
2489		return -EINVAL;
2490
2491	if (!writable)
2492		return -EPERM;
2493
2494	if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input)))
2495		return -EFAULT;
2496
2497	if (input.len != sizeof(input) || input.vlek_wrapped_version != 0)
2498		return -EINVAL;
2499
2500	blob = psp_copy_user_blob(input.vlek_wrapped_address,
2501				  sizeof(struct sev_user_data_snp_wrapped_vlek_hashstick));
2502	if (IS_ERR(blob))
2503		return PTR_ERR(blob);
2504
2505	input.vlek_wrapped_address = __psp_pa(blob);
2506
2507	if (!sev->snp_initialized) {
2508		ret = snp_move_to_init_state(argp, &shutdown_required);
2509		if (ret)
2510			goto cleanup;
2511	}
2512
2513	ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error);
2514
2515	if (shutdown_required)
2516		__sev_snp_shutdown_locked(&error, false);
2517
2518cleanup:
2519	kfree(blob);
2520
2521	return ret;
2522}
2523
2524static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
2525{
2526	void __user *argp = (void __user *)arg;
2527	struct sev_issue_cmd input;
2528	int ret = -EFAULT;
2529	bool writable = file->f_mode & FMODE_WRITE;
2530
2531	if (!psp_master || !psp_master->sev_data)
2532		return -ENODEV;
2533
2534	if (ioctl != SEV_ISSUE_CMD)
2535		return -EINVAL;
2536
2537	if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd)))
2538		return -EFAULT;
2539
2540	if (input.cmd > SEV_MAX)
2541		return -EINVAL;
2542
2543	mutex_lock(&sev_cmd_mutex);
2544
2545	switch (input.cmd) {
2546
2547	case SEV_FACTORY_RESET:
2548		ret = sev_ioctl_do_reset(&input, writable);
2549		break;
2550	case SEV_PLATFORM_STATUS:
2551		ret = sev_ioctl_do_platform_status(&input);
2552		break;
2553	case SEV_PEK_GEN:
2554		ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable);
2555		break;
2556	case SEV_PDH_GEN:
2557		ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable);
2558		break;
2559	case SEV_PEK_CSR:
2560		ret = sev_ioctl_do_pek_csr(&input, writable);
2561		break;
2562	case SEV_PEK_CERT_IMPORT:
2563		ret = sev_ioctl_do_pek_import(&input, writable);
2564		break;
2565	case SEV_PDH_CERT_EXPORT:
2566		ret = sev_ioctl_do_pdh_export(&input, writable);
2567		break;
2568	case SEV_GET_ID:
2569		pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n");
2570		ret = sev_ioctl_do_get_id(&input);
2571		break;
2572	case SEV_GET_ID2:
2573		ret = sev_ioctl_do_get_id2(&input);
2574		break;
2575	case SNP_PLATFORM_STATUS:
2576		ret = sev_ioctl_do_snp_platform_status(&input);
2577		break;
2578	case SNP_COMMIT:
2579		ret = sev_ioctl_do_snp_commit(&input);
2580		break;
2581	case SNP_SET_CONFIG:
2582		ret = sev_ioctl_do_snp_set_config(&input, writable);
2583		break;
2584	case SNP_VLEK_LOAD:
2585		ret = sev_ioctl_do_snp_vlek_load(&input, writable);
2586		break;
2587	default:
2588		ret = -EINVAL;
2589		goto out;
2590	}
2591
2592	if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd)))
2593		ret = -EFAULT;
2594out:
2595	mutex_unlock(&sev_cmd_mutex);
2596
2597	return ret;
2598}
2599
2600static const struct file_operations sev_fops = {
2601	.owner	= THIS_MODULE,
2602	.unlocked_ioctl = sev_ioctl,
2603};
2604
2605int sev_platform_status(struct sev_user_data_status *data, int *error)
2606{
2607	return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error);
2608}
2609EXPORT_SYMBOL_GPL(sev_platform_status);
2610
2611int sev_guest_deactivate(struct sev_data_deactivate *data, int *error)
2612{
2613	return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error);
2614}
2615EXPORT_SYMBOL_GPL(sev_guest_deactivate);
2616
2617int sev_guest_activate(struct sev_data_activate *data, int *error)
2618{
2619	return sev_do_cmd(SEV_CMD_ACTIVATE, data, error);
2620}
2621EXPORT_SYMBOL_GPL(sev_guest_activate);
2622
2623int sev_guest_decommission(struct sev_data_decommission *data, int *error)
2624{
2625	return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error);
2626}
2627EXPORT_SYMBOL_GPL(sev_guest_decommission);
2628
2629int sev_guest_df_flush(int *error)
2630{
2631	return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error);
2632}
2633EXPORT_SYMBOL_GPL(sev_guest_df_flush);
2634
2635static void sev_exit(struct kref *ref)
2636{
2637	misc_deregister(&misc_dev->misc);
2638	kfree(misc_dev);
2639	misc_dev = NULL;
2640}
2641
2642static int sev_misc_init(struct sev_device *sev)
2643{
2644	struct device *dev = sev->dev;
2645	int ret;
2646
2647	/*
2648	 * SEV feature support can be detected on multiple devices but the SEV
2649	 * FW commands must be issued on the master. During probe, we do not
2650	 * know the master hence we create /dev/sev on the first device probe.
2651	 * sev_do_cmd() finds the right master device to which to issue the
2652	 * command to the firmware.
2653	 */
2654	if (!misc_dev) {
2655		struct miscdevice *misc;
2656
2657		misc_dev = kzalloc(sizeof(*misc_dev), GFP_KERNEL);
2658		if (!misc_dev)
2659			return -ENOMEM;
2660
2661		misc = &misc_dev->misc;
2662		misc->minor = MISC_DYNAMIC_MINOR;
2663		misc->name = DEVICE_NAME;
2664		misc->fops = &sev_fops;
2665
2666		ret = misc_register(misc);
2667		if (ret)
2668			return ret;
2669
2670		kref_init(&misc_dev->refcount);
2671	} else {
2672		kref_get(&misc_dev->refcount);
2673	}
2674
2675	init_waitqueue_head(&sev->int_queue);
2676	sev->misc = misc_dev;
2677	dev_dbg(dev, "registered SEV device\n");
2678
2679	return 0;
2680}
2681
2682int sev_dev_init(struct psp_device *psp)
2683{
2684	struct device *dev = psp->dev;
2685	struct sev_device *sev;
2686	int ret = -ENOMEM;
2687
2688	if (!boot_cpu_has(X86_FEATURE_SEV)) {
2689		dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n");
2690		return 0;
2691	}
2692
2693	sev = devm_kzalloc(dev, sizeof(*sev), GFP_KERNEL);
2694	if (!sev)
2695		goto e_err;
2696
2697	sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 1);
2698	if (!sev->cmd_buf)
2699		goto e_sev;
2700
2701	sev->cmd_buf_backup = (uint8_t *)sev->cmd_buf + PAGE_SIZE;
2702
2703	psp->sev_data = sev;
2704
2705	sev->dev = dev;
2706	sev->psp = psp;
2707
2708	sev->io_regs = psp->io_regs;
2709
2710	sev->vdata = (struct sev_vdata *)psp->vdata->sev;
2711	if (!sev->vdata) {
2712		ret = -ENODEV;
2713		dev_err(dev, "sev: missing driver data\n");
2714		goto e_buf;
2715	}
2716
2717	psp_set_sev_irq_handler(psp, sev_irq_handler, sev);
2718
2719	ret = sev_misc_init(sev);
2720	if (ret)
2721		goto e_irq;
2722
2723	dev_notice(dev, "sev enabled\n");
2724
2725	return 0;
2726
2727e_irq:
2728	psp_clear_sev_irq_handler(psp);
2729e_buf:
2730	devm_free_pages(dev, (unsigned long)sev->cmd_buf);
2731e_sev:
2732	devm_kfree(dev, sev);
2733e_err:
2734	psp->sev_data = NULL;
2735
2736	dev_notice(dev, "sev initialization failed\n");
2737
2738	return ret;
2739}
2740
2741static void __sev_firmware_shutdown(struct sev_device *sev, bool panic)
2742{
2743	int error;
2744
2745	__sev_platform_shutdown_locked(&error);
2746
2747	if (sev_es_tmr) {
2748		/*
2749		 * The TMR area was encrypted, flush it from the cache.
2750		 *
2751		 * If invoked during panic handling, local interrupts are
2752		 * disabled and all CPUs are stopped, so wbinvd_on_all_cpus()
2753		 * can't be used. In that case, wbinvd() is done on remote CPUs
2754		 * via the NMI callback, and done for this CPU later during
2755		 * SNP shutdown, so wbinvd_on_all_cpus() can be skipped.
2756		 */
2757		if (!panic)
2758			wbinvd_on_all_cpus();
2759
2760		__snp_free_firmware_pages(virt_to_page(sev_es_tmr),
2761					  get_order(sev_es_tmr_size),
2762					  true);
2763		sev_es_tmr = NULL;
2764	}
2765
2766	if (sev_init_ex_buffer) {
2767		__snp_free_firmware_pages(virt_to_page(sev_init_ex_buffer),
2768					  get_order(NV_LENGTH),
2769					  true);
2770		sev_init_ex_buffer = NULL;
2771	}
2772
2773	__sev_snp_shutdown_locked(&error, panic);
2774}
2775
2776static void sev_firmware_shutdown(struct sev_device *sev)
2777{
2778	/*
2779	 * Calling without sev_cmd_mutex held as TSM will likely try disconnecting
2780	 * IDE and this ends up calling sev_do_cmd() which locks sev_cmd_mutex.
2781	 */
2782	if (sev->tio_status)
2783		sev_tsm_uninit(sev);
2784
2785	mutex_lock(&sev_cmd_mutex);
2786
2787	__sev_firmware_shutdown(sev, false);
2788
2789	kfree(sev->tio_status);
2790	sev->tio_status = NULL;
2791
2792	mutex_unlock(&sev_cmd_mutex);
2793}
2794
2795void sev_platform_shutdown(void)
2796{
2797	if (!psp_master || !psp_master->sev_data)
2798		return;
2799
2800	sev_firmware_shutdown(psp_master->sev_data);
2801}
2802EXPORT_SYMBOL_GPL(sev_platform_shutdown);
2803
2804u64 sev_get_snp_policy_bits(void)
2805{
2806	struct psp_device *psp = psp_master;
2807	struct sev_device *sev;
2808	u64 policy_bits;
2809
2810	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
2811		return 0;
2812
2813	if (!psp || !psp->sev_data)
2814		return 0;
2815
2816	sev = psp->sev_data;
2817
2818	policy_bits = SNP_POLICY_MASK_BASE;
2819
2820	if (sev->snp_plat_status.feature_info) {
2821		if (sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED)
2822			policy_bits |= SNP_POLICY_MASK_RAPL_DIS;
2823
2824		if (sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED)
2825			policy_bits |= SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM;
2826
2827		if (sev->snp_feat_info_0.ecx & SNP_AES_256_XTS_POLICY_SUPPORTED)
2828			policy_bits |= SNP_POLICY_MASK_MEM_AES_256_XTS;
2829
2830		if (sev->snp_feat_info_0.ecx & SNP_CXL_ALLOW_POLICY_SUPPORTED)
2831			policy_bits |= SNP_POLICY_MASK_CXL_ALLOW;
2832
2833		if (sev_version_greater_or_equal(1, 58))
2834			policy_bits |= SNP_POLICY_MASK_PAGE_SWAP_DISABLE;
2835	}
2836
2837	return policy_bits;
2838}
2839EXPORT_SYMBOL_GPL(sev_get_snp_policy_bits);
2840
2841void sev_dev_destroy(struct psp_device *psp)
2842{
2843	struct sev_device *sev = psp->sev_data;
2844
2845	if (!sev)
2846		return;
2847
2848	sev_firmware_shutdown(sev);
2849
2850	if (sev->misc)
2851		kref_put(&misc_dev->refcount, sev_exit);
2852
2853	psp_clear_sev_irq_handler(psp);
2854}
2855
2856static int snp_shutdown_on_panic(struct notifier_block *nb,
2857				 unsigned long reason, void *arg)
2858{
2859	struct sev_device *sev = psp_master->sev_data;
2860
2861	/*
2862	 * If sev_cmd_mutex is already acquired, then it's likely
2863	 * another PSP command is in flight and issuing a shutdown
2864	 * would fail in unexpected ways. Rather than create even
2865	 * more confusion during a panic, just bail out here.
2866	 */
2867	if (mutex_is_locked(&sev_cmd_mutex))
2868		return NOTIFY_DONE;
2869
2870	__sev_firmware_shutdown(sev, true);
2871
2872	return NOTIFY_DONE;
2873}
2874
2875int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd,
2876				void *data, int *error)
2877{
2878	if (!filep || filep->f_op != &sev_fops)
2879		return -EBADF;
2880
2881	return sev_do_cmd(cmd, data, error);
2882}
2883EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
2884
2885void sev_pci_init(void)
2886{
2887	struct sev_device *sev = psp_master->sev_data;
2888	u8 api_major, api_minor, build;
2889
2890	if (!sev)
2891		return;
2892
2893	psp_timeout = psp_probe_timeout;
2894
2895	if (sev_get_api_version())
2896		goto err;
2897
2898	api_major = sev->api_major;
2899	api_minor = sev->api_minor;
2900	build     = sev->build;
2901
2902	if (sev_update_firmware(sev->dev) == 0)
2903		sev_get_api_version();
2904
2905	if (api_major != sev->api_major || api_minor != sev->api_minor ||
2906	    build != sev->build)
2907		dev_info(sev->dev, "SEV firmware updated from %d.%d.%d to %d.%d.%d\n",
2908			 api_major, api_minor, build,
2909			 sev->api_major, sev->api_minor, sev->build);
2910
2911	return;
2912
2913err:
2914	sev_dev_destroy(psp_master);
2915
2916	psp_master->sev_data = NULL;
2917}
2918
2919void sev_pci_exit(void)
2920{
2921	struct sev_device *sev = psp_master->sev_data;
2922
2923	if (!sev)
2924		return;
2925
2926	sev_firmware_shutdown(sev);
2927}