mbox.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3#include <linux/security.h>
   4#include <linux/debugfs.h>
   5#include <linux/ktime.h>
   6#include <linux/mutex.h>
   7#include <linux/unaligned.h>
   8#include <cxlpci.h>
   9#include <cxlmem.h>
  10#include <cxl.h>
  11
  12#include "core.h"
  13#include "trace.h"
  14#include "mce.h"
  15
  16static bool cxl_raw_allow_all;
  17
  18/**
  19 * DOC: cxl mbox
  20 *
  21 * Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The
  22 * implementation is used by the cxl_pci driver to initialize the device
  23 * and implement the cxl_mem.h IOCTL UAPI. It also implements the
  24 * backend of the cxl_pmem_ctl() transport for LIBNVDIMM.
  25 */
  26
  27#define cxl_for_each_cmd(cmd)                                                  \
  28	for ((cmd) = &cxl_mem_commands[0];                                     \
  29	     ((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)
  30
  31#define CXL_CMD(_id, sin, sout, _flags)                                        \
  32	[CXL_MEM_COMMAND_ID_##_id] = {                                         \
  33	.info =	{                                                              \
  34			.id = CXL_MEM_COMMAND_ID_##_id,                        \
  35			.size_in = sin,                                        \
  36			.size_out = sout,                                      \
  37		},                                                             \
  38	.opcode = CXL_MBOX_OP_##_id,                                           \
  39	.flags = _flags,                                                       \
  40	}
  41
  42#define CXL_VARIABLE_PAYLOAD	~0U
  43/*
  44 * This table defines the supported mailbox commands for the driver. This table
  45 * is made up of a UAPI structure. Non-negative values as parameters in the
  46 * table will be validated against the user's input. For example, if size_in is
  47 * 0, and the user passed in 1, it is an error.
  48 */
  49static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
  50	CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
  51#ifdef CONFIG_CXL_MEM_RAW_COMMANDS
  52	CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
  53#endif
  54	CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
  55	CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
  56	CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
  57	CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),
  58	CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
  59	CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
  60	CXL_CMD(GET_LOG_CAPS, 0x10, 0x4, 0),
  61	CXL_CMD(CLEAR_LOG, 0x10, 0, 0),
  62	CXL_CMD(GET_SUP_LOG_SUBLIST, 0x2, CXL_VARIABLE_PAYLOAD, 0),
  63	CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
  64	CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),
  65	CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
  66	CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
  67	CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
  68	CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
  69	CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
  70	CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0),
  71};
  72
  73/*
  74 * Commands that RAW doesn't permit. The rationale for each:
  75 *
  76 * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
  77 * coordination of transaction timeout values at the root bridge level.
  78 *
  79 * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
  80 * and needs to be coordinated with HDM updates.
  81 *
  82 * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
  83 * driver and any writes from userspace invalidates those contents.
  84 *
  85 * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
  86 * to the device after it is marked clean, userspace can not make that
  87 * assertion.
  88 *
  89 * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
  90 * is kept up to date with patrol notifications and error management.
  91 *
  92 * CXL_MBOX_OP_[GET_,INJECT_,CLEAR_]POISON: These commands require kernel
  93 * driver orchestration for safety.
  94 */
  95static u16 cxl_disabled_raw_commands[] = {
  96	CXL_MBOX_OP_ACTIVATE_FW,
  97	CXL_MBOX_OP_SET_PARTITION_INFO,
  98	CXL_MBOX_OP_SET_LSA,
  99	CXL_MBOX_OP_SET_SHUTDOWN_STATE,
 100	CXL_MBOX_OP_SCAN_MEDIA,
 101	CXL_MBOX_OP_GET_SCAN_MEDIA,
 102	CXL_MBOX_OP_GET_POISON,
 103	CXL_MBOX_OP_INJECT_POISON,
 104	CXL_MBOX_OP_CLEAR_POISON,
 105};
 106
 107/*
 108 * Command sets that RAW doesn't permit. All opcodes in this set are
 109 * disabled because they pass plain text security payloads over the
 110 * user/kernel boundary. This functionality is intended to be wrapped
 111 * behind the keys ABI which allows for encrypted payloads in the UAPI
 112 */
 113static u8 security_command_sets[] = {
 114	0x44, /* Sanitize */
 115	0x45, /* Persistent Memory Data-at-rest Security */
 116	0x46, /* Security Passthrough */
 117};
 118
 119static bool cxl_is_security_command(u16 opcode)
 120{
 121	int i;
 122
 123	for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
 124		if (security_command_sets[i] == (opcode >> 8))
 125			return true;
 126	return false;
 127}
 128
 129static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,
 130					 u16 opcode)
 131{
 132	switch (opcode) {
 133	case CXL_MBOX_OP_SANITIZE:
 134		set_bit(CXL_SEC_ENABLED_SANITIZE, security->enabled_cmds);
 135		break;
 136	case CXL_MBOX_OP_SECURE_ERASE:
 137		set_bit(CXL_SEC_ENABLED_SECURE_ERASE,
 138			security->enabled_cmds);
 139		break;
 140	case CXL_MBOX_OP_GET_SECURITY_STATE:
 141		set_bit(CXL_SEC_ENABLED_GET_SECURITY_STATE,
 142			security->enabled_cmds);
 143		break;
 144	case CXL_MBOX_OP_SET_PASSPHRASE:
 145		set_bit(CXL_SEC_ENABLED_SET_PASSPHRASE,
 146			security->enabled_cmds);
 147		break;
 148	case CXL_MBOX_OP_DISABLE_PASSPHRASE:
 149		set_bit(CXL_SEC_ENABLED_DISABLE_PASSPHRASE,
 150			security->enabled_cmds);
 151		break;
 152	case CXL_MBOX_OP_UNLOCK:
 153		set_bit(CXL_SEC_ENABLED_UNLOCK, security->enabled_cmds);
 154		break;
 155	case CXL_MBOX_OP_FREEZE_SECURITY:
 156		set_bit(CXL_SEC_ENABLED_FREEZE_SECURITY,
 157			security->enabled_cmds);
 158		break;
 159	case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:
 160		set_bit(CXL_SEC_ENABLED_PASSPHRASE_SECURE_ERASE,
 161			security->enabled_cmds);
 162		break;
 163	default:
 164		break;
 165	}
 166}
 167
 168static bool cxl_is_poison_command(u16 opcode)
 169{
 170#define CXL_MBOX_OP_POISON_CMDS 0x43
 171
 172	if ((opcode >> 8) == CXL_MBOX_OP_POISON_CMDS)
 173		return true;
 174
 175	return false;
 176}
 177
 178static void cxl_set_poison_cmd_enabled(struct cxl_poison_state *poison,
 179				       u16 opcode)
 180{
 181	switch (opcode) {
 182	case CXL_MBOX_OP_GET_POISON:
 183		set_bit(CXL_POISON_ENABLED_LIST, poison->enabled_cmds);
 184		break;
 185	case CXL_MBOX_OP_INJECT_POISON:
 186		set_bit(CXL_POISON_ENABLED_INJECT, poison->enabled_cmds);
 187		break;
 188	case CXL_MBOX_OP_CLEAR_POISON:
 189		set_bit(CXL_POISON_ENABLED_CLEAR, poison->enabled_cmds);
 190		break;
 191	case CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS:
 192		set_bit(CXL_POISON_ENABLED_SCAN_CAPS, poison->enabled_cmds);
 193		break;
 194	case CXL_MBOX_OP_SCAN_MEDIA:
 195		set_bit(CXL_POISON_ENABLED_SCAN_MEDIA, poison->enabled_cmds);
 196		break;
 197	case CXL_MBOX_OP_GET_SCAN_MEDIA:
 198		set_bit(CXL_POISON_ENABLED_SCAN_RESULTS, poison->enabled_cmds);
 199		break;
 200	default:
 201		break;
 202	}
 203}
 204
 205static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
 206{
 207	struct cxl_mem_command *c;
 208
 209	cxl_for_each_cmd(c)
 210		if (c->opcode == opcode)
 211			return c;
 212
 213	return NULL;
 214}
 215
 216static const char *cxl_mem_opcode_to_name(u16 opcode)
 217{
 218	struct cxl_mem_command *c;
 219
 220	c = cxl_mem_find_command(opcode);
 221	if (!c)
 222		return NULL;
 223
 224	return cxl_command_names[c->info.id].name;
 225}
 226
 227/**
 228 * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
 229 * @cxl_mbox: CXL mailbox context
 230 * @mbox_cmd: initialized command to execute
 231 *
 232 * Context: Any context.
 233 * Return:
 234 *  * %>=0	- Number of bytes returned in @out.
 235 *  * %-E2BIG	- Payload is too large for hardware.
 236 *  * %-EBUSY	- Couldn't acquire exclusive mailbox access.
 237 *  * %-EFAULT	- Hardware error occurred.
 238 *  * %-ENXIO	- Command completed, but device reported an error.
 239 *  * %-EIO	- Unexpected output size.
 240 *
 241 * Mailbox commands may execute successfully yet the device itself reported an
 242 * error. While this distinction can be useful for commands from userspace, the
 243 * kernel will only be able to use results when both are successful.
 244 */
 245int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
 246			  struct cxl_mbox_cmd *mbox_cmd)
 247{
 248	size_t out_size, min_out;
 249	int rc;
 250
 251	if (mbox_cmd->size_in > cxl_mbox->payload_size ||
 252	    mbox_cmd->size_out > cxl_mbox->payload_size)
 253		return -E2BIG;
 254
 255	out_size = mbox_cmd->size_out;
 256	min_out = mbox_cmd->min_out;
 257	rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
 258	/*
 259	 * EIO is reserved for a payload size mismatch and mbox_send()
 260	 * may not return this error.
 261	 */
 262	if (WARN_ONCE(rc == -EIO, "Bad return code: -EIO"))
 263		return -ENXIO;
 264	if (rc)
 265		return rc;
 266
 267	if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&
 268	    mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)
 269		return cxl_mbox_cmd_rc2errno(mbox_cmd);
 270
 271	if (!out_size)
 272		return 0;
 273
 274	/*
 275	 * Variable sized output needs to at least satisfy the caller's
 276	 * minimum if not the fully requested size.
 277	 */
 278	if (min_out == 0)
 279		min_out = out_size;
 280
 281	if (mbox_cmd->size_out < min_out)
 282		return -EIO;
 283	return 0;
 284}
 285EXPORT_SYMBOL_NS_GPL(cxl_internal_send_cmd, "CXL");
 286
 287static bool cxl_mem_raw_command_allowed(u16 opcode)
 288{
 289	int i;
 290
 291	if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
 292		return false;
 293
 294	if (security_locked_down(LOCKDOWN_PCI_ACCESS))
 295		return false;
 296
 297	if (cxl_raw_allow_all)
 298		return true;
 299
 300	if (cxl_is_security_command(opcode))
 301		return false;
 302
 303	for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
 304		if (cxl_disabled_raw_commands[i] == opcode)
 305			return false;
 306
 307	return true;
 308}
 309
 310/**
 311 * cxl_payload_from_user_allowed() - Check contents of in_payload.
 312 * @opcode: The mailbox command opcode.
 313 * @payload_in: Pointer to the input payload passed in from user space.
 314 * @in_size: Size of @payload_in in bytes.
 315 *
 316 * Return:
 317 *  * true	- payload_in passes check for @opcode.
 318 *  * false	- payload_in contains invalid or unsupported values.
 319 *
 320 * The driver may inspect payload contents before sending a mailbox
 321 * command from user space to the device. The intent is to reject
 322 * commands with input payloads that are known to be unsafe. This
 323 * check is not intended to replace the users careful selection of
 324 * mailbox command parameters and makes no guarantee that the user
 325 * command will succeed, nor that it is appropriate.
 326 *
 327 * The specific checks are determined by the opcode.
 328 */
 329static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in,
 330					  size_t in_size)
 331{
 332	switch (opcode) {
 333	case CXL_MBOX_OP_SET_PARTITION_INFO: {
 334		struct cxl_mbox_set_partition_info *pi = payload_in;
 335
 336		if (in_size < sizeof(*pi))
 337			return false;
 338		if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)
 339			return false;
 340		break;
 341	}
 342	case CXL_MBOX_OP_CLEAR_LOG: {
 343		const uuid_t *uuid = (uuid_t *)payload_in;
 344
 345		if (in_size < sizeof(uuid_t))
 346			return false;
 347		/*
 348		 * Restrict the ‘Clear log’ action to only apply to
 349		 * Vendor debug logs.
 350		 */
 351		return uuid_equal(uuid, &DEFINE_CXL_VENDOR_DEBUG_UUID);
 352	}
 353	default:
 354		break;
 355	}
 356	return true;
 357}
 358
 359static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd,
 360			     struct cxl_mailbox *cxl_mbox, u16 opcode,
 361			     size_t in_size, size_t out_size, u64 in_payload)
 362{
 363	*mbox_cmd = (struct cxl_mbox_cmd) {
 364		.opcode = opcode,
 365		.size_in = in_size,
 366	};
 367
 368	if (in_size) {
 369		mbox_cmd->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
 370						    in_size);
 371		if (IS_ERR(mbox_cmd->payload_in))
 372			return PTR_ERR(mbox_cmd->payload_in);
 373
 374		if (!cxl_payload_from_user_allowed(opcode, mbox_cmd->payload_in,
 375						  in_size)) {
 376			dev_dbg(cxl_mbox->host, "%s: input payload not allowed\n",
 377				cxl_mem_opcode_to_name(opcode));
 378			kvfree(mbox_cmd->payload_in);
 379			return -EBUSY;
 380		}
 381	}
 382
 383	/* Prepare to handle a full payload for variable sized output */
 384	if (out_size == CXL_VARIABLE_PAYLOAD)
 385		mbox_cmd->size_out = cxl_mbox->payload_size;
 386	else
 387		mbox_cmd->size_out = out_size;
 388
 389	if (mbox_cmd->size_out) {
 390		mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL);
 391		if (!mbox_cmd->payload_out) {
 392			kvfree(mbox_cmd->payload_in);
 393			return -ENOMEM;
 394		}
 395	}
 396	return 0;
 397}
 398
 399static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
 400{
 401	kvfree(mbox->payload_in);
 402	kvfree(mbox->payload_out);
 403}
 404
 405static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
 406			      const struct cxl_send_command *send_cmd,
 407			      struct cxl_mailbox *cxl_mbox)
 408{
 409	if (send_cmd->raw.rsvd)
 410		return -EINVAL;
 411
 412	/*
 413	 * Unlike supported commands, the output size of RAW commands
 414	 * gets passed along without further checking, so it must be
 415	 * validated here.
 416	 */
 417	if (send_cmd->out.size > cxl_mbox->payload_size)
 418		return -EINVAL;
 419
 420	if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
 421		return -EPERM;
 422
 423	dev_WARN_ONCE(cxl_mbox->host, true, "raw command path used\n");
 424
 425	*mem_cmd = (struct cxl_mem_command) {
 426		.info = {
 427			.id = CXL_MEM_COMMAND_ID_RAW,
 428			.size_in = send_cmd->in.size,
 429			.size_out = send_cmd->out.size,
 430		},
 431		.opcode = send_cmd->raw.opcode
 432	};
 433
 434	return 0;
 435}
 436
 437static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
 438			  const struct cxl_send_command *send_cmd,
 439			  struct cxl_mailbox *cxl_mbox)
 440{
 441	struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
 442	const struct cxl_command_info *info = &c->info;
 443
 444	if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
 445		return -EINVAL;
 446
 447	if (send_cmd->rsvd)
 448		return -EINVAL;
 449
 450	if (send_cmd->in.rsvd || send_cmd->out.rsvd)
 451		return -EINVAL;
 452
 453	/* Check that the command is enabled for hardware */
 454	if (!test_bit(info->id, cxl_mbox->enabled_cmds))
 455		return -ENOTTY;
 456
 457	/* Check that the command is not claimed for exclusive kernel use */
 458	if (test_bit(info->id, cxl_mbox->exclusive_cmds))
 459		return -EBUSY;
 460
 461	/* Check the input buffer is the expected size */
 462	if ((info->size_in != CXL_VARIABLE_PAYLOAD) &&
 463	    (info->size_in != send_cmd->in.size))
 464		return -ENOMEM;
 465
 466	/* Check the output buffer is at least large enough */
 467	if ((info->size_out != CXL_VARIABLE_PAYLOAD) &&
 468	    (send_cmd->out.size < info->size_out))
 469		return -ENOMEM;
 470
 471	*mem_cmd = (struct cxl_mem_command) {
 472		.info = {
 473			.id = info->id,
 474			.flags = info->flags,
 475			.size_in = send_cmd->in.size,
 476			.size_out = send_cmd->out.size,
 477		},
 478		.opcode = c->opcode
 479	};
 480
 481	return 0;
 482}
 483
 484/**
 485 * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
 486 * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
 487 * @cxl_mbox: CXL mailbox context
 488 * @send_cmd: &struct cxl_send_command copied in from userspace.
 489 *
 490 * Return:
 491 *  * %0	- @out_cmd is ready to send.
 492 *  * %-ENOTTY	- Invalid command specified.
 493 *  * %-EINVAL	- Reserved fields or invalid values were used.
 494 *  * %-ENOMEM	- Input or output buffer wasn't sized properly.
 495 *  * %-EPERM	- Attempted to use a protected command.
 496 *  * %-EBUSY	- Kernel has claimed exclusive access to this opcode
 497 *
 498 * The result of this command is a fully validated command in @mbox_cmd that is
 499 * safe to send to the hardware.
 500 */
 501static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
 502				      struct cxl_mailbox *cxl_mbox,
 503				      const struct cxl_send_command *send_cmd)
 504{
 505	struct cxl_mem_command mem_cmd;
 506	int rc;
 507
 508	if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
 509		return -ENOTTY;
 510
 511	/*
 512	 * The user can never specify an input payload larger than what hardware
 513	 * supports, but output can be arbitrarily large (simply write out as
 514	 * much data as the hardware provides).
 515	 */
 516	if (send_cmd->in.size > cxl_mbox->payload_size)
 517		return -EINVAL;
 518
 519	/* Sanitize and construct a cxl_mem_command */
 520	if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
 521		rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, cxl_mbox);
 522	else
 523		rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, cxl_mbox);
 524
 525	if (rc)
 526		return rc;
 527
 528	/* Sanitize and construct a cxl_mbox_cmd */
 529	return cxl_mbox_cmd_ctor(mbox_cmd, cxl_mbox, mem_cmd.opcode,
 530				 mem_cmd.info.size_in, mem_cmd.info.size_out,
 531				 send_cmd->in.payload);
 532}
 533
 534int cxl_query_cmd(struct cxl_mailbox *cxl_mbox,
 535		  struct cxl_mem_query_commands __user *q)
 536{
 537	struct device *dev = cxl_mbox->host;
 538	struct cxl_mem_command *cmd;
 539	u32 n_commands;
 540	int j = 0;
 541
 542	dev_dbg(dev, "Query IOCTL\n");
 543
 544	if (get_user(n_commands, &q->n_commands))
 545		return -EFAULT;
 546
 547	/* returns the total number if 0 elements are requested. */
 548	if (n_commands == 0)
 549		return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
 550
 551	/*
 552	 * otherwise, return min(n_commands, total commands) cxl_command_info
 553	 * structures.
 554	 */
 555	cxl_for_each_cmd(cmd) {
 556		struct cxl_command_info info = cmd->info;
 557
 558		if (test_bit(info.id, cxl_mbox->enabled_cmds))
 559			info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
 560		if (test_bit(info.id, cxl_mbox->exclusive_cmds))
 561			info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
 562
 563		if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
 564			return -EFAULT;
 565
 566		if (j == n_commands)
 567			break;
 568	}
 569
 570	return 0;
 571}
 572
 573/**
 574 * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
 575 * @cxl_mbox: The mailbox context for the operation.
 576 * @mbox_cmd: The validated mailbox command.
 577 * @out_payload: Pointer to userspace's output payload.
 578 * @size_out: (Input) Max payload size to copy out.
 579 *            (Output) Payload size hardware generated.
 580 * @retval: Hardware generated return code from the operation.
 581 *
 582 * Return:
 583 *  * %0	- Mailbox transaction succeeded. This implies the mailbox
 584 *		  protocol completed successfully not that the operation itself
 585 *		  was successful.
 586 *  * %-ENOMEM  - Couldn't allocate a bounce buffer.
 587 *  * %-EFAULT	- Something happened with copy_to/from_user.
 588 *  * %-EINTR	- Mailbox acquisition interrupted.
 589 *  * %-EXXX	- Transaction level failures.
 590 *
 591 * Dispatches a mailbox command on behalf of a userspace request.
 592 * The output payload is copied to userspace.
 593 *
 594 * See cxl_send_cmd().
 595 */
 596static int handle_mailbox_cmd_from_user(struct cxl_mailbox *cxl_mbox,
 597					struct cxl_mbox_cmd *mbox_cmd,
 598					u64 out_payload, s32 *size_out,
 599					u32 *retval)
 600{
 601	struct device *dev = cxl_mbox->host;
 602	int rc;
 603
 604	dev_dbg(dev,
 605		"Submitting %s command for user\n"
 606		"\topcode: %x\n"
 607		"\tsize: %zx\n",
 608		cxl_mem_opcode_to_name(mbox_cmd->opcode),
 609		mbox_cmd->opcode, mbox_cmd->size_in);
 610
 611	rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd);
 612	if (rc)
 613		goto out;
 614
 615	/*
 616	 * @size_out contains the max size that's allowed to be written back out
 617	 * to userspace. While the payload may have written more output than
 618	 * this it will have to be ignored.
 619	 */
 620	if (mbox_cmd->size_out) {
 621		dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,
 622			      "Invalid return size\n");
 623		if (copy_to_user(u64_to_user_ptr(out_payload),
 624				 mbox_cmd->payload_out, mbox_cmd->size_out)) {
 625			rc = -EFAULT;
 626			goto out;
 627		}
 628	}
 629
 630	*size_out = mbox_cmd->size_out;
 631	*retval = mbox_cmd->return_code;
 632
 633out:
 634	cxl_mbox_cmd_dtor(mbox_cmd);
 635	return rc;
 636}
 637
 638int cxl_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_send_command __user *s)
 639{
 640	struct device *dev = cxl_mbox->host;
 641	struct cxl_send_command send;
 642	struct cxl_mbox_cmd mbox_cmd;
 643	int rc;
 644
 645	dev_dbg(dev, "Send IOCTL\n");
 646
 647	if (copy_from_user(&send, s, sizeof(send)))
 648		return -EFAULT;
 649
 650	rc = cxl_validate_cmd_from_user(&mbox_cmd, cxl_mbox, &send);
 651	if (rc)
 652		return rc;
 653
 654	rc = handle_mailbox_cmd_from_user(cxl_mbox, &mbox_cmd, send.out.payload,
 655					  &send.out.size, &send.retval);
 656	if (rc)
 657		return rc;
 658
 659	if (copy_to_user(s, &send, sizeof(send)))
 660		return -EFAULT;
 661
 662	return 0;
 663}
 664
 665static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
 666			u32 *size, u8 *out)
 667{
 668	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 669	u32 remaining = *size;
 670	u32 offset = 0;
 671
 672	while (remaining) {
 673		u32 xfer_size = min_t(u32, remaining, cxl_mbox->payload_size);
 674		struct cxl_mbox_cmd mbox_cmd;
 675		struct cxl_mbox_get_log log;
 676		int rc;
 677
 678		log = (struct cxl_mbox_get_log) {
 679			.uuid = *uuid,
 680			.offset = cpu_to_le32(offset),
 681			.length = cpu_to_le32(xfer_size),
 682		};
 683
 684		mbox_cmd = (struct cxl_mbox_cmd) {
 685			.opcode = CXL_MBOX_OP_GET_LOG,
 686			.size_in = sizeof(log),
 687			.payload_in = &log,
 688			.size_out = xfer_size,
 689			.payload_out = out,
 690		};
 691
 692		rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 693
 694		/*
 695		 * The output payload length that indicates the number
 696		 * of valid bytes can be smaller than the Log buffer
 697		 * size.
 698		 */
 699		if (rc == -EIO && mbox_cmd.size_out < xfer_size) {
 700			offset += mbox_cmd.size_out;
 701			break;
 702		}
 703
 704		if (rc < 0)
 705			return rc;
 706
 707		out += xfer_size;
 708		remaining -= xfer_size;
 709		offset += xfer_size;
 710	}
 711
 712	*size = offset;
 713
 714	return 0;
 715}
 716
 717static int check_features_opcodes(u16 opcode, int *ro_cmds, int *wr_cmds)
 718{
 719	switch (opcode) {
 720	case CXL_MBOX_OP_GET_SUPPORTED_FEATURES:
 721	case CXL_MBOX_OP_GET_FEATURE:
 722		(*ro_cmds)++;
 723		return 1;
 724	case CXL_MBOX_OP_SET_FEATURE:
 725		(*wr_cmds)++;
 726		return 1;
 727	default:
 728		return 0;
 729	}
 730}
 731
 732/* 'Get Supported Features' and 'Get Feature' */
 733#define MAX_FEATURES_READ_CMDS	2
 734static void set_features_cap(struct cxl_mailbox *cxl_mbox,
 735			     int ro_cmds, int wr_cmds)
 736{
 737	/* Setting up Features capability while walking the CEL */
 738	if (ro_cmds == MAX_FEATURES_READ_CMDS) {
 739		if (wr_cmds)
 740			cxl_mbox->feat_cap = CXL_FEATURES_RW;
 741		else
 742			cxl_mbox->feat_cap = CXL_FEATURES_RO;
 743	}
 744}
 745
 746/**
 747 * cxl_walk_cel() - Walk through the Command Effects Log.
 748 * @mds: The driver data for the operation
 749 * @size: Length of the Command Effects Log.
 750 * @cel: CEL
 751 *
 752 * Iterate over each entry in the CEL and determine if the driver supports the
 753 * command. If so, the command is enabled for the device and can be used later.
 754 */
 755static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
 756{
 757	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 758	struct cxl_cel_entry *cel_entry;
 759	const int cel_entries = size / sizeof(*cel_entry);
 760	struct device *dev = mds->cxlds.dev;
 761	int i, ro_cmds = 0, wr_cmds = 0;
 762
 763	cel_entry = (struct cxl_cel_entry *) cel;
 764
 765	for (i = 0; i < cel_entries; i++) {
 766		u16 opcode = le16_to_cpu(cel_entry[i].opcode);
 767		struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
 768		int enabled = 0;
 769
 770		if (cmd) {
 771			set_bit(cmd->info.id, cxl_mbox->enabled_cmds);
 772			enabled++;
 773		}
 774
 775		enabled += check_features_opcodes(opcode, &ro_cmds,
 776						  &wr_cmds);
 777
 778		if (cxl_is_poison_command(opcode)) {
 779			cxl_set_poison_cmd_enabled(&mds->poison, opcode);
 780			enabled++;
 781		}
 782
 783		if (cxl_is_security_command(opcode)) {
 784			cxl_set_security_cmd_enabled(&mds->security, opcode);
 785			enabled++;
 786		}
 787
 788		dev_dbg(dev, "Opcode 0x%04x %s\n", opcode,
 789			enabled ? "enabled" : "unsupported by driver");
 790	}
 791
 792	set_features_cap(cxl_mbox, ro_cmds, wr_cmds);
 793}
 794
 795static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
 796{
 797	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 798	struct cxl_mbox_get_supported_logs *ret;
 799	struct cxl_mbox_cmd mbox_cmd;
 800	int rc;
 801
 802	ret = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
 803	if (!ret)
 804		return ERR_PTR(-ENOMEM);
 805
 806	mbox_cmd = (struct cxl_mbox_cmd) {
 807		.opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
 808		.size_out = cxl_mbox->payload_size,
 809		.payload_out = ret,
 810		/* At least the record number field must be valid */
 811		.min_out = 2,
 812	};
 813	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
 814	if (rc < 0) {
 815		kvfree(ret);
 816		return ERR_PTR(rc);
 817	}
 818
 819
 820	return ret;
 821}
 822
 823enum {
 824	CEL_UUID,
 825	VENDOR_DEBUG_UUID,
 826};
 827
 828/* See CXL 2.0 Table 170. Get Log Input Payload */
 829static const uuid_t log_uuid[] = {
 830	[CEL_UUID] = DEFINE_CXL_CEL_UUID,
 831	[VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
 832};
 833
 834/**
 835 * cxl_enumerate_cmds() - Enumerate commands for a device.
 836 * @mds: The driver data for the operation
 837 *
 838 * Returns 0 if enumerate completed successfully.
 839 *
 840 * CXL devices have optional support for certain commands. This function will
 841 * determine the set of supported commands for the hardware and update the
 842 * enabled_cmds bitmap in the @mds.
 843 */
 844int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
 845{
 846	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 847	struct cxl_mbox_get_supported_logs *gsl;
 848	struct device *dev = mds->cxlds.dev;
 849	struct cxl_mem_command *cmd;
 850	int i, rc;
 851
 852	gsl = cxl_get_gsl(mds);
 853	if (IS_ERR(gsl))
 854		return PTR_ERR(gsl);
 855
 856	rc = -ENOENT;
 857	for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
 858		u32 size = le32_to_cpu(gsl->entry[i].size);
 859		uuid_t uuid = gsl->entry[i].uuid;
 860		u8 *log;
 861
 862		dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
 863
 864		if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
 865			continue;
 866
 867		log = kvmalloc(size, GFP_KERNEL);
 868		if (!log) {
 869			rc = -ENOMEM;
 870			goto out;
 871		}
 872
 873		rc = cxl_xfer_log(mds, &uuid, &size, log);
 874		if (rc) {
 875			kvfree(log);
 876			goto out;
 877		}
 878
 879		cxl_walk_cel(mds, size, log);
 880		kvfree(log);
 881
 882		/* In case CEL was bogus, enable some default commands. */
 883		cxl_for_each_cmd(cmd)
 884			if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
 885				set_bit(cmd->info.id, cxl_mbox->enabled_cmds);
 886
 887		/* Found the required CEL */
 888		rc = 0;
 889	}
 890out:
 891	kvfree(gsl);
 892	return rc;
 893}
 894EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, "CXL");
 895
 896void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
 897			    enum cxl_event_log_type type,
 898			    enum cxl_event_type event_type,
 899			    const uuid_t *uuid, union cxl_event *evt)
 900{
 901	if (event_type == CXL_CPER_EVENT_MEM_MODULE) {
 902		trace_cxl_memory_module(cxlmd, type, &evt->mem_module);
 903		return;
 904	}
 905	if (event_type == CXL_CPER_EVENT_GENERIC) {
 906		trace_cxl_generic_event(cxlmd, type, uuid, &evt->generic);
 907		return;
 908	}
 909	if (event_type == CXL_CPER_EVENT_MEM_SPARING) {
 910		trace_cxl_memory_sparing(cxlmd, type, &evt->mem_sparing);
 911		return;
 912	}
 913
 914	if (trace_cxl_general_media_enabled() || trace_cxl_dram_enabled()) {
 915		u64 dpa, hpa = ULLONG_MAX, hpa_alias = ULLONG_MAX;
 916		struct cxl_region *cxlr;
 917
 918		/*
 919		 * These trace points are annotated with HPA and region
 920		 * translations. Take topology mutation locks and lookup
 921		 * { HPA, REGION } from { DPA, MEMDEV } in the event record.
 922		 */
 923		guard(rwsem_read)(&cxl_rwsem.region);
 924		guard(rwsem_read)(&cxl_rwsem.dpa);
 925
 926		dpa = le64_to_cpu(evt->media_hdr.phys_addr) & CXL_DPA_MASK;
 927		cxlr = cxl_dpa_to_region(cxlmd, dpa);
 928		if (cxlr) {
 929			u64 cache_size = cxlr->params.cache_size;
 930
 931			hpa = cxl_dpa_to_hpa(cxlr, cxlmd, dpa);
 932			if (cache_size)
 933				hpa_alias = hpa - cache_size;
 934		}
 935
 936		if (event_type == CXL_CPER_EVENT_GEN_MEDIA) {
 937			if (cxl_store_rec_gen_media((struct cxl_memdev *)cxlmd, evt))
 938				dev_dbg(&cxlmd->dev, "CXL store rec_gen_media failed\n");
 939
 940			if (evt->gen_media.media_hdr.descriptor &
 941			    CXL_GMER_EVT_DESC_THRESHOLD_EVENT)
 942				WARN_ON_ONCE((evt->gen_media.media_hdr.type &
 943					      CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) &&
 944					     !get_unaligned_le24(evt->gen_media.cme_count));
 945			else
 946				WARN_ON_ONCE(evt->gen_media.media_hdr.type &
 947					     CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE);
 948
 949			trace_cxl_general_media(cxlmd, type, cxlr, hpa,
 950						hpa_alias, &evt->gen_media);
 951		} else if (event_type == CXL_CPER_EVENT_DRAM) {
 952			if (cxl_store_rec_dram((struct cxl_memdev *)cxlmd, evt))
 953				dev_dbg(&cxlmd->dev, "CXL store rec_dram failed\n");
 954
 955			if (evt->dram.media_hdr.descriptor &
 956			    CXL_GMER_EVT_DESC_THRESHOLD_EVENT)
 957				WARN_ON_ONCE((evt->dram.media_hdr.type &
 958					      CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) &&
 959					     !get_unaligned_le24(evt->dram.cvme_count));
 960			else
 961				WARN_ON_ONCE(evt->dram.media_hdr.type &
 962					     CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE);
 963
 964			trace_cxl_dram(cxlmd, type, cxlr, hpa, hpa_alias,
 965				       &evt->dram);
 966		}
 967	}
 968}
 969EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, "CXL");
 970
 971static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd,
 972				     enum cxl_event_log_type type,
 973				     struct cxl_event_record_raw *record)
 974{
 975	enum cxl_event_type ev_type = CXL_CPER_EVENT_GENERIC;
 976	const uuid_t *uuid = &record->id;
 977
 978	if (uuid_equal(uuid, &CXL_EVENT_GEN_MEDIA_UUID))
 979		ev_type = CXL_CPER_EVENT_GEN_MEDIA;
 980	else if (uuid_equal(uuid, &CXL_EVENT_DRAM_UUID))
 981		ev_type = CXL_CPER_EVENT_DRAM;
 982	else if (uuid_equal(uuid, &CXL_EVENT_MEM_MODULE_UUID))
 983		ev_type = CXL_CPER_EVENT_MEM_MODULE;
 984	else if (uuid_equal(uuid, &CXL_EVENT_MEM_SPARING_UUID))
 985		ev_type = CXL_CPER_EVENT_MEM_SPARING;
 986
 987	cxl_event_trace_record(cxlmd, type, ev_type, uuid, &record->event);
 988}
 989
 990static int cxl_clear_event_record(struct cxl_memdev_state *mds,
 991				  enum cxl_event_log_type log,
 992				  struct cxl_get_event_payload *get_pl)
 993{
 994	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 995	struct cxl_mbox_clear_event_payload *payload;
 996	u16 total = le16_to_cpu(get_pl->record_count);
 997	u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
 998	size_t pl_size = struct_size(payload, handles, max_handles);
 999	struct cxl_mbox_cmd mbox_cmd;
1000	u16 cnt;
1001	int rc = 0;
1002	int i;
1003
1004	/* Payload size may limit the max handles */
1005	if (pl_size > cxl_mbox->payload_size) {
1006		max_handles = (cxl_mbox->payload_size - sizeof(*payload)) /
1007			      sizeof(__le16);
1008		pl_size = struct_size(payload, handles, max_handles);
1009	}
1010
1011	payload = kvzalloc(pl_size, GFP_KERNEL);
1012	if (!payload)
1013		return -ENOMEM;
1014
1015	*payload = (struct cxl_mbox_clear_event_payload) {
1016		.event_log = log,
1017	};
1018
1019	mbox_cmd = (struct cxl_mbox_cmd) {
1020		.opcode = CXL_MBOX_OP_CLEAR_EVENT_RECORD,
1021		.payload_in = payload,
1022		.size_in = pl_size,
1023	};
1024
1025	/*
1026	 * Clear Event Records uses u8 for the handle cnt while Get Event
1027	 * Record can return up to 0xffff records.
1028	 */
1029	i = 0;
1030	for (cnt = 0; cnt < total; cnt++) {
1031		struct cxl_event_record_raw *raw = &get_pl->records[cnt];
1032		struct cxl_event_generic *gen = &raw->event.generic;
1033
1034		payload->handles[i++] = gen->hdr.handle;
1035		dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,
1036			le16_to_cpu(payload->handles[i - 1]));
1037
1038		if (i == max_handles) {
1039			payload->nr_recs = i;
1040			rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1041			if (rc)
1042				goto free_pl;
1043			i = 0;
1044		}
1045	}
1046
1047	/* Clear what is left if any */
1048	if (i) {
1049		payload->nr_recs = i;
1050		mbox_cmd.size_in = struct_size(payload, handles, i);
1051		rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1052		if (rc)
1053			goto free_pl;
1054	}
1055
1056free_pl:
1057	kvfree(payload);
1058	return rc;
1059}
1060
1061static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
1062				    enum cxl_event_log_type type)
1063{
1064	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1065	struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
1066	struct device *dev = mds->cxlds.dev;
1067	struct cxl_get_event_payload *payload;
1068	u8 log_type = type;
1069	u16 nr_rec;
1070
1071	mutex_lock(&mds->event.log_lock);
1072	payload = mds->event.buf;
1073
1074	do {
1075		int rc, i;
1076		struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) {
1077			.opcode = CXL_MBOX_OP_GET_EVENT_RECORD,
1078			.payload_in = &log_type,
1079			.size_in = sizeof(log_type),
1080			.payload_out = payload,
1081			.size_out = cxl_mbox->payload_size,
1082			.min_out = struct_size(payload, records, 0),
1083		};
1084
1085		rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1086		if (rc) {
1087			dev_err_ratelimited(dev,
1088				"Event log '%d': Failed to query event records : %d",
1089				type, rc);
1090			break;
1091		}
1092
1093		nr_rec = le16_to_cpu(payload->record_count);
1094		if (!nr_rec)
1095			break;
1096
1097		for (i = 0; i < nr_rec; i++)
1098			__cxl_event_trace_record(cxlmd, type,
1099						 &payload->records[i]);
1100
1101		if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)
1102			trace_cxl_overflow(cxlmd, type, payload);
1103
1104		rc = cxl_clear_event_record(mds, type, payload);
1105		if (rc) {
1106			dev_err_ratelimited(dev,
1107				"Event log '%d': Failed to clear events : %d",
1108				type, rc);
1109			break;
1110		}
1111	} while (nr_rec);
1112
1113	mutex_unlock(&mds->event.log_lock);
1114}
1115
1116/**
1117 * cxl_mem_get_event_records - Get Event Records from the device
1118 * @mds: The driver data for the operation
1119 * @status: Event Status register value identifying which events are available.
1120 *
1121 * Retrieve all event records available on the device, report them as trace
1122 * events, and clear them.
1123 *
1124 * See CXL rev 3.0 @8.2.9.2.2 Get Event Records
1125 * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records
1126 */
1127void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)
1128{
1129	dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);
1130
1131	if (status & CXLDEV_EVENT_STATUS_FATAL)
1132		cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);
1133	if (status & CXLDEV_EVENT_STATUS_FAIL)
1134		cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);
1135	if (status & CXLDEV_EVENT_STATUS_WARN)
1136		cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);
1137	if (status & CXLDEV_EVENT_STATUS_INFO)
1138		cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);
1139}
1140EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, "CXL");
1141
1142/**
1143 * cxl_mem_get_partition_info - Get partition info
1144 * @mds: The driver data for the operation
1145 *
1146 * Retrieve the current partition info for the device specified.  The active
1147 * values are the current capacity in bytes.  If not 0, the 'next' values are
1148 * the pending values, in bytes, which take affect on next cold reset.
1149 *
1150 * Return: 0 if no error: or the result of the mailbox command.
1151 *
1152 * See CXL @8.2.9.5.2.1 Get Partition Info
1153 */
1154static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
1155{
1156	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1157	struct cxl_mbox_get_partition_info pi;
1158	struct cxl_mbox_cmd mbox_cmd;
1159	int rc;
1160
1161	mbox_cmd = (struct cxl_mbox_cmd) {
1162		.opcode = CXL_MBOX_OP_GET_PARTITION_INFO,
1163		.size_out = sizeof(pi),
1164		.payload_out = &pi,
1165	};
1166	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1167	if (rc)
1168		return rc;
1169
1170	mds->active_volatile_bytes =
1171		le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1172	mds->active_persistent_bytes =
1173		le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
1174
1175	return 0;
1176}
1177
1178/**
1179 * cxl_dev_state_identify() - Send the IDENTIFY command to the device.
1180 * @mds: The driver data for the operation
1181 *
1182 * Return: 0 if identify was executed successfully or media not ready.
1183 *
1184 * This will dispatch the identify command to the device and on success populate
1185 * structures to be exported to sysfs.
1186 */
1187int cxl_dev_state_identify(struct cxl_memdev_state *mds)
1188{
1189	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1190	/* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
1191	struct cxl_mbox_identify id;
1192	struct cxl_mbox_cmd mbox_cmd;
1193	u32 val;
1194	int rc;
1195
1196	if (!mds->cxlds.media_ready)
1197		return 0;
1198
1199	mbox_cmd = (struct cxl_mbox_cmd) {
1200		.opcode = CXL_MBOX_OP_IDENTIFY,
1201		.size_out = sizeof(id),
1202		.payload_out = &id,
1203	};
1204	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1205	if (rc < 0)
1206		return rc;
1207
1208	mds->total_bytes =
1209		le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
1210	mds->volatile_only_bytes =
1211		le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
1212	mds->persistent_only_bytes =
1213		le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
1214	mds->partition_align_bytes =
1215		le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
1216
1217	mds->lsa_size = le32_to_cpu(id.lsa_size);
1218	memcpy(mds->firmware_version, id.fw_revision,
1219	       sizeof(id.fw_revision));
1220
1221	if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {
1222		val = get_unaligned_le24(id.poison_list_max_mer);
1223		mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
1224	}
1225
1226	return 0;
1227}
1228EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, "CXL");
1229
1230static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
1231{
1232	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1233	int rc;
1234	u32 sec_out = 0;
1235	struct cxl_get_security_output {
1236		__le32 flags;
1237	} out;
1238	struct cxl_mbox_cmd sec_cmd = {
1239		.opcode = CXL_MBOX_OP_GET_SECURITY_STATE,
1240		.payload_out = &out,
1241		.size_out = sizeof(out),
1242	};
1243	struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
1244
1245	if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
1246		return -EINVAL;
1247
1248	rc = cxl_internal_send_cmd(cxl_mbox, &sec_cmd);
1249	if (rc < 0) {
1250		dev_err(cxl_mbox->host, "Failed to get security state : %d", rc);
1251		return rc;
1252	}
1253
1254	/*
1255	 * Prior to using these commands, any security applied to
1256	 * the user data areas of the device shall be DISABLED (or
1257	 * UNLOCKED for secure erase case).
1258	 */
1259	sec_out = le32_to_cpu(out.flags);
1260	if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)
1261		return -EINVAL;
1262
1263	if (cmd == CXL_MBOX_OP_SECURE_ERASE &&
1264	    sec_out & CXL_PMEM_SEC_STATE_LOCKED)
1265		return -EINVAL;
1266
1267	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1268	if (rc < 0) {
1269		dev_err(cxl_mbox->host, "Failed to sanitize device : %d", rc);
1270		return rc;
1271	}
1272
1273	return 0;
1274}
1275
1276
1277/**
1278 * cxl_mem_sanitize() - Send a sanitization command to the device.
1279 * @cxlmd: The device for the operation
1280 * @cmd: The specific sanitization command opcode
1281 *
1282 * Return: 0 if the command was executed successfully, regardless of
1283 * whether or not the actual security operation is done in the background,
1284 * such as for the Sanitize case.
1285 * Error return values can be the result of the mailbox command, -EINVAL
1286 * when security requirements are not met or invalid contexts, or -EBUSY
1287 * if the sanitize operation is already in flight.
1288 *
1289 * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.
1290 */
1291int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
1292{
1293	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1294	struct cxl_port  *endpoint;
1295
1296	/* synchronize with cxl_mem_probe() and decoder write operations */
1297	guard(device)(&cxlmd->dev);
1298	endpoint = cxlmd->endpoint;
1299	guard(rwsem_read)(&cxl_rwsem.region);
1300	/*
1301	 * Require an endpoint to be safe otherwise the driver can not
1302	 * be sure that the device is unmapped.
1303	 */
1304	if (endpoint && cxl_num_decoders_committed(endpoint) == 0)
1305		return __cxl_mem_sanitize(mds, cmd);
1306
1307	return -EBUSY;
1308}
1309
1310static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
1311{
1312	int i = info->nr_partitions;
1313
1314	if (size == 0)
1315		return;
1316
1317	info->part[i].range = (struct range) {
1318		.start = start,
1319		.end = start + size - 1,
1320	};
1321	info->part[i].mode = mode;
1322	info->nr_partitions++;
1323}
1324
1325int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
1326{
1327	struct cxl_dev_state *cxlds = &mds->cxlds;
1328	struct device *dev = cxlds->dev;
1329	int rc;
1330
1331	if (!cxlds->media_ready) {
1332		info->size = 0;
1333		return 0;
1334	}
1335
1336	info->size = mds->total_bytes;
1337
1338	if (mds->partition_align_bytes == 0) {
1339		add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
1340		add_part(info, mds->volatile_only_bytes,
1341			 mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
1342		return 0;
1343	}
1344
1345	rc = cxl_mem_get_partition_info(mds);
1346	if (rc) {
1347		dev_err(dev, "Failed to query partition information\n");
1348		return rc;
1349	}
1350
1351	add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
1352	add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
1353		 CXL_PARTMODE_PMEM);
1354
1355	return 0;
1356}
1357EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
1358
1359int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count)
1360{
1361	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1362	struct cxl_mbox_get_health_info_out hi;
1363	struct cxl_mbox_cmd mbox_cmd;
1364	int rc;
1365
1366	mbox_cmd = (struct cxl_mbox_cmd) {
1367		.opcode = CXL_MBOX_OP_GET_HEALTH_INFO,
1368		.size_out = sizeof(hi),
1369		.payload_out = &hi,
1370	};
1371
1372	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1373	if (!rc)
1374		*count = le32_to_cpu(hi.dirty_shutdown_cnt);
1375
1376	return rc;
1377}
1378EXPORT_SYMBOL_NS_GPL(cxl_get_dirty_count, "CXL");
1379
1380int cxl_arm_dirty_shutdown(struct cxl_memdev_state *mds)
1381{
1382	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1383	struct cxl_mbox_cmd mbox_cmd;
1384	struct cxl_mbox_set_shutdown_state_in in = {
1385		.state = 1
1386	};
1387
1388	mbox_cmd = (struct cxl_mbox_cmd) {
1389		.opcode = CXL_MBOX_OP_SET_SHUTDOWN_STATE,
1390		.size_in = sizeof(in),
1391		.payload_in = &in,
1392	};
1393
1394	return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1395}
1396EXPORT_SYMBOL_NS_GPL(cxl_arm_dirty_shutdown, "CXL");
1397
1398int cxl_set_timestamp(struct cxl_memdev_state *mds)
1399{
1400	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1401	struct cxl_mbox_cmd mbox_cmd;
1402	struct cxl_mbox_set_timestamp_in pi;
1403	int rc;
1404
1405	pi.timestamp = cpu_to_le64(ktime_get_real_ns());
1406	mbox_cmd = (struct cxl_mbox_cmd) {
1407		.opcode = CXL_MBOX_OP_SET_TIMESTAMP,
1408		.size_in = sizeof(pi),
1409		.payload_in = &pi,
1410	};
1411
1412	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1413	/*
1414	 * Command is optional. Devices may have another way of providing
1415	 * a timestamp, or may return all 0s in timestamp fields.
1416	 * Don't report an error if this command isn't supported
1417	 */
1418	if (rc && (mbox_cmd.return_code != CXL_MBOX_CMD_RC_UNSUPPORTED))
1419		return rc;
1420
1421	return 0;
1422}
1423EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, "CXL");
1424
1425int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
1426		       struct cxl_region *cxlr)
1427{
1428	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1429	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
1430	struct cxl_mbox_poison_out *po;
1431	struct cxl_mbox_poison_in pi;
1432	int nr_records = 0;
1433	int rc;
1434
1435	ACQUIRE(mutex_intr, lock)(&mds->poison.mutex);
1436	if ((rc = ACQUIRE_ERR(mutex_intr, &lock)))
1437		return rc;
1438
1439	po = mds->poison.list_out;
1440	pi.offset = cpu_to_le64(offset);
1441	pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);
1442
1443	do {
1444		struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd){
1445			.opcode = CXL_MBOX_OP_GET_POISON,
1446			.size_in = sizeof(pi),
1447			.payload_in = &pi,
1448			.size_out = cxl_mbox->payload_size,
1449			.payload_out = po,
1450			.min_out = struct_size(po, record, 0),
1451		};
1452
1453		rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1454		if (rc)
1455			break;
1456
1457		for (int i = 0; i < le16_to_cpu(po->count); i++)
1458			trace_cxl_poison(cxlmd, cxlr, &po->record[i],
1459					 po->flags, po->overflow_ts,
1460					 CXL_POISON_TRACE_LIST);
1461
1462		/* Protect against an uncleared _FLAG_MORE */
1463		nr_records = nr_records + le16_to_cpu(po->count);
1464		if (nr_records >= mds->poison.max_errors) {
1465			dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",
1466				nr_records);
1467			break;
1468		}
1469	} while (po->flags & CXL_POISON_FLAG_MORE);
1470
1471	return rc;
1472}
1473EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, "CXL");
1474
1475static void free_poison_buf(void *buf)
1476{
1477	kvfree(buf);
1478}
1479
1480/* Get Poison List output buffer is protected by mds->poison.lock */
1481static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
1482{
1483	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1484
1485	mds->poison.list_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
1486	if (!mds->poison.list_out)
1487		return -ENOMEM;
1488
1489	return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,
1490					mds->poison.list_out);
1491}
1492
1493int cxl_poison_state_init(struct cxl_memdev_state *mds)
1494{
1495	int rc;
1496
1497	if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))
1498		return 0;
1499
1500	rc = cxl_poison_alloc_buf(mds);
1501	if (rc) {
1502		clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);
1503		return rc;
1504	}
1505
1506	mutex_init(&mds->poison.mutex);
1507	return 0;
1508}
1509EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, "CXL");
1510
1511int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host)
1512{
1513	if (!cxl_mbox || !host)
1514		return -EINVAL;
1515
1516	cxl_mbox->host = host;
1517	mutex_init(&cxl_mbox->mbox_mutex);
1518	rcuwait_init(&cxl_mbox->mbox_wait);
1519
1520	return 0;
1521}
1522EXPORT_SYMBOL_NS_GPL(cxl_mailbox_init, "CXL");
1523
1524struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
1525{
1526	struct cxl_memdev_state *mds;
1527	int rc;
1528
1529	mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
1530	if (!mds) {
1531		dev_err(dev, "No memory available\n");
1532		return ERR_PTR(-ENOMEM);
1533	}
1534
1535	mutex_init(&mds->event.log_lock);
1536	mds->cxlds.dev = dev;
1537	mds->cxlds.reg_map.host = dev;
1538	mds->cxlds.cxl_mbox.host = dev;
1539	mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE;
1540	mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
1541
1542	rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier);
1543	if (rc == -EOPNOTSUPP)
1544		dev_warn(dev, "CXL MCE unsupported\n");
1545	else if (rc)
1546		return ERR_PTR(rc);
1547
1548	return mds;
1549}
1550EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL");
1551
1552void __init cxl_mbox_init(void)
1553{
1554	struct dentry *mbox_debugfs;
1555
1556	mbox_debugfs = cxl_debugfs_create_dir("mbox");
1557	debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
1558			    &cxl_raw_allow_all);
1559}