raw_mode.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * System Control and Management Interface (SCMI) Raw mode support
   4 *
   5 * Copyright (C) 2022 ARM Ltd.
   6 */
   7/**
   8 * DOC: Theory of operation
   9 *
  10 * When enabled the SCMI Raw mode support exposes a userspace API which allows
  11 * to send and receive SCMI commands, replies and notifications from a user
  12 * application through injection and snooping of bare SCMI messages in binary
  13 * little-endian format.
  14 *
  15 * Such injected SCMI transactions will then be routed through the SCMI core
  16 * stack towards the SCMI backend server using whatever SCMI transport is
  17 * currently configured on the system under test.
  18 *
  19 * It is meant to help in running any sort of SCMI backend server testing, no
  20 * matter where the server is placed, as long as it is normally reachable via
  21 * the transport configured on the system.
  22 *
  23 * It is activated by a Kernel configuration option since it is NOT meant to
  24 * be used in production but only during development and in CI deployments.
  25 *
  26 * In order to avoid possible interferences between the SCMI Raw transactions
  27 * originated from a test-suite and the normal operations of the SCMI drivers,
  28 * when Raw mode is enabled, by default, all the regular SCMI drivers are
  29 * inhibited, unless CONFIG_ARM_SCMI_RAW_MODE_SUPPORT_COEX is enabled: in this
  30 * latter case the regular SCMI stack drivers will be loaded as usual and it is
  31 * up to the user of this interface to take care of manually inhibiting the
  32 * regular SCMI drivers in order to avoid interferences during the test runs.
  33 *
  34 * The exposed API is as follows.
  35 *
  36 * All SCMI Raw entries are rooted under a common top /raw debugfs top directory
  37 * which in turn is rooted under the corresponding underlying  SCMI instance.
  38 *
  39 * /sys/kernel/debug/scmi/
  40 * `-- 0
  41 *     |-- atomic_threshold_us
  42 *     |-- instance_name
  43 *     |-- raw
  44 *     |   |-- channels
  45 *     |   |   |-- 0x10
  46 *     |   |   |   |-- message
  47 *     |   |   |   `-- message_async
  48 *     |   |   `-- 0x13
  49 *     |   |       |-- message
  50 *     |   |       `-- message_async
  51 *     |   |-- errors
  52 *     |   |-- message
  53 *     |   |-- message_async
  54 *     |   |-- notification
  55 *     |   `-- reset
  56 *     `-- transport
  57 *         |-- is_atomic
  58 *         |-- max_msg_size
  59 *         |-- max_rx_timeout_ms
  60 *         |-- rx_max_msg
  61 *         |-- tx_max_msg
  62 *         `-- type
  63 *
  64 * where:
  65 *
  66 *  - errors: used to read back timed-out and unexpected replies
  67 *  - message*: used to send sync/async commands and read back immediate and
  68 *		delayed reponses (if any)
  69 *  - notification: used to read any notification being emitted by the system
  70 *		    (if previously enabled by the user app)
  71 *  - reset: used to flush the queues of messages (of any kind) still pending
  72 *	     to be read; this is useful at test-suite start/stop to get
  73 *	     rid of any unread messages from the previous run.
  74 *
  75 * with the per-channel entries rooted at /channels being present only on a
  76 * system where multiple transport channels have been configured.
  77 *
  78 * Such per-channel entries can be used to explicitly choose a specific channel
  79 * for SCMI bare message injection, in contrast with the general entries above
  80 * where, instead, the selection of the proper channel to use is automatically
  81 * performed based the protocol embedded in the injected message and on how the
  82 * transport is configured on the system.
  83 *
  84 * Note that other common general entries are available under transport/ to let
  85 * the user applications properly make up their expectations in terms of
  86 * timeouts and message characteristics.
  87 *
  88 * Each write to the message* entries causes one command request to be built
  89 * and sent while the replies or delayed response are read back from those same
  90 * entries one message at time (receiving an EOF at each message boundary).
  91 *
  92 * The user application running the test is in charge of handling timeouts
  93 * on replies and properly choosing SCMI sequence numbers for the outgoing
  94 * requests (using the same sequence number is supported but discouraged).
  95 *
  96 * Injection of multiple in-flight requests is supported as long as the user
  97 * application uses properly distinct sequence numbers for concurrent requests
  98 * and takes care to properly manage all the related issues about concurrency
  99 * and command/reply pairing. Keep in mind that, anyway, the real level of
 100 * parallelism attainable in such scenario is dependent on the characteristics
 101 * of the underlying transport being used.
 102 *
 103 * Since the SCMI core regular stack is partially used to deliver and collect
 104 * the messages, late replies arrived after timeouts and any other sort of
 105 * unexpected message can be identified by the SCMI core as usual and they will
 106 * be reported as messages under "errors" for later analysis.
 107 */
 108
 109#include <linux/bitmap.h>
 110#include <linux/debugfs.h>
 111#include <linux/delay.h>
 112#include <linux/device.h>
 113#include <linux/export.h>
 114#include <linux/io.h>
 115#include <linux/kernel.h>
 116#include <linux/fs.h>
 117#include <linux/list.h>
 118#include <linux/module.h>
 119#include <linux/poll.h>
 120#include <linux/of.h>
 121#include <linux/slab.h>
 122#include <linux/xarray.h>
 123
 124#include "common.h"
 125
 126#include "raw_mode.h"
 127
 128#include <trace/events/scmi.h>
 129
 130#define SCMI_XFER_RAW_MAX_RETRIES	10
 131
 132/**
 133 * struct scmi_raw_queue  - Generic Raw queue descriptor
 134 *
 135 * @free_bufs: A freelists listhead used to keep unused raw buffers
 136 * @free_bufs_lock: Spinlock used to protect access to @free_bufs
 137 * @msg_q: A listhead to a queue of snooped messages waiting to be read out
 138 * @msg_q_lock: Spinlock used to protect access to @msg_q
 139 * @wq: A waitqueue used to wait and poll on related @msg_q
 140 */
 141struct scmi_raw_queue {
 142	struct list_head free_bufs;
 143	/* Protect free_bufs[] lists */
 144	spinlock_t free_bufs_lock;
 145	struct list_head msg_q;
 146	/* Protect msg_q[] lists */
 147	spinlock_t msg_q_lock;
 148	wait_queue_head_t wq;
 149};
 150
 151/**
 152 * struct scmi_raw_mode_info  - Structure holding SCMI Raw instance data
 153 *
 154 * @id: Sequential Raw instance ID.
 155 * @handle: Pointer to SCMI entity handle to use
 156 * @desc: Pointer to the transport descriptor to use
 157 * @tx_max_msg: Maximum number of concurrent TX in-flight messages
 158 * @q: An array of Raw queue descriptors
 159 * @chans_q: An XArray mapping optional additional per-channel queues
 160 * @free_waiters: Head of freelist for unused waiters
 161 * @free_mtx: A mutex to protect the waiters freelist
 162 * @active_waiters: Head of list for currently active and used waiters
 163 * @active_mtx: A mutex to protect the active waiters list
 164 * @waiters_work: A work descriptor to be used with the workqueue machinery
 165 * @wait_wq: A workqueue reference to the created workqueue
 166 * @dentry: Top debugfs root dentry for SCMI Raw
 167 * @gid: A group ID used for devres accounting
 168 *
 169 * Note that this descriptor is passed back to the core after SCMI Raw is
 170 * initialized as an opaque handle to use by subsequent SCMI Raw call hooks.
 171 *
 172 */
 173struct scmi_raw_mode_info {
 174	unsigned int id;
 175	const struct scmi_handle *handle;
 176	const struct scmi_desc *desc;
 177	int tx_max_msg;
 178	struct scmi_raw_queue *q[SCMI_RAW_MAX_QUEUE];
 179	struct xarray chans_q;
 180	struct list_head free_waiters;
 181	/* Protect free_waiters list */
 182	struct mutex free_mtx;
 183	struct list_head active_waiters;
 184	/* Protect active_waiters list */
 185	struct mutex active_mtx;
 186	struct work_struct waiters_work;
 187	struct workqueue_struct	*wait_wq;
 188	struct dentry *dentry;
 189	void *gid;
 190};
 191
 192/**
 193 * struct scmi_xfer_raw_waiter  - Structure to describe an xfer to be waited for
 194 *
 195 * @start_jiffies: The timestamp in jiffies of when this structure was queued.
 196 * @cinfo: A reference to the channel to use for this transaction
 197 * @xfer: A reference to the xfer to be waited for
 198 * @async_response: A completion to be, optionally, used for async waits: it
 199 *		    will be setup by @scmi_do_xfer_raw_start, if needed, to be
 200 *		    pointed at by xfer->async_done.
 201 * @node: A list node.
 202 */
 203struct scmi_xfer_raw_waiter {
 204	unsigned long start_jiffies;
 205	struct scmi_chan_info *cinfo;
 206	struct scmi_xfer *xfer;
 207	struct completion async_response;
 208	struct list_head node;
 209};
 210
 211/**
 212 * struct scmi_raw_buffer  - Structure to hold a full SCMI message
 213 *
 214 * @max_len: The maximum allowed message size (header included) that can be
 215 *	     stored into @msg
 216 * @msg: A message buffer used to collect a full message grabbed from an xfer.
 217 * @node: A list node.
 218 */
 219struct scmi_raw_buffer {
 220	size_t max_len;
 221	struct scmi_msg msg;
 222	struct list_head node;
 223};
 224
 225/**
 226 * struct scmi_dbg_raw_data  - Structure holding data needed by the debugfs
 227 * layer
 228 *
 229 * @chan_id: The preferred channel to use: if zero the channel is automatically
 230 *	     selected based on protocol.
 231 * @raw: A reference to the Raw instance.
 232 * @tx: A message buffer used to collect TX message on write.
 233 * @tx_size: The effective size of the TX message.
 234 * @tx_req_size: The final expected size of the complete TX message.
 235 * @rx: A message buffer to collect RX message on read.
 236 * @rx_size: The effective size of the RX message.
 237 */
 238struct scmi_dbg_raw_data {
 239	u8 chan_id;
 240	struct scmi_raw_mode_info *raw;
 241	struct scmi_msg tx;
 242	size_t tx_size;
 243	size_t tx_req_size;
 244	struct scmi_msg rx;
 245	size_t rx_size;
 246};
 247
 248static struct scmi_raw_queue *
 249scmi_raw_queue_select(struct scmi_raw_mode_info *raw, unsigned int idx,
 250		      unsigned int chan_id)
 251{
 252	if (!chan_id)
 253		return raw->q[idx];
 254
 255	return xa_load(&raw->chans_q, chan_id);
 256}
 257
 258static struct scmi_raw_buffer *scmi_raw_buffer_get(struct scmi_raw_queue *q)
 259{
 260	unsigned long flags;
 261	struct scmi_raw_buffer *rb = NULL;
 262	struct list_head *head = &q->free_bufs;
 263
 264	spin_lock_irqsave(&q->free_bufs_lock, flags);
 265	if (!list_empty(head)) {
 266		rb = list_first_entry(head, struct scmi_raw_buffer, node);
 267		list_del_init(&rb->node);
 268	}
 269	spin_unlock_irqrestore(&q->free_bufs_lock, flags);
 270
 271	return rb;
 272}
 273
 274static void scmi_raw_buffer_put(struct scmi_raw_queue *q,
 275				struct scmi_raw_buffer *rb)
 276{
 277	unsigned long flags;
 278
 279	/* Reset to full buffer length */
 280	rb->msg.len = rb->max_len;
 281
 282	spin_lock_irqsave(&q->free_bufs_lock, flags);
 283	list_add_tail(&rb->node, &q->free_bufs);
 284	spin_unlock_irqrestore(&q->free_bufs_lock, flags);
 285}
 286
 287static void scmi_raw_buffer_enqueue(struct scmi_raw_queue *q,
 288				    struct scmi_raw_buffer *rb)
 289{
 290	unsigned long flags;
 291
 292	spin_lock_irqsave(&q->msg_q_lock, flags);
 293	list_add_tail(&rb->node, &q->msg_q);
 294	spin_unlock_irqrestore(&q->msg_q_lock, flags);
 295
 296	wake_up_interruptible(&q->wq);
 297}
 298
 299static struct scmi_raw_buffer*
 300scmi_raw_buffer_dequeue_unlocked(struct scmi_raw_queue *q)
 301{
 302	struct scmi_raw_buffer *rb = NULL;
 303
 304	if (!list_empty(&q->msg_q)) {
 305		rb = list_first_entry(&q->msg_q, struct scmi_raw_buffer, node);
 306		list_del_init(&rb->node);
 307	}
 308
 309	return rb;
 310}
 311
 312static struct scmi_raw_buffer *scmi_raw_buffer_dequeue(struct scmi_raw_queue *q)
 313{
 314	unsigned long flags;
 315	struct scmi_raw_buffer *rb;
 316
 317	spin_lock_irqsave(&q->msg_q_lock, flags);
 318	rb = scmi_raw_buffer_dequeue_unlocked(q);
 319	spin_unlock_irqrestore(&q->msg_q_lock, flags);
 320
 321	return rb;
 322}
 323
 324static void scmi_raw_buffer_queue_flush(struct scmi_raw_queue *q)
 325{
 326	struct scmi_raw_buffer *rb;
 327
 328	do {
 329		rb = scmi_raw_buffer_dequeue(q);
 330		if (rb)
 331			scmi_raw_buffer_put(q, rb);
 332	} while (rb);
 333}
 334
 335static struct scmi_xfer_raw_waiter *
 336scmi_xfer_raw_waiter_get(struct scmi_raw_mode_info *raw, struct scmi_xfer *xfer,
 337			 struct scmi_chan_info *cinfo, bool async)
 338{
 339	struct scmi_xfer_raw_waiter *rw = NULL;
 340
 341	mutex_lock(&raw->free_mtx);
 342	if (!list_empty(&raw->free_waiters)) {
 343		rw = list_first_entry(&raw->free_waiters,
 344				      struct scmi_xfer_raw_waiter, node);
 345		list_del_init(&rw->node);
 346
 347		if (async) {
 348			reinit_completion(&rw->async_response);
 349			xfer->async_done = &rw->async_response;
 350		}
 351
 352		rw->cinfo = cinfo;
 353		rw->xfer = xfer;
 354	}
 355	mutex_unlock(&raw->free_mtx);
 356
 357	return rw;
 358}
 359
 360static void scmi_xfer_raw_waiter_put(struct scmi_raw_mode_info *raw,
 361				     struct scmi_xfer_raw_waiter *rw)
 362{
 363	if (rw->xfer) {
 364		rw->xfer->async_done = NULL;
 365		rw->xfer = NULL;
 366	}
 367
 368	mutex_lock(&raw->free_mtx);
 369	list_add_tail(&rw->node, &raw->free_waiters);
 370	mutex_unlock(&raw->free_mtx);
 371}
 372
 373static void scmi_xfer_raw_waiter_enqueue(struct scmi_raw_mode_info *raw,
 374					 struct scmi_xfer_raw_waiter *rw)
 375{
 376	/* A timestamp for the deferred worker to know how much this has aged */
 377	rw->start_jiffies = jiffies;
 378
 379	trace_scmi_xfer_response_wait(rw->xfer->transfer_id, rw->xfer->hdr.id,
 380				      rw->xfer->hdr.protocol_id,
 381				      rw->xfer->hdr.seq,
 382				      raw->desc->max_rx_timeout_ms,
 383				      rw->xfer->hdr.poll_completion);
 384
 385	mutex_lock(&raw->active_mtx);
 386	list_add_tail(&rw->node, &raw->active_waiters);
 387	mutex_unlock(&raw->active_mtx);
 388
 389	/* kick waiter work */
 390	queue_work(raw->wait_wq, &raw->waiters_work);
 391}
 392
 393static struct scmi_xfer_raw_waiter *
 394scmi_xfer_raw_waiter_dequeue(struct scmi_raw_mode_info *raw)
 395{
 396	struct scmi_xfer_raw_waiter *rw = NULL;
 397
 398	mutex_lock(&raw->active_mtx);
 399	if (!list_empty(&raw->active_waiters)) {
 400		rw = list_first_entry(&raw->active_waiters,
 401				      struct scmi_xfer_raw_waiter, node);
 402		list_del_init(&rw->node);
 403	}
 404	mutex_unlock(&raw->active_mtx);
 405
 406	return rw;
 407}
 408
 409/**
 410 * scmi_xfer_raw_worker  - Work function to wait for Raw xfers completions
 411 *
 412 * @work: A reference to the work.
 413 *
 414 * In SCMI Raw mode, once a user-provided injected SCMI message is sent, we
 415 * cannot wait to receive its response (if any) in the context of the injection
 416 * routines so as not to leave the userspace write syscall, which delivered the
 417 * SCMI message to send, pending till eventually a reply is received.
 418 * Userspace should and will poll/wait instead on the read syscalls which will
 419 * be in charge of reading a received reply (if any).
 420 *
 421 * Even though reply messages are collected and reported into the SCMI Raw layer
 422 * on the RX path, nonetheless we have to properly wait for their completion as
 423 * usual (and async_completion too if needed) in order to properly release the
 424 * xfer structure at the end: to do this out of the context of the write/send
 425 * these waiting jobs are delegated to this deferred worker.
 426 *
 427 * Any sent xfer, to be waited for, is timestamped and queued for later
 428 * consumption by this worker: queue aging is accounted for while choosing a
 429 * timeout for the completion, BUT we do not really care here if we end up
 430 * accidentally waiting for a bit too long.
 431 */
 432static void scmi_xfer_raw_worker(struct work_struct *work)
 433{
 434	struct scmi_raw_mode_info *raw;
 435	struct device *dev;
 436	unsigned long max_tmo;
 437
 438	raw = container_of(work, struct scmi_raw_mode_info, waiters_work);
 439	dev = raw->handle->dev;
 440	max_tmo = msecs_to_jiffies(raw->desc->max_rx_timeout_ms);
 441
 442	do {
 443		int ret = 0;
 444		unsigned int timeout_ms;
 445		unsigned long aging;
 446		struct scmi_xfer *xfer;
 447		struct scmi_xfer_raw_waiter *rw;
 448		struct scmi_chan_info *cinfo;
 449
 450		rw = scmi_xfer_raw_waiter_dequeue(raw);
 451		if (!rw)
 452			return;
 453
 454		cinfo = rw->cinfo;
 455		xfer = rw->xfer;
 456		/*
 457		 * Waiters are queued by wait-deadline at the end, so some of
 458		 * them could have been already expired when processed, BUT we
 459		 * have to check the completion status anyway just in case a
 460		 * virtually expired (aged) transaction was indeed completed
 461		 * fine and we'll have to wait for the asynchronous part (if
 462		 * any): for this reason a 1 ms timeout is used for already
 463		 * expired/aged xfers.
 464		 */
 465		aging = jiffies - rw->start_jiffies;
 466		timeout_ms = max_tmo > aging ?
 467			jiffies_to_msecs(max_tmo - aging) : 1;
 468
 469		ret = scmi_xfer_raw_wait_for_message_response(cinfo, xfer,
 470							      timeout_ms);
 471		if (!ret && xfer->hdr.status)
 472			ret = scmi_to_linux_errno(xfer->hdr.status);
 473
 474		if (raw->desc->ops->mark_txdone)
 475			raw->desc->ops->mark_txdone(rw->cinfo, ret, xfer);
 476
 477		trace_scmi_xfer_end(xfer->transfer_id, xfer->hdr.id,
 478				    xfer->hdr.protocol_id, xfer->hdr.seq,
 479				    ret, scmi_inflight_count(raw->handle));
 480
 481		/* Wait also for an async delayed response if needed */
 482		if (!ret && xfer->async_done) {
 483			unsigned long tmo = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT);
 484
 485			if (!wait_for_completion_timeout(xfer->async_done, tmo))
 486				dev_err(dev,
 487					"timed out in RAW delayed resp - HDR:%08X\n",
 488					pack_scmi_header(&xfer->hdr));
 489		}
 490
 491		/* Release waiter and xfer */
 492		scmi_xfer_raw_put(raw->handle, xfer);
 493		scmi_xfer_raw_waiter_put(raw, rw);
 494	} while (1);
 495}
 496
 497static void scmi_xfer_raw_reset(struct scmi_raw_mode_info *raw)
 498{
 499	int i;
 500
 501	dev_info(raw->handle->dev, "Resetting SCMI Raw stack.\n");
 502
 503	for (i = 0; i < SCMI_RAW_MAX_QUEUE; i++)
 504		scmi_raw_buffer_queue_flush(raw->q[i]);
 505}
 506
 507/**
 508 * scmi_xfer_raw_get_init  - An helper to build a valid xfer from the provided
 509 * bare SCMI message.
 510 *
 511 * @raw: A reference to the Raw instance.
 512 * @buf: A buffer containing the whole SCMI message to send (including the
 513 *	 header) in little-endian binary formmat.
 514 * @len: Length of the message in @buf.
 515 * @p: A pointer to return the initialized Raw xfer.
 516 *
 517 * After an xfer is picked from the TX pool and filled in with the message
 518 * content, the xfer is registered as pending with the core in the usual way
 519 * using the original sequence number provided by the user with the message.
 520 *
 521 * Note that, in case the testing user application is NOT using distinct
 522 * sequence-numbers between successive SCMI messages such registration could
 523 * fail temporarily if the previous message, using the same sequence number,
 524 * had still not released; in such a case we just wait and retry.
 525 *
 526 * Return: 0 on Success
 527 */
 528static int scmi_xfer_raw_get_init(struct scmi_raw_mode_info *raw, void *buf,
 529				  size_t len, struct scmi_xfer **p)
 530{
 531	u32 msg_hdr;
 532	size_t tx_size;
 533	struct scmi_xfer *xfer;
 534	int ret, retry = SCMI_XFER_RAW_MAX_RETRIES;
 535	struct device *dev = raw->handle->dev;
 536
 537	if (!buf || len < sizeof(u32))
 538		return -EINVAL;
 539
 540	tx_size = len - sizeof(u32);
 541	/* Ensure we have sane transfer sizes */
 542	if (tx_size > raw->desc->max_msg_size)
 543		return -ERANGE;
 544
 545	xfer = scmi_xfer_raw_get(raw->handle);
 546	if (IS_ERR(xfer)) {
 547		dev_warn(dev, "RAW - Cannot get a free RAW xfer !\n");
 548		return PTR_ERR(xfer);
 549	}
 550
 551	/* Build xfer from the provided SCMI bare LE message */
 552	msg_hdr = le32_to_cpu(*((__le32 *)buf));
 553	unpack_scmi_header(msg_hdr, &xfer->hdr);
 554	xfer->hdr.seq = (u16)MSG_XTRACT_TOKEN(msg_hdr);
 555	/* Polling not supported */
 556	xfer->hdr.poll_completion = false;
 557	xfer->hdr.status = SCMI_SUCCESS;
 558	xfer->tx.len = tx_size;
 559	xfer->rx.len = raw->desc->max_msg_size;
 560	/* Clear the whole TX buffer */
 561	memset(xfer->tx.buf, 0x00, raw->desc->max_msg_size);
 562	if (xfer->tx.len)
 563		memcpy(xfer->tx.buf, (u8 *)buf + sizeof(msg_hdr), xfer->tx.len);
 564	*p = xfer;
 565
 566	/*
 567	 * In flight registration can temporarily fail in case of Raw messages
 568	 * if the user injects messages without using monotonically increasing
 569	 * sequence numbers since, in Raw mode, the xfer (and the token) is
 570	 * finally released later by a deferred worker. Just retry for a while.
 571	 */
 572	do {
 573		ret = scmi_xfer_raw_inflight_register(raw->handle, xfer);
 574		if (ret) {
 575			dev_dbg(dev,
 576				"...retrying[%d] inflight registration\n",
 577				retry);
 578			msleep(raw->desc->max_rx_timeout_ms /
 579			       SCMI_XFER_RAW_MAX_RETRIES);
 580		}
 581	} while (ret && --retry);
 582
 583	if (ret) {
 584		dev_warn(dev,
 585			 "RAW - Could NOT register xfer %d in-flight HDR:0x%08X\n",
 586			 xfer->hdr.seq, msg_hdr);
 587		scmi_xfer_raw_put(raw->handle, xfer);
 588	}
 589
 590	return ret;
 591}
 592
 593/**
 594 * scmi_do_xfer_raw_start  - An helper to send a valid raw xfer
 595 *
 596 * @raw: A reference to the Raw instance.
 597 * @xfer: The xfer to send
 598 * @chan_id: The channel ID to use, if zero the channels is automatically
 599 *	     selected based on the protocol used.
 600 * @async: A flag stating if an asynchronous command is required.
 601 *
 602 * This function send a previously built raw xfer using an appropriate channel
 603 * and queues the related waiting work.
 604 *
 605 * Note that we need to know explicitly if the required command is meant to be
 606 * asynchronous in kind since we have to properly setup the waiter.
 607 * (and deducing this from the payload is weak and do not scale given there is
 608 *  NOT a common header-flag stating if the command is asynchronous or not)
 609 *
 610 * Return: 0 on Success
 611 */
 612static int scmi_do_xfer_raw_start(struct scmi_raw_mode_info *raw,
 613				  struct scmi_xfer *xfer, u8 chan_id,
 614				  bool async)
 615{
 616	int ret;
 617	struct scmi_chan_info *cinfo;
 618	struct scmi_xfer_raw_waiter *rw;
 619	struct device *dev = raw->handle->dev;
 620
 621	if (!chan_id)
 622		chan_id = xfer->hdr.protocol_id;
 623	else
 624		xfer->flags |= SCMI_XFER_FLAG_CHAN_SET;
 625
 626	cinfo = scmi_xfer_raw_channel_get(raw->handle, chan_id);
 627	if (IS_ERR(cinfo))
 628		return PTR_ERR(cinfo);
 629
 630	rw = scmi_xfer_raw_waiter_get(raw, xfer, cinfo, async);
 631	if (!rw) {
 632		dev_warn(dev, "RAW - Cannot get a free waiter !\n");
 633		return -ENOMEM;
 634	}
 635
 636	/* True ONLY if also supported by transport. */
 637	if (is_polling_enabled(cinfo, raw->desc))
 638		xfer->hdr.poll_completion = true;
 639
 640	reinit_completion(&xfer->done);
 641	/* Make sure xfer state update is visible before sending */
 642	smp_store_mb(xfer->state, SCMI_XFER_SENT_OK);
 643
 644	trace_scmi_xfer_begin(xfer->transfer_id, xfer->hdr.id,
 645			      xfer->hdr.protocol_id, xfer->hdr.seq,
 646			      xfer->hdr.poll_completion,
 647			      scmi_inflight_count(raw->handle));
 648
 649	ret = raw->desc->ops->send_message(rw->cinfo, xfer);
 650	if (ret) {
 651		dev_err(dev, "Failed to send RAW message %d\n", ret);
 652		scmi_xfer_raw_waiter_put(raw, rw);
 653		return ret;
 654	}
 655
 656	trace_scmi_msg_dump(raw->id, cinfo->id, xfer->hdr.protocol_id,
 657			    xfer->hdr.id, "cmnd", xfer->hdr.seq,
 658			    xfer->hdr.status,
 659			    xfer->tx.buf, xfer->tx.len);
 660
 661	scmi_xfer_raw_waiter_enqueue(raw, rw);
 662
 663	return ret;
 664}
 665
 666/**
 667 * scmi_raw_message_send  - An helper to build and send an SCMI command using
 668 * the provided SCMI bare message buffer
 669 *
 670 * @raw: A reference to the Raw instance.
 671 * @buf: A buffer containing the whole SCMI message to send (including the
 672 *	 header) in little-endian binary format.
 673 * @len: Length of the message in @buf.
 674 * @chan_id: The channel ID to use.
 675 * @async: A flag stating if an asynchronous command is required.
 676 * @poll: A flag stating if a polling transmission is required.
 677 *
 678 * Return: 0 on Success
 679 */
 680static int scmi_raw_message_send(struct scmi_raw_mode_info *raw,
 681				 void *buf, size_t len, u8 chan_id,
 682				 bool async, bool poll)
 683{
 684	int ret;
 685	struct scmi_xfer *xfer;
 686
 687	ret = scmi_xfer_raw_get_init(raw, buf, len, &xfer);
 688	if (ret)
 689		return ret;
 690
 691	if (poll) {
 692		if (is_transport_polling_capable(raw->desc)) {
 693			xfer->hdr.poll_completion = true;
 694		} else {
 695			dev_err(raw->handle->dev,
 696				"Failed to send RAW message - Polling NOT supported\n");
 697			return -EINVAL;
 698		}
 699	}
 700
 701	ret = scmi_do_xfer_raw_start(raw, xfer, chan_id, async);
 702	if (ret)
 703		scmi_xfer_raw_put(raw->handle, xfer);
 704
 705	return ret;
 706}
 707
 708static struct scmi_raw_buffer *
 709scmi_raw_message_dequeue(struct scmi_raw_queue *q, bool o_nonblock)
 710{
 711	unsigned long flags;
 712	struct scmi_raw_buffer *rb;
 713
 714	spin_lock_irqsave(&q->msg_q_lock, flags);
 715	while (list_empty(&q->msg_q)) {
 716		spin_unlock_irqrestore(&q->msg_q_lock, flags);
 717
 718		if (o_nonblock)
 719			return ERR_PTR(-EAGAIN);
 720
 721		if (wait_event_interruptible(q->wq, !list_empty(&q->msg_q)))
 722			return ERR_PTR(-ERESTARTSYS);
 723
 724		spin_lock_irqsave(&q->msg_q_lock, flags);
 725	}
 726
 727	rb = scmi_raw_buffer_dequeue_unlocked(q);
 728
 729	spin_unlock_irqrestore(&q->msg_q_lock, flags);
 730
 731	return rb;
 732}
 733
 734/**
 735 * scmi_raw_message_receive  - An helper to dequeue and report the next
 736 * available enqueued raw message payload that has been collected.
 737 *
 738 * @raw: A reference to the Raw instance.
 739 * @buf: A buffer to get hold of the whole SCMI message received and represented
 740 *	 in little-endian binary format.
 741 * @len: Length of @buf.
 742 * @size: The effective size of the message copied into @buf
 743 * @idx: The index of the queue to pick the next queued message from.
 744 * @chan_id: The channel ID to use.
 745 * @o_nonblock: A flag to request a non-blocking message dequeue.
 746 *
 747 * Return: 0 on Success
 748 */
 749static int scmi_raw_message_receive(struct scmi_raw_mode_info *raw,
 750				    void *buf, size_t len, size_t *size,
 751				    unsigned int idx, unsigned int chan_id,
 752				    bool o_nonblock)
 753{
 754	int ret = 0;
 755	struct scmi_raw_buffer *rb;
 756	struct scmi_raw_queue *q;
 757
 758	q = scmi_raw_queue_select(raw, idx, chan_id);
 759	if (!q)
 760		return -ENODEV;
 761
 762	rb = scmi_raw_message_dequeue(q, o_nonblock);
 763	if (IS_ERR(rb)) {
 764		dev_dbg(raw->handle->dev, "RAW - No message available!\n");
 765		return PTR_ERR(rb);
 766	}
 767
 768	if (rb->msg.len <= len) {
 769		memcpy(buf, rb->msg.buf, rb->msg.len);
 770		*size = rb->msg.len;
 771	} else {
 772		ret = -ENOSPC;
 773	}
 774
 775	scmi_raw_buffer_put(q, rb);
 776
 777	return ret;
 778}
 779
 780/* SCMI Raw debugfs helpers */
 781
 782static ssize_t scmi_dbg_raw_mode_common_read(struct file *filp,
 783					     char __user *buf,
 784					     size_t count, loff_t *ppos,
 785					     unsigned int idx)
 786{
 787	ssize_t cnt;
 788	struct scmi_dbg_raw_data *rd = filp->private_data;
 789
 790	if (!rd->rx_size) {
 791		int ret;
 792
 793		ret = scmi_raw_message_receive(rd->raw, rd->rx.buf, rd->rx.len,
 794					       &rd->rx_size, idx, rd->chan_id,
 795					       filp->f_flags & O_NONBLOCK);
 796		if (ret) {
 797			rd->rx_size = 0;
 798			return ret;
 799		}
 800
 801		/* Reset any previous filepos change, including writes */
 802		*ppos = 0;
 803	} else if (*ppos == rd->rx_size) {
 804		/* Return EOF once all the message has been read-out */
 805		rd->rx_size = 0;
 806		return 0;
 807	}
 808
 809	cnt = simple_read_from_buffer(buf, count, ppos,
 810				      rd->rx.buf, rd->rx_size);
 811
 812	return cnt;
 813}
 814
 815static ssize_t scmi_dbg_raw_mode_common_write(struct file *filp,
 816					      const char __user *buf,
 817					      size_t count, loff_t *ppos,
 818					      bool async, bool poll)
 819{
 820	int ret;
 821	struct scmi_dbg_raw_data *rd = filp->private_data;
 822
 823	if (count > rd->tx.len - rd->tx_size)
 824		return -ENOSPC;
 825
 826	/* On first write attempt @count carries the total full message size. */
 827	if (!rd->tx_size)
 828		rd->tx_req_size = count;
 829
 830	/*
 831	 * Gather a full message, possibly across multiple interrupted wrrtes,
 832	 * before sending it with a single RAW xfer.
 833	 */
 834	if (rd->tx_size < rd->tx_req_size) {
 835		ssize_t cnt;
 836
 837		cnt = simple_write_to_buffer(rd->tx.buf, rd->tx.len, ppos,
 838					     buf, count);
 839		if (cnt < 0)
 840			return cnt;
 841
 842		rd->tx_size += cnt;
 843		if (cnt < count)
 844			return cnt;
 845	}
 846
 847	ret = scmi_raw_message_send(rd->raw, rd->tx.buf, rd->tx_size,
 848				    rd->chan_id, async, poll);
 849
 850	/* Reset ppos for next message ... */
 851	rd->tx_size = 0;
 852	*ppos = 0;
 853
 854	return ret ?: count;
 855}
 856
 857static __poll_t scmi_test_dbg_raw_common_poll(struct file *filp,
 858					      struct poll_table_struct *wait,
 859					      unsigned int idx)
 860{
 861	unsigned long flags;
 862	struct scmi_dbg_raw_data *rd = filp->private_data;
 863	struct scmi_raw_queue *q;
 864	__poll_t mask = 0;
 865
 866	q = scmi_raw_queue_select(rd->raw, idx, rd->chan_id);
 867	if (!q)
 868		return mask;
 869
 870	poll_wait(filp, &q->wq, wait);
 871
 872	spin_lock_irqsave(&q->msg_q_lock, flags);
 873	if (!list_empty(&q->msg_q))
 874		mask = EPOLLIN | EPOLLRDNORM;
 875	spin_unlock_irqrestore(&q->msg_q_lock, flags);
 876
 877	return mask;
 878}
 879
 880static ssize_t scmi_dbg_raw_mode_message_read(struct file *filp,
 881					      char __user *buf,
 882					      size_t count, loff_t *ppos)
 883{
 884	return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos,
 885					     SCMI_RAW_REPLY_QUEUE);
 886}
 887
 888static ssize_t scmi_dbg_raw_mode_message_write(struct file *filp,
 889					       const char __user *buf,
 890					       size_t count, loff_t *ppos)
 891{
 892	return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos,
 893					      false, false);
 894}
 895
 896static __poll_t scmi_dbg_raw_mode_message_poll(struct file *filp,
 897					       struct poll_table_struct *wait)
 898{
 899	return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_REPLY_QUEUE);
 900}
 901
 902static int scmi_dbg_raw_mode_open(struct inode *inode, struct file *filp)
 903{
 904	struct scmi_raw_mode_info *raw;
 905	struct scmi_dbg_raw_data *rd;
 906
 907	if (!inode->i_private)
 908		return -ENODEV;
 909
 910	raw = inode->i_private;
 911	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
 912	if (!rd)
 913		return -ENOMEM;
 914
 915	rd->rx.len = raw->desc->max_msg_size + sizeof(u32);
 916	rd->rx.buf = kzalloc(rd->rx.len, GFP_KERNEL);
 917	if (!rd->rx.buf) {
 918		kfree(rd);
 919		return -ENOMEM;
 920	}
 921
 922	rd->tx.len = raw->desc->max_msg_size + sizeof(u32);
 923	rd->tx.buf = kzalloc(rd->tx.len, GFP_KERNEL);
 924	if (!rd->tx.buf) {
 925		kfree(rd->rx.buf);
 926		kfree(rd);
 927		return -ENOMEM;
 928	}
 929
 930	/* Grab channel ID from debugfs entry naming if any */
 931	/* not set - reassing 0 we already had after kzalloc() */
 932	rd->chan_id = debugfs_get_aux_num(filp);
 933
 934	rd->raw = raw;
 935	filp->private_data = rd;
 936
 937	return nonseekable_open(inode, filp);
 938}
 939
 940static int scmi_dbg_raw_mode_release(struct inode *inode, struct file *filp)
 941{
 942	struct scmi_dbg_raw_data *rd = filp->private_data;
 943
 944	kfree(rd->rx.buf);
 945	kfree(rd->tx.buf);
 946	kfree(rd);
 947
 948	return 0;
 949}
 950
 951static ssize_t scmi_dbg_raw_mode_reset_write(struct file *filp,
 952					     const char __user *buf,
 953					     size_t count, loff_t *ppos)
 954{
 955	struct scmi_dbg_raw_data *rd = filp->private_data;
 956
 957	scmi_xfer_raw_reset(rd->raw);
 958
 959	return count;
 960}
 961
 962static const struct file_operations scmi_dbg_raw_mode_reset_fops = {
 963	.open = scmi_dbg_raw_mode_open,
 964	.release = scmi_dbg_raw_mode_release,
 965	.write = scmi_dbg_raw_mode_reset_write,
 966	.owner = THIS_MODULE,
 967};
 968
 969static const struct file_operations scmi_dbg_raw_mode_message_fops = {
 970	.open = scmi_dbg_raw_mode_open,
 971	.release = scmi_dbg_raw_mode_release,
 972	.read = scmi_dbg_raw_mode_message_read,
 973	.write = scmi_dbg_raw_mode_message_write,
 974	.poll = scmi_dbg_raw_mode_message_poll,
 975	.owner = THIS_MODULE,
 976};
 977
 978static ssize_t scmi_dbg_raw_mode_message_async_write(struct file *filp,
 979						     const char __user *buf,
 980						     size_t count, loff_t *ppos)
 981{
 982	return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos,
 983					      true, false);
 984}
 985
 986static const struct file_operations scmi_dbg_raw_mode_message_async_fops = {
 987	.open = scmi_dbg_raw_mode_open,
 988	.release = scmi_dbg_raw_mode_release,
 989	.read = scmi_dbg_raw_mode_message_read,
 990	.write = scmi_dbg_raw_mode_message_async_write,
 991	.poll = scmi_dbg_raw_mode_message_poll,
 992	.owner = THIS_MODULE,
 993};
 994
 995static ssize_t scmi_dbg_raw_mode_message_poll_write(struct file *filp,
 996						    const char __user *buf,
 997						    size_t count, loff_t *ppos)
 998{
 999	return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos,
1000					      false, true);
1001}
1002
1003static const struct file_operations scmi_dbg_raw_mode_message_poll_fops = {
1004	.open = scmi_dbg_raw_mode_open,
1005	.release = scmi_dbg_raw_mode_release,
1006	.read = scmi_dbg_raw_mode_message_read,
1007	.write = scmi_dbg_raw_mode_message_poll_write,
1008	.poll = scmi_dbg_raw_mode_message_poll,
1009	.owner = THIS_MODULE,
1010};
1011
1012static ssize_t scmi_dbg_raw_mode_message_poll_async_write(struct file *filp,
1013							  const char __user *buf,
1014							  size_t count, loff_t *ppos)
1015{
1016	return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos,
1017					      true, true);
1018}
1019
1020static const struct file_operations scmi_dbg_raw_mode_message_poll_async_fops = {
1021	.open = scmi_dbg_raw_mode_open,
1022	.release = scmi_dbg_raw_mode_release,
1023	.read = scmi_dbg_raw_mode_message_read,
1024	.write = scmi_dbg_raw_mode_message_poll_async_write,
1025	.poll = scmi_dbg_raw_mode_message_poll,
1026	.owner = THIS_MODULE,
1027};
1028
1029static ssize_t scmi_test_dbg_raw_mode_notif_read(struct file *filp,
1030						 char __user *buf,
1031						 size_t count, loff_t *ppos)
1032{
1033	return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos,
1034					     SCMI_RAW_NOTIF_QUEUE);
1035}
1036
1037static __poll_t
1038scmi_test_dbg_raw_mode_notif_poll(struct file *filp,
1039				  struct poll_table_struct *wait)
1040{
1041	return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_NOTIF_QUEUE);
1042}
1043
1044static const struct file_operations scmi_dbg_raw_mode_notification_fops = {
1045	.open = scmi_dbg_raw_mode_open,
1046	.release = scmi_dbg_raw_mode_release,
1047	.read = scmi_test_dbg_raw_mode_notif_read,
1048	.poll = scmi_test_dbg_raw_mode_notif_poll,
1049	.owner = THIS_MODULE,
1050};
1051
1052static ssize_t scmi_test_dbg_raw_mode_errors_read(struct file *filp,
1053						  char __user *buf,
1054						  size_t count, loff_t *ppos)
1055{
1056	return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos,
1057					     SCMI_RAW_ERRS_QUEUE);
1058}
1059
1060static __poll_t
1061scmi_test_dbg_raw_mode_errors_poll(struct file *filp,
1062				   struct poll_table_struct *wait)
1063{
1064	return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_ERRS_QUEUE);
1065}
1066
1067static const struct file_operations scmi_dbg_raw_mode_errors_fops = {
1068	.open = scmi_dbg_raw_mode_open,
1069	.release = scmi_dbg_raw_mode_release,
1070	.read = scmi_test_dbg_raw_mode_errors_read,
1071	.poll = scmi_test_dbg_raw_mode_errors_poll,
1072	.owner = THIS_MODULE,
1073};
1074
1075static struct scmi_raw_queue *
1076scmi_raw_queue_init(struct scmi_raw_mode_info *raw)
1077{
1078	int i;
1079	struct scmi_raw_buffer *rb;
1080	struct device *dev = raw->handle->dev;
1081	struct scmi_raw_queue *q;
1082
1083	q = devm_kzalloc(dev, sizeof(*q), GFP_KERNEL);
1084	if (!q)
1085		return ERR_PTR(-ENOMEM);
1086
1087	rb = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rb), GFP_KERNEL);
1088	if (!rb)
1089		return ERR_PTR(-ENOMEM);
1090
1091	spin_lock_init(&q->free_bufs_lock);
1092	INIT_LIST_HEAD(&q->free_bufs);
1093	for (i = 0; i < raw->tx_max_msg; i++, rb++) {
1094		rb->max_len = raw->desc->max_msg_size + sizeof(u32);
1095		rb->msg.buf = devm_kzalloc(dev, rb->max_len, GFP_KERNEL);
1096		if (!rb->msg.buf)
1097			return ERR_PTR(-ENOMEM);
1098		scmi_raw_buffer_put(q, rb);
1099	}
1100
1101	spin_lock_init(&q->msg_q_lock);
1102	INIT_LIST_HEAD(&q->msg_q);
1103	init_waitqueue_head(&q->wq);
1104
1105	return q;
1106}
1107
1108static int scmi_xfer_raw_worker_init(struct scmi_raw_mode_info *raw)
1109{
1110	int i;
1111	struct scmi_xfer_raw_waiter *rw;
1112	struct device *dev = raw->handle->dev;
1113
1114	rw = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rw), GFP_KERNEL);
1115	if (!rw)
1116		return -ENOMEM;
1117
1118	raw->wait_wq = alloc_workqueue("scmi-raw-wait-wq-%d",
1119				       WQ_UNBOUND | WQ_FREEZABLE |
1120				       WQ_HIGHPRI | WQ_SYSFS, 0, raw->id);
1121	if (!raw->wait_wq)
1122		return -ENOMEM;
1123
1124	mutex_init(&raw->free_mtx);
1125	INIT_LIST_HEAD(&raw->free_waiters);
1126	mutex_init(&raw->active_mtx);
1127	INIT_LIST_HEAD(&raw->active_waiters);
1128
1129	for (i = 0; i < raw->tx_max_msg; i++, rw++) {
1130		init_completion(&rw->async_response);
1131		scmi_xfer_raw_waiter_put(raw, rw);
1132	}
1133	INIT_WORK(&raw->waiters_work, scmi_xfer_raw_worker);
1134
1135	return 0;
1136}
1137
1138static int scmi_raw_mode_setup(struct scmi_raw_mode_info *raw,
1139			       u8 *channels, int num_chans)
1140{
1141	int ret, idx;
1142	void *gid;
1143	struct device *dev = raw->handle->dev;
1144
1145	gid = devres_open_group(dev, NULL, GFP_KERNEL);
1146	if (!gid)
1147		return -ENOMEM;
1148
1149	for (idx = 0; idx < SCMI_RAW_MAX_QUEUE; idx++) {
1150		raw->q[idx] = scmi_raw_queue_init(raw);
1151		if (IS_ERR(raw->q[idx])) {
1152			ret = PTR_ERR(raw->q[idx]);
1153			goto err;
1154		}
1155	}
1156
1157	xa_init(&raw->chans_q);
1158	if (num_chans > 1) {
1159		int i;
1160
1161		for (i = 0; i < num_chans; i++) {
1162			struct scmi_raw_queue *q;
1163
1164			q = scmi_raw_queue_init(raw);
1165			if (IS_ERR(q)) {
1166				ret = PTR_ERR(q);
1167				goto err_xa;
1168			}
1169
1170			ret = xa_insert(&raw->chans_q, channels[i], q,
1171					GFP_KERNEL);
1172			if (ret) {
1173				dev_err(dev,
1174					"Fail to allocate Raw queue 0x%02X\n",
1175					channels[i]);
1176				goto err_xa;
1177			}
1178		}
1179	}
1180
1181	ret = scmi_xfer_raw_worker_init(raw);
1182	if (ret)
1183		goto err_xa;
1184
1185	devres_close_group(dev, gid);
1186	raw->gid = gid;
1187
1188	return 0;
1189
1190err_xa:
1191	xa_destroy(&raw->chans_q);
1192err:
1193	devres_release_group(dev, gid);
1194	return ret;
1195}
1196
1197/**
1198 * scmi_raw_mode_init  - Function to initialize the SCMI Raw stack
1199 *
1200 * @handle: Pointer to SCMI entity handle
1201 * @top_dentry: A reference to the top Raw debugfs dentry
1202 * @instance_id: The ID of the underlying SCMI platform instance represented by
1203 *		 this Raw instance
1204 * @channels: The list of the existing channels
1205 * @num_chans: The number of entries in @channels
1206 * @desc: Reference to the transport operations
1207 * @tx_max_msg: Max number of in-flight messages allowed by the transport
1208 *
1209 * This function prepare the SCMI Raw stack and creates the debugfs API.
1210 *
1211 * Return: An opaque handle to the Raw instance on Success, an ERR_PTR otherwise
1212 */
1213void *scmi_raw_mode_init(const struct scmi_handle *handle,
1214			 struct dentry *top_dentry, int instance_id,
1215			 u8 *channels, int num_chans,
1216			 const struct scmi_desc *desc, int tx_max_msg)
1217{
1218	int ret;
1219	struct scmi_raw_mode_info *raw;
1220	struct device *dev;
1221
1222	if (!handle || !desc)
1223		return ERR_PTR(-EINVAL);
1224
1225	dev = handle->dev;
1226	raw = devm_kzalloc(dev, sizeof(*raw), GFP_KERNEL);
1227	if (!raw)
1228		return ERR_PTR(-ENOMEM);
1229
1230	raw->handle = handle;
1231	raw->desc = desc;
1232	raw->tx_max_msg = tx_max_msg;
1233	raw->id = instance_id;
1234
1235	ret = scmi_raw_mode_setup(raw, channels, num_chans);
1236	if (ret) {
1237		devm_kfree(dev, raw);
1238		return ERR_PTR(ret);
1239	}
1240
1241	raw->dentry = debugfs_create_dir("raw", top_dentry);
1242
1243	debugfs_create_file("reset", 0200, raw->dentry, raw,
1244			    &scmi_dbg_raw_mode_reset_fops);
1245
1246	debugfs_create_file("message", 0600, raw->dentry, raw,
1247			    &scmi_dbg_raw_mode_message_fops);
1248
1249	debugfs_create_file("message_async", 0600, raw->dentry, raw,
1250			    &scmi_dbg_raw_mode_message_async_fops);
1251
1252	debugfs_create_file("message_poll", 0600, raw->dentry, raw,
1253			    &scmi_dbg_raw_mode_message_poll_fops);
1254
1255	debugfs_create_file("message_poll_async", 0600, raw->dentry, raw,
1256			    &scmi_dbg_raw_mode_message_poll_async_fops);
1257
1258	debugfs_create_file("notification", 0400, raw->dentry, raw,
1259			    &scmi_dbg_raw_mode_notification_fops);
1260
1261	debugfs_create_file("errors", 0400, raw->dentry, raw,
1262			    &scmi_dbg_raw_mode_errors_fops);
1263
1264	/*
1265	 * Expose per-channel entries if multiple channels available.
1266	 * Just ignore errors while setting up these interfaces since we
1267	 * have anyway already a working core Raw support.
1268	 */
1269	if (num_chans > 1) {
1270		int i;
1271		struct dentry *top_chans;
1272
1273		top_chans = debugfs_create_dir("channels", raw->dentry);
1274
1275		for (i = 0; i < num_chans; i++) {
1276			char cdir[8];
1277			struct dentry *chd;
1278
1279			snprintf(cdir, 8, "0x%02X", channels[i]);
1280			chd = debugfs_create_dir(cdir, top_chans);
1281
1282			debugfs_create_file_aux_num("message", 0600, chd,
1283					    raw, channels[i],
1284					    &scmi_dbg_raw_mode_message_fops);
1285
1286			debugfs_create_file_aux_num("message_async", 0600, chd,
1287					    raw, channels[i],
1288					    &scmi_dbg_raw_mode_message_async_fops);
1289
1290			debugfs_create_file_aux_num("message_poll", 0600, chd,
1291						    raw, channels[i],
1292						    &scmi_dbg_raw_mode_message_poll_fops);
1293
1294			debugfs_create_file_aux_num("message_poll_async", 0600,
1295						    chd, raw, channels[i],
1296						    &scmi_dbg_raw_mode_message_poll_async_fops);
1297		}
1298	}
1299
1300	dev_info(dev, "SCMI RAW Mode initialized for instance %d\n", raw->id);
1301
1302	return raw;
1303}
1304
1305/**
1306 * scmi_raw_mode_cleanup  - Function to cleanup the SCMI Raw stack
1307 *
1308 * @r: An opaque handle to an initialized SCMI Raw instance
1309 */
1310void scmi_raw_mode_cleanup(void *r)
1311{
1312	struct scmi_raw_mode_info *raw = r;
1313
1314	if (!raw)
1315		return;
1316
1317	debugfs_remove_recursive(raw->dentry);
1318
1319	cancel_work_sync(&raw->waiters_work);
1320	destroy_workqueue(raw->wait_wq);
1321	xa_destroy(&raw->chans_q);
1322}
1323
1324static int scmi_xfer_raw_collect(void *msg, size_t *msg_len,
1325				 struct scmi_xfer *xfer)
1326{
1327	__le32 *m;
1328	size_t msg_size;
1329
1330	if (!xfer || !msg || !msg_len)
1331		return -EINVAL;
1332
1333	/* Account for hdr ...*/
1334	msg_size = xfer->rx.len + sizeof(u32);
1335	/* ... and status if needed */
1336	if (xfer->hdr.type != MSG_TYPE_NOTIFICATION)
1337		msg_size += sizeof(u32);
1338
1339	if (msg_size > *msg_len)
1340		return -ENOSPC;
1341
1342	m = msg;
1343	*m = cpu_to_le32(pack_scmi_header(&xfer->hdr));
1344	if (xfer->hdr.type != MSG_TYPE_NOTIFICATION)
1345		*++m = cpu_to_le32(xfer->hdr.status);
1346
1347	memcpy(++m, xfer->rx.buf, xfer->rx.len);
1348
1349	*msg_len = msg_size;
1350
1351	return 0;
1352}
1353
1354/**
1355 * scmi_raw_message_report  - Helper to report back valid reponses/notifications
1356 * to raw message requests.
1357 *
1358 * @r: An opaque reference to the raw instance configuration
1359 * @xfer: The xfer containing the message to be reported
1360 * @idx: The index of the queue.
1361 * @chan_id: The channel ID to use.
1362 *
1363 * If Raw mode is enabled, this is called from the SCMI core on the regular RX
1364 * path to save and enqueue the response/notification payload carried by this
1365 * xfer into a dedicated scmi_raw_buffer for later consumption by the user.
1366 *
1367 * This way the caller can free the related xfer immediately afterwards and the
1368 * user can read back the raw message payload at its own pace (if ever) without
1369 * holding an xfer for too long.
1370 */
1371void scmi_raw_message_report(void *r, struct scmi_xfer *xfer,
1372			     unsigned int idx, unsigned int chan_id)
1373{
1374	int ret;
1375	unsigned long flags;
1376	struct scmi_raw_buffer *rb;
1377	struct device *dev;
1378	struct scmi_raw_queue *q;
1379	struct scmi_raw_mode_info *raw = r;
1380
1381	if (!raw || (idx == SCMI_RAW_REPLY_QUEUE && !SCMI_XFER_IS_RAW(xfer)))
1382		return;
1383
1384	dev = raw->handle->dev;
1385	q = scmi_raw_queue_select(raw, idx,
1386				  SCMI_XFER_IS_CHAN_SET(xfer) ? chan_id : 0);
1387	if (!q) {
1388		dev_warn(dev,
1389			 "RAW[%d] - NO queue for chan 0x%X. Dropping report.\n",
1390			 idx, chan_id);
1391		return;
1392	}
1393
1394	/*
1395	 * Grab the msg_q_lock upfront to avoid a possible race between
1396	 * realizing the free list was empty and effectively picking the next
1397	 * buffer to use from the oldest one enqueued and still unread on this
1398	 * msg_q.
1399	 *
1400	 * Note that nowhere else these locks are taken together, so no risk of
1401	 * deadlocks du eto inversion.
1402	 */
1403	spin_lock_irqsave(&q->msg_q_lock, flags);
1404	rb = scmi_raw_buffer_get(q);
1405	if (!rb) {
1406		/*
1407		 * Immediate and delayed replies to previously injected Raw
1408		 * commands MUST be read back from userspace to free the buffers:
1409		 * if this is not happening something is seriously broken and
1410		 * must be fixed at the application level: complain loudly.
1411		 */
1412		if (idx == SCMI_RAW_REPLY_QUEUE) {
1413			spin_unlock_irqrestore(&q->msg_q_lock, flags);
1414			dev_warn(dev,
1415				 "RAW[%d] - Buffers exhausted. Dropping report.\n",
1416				 idx);
1417			return;
1418		}
1419
1420		/*
1421		 * Notifications and errors queues are instead handled in a
1422		 * circular manner: unread old buffers are just overwritten by
1423		 * newer ones.
1424		 *
1425		 * The main reason for this is that notifications originated
1426		 * by Raw requests cannot be distinguished from normal ones, so
1427		 * your Raw buffers queues risk to be flooded and depleted by
1428		 * notifications if you left it mistakenly enabled or when in
1429		 * coexistence mode.
1430		 */
1431		rb = scmi_raw_buffer_dequeue_unlocked(q);
1432		if (WARN_ON(!rb)) {
1433			spin_unlock_irqrestore(&q->msg_q_lock, flags);
1434			return;
1435		}
1436
1437		/* Reset to full buffer length */
1438		rb->msg.len = rb->max_len;
1439
1440		dev_warn_once(dev,
1441			      "RAW[%d] - Buffers exhausted. Re-using oldest.\n",
1442			      idx);
1443	}
1444	spin_unlock_irqrestore(&q->msg_q_lock, flags);
1445
1446	ret = scmi_xfer_raw_collect(rb->msg.buf, &rb->msg.len, xfer);
1447	if (ret) {
1448		dev_warn(dev, "RAW - Cannot collect xfer into buffer !\n");
1449		scmi_raw_buffer_put(q, rb);
1450		return;
1451	}
1452
1453	scmi_raw_buffer_enqueue(q, rb);
1454}
1455
1456static void scmi_xfer_raw_fill(struct scmi_raw_mode_info *raw,
1457			       struct scmi_chan_info *cinfo,
1458			       struct scmi_xfer *xfer, u32 msg_hdr)
1459{
1460	/* Unpack received HDR as it is */
1461	unpack_scmi_header(msg_hdr, &xfer->hdr);
1462	xfer->hdr.seq = MSG_XTRACT_TOKEN(msg_hdr);
1463
1464	memset(xfer->rx.buf, 0x00, xfer->rx.len);
1465
1466	raw->desc->ops->fetch_response(cinfo, xfer);
1467}
1468
1469/**
1470 * scmi_raw_error_report  - Helper to report back timed-out or generally
1471 * unexpected replies.
1472 *
1473 * @r: An opaque reference to the raw instance configuration
1474 * @cinfo: A reference to the channel to use to retrieve the broken xfer
1475 * @msg_hdr: The SCMI message header of the message to fetch and report
1476 * @priv: Any private data related to the xfer.
1477 *
1478 * If Raw mode is enabled, this is called from the SCMI core on the RX path in
1479 * case of errors to save and enqueue the bad message payload carried by the
1480 * message that has just been received.
1481 *
1482 * Note that we have to manually fetch any available payload into a temporary
1483 * xfer to be able to save and enqueue the message, since the regular RX error
1484 * path which had called this would have not fetched the message payload having
1485 * classified it as an error.
1486 */
1487void scmi_raw_error_report(void *r, struct scmi_chan_info *cinfo,
1488			   u32 msg_hdr, void *priv)
1489{
1490	struct scmi_xfer xfer;
1491	struct scmi_raw_mode_info *raw = r;
1492
1493	if (!raw)
1494		return;
1495
1496	xfer.rx.len = raw->desc->max_msg_size;
1497	xfer.rx.buf = kzalloc(xfer.rx.len, GFP_ATOMIC);
1498	if (!xfer.rx.buf) {
1499		dev_info(raw->handle->dev,
1500			 "Cannot report Raw error for HDR:0x%X - ENOMEM\n",
1501			 msg_hdr);
1502		return;
1503	}
1504
1505	/* Any transport-provided priv must be passed back down to transport */
1506	if (priv)
1507		/* Ensure priv is visible */
1508		smp_store_mb(xfer.priv, priv);
1509
1510	scmi_xfer_raw_fill(raw, cinfo, &xfer, msg_hdr);
1511	scmi_raw_message_report(raw, &xfer, SCMI_RAW_ERRS_QUEUE, 0);
1512
1513	kfree(xfer.rx.buf);
1514}