qcom_glink_native.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2016-2017, Linaro Ltd
   4 */
   5
   6#include <linux/idr.h>
   7#include <linux/interrupt.h>
   8#include <linux/io.h>
   9#include <linux/list.h>
  10#include <linux/mfd/syscon.h>
  11#include <linux/module.h>
  12#include <linux/of.h>
  13#include <linux/of_address.h>
  14#include <linux/platform_device.h>
  15#include <linux/regmap.h>
  16#include <linux/rpmsg.h>
  17#include <linux/sizes.h>
  18#include <linux/slab.h>
  19#include <linux/wait.h>
  20#include <linux/workqueue.h>
  21#include <linux/mailbox_client.h>
  22
  23#include "rpmsg_internal.h"
  24#include "qcom_glink_native.h"
  25
  26#define CREATE_TRACE_POINTS
  27#include "qcom_glink_trace.h"
  28
  29#define GLINK_NAME_SIZE		32
  30#define GLINK_VERSION_1		1
  31
  32#define RPM_GLINK_CID_MIN	1
  33#define RPM_GLINK_CID_MAX	65536
  34
  35struct glink_msg {
  36	/* New members MUST be added within the __struct_group() macro below. */
  37	__struct_group(glink_msg_hdr, hdr, __packed,
  38		__le16 cmd;
  39		__le16 param1;
  40		__le32 param2;
  41	);
  42	u8 data[];
  43} __packed;
  44static_assert(offsetof(struct glink_msg, data) == sizeof(struct glink_msg_hdr),
  45	      "struct member likely outside of __struct_group()");
  46
  47/**
  48 * struct glink_defer_cmd - deferred incoming control message
  49 * @node:	list node
  50 * @msg:	message header
  51 * @data:	payload of the message
  52 *
  53 * Copy of a received control message, to be added to @rx_queue and processed
  54 * by @rx_work of @qcom_glink.
  55 */
  56struct glink_defer_cmd {
  57	struct list_head node;
  58
  59	struct glink_msg_hdr msg;
  60	u8 data[];
  61};
  62
  63/**
  64 * struct glink_core_rx_intent - RX intent
  65 * RX intent
  66 *
  67 * @data: pointer to the data (may be NULL for zero-copy)
  68 * @id: remote or local intent ID
  69 * @size: size of the original intent (do not modify)
  70 * @reuse: To mark if the intent can be reused after first use
  71 * @in_use: To mark if intent is already in use for the channel
  72 * @offset: next write offset (initially 0)
  73 * @node:	list node
  74 */
  75struct glink_core_rx_intent {
  76	void *data;
  77	u32 id;
  78	size_t size;
  79	bool reuse;
  80	bool in_use;
  81	u32 offset;
  82
  83	struct list_head node;
  84};
  85
  86/**
  87 * struct qcom_glink - driver context, relates to one remote subsystem
  88 * @dev:	reference to the associated struct device
  89 * @label:	identifier of the glink edge
  90 * @rx_pipe:	pipe object for receive FIFO
  91 * @tx_pipe:	pipe object for transmit FIFO
  92 * @rx_work:	worker for handling received control messages
  93 * @rx_lock:	protects the @rx_queue
  94 * @rx_queue:	queue of received control messages to be processed in @rx_work
  95 * @tx_lock:	synchronizes operations on the tx fifo
  96 * @idr_lock:	synchronizes @lcids and @rcids modifications
  97 * @lcids:	idr of all channels with a known local channel id
  98 * @rcids:	idr of all channels with a known remote channel id
  99 * @features:	remote features
 100 * @intentless:	flag to indicate that there is no intent
 101 * @tx_avail_notify: Waitqueue for pending tx tasks
 102 * @sent_read_notify: flag to check cmd sent or not
 103 * @abort_tx:	flag indicating that all tx attempts should fail
 104 */
 105struct qcom_glink {
 106	struct device *dev;
 107
 108	const char *label;
 109
 110	struct qcom_glink_pipe *rx_pipe;
 111	struct qcom_glink_pipe *tx_pipe;
 112
 113	struct work_struct rx_work;
 114	spinlock_t rx_lock;
 115	struct list_head rx_queue;
 116
 117	spinlock_t tx_lock;
 118
 119	spinlock_t idr_lock;
 120	struct idr lcids;
 121	struct idr rcids;
 122	unsigned long features;
 123
 124	bool intentless;
 125	wait_queue_head_t tx_avail_notify;
 126	bool sent_read_notify;
 127
 128	bool abort_tx;
 129};
 130
 131enum {
 132	GLINK_STATE_CLOSED,
 133	GLINK_STATE_OPENING,
 134	GLINK_STATE_OPEN,
 135	GLINK_STATE_CLOSING,
 136};
 137
 138/**
 139 * struct glink_channel - internal representation of a channel
 140 * @rpdev:	rpdev reference, only used for primary endpoints
 141 * @ept:	rpmsg endpoint this channel is associated with
 142 * @glink:	qcom_glink context handle
 143 * @refcount:	refcount for the channel object
 144 * @recv_lock:	guard for @ept.cb
 145 * @name:	unique channel name/identifier
 146 * @lcid:	channel id, in local space
 147 * @rcid:	channel id, in remote space
 148 * @intent_lock: lock for protection of @liids, @riids
 149 * @liids:	idr of all local intents
 150 * @riids:	idr of all remote intents
 151 * @intent_work: worker responsible for transmitting rx_done packets
 152 * @done_intents: list of intents that needs to be announced rx_done
 153 * @buf:	receive buffer, for gathering fragments
 154 * @buf_offset:	write offset in @buf
 155 * @buf_size:	size of current @buf
 156 * @open_ack:	completed once remote has acked the open-request
 157 * @open_req:	completed once open-request has been received
 158 * @intent_req_lock: Synchronises multiple intent requests
 159 * @intent_req_result: Result of intent request
 160 * @intent_received: flag indicating that an intent has been received
 161 * @intent_req_wq: wait queue for intent_req signalling
 162 */
 163struct glink_channel {
 164	struct rpmsg_endpoint ept;
 165
 166	struct rpmsg_device *rpdev;
 167	struct qcom_glink *glink;
 168
 169	struct kref refcount;
 170
 171	spinlock_t recv_lock;
 172
 173	char *name;
 174	unsigned int lcid;
 175	unsigned int rcid;
 176
 177	spinlock_t intent_lock;
 178	struct idr liids;
 179	struct idr riids;
 180	struct work_struct intent_work;
 181	struct list_head done_intents;
 182
 183	struct glink_core_rx_intent *buf;
 184	int buf_offset;
 185	int buf_size;
 186
 187	struct completion open_ack;
 188	struct completion open_req;
 189
 190	struct mutex intent_req_lock;
 191	int intent_req_result;
 192	bool intent_received;
 193	wait_queue_head_t intent_req_wq;
 194};
 195
 196#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
 197
 198static const struct rpmsg_endpoint_ops glink_endpoint_ops;
 199
 200#define GLINK_CMD_VERSION		0
 201#define GLINK_CMD_VERSION_ACK		1
 202#define GLINK_CMD_OPEN			2
 203#define GLINK_CMD_CLOSE			3
 204#define GLINK_CMD_OPEN_ACK		4
 205#define GLINK_CMD_INTENT		5
 206#define GLINK_CMD_RX_DONE		6
 207#define GLINK_CMD_RX_INTENT_REQ		7
 208#define GLINK_CMD_RX_INTENT_REQ_ACK	8
 209#define GLINK_CMD_TX_DATA		9
 210#define GLINK_CMD_CLOSE_ACK		11
 211#define GLINK_CMD_TX_DATA_CONT		12
 212#define GLINK_CMD_READ_NOTIF		13
 213#define GLINK_CMD_RX_DONE_W_REUSE	14
 214#define GLINK_CMD_SIGNALS		15
 215
 216#define GLINK_FEATURE_INTENTLESS	BIT(1)
 217
 218#define NATIVE_DTR_SIG			NATIVE_DSR_SIG
 219#define NATIVE_DSR_SIG			BIT(31)
 220#define NATIVE_RTS_SIG			NATIVE_CTS_SIG
 221#define NATIVE_CTS_SIG			BIT(30)
 222
 223static void qcom_glink_rx_done_work(struct work_struct *work);
 224
 225static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
 226						      const char *name)
 227{
 228	struct glink_channel *channel;
 229
 230	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 231	if (!channel)
 232		return ERR_PTR(-ENOMEM);
 233
 234	/* Setup glink internal glink_channel data */
 235	spin_lock_init(&channel->recv_lock);
 236	spin_lock_init(&channel->intent_lock);
 237	mutex_init(&channel->intent_req_lock);
 238
 239	channel->glink = glink;
 240	channel->name = kstrdup(name, GFP_KERNEL);
 241	if (!channel->name) {
 242		kfree(channel);
 243		return ERR_PTR(-ENOMEM);
 244	}
 245
 246	init_completion(&channel->open_req);
 247	init_completion(&channel->open_ack);
 248	init_waitqueue_head(&channel->intent_req_wq);
 249
 250	INIT_LIST_HEAD(&channel->done_intents);
 251	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
 252
 253	idr_init(&channel->liids);
 254	idr_init(&channel->riids);
 255	kref_init(&channel->refcount);
 256
 257	return channel;
 258}
 259
 260static void qcom_glink_channel_release(struct kref *ref)
 261{
 262	struct glink_channel *channel = container_of(ref, struct glink_channel,
 263						     refcount);
 264	struct glink_core_rx_intent *intent;
 265	struct glink_core_rx_intent *tmp;
 266	unsigned long flags;
 267	int iid;
 268
 269	/* cancel pending rx_done work */
 270	cancel_work_sync(&channel->intent_work);
 271
 272	spin_lock_irqsave(&channel->intent_lock, flags);
 273	/* Free all non-reuse intents pending rx_done work */
 274	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 275		if (!intent->reuse) {
 276			kfree(intent->data);
 277			kfree(intent);
 278		}
 279	}
 280
 281	idr_for_each_entry(&channel->liids, tmp, iid) {
 282		kfree(tmp->data);
 283		kfree(tmp);
 284	}
 285	idr_destroy(&channel->liids);
 286
 287	idr_for_each_entry(&channel->riids, tmp, iid)
 288		kfree(tmp);
 289	idr_destroy(&channel->riids);
 290	spin_unlock_irqrestore(&channel->intent_lock, flags);
 291
 292	kfree(channel->name);
 293	kfree(channel);
 294}
 295
 296static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
 297{
 298	return glink->rx_pipe->avail(glink->rx_pipe);
 299}
 300
 301static void qcom_glink_rx_peek(struct qcom_glink *glink,
 302			       void *data, unsigned int offset, size_t count)
 303{
 304	glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
 305}
 306
 307static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
 308{
 309	glink->rx_pipe->advance(glink->rx_pipe, count);
 310}
 311
 312static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
 313{
 314	return glink->tx_pipe->avail(glink->tx_pipe);
 315}
 316
 317static void qcom_glink_tx_write(struct qcom_glink *glink,
 318				const void *hdr, size_t hlen,
 319				const void *data, size_t dlen)
 320{
 321	glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
 322}
 323
 324static void qcom_glink_tx_kick(struct qcom_glink *glink)
 325{
 326	glink->tx_pipe->kick(glink->tx_pipe);
 327}
 328
 329static void qcom_glink_send_read_notify(struct qcom_glink *glink)
 330{
 331	struct glink_msg msg;
 332
 333	msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
 334	msg.param1 = 0;
 335	msg.param2 = 0;
 336
 337	qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0);
 338
 339	qcom_glink_tx_kick(glink);
 340}
 341
 342static int qcom_glink_tx(struct qcom_glink *glink,
 343			 const void *hdr, size_t hlen,
 344			 const void *data, size_t dlen, bool wait)
 345{
 346	unsigned int tlen = hlen + dlen;
 347	unsigned long flags;
 348	int ret = 0;
 349
 350	/* Reject packets that are too big */
 351	if (tlen >= glink->tx_pipe->length)
 352		return -EINVAL;
 353
 354	spin_lock_irqsave(&glink->tx_lock, flags);
 355
 356	if (glink->abort_tx) {
 357		ret = -EIO;
 358		goto out;
 359	}
 360
 361	while (qcom_glink_tx_avail(glink) < tlen) {
 362		if (!wait) {
 363			ret = -EAGAIN;
 364			goto out;
 365		}
 366
 367		if (glink->abort_tx) {
 368			ret = -EIO;
 369			goto out;
 370		}
 371
 372		if (!glink->sent_read_notify) {
 373			glink->sent_read_notify = true;
 374			qcom_glink_send_read_notify(glink);
 375		}
 376
 377		/* Wait without holding the tx_lock */
 378		spin_unlock_irqrestore(&glink->tx_lock, flags);
 379
 380		wait_event_timeout(glink->tx_avail_notify,
 381				   qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
 382
 383		spin_lock_irqsave(&glink->tx_lock, flags);
 384
 385		if (qcom_glink_tx_avail(glink) >= tlen)
 386			glink->sent_read_notify = false;
 387	}
 388
 389	qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
 390	qcom_glink_tx_kick(glink);
 391
 392out:
 393	spin_unlock_irqrestore(&glink->tx_lock, flags);
 394
 395	return ret;
 396}
 397
 398static int qcom_glink_send_version(struct qcom_glink *glink)
 399{
 400	struct glink_msg msg;
 401
 402	msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
 403	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 404	msg.param2 = cpu_to_le32(glink->features);
 405
 406	trace_qcom_glink_cmd_version_tx(glink->label, GLINK_VERSION_1, glink->features);
 407
 408	return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 409}
 410
 411static void qcom_glink_send_version_ack(struct qcom_glink *glink)
 412{
 413	struct glink_msg msg;
 414
 415	msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
 416	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
 417	msg.param2 = cpu_to_le32(glink->features);
 418
 419	trace_qcom_glink_cmd_version_ack_tx(glink->label, msg.param1, msg.param2);
 420
 421	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 422}
 423
 424static void qcom_glink_send_open_ack(struct qcom_glink *glink,
 425				     struct glink_channel *channel)
 426{
 427	struct glink_msg msg;
 428
 429	msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
 430	msg.param1 = cpu_to_le16(channel->rcid);
 431	msg.param2 = cpu_to_le32(0);
 432
 433	trace_qcom_glink_cmd_open_ack_tx(glink->label, channel->name,
 434					 channel->lcid, channel->rcid);
 435
 436	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 437}
 438
 439static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
 440					     unsigned int cid, bool granted)
 441{
 442	struct glink_channel *channel;
 443	unsigned long flags;
 444
 445	qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
 446
 447	spin_lock_irqsave(&glink->idr_lock, flags);
 448	channel = idr_find(&glink->rcids, cid);
 449	spin_unlock_irqrestore(&glink->idr_lock, flags);
 450
 451	trace_qcom_glink_cmd_rx_intent_req_ack_rx(glink->label,
 452						  channel ? channel->name : NULL,
 453						  channel ? channel->lcid : 0,
 454						  cid, granted);
 455	if (!channel) {
 456		dev_err(glink->dev, "unable to find channel\n");
 457		return;
 458	}
 459
 460	WRITE_ONCE(channel->intent_req_result, granted);
 461	wake_up_all(&channel->intent_req_wq);
 462}
 463
 464static void qcom_glink_intent_req_abort(struct glink_channel *channel)
 465{
 466	WRITE_ONCE(channel->intent_req_result, 0);
 467	wake_up_all(&channel->intent_req_wq);
 468}
 469
 470/**
 471 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
 472 * @glink: Ptr to the glink edge
 473 * @channel: Ptr to the channel that the open req is sent
 474 *
 475 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
 476 * Will return with refcount held, regardless of outcome.
 477 *
 478 * Return: 0 on success, negative errno otherwise.
 479 */
 480static int qcom_glink_send_open_req(struct qcom_glink *glink,
 481				    struct glink_channel *channel)
 482{
 483	DEFINE_RAW_FLEX(struct glink_msg, req, data, GLINK_NAME_SIZE);
 484	int name_len = strlen(channel->name) + 1;
 485	int req_len = ALIGN(sizeof(*req) + name_len, 8);
 486	int ret;
 487	unsigned long flags;
 488
 489	kref_get(&channel->refcount);
 490
 491	spin_lock_irqsave(&glink->idr_lock, flags);
 492	ret = idr_alloc_cyclic(&glink->lcids, channel,
 493			       RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
 494			       GFP_ATOMIC);
 495	spin_unlock_irqrestore(&glink->idr_lock, flags);
 496	if (ret < 0)
 497		return ret;
 498
 499	channel->lcid = ret;
 500
 501	req->cmd = cpu_to_le16(GLINK_CMD_OPEN);
 502	req->param1 = cpu_to_le16(channel->lcid);
 503	req->param2 = cpu_to_le32(name_len);
 504	strcpy(req->data, channel->name);
 505
 506	trace_qcom_glink_cmd_open_tx(glink->label, channel->name,
 507				     channel->lcid, channel->rcid);
 508
 509	ret = qcom_glink_tx(glink, req, req_len, NULL, 0, true);
 510	if (ret)
 511		goto remove_idr;
 512
 513	return 0;
 514
 515remove_idr:
 516	spin_lock_irqsave(&glink->idr_lock, flags);
 517	idr_remove(&glink->lcids, channel->lcid);
 518	channel->lcid = 0;
 519	spin_unlock_irqrestore(&glink->idr_lock, flags);
 520
 521	return ret;
 522}
 523
 524static void qcom_glink_send_close_req(struct qcom_glink *glink,
 525				      struct glink_channel *channel)
 526{
 527	struct glink_msg req;
 528
 529	req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
 530	req.param1 = cpu_to_le16(channel->lcid);
 531	req.param2 = 0;
 532
 533	trace_qcom_glink_cmd_close_tx(glink->label, channel->name,
 534				      channel->lcid, channel->rcid);
 535
 536	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 537}
 538
 539static void qcom_glink_send_close_ack(struct qcom_glink *glink,
 540				      struct glink_channel *channel)
 541{
 542	struct glink_msg req;
 543
 544	req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
 545	req.param1 = cpu_to_le16(channel->rcid);
 546	req.param2 = 0;
 547
 548	trace_qcom_glink_cmd_close_ack_tx(glink->label, channel->name,
 549					  channel->lcid, channel->rcid);
 550
 551	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
 552}
 553
 554static void qcom_glink_rx_done_work(struct work_struct *work)
 555{
 556	struct glink_channel *channel = container_of(work, struct glink_channel,
 557						     intent_work);
 558	struct qcom_glink *glink = channel->glink;
 559	struct glink_core_rx_intent *intent, *tmp;
 560	struct {
 561		u16 id;
 562		u16 lcid;
 563		u32 liid;
 564	} __packed cmd;
 565
 566	unsigned int cid = channel->lcid;
 567	unsigned int iid;
 568	bool reuse;
 569	unsigned long flags;
 570
 571	spin_lock_irqsave(&channel->intent_lock, flags);
 572	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
 573		list_del(&intent->node);
 574		spin_unlock_irqrestore(&channel->intent_lock, flags);
 575		iid = intent->id;
 576		reuse = intent->reuse;
 577
 578		cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
 579		cmd.lcid = cid;
 580		cmd.liid = iid;
 581
 582		trace_qcom_glink_cmd_rx_done_tx(glink->label, channel->name,
 583						channel->lcid, channel->rcid, cmd.liid, reuse);
 584
 585		qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 586		if (!reuse) {
 587			kfree(intent->data);
 588			kfree(intent);
 589		}
 590		spin_lock_irqsave(&channel->intent_lock, flags);
 591	}
 592	spin_unlock_irqrestore(&channel->intent_lock, flags);
 593}
 594
 595static void qcom_glink_rx_done(struct qcom_glink *glink,
 596			       struct glink_channel *channel,
 597			       struct glink_core_rx_intent *intent)
 598{
 599	/* We don't send RX_DONE to intentless systems */
 600	if (glink->intentless) {
 601		kfree(intent->data);
 602		kfree(intent);
 603		return;
 604	}
 605
 606	/* Take it off the tree of receive intents */
 607	if (!intent->reuse) {
 608		spin_lock(&channel->intent_lock);
 609		idr_remove(&channel->liids, intent->id);
 610		spin_unlock(&channel->intent_lock);
 611	}
 612
 613	/* Schedule the sending of a rx_done indication */
 614	spin_lock(&channel->intent_lock);
 615	list_add_tail(&intent->node, &channel->done_intents);
 616	spin_unlock(&channel->intent_lock);
 617
 618	schedule_work(&channel->intent_work);
 619}
 620
 621/**
 622 * qcom_glink_receive_version() - receive version/features from remote system
 623 *
 624 * @glink:	pointer to transport interface
 625 * @version:	remote version
 626 * @features:	remote features
 627 *
 628 * This function is called in response to a remote-initiated version/feature
 629 * negotiation sequence.
 630 */
 631static void qcom_glink_receive_version(struct qcom_glink *glink,
 632				       u32 version,
 633				       u32 features)
 634{
 635	trace_qcom_glink_cmd_version_rx(glink->label, version, features);
 636
 637	switch (version) {
 638	case 0:
 639		break;
 640	case GLINK_VERSION_1:
 641		glink->features &= features;
 642		fallthrough;
 643	default:
 644		qcom_glink_send_version_ack(glink);
 645		break;
 646	}
 647}
 648
 649/**
 650 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
 651 *
 652 * @glink:	pointer to transport interface
 653 * @version:	remote version response
 654 * @features:	remote features response
 655 *
 656 * This function is called in response to a local-initiated version/feature
 657 * negotiation sequence and is the counter-offer from the remote side based
 658 * upon the initial version and feature set requested.
 659 */
 660static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
 661					   u32 version,
 662					   u32 features)
 663{
 664	trace_qcom_glink_cmd_version_ack_rx(glink->label, version, features);
 665
 666	switch (version) {
 667	case 0:
 668		/* Version negotiation failed */
 669		break;
 670	case GLINK_VERSION_1:
 671		if (features == glink->features)
 672			break;
 673
 674		glink->features &= features;
 675		fallthrough;
 676	default:
 677		qcom_glink_send_version(glink);
 678		break;
 679	}
 680}
 681
 682/**
 683 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
 684 * 	wire format and transmit
 685 * @glink:	The transport to transmit on.
 686 * @channel:	The glink channel
 687 * @granted:	The request response to encode.
 688 *
 689 * Return: 0 on success or standard Linux error code.
 690 */
 691static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
 692					  struct glink_channel *channel,
 693					  bool granted)
 694{
 695	struct glink_msg msg;
 696
 697	trace_qcom_glink_cmd_rx_intent_req_ack_tx(glink->label, channel->name,
 698						  channel->lcid, channel->rcid,
 699						  granted);
 700
 701	msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
 702	msg.param1 = cpu_to_le16(channel->lcid);
 703	msg.param2 = cpu_to_le32(granted);
 704
 705	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
 706
 707	return 0;
 708}
 709
 710/**
 711 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
 712 *			   transmit
 713 * @glink:	The transport to transmit on.
 714 * @channel:	The local channel
 715 * @intent:	The intent to pass on to remote.
 716 *
 717 * Return: 0 on success or standard Linux error code.
 718 */
 719static int qcom_glink_advertise_intent(struct qcom_glink *glink,
 720				       struct glink_channel *channel,
 721				       struct glink_core_rx_intent *intent)
 722{
 723	struct command {
 724		__le16 id;
 725		__le16 lcid;
 726		__le32 count;
 727		__le32 size;
 728		__le32 liid;
 729	} __packed;
 730	struct command cmd;
 731
 732	cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
 733	cmd.lcid = cpu_to_le16(channel->lcid);
 734	cmd.count = cpu_to_le32(1);
 735	cmd.size = cpu_to_le32(intent->size);
 736	cmd.liid = cpu_to_le32(intent->id);
 737
 738	trace_qcom_glink_cmd_intent_tx(glink->label, channel->name,
 739				       channel->lcid, channel->rcid,
 740				       cmd.count, cmd.size, cmd.liid);
 741
 742	qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
 743
 744	return 0;
 745}
 746
 747static struct glink_core_rx_intent *
 748qcom_glink_alloc_intent(struct qcom_glink *glink,
 749			struct glink_channel *channel,
 750			size_t size,
 751			bool reuseable)
 752{
 753	struct glink_core_rx_intent *intent;
 754	int ret;
 755	unsigned long flags;
 756
 757	intent = kzalloc(sizeof(*intent), GFP_KERNEL);
 758	if (!intent)
 759		return NULL;
 760
 761	intent->data = kzalloc(size, GFP_KERNEL);
 762	if (!intent->data)
 763		goto free_intent;
 764
 765	spin_lock_irqsave(&channel->intent_lock, flags);
 766	ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
 767	if (ret < 0) {
 768		spin_unlock_irqrestore(&channel->intent_lock, flags);
 769		goto free_data;
 770	}
 771	spin_unlock_irqrestore(&channel->intent_lock, flags);
 772
 773	intent->id = ret;
 774	intent->size = size;
 775	intent->reuse = reuseable;
 776
 777	return intent;
 778
 779free_data:
 780	kfree(intent->data);
 781free_intent:
 782	kfree(intent);
 783	return NULL;
 784}
 785
 786static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
 787				      u32 cid, uint32_t iid,
 788				      bool reuse)
 789{
 790	struct glink_core_rx_intent *intent;
 791	struct glink_channel *channel;
 792	unsigned long flags;
 793
 794	qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
 795
 796	spin_lock_irqsave(&glink->idr_lock, flags);
 797	channel = idr_find(&glink->rcids, cid);
 798	spin_unlock_irqrestore(&glink->idr_lock, flags);
 799
 800	trace_qcom_glink_cmd_rx_done_rx(glink->label, channel ? channel->name : NULL,
 801					channel ? channel->lcid : 0, cid, iid, reuse);
 802	if (!channel) {
 803		dev_err(glink->dev, "invalid channel id received\n");
 804		return;
 805	}
 806
 807	spin_lock_irqsave(&channel->intent_lock, flags);
 808	intent = idr_find(&channel->riids, iid);
 809
 810	if (!intent) {
 811		spin_unlock_irqrestore(&channel->intent_lock, flags);
 812		dev_err(glink->dev, "invalid intent id received\n");
 813		return;
 814	}
 815
 816	intent->in_use = false;
 817
 818	if (!reuse) {
 819		idr_remove(&channel->riids, intent->id);
 820		kfree(intent);
 821	}
 822	spin_unlock_irqrestore(&channel->intent_lock, flags);
 823
 824	if (reuse) {
 825		WRITE_ONCE(channel->intent_received, true);
 826		wake_up_all(&channel->intent_req_wq);
 827	}
 828}
 829
 830/**
 831 * qcom_glink_handle_intent_req() - Receive a request for rx_intent
 832 *					    from remote side
 833 * @glink:      Pointer to the transport interface
 834 * @cid:	Remote channel ID
 835 * @size:	size of the intent
 836 *
 837 * The function searches for the local channel to which the request for
 838 * rx_intent has arrived and allocates and notifies the remote back
 839 */
 840static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
 841					 u32 cid, size_t size)
 842{
 843	struct glink_core_rx_intent *intent;
 844	struct glink_channel *channel;
 845	unsigned long flags;
 846
 847	spin_lock_irqsave(&glink->idr_lock, flags);
 848	channel = idr_find(&glink->rcids, cid);
 849	spin_unlock_irqrestore(&glink->idr_lock, flags);
 850
 851	trace_qcom_glink_cmd_rx_intent_req_rx(glink->label,
 852					      channel ? channel->name : NULL,
 853					      channel ? channel->lcid : 0,
 854					      cid, size);
 855	if (!channel) {
 856		pr_err("%s channel not found for cid %d\n", __func__, cid);
 857		return;
 858	}
 859
 860	intent = qcom_glink_alloc_intent(glink, channel, size, false);
 861	if (intent)
 862		qcom_glink_advertise_intent(glink, channel, intent);
 863
 864	qcom_glink_send_intent_req_ack(glink, channel, !!intent);
 865}
 866
 867static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
 868{
 869	struct glink_defer_cmd *dcmd;
 870
 871	extra = ALIGN(extra, 8);
 872
 873	if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
 874		dev_dbg(glink->dev, "Insufficient data in rx fifo");
 875		return -ENXIO;
 876	}
 877
 878	dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
 879	if (!dcmd)
 880		return -ENOMEM;
 881
 882	INIT_LIST_HEAD(&dcmd->node);
 883
 884	qcom_glink_rx_peek(glink,
 885			   container_of(&dcmd->msg, struct glink_msg, hdr), 0,
 886			   sizeof(dcmd->msg) + extra);
 887
 888	spin_lock(&glink->rx_lock);
 889	list_add_tail(&dcmd->node, &glink->rx_queue);
 890	spin_unlock(&glink->rx_lock);
 891
 892	schedule_work(&glink->rx_work);
 893	qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
 894
 895	return 0;
 896}
 897
 898static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
 899{
 900	struct glink_core_rx_intent *intent;
 901	struct glink_channel *channel;
 902	struct {
 903		struct glink_msg_hdr msg;
 904		__le32 chunk_size;
 905		__le32 left_size;
 906	} __packed hdr;
 907	unsigned int chunk_size;
 908	unsigned int left_size;
 909	unsigned int rcid;
 910	unsigned int liid;
 911	int ret = 0;
 912	unsigned long flags;
 913
 914	if (avail < sizeof(hdr)) {
 915		dev_dbg(glink->dev, "Not enough data in fifo\n");
 916		return -EAGAIN;
 917	}
 918
 919	qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr));
 920	chunk_size = le32_to_cpu(hdr.chunk_size);
 921	left_size = le32_to_cpu(hdr.left_size);
 922
 923	if (avail < sizeof(hdr) + chunk_size) {
 924		dev_dbg(glink->dev, "Payload not yet in fifo\n");
 925		return -EAGAIN;
 926	}
 927
 928	rcid = le16_to_cpu(hdr.msg.param1);
 929	liid = le32_to_cpu(hdr.msg.param2);
 930	spin_lock_irqsave(&glink->idr_lock, flags);
 931	channel = idr_find(&glink->rcids, rcid);
 932	spin_unlock_irqrestore(&glink->idr_lock, flags);
 933
 934	trace_qcom_glink_cmd_tx_data_rx(glink->label, channel ? channel->name : NULL,
 935					channel ? channel->lcid : 0, rcid,
 936					liid, chunk_size, left_size,
 937					hdr.msg.cmd == GLINK_CMD_TX_DATA_CONT);
 938	if (!channel) {
 939		dev_dbg(glink->dev, "Data on non-existing channel\n");
 940
 941		/* Drop the message */
 942		goto advance_rx;
 943	}
 944
 945	if (glink->intentless) {
 946		/* Might have an ongoing, fragmented, message to append */
 947		if (!channel->buf) {
 948			intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
 949			if (!intent)
 950				return -ENOMEM;
 951
 952			intent->data = kmalloc(chunk_size + left_size,
 953					       GFP_ATOMIC);
 954			if (!intent->data) {
 955				kfree(intent);
 956				return -ENOMEM;
 957			}
 958
 959			intent->id = 0xbabababa;
 960			intent->size = chunk_size + left_size;
 961			intent->offset = 0;
 962
 963			channel->buf = intent;
 964		} else {
 965			intent = channel->buf;
 966		}
 967	} else {
 968		spin_lock_irqsave(&channel->intent_lock, flags);
 969		intent = idr_find(&channel->liids, liid);
 970		spin_unlock_irqrestore(&channel->intent_lock, flags);
 971
 972		if (!intent) {
 973			dev_err(glink->dev,
 974				"no intent found for channel %s intent %d",
 975				channel->name, liid);
 976			ret = -ENOENT;
 977			goto advance_rx;
 978		}
 979	}
 980
 981	if (intent->size - intent->offset < chunk_size) {
 982		dev_err(glink->dev, "Insufficient space in intent\n");
 983
 984		/* The packet header lied, drop payload */
 985		goto advance_rx;
 986	}
 987
 988	qcom_glink_rx_peek(glink, intent->data + intent->offset,
 989			   sizeof(hdr), chunk_size);
 990	intent->offset += chunk_size;
 991
 992	/* Handle message when no fragments remain to be received */
 993	if (!left_size) {
 994		spin_lock(&channel->recv_lock);
 995		if (channel->ept.cb) {
 996			channel->ept.cb(channel->ept.rpdev,
 997					intent->data,
 998					intent->offset,
 999					channel->ept.priv,
1000					RPMSG_ADDR_ANY);
1001		}
1002		spin_unlock(&channel->recv_lock);
1003
1004		intent->offset = 0;
1005		channel->buf = NULL;
1006
1007		qcom_glink_rx_done(glink, channel, intent);
1008	}
1009
1010advance_rx:
1011	qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
1012
1013	return ret;
1014}
1015
1016static void qcom_glink_rx_read_notif(struct qcom_glink *glink)
1017{
1018	trace_qcom_glink_cmd_read_notif_rx(glink->label);
1019
1020	qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1021	qcom_glink_tx_kick(glink);
1022}
1023
1024static void qcom_glink_handle_intent(struct qcom_glink *glink,
1025				     unsigned int cid,
1026				     unsigned int count,
1027				     size_t avail)
1028{
1029	struct glink_core_rx_intent *intent;
1030	struct glink_channel *channel;
1031	struct intent_pair {
1032		__le32 size;
1033		__le32 iid;
1034	};
1035
1036	struct {
1037		struct glink_msg_hdr msg;
1038		struct intent_pair intents[];
1039	} __packed * msg;
1040
1041	const size_t msglen = struct_size(msg, intents, count);
1042	int ret;
1043	int i;
1044	unsigned long flags;
1045
1046	if (avail < msglen) {
1047		dev_dbg(glink->dev, "Not enough data in fifo\n");
1048		return;
1049	}
1050
1051	spin_lock_irqsave(&glink->idr_lock, flags);
1052	channel = idr_find(&glink->rcids, cid);
1053	spin_unlock_irqrestore(&glink->idr_lock, flags);
1054	if (!channel) {
1055		trace_qcom_glink_cmd_intent_rx(glink->label, NULL, 0, cid, count, 0, 0);
1056		dev_err(glink->dev, "intents for non-existing channel\n");
1057		qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1058		return;
1059	}
1060
1061	msg = kmalloc(msglen, GFP_ATOMIC);
1062	if (!msg)
1063		return;
1064
1065	qcom_glink_rx_peek(glink, msg, 0, msglen);
1066
1067	trace_qcom_glink_cmd_intent_rx(glink->label, channel->name,
1068				       channel->lcid, cid, count,
1069				       count > 0 ? msg->intents[0].size : 0,
1070				       count > 0 ? msg->intents[0].iid : 0);
1071
1072	for (i = 0; i < count; ++i) {
1073		intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
1074		if (!intent)
1075			break;
1076
1077		intent->id = le32_to_cpu(msg->intents[i].iid);
1078		intent->size = le32_to_cpu(msg->intents[i].size);
1079
1080		spin_lock_irqsave(&channel->intent_lock, flags);
1081		ret = idr_alloc(&channel->riids, intent,
1082				intent->id, intent->id + 1, GFP_ATOMIC);
1083		spin_unlock_irqrestore(&channel->intent_lock, flags);
1084
1085		if (ret < 0)
1086			dev_err(glink->dev, "failed to store remote intent\n");
1087	}
1088
1089	WRITE_ONCE(channel->intent_received, true);
1090	wake_up_all(&channel->intent_req_wq);
1091
1092	kfree(msg);
1093	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1094}
1095
1096static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1097{
1098	struct glink_channel *channel;
1099
1100	qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1101
1102	spin_lock(&glink->idr_lock);
1103	channel = idr_find(&glink->lcids, lcid);
1104	spin_unlock(&glink->idr_lock);
1105
1106	trace_qcom_glink_cmd_open_ack_rx(glink->label, channel ? channel->name : NULL,
1107					 lcid, channel ? channel->rcid : 0);
1108	if (!channel) {
1109		dev_err(glink->dev, "Invalid open ack packet\n");
1110		return -EINVAL;
1111	}
1112
1113	complete_all(&channel->open_ack);
1114
1115	return 0;
1116}
1117
1118/**
1119 * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1120 * @ept:	Rpmsg endpoint for channel.
1121 * @pause:	Pause transmission
1122 * @dst:	destination address of the endpoint
1123 *
1124 * Return: 0 on success or standard Linux error code.
1125 */
1126static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1127{
1128	struct glink_channel *channel = to_glink_channel(ept);
1129	struct qcom_glink *glink = channel->glink;
1130	struct glink_msg msg;
1131	u32 sigs = 0;
1132
1133	if (pause)
1134		sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1135
1136	msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1137	msg.param1 = cpu_to_le16(channel->lcid);
1138	msg.param2 = cpu_to_le32(sigs);
1139
1140	trace_qcom_glink_cmd_signal_tx(glink->label, channel->name,
1141				       channel->lcid, channel->rcid, sigs);
1142
1143	return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
1144}
1145
1146static void qcom_glink_handle_signals(struct qcom_glink *glink,
1147				      unsigned int rcid, unsigned int sigs)
1148{
1149	struct glink_channel *channel;
1150	unsigned long flags;
1151	bool enable;
1152
1153	qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1154
1155	spin_lock_irqsave(&glink->idr_lock, flags);
1156	channel = idr_find(&glink->rcids, rcid);
1157	spin_unlock_irqrestore(&glink->idr_lock, flags);
1158
1159	trace_qcom_glink_cmd_signal_rx(glink->label, channel ? channel->name : NULL,
1160				       channel ? channel->lcid : 0, rcid, sigs);
1161	if (!channel) {
1162		dev_err(glink->dev, "signal for non-existing channel\n");
1163		return;
1164	}
1165
1166	enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1167
1168	if (channel->ept.flow_cb)
1169		channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1170}
1171
1172void qcom_glink_native_rx(struct qcom_glink *glink)
1173{
1174	struct glink_msg msg;
1175	unsigned int param1;
1176	unsigned int param2;
1177	unsigned int avail;
1178	unsigned int cmd;
1179	int ret = 0;
1180
1181	/* To wakeup any blocking writers */
1182	wake_up_all(&glink->tx_avail_notify);
1183
1184	for (;;) {
1185		avail = qcom_glink_rx_avail(glink);
1186		if (avail < sizeof(msg))
1187			break;
1188
1189		qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg));
1190
1191		cmd = le16_to_cpu(msg.cmd);
1192		param1 = le16_to_cpu(msg.param1);
1193		param2 = le32_to_cpu(msg.param2);
1194
1195		switch (cmd) {
1196		case GLINK_CMD_VERSION:
1197		case GLINK_CMD_VERSION_ACK:
1198		case GLINK_CMD_CLOSE:
1199		case GLINK_CMD_CLOSE_ACK:
1200		case GLINK_CMD_RX_INTENT_REQ:
1201			ret = qcom_glink_rx_defer(glink, 0);
1202			break;
1203		case GLINK_CMD_OPEN_ACK:
1204			ret = qcom_glink_rx_open_ack(glink, param1);
1205			break;
1206		case GLINK_CMD_OPEN:
1207			/* upper 16 bits of param2 are the "prio" field */
1208			ret = qcom_glink_rx_defer(glink, param2 & 0xffff);
1209			break;
1210		case GLINK_CMD_TX_DATA:
1211		case GLINK_CMD_TX_DATA_CONT:
1212			ret = qcom_glink_rx_data(glink, avail);
1213			break;
1214		case GLINK_CMD_READ_NOTIF:
1215			qcom_glink_rx_read_notif(glink);
1216			break;
1217		case GLINK_CMD_INTENT:
1218			qcom_glink_handle_intent(glink, param1, param2, avail);
1219			break;
1220		case GLINK_CMD_RX_DONE:
1221			qcom_glink_handle_rx_done(glink, param1, param2, false);
1222			break;
1223		case GLINK_CMD_RX_DONE_W_REUSE:
1224			qcom_glink_handle_rx_done(glink, param1, param2, true);
1225			break;
1226		case GLINK_CMD_RX_INTENT_REQ_ACK:
1227			qcom_glink_handle_intent_req_ack(glink, param1, param2);
1228			break;
1229		case GLINK_CMD_SIGNALS:
1230			qcom_glink_handle_signals(glink, param1, param2);
1231			break;
1232		default:
1233			dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1234			ret = -EINVAL;
1235			break;
1236		}
1237
1238		if (ret)
1239			break;
1240	}
1241}
1242EXPORT_SYMBOL(qcom_glink_native_rx);
1243
1244/* Locally initiated rpmsg_create_ept */
1245static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1246						     const char *name)
1247{
1248	struct glink_channel *channel;
1249	int ret;
1250	unsigned long flags;
1251
1252	channel = qcom_glink_alloc_channel(glink, name);
1253	if (IS_ERR(channel))
1254		return ERR_CAST(channel);
1255
1256	ret = qcom_glink_send_open_req(glink, channel);
1257	if (ret)
1258		goto release_channel;
1259
1260	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1261	if (!ret)
1262		goto err_timeout;
1263
1264	ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1265	if (!ret)
1266		goto err_timeout;
1267
1268	qcom_glink_send_open_ack(glink, channel);
1269
1270	return channel;
1271
1272err_timeout:
1273	/* qcom_glink_send_open_req() did register the channel in lcids*/
1274	spin_lock_irqsave(&glink->idr_lock, flags);
1275	idr_remove(&glink->lcids, channel->lcid);
1276	spin_unlock_irqrestore(&glink->idr_lock, flags);
1277
1278release_channel:
1279	/* Release qcom_glink_send_open_req() reference */
1280	kref_put(&channel->refcount, qcom_glink_channel_release);
1281	/* Release qcom_glink_alloc_channel() reference */
1282	kref_put(&channel->refcount, qcom_glink_channel_release);
1283
1284	return ERR_PTR(-ETIMEDOUT);
1285}
1286
1287/* Remote initiated rpmsg_create_ept */
1288static int qcom_glink_create_remote(struct qcom_glink *glink,
1289				    struct glink_channel *channel)
1290{
1291	int ret;
1292
1293	qcom_glink_send_open_ack(glink, channel);
1294
1295	ret = qcom_glink_send_open_req(glink, channel);
1296	if (ret)
1297		goto close_link;
1298
1299	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1300	if (!ret) {
1301		ret = -ETIMEDOUT;
1302		goto close_link;
1303	}
1304
1305	return 0;
1306
1307close_link:
1308	/*
1309	 * Send a close request to "undo" our open-ack. The close-ack will
1310	 * release qcom_glink_send_open_req() reference and the last reference
1311	 * will be relesed after receiving remote_close or transport unregister
1312	 * by calling qcom_glink_native_remove().
1313	 */
1314	qcom_glink_send_close_req(glink, channel);
1315
1316	return ret;
1317}
1318
1319static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1320						    rpmsg_rx_cb_t cb,
1321						    void *priv,
1322						    struct rpmsg_channel_info
1323									chinfo)
1324{
1325	struct glink_channel *parent = to_glink_channel(rpdev->ept);
1326	struct glink_channel *channel;
1327	struct qcom_glink *glink = parent->glink;
1328	struct rpmsg_endpoint *ept;
1329	const char *name = chinfo.name;
1330	int cid;
1331	int ret;
1332	unsigned long flags;
1333
1334	spin_lock_irqsave(&glink->idr_lock, flags);
1335	idr_for_each_entry(&glink->rcids, channel, cid) {
1336		if (!strcmp(channel->name, name))
1337			break;
1338	}
1339	spin_unlock_irqrestore(&glink->idr_lock, flags);
1340
1341	if (!channel) {
1342		channel = qcom_glink_create_local(glink, name);
1343		if (IS_ERR(channel))
1344			return NULL;
1345	} else {
1346		ret = qcom_glink_create_remote(glink, channel);
1347		if (ret)
1348			return NULL;
1349	}
1350
1351	ept = &channel->ept;
1352	ept->rpdev = rpdev;
1353	ept->cb = cb;
1354	ept->priv = priv;
1355	ept->ops = &glink_endpoint_ops;
1356
1357	return ept;
1358}
1359
1360static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1361{
1362	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1363	struct device_node *np = rpdev->dev.of_node;
1364	struct qcom_glink *glink = channel->glink;
1365	struct glink_core_rx_intent *intent;
1366	const struct property *prop = NULL;
1367	__be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1368	int num_intents;
1369	int num_groups = 1;
1370	__be32 *val = defaults;
1371	int size;
1372
1373	if (glink->intentless || !completion_done(&channel->open_ack))
1374		return 0;
1375
1376	prop = of_find_property(np, "qcom,intents", NULL);
1377	if (prop) {
1378		val = prop->value;
1379		num_groups = prop->length / sizeof(u32) / 2;
1380	}
1381
1382	/* Channel is now open, advertise base set of intents */
1383	while (num_groups--) {
1384		size = be32_to_cpup(val++);
1385		num_intents = be32_to_cpup(val++);
1386		while (num_intents--) {
1387			intent = qcom_glink_alloc_intent(glink, channel, size,
1388							 true);
1389			if (!intent)
1390				break;
1391
1392			qcom_glink_advertise_intent(glink, channel, intent);
1393		}
1394	}
1395	return 0;
1396}
1397
1398static void qcom_glink_remove_rpmsg_device(struct qcom_glink *glink, struct glink_channel *channel)
1399{
1400	struct rpmsg_channel_info chinfo;
1401
1402	if (channel->rpdev) {
1403		strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1404		chinfo.src = RPMSG_ADDR_ANY;
1405		chinfo.dst = RPMSG_ADDR_ANY;
1406		rpmsg_unregister_device(glink->dev, &chinfo);
1407	}
1408	channel->rpdev = NULL;
1409}
1410
1411static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1412{
1413	struct glink_channel *channel = to_glink_channel(ept);
1414	struct qcom_glink *glink = channel->glink;
1415	unsigned long flags;
1416
1417	spin_lock_irqsave(&channel->recv_lock, flags);
1418	channel->ept.cb = NULL;
1419	spin_unlock_irqrestore(&channel->recv_lock, flags);
1420
1421	/* Decouple the potential rpdev from the channel */
1422	qcom_glink_remove_rpmsg_device(glink, channel);
1423
1424	qcom_glink_send_close_req(glink, channel);
1425}
1426
1427static int qcom_glink_request_intent(struct qcom_glink *glink,
1428				     struct glink_channel *channel,
1429				     size_t size)
1430{
1431	struct {
1432		u16 id;
1433		u16 cid;
1434		u32 size;
1435	} __packed cmd;
1436
1437	int ret;
1438
1439	mutex_lock(&channel->intent_req_lock);
1440
1441	WRITE_ONCE(channel->intent_req_result, -1);
1442	WRITE_ONCE(channel->intent_received, false);
1443
1444	cmd.id = GLINK_CMD_RX_INTENT_REQ;
1445	cmd.cid = channel->lcid;
1446	cmd.size = size;
1447
1448	trace_qcom_glink_cmd_rx_intent_req_tx(glink->label, channel->name,
1449					      channel->lcid, channel->rcid,
1450					      cmd.size);
1451
1452	ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1453	if (ret)
1454		goto unlock;
1455
1456	ret = wait_event_timeout(channel->intent_req_wq,
1457				 READ_ONCE(channel->intent_req_result) == 0 ||
1458				 (READ_ONCE(channel->intent_req_result) > 0 &&
1459				  READ_ONCE(channel->intent_received)) ||
1460				 glink->abort_tx,
1461				 10 * HZ);
1462	if (!ret) {
1463		dev_err(glink->dev, "intent request timed out\n");
1464		ret = -ETIMEDOUT;
1465	} else if (glink->abort_tx) {
1466		ret = -ECANCELED;
1467	} else {
1468		ret = READ_ONCE(channel->intent_req_result) ? 0 : -EAGAIN;
1469	}
1470
1471unlock:
1472	mutex_unlock(&channel->intent_req_lock);
1473	return ret;
1474}
1475
1476static int __qcom_glink_send(struct glink_channel *channel,
1477			     void *data, int len, bool wait)
1478{
1479	struct qcom_glink *glink = channel->glink;
1480	struct glink_core_rx_intent *intent = NULL;
1481	struct glink_core_rx_intent *tmp;
1482	int iid = 0;
1483	struct {
1484		struct glink_msg_hdr msg;
1485		__le32 chunk_size;
1486		__le32 left_size;
1487	} __packed req;
1488	int ret;
1489	unsigned long flags;
1490	int chunk_size = len;
1491	size_t offset = 0;
1492
1493	if (!glink->intentless) {
1494		while (!intent) {
1495			spin_lock_irqsave(&channel->intent_lock, flags);
1496			idr_for_each_entry(&channel->riids, tmp, iid) {
1497				if (tmp->size >= len && !tmp->in_use) {
1498					if (!intent)
1499						intent = tmp;
1500					else if (intent->size > tmp->size)
1501						intent = tmp;
1502					if (intent->size == len)
1503						break;
1504				}
1505			}
1506			if (intent)
1507				intent->in_use = true;
1508			spin_unlock_irqrestore(&channel->intent_lock, flags);
1509
1510			/* We found an available intent */
1511			if (intent)
1512				break;
1513
1514			if (!wait)
1515				return -EBUSY;
1516
1517			ret = qcom_glink_request_intent(glink, channel, len);
1518			if (ret < 0)
1519				return ret;
1520		}
1521
1522		iid = intent->id;
1523	}
1524
1525	while (offset < len) {
1526		chunk_size = len - offset;
1527		if (chunk_size > SZ_8K && wait)
1528			chunk_size = SZ_8K;
1529
1530		req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1531		req.msg.param1 = cpu_to_le16(channel->lcid);
1532		req.msg.param2 = cpu_to_le32(iid);
1533		req.chunk_size = cpu_to_le32(chunk_size);
1534		req.left_size = cpu_to_le32(len - offset - chunk_size);
1535
1536		trace_qcom_glink_cmd_tx_data_tx(glink->label, channel->name,
1537						channel->lcid, channel->rcid,
1538						iid, chunk_size,
1539						len - offset - chunk_size,
1540						offset > 0);
1541
1542		ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait);
1543		if (ret) {
1544			/* Mark intent available if we failed */
1545			if (intent)
1546				intent->in_use = false;
1547			return ret;
1548		}
1549
1550		offset += chunk_size;
1551	}
1552
1553	return 0;
1554}
1555
1556static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1557{
1558	struct glink_channel *channel = to_glink_channel(ept);
1559
1560	return __qcom_glink_send(channel, data, len, true);
1561}
1562
1563static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1564{
1565	struct glink_channel *channel = to_glink_channel(ept);
1566
1567	return __qcom_glink_send(channel, data, len, false);
1568}
1569
1570static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1571{
1572	struct glink_channel *channel = to_glink_channel(ept);
1573
1574	return __qcom_glink_send(channel, data, len, true);
1575}
1576
1577static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1578{
1579	struct glink_channel *channel = to_glink_channel(ept);
1580
1581	return __qcom_glink_send(channel, data, len, false);
1582}
1583
1584/*
1585 * Finds the device_node for the glink child interested in this channel.
1586 */
1587static struct device_node *qcom_glink_match_channel(struct device_node *node,
1588						    const char *channel)
1589{
1590	struct device_node *child;
1591	const char *name;
1592	const char *key;
1593	int ret;
1594
1595	for_each_available_child_of_node(node, child) {
1596		key = "qcom,glink-channels";
1597		ret = of_property_read_string(child, key, &name);
1598		if (ret)
1599			continue;
1600
1601		if (strcmp(name, channel) == 0)
1602			return child;
1603	}
1604
1605	return NULL;
1606}
1607
1608static const struct rpmsg_device_ops glink_device_ops = {
1609	.create_ept = qcom_glink_create_ept,
1610	.announce_create = qcom_glink_announce_create,
1611};
1612
1613static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1614	.destroy_ept = qcom_glink_destroy_ept,
1615	.send = qcom_glink_send,
1616	.sendto = qcom_glink_sendto,
1617	.trysend = qcom_glink_trysend,
1618	.trysendto = qcom_glink_trysendto,
1619	.set_flow_control = qcom_glink_set_flow_control,
1620};
1621
1622static void qcom_glink_rpdev_release(struct device *dev)
1623{
1624	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1625
1626	kfree(rpdev->driver_override);
1627	kfree(rpdev);
1628}
1629
1630static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1631			      char *name)
1632{
1633	struct glink_channel *channel;
1634	struct rpmsg_device *rpdev;
1635	bool create_device = false;
1636	struct device_node *node;
1637	int lcid;
1638	int ret;
1639	unsigned long flags;
1640
1641	spin_lock_irqsave(&glink->idr_lock, flags);
1642	idr_for_each_entry(&glink->lcids, channel, lcid) {
1643		if (!strcmp(channel->name, name))
1644			break;
1645	}
1646	spin_unlock_irqrestore(&glink->idr_lock, flags);
1647
1648	if (!channel) {
1649		channel = qcom_glink_alloc_channel(glink, name);
1650		if (IS_ERR(channel))
1651			return PTR_ERR(channel);
1652
1653		/* The opening dance was initiated by the remote */
1654		create_device = true;
1655	}
1656
1657	trace_qcom_glink_cmd_open_rx(glink->label, name, channel->lcid, rcid);
1658
1659	spin_lock_irqsave(&glink->idr_lock, flags);
1660	ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1661	if (ret < 0) {
1662		dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1663		spin_unlock_irqrestore(&glink->idr_lock, flags);
1664		goto free_channel;
1665	}
1666	channel->rcid = ret;
1667	spin_unlock_irqrestore(&glink->idr_lock, flags);
1668
1669	complete_all(&channel->open_req);
1670
1671	if (create_device) {
1672		rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1673		if (!rpdev) {
1674			ret = -ENOMEM;
1675			goto rcid_remove;
1676		}
1677
1678		rpdev->ept = &channel->ept;
1679		strscpy(rpdev->id.name, name);
1680		rpdev->src = RPMSG_ADDR_ANY;
1681		rpdev->dst = RPMSG_ADDR_ANY;
1682		rpdev->ops = &glink_device_ops;
1683
1684		node = qcom_glink_match_channel(glink->dev->of_node, name);
1685		rpdev->dev.of_node = node;
1686		rpdev->dev.parent = glink->dev;
1687		rpdev->dev.release = qcom_glink_rpdev_release;
1688
1689		ret = rpmsg_register_device(rpdev);
1690		if (ret)
1691			goto rcid_remove;
1692
1693		channel->rpdev = rpdev;
1694	}
1695
1696	return 0;
1697
1698rcid_remove:
1699	spin_lock_irqsave(&glink->idr_lock, flags);
1700	idr_remove(&glink->rcids, channel->rcid);
1701	channel->rcid = 0;
1702	spin_unlock_irqrestore(&glink->idr_lock, flags);
1703free_channel:
1704	/* Release the reference, iff we took it */
1705	if (create_device)
1706		kref_put(&channel->refcount, qcom_glink_channel_release);
1707
1708	return ret;
1709}
1710
1711static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1712{
1713	struct glink_channel *channel;
1714	unsigned long flags;
1715
1716	spin_lock_irqsave(&glink->idr_lock, flags);
1717	channel = idr_find(&glink->rcids, rcid);
1718	spin_unlock_irqrestore(&glink->idr_lock, flags);
1719
1720	trace_qcom_glink_cmd_close_rx(glink->label, channel ? channel->name : NULL,
1721				      channel ? channel->lcid : 0, rcid);
1722	if (WARN(!channel, "close request on unknown channel\n"))
1723		return;
1724
1725	/* cancel pending rx_done work */
1726	cancel_work_sync(&channel->intent_work);
1727
1728	qcom_glink_remove_rpmsg_device(glink, channel);
1729
1730	qcom_glink_send_close_ack(glink, channel);
1731
1732	spin_lock_irqsave(&glink->idr_lock, flags);
1733	idr_remove(&glink->rcids, channel->rcid);
1734	channel->rcid = 0;
1735	spin_unlock_irqrestore(&glink->idr_lock, flags);
1736
1737	kref_put(&channel->refcount, qcom_glink_channel_release);
1738}
1739
1740static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1741{
1742	struct glink_channel *channel;
1743	unsigned long flags;
1744
1745	/* To wakeup any blocking writers */
1746	wake_up_all(&glink->tx_avail_notify);
1747
1748	spin_lock_irqsave(&glink->idr_lock, flags);
1749	channel = idr_find(&glink->lcids, lcid);
1750
1751	trace_qcom_glink_cmd_close_ack_rx(glink->label, channel ? channel->name : NULL,
1752					  lcid, channel ? channel->rcid : 0);
1753	if (WARN(!channel, "close ack on unknown channel\n")) {
1754		spin_unlock_irqrestore(&glink->idr_lock, flags);
1755		return;
1756	}
1757
1758	idr_remove(&glink->lcids, channel->lcid);
1759	channel->lcid = 0;
1760	spin_unlock_irqrestore(&glink->idr_lock, flags);
1761
1762	/* Decouple the potential rpdev from the channel */
1763	qcom_glink_remove_rpmsg_device(glink, channel);
1764
1765	kref_put(&channel->refcount, qcom_glink_channel_release);
1766}
1767
1768static void qcom_glink_work(struct work_struct *work)
1769{
1770	struct qcom_glink *glink = container_of(work, struct qcom_glink,
1771						rx_work);
1772	struct glink_defer_cmd *dcmd;
1773	struct glink_msg *msg;
1774	unsigned long flags;
1775	unsigned int param1;
1776	unsigned int param2;
1777	unsigned int cmd;
1778
1779	for (;;) {
1780		spin_lock_irqsave(&glink->rx_lock, flags);
1781		if (list_empty(&glink->rx_queue)) {
1782			spin_unlock_irqrestore(&glink->rx_lock, flags);
1783			break;
1784		}
1785		dcmd = list_first_entry(&glink->rx_queue,
1786					struct glink_defer_cmd, node);
1787		list_del(&dcmd->node);
1788		spin_unlock_irqrestore(&glink->rx_lock, flags);
1789
1790		msg = container_of(&dcmd->msg, struct glink_msg, hdr);
1791		cmd = le16_to_cpu(msg->cmd);
1792		param1 = le16_to_cpu(msg->param1);
1793		param2 = le32_to_cpu(msg->param2);
1794
1795		switch (cmd) {
1796		case GLINK_CMD_VERSION:
1797			qcom_glink_receive_version(glink, param1, param2);
1798			break;
1799		case GLINK_CMD_VERSION_ACK:
1800			qcom_glink_receive_version_ack(glink, param1, param2);
1801			break;
1802		case GLINK_CMD_OPEN:
1803			qcom_glink_rx_open(glink, param1, msg->data);
1804			break;
1805		case GLINK_CMD_CLOSE:
1806			qcom_glink_rx_close(glink, param1);
1807			break;
1808		case GLINK_CMD_CLOSE_ACK:
1809			qcom_glink_rx_close_ack(glink, param1);
1810			break;
1811		case GLINK_CMD_RX_INTENT_REQ:
1812			qcom_glink_handle_intent_req(glink, param1, param2);
1813			break;
1814		default:
1815			WARN(1, "Unknown defer object %d\n", cmd);
1816			break;
1817		}
1818
1819		kfree(dcmd);
1820	}
1821}
1822
1823static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1824{
1825	struct glink_defer_cmd *dcmd;
1826	struct glink_defer_cmd *tmp;
1827
1828	/* cancel any pending deferred rx_work */
1829	cancel_work_sync(&glink->rx_work);
1830
1831	list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1832		kfree(dcmd);
1833}
1834
1835static ssize_t rpmsg_name_show(struct device *dev,
1836			       struct device_attribute *attr, char *buf)
1837{
1838	int ret = 0;
1839	const char *name;
1840
1841	ret = of_property_read_string(dev->of_node, "label", &name);
1842	if (ret < 0)
1843		name = dev->of_node->name;
1844
1845	return sysfs_emit(buf, "%s\n", name);
1846}
1847static DEVICE_ATTR_RO(rpmsg_name);
1848
1849static struct attribute *qcom_glink_attrs[] = {
1850	&dev_attr_rpmsg_name.attr,
1851	NULL
1852};
1853ATTRIBUTE_GROUPS(qcom_glink);
1854
1855static void qcom_glink_device_release(struct device *dev)
1856{
1857	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1858	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1859
1860	/* Release qcom_glink_alloc_channel() reference */
1861	kref_put(&channel->refcount, qcom_glink_channel_release);
1862	kfree(rpdev->driver_override);
1863	kfree(rpdev);
1864}
1865
1866static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1867{
1868	struct rpmsg_device *rpdev;
1869	struct glink_channel *channel;
1870
1871	rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1872	if (!rpdev)
1873		return -ENOMEM;
1874
1875	channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");
1876	if (IS_ERR(channel)) {
1877		kfree(rpdev);
1878		return PTR_ERR(channel);
1879	}
1880	channel->rpdev = rpdev;
1881
1882	rpdev->ept = &channel->ept;
1883	rpdev->ops = &glink_device_ops;
1884	rpdev->dev.parent = glink->dev;
1885	rpdev->dev.release = qcom_glink_device_release;
1886
1887	return rpmsg_ctrldev_register_device(rpdev);
1888}
1889
1890struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1891					   unsigned long features,
1892					   struct qcom_glink_pipe *rx,
1893					   struct qcom_glink_pipe *tx,
1894					   bool intentless)
1895{
1896	int ret;
1897	struct qcom_glink *glink;
1898
1899	glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1900	if (!glink)
1901		return ERR_PTR(-ENOMEM);
1902
1903	glink->dev = dev;
1904	glink->tx_pipe = tx;
1905	glink->rx_pipe = rx;
1906
1907	glink->features = features;
1908	glink->intentless = intentless;
1909
1910	spin_lock_init(&glink->tx_lock);
1911	spin_lock_init(&glink->rx_lock);
1912	INIT_LIST_HEAD(&glink->rx_queue);
1913	INIT_WORK(&glink->rx_work, qcom_glink_work);
1914	init_waitqueue_head(&glink->tx_avail_notify);
1915
1916	spin_lock_init(&glink->idr_lock);
1917	idr_init(&glink->lcids);
1918	idr_init(&glink->rcids);
1919
1920	ret = of_property_read_string(dev->of_node, "label", &glink->label);
1921	if (ret < 0)
1922		glink->label = dev->of_node->name;
1923
1924	glink->dev->groups = qcom_glink_groups;
1925
1926	ret = device_add_groups(dev, qcom_glink_groups);
1927	if (ret)
1928		dev_err(dev, "failed to add groups\n");
1929
1930	ret = qcom_glink_send_version(glink);
1931	if (ret)
1932		return ERR_PTR(ret);
1933
1934	ret = qcom_glink_create_chrdev(glink);
1935	if (ret)
1936		dev_err(glink->dev, "failed to register chrdev\n");
1937
1938	return glink;
1939}
1940EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1941
1942static int qcom_glink_remove_device(struct device *dev, void *data)
1943{
1944	device_unregister(dev);
1945
1946	return 0;
1947}
1948
1949void qcom_glink_native_remove(struct qcom_glink *glink)
1950{
1951	struct glink_channel *channel;
1952	unsigned long flags;
1953	int cid;
1954	int ret;
1955
1956	qcom_glink_cancel_rx_work(glink);
1957
1958	/* Fail all attempts at sending messages */
1959	spin_lock_irqsave(&glink->tx_lock, flags);
1960	glink->abort_tx = true;
1961	wake_up_all(&glink->tx_avail_notify);
1962	spin_unlock_irqrestore(&glink->tx_lock, flags);
1963
1964	/* Abort any senders waiting for intent requests */
1965	spin_lock_irqsave(&glink->idr_lock, flags);
1966	idr_for_each_entry(&glink->lcids, channel, cid)
1967		qcom_glink_intent_req_abort(channel);
1968	spin_unlock_irqrestore(&glink->idr_lock, flags);
1969
1970	ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1971	if (ret)
1972		dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1973
1974	/* Release any defunct local channels, waiting for close-ack */
1975	idr_for_each_entry(&glink->lcids, channel, cid)
1976		kref_put(&channel->refcount, qcom_glink_channel_release);
1977
1978	/* Release any defunct local channels, waiting for close-req */
1979	idr_for_each_entry(&glink->rcids, channel, cid)
1980		kref_put(&channel->refcount, qcom_glink_channel_release);
1981
1982	idr_destroy(&glink->lcids);
1983	idr_destroy(&glink->rcids);
1984}
1985EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1986
1987MODULE_DESCRIPTION("Qualcomm GLINK driver");
1988MODULE_LICENSE("GPL v2");