main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * MHI Endpoint bus stack
   4 *
   5 * Copyright (C) 2022 Linaro Ltd.
   6 * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
   7 */
   8
   9#include <linux/bitfield.h>
  10#include <linux/delay.h>
  11#include <linux/dma-direction.h>
  12#include <linux/interrupt.h>
  13#include <linux/io.h>
  14#include <linux/irq.h>
  15#include <linux/mhi_ep.h>
  16#include <linux/mod_devicetable.h>
  17#include <linux/module.h>
  18#include "internal.h"
  19
  20#define M0_WAIT_DELAY_MS	100
  21#define M0_WAIT_COUNT		100
  22
  23static DEFINE_IDA(mhi_ep_cntrl_ida);
  24
  25static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id);
  26static int mhi_ep_destroy_device(struct device *dev, void *data);
  27
  28static int mhi_ep_send_event(struct mhi_ep_cntrl *mhi_cntrl, u32 ring_idx,
  29			     struct mhi_ring_element *el, bool bei)
  30{
  31	struct device *dev = &mhi_cntrl->mhi_dev->dev;
  32	union mhi_ep_ring_ctx *ctx;
  33	struct mhi_ep_ring *ring;
  34	int ret;
  35
  36	mutex_lock(&mhi_cntrl->event_lock);
  37	ring = &mhi_cntrl->mhi_event[ring_idx].ring;
  38	ctx = (union mhi_ep_ring_ctx *)&mhi_cntrl->ev_ctx_cache[ring_idx];
  39	if (!ring->started) {
  40		ret = mhi_ep_ring_start(mhi_cntrl, ring, ctx);
  41		if (ret) {
  42			dev_err(dev, "Error starting event ring (%u)\n", ring_idx);
  43			goto err_unlock;
  44		}
  45	}
  46
  47	/* Add element to the event ring */
  48	ret = mhi_ep_ring_add_element(ring, el);
  49	if (ret) {
  50		dev_err(dev, "Error adding element to event ring (%u)\n", ring_idx);
  51		goto err_unlock;
  52	}
  53
  54	mutex_unlock(&mhi_cntrl->event_lock);
  55
  56	/*
  57	 * As per the MHI specification, section 4.3, Interrupt moderation:
  58	 *
  59	 * 1. If BEI flag is not set, cancel any pending intmodt work if started
  60	 * for the event ring and raise IRQ immediately.
  61	 *
  62	 * 2. If both BEI and intmodt are set, and if no IRQ is pending for the
  63	 * same event ring, start the IRQ delayed work as per the value of
  64	 * intmodt. If previous IRQ is pending, then do nothing as the pending
  65	 * IRQ is enough for the host to process the current event ring element.
  66	 *
  67	 * 3. If BEI is set and intmodt is not set, no need to raise IRQ.
  68	 */
  69	if (!bei) {
  70		if (READ_ONCE(ring->irq_pending))
  71			cancel_delayed_work(&ring->intmodt_work);
  72
  73		mhi_cntrl->raise_irq(mhi_cntrl, ring->irq_vector);
  74	} else if (ring->intmodt && !READ_ONCE(ring->irq_pending)) {
  75		WRITE_ONCE(ring->irq_pending, true);
  76		schedule_delayed_work(&ring->intmodt_work, msecs_to_jiffies(ring->intmodt));
  77	}
  78
  79	return 0;
  80
  81err_unlock:
  82	mutex_unlock(&mhi_cntrl->event_lock);
  83
  84	return ret;
  85}
  86
  87static int mhi_ep_send_completion_event(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_ring *ring,
  88					struct mhi_ring_element *tre, u32 len, enum mhi_ev_ccs code)
  89{
  90	struct mhi_ring_element *event;
  91	int ret;
  92
  93	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
  94	if (!event)
  95		return -ENOMEM;
  96
  97	event->ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(*tre));
  98	event->dword[0] = MHI_TRE_EV_DWORD0(code, len);
  99	event->dword[1] = MHI_TRE_EV_DWORD1(ring->ch_id, MHI_PKT_TYPE_TX_EVENT);
 100
 101	ret = mhi_ep_send_event(mhi_cntrl, ring->er_index, event, MHI_TRE_DATA_GET_BEI(tre));
 102	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
 103
 104	return ret;
 105}
 106
 107int mhi_ep_send_state_change_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_state state)
 108{
 109	struct mhi_ring_element *event;
 110	int ret;
 111
 112	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
 113	if (!event)
 114		return -ENOMEM;
 115
 116	event->dword[0] = MHI_SC_EV_DWORD0(state);
 117	event->dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_STATE_CHANGE_EVENT);
 118
 119	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
 120	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
 121
 122	return ret;
 123}
 124
 125int mhi_ep_send_ee_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ee_type exec_env)
 126{
 127	struct mhi_ring_element *event;
 128	int ret;
 129
 130	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
 131	if (!event)
 132		return -ENOMEM;
 133
 134	event->dword[0] = MHI_EE_EV_DWORD0(exec_env);
 135	event->dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_EE_EVENT);
 136
 137	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
 138	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
 139
 140	return ret;
 141}
 142
 143static int mhi_ep_send_cmd_comp_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ev_ccs code)
 144{
 145	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
 146	struct mhi_ring_element *event;
 147	int ret;
 148
 149	event = kmem_cache_zalloc(mhi_cntrl->ev_ring_el_cache, GFP_KERNEL);
 150	if (!event)
 151		return -ENOMEM;
 152
 153	event->ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(struct mhi_ring_element));
 154	event->dword[0] = MHI_CC_EV_DWORD0(code);
 155	event->dword[1] = MHI_CC_EV_DWORD1(MHI_PKT_TYPE_CMD_COMPLETION_EVENT);
 156
 157	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
 158	kmem_cache_free(mhi_cntrl->ev_ring_el_cache, event);
 159
 160	return ret;
 161}
 162
 163static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_element *el)
 164{
 165	struct mhi_ep_cntrl *mhi_cntrl = ring->mhi_cntrl;
 166	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 167	struct mhi_result result = {};
 168	struct mhi_ep_chan *mhi_chan;
 169	struct mhi_ep_ring *ch_ring;
 170	u32 tmp, ch_id;
 171	int ret;
 172
 173	ch_id = MHI_TRE_GET_CMD_CHID(el);
 174
 175	/* Check if the channel is supported by the controller */
 176	if ((ch_id >= mhi_cntrl->max_chan) || !mhi_cntrl->mhi_chan[ch_id].name) {
 177		dev_dbg(dev, "Channel (%u) not supported!\n", ch_id);
 178		return -ENODEV;
 179	}
 180
 181	mhi_chan = &mhi_cntrl->mhi_chan[ch_id];
 182	ch_ring = &mhi_cntrl->mhi_chan[ch_id].ring;
 183
 184	switch (MHI_TRE_GET_CMD_TYPE(el)) {
 185	case MHI_PKT_TYPE_START_CHAN_CMD:
 186		dev_dbg(dev, "Received START command for channel (%u)\n", ch_id);
 187
 188		mutex_lock(&mhi_chan->lock);
 189		/* Initialize and configure the corresponding channel ring */
 190		if (!ch_ring->started) {
 191			ret = mhi_ep_ring_start(mhi_cntrl, ch_ring,
 192				(union mhi_ep_ring_ctx *)&mhi_cntrl->ch_ctx_cache[ch_id]);
 193			if (ret) {
 194				dev_err(dev, "Failed to start ring for channel (%u)\n", ch_id);
 195				ret = mhi_ep_send_cmd_comp_event(mhi_cntrl,
 196							MHI_EV_CC_UNDEFINED_ERR);
 197				if (ret)
 198					dev_err(dev, "Error sending completion event: %d\n", ret);
 199
 200				goto err_unlock;
 201			}
 202
 203			mhi_chan->rd_offset = ch_ring->rd_offset;
 204		}
 205
 206		/* Set channel state to RUNNING */
 207		mhi_chan->state = MHI_CH_STATE_RUNNING;
 208		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
 209		tmp &= ~CHAN_CTX_CHSTATE_MASK;
 210		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_RUNNING);
 211		mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
 212
 213		ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
 214		if (ret) {
 215			dev_err(dev, "Error sending command completion event (%u)\n",
 216				MHI_EV_CC_SUCCESS);
 217			goto err_unlock;
 218		}
 219
 220		mutex_unlock(&mhi_chan->lock);
 221
 222		/*
 223		 * Create MHI device only during UL channel start. Since the MHI
 224		 * channels operate in a pair, we'll associate both UL and DL
 225		 * channels to the same device.
 226		 *
 227		 * We also need to check for mhi_dev != NULL because, the host
 228		 * will issue START_CHAN command during resume and we don't
 229		 * destroy the device during suspend.
 230		 */
 231		if (!(ch_id % 2) && !mhi_chan->mhi_dev) {
 232			ret = mhi_ep_create_device(mhi_cntrl, ch_id);
 233			if (ret) {
 234				dev_err(dev, "Error creating device for channel (%u)\n", ch_id);
 235				mhi_ep_handle_syserr(mhi_cntrl);
 236				return ret;
 237			}
 238		}
 239
 240		/* Finally, enable DB for the channel */
 241		mhi_ep_mmio_enable_chdb(mhi_cntrl, ch_id);
 242
 243		break;
 244	case MHI_PKT_TYPE_STOP_CHAN_CMD:
 245		dev_dbg(dev, "Received STOP command for channel (%u)\n", ch_id);
 246		if (!ch_ring->started) {
 247			dev_err(dev, "Channel (%u) not opened\n", ch_id);
 248			return -ENODEV;
 249		}
 250
 251		mutex_lock(&mhi_chan->lock);
 252		/* Disable DB for the channel */
 253		mhi_ep_mmio_disable_chdb(mhi_cntrl, ch_id);
 254
 255		/* Send channel disconnect status to client drivers */
 256		if (mhi_chan->xfer_cb) {
 257			result.transaction_status = -ENOTCONN;
 258			result.bytes_xferd = 0;
 259			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
 260		}
 261
 262		/* Set channel state to STOP */
 263		mhi_chan->state = MHI_CH_STATE_STOP;
 264		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
 265		tmp &= ~CHAN_CTX_CHSTATE_MASK;
 266		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_STOP);
 267		mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
 268
 269		ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
 270		if (ret) {
 271			dev_err(dev, "Error sending command completion event (%u)\n",
 272				MHI_EV_CC_SUCCESS);
 273			goto err_unlock;
 274		}
 275
 276		mutex_unlock(&mhi_chan->lock);
 277		break;
 278	case MHI_PKT_TYPE_RESET_CHAN_CMD:
 279		dev_dbg(dev, "Received RESET command for channel (%u)\n", ch_id);
 280		if (!ch_ring->started) {
 281			dev_err(dev, "Channel (%u) not opened\n", ch_id);
 282			return -ENODEV;
 283		}
 284
 285		mutex_lock(&mhi_chan->lock);
 286		/* Stop and reset the transfer ring */
 287		mhi_ep_ring_reset(mhi_cntrl, ch_ring);
 288
 289		/* Send channel disconnect status to client driver */
 290		if (mhi_chan->xfer_cb) {
 291			result.transaction_status = -ENOTCONN;
 292			result.bytes_xferd = 0;
 293			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
 294		}
 295
 296		/* Set channel state to DISABLED */
 297		mhi_chan->state = MHI_CH_STATE_DISABLED;
 298		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[ch_id].chcfg);
 299		tmp &= ~CHAN_CTX_CHSTATE_MASK;
 300		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_DISABLED);
 301		mhi_cntrl->ch_ctx_cache[ch_id].chcfg = cpu_to_le32(tmp);
 302
 303		ret = mhi_ep_send_cmd_comp_event(mhi_cntrl, MHI_EV_CC_SUCCESS);
 304		if (ret) {
 305			dev_err(dev, "Error sending command completion event (%u)\n",
 306				MHI_EV_CC_SUCCESS);
 307			goto err_unlock;
 308		}
 309
 310		mutex_unlock(&mhi_chan->lock);
 311		break;
 312	default:
 313		dev_err(dev, "Invalid command received: %lu for channel (%u)\n",
 314			MHI_TRE_GET_CMD_TYPE(el), ch_id);
 315		return -EINVAL;
 316	}
 317
 318	return 0;
 319
 320err_unlock:
 321	mutex_unlock(&mhi_chan->lock);
 322
 323	return ret;
 324}
 325
 326bool mhi_ep_queue_is_empty(struct mhi_ep_device *mhi_dev, enum dma_data_direction dir)
 327{
 328	struct mhi_ep_chan *mhi_chan = (dir == DMA_FROM_DEVICE) ? mhi_dev->dl_chan :
 329								mhi_dev->ul_chan;
 330	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
 331	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
 332
 333	return !!(mhi_chan->rd_offset == ring->wr_offset);
 334}
 335EXPORT_SYMBOL_GPL(mhi_ep_queue_is_empty);
 336
 337static void mhi_ep_read_completion(struct mhi_ep_buf_info *buf_info)
 338{
 339	struct mhi_ep_device *mhi_dev = buf_info->mhi_dev;
 340	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
 341	struct mhi_ep_chan *mhi_chan = mhi_dev->ul_chan;
 342	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
 343	struct mhi_ring_element *el = &ring->ring_cache[ring->rd_offset];
 344	struct mhi_result result = {};
 345	int ret;
 346
 347	if (mhi_chan->xfer_cb) {
 348		result.buf_addr = buf_info->cb_buf;
 349		result.dir = mhi_chan->dir;
 350		result.bytes_xferd = buf_info->size;
 351
 352		mhi_chan->xfer_cb(mhi_dev, &result);
 353	}
 354
 355	/*
 356	 * The host will split the data packet into multiple TREs if it can't fit
 357	 * the packet in a single TRE. In that case, CHAIN flag will be set by the
 358	 * host for all TREs except the last one.
 359	 */
 360	if (buf_info->code != MHI_EV_CC_OVERFLOW) {
 361		if (MHI_TRE_DATA_GET_CHAIN(el)) {
 362			/*
 363			 * IEOB (Interrupt on End of Block) flag will be set by the host if
 364			 * it expects the completion event for all TREs of a TD.
 365			 */
 366			if (MHI_TRE_DATA_GET_IEOB(el)) {
 367				ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
 368							     MHI_TRE_DATA_GET_LEN(el),
 369							     MHI_EV_CC_EOB);
 370				if (ret < 0) {
 371					dev_err(&mhi_chan->mhi_dev->dev,
 372						"Error sending transfer compl. event\n");
 373					goto err_free_tre_buf;
 374				}
 375			}
 376		} else {
 377			/*
 378			 * IEOT (Interrupt on End of Transfer) flag will be set by the host
 379			 * for the last TRE of the TD and expects the completion event for
 380			 * the same.
 381			 */
 382			if (MHI_TRE_DATA_GET_IEOT(el)) {
 383				ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el,
 384							     MHI_TRE_DATA_GET_LEN(el),
 385							     MHI_EV_CC_EOT);
 386				if (ret < 0) {
 387					dev_err(&mhi_chan->mhi_dev->dev,
 388						"Error sending transfer compl. event\n");
 389					goto err_free_tre_buf;
 390				}
 391			}
 392		}
 393	}
 394
 395	mhi_ep_ring_inc_index(ring);
 396
 397err_free_tre_buf:
 398	kmem_cache_free(mhi_cntrl->tre_buf_cache, buf_info->cb_buf);
 399}
 400
 401static int mhi_ep_read_channel(struct mhi_ep_cntrl *mhi_cntrl,
 402			       struct mhi_ep_ring *ring)
 403{
 404	struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id];
 405	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 406	size_t tr_len, read_offset;
 407	struct mhi_ep_buf_info buf_info = {};
 408	u32 len = MHI_EP_DEFAULT_MTU;
 409	struct mhi_ring_element *el;
 410	void *buf_addr;
 411	int ret;
 412
 413	do {
 414		/* Don't process the transfer ring if the channel is not in RUNNING state */
 415		if (mhi_chan->state != MHI_CH_STATE_RUNNING) {
 416			dev_err(dev, "Channel not available\n");
 417			return -ENODEV;
 418		}
 419
 420		el = &ring->ring_cache[mhi_chan->rd_offset];
 421
 422		/* Check if there is data pending to be read from previous read operation */
 423		if (mhi_chan->tre_bytes_left) {
 424			dev_dbg(dev, "TRE bytes remaining: %u\n", mhi_chan->tre_bytes_left);
 425			tr_len = min(len, mhi_chan->tre_bytes_left);
 426		} else {
 427			mhi_chan->tre_loc = MHI_TRE_DATA_GET_PTR(el);
 428			mhi_chan->tre_size = MHI_TRE_DATA_GET_LEN(el);
 429			mhi_chan->tre_bytes_left = mhi_chan->tre_size;
 430
 431			tr_len = min(len, mhi_chan->tre_size);
 432		}
 433
 434		read_offset = mhi_chan->tre_size - mhi_chan->tre_bytes_left;
 435
 436		buf_addr = kmem_cache_zalloc(mhi_cntrl->tre_buf_cache, GFP_KERNEL);
 437		if (!buf_addr)
 438			return -ENOMEM;
 439
 440		buf_info.host_addr = mhi_chan->tre_loc + read_offset;
 441		buf_info.dev_addr = buf_addr;
 442		buf_info.size = tr_len;
 443		buf_info.cb = mhi_ep_read_completion;
 444		buf_info.cb_buf = buf_addr;
 445		buf_info.mhi_dev = mhi_chan->mhi_dev;
 446
 447		if (mhi_chan->tre_bytes_left - tr_len)
 448			buf_info.code = MHI_EV_CC_OVERFLOW;
 449
 450		dev_dbg(dev, "Reading %zd bytes from channel (%u)\n", tr_len, ring->ch_id);
 451		ret = mhi_cntrl->read_async(mhi_cntrl, &buf_info);
 452		if (ret < 0) {
 453			dev_err(&mhi_chan->mhi_dev->dev, "Error reading from channel\n");
 454			goto err_free_buf_addr;
 455		}
 456
 457		mhi_chan->tre_bytes_left -= tr_len;
 458
 459		if (!mhi_chan->tre_bytes_left)
 460			mhi_chan->rd_offset = (mhi_chan->rd_offset + 1) % ring->ring_size;
 461	/* Read until the some buffer is left or the ring becomes not empty */
 462	} while (!mhi_ep_queue_is_empty(mhi_chan->mhi_dev, DMA_TO_DEVICE));
 463
 464	return 0;
 465
 466err_free_buf_addr:
 467	kmem_cache_free(mhi_cntrl->tre_buf_cache, buf_addr);
 468
 469	return ret;
 470}
 471
 472static int mhi_ep_process_ch_ring(struct mhi_ep_ring *ring)
 473{
 474	struct mhi_ep_cntrl *mhi_cntrl = ring->mhi_cntrl;
 475	struct mhi_result result = {};
 476	struct mhi_ep_chan *mhi_chan;
 477	int ret;
 478
 479	mhi_chan = &mhi_cntrl->mhi_chan[ring->ch_id];
 480
 481	/*
 482	 * Bail out if transfer callback is not registered for the channel.
 483	 * This is most likely due to the client driver not loaded at this point.
 484	 */
 485	if (!mhi_chan->xfer_cb) {
 486		dev_err(&mhi_chan->mhi_dev->dev, "Client driver not available\n");
 487		return -ENODEV;
 488	}
 489
 490	if (ring->ch_id % 2) {
 491		/* DL channel */
 492		result.dir = mhi_chan->dir;
 493		mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
 494	} else {
 495		/* UL channel */
 496		ret = mhi_ep_read_channel(mhi_cntrl, ring);
 497		if (ret < 0) {
 498			dev_err(&mhi_chan->mhi_dev->dev, "Failed to read channel\n");
 499			return ret;
 500		}
 501	}
 502
 503	return 0;
 504}
 505
 506static void mhi_ep_skb_completion(struct mhi_ep_buf_info *buf_info)
 507{
 508	struct mhi_ep_device *mhi_dev = buf_info->mhi_dev;
 509	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
 510	struct mhi_ep_chan *mhi_chan = mhi_dev->dl_chan;
 511	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
 512	struct mhi_ring_element *el = &ring->ring_cache[ring->rd_offset];
 513	struct device *dev = &mhi_dev->dev;
 514	struct mhi_result result = {};
 515	int ret;
 516
 517	if (mhi_chan->xfer_cb) {
 518		result.buf_addr = buf_info->cb_buf;
 519		result.dir = mhi_chan->dir;
 520		result.bytes_xferd = buf_info->size;
 521
 522		mhi_chan->xfer_cb(mhi_dev, &result);
 523	}
 524
 525	ret = mhi_ep_send_completion_event(mhi_cntrl, ring, el, buf_info->size,
 526					   buf_info->code);
 527	if (ret) {
 528		dev_err(dev, "Error sending transfer completion event\n");
 529		return;
 530	}
 531
 532	mhi_ep_ring_inc_index(ring);
 533}
 534
 535/* TODO: Handle partially formed TDs */
 536int mhi_ep_queue_skb(struct mhi_ep_device *mhi_dev, struct sk_buff *skb)
 537{
 538	struct mhi_ep_cntrl *mhi_cntrl = mhi_dev->mhi_cntrl;
 539	struct mhi_ep_chan *mhi_chan = mhi_dev->dl_chan;
 540	struct device *dev = &mhi_chan->mhi_dev->dev;
 541	struct mhi_ep_buf_info buf_info = {};
 542	struct mhi_ring_element *el;
 543	u32 buf_left, read_offset;
 544	struct mhi_ep_ring *ring;
 545	size_t tr_len;
 546	u32 tre_len;
 547	int ret;
 548
 549	buf_left = skb->len;
 550	ring = &mhi_cntrl->mhi_chan[mhi_chan->chan].ring;
 551
 552	mutex_lock(&mhi_chan->lock);
 553
 554	do {
 555		/* Don't process the transfer ring if the channel is not in RUNNING state */
 556		if (mhi_chan->state != MHI_CH_STATE_RUNNING) {
 557			dev_err(dev, "Channel not available\n");
 558			ret = -ENODEV;
 559			goto err_exit;
 560		}
 561
 562		if (mhi_ep_queue_is_empty(mhi_dev, DMA_FROM_DEVICE)) {
 563			dev_err(dev, "TRE not available!\n");
 564			ret = -ENOSPC;
 565			goto err_exit;
 566		}
 567
 568		el = &ring->ring_cache[mhi_chan->rd_offset];
 569		tre_len = MHI_TRE_DATA_GET_LEN(el);
 570
 571		tr_len = min(buf_left, tre_len);
 572		read_offset = skb->len - buf_left;
 573
 574		buf_info.dev_addr = skb->data + read_offset;
 575		buf_info.host_addr = MHI_TRE_DATA_GET_PTR(el);
 576		buf_info.size = tr_len;
 577		buf_info.cb = mhi_ep_skb_completion;
 578		buf_info.cb_buf = skb;
 579		buf_info.mhi_dev = mhi_dev;
 580
 581		/*
 582		 * For all TREs queued by the host for DL channel, only the EOT flag will be set.
 583		 * If the packet doesn't fit into a single TRE, send the OVERFLOW event to
 584		 * the host so that the host can adjust the packet boundary to next TREs. Else send
 585		 * the EOT event to the host indicating the packet boundary.
 586		 */
 587		if (buf_left - tr_len)
 588			buf_info.code = MHI_EV_CC_OVERFLOW;
 589		else
 590			buf_info.code = MHI_EV_CC_EOT;
 591
 592		dev_dbg(dev, "Writing %zd bytes to channel (%u)\n", tr_len, ring->ch_id);
 593		ret = mhi_cntrl->write_async(mhi_cntrl, &buf_info);
 594		if (ret < 0) {
 595			dev_err(dev, "Error writing to the channel\n");
 596			goto err_exit;
 597		}
 598
 599		buf_left -= tr_len;
 600
 601		/*
 602		 * Update the read offset cached in mhi_chan. Actual read offset
 603		 * will be updated by the completion handler.
 604		 */
 605		mhi_chan->rd_offset = (mhi_chan->rd_offset + 1) % ring->ring_size;
 606	} while (buf_left);
 607
 608	mutex_unlock(&mhi_chan->lock);
 609
 610	return 0;
 611
 612err_exit:
 613	mutex_unlock(&mhi_chan->lock);
 614
 615	return ret;
 616}
 617EXPORT_SYMBOL_GPL(mhi_ep_queue_skb);
 618
 619static int mhi_ep_cache_host_cfg(struct mhi_ep_cntrl *mhi_cntrl)
 620{
 621	size_t cmd_ctx_host_size, ch_ctx_host_size, ev_ctx_host_size;
 622	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 623	int ret;
 624
 625	/* Update the number of event rings (NER) programmed by the host */
 626	mhi_ep_mmio_update_ner(mhi_cntrl);
 627
 628	dev_dbg(dev, "Number of Event rings: %u, HW Event rings: %u\n",
 629		 mhi_cntrl->event_rings, mhi_cntrl->hw_event_rings);
 630
 631	ch_ctx_host_size = sizeof(struct mhi_chan_ctxt) * mhi_cntrl->max_chan;
 632	ev_ctx_host_size = sizeof(struct mhi_event_ctxt) * mhi_cntrl->event_rings;
 633	cmd_ctx_host_size = sizeof(struct mhi_cmd_ctxt) * NR_OF_CMD_RINGS;
 634
 635	/* Get the channel context base pointer from host */
 636	mhi_ep_mmio_get_chc_base(mhi_cntrl);
 637
 638	/* Allocate and map memory for caching host channel context */
 639	ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa,
 640				   &mhi_cntrl->ch_ctx_cache_phys,
 641				   (void __iomem **) &mhi_cntrl->ch_ctx_cache,
 642				   ch_ctx_host_size);
 643	if (ret) {
 644		dev_err(dev, "Failed to allocate and map ch_ctx_cache\n");
 645		return ret;
 646	}
 647
 648	/* Get the event context base pointer from host */
 649	mhi_ep_mmio_get_erc_base(mhi_cntrl);
 650
 651	/* Allocate and map memory for caching host event context */
 652	ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa,
 653				   &mhi_cntrl->ev_ctx_cache_phys,
 654				   (void __iomem **) &mhi_cntrl->ev_ctx_cache,
 655				   ev_ctx_host_size);
 656	if (ret) {
 657		dev_err(dev, "Failed to allocate and map ev_ctx_cache\n");
 658		goto err_ch_ctx;
 659	}
 660
 661	/* Get the command context base pointer from host */
 662	mhi_ep_mmio_get_crc_base(mhi_cntrl);
 663
 664	/* Allocate and map memory for caching host command context */
 665	ret = mhi_cntrl->alloc_map(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa,
 666				   &mhi_cntrl->cmd_ctx_cache_phys,
 667				   (void __iomem **) &mhi_cntrl->cmd_ctx_cache,
 668				   cmd_ctx_host_size);
 669	if (ret) {
 670		dev_err(dev, "Failed to allocate and map cmd_ctx_cache\n");
 671		goto err_ev_ctx;
 672	}
 673
 674	/* Initialize command ring */
 675	ret = mhi_ep_ring_start(mhi_cntrl, &mhi_cntrl->mhi_cmd->ring,
 676				(union mhi_ep_ring_ctx *)mhi_cntrl->cmd_ctx_cache);
 677	if (ret) {
 678		dev_err(dev, "Failed to start the command ring\n");
 679		goto err_cmd_ctx;
 680	}
 681
 682	return ret;
 683
 684err_cmd_ctx:
 685	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa, mhi_cntrl->cmd_ctx_cache_phys,
 686			      (void __iomem *) mhi_cntrl->cmd_ctx_cache, cmd_ctx_host_size);
 687
 688err_ev_ctx:
 689	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa, mhi_cntrl->ev_ctx_cache_phys,
 690			      (void __iomem *) mhi_cntrl->ev_ctx_cache, ev_ctx_host_size);
 691
 692err_ch_ctx:
 693	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa, mhi_cntrl->ch_ctx_cache_phys,
 694			      (void __iomem *) mhi_cntrl->ch_ctx_cache, ch_ctx_host_size);
 695
 696	return ret;
 697}
 698
 699static void mhi_ep_free_host_cfg(struct mhi_ep_cntrl *mhi_cntrl)
 700{
 701	size_t cmd_ctx_host_size, ch_ctx_host_size, ev_ctx_host_size;
 702
 703	ch_ctx_host_size = sizeof(struct mhi_chan_ctxt) * mhi_cntrl->max_chan;
 704	ev_ctx_host_size = sizeof(struct mhi_event_ctxt) * mhi_cntrl->event_rings;
 705	cmd_ctx_host_size = sizeof(struct mhi_cmd_ctxt) * NR_OF_CMD_RINGS;
 706
 707	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->cmd_ctx_host_pa, mhi_cntrl->cmd_ctx_cache_phys,
 708			      (void __iomem *) mhi_cntrl->cmd_ctx_cache, cmd_ctx_host_size);
 709
 710	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ev_ctx_host_pa, mhi_cntrl->ev_ctx_cache_phys,
 711			      (void __iomem *) mhi_cntrl->ev_ctx_cache, ev_ctx_host_size);
 712
 713	mhi_cntrl->unmap_free(mhi_cntrl, mhi_cntrl->ch_ctx_host_pa, mhi_cntrl->ch_ctx_cache_phys,
 714			      (void __iomem *) mhi_cntrl->ch_ctx_cache, ch_ctx_host_size);
 715}
 716
 717static void mhi_ep_enable_int(struct mhi_ep_cntrl *mhi_cntrl)
 718{
 719	/*
 720	 * Doorbell interrupts are enabled when the corresponding channel gets started.
 721	 * Enabling all interrupts here triggers spurious irqs as some of the interrupts
 722	 * associated with hw channels always get triggered.
 723	 */
 724	mhi_ep_mmio_enable_ctrl_interrupt(mhi_cntrl);
 725	mhi_ep_mmio_enable_cmdb_interrupt(mhi_cntrl);
 726}
 727
 728static int mhi_ep_enable(struct mhi_ep_cntrl *mhi_cntrl)
 729{
 730	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 731	enum mhi_state state;
 732	bool mhi_reset;
 733	u32 count = 0;
 734	int ret;
 735
 736	/* Wait for Host to set the M0 state */
 737	do {
 738		msleep(M0_WAIT_DELAY_MS);
 739		mhi_ep_mmio_get_mhi_state(mhi_cntrl, &state, &mhi_reset);
 740		if (mhi_reset) {
 741			/* Clear the MHI reset if host is in reset state */
 742			mhi_ep_mmio_clear_reset(mhi_cntrl);
 743			dev_info(dev, "Detected Host reset while waiting for M0\n");
 744		}
 745		count++;
 746	} while (state != MHI_STATE_M0 && count < M0_WAIT_COUNT);
 747
 748	if (state != MHI_STATE_M0) {
 749		dev_err(dev, "Host failed to enter M0\n");
 750		return -ETIMEDOUT;
 751	}
 752
 753	ret = mhi_ep_cache_host_cfg(mhi_cntrl);
 754	if (ret) {
 755		dev_err(dev, "Failed to cache host config\n");
 756		return ret;
 757	}
 758
 759	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
 760
 761	/* Enable all interrupts now */
 762	mhi_ep_enable_int(mhi_cntrl);
 763
 764	return 0;
 765}
 766
 767static void mhi_ep_cmd_ring_worker(struct work_struct *work)
 768{
 769	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, cmd_ring_work);
 770	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
 771	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 772	struct mhi_ring_element *el;
 773	int ret;
 774
 775	/* Update the write offset for the ring */
 776	ret = mhi_ep_update_wr_offset(ring);
 777	if (ret) {
 778		dev_err(dev, "Error updating write offset for ring\n");
 779		return;
 780	}
 781
 782	/* Sanity check to make sure there are elements in the ring */
 783	if (ring->rd_offset == ring->wr_offset)
 784		return;
 785
 786	/*
 787	 * Process command ring element till write offset. In case of an error, just try to
 788	 * process next element.
 789	 */
 790	while (ring->rd_offset != ring->wr_offset) {
 791		el = &ring->ring_cache[ring->rd_offset];
 792
 793		ret = mhi_ep_process_cmd_ring(ring, el);
 794		if (ret && ret != -ENODEV)
 795			dev_err(dev, "Error processing cmd ring element: %zu\n", ring->rd_offset);
 796
 797		mhi_ep_ring_inc_index(ring);
 798	}
 799}
 800
 801static void mhi_ep_ch_ring_worker(struct work_struct *work)
 802{
 803	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, ch_ring_work);
 804	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 805	struct mhi_ep_ring_item *itr, *tmp;
 806	struct mhi_ep_ring *ring;
 807	struct mhi_ep_chan *chan;
 808	unsigned long flags;
 809	LIST_HEAD(head);
 810	int ret;
 811
 812	spin_lock_irqsave(&mhi_cntrl->list_lock, flags);
 813	list_splice_tail_init(&mhi_cntrl->ch_db_list, &head);
 814	spin_unlock_irqrestore(&mhi_cntrl->list_lock, flags);
 815
 816	/* Process each queued channel ring. In case of an error, just process next element. */
 817	list_for_each_entry_safe(itr, tmp, &head, node) {
 818		list_del(&itr->node);
 819		ring = itr->ring;
 820
 821		chan = &mhi_cntrl->mhi_chan[ring->ch_id];
 822		mutex_lock(&chan->lock);
 823
 824		/*
 825		 * The ring could've stopped while we waited to grab the (chan->lock), so do
 826		 * a sanity check before going further.
 827		 */
 828		if (!ring->started) {
 829			mutex_unlock(&chan->lock);
 830			kfree(itr);
 831			continue;
 832		}
 833
 834		/* Update the write offset for the ring */
 835		ret = mhi_ep_update_wr_offset(ring);
 836		if (ret) {
 837			dev_err(dev, "Error updating write offset for ring\n");
 838			mutex_unlock(&chan->lock);
 839			kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
 840			continue;
 841		}
 842
 843		/* Sanity check to make sure there are elements in the ring */
 844		if (chan->rd_offset == ring->wr_offset) {
 845			mutex_unlock(&chan->lock);
 846			kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
 847			continue;
 848		}
 849
 850		dev_dbg(dev, "Processing the ring for channel (%u)\n", ring->ch_id);
 851		ret = mhi_ep_process_ch_ring(ring);
 852		if (ret) {
 853			dev_err(dev, "Error processing ring for channel (%u): %d\n",
 854				ring->ch_id, ret);
 855			mutex_unlock(&chan->lock);
 856			kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
 857			continue;
 858		}
 859
 860		mutex_unlock(&chan->lock);
 861		kmem_cache_free(mhi_cntrl->ring_item_cache, itr);
 862	}
 863}
 864
 865static void mhi_ep_state_worker(struct work_struct *work)
 866{
 867	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, state_work);
 868	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 869	struct mhi_ep_state_transition *itr, *tmp;
 870	unsigned long flags;
 871	LIST_HEAD(head);
 872	int ret;
 873
 874	spin_lock_irqsave(&mhi_cntrl->list_lock, flags);
 875	list_splice_tail_init(&mhi_cntrl->st_transition_list, &head);
 876	spin_unlock_irqrestore(&mhi_cntrl->list_lock, flags);
 877
 878	list_for_each_entry_safe(itr, tmp, &head, node) {
 879		list_del(&itr->node);
 880		dev_dbg(dev, "Handling MHI state transition to %s\n",
 881			 mhi_state_str(itr->state));
 882
 883		switch (itr->state) {
 884		case MHI_STATE_M0:
 885			ret = mhi_ep_set_m0_state(mhi_cntrl);
 886			if (ret)
 887				dev_err(dev, "Failed to transition to M0 state\n");
 888			break;
 889		case MHI_STATE_M3:
 890			ret = mhi_ep_set_m3_state(mhi_cntrl);
 891			if (ret)
 892				dev_err(dev, "Failed to transition to M3 state\n");
 893			break;
 894		default:
 895			dev_err(dev, "Invalid MHI state transition: %d\n", itr->state);
 896			break;
 897		}
 898		kfree(itr);
 899	}
 900}
 901
 902static void mhi_ep_queue_channel_db(struct mhi_ep_cntrl *mhi_cntrl, unsigned long ch_int,
 903				    u32 ch_idx)
 904{
 905	struct mhi_ep_ring_item *item;
 906	struct mhi_ep_ring *ring;
 907	bool work = !!ch_int;
 908	LIST_HEAD(head);
 909	u32 i;
 910
 911	/* First add the ring items to a local list */
 912	for_each_set_bit(i, &ch_int, 32) {
 913		/* Channel index varies for each register: 0, 32, 64, 96 */
 914		u32 ch_id = ch_idx + i;
 915
 916		ring = &mhi_cntrl->mhi_chan[ch_id].ring;
 917		item = kmem_cache_zalloc(mhi_cntrl->ring_item_cache, GFP_ATOMIC);
 918		if (!item)
 919			return;
 920
 921		item->ring = ring;
 922		list_add_tail(&item->node, &head);
 923	}
 924
 925	/* Now, splice the local list into ch_db_list and queue the work item */
 926	if (work) {
 927		spin_lock(&mhi_cntrl->list_lock);
 928		list_splice_tail_init(&head, &mhi_cntrl->ch_db_list);
 929		spin_unlock(&mhi_cntrl->list_lock);
 930
 931		queue_work(mhi_cntrl->wq, &mhi_cntrl->ch_ring_work);
 932	}
 933}
 934
 935/*
 936 * Channel interrupt statuses are contained in 4 registers each of 32bit length.
 937 * For checking all interrupts, we need to loop through each registers and then
 938 * check for bits set.
 939 */
 940static void mhi_ep_check_channel_interrupt(struct mhi_ep_cntrl *mhi_cntrl)
 941{
 942	u32 ch_int, ch_idx, i;
 943
 944	/* Bail out if there is no channel doorbell interrupt */
 945	if (!mhi_ep_mmio_read_chdb_status_interrupts(mhi_cntrl))
 946		return;
 947
 948	for (i = 0; i < MHI_MASK_ROWS_CH_DB; i++) {
 949		ch_idx = i * MHI_MASK_CH_LEN;
 950
 951		/* Only process channel interrupt if the mask is enabled */
 952		ch_int = mhi_cntrl->chdb[i].status & mhi_cntrl->chdb[i].mask;
 953		if (ch_int) {
 954			mhi_ep_queue_channel_db(mhi_cntrl, ch_int, ch_idx);
 955			mhi_ep_mmio_write(mhi_cntrl, MHI_CHDB_INT_CLEAR_n(i),
 956							mhi_cntrl->chdb[i].status);
 957		}
 958	}
 959}
 960
 961static void mhi_ep_process_ctrl_interrupt(struct mhi_ep_cntrl *mhi_cntrl,
 962					 enum mhi_state state)
 963{
 964	struct mhi_ep_state_transition *item;
 965
 966	item = kzalloc(sizeof(*item), GFP_ATOMIC);
 967	if (!item)
 968		return;
 969
 970	item->state = state;
 971	spin_lock(&mhi_cntrl->list_lock);
 972	list_add_tail(&item->node, &mhi_cntrl->st_transition_list);
 973	spin_unlock(&mhi_cntrl->list_lock);
 974
 975	queue_work(mhi_cntrl->wq, &mhi_cntrl->state_work);
 976}
 977
 978/*
 979 * Interrupt handler that services interrupts raised by the host writing to
 980 * MHICTRL and Command ring doorbell (CRDB) registers for state change and
 981 * channel interrupts.
 982 */
 983static irqreturn_t mhi_ep_irq(int irq, void *data)
 984{
 985	struct mhi_ep_cntrl *mhi_cntrl = data;
 986	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 987	enum mhi_state state;
 988	u32 int_value;
 989	bool mhi_reset;
 990
 991	/* Acknowledge the ctrl interrupt */
 992	int_value = mhi_ep_mmio_read(mhi_cntrl, MHI_CTRL_INT_STATUS);
 993	mhi_ep_mmio_write(mhi_cntrl, MHI_CTRL_INT_CLEAR, int_value);
 994
 995	/* Check for ctrl interrupt */
 996	if (FIELD_GET(MHI_CTRL_INT_STATUS_MSK, int_value)) {
 997		dev_dbg(dev, "Processing ctrl interrupt\n");
 998		mhi_ep_mmio_get_mhi_state(mhi_cntrl, &state, &mhi_reset);
 999		if (mhi_reset) {
1000			dev_info(dev, "Host triggered MHI reset!\n");
1001			disable_irq_nosync(mhi_cntrl->irq);
1002			schedule_work(&mhi_cntrl->reset_work);
1003			return IRQ_HANDLED;
1004		}
1005
1006		mhi_ep_process_ctrl_interrupt(mhi_cntrl, state);
1007	}
1008
1009	/* Check for command doorbell interrupt */
1010	if (FIELD_GET(MHI_CTRL_INT_STATUS_CRDB_MSK, int_value)) {
1011		dev_dbg(dev, "Processing command doorbell interrupt\n");
1012		queue_work(mhi_cntrl->wq, &mhi_cntrl->cmd_ring_work);
1013	}
1014
1015	/* Check for channel interrupts */
1016	mhi_ep_check_channel_interrupt(mhi_cntrl);
1017
1018	return IRQ_HANDLED;
1019}
1020
1021static void mhi_ep_abort_transfer(struct mhi_ep_cntrl *mhi_cntrl)
1022{
1023	struct mhi_ep_ring *ch_ring, *ev_ring;
1024	struct mhi_result result = {};
1025	struct mhi_ep_chan *mhi_chan;
1026	int i;
1027
1028	/* Stop all the channels */
1029	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1030		mhi_chan = &mhi_cntrl->mhi_chan[i];
1031		if (!mhi_chan->ring.started)
1032			continue;
1033
1034		mutex_lock(&mhi_chan->lock);
1035		/* Send channel disconnect status to client drivers */
1036		if (mhi_chan->xfer_cb) {
1037			result.transaction_status = -ENOTCONN;
1038			result.bytes_xferd = 0;
1039			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1040		}
1041
1042		mhi_chan->state = MHI_CH_STATE_DISABLED;
1043		mutex_unlock(&mhi_chan->lock);
1044	}
1045
1046	flush_workqueue(mhi_cntrl->wq);
1047
1048	/* Destroy devices associated with all channels */
1049	device_for_each_child(&mhi_cntrl->mhi_dev->dev, NULL, mhi_ep_destroy_device);
1050
1051	/* Stop and reset the transfer rings */
1052	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1053		mhi_chan = &mhi_cntrl->mhi_chan[i];
1054		if (!mhi_chan->ring.started)
1055			continue;
1056
1057		ch_ring = &mhi_cntrl->mhi_chan[i].ring;
1058		mutex_lock(&mhi_chan->lock);
1059		mhi_ep_ring_reset(mhi_cntrl, ch_ring);
1060		mutex_unlock(&mhi_chan->lock);
1061	}
1062
1063	/* Stop and reset the event rings */
1064	for (i = 0; i < mhi_cntrl->event_rings; i++) {
1065		ev_ring = &mhi_cntrl->mhi_event[i].ring;
1066		if (!ev_ring->started)
1067			continue;
1068
1069		mutex_lock(&mhi_cntrl->event_lock);
1070		mhi_ep_ring_reset(mhi_cntrl, ev_ring);
1071		mutex_unlock(&mhi_cntrl->event_lock);
1072	}
1073
1074	/* Stop and reset the command ring */
1075	mhi_ep_ring_reset(mhi_cntrl, &mhi_cntrl->mhi_cmd->ring);
1076
1077	mhi_ep_free_host_cfg(mhi_cntrl);
1078	mhi_ep_mmio_mask_interrupts(mhi_cntrl);
1079
1080	mhi_cntrl->enabled = false;
1081}
1082
1083static void mhi_ep_reset_worker(struct work_struct *work)
1084{
1085	struct mhi_ep_cntrl *mhi_cntrl = container_of(work, struct mhi_ep_cntrl, reset_work);
1086	enum mhi_state cur_state;
1087
1088	mhi_ep_power_down(mhi_cntrl);
1089
1090	mutex_lock(&mhi_cntrl->state_lock);
1091
1092	/* Reset MMIO to signal host that the MHI_RESET is completed in endpoint */
1093	mhi_ep_mmio_reset(mhi_cntrl);
1094	cur_state = mhi_cntrl->mhi_state;
1095
1096	/*
1097	 * Only proceed further if the reset is due to SYS_ERR. The host will
1098	 * issue reset during shutdown also and we don't need to do re-init in
1099	 * that case.
1100	 */
1101	if (cur_state == MHI_STATE_SYS_ERR)
1102		mhi_ep_power_up(mhi_cntrl);
1103
1104	mutex_unlock(&mhi_cntrl->state_lock);
1105}
1106
1107/*
1108 * We don't need to do anything special other than setting the MHI SYS_ERR
1109 * state. The host will reset all contexts and issue MHI RESET so that we
1110 * could also recover from error state.
1111 */
1112void mhi_ep_handle_syserr(struct mhi_ep_cntrl *mhi_cntrl)
1113{
1114	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1115	int ret;
1116
1117	ret = mhi_ep_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR);
1118	if (ret)
1119		return;
1120
1121	/* Signal host that the device went to SYS_ERR state */
1122	ret = mhi_ep_send_state_change_event(mhi_cntrl, MHI_STATE_SYS_ERR);
1123	if (ret)
1124		dev_err(dev, "Failed sending SYS_ERR state change event: %d\n", ret);
1125}
1126
1127int mhi_ep_power_up(struct mhi_ep_cntrl *mhi_cntrl)
1128{
1129	struct device *dev = &mhi_cntrl->mhi_dev->dev;
1130	int ret, i;
1131
1132	/*
1133	 * Mask all interrupts until the state machine is ready. Interrupts will
1134	 * be enabled later with mhi_ep_enable().
1135	 */
1136	mhi_ep_mmio_mask_interrupts(mhi_cntrl);
1137	mhi_ep_mmio_init(mhi_cntrl);
1138
1139	mhi_cntrl->mhi_event = kcalloc(mhi_cntrl->event_rings,
1140				       sizeof(*mhi_cntrl->mhi_event),
1141				       GFP_KERNEL);
1142	if (!mhi_cntrl->mhi_event)
1143		return -ENOMEM;
1144
1145	/* Initialize command, channel and event rings */
1146	mhi_ep_ring_init(&mhi_cntrl->mhi_cmd->ring, RING_TYPE_CMD, 0);
1147	for (i = 0; i < mhi_cntrl->max_chan; i++)
1148		mhi_ep_ring_init(&mhi_cntrl->mhi_chan[i].ring, RING_TYPE_CH, i);
1149	for (i = 0; i < mhi_cntrl->event_rings; i++)
1150		mhi_ep_ring_init(&mhi_cntrl->mhi_event[i].ring, RING_TYPE_ER, i);
1151
1152	mhi_cntrl->mhi_state = MHI_STATE_RESET;
1153
1154	/* Set AMSS EE before signaling ready state */
1155	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
1156
1157	/* All set, notify the host that we are ready */
1158	ret = mhi_ep_set_ready_state(mhi_cntrl);
1159	if (ret)
1160		goto err_free_event;
1161
1162	dev_dbg(dev, "READY state notification sent to the host\n");
1163
1164	ret = mhi_ep_enable(mhi_cntrl);
1165	if (ret) {
1166		dev_err(dev, "Failed to enable MHI endpoint\n");
1167		goto err_free_event;
1168	}
1169
1170	enable_irq(mhi_cntrl->irq);
1171	mhi_cntrl->enabled = true;
1172
1173	return 0;
1174
1175err_free_event:
1176	kfree(mhi_cntrl->mhi_event);
1177
1178	return ret;
1179}
1180EXPORT_SYMBOL_GPL(mhi_ep_power_up);
1181
1182void mhi_ep_power_down(struct mhi_ep_cntrl *mhi_cntrl)
1183{
1184	if (mhi_cntrl->enabled) {
1185		mhi_ep_abort_transfer(mhi_cntrl);
1186		kfree(mhi_cntrl->mhi_event);
1187		disable_irq(mhi_cntrl->irq);
1188	}
1189}
1190EXPORT_SYMBOL_GPL(mhi_ep_power_down);
1191
1192void mhi_ep_suspend_channels(struct mhi_ep_cntrl *mhi_cntrl)
1193{
1194	struct mhi_ep_chan *mhi_chan;
1195	u32 tmp;
1196	int i;
1197
1198	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1199		mhi_chan = &mhi_cntrl->mhi_chan[i];
1200
1201		if (!mhi_chan->mhi_dev)
1202			continue;
1203
1204		mutex_lock(&mhi_chan->lock);
1205		/* Skip if the channel is not currently running */
1206		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[i].chcfg);
1207		if (FIELD_GET(CHAN_CTX_CHSTATE_MASK, tmp) != MHI_CH_STATE_RUNNING) {
1208			mutex_unlock(&mhi_chan->lock);
1209			continue;
1210		}
1211
1212		dev_dbg(&mhi_chan->mhi_dev->dev, "Suspending channel\n");
1213		/* Set channel state to SUSPENDED */
1214		mhi_chan->state = MHI_CH_STATE_SUSPENDED;
1215		tmp &= ~CHAN_CTX_CHSTATE_MASK;
1216		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_SUSPENDED);
1217		mhi_cntrl->ch_ctx_cache[i].chcfg = cpu_to_le32(tmp);
1218		mutex_unlock(&mhi_chan->lock);
1219	}
1220}
1221
1222void mhi_ep_resume_channels(struct mhi_ep_cntrl *mhi_cntrl)
1223{
1224	struct mhi_ep_chan *mhi_chan;
1225	u32 tmp;
1226	int i;
1227
1228	for (i = 0; i < mhi_cntrl->max_chan; i++) {
1229		mhi_chan = &mhi_cntrl->mhi_chan[i];
1230
1231		if (!mhi_chan->mhi_dev)
1232			continue;
1233
1234		mutex_lock(&mhi_chan->lock);
1235		/* Skip if the channel is not currently suspended */
1236		tmp = le32_to_cpu(mhi_cntrl->ch_ctx_cache[i].chcfg);
1237		if (FIELD_GET(CHAN_CTX_CHSTATE_MASK, tmp) != MHI_CH_STATE_SUSPENDED) {
1238			mutex_unlock(&mhi_chan->lock);
1239			continue;
1240		}
1241
1242		dev_dbg(&mhi_chan->mhi_dev->dev, "Resuming channel\n");
1243		/* Set channel state to RUNNING */
1244		mhi_chan->state = MHI_CH_STATE_RUNNING;
1245		tmp &= ~CHAN_CTX_CHSTATE_MASK;
1246		tmp |= FIELD_PREP(CHAN_CTX_CHSTATE_MASK, MHI_CH_STATE_RUNNING);
1247		mhi_cntrl->ch_ctx_cache[i].chcfg = cpu_to_le32(tmp);
1248		mutex_unlock(&mhi_chan->lock);
1249	}
1250}
1251
1252static void mhi_ep_release_device(struct device *dev)
1253{
1254	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1255
1256	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1257		mhi_dev->mhi_cntrl->mhi_dev = NULL;
1258
1259	/*
1260	 * We need to set the mhi_chan->mhi_dev to NULL here since the MHI
1261	 * devices for the channels will only get created in mhi_ep_create_device()
1262	 * if the mhi_dev associated with it is NULL.
1263	 */
1264	if (mhi_dev->ul_chan)
1265		mhi_dev->ul_chan->mhi_dev = NULL;
1266
1267	if (mhi_dev->dl_chan)
1268		mhi_dev->dl_chan->mhi_dev = NULL;
1269
1270	kfree(mhi_dev);
1271}
1272
1273static struct mhi_ep_device *mhi_ep_alloc_device(struct mhi_ep_cntrl *mhi_cntrl,
1274						 enum mhi_device_type dev_type)
1275{
1276	struct mhi_ep_device *mhi_dev;
1277	struct device *dev;
1278
1279	mhi_dev = kzalloc(sizeof(*mhi_dev), GFP_KERNEL);
1280	if (!mhi_dev)
1281		return ERR_PTR(-ENOMEM);
1282
1283	dev = &mhi_dev->dev;
1284	device_initialize(dev);
1285	dev->bus = &mhi_ep_bus_type;
1286	dev->release = mhi_ep_release_device;
1287
1288	/* Controller device is always allocated first */
1289	if (dev_type == MHI_DEVICE_CONTROLLER)
1290		/* for MHI controller device, parent is the bus device (e.g. PCI EPF) */
1291		dev->parent = mhi_cntrl->cntrl_dev;
1292	else
1293		/* for MHI client devices, parent is the MHI controller device */
1294		dev->parent = &mhi_cntrl->mhi_dev->dev;
1295
1296	mhi_dev->mhi_cntrl = mhi_cntrl;
1297	mhi_dev->dev_type = dev_type;
1298
1299	return mhi_dev;
1300}
1301
1302/*
1303 * MHI channels are always defined in pairs with UL as the even numbered
1304 * channel and DL as odd numbered one. This function gets UL channel (primary)
1305 * as the ch_id and always looks after the next entry in channel list for
1306 * the corresponding DL channel (secondary).
1307 */
1308static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id)
1309{
1310	struct mhi_ep_chan *mhi_chan = &mhi_cntrl->mhi_chan[ch_id];
1311	struct device *dev = mhi_cntrl->cntrl_dev;
1312	struct mhi_ep_device *mhi_dev;
1313	int ret;
1314
1315	/* Check if the channel name is same for both UL and DL */
1316	if (strcmp(mhi_chan->name, mhi_chan[1].name)) {
1317		dev_err(dev, "UL and DL channel names are not same: (%s) != (%s)\n",
1318			mhi_chan->name, mhi_chan[1].name);
1319		return -EINVAL;
1320	}
1321
1322	mhi_dev = mhi_ep_alloc_device(mhi_cntrl, MHI_DEVICE_XFER);
1323	if (IS_ERR(mhi_dev))
1324		return PTR_ERR(mhi_dev);
1325
1326	/* Configure primary channel */
1327	mhi_dev->ul_chan = mhi_chan;
1328	get_device(&mhi_dev->dev);
1329	mhi_chan->mhi_dev = mhi_dev;
1330
1331	/* Configure secondary channel as well */
1332	mhi_chan++;
1333	mhi_dev->dl_chan = mhi_chan;
1334	get_device(&mhi_dev->dev);
1335	mhi_chan->mhi_dev = mhi_dev;
1336
1337	/* Channel name is same for both UL and DL */
1338	mhi_dev->name = mhi_chan->name;
1339	ret = dev_set_name(&mhi_dev->dev, "%s_%s",
1340		     dev_name(&mhi_cntrl->mhi_dev->dev),
1341		     mhi_dev->name);
1342	if (ret) {
1343		put_device(&mhi_dev->dev);
1344		return ret;
1345	}
1346
1347	ret = device_add(&mhi_dev->dev);
1348	if (ret)
1349		put_device(&mhi_dev->dev);
1350
1351	return ret;
1352}
1353
1354static int mhi_ep_destroy_device(struct device *dev, void *data)
1355{
1356	struct mhi_ep_device *mhi_dev;
1357	struct mhi_ep_cntrl *mhi_cntrl;
1358	struct mhi_ep_chan *ul_chan, *dl_chan;
1359
1360	if (dev->bus != &mhi_ep_bus_type)
1361		return 0;
1362
1363	mhi_dev = to_mhi_ep_device(dev);
1364	mhi_cntrl = mhi_dev->mhi_cntrl;
1365
1366	/* Only destroy devices created for channels */
1367	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1368		return 0;
1369
1370	ul_chan = mhi_dev->ul_chan;
1371	dl_chan = mhi_dev->dl_chan;
1372
1373	if (ul_chan)
1374		put_device(&ul_chan->mhi_dev->dev);
1375
1376	if (dl_chan)
1377		put_device(&dl_chan->mhi_dev->dev);
1378
1379	dev_dbg(&mhi_cntrl->mhi_dev->dev, "Destroying device for chan:%s\n",
1380		 mhi_dev->name);
1381
1382	/* Notify the client and remove the device from MHI bus */
1383	device_del(dev);
1384	put_device(dev);
1385
1386	return 0;
1387}
1388
1389static int mhi_ep_chan_init(struct mhi_ep_cntrl *mhi_cntrl,
1390			    const struct mhi_ep_cntrl_config *config)
1391{
1392	const struct mhi_ep_channel_config *ch_cfg;
1393	struct device *dev = mhi_cntrl->cntrl_dev;
1394	u32 chan, i;
1395	int ret = -EINVAL;
1396
1397	mhi_cntrl->max_chan = config->max_channels;
1398
1399	/*
1400	 * Allocate max_channels supported by the MHI endpoint and populate
1401	 * only the defined channels
1402	 */
1403	mhi_cntrl->mhi_chan = kcalloc(mhi_cntrl->max_chan, sizeof(*mhi_cntrl->mhi_chan),
1404				      GFP_KERNEL);
1405	if (!mhi_cntrl->mhi_chan)
1406		return -ENOMEM;
1407
1408	for (i = 0; i < config->num_channels; i++) {
1409		struct mhi_ep_chan *mhi_chan;
1410
1411		ch_cfg = &config->ch_cfg[i];
1412
1413		chan = ch_cfg->num;
1414		if (chan >= mhi_cntrl->max_chan) {
1415			dev_err(dev, "Channel (%u) exceeds maximum available channels (%u)\n",
1416				chan, mhi_cntrl->max_chan);
1417			goto error_chan_cfg;
1418		}
1419
1420		/* Bi-directional and direction less channels are not supported */
1421		if (ch_cfg->dir == DMA_BIDIRECTIONAL || ch_cfg->dir == DMA_NONE) {
1422			dev_err(dev, "Invalid direction (%u) for channel (%u)\n",
1423				ch_cfg->dir, chan);
1424			goto error_chan_cfg;
1425		}
1426
1427		mhi_chan = &mhi_cntrl->mhi_chan[chan];
1428		mhi_chan->name = ch_cfg->name;
1429		mhi_chan->chan = chan;
1430		mhi_chan->dir = ch_cfg->dir;
1431		mutex_init(&mhi_chan->lock);
1432	}
1433
1434	return 0;
1435
1436error_chan_cfg:
1437	kfree(mhi_cntrl->mhi_chan);
1438
1439	return ret;
1440}
1441
1442/*
1443 * Allocate channel and command rings here. Event rings will be allocated
1444 * in mhi_ep_power_up() as the config comes from the host.
1445 */
1446int mhi_ep_register_controller(struct mhi_ep_cntrl *mhi_cntrl,
1447				const struct mhi_ep_cntrl_config *config)
1448{
1449	struct mhi_ep_device *mhi_dev;
1450	int ret;
1451
1452	if (!mhi_cntrl || !mhi_cntrl->cntrl_dev || !mhi_cntrl->mmio || !mhi_cntrl->irq)
1453		return -EINVAL;
1454
1455	if (!mhi_cntrl->read_sync || !mhi_cntrl->write_sync ||
1456	    !mhi_cntrl->read_async || !mhi_cntrl->write_async)
1457		return -EINVAL;
1458
1459	ret = mhi_ep_chan_init(mhi_cntrl, config);
1460	if (ret)
1461		return ret;
1462
1463	mhi_cntrl->mhi_cmd = kcalloc(NR_OF_CMD_RINGS, sizeof(*mhi_cntrl->mhi_cmd), GFP_KERNEL);
1464	if (!mhi_cntrl->mhi_cmd) {
1465		ret = -ENOMEM;
1466		goto err_free_ch;
1467	}
1468
1469	mhi_cntrl->ev_ring_el_cache = kmem_cache_create("mhi_ep_event_ring_el",
1470							sizeof(struct mhi_ring_element), 0,
1471							0, NULL);
1472	if (!mhi_cntrl->ev_ring_el_cache) {
1473		ret = -ENOMEM;
1474		goto err_free_cmd;
1475	}
1476
1477	mhi_cntrl->tre_buf_cache = kmem_cache_create("mhi_ep_tre_buf", MHI_EP_DEFAULT_MTU, 0,
1478						      0, NULL);
1479	if (!mhi_cntrl->tre_buf_cache) {
1480		ret = -ENOMEM;
1481		goto err_destroy_ev_ring_el_cache;
1482	}
1483
1484	mhi_cntrl->ring_item_cache = kmem_cache_create("mhi_ep_ring_item",
1485							sizeof(struct mhi_ep_ring_item), 0,
1486							0, NULL);
1487	if (!mhi_cntrl->ring_item_cache) {
1488		ret = -ENOMEM;
1489		goto err_destroy_tre_buf_cache;
1490	}
1491
1492	INIT_WORK(&mhi_cntrl->state_work, mhi_ep_state_worker);
1493	INIT_WORK(&mhi_cntrl->reset_work, mhi_ep_reset_worker);
1494	INIT_WORK(&mhi_cntrl->cmd_ring_work, mhi_ep_cmd_ring_worker);
1495	INIT_WORK(&mhi_cntrl->ch_ring_work, mhi_ep_ch_ring_worker);
1496
1497	mhi_cntrl->wq = alloc_workqueue("mhi_ep_wq", WQ_PERCPU, 0);
1498	if (!mhi_cntrl->wq) {
1499		ret = -ENOMEM;
1500		goto err_destroy_ring_item_cache;
1501	}
1502
1503	INIT_LIST_HEAD(&mhi_cntrl->st_transition_list);
1504	INIT_LIST_HEAD(&mhi_cntrl->ch_db_list);
1505	spin_lock_init(&mhi_cntrl->list_lock);
1506	mutex_init(&mhi_cntrl->state_lock);
1507	mutex_init(&mhi_cntrl->event_lock);
1508
1509	/* Set MHI version and AMSS EE before enumeration */
1510	mhi_ep_mmio_write(mhi_cntrl, EP_MHIVER, config->mhi_version);
1511	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
1512
1513	/* Set controller index */
1514	ret = ida_alloc(&mhi_ep_cntrl_ida, GFP_KERNEL);
1515	if (ret < 0)
1516		goto err_destroy_wq;
1517
1518	mhi_cntrl->index = ret;
1519
1520	irq_set_status_flags(mhi_cntrl->irq, IRQ_NOAUTOEN);
1521	ret = request_irq(mhi_cntrl->irq, mhi_ep_irq, IRQF_TRIGGER_HIGH,
1522			  "doorbell_irq", mhi_cntrl);
1523	if (ret) {
1524		dev_err(mhi_cntrl->cntrl_dev, "Failed to request Doorbell IRQ\n");
1525		goto err_ida_free;
1526	}
1527
1528	/* Allocate the controller device */
1529	mhi_dev = mhi_ep_alloc_device(mhi_cntrl, MHI_DEVICE_CONTROLLER);
1530	if (IS_ERR(mhi_dev)) {
1531		dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate controller device\n");
1532		ret = PTR_ERR(mhi_dev);
1533		goto err_free_irq;
1534	}
1535
1536	ret = dev_set_name(&mhi_dev->dev, "mhi_ep%u", mhi_cntrl->index);
1537	if (ret)
1538		goto err_put_dev;
1539
1540	mhi_dev->name = dev_name(&mhi_dev->dev);
1541	mhi_cntrl->mhi_dev = mhi_dev;
1542
1543	ret = device_add(&mhi_dev->dev);
1544	if (ret)
1545		goto err_put_dev;
1546
1547	dev_dbg(&mhi_dev->dev, "MHI EP Controller registered\n");
1548
1549	return 0;
1550
1551err_put_dev:
1552	put_device(&mhi_dev->dev);
1553err_free_irq:
1554	free_irq(mhi_cntrl->irq, mhi_cntrl);
1555err_ida_free:
1556	ida_free(&mhi_ep_cntrl_ida, mhi_cntrl->index);
1557err_destroy_wq:
1558	destroy_workqueue(mhi_cntrl->wq);
1559err_destroy_ring_item_cache:
1560	kmem_cache_destroy(mhi_cntrl->ring_item_cache);
1561err_destroy_ev_ring_el_cache:
1562	kmem_cache_destroy(mhi_cntrl->ev_ring_el_cache);
1563err_destroy_tre_buf_cache:
1564	kmem_cache_destroy(mhi_cntrl->tre_buf_cache);
1565err_free_cmd:
1566	kfree(mhi_cntrl->mhi_cmd);
1567err_free_ch:
1568	kfree(mhi_cntrl->mhi_chan);
1569
1570	return ret;
1571}
1572EXPORT_SYMBOL_GPL(mhi_ep_register_controller);
1573
1574/*
1575 * It is expected that the controller drivers will power down the MHI EP stack
1576 * using "mhi_ep_power_down()" before calling this function to unregister themselves.
1577 */
1578void mhi_ep_unregister_controller(struct mhi_ep_cntrl *mhi_cntrl)
1579{
1580	struct mhi_ep_device *mhi_dev = mhi_cntrl->mhi_dev;
1581
1582	destroy_workqueue(mhi_cntrl->wq);
1583
1584	free_irq(mhi_cntrl->irq, mhi_cntrl);
1585
1586	kmem_cache_destroy(mhi_cntrl->tre_buf_cache);
1587	kmem_cache_destroy(mhi_cntrl->ev_ring_el_cache);
1588	kmem_cache_destroy(mhi_cntrl->ring_item_cache);
1589	kfree(mhi_cntrl->mhi_cmd);
1590	kfree(mhi_cntrl->mhi_chan);
1591
1592	device_del(&mhi_dev->dev);
1593	put_device(&mhi_dev->dev);
1594
1595	ida_free(&mhi_ep_cntrl_ida, mhi_cntrl->index);
1596}
1597EXPORT_SYMBOL_GPL(mhi_ep_unregister_controller);
1598
1599static int mhi_ep_driver_probe(struct device *dev)
1600{
1601	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1602	struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(dev->driver);
1603	struct mhi_ep_chan *ul_chan = mhi_dev->ul_chan;
1604	struct mhi_ep_chan *dl_chan = mhi_dev->dl_chan;
1605
1606	ul_chan->xfer_cb = mhi_drv->ul_xfer_cb;
1607	dl_chan->xfer_cb = mhi_drv->dl_xfer_cb;
1608
1609	return mhi_drv->probe(mhi_dev, mhi_dev->id);
1610}
1611
1612static int mhi_ep_driver_remove(struct device *dev)
1613{
1614	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1615	struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(dev->driver);
1616	struct mhi_result result = {};
1617	struct mhi_ep_chan *mhi_chan;
1618	int dir;
1619
1620	/* Skip if it is a controller device */
1621	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1622		return 0;
1623
1624	/* Disconnect the channels associated with the driver */
1625	for (dir = 0; dir < 2; dir++) {
1626		mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
1627
1628		if (!mhi_chan)
1629			continue;
1630
1631		mutex_lock(&mhi_chan->lock);
1632		/* Send channel disconnect status to the client driver */
1633		if (mhi_chan->xfer_cb) {
1634			result.transaction_status = -ENOTCONN;
1635			result.bytes_xferd = 0;
1636			mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
1637		}
1638
1639		mhi_chan->state = MHI_CH_STATE_DISABLED;
1640		mhi_chan->xfer_cb = NULL;
1641		mutex_unlock(&mhi_chan->lock);
1642	}
1643
1644	/* Remove the client driver now */
1645	mhi_drv->remove(mhi_dev);
1646
1647	return 0;
1648}
1649
1650int __mhi_ep_driver_register(struct mhi_ep_driver *mhi_drv, struct module *owner)
1651{
1652	struct device_driver *driver = &mhi_drv->driver;
1653
1654	if (!mhi_drv->probe || !mhi_drv->remove)
1655		return -EINVAL;
1656
1657	/* Client drivers should have callbacks defined for both channels */
1658	if (!mhi_drv->ul_xfer_cb || !mhi_drv->dl_xfer_cb)
1659		return -EINVAL;
1660
1661	driver->bus = &mhi_ep_bus_type;
1662	driver->owner = owner;
1663	driver->probe = mhi_ep_driver_probe;
1664	driver->remove = mhi_ep_driver_remove;
1665
1666	return driver_register(driver);
1667}
1668EXPORT_SYMBOL_GPL(__mhi_ep_driver_register);
1669
1670void mhi_ep_driver_unregister(struct mhi_ep_driver *mhi_drv)
1671{
1672	driver_unregister(&mhi_drv->driver);
1673}
1674EXPORT_SYMBOL_GPL(mhi_ep_driver_unregister);
1675
1676static int mhi_ep_uevent(const struct device *dev, struct kobj_uevent_env *env)
1677{
1678	const struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1679
1680	return add_uevent_var(env, "MODALIAS=" MHI_EP_DEVICE_MODALIAS_FMT,
1681					mhi_dev->name);
1682}
1683
1684static int mhi_ep_match(struct device *dev, const struct device_driver *drv)
1685{
1686	struct mhi_ep_device *mhi_dev = to_mhi_ep_device(dev);
1687	const struct mhi_ep_driver *mhi_drv = to_mhi_ep_driver(drv);
1688	const struct mhi_device_id *id;
1689
1690	/*
1691	 * If the device is a controller type then there is no client driver
1692	 * associated with it
1693	 */
1694	if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER)
1695		return 0;
1696
1697	for (id = mhi_drv->id_table; id->chan[0]; id++)
1698		if (!strcmp(mhi_dev->name, id->chan)) {
1699			mhi_dev->id = id;
1700			return 1;
1701		}
1702
1703	return 0;
1704};
1705
1706const struct bus_type mhi_ep_bus_type = {
1707	.name = "mhi_ep",
1708	.dev_name = "mhi_ep",
1709	.match = mhi_ep_match,
1710	.uevent = mhi_ep_uevent,
1711};
1712
1713static int __init mhi_ep_init(void)
1714{
1715	return bus_register(&mhi_ep_bus_type);
1716}
1717
1718static void __exit mhi_ep_exit(void)
1719{
1720	bus_unregister(&mhi_ep_bus_type);
1721}
1722
1723postcore_initcall(mhi_ep_init);
1724module_exit(mhi_ep_exit);
1725
1726MODULE_LICENSE("GPL v2");
1727MODULE_DESCRIPTION("MHI Bus Endpoint stack");
1728MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");