i2c-imx-lpi2c.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * This is i.MX low power i2c controller driver.
   4 *
   5 * Copyright 2016 Freescale Semiconductor, Inc.
   6 */
   7
   8#include <linux/bitfield.h>
   9#include <linux/clk.h>
  10#include <linux/completion.h>
  11#include <linux/delay.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/dmaengine.h>
  14#include <linux/err.h>
  15#include <linux/errno.h>
  16#include <linux/i2c.h>
  17#include <linux/init.h>
  18#include <linux/interrupt.h>
  19#include <linux/io.h>
  20#include <linux/iopoll.h>
  21#include <linux/kernel.h>
  22#include <linux/module.h>
  23#include <linux/of.h>
  24#include <linux/pinctrl/consumer.h>
  25#include <linux/platform_device.h>
  26#include <linux/pm_runtime.h>
  27#include <linux/sched.h>
  28#include <linux/slab.h>
  29
  30#define DRIVER_NAME "imx-lpi2c"
  31
  32#define LPI2C_PARAM	0x04	/* i2c RX/TX FIFO size */
  33#define LPI2C_MCR	0x10	/* i2c contrl register */
  34#define LPI2C_MSR	0x14	/* i2c status register */
  35#define LPI2C_MIER	0x18	/* i2c interrupt enable */
  36#define LPI2C_MDER	0x1C	/* i2c DMA enable */
  37#define LPI2C_MCFGR0	0x20	/* i2c master configuration */
  38#define LPI2C_MCFGR1	0x24	/* i2c master configuration */
  39#define LPI2C_MCFGR2	0x28	/* i2c master configuration */
  40#define LPI2C_MCFGR3	0x2C	/* i2c master configuration */
  41#define LPI2C_MCCR0	0x48	/* i2c master clk configuration */
  42#define LPI2C_MCCR1	0x50	/* i2c master clk configuration */
  43#define LPI2C_MFCR	0x58	/* i2c master FIFO control */
  44#define LPI2C_MFSR	0x5C	/* i2c master FIFO status */
  45#define LPI2C_MTDR	0x60	/* i2c master TX data register */
  46#define LPI2C_MRDR	0x70	/* i2c master RX data register */
  47
  48#define LPI2C_SCR	0x110	/* i2c target control register */
  49#define LPI2C_SSR	0x114	/* i2c target status register */
  50#define LPI2C_SIER	0x118	/* i2c target interrupt enable */
  51#define LPI2C_SDER	0x11C	/* i2c target DMA enable */
  52#define LPI2C_SCFGR0	0x120	/* i2c target configuration */
  53#define LPI2C_SCFGR1	0x124	/* i2c target configuration */
  54#define LPI2C_SCFGR2	0x128	/* i2c target configuration */
  55#define LPI2C_SAMR	0x140	/* i2c target address match */
  56#define LPI2C_SASR	0x150	/* i2c target address status */
  57#define LPI2C_STAR	0x154	/* i2c target transmit ACK */
  58#define LPI2C_STDR	0x160	/* i2c target transmit data */
  59#define LPI2C_SRDR	0x170	/* i2c target receive data */
  60#define LPI2C_SRDROR	0x178	/* i2c target receive data read only */
  61
  62/* i2c command */
  63#define TRAN_DATA	0X00
  64#define RECV_DATA	0X01
  65#define GEN_STOP	0X02
  66#define RECV_DISCARD	0X03
  67#define GEN_START	0X04
  68#define START_NACK	0X05
  69#define START_HIGH	0X06
  70#define START_HIGH_NACK	0X07
  71
  72#define MCR_MEN		BIT(0)
  73#define MCR_RST		BIT(1)
  74#define MCR_DOZEN	BIT(2)
  75#define MCR_DBGEN	BIT(3)
  76#define MCR_RTF		BIT(8)
  77#define MCR_RRF		BIT(9)
  78#define MSR_TDF		BIT(0)
  79#define MSR_RDF		BIT(1)
  80#define MSR_SDF		BIT(9)
  81#define MSR_NDF		BIT(10)
  82#define MSR_ALF		BIT(11)
  83#define MSR_MBF		BIT(24)
  84#define MSR_BBF		BIT(25)
  85#define MIER_TDIE	BIT(0)
  86#define MIER_RDIE	BIT(1)
  87#define MIER_SDIE	BIT(9)
  88#define MIER_NDIE	BIT(10)
  89#define MCFGR1_AUTOSTOP	BIT(8)
  90#define MCFGR1_IGNACK	BIT(9)
  91#define MRDR_RXEMPTY	BIT(14)
  92#define MDER_TDDE	BIT(0)
  93#define MDER_RDDE	BIT(1)
  94#define MSR_RDF_ASSERTED(x) FIELD_GET(MSR_RDF, (x))
  95
  96#define SCR_SEN		BIT(0)
  97#define SCR_RST		BIT(1)
  98#define SCR_FILTEN	BIT(4)
  99#define SCR_RTF		BIT(8)
 100#define SCR_RRF		BIT(9)
 101#define SSR_TDF		BIT(0)
 102#define SSR_RDF		BIT(1)
 103#define SSR_AVF		BIT(2)
 104#define SSR_TAF		BIT(3)
 105#define SSR_RSF		BIT(8)
 106#define SSR_SDF		BIT(9)
 107#define SSR_BEF		BIT(10)
 108#define SSR_FEF		BIT(11)
 109#define SSR_SBF		BIT(24)
 110#define SSR_BBF		BIT(25)
 111#define SSR_CLEAR_BITS	(SSR_RSF | SSR_SDF | SSR_BEF | SSR_FEF)
 112#define SIER_TDIE	BIT(0)
 113#define SIER_RDIE	BIT(1)
 114#define SIER_AVIE	BIT(2)
 115#define SIER_TAIE	BIT(3)
 116#define SIER_RSIE	BIT(8)
 117#define SIER_SDIE	BIT(9)
 118#define SIER_BEIE	BIT(10)
 119#define SIER_FEIE	BIT(11)
 120#define SIER_AM0F	BIT(12)
 121#define SCFGR1_RXSTALL	BIT(1)
 122#define SCFGR1_TXDSTALL	BIT(2)
 123#define SCFGR2_FILTSDA_SHIFT	24
 124#define SCFGR2_FILTSCL_SHIFT	16
 125#define SCFGR2_CLKHOLD(x)	(x)
 126#define SCFGR2_FILTSDA(x)	((x) << SCFGR2_FILTSDA_SHIFT)
 127#define SCFGR2_FILTSCL(x)	((x) << SCFGR2_FILTSCL_SHIFT)
 128#define SASR_READ_REQ	0x1
 129#define SLAVE_INT_FLAG	(SIER_TDIE | SIER_RDIE | SIER_AVIE | \
 130			 SIER_SDIE | SIER_BEIE)
 131
 132#define I2C_CLK_RATIO	2
 133#define CHUNK_DATA	256
 134
 135#define I2C_PM_TIMEOUT		10 /* ms */
 136#define I2C_DMA_THRESHOLD	8 /* bytes */
 137
 138enum lpi2c_imx_mode {
 139	STANDARD,	/* 100+Kbps */
 140	FAST,		/* 400+Kbps */
 141	FAST_PLUS,	/* 1.0+Mbps */
 142	HS,		/* 3.4+Mbps */
 143	ULTRA_FAST,	/* 5.0+Mbps */
 144};
 145
 146enum lpi2c_imx_pincfg {
 147	TWO_PIN_OD,
 148	TWO_PIN_OO,
 149	TWO_PIN_PP,
 150	FOUR_PIN_PP,
 151};
 152
 153struct lpi2c_imx_dma {
 154	bool		using_pio_mode;
 155	u8		rx_cmd_buf_len;
 156	u8		*dma_buf;
 157	u16		*rx_cmd_buf;
 158	unsigned int	dma_len;
 159	unsigned int	tx_burst_num;
 160	unsigned int	rx_burst_num;
 161	unsigned long	dma_msg_flag;
 162	resource_size_t	phy_addr;
 163	dma_addr_t	dma_tx_addr;
 164	dma_addr_t	dma_addr;
 165	enum dma_data_direction dma_data_dir;
 166	enum dma_transfer_direction dma_transfer_dir;
 167	struct dma_chan	*chan_tx;
 168	struct dma_chan	*chan_rx;
 169};
 170
 171struct lpi2c_imx_struct {
 172	struct i2c_adapter	adapter;
 173	int			num_clks;
 174	struct clk_bulk_data	*clks;
 175	void __iomem		*base;
 176	__u8			*rx_buf;
 177	__u8			*tx_buf;
 178	struct completion	complete;
 179	unsigned long		rate_per;
 180	unsigned int		msglen;
 181	unsigned int		delivered;
 182	unsigned int		block_data;
 183	unsigned int		bitrate;
 184	unsigned int		txfifosize;
 185	unsigned int		rxfifosize;
 186	enum lpi2c_imx_mode	mode;
 187	struct i2c_bus_recovery_info rinfo;
 188	bool			can_use_dma;
 189	struct lpi2c_imx_dma	*dma;
 190	struct i2c_client	*target;
 191};
 192
 193#define lpi2c_imx_read_msr_poll_timeout(atomic, val, cond)                    \
 194	(atomic ? readl_poll_timeout_atomic(lpi2c_imx->base + LPI2C_MSR, val, \
 195					    cond, 0, 500000) :                \
 196		  readl_poll_timeout(lpi2c_imx->base + LPI2C_MSR, val, cond,  \
 197				     0, 500000))
 198
 199static void lpi2c_imx_intctrl(struct lpi2c_imx_struct *lpi2c_imx,
 200			      unsigned int enable)
 201{
 202	writel(enable, lpi2c_imx->base + LPI2C_MIER);
 203}
 204
 205static int lpi2c_imx_bus_busy(struct lpi2c_imx_struct *lpi2c_imx, bool atomic)
 206{
 207	unsigned int temp;
 208	int err;
 209
 210	err = lpi2c_imx_read_msr_poll_timeout(atomic, temp,
 211					      temp & (MSR_ALF | MSR_BBF | MSR_MBF));
 212
 213	/* check for arbitration lost, clear if set */
 214	if (temp & MSR_ALF) {
 215		writel(temp, lpi2c_imx->base + LPI2C_MSR);
 216		return -EAGAIN;
 217	}
 218
 219	/* check for bus not busy */
 220	if (err) {
 221		dev_dbg(&lpi2c_imx->adapter.dev, "bus not work\n");
 222		if (lpi2c_imx->adapter.bus_recovery_info)
 223			i2c_recover_bus(&lpi2c_imx->adapter);
 224		return -ETIMEDOUT;
 225	}
 226
 227	return 0;
 228}
 229
 230static u32 lpi2c_imx_txfifo_cnt(struct lpi2c_imx_struct *lpi2c_imx)
 231{
 232	return readl(lpi2c_imx->base + LPI2C_MFSR) & 0xff;
 233}
 234
 235static void lpi2c_imx_set_mode(struct lpi2c_imx_struct *lpi2c_imx)
 236{
 237	unsigned int bitrate = lpi2c_imx->bitrate;
 238	enum lpi2c_imx_mode mode;
 239
 240	if (bitrate < I2C_MAX_FAST_MODE_FREQ)
 241		mode = STANDARD;
 242	else if (bitrate < I2C_MAX_FAST_MODE_PLUS_FREQ)
 243		mode = FAST;
 244	else if (bitrate < I2C_MAX_HIGH_SPEED_MODE_FREQ)
 245		mode = FAST_PLUS;
 246	else if (bitrate < I2C_MAX_ULTRA_FAST_MODE_FREQ)
 247		mode = HS;
 248	else
 249		mode = ULTRA_FAST;
 250
 251	lpi2c_imx->mode = mode;
 252}
 253
 254static int lpi2c_imx_start(struct lpi2c_imx_struct *lpi2c_imx,
 255			   struct i2c_msg *msgs, bool atomic)
 256{
 257	unsigned int temp;
 258
 259	temp = readl(lpi2c_imx->base + LPI2C_MCR);
 260	temp |= MCR_RRF | MCR_RTF;
 261	writel(temp, lpi2c_imx->base + LPI2C_MCR);
 262	writel(0x7f00, lpi2c_imx->base + LPI2C_MSR);
 263
 264	temp = i2c_8bit_addr_from_msg(msgs) | (GEN_START << 8);
 265	writel(temp, lpi2c_imx->base + LPI2C_MTDR);
 266
 267	return lpi2c_imx_bus_busy(lpi2c_imx, atomic);
 268}
 269
 270static void lpi2c_imx_stop(struct lpi2c_imx_struct *lpi2c_imx, bool atomic)
 271{
 272	unsigned int temp;
 273	int err;
 274
 275	writel(GEN_STOP << 8, lpi2c_imx->base + LPI2C_MTDR);
 276
 277	err = lpi2c_imx_read_msr_poll_timeout(atomic, temp, temp & MSR_SDF);
 278
 279	if (err) {
 280		dev_dbg(&lpi2c_imx->adapter.dev, "stop timeout\n");
 281		if (lpi2c_imx->adapter.bus_recovery_info)
 282			i2c_recover_bus(&lpi2c_imx->adapter);
 283	}
 284}
 285
 286/* CLKLO = I2C_CLK_RATIO * CLKHI, SETHOLD = CLKHI, DATAVD = CLKHI/2 */
 287static int lpi2c_imx_config(struct lpi2c_imx_struct *lpi2c_imx)
 288{
 289	u8 prescale, filt, sethold, datavd;
 290	unsigned int clk_rate, clk_cycle, clkhi, clklo;
 291	enum lpi2c_imx_pincfg pincfg;
 292	unsigned int temp;
 293
 294	lpi2c_imx_set_mode(lpi2c_imx);
 295
 296	clk_rate = lpi2c_imx->rate_per;
 297
 298	if (lpi2c_imx->mode == HS || lpi2c_imx->mode == ULTRA_FAST)
 299		filt = 0;
 300	else
 301		filt = 2;
 302
 303	for (prescale = 0; prescale <= 7; prescale++) {
 304		clk_cycle = clk_rate / ((1 << prescale) * lpi2c_imx->bitrate)
 305			    - 3 - (filt >> 1);
 306		clkhi = DIV_ROUND_UP(clk_cycle, I2C_CLK_RATIO + 1);
 307		clklo = clk_cycle - clkhi;
 308		if (clklo < 64)
 309			break;
 310	}
 311
 312	if (prescale > 7)
 313		return -EINVAL;
 314
 315	/* set MCFGR1: PINCFG, PRESCALE, IGNACK */
 316	if (lpi2c_imx->mode == ULTRA_FAST)
 317		pincfg = TWO_PIN_OO;
 318	else
 319		pincfg = TWO_PIN_OD;
 320	temp = prescale | pincfg << 24;
 321
 322	if (lpi2c_imx->mode == ULTRA_FAST)
 323		temp |= MCFGR1_IGNACK;
 324
 325	writel(temp, lpi2c_imx->base + LPI2C_MCFGR1);
 326
 327	/* set MCFGR2: FILTSDA, FILTSCL */
 328	temp = (filt << 16) | (filt << 24);
 329	writel(temp, lpi2c_imx->base + LPI2C_MCFGR2);
 330
 331	/* set MCCR: DATAVD, SETHOLD, CLKHI, CLKLO */
 332	sethold = clkhi;
 333	datavd = clkhi >> 1;
 334	temp = datavd << 24 | sethold << 16 | clkhi << 8 | clklo;
 335
 336	if (lpi2c_imx->mode == HS)
 337		writel(temp, lpi2c_imx->base + LPI2C_MCCR1);
 338	else
 339		writel(temp, lpi2c_imx->base + LPI2C_MCCR0);
 340
 341	return 0;
 342}
 343
 344static int lpi2c_imx_master_enable(struct lpi2c_imx_struct *lpi2c_imx)
 345{
 346	unsigned int temp;
 347	int ret;
 348
 349	ret = pm_runtime_resume_and_get(lpi2c_imx->adapter.dev.parent);
 350	if (ret < 0)
 351		return ret;
 352
 353	temp = MCR_RST;
 354	writel(temp, lpi2c_imx->base + LPI2C_MCR);
 355	writel(0, lpi2c_imx->base + LPI2C_MCR);
 356
 357	ret = lpi2c_imx_config(lpi2c_imx);
 358	if (ret)
 359		goto rpm_put;
 360
 361	temp = readl(lpi2c_imx->base + LPI2C_MCR);
 362	temp |= MCR_MEN;
 363	writel(temp, lpi2c_imx->base + LPI2C_MCR);
 364
 365	return 0;
 366
 367rpm_put:
 368	pm_runtime_put_autosuspend(lpi2c_imx->adapter.dev.parent);
 369
 370	return ret;
 371}
 372
 373static int lpi2c_imx_master_disable(struct lpi2c_imx_struct *lpi2c_imx)
 374{
 375	u32 temp;
 376
 377	temp = readl(lpi2c_imx->base + LPI2C_MCR);
 378	temp &= ~MCR_MEN;
 379	writel(temp, lpi2c_imx->base + LPI2C_MCR);
 380
 381	pm_runtime_put_autosuspend(lpi2c_imx->adapter.dev.parent);
 382
 383	return 0;
 384}
 385
 386static int lpi2c_imx_pio_msg_complete(struct lpi2c_imx_struct *lpi2c_imx)
 387{
 388	unsigned long time_left;
 389
 390	time_left = wait_for_completion_timeout(&lpi2c_imx->complete, HZ);
 391
 392	return time_left ? 0 : -ETIMEDOUT;
 393}
 394
 395static int lpi2c_imx_txfifo_empty(struct lpi2c_imx_struct *lpi2c_imx, bool atomic)
 396{
 397	unsigned int temp;
 398	int err;
 399
 400	err = lpi2c_imx_read_msr_poll_timeout(atomic, temp,
 401					      (temp & MSR_NDF) || !lpi2c_imx_txfifo_cnt(lpi2c_imx));
 402
 403	if (temp & MSR_NDF) {
 404		dev_dbg(&lpi2c_imx->adapter.dev, "NDF detected\n");
 405		return -EIO;
 406	}
 407
 408	if (err) {
 409		dev_dbg(&lpi2c_imx->adapter.dev, "txfifo empty timeout\n");
 410		if (lpi2c_imx->adapter.bus_recovery_info)
 411			i2c_recover_bus(&lpi2c_imx->adapter);
 412		return -ETIMEDOUT;
 413	}
 414
 415	return 0;
 416}
 417
 418static void lpi2c_imx_set_tx_watermark(struct lpi2c_imx_struct *lpi2c_imx)
 419{
 420	writel(lpi2c_imx->txfifosize >> 1, lpi2c_imx->base + LPI2C_MFCR);
 421}
 422
 423static void lpi2c_imx_set_rx_watermark(struct lpi2c_imx_struct *lpi2c_imx)
 424{
 425	unsigned int temp, remaining;
 426
 427	remaining = lpi2c_imx->msglen - lpi2c_imx->delivered;
 428
 429	if (remaining > (lpi2c_imx->rxfifosize >> 1))
 430		temp = lpi2c_imx->rxfifosize >> 1;
 431	else
 432		temp = 0;
 433
 434	writel(temp << 16, lpi2c_imx->base + LPI2C_MFCR);
 435}
 436
 437static bool lpi2c_imx_write_txfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomic)
 438{
 439	unsigned int data, txcnt;
 440
 441	txcnt = readl(lpi2c_imx->base + LPI2C_MFSR) & 0xff;
 442
 443	while (txcnt < lpi2c_imx->txfifosize) {
 444		if (lpi2c_imx->delivered == lpi2c_imx->msglen)
 445			break;
 446
 447		data = lpi2c_imx->tx_buf[lpi2c_imx->delivered++];
 448		writel(data, lpi2c_imx->base + LPI2C_MTDR);
 449		txcnt++;
 450	}
 451
 452	if (lpi2c_imx->delivered < lpi2c_imx->msglen) {
 453		if (!atomic)
 454			lpi2c_imx_intctrl(lpi2c_imx, MIER_TDIE | MIER_NDIE);
 455		return false;
 456	}
 457
 458	if (!atomic)
 459		complete(&lpi2c_imx->complete);
 460
 461	return true;
 462}
 463
 464static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomic)
 465{
 466	unsigned int remaining;
 467	unsigned int temp, data;
 468
 469	do {
 470		data = readl(lpi2c_imx->base + LPI2C_MRDR);
 471		if (data & MRDR_RXEMPTY)
 472			break;
 473
 474		lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff;
 475	} while (1);
 476
 477	remaining = lpi2c_imx->msglen - lpi2c_imx->delivered;
 478
 479	if (!remaining) {
 480		if (!atomic)
 481			complete(&lpi2c_imx->complete);
 482		return true;
 483	}
 484
 485	/* not finished, still waiting for rx data */
 486	lpi2c_imx_set_rx_watermark(lpi2c_imx);
 487
 488	/* multiple receive commands */
 489	if (!(lpi2c_imx->delivered & 0xff)) {
 490		temp = (remaining > CHUNK_DATA ? CHUNK_DATA : remaining) - 1;
 491		temp |= (RECV_DATA << 8);
 492		writel(temp, lpi2c_imx->base + LPI2C_MTDR);
 493	}
 494
 495	if (!atomic)
 496		lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE);
 497
 498	return false;
 499}
 500
 501static void lpi2c_imx_write(struct lpi2c_imx_struct *lpi2c_imx,
 502			    struct i2c_msg *msgs)
 503{
 504	lpi2c_imx->tx_buf = msgs->buf;
 505	lpi2c_imx_set_tx_watermark(lpi2c_imx);
 506	lpi2c_imx_write_txfifo(lpi2c_imx, false);
 507}
 508
 509static int lpi2c_imx_write_atomic(struct lpi2c_imx_struct *lpi2c_imx,
 510				  struct i2c_msg *msgs)
 511{
 512	u32 temp;
 513	int err;
 514
 515	lpi2c_imx->tx_buf = msgs->buf;
 516
 517	err = lpi2c_imx_read_msr_poll_timeout(true, temp,
 518					      (temp & MSR_NDF) ||
 519					      lpi2c_imx_write_txfifo(lpi2c_imx, true));
 520
 521	if (temp & MSR_NDF)
 522		return -EIO;
 523
 524	return err;
 525}
 526
 527static unsigned int lpi2c_SMBus_block_read_length_byte(struct lpi2c_imx_struct *lpi2c_imx)
 528{
 529	unsigned int data;
 530
 531	data = readl(lpi2c_imx->base + LPI2C_MRDR);
 532	lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff;
 533
 534	return data;
 535}
 536
 537static int lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx,
 538			       struct i2c_msg *msgs)
 539{
 540	unsigned int temp, val, block_len;
 541	int ret;
 542
 543	lpi2c_imx->rx_buf = msgs->buf;
 544	lpi2c_imx->block_data = msgs->flags & I2C_M_RECV_LEN;
 545
 546	lpi2c_imx_set_rx_watermark(lpi2c_imx);
 547
 548	if (!lpi2c_imx->block_data) {
 549		temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1;
 550		temp |= (RECV_DATA << 8);
 551		writel(temp, lpi2c_imx->base + LPI2C_MTDR);
 552	} else {
 553		/*
 554		 * The LPI2C controller automatically sends a NACK after the last byte of a
 555		 * receive command, unless the next command in MTDR is also a receive command.
 556		 * If MTDR is empty when a receive completes, a NACK is sent by default.
 557		 *
 558		 * To comply with the SMBus block read spec, we start with a 2-byte read:
 559		 * The first byte in RXFIFO is the block length. Once this byte arrives, the
 560		 * controller immediately updates MTDR with the next read command, ensuring
 561		 * continuous ACK instead of NACK.
 562		 *
 563		 * The second byte is the first block data byte. Therefore, the subsequent
 564		 * read command should request (block_len - 1) bytes, since one data byte
 565		 * has already been read.
 566		 */
 567
 568		writel((RECV_DATA << 8) | 0x01, lpi2c_imx->base + LPI2C_MTDR);
 569
 570		ret = readl_poll_timeout(lpi2c_imx->base + LPI2C_MSR, val,
 571					 MSR_RDF_ASSERTED(val), 1, 1000);
 572		if (ret) {
 573			dev_err(&lpi2c_imx->adapter.dev, "SMBus read count failed %d\n", ret);
 574			return ret;
 575		}
 576
 577		/* Read block length byte and confirm this SMBus transfer meets protocol */
 578		block_len = lpi2c_SMBus_block_read_length_byte(lpi2c_imx);
 579		if (block_len == 0 || block_len > I2C_SMBUS_BLOCK_MAX) {
 580			dev_err(&lpi2c_imx->adapter.dev, "Invalid SMBus block read length\n");
 581			return -EPROTO;
 582		}
 583
 584		/*
 585		 * When block_len shows more bytes need to be read, update second read command to
 586		 * keep MTDR non-empty and ensuring continuous ACKs. Only update command register
 587		 * here. All block bytes will be read out at IRQ handler or lpi2c_imx_read_atomic()
 588		 * function.
 589		 */
 590		if (block_len > 1)
 591			writel((RECV_DATA << 8) | (block_len - 2), lpi2c_imx->base + LPI2C_MTDR);
 592
 593		lpi2c_imx->msglen += block_len;
 594		msgs->len += block_len;
 595	}
 596
 597	return 0;
 598}
 599
 600static bool lpi2c_imx_read_chunk_atomic(struct lpi2c_imx_struct *lpi2c_imx)
 601{
 602	u32 rxcnt;
 603
 604	rxcnt = (readl(lpi2c_imx->base + LPI2C_MFSR) >> 16) & 0xFF;
 605	if (!rxcnt)
 606		return false;
 607
 608	if (!lpi2c_imx_read_rxfifo(lpi2c_imx, true))
 609		return false;
 610
 611	return true;
 612}
 613
 614static int lpi2c_imx_read_atomic(struct lpi2c_imx_struct *lpi2c_imx,
 615				 struct i2c_msg *msgs)
 616{
 617	u32 temp;
 618	int tmo_us;
 619
 620	tmo_us = 1000000;
 621	do {
 622		if (lpi2c_imx_read_chunk_atomic(lpi2c_imx))
 623			return 0;
 624
 625		temp = readl(lpi2c_imx->base + LPI2C_MSR);
 626
 627		if (temp & MSR_NDF)
 628			return -EIO;
 629
 630		udelay(100);
 631		tmo_us -= 100;
 632	} while (tmo_us > 0);
 633
 634	return -ETIMEDOUT;
 635}
 636
 637static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg)
 638{
 639	if (!lpi2c_imx->can_use_dma)
 640		return false;
 641
 642	/* DMA is not suitable for SMBus block read */
 643	if (msg->flags & I2C_M_RECV_LEN)
 644		return false;
 645
 646	/*
 647	 * A system-wide suspend or resume transition is in progress. LPI2C should use PIO to
 648	 * transfer data to avoid issue caused by no ready DMA HW resource.
 649	 */
 650	if (pm_suspend_in_progress())
 651		return false;
 652
 653	/*
 654	 * When the length of data is less than I2C_DMA_THRESHOLD,
 655	 * cpu mode is used directly to avoid low performance.
 656	 */
 657	return !(msg->len < I2C_DMA_THRESHOLD);
 658}
 659
 660static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx,
 661			      struct i2c_msg *msg)
 662{
 663	int ret;
 664
 665	reinit_completion(&lpi2c_imx->complete);
 666
 667	if (msg->flags & I2C_M_RD) {
 668		ret = lpi2c_imx_read_init(lpi2c_imx, msg);
 669		if (ret)
 670			return ret;
 671		lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE | MIER_NDIE);
 672	} else {
 673		lpi2c_imx_write(lpi2c_imx, msg);
 674	}
 675
 676	return lpi2c_imx_pio_msg_complete(lpi2c_imx);
 677}
 678
 679static int lpi2c_imx_pio_xfer_atomic(struct lpi2c_imx_struct *lpi2c_imx,
 680				     struct i2c_msg *msg)
 681{
 682	int ret;
 683
 684	if (msg->flags & I2C_M_RD) {
 685		ret = lpi2c_imx_read_init(lpi2c_imx, msg);
 686		if (ret)
 687			return ret;
 688		return lpi2c_imx_read_atomic(lpi2c_imx, msg);
 689	}
 690
 691	return lpi2c_imx_write_atomic(lpi2c_imx, msg);
 692}
 693
 694static int lpi2c_imx_dma_timeout_calculate(struct lpi2c_imx_struct *lpi2c_imx)
 695{
 696	unsigned long time = 0;
 697
 698	time = 8 * lpi2c_imx->dma->dma_len * 1000 / lpi2c_imx->bitrate;
 699
 700	/* Add extra second for scheduler related activities */
 701	time += 1;
 702
 703	/* Double calculated time */
 704	return secs_to_jiffies(time);
 705}
 706
 707static int lpi2c_imx_alloc_rx_cmd_buf(struct lpi2c_imx_struct *lpi2c_imx)
 708{
 709	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
 710	u16 rx_remain = dma->dma_len;
 711	int cmd_num;
 712	u16 temp;
 713
 714	/*
 715	 * Calculate the number of rx command words via the DMA TX channel
 716	 * writing into command register based on the i2c msg len, and build
 717	 * the rx command words buffer.
 718	 */
 719	cmd_num = DIV_ROUND_UP(rx_remain, CHUNK_DATA);
 720	dma->rx_cmd_buf = kcalloc(cmd_num, sizeof(u16), GFP_KERNEL);
 721	dma->rx_cmd_buf_len = cmd_num * sizeof(u16);
 722
 723	if (!dma->rx_cmd_buf) {
 724		dev_err(&lpi2c_imx->adapter.dev, "Alloc RX cmd buffer failed\n");
 725		return -ENOMEM;
 726	}
 727
 728	for (int i = 0; i < cmd_num ; i++) {
 729		temp = rx_remain > CHUNK_DATA ? CHUNK_DATA - 1 : rx_remain - 1;
 730		temp |= (RECV_DATA << 8);
 731		rx_remain -= CHUNK_DATA;
 732		dma->rx_cmd_buf[i] = temp;
 733	}
 734
 735	return 0;
 736}
 737
 738static int lpi2c_imx_dma_msg_complete(struct lpi2c_imx_struct *lpi2c_imx)
 739{
 740	unsigned long time_left, time;
 741
 742	time = lpi2c_imx_dma_timeout_calculate(lpi2c_imx);
 743	time_left = wait_for_completion_timeout(&lpi2c_imx->complete, time);
 744	if (time_left == 0) {
 745		dev_err(&lpi2c_imx->adapter.dev, "I/O Error in DMA Data Transfer\n");
 746		return -ETIMEDOUT;
 747	}
 748
 749	return 0;
 750}
 751
 752static void lpi2c_dma_unmap(struct lpi2c_imx_dma *dma)
 753{
 754	struct dma_chan *chan = dma->dma_data_dir == DMA_FROM_DEVICE
 755				? dma->chan_rx : dma->chan_tx;
 756
 757	dma_unmap_single(chan->device->dev, dma->dma_addr,
 758			 dma->dma_len, dma->dma_data_dir);
 759
 760	dma->dma_data_dir = DMA_NONE;
 761}
 762
 763static void lpi2c_cleanup_rx_cmd_dma(struct lpi2c_imx_dma *dma)
 764{
 765	dmaengine_terminate_sync(dma->chan_tx);
 766	dma_unmap_single(dma->chan_tx->device->dev, dma->dma_tx_addr,
 767			 dma->rx_cmd_buf_len, DMA_TO_DEVICE);
 768}
 769
 770static void lpi2c_cleanup_dma(struct lpi2c_imx_dma *dma)
 771{
 772	if (dma->dma_data_dir == DMA_FROM_DEVICE)
 773		dmaengine_terminate_sync(dma->chan_rx);
 774	else if (dma->dma_data_dir == DMA_TO_DEVICE)
 775		dmaengine_terminate_sync(dma->chan_tx);
 776
 777	lpi2c_dma_unmap(dma);
 778}
 779
 780static void lpi2c_dma_callback(void *data)
 781{
 782	struct lpi2c_imx_struct *lpi2c_imx = (struct lpi2c_imx_struct *)data;
 783
 784	complete(&lpi2c_imx->complete);
 785}
 786
 787static int lpi2c_dma_rx_cmd_submit(struct lpi2c_imx_struct *lpi2c_imx)
 788{
 789	struct dma_async_tx_descriptor *rx_cmd_desc;
 790	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
 791	struct dma_chan *txchan = dma->chan_tx;
 792	dma_cookie_t cookie;
 793
 794	dma->dma_tx_addr = dma_map_single(txchan->device->dev,
 795					  dma->rx_cmd_buf, dma->rx_cmd_buf_len,
 796					  DMA_TO_DEVICE);
 797	if (dma_mapping_error(txchan->device->dev, dma->dma_tx_addr)) {
 798		dev_err(&lpi2c_imx->adapter.dev, "DMA map failed, use pio\n");
 799		return -EINVAL;
 800	}
 801
 802	rx_cmd_desc = dmaengine_prep_slave_single(txchan, dma->dma_tx_addr,
 803						  dma->rx_cmd_buf_len, DMA_MEM_TO_DEV,
 804						  DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 805	if (!rx_cmd_desc) {
 806		dev_err(&lpi2c_imx->adapter.dev, "DMA prep slave sg failed, use pio\n");
 807		goto desc_prepare_err_exit;
 808	}
 809
 810	cookie = dmaengine_submit(rx_cmd_desc);
 811	if (dma_submit_error(cookie)) {
 812		dev_err(&lpi2c_imx->adapter.dev, "submitting DMA failed, use pio\n");
 813		goto submit_err_exit;
 814	}
 815
 816	dma_async_issue_pending(txchan);
 817
 818	return 0;
 819
 820desc_prepare_err_exit:
 821	dma_unmap_single(txchan->device->dev, dma->dma_tx_addr,
 822			 dma->rx_cmd_buf_len, DMA_TO_DEVICE);
 823	return -EINVAL;
 824
 825submit_err_exit:
 826	dma_unmap_single(txchan->device->dev, dma->dma_tx_addr,
 827			 dma->rx_cmd_buf_len, DMA_TO_DEVICE);
 828	dmaengine_desc_free(rx_cmd_desc);
 829	return -EINVAL;
 830}
 831
 832static int lpi2c_dma_submit(struct lpi2c_imx_struct *lpi2c_imx)
 833{
 834	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
 835	struct dma_async_tx_descriptor *desc;
 836	struct dma_chan *chan;
 837	dma_cookie_t cookie;
 838
 839	if (dma->dma_msg_flag & I2C_M_RD) {
 840		chan = dma->chan_rx;
 841		dma->dma_data_dir = DMA_FROM_DEVICE;
 842		dma->dma_transfer_dir = DMA_DEV_TO_MEM;
 843	} else {
 844		chan = dma->chan_tx;
 845		dma->dma_data_dir = DMA_TO_DEVICE;
 846		dma->dma_transfer_dir = DMA_MEM_TO_DEV;
 847	}
 848
 849	dma->dma_addr = dma_map_single(chan->device->dev,
 850				       dma->dma_buf, dma->dma_len, dma->dma_data_dir);
 851	if (dma_mapping_error(chan->device->dev, dma->dma_addr)) {
 852		dev_err(&lpi2c_imx->adapter.dev, "DMA map failed, use pio\n");
 853		return -EINVAL;
 854	}
 855
 856	desc = dmaengine_prep_slave_single(chan, dma->dma_addr,
 857					   dma->dma_len, dma->dma_transfer_dir,
 858					   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 859	if (!desc) {
 860		dev_err(&lpi2c_imx->adapter.dev, "DMA prep slave sg failed, use pio\n");
 861		goto desc_prepare_err_exit;
 862	}
 863
 864	reinit_completion(&lpi2c_imx->complete);
 865	desc->callback = lpi2c_dma_callback;
 866	desc->callback_param = lpi2c_imx;
 867
 868	cookie = dmaengine_submit(desc);
 869	if (dma_submit_error(cookie)) {
 870		dev_err(&lpi2c_imx->adapter.dev, "submitting DMA failed, use pio\n");
 871		goto submit_err_exit;
 872	}
 873
 874	/* Can't switch to PIO mode when DMA have started transfer */
 875	dma->using_pio_mode = false;
 876
 877	dma_async_issue_pending(chan);
 878
 879	return 0;
 880
 881desc_prepare_err_exit:
 882	lpi2c_dma_unmap(dma);
 883	return -EINVAL;
 884
 885submit_err_exit:
 886	lpi2c_dma_unmap(dma);
 887	dmaengine_desc_free(desc);
 888	return -EINVAL;
 889}
 890
 891static int lpi2c_imx_find_max_burst_num(unsigned int fifosize, unsigned int len)
 892{
 893	unsigned int i;
 894
 895	for (i = fifosize / 2; i > 0; i--)
 896		if (!(len % i))
 897			break;
 898
 899	return i;
 900}
 901
 902/*
 903 * For a highest DMA efficiency, tx/rx burst number should be calculated according
 904 * to the FIFO depth.
 905 */
 906static void lpi2c_imx_dma_burst_num_calculate(struct lpi2c_imx_struct *lpi2c_imx)
 907{
 908	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
 909	unsigned int cmd_num;
 910
 911	if (dma->dma_msg_flag & I2C_M_RD) {
 912		/*
 913		 * One RX cmd word can trigger DMA receive no more than 256 bytes.
 914		 * The number of RX cmd words should be calculated based on the data
 915		 * length.
 916		 */
 917		cmd_num = DIV_ROUND_UP(dma->dma_len, CHUNK_DATA);
 918		dma->tx_burst_num = lpi2c_imx_find_max_burst_num(lpi2c_imx->txfifosize,
 919								 cmd_num);
 920		dma->rx_burst_num = lpi2c_imx_find_max_burst_num(lpi2c_imx->rxfifosize,
 921								 dma->dma_len);
 922	} else {
 923		dma->tx_burst_num = lpi2c_imx_find_max_burst_num(lpi2c_imx->txfifosize,
 924								 dma->dma_len);
 925	}
 926}
 927
 928static int lpi2c_dma_config(struct lpi2c_imx_struct *lpi2c_imx)
 929{
 930	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
 931	struct dma_slave_config rx = {}, tx = {};
 932	int ret;
 933
 934	lpi2c_imx_dma_burst_num_calculate(lpi2c_imx);
 935
 936	if (dma->dma_msg_flag & I2C_M_RD) {
 937		tx.dst_addr = dma->phy_addr + LPI2C_MTDR;
 938		tx.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 939		tx.dst_maxburst = dma->tx_burst_num;
 940		tx.direction = DMA_MEM_TO_DEV;
 941		ret = dmaengine_slave_config(dma->chan_tx, &tx);
 942		if (ret < 0)
 943			return ret;
 944
 945		rx.src_addr = dma->phy_addr + LPI2C_MRDR;
 946		rx.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 947		rx.src_maxburst = dma->rx_burst_num;
 948		rx.direction = DMA_DEV_TO_MEM;
 949		ret = dmaengine_slave_config(dma->chan_rx, &rx);
 950		if (ret < 0)
 951			return ret;
 952	} else {
 953		tx.dst_addr = dma->phy_addr + LPI2C_MTDR;
 954		tx.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 955		tx.dst_maxburst = dma->tx_burst_num;
 956		tx.direction = DMA_MEM_TO_DEV;
 957		ret = dmaengine_slave_config(dma->chan_tx, &tx);
 958		if (ret < 0)
 959			return ret;
 960	}
 961
 962	return 0;
 963}
 964
 965static void lpi2c_dma_enable(struct lpi2c_imx_struct *lpi2c_imx)
 966{
 967	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
 968	/*
 969	 * TX interrupt will be triggered when the number of words in
 970	 * the transmit FIFO is equal or less than TX watermark.
 971	 * RX interrupt will be triggered when the number of words in
 972	 * the receive FIFO is greater than RX watermark.
 973	 * In order to trigger the DMA interrupt, TX watermark should be
 974	 * set equal to the DMA TX burst number but RX watermark should
 975	 * be set less than the DMA RX burst number.
 976	 */
 977	if (dma->dma_msg_flag & I2C_M_RD) {
 978		/* Set I2C TX/RX watermark */
 979		writel(dma->tx_burst_num | (dma->rx_burst_num - 1) << 16,
 980		       lpi2c_imx->base + LPI2C_MFCR);
 981		/* Enable I2C DMA TX/RX function */
 982		writel(MDER_TDDE | MDER_RDDE, lpi2c_imx->base + LPI2C_MDER);
 983	} else {
 984		/* Set I2C TX watermark */
 985		writel(dma->tx_burst_num, lpi2c_imx->base + LPI2C_MFCR);
 986		/* Enable I2C DMA TX function */
 987		writel(MDER_TDDE, lpi2c_imx->base + LPI2C_MDER);
 988	}
 989
 990	/* Enable NACK detected */
 991	lpi2c_imx_intctrl(lpi2c_imx, MIER_NDIE);
 992};
 993
 994/*
 995 * When lpi2c is in TX DMA mode we can use one DMA TX channel to write
 996 * data word into TXFIFO, but in RX DMA mode it is different.
 997 *
 998 * The LPI2C MTDR register is a command data and transmit data register.
 999 * Bits 8-10 are the command data field and Bits 0-7 are the transmit
1000 * data field. When the LPI2C master needs to read data, the number of
1001 * bytes to read should be set in the command field and RECV_DATA should
1002 * be set into the command data field to receive (DATA[7:0] + 1) bytes.
1003 * The recv data command word is made of RECV_DATA in the command data
1004 * field and the number of bytes to read in transmit data field. When the
1005 * length of data to be read exceeds 256 bytes, recv data command word
1006 * needs to be written to TXFIFO multiple times.
1007 *
1008 * So when in RX DMA mode, the TX channel also must to be configured to
1009 * send RX command words and the RX command word must be set in advance
1010 * before transmitting.
1011 */
1012static int lpi2c_imx_dma_xfer(struct lpi2c_imx_struct *lpi2c_imx,
1013			      struct i2c_msg *msg)
1014{
1015	struct lpi2c_imx_dma *dma = lpi2c_imx->dma;
1016	int ret;
1017
1018	/* When DMA mode fails before transferring, CPU mode can be used. */
1019	dma->using_pio_mode = true;
1020
1021	dma->dma_len = msg->len;
1022	dma->dma_msg_flag = msg->flags;
1023	dma->dma_buf = i2c_get_dma_safe_msg_buf(msg, I2C_DMA_THRESHOLD);
1024	if (!dma->dma_buf)
1025		return -ENOMEM;
1026
1027	ret = lpi2c_dma_config(lpi2c_imx);
1028	if (ret) {
1029		dev_err(&lpi2c_imx->adapter.dev, "Failed to configure DMA (%d)\n", ret);
1030		goto disable_dma;
1031	}
1032
1033	lpi2c_dma_enable(lpi2c_imx);
1034
1035	ret = lpi2c_dma_submit(lpi2c_imx);
1036	if (ret) {
1037		dev_err(&lpi2c_imx->adapter.dev, "DMA submission failed (%d)\n", ret);
1038		goto disable_dma;
1039	}
1040
1041	if (dma->dma_msg_flag & I2C_M_RD) {
1042		ret = lpi2c_imx_alloc_rx_cmd_buf(lpi2c_imx);
1043		if (ret)
1044			goto disable_cleanup_data_dma;
1045
1046		ret = lpi2c_dma_rx_cmd_submit(lpi2c_imx);
1047		if (ret)
1048			goto disable_cleanup_data_dma;
1049	}
1050
1051	ret = lpi2c_imx_dma_msg_complete(lpi2c_imx);
1052	if (ret)
1053		goto disable_cleanup_all_dma;
1054
1055	/* When encountering NACK in transfer, clean up all DMA transfers */
1056	if ((readl(lpi2c_imx->base + LPI2C_MSR) & MSR_NDF) && !ret) {
1057		ret = -EIO;
1058		goto disable_cleanup_all_dma;
1059	}
1060
1061	if (dma->dma_msg_flag & I2C_M_RD)
1062		dma_unmap_single(dma->chan_tx->device->dev, dma->dma_tx_addr,
1063				 dma->rx_cmd_buf_len, DMA_TO_DEVICE);
1064	lpi2c_dma_unmap(dma);
1065
1066	goto disable_dma;
1067
1068disable_cleanup_all_dma:
1069	if (dma->dma_msg_flag & I2C_M_RD)
1070		lpi2c_cleanup_rx_cmd_dma(dma);
1071disable_cleanup_data_dma:
1072	lpi2c_cleanup_dma(dma);
1073disable_dma:
1074	/* Disable I2C DMA function */
1075	writel(0, lpi2c_imx->base + LPI2C_MDER);
1076
1077	if (dma->dma_msg_flag & I2C_M_RD)
1078		kfree(dma->rx_cmd_buf);
1079
1080	if (ret)
1081		i2c_put_dma_safe_msg_buf(dma->dma_buf, msg, false);
1082	else
1083		i2c_put_dma_safe_msg_buf(dma->dma_buf, msg, true);
1084
1085	return ret;
1086}
1087
1088static int lpi2c_imx_xfer_common(struct i2c_adapter *adapter,
1089				 struct i2c_msg *msgs, int num, bool atomic)
1090{
1091	struct lpi2c_imx_struct *lpi2c_imx = i2c_get_adapdata(adapter);
1092	unsigned int temp;
1093	int i, result;
1094
1095	result = lpi2c_imx_master_enable(lpi2c_imx);
1096	if (result)
1097		return result;
1098
1099	for (i = 0; i < num; i++) {
1100		result = lpi2c_imx_start(lpi2c_imx, &msgs[i], atomic);
1101		if (result)
1102			goto disable;
1103
1104		/* quick smbus */
1105		if (num == 1 && msgs[0].len == 0)
1106			goto stop;
1107
1108		lpi2c_imx->rx_buf = NULL;
1109		lpi2c_imx->tx_buf = NULL;
1110		lpi2c_imx->delivered = 0;
1111		lpi2c_imx->msglen = msgs[i].len;
1112
1113		if (atomic) {
1114			result = lpi2c_imx_pio_xfer_atomic(lpi2c_imx, &msgs[i]);
1115		} else {
1116			init_completion(&lpi2c_imx->complete);
1117
1118			if (is_use_dma(lpi2c_imx, &msgs[i])) {
1119				result = lpi2c_imx_dma_xfer(lpi2c_imx, &msgs[i]);
1120				if (result && lpi2c_imx->dma->using_pio_mode)
1121					result = lpi2c_imx_pio_xfer(lpi2c_imx, &msgs[i]);
1122			} else {
1123				result = lpi2c_imx_pio_xfer(lpi2c_imx, &msgs[i]);
1124			}
1125		}
1126
1127		if (result)
1128			goto stop;
1129
1130		if (!(msgs[i].flags & I2C_M_RD)) {
1131			result = lpi2c_imx_txfifo_empty(lpi2c_imx, atomic);
1132			if (result)
1133				goto stop;
1134		}
1135	}
1136
1137stop:
1138	lpi2c_imx_stop(lpi2c_imx, atomic);
1139
1140	temp = readl(lpi2c_imx->base + LPI2C_MSR);
1141	if ((temp & MSR_NDF) && !result)
1142		result = -EIO;
1143
1144disable:
1145	lpi2c_imx_master_disable(lpi2c_imx);
1146
1147	dev_dbg(&lpi2c_imx->adapter.dev, "<%s> exit with: %s: %d\n", __func__,
1148		(result < 0) ? "error" : "success msg",
1149		(result < 0) ? result : num);
1150
1151	return (result < 0) ? result : num;
1152}
1153
1154static int lpi2c_imx_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
1155{
1156	return lpi2c_imx_xfer_common(adapter, msgs, num, false);
1157}
1158
1159static int lpi2c_imx_xfer_atomic(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
1160{
1161	return lpi2c_imx_xfer_common(adapter, msgs, num, true);
1162}
1163
1164static irqreturn_t lpi2c_imx_target_isr(struct lpi2c_imx_struct *lpi2c_imx,
1165					u32 ssr, u32 sier_filter)
1166{
1167	u8 value;
1168	u32 sasr;
1169
1170	/* Arbitration lost */
1171	if (sier_filter & SSR_BEF) {
1172		writel(0, lpi2c_imx->base + LPI2C_SIER);
1173		return IRQ_HANDLED;
1174	}
1175
1176	/* Address detected */
1177	if (sier_filter & SSR_AVF) {
1178		sasr = readl(lpi2c_imx->base + LPI2C_SASR);
1179		if (SASR_READ_REQ & sasr) {
1180			/* Read request */
1181			i2c_slave_event(lpi2c_imx->target, I2C_SLAVE_READ_REQUESTED, &value);
1182			writel(value, lpi2c_imx->base + LPI2C_STDR);
1183			goto ret;
1184		} else {
1185			/* Write request */
1186			i2c_slave_event(lpi2c_imx->target, I2C_SLAVE_WRITE_REQUESTED, &value);
1187		}
1188	}
1189
1190	if (sier_filter & SSR_SDF)
1191		/* STOP */
1192		i2c_slave_event(lpi2c_imx->target, I2C_SLAVE_STOP, &value);
1193
1194	if (sier_filter & SSR_TDF) {
1195		/* Target send data */
1196		i2c_slave_event(lpi2c_imx->target, I2C_SLAVE_READ_PROCESSED, &value);
1197		writel(value, lpi2c_imx->base + LPI2C_STDR);
1198	}
1199
1200	if (sier_filter & SSR_RDF) {
1201		/* Target receive data */
1202		value = readl(lpi2c_imx->base + LPI2C_SRDR);
1203		i2c_slave_event(lpi2c_imx->target, I2C_SLAVE_WRITE_RECEIVED, &value);
1204	}
1205
1206ret:
1207	/* Clear SSR */
1208	writel(ssr & SSR_CLEAR_BITS, lpi2c_imx->base + LPI2C_SSR);
1209	return IRQ_HANDLED;
1210}
1211
1212static irqreturn_t lpi2c_imx_master_isr(struct lpi2c_imx_struct *lpi2c_imx)
1213{
1214	unsigned int enabled;
1215	unsigned int temp;
1216
1217	enabled = readl(lpi2c_imx->base + LPI2C_MIER);
1218
1219	lpi2c_imx_intctrl(lpi2c_imx, 0);
1220	temp = readl(lpi2c_imx->base + LPI2C_MSR);
1221	temp &= enabled;
1222
1223	if (temp & MSR_NDF)
1224		complete(&lpi2c_imx->complete);
1225	else if (temp & MSR_RDF)
1226		lpi2c_imx_read_rxfifo(lpi2c_imx, false);
1227	else if (temp & MSR_TDF)
1228		lpi2c_imx_write_txfifo(lpi2c_imx, false);
1229
1230	return IRQ_HANDLED;
1231}
1232
1233static irqreturn_t lpi2c_imx_isr(int irq, void *dev_id)
1234{
1235	struct lpi2c_imx_struct *lpi2c_imx = dev_id;
1236
1237	if (lpi2c_imx->target) {
1238		u32 scr = readl(lpi2c_imx->base + LPI2C_SCR);
1239		u32 ssr = readl(lpi2c_imx->base + LPI2C_SSR);
1240		u32 sier_filter = ssr & readl(lpi2c_imx->base + LPI2C_SIER);
1241
1242		/*
1243		 * The target is enabled and an interrupt has been triggered.
1244		 * Enter the target's irq handler.
1245		 */
1246		if ((scr & SCR_SEN) && sier_filter)
1247			return lpi2c_imx_target_isr(lpi2c_imx, ssr, sier_filter);
1248	}
1249
1250	/*
1251	 * Otherwise the interrupt has been triggered by the master.
1252	 * Enter the master's irq handler.
1253	 */
1254	return lpi2c_imx_master_isr(lpi2c_imx);
1255}
1256
1257static void lpi2c_imx_target_init(struct lpi2c_imx_struct *lpi2c_imx)
1258{
1259	u32 temp;
1260
1261	/* reset target module */
1262	writel(SCR_RST, lpi2c_imx->base + LPI2C_SCR);
1263	writel(0, lpi2c_imx->base + LPI2C_SCR);
1264
1265	/* Set target address */
1266	writel((lpi2c_imx->target->addr << 1), lpi2c_imx->base + LPI2C_SAMR);
1267
1268	writel(SCFGR1_RXSTALL | SCFGR1_TXDSTALL, lpi2c_imx->base + LPI2C_SCFGR1);
1269
1270	/*
1271	 * set SCFGR2: FILTSDA, FILTSCL and CLKHOLD
1272	 *
1273	 * FILTSCL/FILTSDA can eliminate signal skew. It should generally be
1274	 * set to the same value and should be set >= 50ns.
1275	 *
1276	 * CLKHOLD is only used when clock stretching is enabled, but it will
1277	 * extend the clock stretching to ensure there is an additional delay
1278	 * between the target driving SDA and the target releasing the SCL pin.
1279	 *
1280	 * CLKHOLD setting is crucial for lpi2c target. When master read data
1281	 * from target, if there is a delay caused by cpu idle, excessive load,
1282	 * or other delays between two bytes in one message transmission, it
1283	 * will cause a short interval time between the driving SDA signal and
1284	 * releasing SCL signal. The lpi2c master will mistakenly think it is a stop
1285	 * signal resulting in an arbitration failure. This issue can be avoided
1286	 * by setting CLKHOLD.
1287	 *
1288	 * In order to ensure lpi2c function normally when the lpi2c speed is as
1289	 * low as 100kHz, CLKHOLD should be set to 3 and it is also compatible with
1290	 * higher clock frequency like 400kHz and 1MHz.
1291	 */
1292	temp = SCFGR2_FILTSDA(2) | SCFGR2_FILTSCL(2) | SCFGR2_CLKHOLD(3);
1293	writel(temp, lpi2c_imx->base + LPI2C_SCFGR2);
1294
1295	/*
1296	 * Enable module:
1297	 * SCR_FILTEN can enable digital filter and output delay counter for LPI2C
1298	 * target mode. So SCR_FILTEN need be asserted when enable SDA/SCL FILTER
1299	 * and CLKHOLD.
1300	 */
1301	writel(SCR_SEN | SCR_FILTEN, lpi2c_imx->base + LPI2C_SCR);
1302
1303	/* Enable interrupt from i2c module */
1304	writel(SLAVE_INT_FLAG, lpi2c_imx->base + LPI2C_SIER);
1305}
1306
1307static int lpi2c_imx_register_target(struct i2c_client *client)
1308{
1309	struct lpi2c_imx_struct *lpi2c_imx = i2c_get_adapdata(client->adapter);
1310	int ret;
1311
1312	if (lpi2c_imx->target)
1313		return -EBUSY;
1314
1315	lpi2c_imx->target = client;
1316
1317	ret = pm_runtime_resume_and_get(lpi2c_imx->adapter.dev.parent);
1318	if (ret < 0) {
1319		dev_err(&lpi2c_imx->adapter.dev, "failed to resume i2c controller");
1320		return ret;
1321	}
1322
1323	lpi2c_imx_target_init(lpi2c_imx);
1324
1325	return 0;
1326}
1327
1328static int lpi2c_imx_unregister_target(struct i2c_client *client)
1329{
1330	struct lpi2c_imx_struct *lpi2c_imx = i2c_get_adapdata(client->adapter);
1331	int ret;
1332
1333	if (!lpi2c_imx->target)
1334		return -EINVAL;
1335
1336	/* Reset target address. */
1337	writel(0, lpi2c_imx->base + LPI2C_SAMR);
1338
1339	writel(SCR_RST, lpi2c_imx->base + LPI2C_SCR);
1340	writel(0, lpi2c_imx->base + LPI2C_SCR);
1341
1342	lpi2c_imx->target = NULL;
1343
1344	ret = pm_runtime_put_sync(lpi2c_imx->adapter.dev.parent);
1345	if (ret < 0)
1346		dev_err(&lpi2c_imx->adapter.dev, "failed to suspend i2c controller");
1347
1348	return ret;
1349}
1350
1351static int lpi2c_imx_init_recovery_info(struct lpi2c_imx_struct *lpi2c_imx,
1352				  struct platform_device *pdev)
1353{
1354	struct i2c_bus_recovery_info *bri = &lpi2c_imx->rinfo;
1355
1356	bri->pinctrl = devm_pinctrl_get(&pdev->dev);
1357	if (IS_ERR(bri->pinctrl))
1358		return PTR_ERR(bri->pinctrl);
1359
1360	lpi2c_imx->adapter.bus_recovery_info = bri;
1361
1362	return 0;
1363}
1364
1365static void dma_exit(struct device *dev, struct lpi2c_imx_dma *dma)
1366{
1367	if (dma->chan_rx)
1368		dma_release_channel(dma->chan_rx);
1369
1370	if (dma->chan_tx)
1371		dma_release_channel(dma->chan_tx);
1372
1373	devm_kfree(dev, dma);
1374}
1375
1376static int lpi2c_dma_init(struct device *dev, dma_addr_t phy_addr)
1377{
1378	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
1379	struct lpi2c_imx_dma *dma;
1380	int ret;
1381
1382	dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL);
1383	if (!dma)
1384		return -ENOMEM;
1385
1386	dma->phy_addr = phy_addr;
1387
1388	/* Prepare for TX DMA: */
1389	dma->chan_tx = dma_request_chan(dev, "tx");
1390	if (IS_ERR(dma->chan_tx)) {
1391		ret = PTR_ERR(dma->chan_tx);
1392		if (ret != -ENODEV && ret != -EPROBE_DEFER)
1393			dev_err(dev, "can't request DMA tx channel (%d)\n", ret);
1394		dma->chan_tx = NULL;
1395		goto dma_exit;
1396	}
1397
1398	/* Prepare for RX DMA: */
1399	dma->chan_rx = dma_request_chan(dev, "rx");
1400	if (IS_ERR(dma->chan_rx)) {
1401		ret = PTR_ERR(dma->chan_rx);
1402		if (ret != -ENODEV && ret != -EPROBE_DEFER)
1403			dev_err(dev, "can't request DMA rx channel (%d)\n", ret);
1404		dma->chan_rx = NULL;
1405		goto dma_exit;
1406	}
1407
1408	lpi2c_imx->can_use_dma = true;
1409	lpi2c_imx->dma = dma;
1410	return 0;
1411
1412dma_exit:
1413	dma_exit(dev, dma);
1414	return ret;
1415}
1416
1417static u32 lpi2c_imx_func(struct i2c_adapter *adapter)
1418{
1419	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
1420		I2C_FUNC_SMBUS_READ_BLOCK_DATA;
1421}
1422
1423static const struct i2c_algorithm lpi2c_imx_algo = {
1424	.xfer = lpi2c_imx_xfer,
1425	.xfer_atomic = lpi2c_imx_xfer_atomic,
1426	.functionality = lpi2c_imx_func,
1427	.reg_target = lpi2c_imx_register_target,
1428	.unreg_target = lpi2c_imx_unregister_target,
1429};
1430
1431static const struct of_device_id lpi2c_imx_of_match[] = {
1432	{ .compatible = "fsl,imx7ulp-lpi2c" },
1433	{ }
1434};
1435MODULE_DEVICE_TABLE(of, lpi2c_imx_of_match);
1436
1437static int lpi2c_imx_probe(struct platform_device *pdev)
1438{
1439	struct lpi2c_imx_struct *lpi2c_imx;
1440	struct resource *res;
1441	dma_addr_t phy_addr;
1442	unsigned int temp;
1443	int irq, ret;
1444
1445	lpi2c_imx = devm_kzalloc(&pdev->dev, sizeof(*lpi2c_imx), GFP_KERNEL);
1446	if (!lpi2c_imx)
1447		return -ENOMEM;
1448
1449	lpi2c_imx->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1450	if (IS_ERR(lpi2c_imx->base))
1451		return PTR_ERR(lpi2c_imx->base);
1452
1453	irq = platform_get_irq(pdev, 0);
1454	if (irq < 0)
1455		return irq;
1456
1457	lpi2c_imx->adapter.owner	= THIS_MODULE;
1458	lpi2c_imx->adapter.algo		= &lpi2c_imx_algo;
1459	lpi2c_imx->adapter.dev.parent	= &pdev->dev;
1460	lpi2c_imx->adapter.dev.of_node	= pdev->dev.of_node;
1461	strscpy(lpi2c_imx->adapter.name, pdev->name,
1462		sizeof(lpi2c_imx->adapter.name));
1463	phy_addr = (dma_addr_t)res->start;
1464
1465	ret = devm_clk_bulk_get_all(&pdev->dev, &lpi2c_imx->clks);
1466	if (ret < 0)
1467		return dev_err_probe(&pdev->dev, ret, "can't get I2C peripheral clock\n");
1468	lpi2c_imx->num_clks = ret;
1469
1470	ret = of_property_read_u32(pdev->dev.of_node,
1471				   "clock-frequency", &lpi2c_imx->bitrate);
1472	if (ret)
1473		lpi2c_imx->bitrate = I2C_MAX_STANDARD_MODE_FREQ;
1474
1475	ret = devm_request_irq(&pdev->dev, irq, lpi2c_imx_isr, IRQF_NO_SUSPEND,
1476			       pdev->name, lpi2c_imx);
1477	if (ret)
1478		return dev_err_probe(&pdev->dev, ret, "can't claim irq %d\n", irq);
1479
1480	i2c_set_adapdata(&lpi2c_imx->adapter, lpi2c_imx);
1481	platform_set_drvdata(pdev, lpi2c_imx);
1482
1483	ret = clk_bulk_prepare_enable(lpi2c_imx->num_clks, lpi2c_imx->clks);
1484	if (ret)
1485		return ret;
1486
1487	/*
1488	 * Lock the parent clock rate to avoid getting parent clock upon
1489	 * each transfer
1490	 */
1491	ret = devm_clk_rate_exclusive_get(&pdev->dev, lpi2c_imx->clks[0].clk);
1492	if (ret)
1493		return dev_err_probe(&pdev->dev, ret,
1494				     "can't lock I2C peripheral clock rate\n");
1495
1496	lpi2c_imx->rate_per = clk_get_rate(lpi2c_imx->clks[0].clk);
1497	if (!lpi2c_imx->rate_per)
1498		return dev_err_probe(&pdev->dev, -EINVAL,
1499				     "can't get I2C peripheral clock rate\n");
1500
1501	pm_runtime_set_autosuspend_delay(&pdev->dev, I2C_PM_TIMEOUT);
1502	pm_runtime_use_autosuspend(&pdev->dev);
1503	pm_runtime_get_noresume(&pdev->dev);
1504	pm_runtime_set_active(&pdev->dev);
1505	pm_runtime_enable(&pdev->dev);
1506
1507	temp = readl(lpi2c_imx->base + LPI2C_PARAM);
1508	lpi2c_imx->txfifosize = 1 << (temp & 0x0f);
1509	lpi2c_imx->rxfifosize = 1 << ((temp >> 8) & 0x0f);
1510
1511	/* Init optional bus recovery function */
1512	ret = lpi2c_imx_init_recovery_info(lpi2c_imx, pdev);
1513	/* Give it another chance if pinctrl used is not ready yet */
1514	if (ret == -EPROBE_DEFER)
1515		goto rpm_disable;
1516
1517	/* Init DMA */
1518	ret = lpi2c_dma_init(&pdev->dev, phy_addr);
1519	if (ret) {
1520		if (ret == -EPROBE_DEFER)
1521			goto rpm_disable;
1522		dev_info(&pdev->dev, "use pio mode\n");
1523	}
1524
1525	ret = i2c_add_adapter(&lpi2c_imx->adapter);
1526	if (ret)
1527		goto rpm_disable;
1528
1529	pm_runtime_put_autosuspend(&pdev->dev);
1530
1531	dev_info(&lpi2c_imx->adapter.dev, "LPI2C adapter registered\n");
1532
1533	return 0;
1534
1535rpm_disable:
1536	pm_runtime_dont_use_autosuspend(&pdev->dev);
1537	pm_runtime_put_sync(&pdev->dev);
1538	pm_runtime_disable(&pdev->dev);
1539
1540	return ret;
1541}
1542
1543static void lpi2c_imx_remove(struct platform_device *pdev)
1544{
1545	struct lpi2c_imx_struct *lpi2c_imx = platform_get_drvdata(pdev);
1546
1547	i2c_del_adapter(&lpi2c_imx->adapter);
1548
1549	pm_runtime_disable(&pdev->dev);
1550	pm_runtime_dont_use_autosuspend(&pdev->dev);
1551}
1552
1553static int __maybe_unused lpi2c_runtime_suspend(struct device *dev)
1554{
1555	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
1556
1557	clk_bulk_disable(lpi2c_imx->num_clks, lpi2c_imx->clks);
1558	pinctrl_pm_select_sleep_state(dev);
1559
1560	return 0;
1561}
1562
1563static int __maybe_unused lpi2c_runtime_resume(struct device *dev)
1564{
1565	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
1566	int ret;
1567
1568	pinctrl_pm_select_default_state(dev);
1569	ret = clk_bulk_enable(lpi2c_imx->num_clks, lpi2c_imx->clks);
1570	if (ret) {
1571		dev_err(dev, "failed to enable I2C clock, ret=%d\n", ret);
1572		return ret;
1573	}
1574
1575	return 0;
1576}
1577
1578static int __maybe_unused lpi2c_suspend_noirq(struct device *dev)
1579{
1580	return pm_runtime_force_suspend(dev);
1581}
1582
1583static int __maybe_unused lpi2c_resume_noirq(struct device *dev)
1584{
1585	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
1586	int ret;
1587
1588	ret = pm_runtime_force_resume(dev);
1589	if (ret)
1590		return ret;
1591
1592	/*
1593	 * If the I2C module powers down during system suspend,
1594	 * the register values will be lost. Therefore, reinitialize
1595	 * the target when the system resumes.
1596	 */
1597	if (lpi2c_imx->target)
1598		lpi2c_imx_target_init(lpi2c_imx);
1599
1600	return 0;
1601}
1602
1603static int lpi2c_suspend(struct device *dev)
1604{
1605	/*
1606	 * Some I2C devices may need the I2C controller to remain active
1607	 * during resume_noirq() or suspend_noirq(). If the controller is
1608	 * autosuspended, there is no way to wake it up once runtime PM is
1609	 * disabled (in suspend_late()).
1610	 *
1611	 * During system resume, the I2C controller will be available only
1612	 * after runtime PM is re-enabled (in resume_early()). However, this
1613	 * may be too late for some devices.
1614	 *
1615	 * Wake up the controller in the suspend() callback while runtime PM
1616	 * is still enabled. The I2C controller will remain available until
1617	 * the suspend_noirq() callback (pm_runtime_force_suspend()) is
1618	 * called. During resume, the I2C controller can be restored by the
1619	 * resume_noirq() callback (pm_runtime_force_resume()).
1620	 *
1621	 * Finally, the resume() callback re-enables autosuspend, ensuring
1622	 * the I2C controller remains available until the system enters
1623	 * suspend_noirq() and from resume_noirq().
1624	 */
1625	return pm_runtime_resume_and_get(dev);
1626}
1627
1628static int lpi2c_resume(struct device *dev)
1629{
1630	pm_runtime_put_autosuspend(dev);
1631
1632	return 0;
1633}
1634
1635static const struct dev_pm_ops lpi2c_pm_ops = {
1636	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(lpi2c_suspend_noirq,
1637				      lpi2c_resume_noirq)
1638	SYSTEM_SLEEP_PM_OPS(lpi2c_suspend, lpi2c_resume)
1639	SET_RUNTIME_PM_OPS(lpi2c_runtime_suspend,
1640			   lpi2c_runtime_resume, NULL)
1641};
1642
1643static struct platform_driver lpi2c_imx_driver = {
1644	.probe = lpi2c_imx_probe,
1645	.remove = lpi2c_imx_remove,
1646	.driver = {
1647		.name = DRIVER_NAME,
1648		.of_match_table = lpi2c_imx_of_match,
1649		.pm = &lpi2c_pm_ops,
1650	},
1651};
1652
1653module_platform_driver(lpi2c_imx_driver);
1654
1655MODULE_AUTHOR("Gao Pan <pandy.gao@nxp.com>");
1656MODULE_DESCRIPTION("I2C adapter driver for LPI2C bus");
1657MODULE_LICENSE("GPL");