fsl_lpuart.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 *  Freescale lpuart serial port driver
   4 *
   5 *  Copyright 2012-2014 Freescale Semiconductor, Inc.
   6 */
   7
   8#include <linux/bitfield.h>
   9#include <linux/bits.h>
  10#include <linux/circ_buf.h>
  11#include <linux/clk.h>
  12#include <linux/console.h>
  13#include <linux/delay.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/dmaengine.h>
  16#include <linux/dmapool.h>
  17#include <linux/io.h>
  18#include <linux/iopoll.h>
  19#include <linux/irq.h>
  20#include <linux/module.h>
  21#include <linux/of.h>
  22#include <linux/of_dma.h>
  23#include <linux/pinctrl/consumer.h>
  24#include <linux/platform_device.h>
  25#include <linux/pm_runtime.h>
  26#include <linux/serial_core.h>
  27#include <linux/slab.h>
  28#include <linux/tty_flip.h>
  29
  30/* All registers are 8-bit width */
  31#define UARTBDH			0x00
  32#define UARTBDL			0x01
  33#define UARTCR1			0x02
  34#define UARTCR2			0x03
  35#define UARTSR1			0x04
  36#define UARTCR3			0x06
  37#define UARTDR			0x07
  38#define UARTCR4			0x0a
  39#define UARTCR5			0x0b
  40#define UARTMODEM		0x0d
  41#define UARTPFIFO		0x10
  42#define UARTCFIFO		0x11
  43#define UARTSFIFO		0x12
  44#define UARTTWFIFO		0x13
  45#define UARTTCFIFO		0x14
  46#define UARTRWFIFO		0x15
  47
  48#define UARTBDH_LBKDIE		0x80
  49#define UARTBDH_RXEDGIE		0x40
  50#define UARTBDH_SBR_MASK	0x1f
  51
  52#define UARTCR1_LOOPS		0x80
  53#define UARTCR1_RSRC		0x20
  54#define UARTCR1_M		0x10
  55#define UARTCR1_WAKE		0x08
  56#define UARTCR1_ILT		0x04
  57#define UARTCR1_PE		0x02
  58#define UARTCR1_PT		0x01
  59
  60#define UARTCR2_TIE		0x80
  61#define UARTCR2_TCIE		0x40
  62#define UARTCR2_RIE		0x20
  63#define UARTCR2_ILIE		0x10
  64#define UARTCR2_TE		0x08
  65#define UARTCR2_RE		0x04
  66#define UARTCR2_RWU		0x02
  67#define UARTCR2_SBK		0x01
  68
  69#define UARTSR1_TDRE		0x80
  70#define UARTSR1_TC		0x40
  71#define UARTSR1_RDRF		0x20
  72#define UARTSR1_IDLE		0x10
  73#define UARTSR1_OR		0x08
  74#define UARTSR1_NF		0x04
  75#define UARTSR1_FE		0x02
  76#define UARTSR1_PE		0x01
  77
  78#define UARTCR3_R8		0x80
  79#define UARTCR3_T8		0x40
  80#define UARTCR3_TXDIR		0x20
  81#define UARTCR3_TXINV		0x10
  82#define UARTCR3_ORIE		0x08
  83#define UARTCR3_NEIE		0x04
  84#define UARTCR3_FEIE		0x02
  85#define UARTCR3_PEIE		0x01
  86
  87#define UARTCR4_MAEN1		0x80
  88#define UARTCR4_MAEN2		0x40
  89#define UARTCR4_M10		0x20
  90#define UARTCR4_BRFA_MASK	0x1f
  91#define UARTCR4_BRFA_OFF	0
  92
  93#define UARTCR5_TDMAS		0x80
  94#define UARTCR5_RDMAS		0x20
  95
  96#define UARTMODEM_RXRTSE	0x08
  97#define UARTMODEM_TXRTSPOL	0x04
  98#define UARTMODEM_TXRTSE	0x02
  99#define UARTMODEM_TXCTSE	0x01
 100
 101#define UARTPFIFO_TXFE		0x80
 102#define UARTPFIFO_FIFOSIZE_MASK	0x7
 103#define UARTPFIFO_TXSIZE_OFF	4
 104#define UARTPFIFO_RXFE		0x08
 105#define UARTPFIFO_RXSIZE_OFF	0
 106
 107#define UARTCFIFO_TXFLUSH	0x80
 108#define UARTCFIFO_RXFLUSH	0x40
 109#define UARTCFIFO_RXOFE		0x04
 110#define UARTCFIFO_TXOFE		0x02
 111#define UARTCFIFO_RXUFE		0x01
 112
 113#define UARTSFIFO_TXEMPT	0x80
 114#define UARTSFIFO_RXEMPT	0x40
 115#define UARTSFIFO_RXOF		0x04
 116#define UARTSFIFO_TXOF		0x02
 117#define UARTSFIFO_RXUF		0x01
 118
 119/* 32-bit global registers only for i.MX7ULP/i.MX8x
 120 * Used to reset all internal logic and registers, except the Global Register.
 121 */
 122#define UART_GLOBAL		0x8
 123
 124/* 32-bit register definition */
 125#define UARTBAUD		0x00
 126#define UARTSTAT		0x04
 127#define UARTCTRL		0x08
 128#define UARTDATA		0x0C
 129#define UARTMATCH		0x10
 130#define UARTMODIR		0x14
 131#define UARTFIFO		0x18
 132#define UARTWATER		0x1c
 133
 134#define UARTBAUD_MAEN1		0x80000000
 135#define UARTBAUD_MAEN2		0x40000000
 136#define UARTBAUD_M10		0x20000000
 137#define UARTBAUD_TDMAE		0x00800000
 138#define UARTBAUD_RDMAE		0x00200000
 139#define UARTBAUD_MATCFG		0x00400000
 140#define UARTBAUD_BOTHEDGE	0x00020000
 141#define UARTBAUD_RESYNCDIS	0x00010000
 142#define UARTBAUD_LBKDIE		0x00008000
 143#define UARTBAUD_RXEDGIE	0x00004000
 144#define UARTBAUD_SBNS		0x00002000
 145#define UARTBAUD_SBR		0x00000000
 146#define UARTBAUD_SBR_MASK	0x1fff
 147#define UARTBAUD_OSR_MASK       0x1f
 148#define UARTBAUD_OSR_SHIFT      24
 149
 150#define UARTSTAT_LBKDIF		0x80000000
 151#define UARTSTAT_RXEDGIF	0x40000000
 152#define UARTSTAT_MSBF		0x20000000
 153#define UARTSTAT_RXINV		0x10000000
 154#define UARTSTAT_RWUID		0x08000000
 155#define UARTSTAT_BRK13		0x04000000
 156#define UARTSTAT_LBKDE		0x02000000
 157#define UARTSTAT_RAF		0x01000000
 158#define UARTSTAT_TDRE		0x00800000
 159#define UARTSTAT_TC		0x00400000
 160#define UARTSTAT_RDRF		0x00200000
 161#define UARTSTAT_IDLE		0x00100000
 162#define UARTSTAT_OR		0x00080000
 163#define UARTSTAT_NF		0x00040000
 164#define UARTSTAT_FE		0x00020000
 165#define UARTSTAT_PE		0x00010000
 166#define UARTSTAT_MA1F		0x00008000
 167#define UARTSTAT_M21F		0x00004000
 168
 169#define UARTCTRL_R8T9		0x80000000
 170#define UARTCTRL_R9T8		0x40000000
 171#define UARTCTRL_TXDIR		0x20000000
 172#define UARTCTRL_TXINV		0x10000000
 173#define UARTCTRL_ORIE		0x08000000
 174#define UARTCTRL_NEIE		0x04000000
 175#define UARTCTRL_FEIE		0x02000000
 176#define UARTCTRL_PEIE		0x01000000
 177#define UARTCTRL_TIE		0x00800000
 178#define UARTCTRL_TCIE		0x00400000
 179#define UARTCTRL_RIE		0x00200000
 180#define UARTCTRL_ILIE		0x00100000
 181#define UARTCTRL_TE		0x00080000
 182#define UARTCTRL_RE		0x00040000
 183#define UARTCTRL_RWU		0x00020000
 184#define UARTCTRL_SBK		0x00010000
 185#define UARTCTRL_MA1IE		0x00008000
 186#define UARTCTRL_MA2IE		0x00004000
 187#define UARTCTRL_M7		0x00000800
 188#define UARTCTRL_IDLECFG	GENMASK(10, 8)
 189#define UARTCTRL_LOOPS		0x00000080
 190#define UARTCTRL_DOZEEN		0x00000040
 191#define UARTCTRL_RSRC		0x00000020
 192#define UARTCTRL_M		0x00000010
 193#define UARTCTRL_WAKE		0x00000008
 194#define UARTCTRL_ILT		0x00000004
 195#define UARTCTRL_PE		0x00000002
 196#define UARTCTRL_PT		0x00000001
 197
 198#define UARTDATA_NOISY		0x00008000
 199#define UARTDATA_PARITYE	0x00004000
 200#define UARTDATA_FRETSC		0x00002000
 201#define UARTDATA_RXEMPT		0x00001000
 202#define UARTDATA_IDLINE		0x00000800
 203#define UARTDATA_MASK		0x3ff
 204
 205#define UARTMODIR_IREN		0x00020000
 206#define UARTMODIR_RTSWATER	GENMASK(10, 8)
 207#define UARTMODIR_TXCTSSRC	0x00000020
 208#define UARTMODIR_TXCTSC	0x00000010
 209#define UARTMODIR_RXRTSE	0x00000008
 210#define UARTMODIR_TXRTSPOL	0x00000004
 211#define UARTMODIR_TXRTSE	0x00000002
 212#define UARTMODIR_TXCTSE	0x00000001
 213
 214#define UARTFIFO_TXEMPT		0x00800000
 215#define UARTFIFO_RXEMPT		0x00400000
 216#define UARTFIFO_TXOF		0x00020000
 217#define UARTFIFO_RXUF		0x00010000
 218#define UARTFIFO_TXFLUSH	0x00008000
 219#define UARTFIFO_RXFLUSH	0x00004000
 220#define UARTFIFO_RXIDEN	GENMASK(12, 10)
 221#define UARTFIFO_TXOFE		0x00000200
 222#define UARTFIFO_RXUFE		0x00000100
 223#define UARTFIFO_TXFE		0x00000080
 224#define UARTFIFO_FIFOSIZE_MASK	0x7
 225#define UARTFIFO_TXSIZE_OFF	4
 226#define UARTFIFO_RXFE		0x00000008
 227#define UARTFIFO_RXSIZE_OFF	0
 228#define UARTFIFO_DEPTH(x)	(0x1 << ((x) ? ((x) + 1) : 0))
 229
 230#define UARTWATER_COUNT_MASK	0xff
 231#define UARTWATER_TXCNT_OFF	8
 232#define UARTWATER_RXCNT_OFF	24
 233#define UARTWATER_WATER_MASK	0xff
 234#define UARTWATER_TXWATER_OFF	0
 235#define UARTWATER_RXWATER_OFF	16
 236
 237#define UART_GLOBAL_RST	0x2
 238#define GLOBAL_RST_MIN_US	20
 239#define GLOBAL_RST_MAX_US	40
 240
 241/* Rx DMA timeout in ms, which is used to calculate Rx ring buffer size */
 242#define DMA_RX_TIMEOUT		(10)
 243#define DMA_RX_IDLE_CHARS	8
 244#define UART_AUTOSUSPEND_TIMEOUT	3000
 245
 246#define DRIVER_NAME	"fsl-lpuart"
 247#define DEV_NAME	"ttyLP"
 248#define UART_NR		12
 249
 250/* IMX lpuart has four extra unused regs located at the beginning */
 251#define IMX_REG_OFF	0x10
 252
 253enum lpuart_type {
 254	VF610_LPUART,
 255	LS1021A_LPUART,
 256	LS1028A_LPUART,
 257	IMX7ULP_LPUART,
 258	IMX8ULP_LPUART,
 259	IMX8QXP_LPUART,
 260	IMXRT1050_LPUART,
 261};
 262
 263struct lpuart_port {
 264	struct uart_port	port;
 265	enum lpuart_type	devtype;
 266	struct clk		*ipg_clk;
 267	struct clk		*baud_clk;
 268	unsigned int		txfifo_size;
 269	unsigned int		rxfifo_size;
 270
 271	u8			rx_watermark;
 272	bool			lpuart_dma_tx_use;
 273	bool			lpuart_dma_rx_use;
 274	struct dma_chan		*dma_tx_chan;
 275	struct dma_chan		*dma_rx_chan;
 276	struct dma_async_tx_descriptor  *dma_tx_desc;
 277	struct dma_async_tx_descriptor  *dma_rx_desc;
 278	dma_cookie_t		dma_tx_cookie;
 279	dma_cookie_t		dma_rx_cookie;
 280	unsigned int		dma_tx_bytes;
 281	unsigned int		dma_rx_bytes;
 282	bool			dma_tx_in_progress;
 283	unsigned int		dma_rx_timeout;
 284	struct timer_list	lpuart_timer;
 285	struct scatterlist	rx_sgl, tx_sgl[2];
 286	struct circ_buf		rx_ring;
 287	int			rx_dma_rng_buf_len;
 288	int                     last_residue;
 289	unsigned int		dma_tx_nents;
 290	wait_queue_head_t	dma_wait;
 291	bool			is_cs7; /* Set to true when character size is 7 */
 292					/* and the parity is enabled		*/
 293	bool			dma_idle_int;
 294};
 295
 296struct lpuart_soc_data {
 297	enum lpuart_type devtype;
 298	char iotype;
 299	u8 reg_off;
 300	u8 rx_watermark;
 301};
 302
 303static const struct lpuart_soc_data vf_data = {
 304	.devtype = VF610_LPUART,
 305	.iotype = UPIO_MEM,
 306	.rx_watermark = 1,
 307};
 308
 309static const struct lpuart_soc_data ls1021a_data = {
 310	.devtype = LS1021A_LPUART,
 311	.iotype = UPIO_MEM32BE,
 312	.rx_watermark = 1,
 313};
 314
 315static const struct lpuart_soc_data ls1028a_data = {
 316	.devtype = LS1028A_LPUART,
 317	.iotype = UPIO_MEM32,
 318	.rx_watermark = 0,
 319};
 320
 321static const struct lpuart_soc_data imx7ulp_data = {
 322	.devtype = IMX7ULP_LPUART,
 323	.iotype = UPIO_MEM32,
 324	.reg_off = IMX_REG_OFF,
 325	.rx_watermark = 1,
 326};
 327
 328static const struct lpuart_soc_data imx8ulp_data = {
 329	.devtype = IMX8ULP_LPUART,
 330	.iotype = UPIO_MEM32,
 331	.reg_off = IMX_REG_OFF,
 332	.rx_watermark = 3,
 333};
 334
 335static const struct lpuart_soc_data imx8qxp_data = {
 336	.devtype = IMX8QXP_LPUART,
 337	.iotype = UPIO_MEM32,
 338	.reg_off = IMX_REG_OFF,
 339	.rx_watermark = 7, /* A lower watermark is ideal for low baud rates. */
 340};
 341static const struct lpuart_soc_data imxrt1050_data = {
 342	.devtype = IMXRT1050_LPUART,
 343	.iotype = UPIO_MEM32,
 344	.reg_off = IMX_REG_OFF,
 345	.rx_watermark = 1,
 346};
 347
 348static const struct of_device_id lpuart_dt_ids[] = {
 349	{ .compatible = "fsl,vf610-lpuart",	.data = &vf_data, },
 350	{ .compatible = "fsl,ls1021a-lpuart",	.data = &ls1021a_data, },
 351	{ .compatible = "fsl,ls1028a-lpuart",	.data = &ls1028a_data, },
 352	{ .compatible = "fsl,imx7ulp-lpuart",	.data = &imx7ulp_data, },
 353	{ .compatible = "fsl,imx8ulp-lpuart",	.data = &imx8ulp_data, },
 354	{ .compatible = "fsl,imx8qxp-lpuart",	.data = &imx8qxp_data, },
 355	{ .compatible = "fsl,imxrt1050-lpuart",	.data = &imxrt1050_data},
 356	{ /* sentinel */ }
 357};
 358MODULE_DEVICE_TABLE(of, lpuart_dt_ids);
 359
 360/* Forward declare this for the dma callbacks*/
 361static void lpuart_dma_tx_complete(void *arg);
 362
 363static inline bool is_layerscape_lpuart(struct lpuart_port *sport)
 364{
 365	return (sport->devtype == LS1021A_LPUART ||
 366		sport->devtype == LS1028A_LPUART);
 367}
 368
 369static inline bool is_imx7ulp_lpuart(struct lpuart_port *sport)
 370{
 371	return sport->devtype == IMX7ULP_LPUART;
 372}
 373
 374static inline bool is_imx8ulp_lpuart(struct lpuart_port *sport)
 375{
 376	return sport->devtype == IMX8ULP_LPUART;
 377}
 378
 379static inline bool is_imx8qxp_lpuart(struct lpuart_port *sport)
 380{
 381	return sport->devtype == IMX8QXP_LPUART;
 382}
 383
 384static inline u32 lpuart32_read(struct uart_port *port, u32 off)
 385{
 386	switch (port->iotype) {
 387	case UPIO_MEM32:
 388		return readl(port->membase + off);
 389	case UPIO_MEM32BE:
 390		return ioread32be(port->membase + off);
 391	default:
 392		return 0;
 393	}
 394}
 395
 396static inline void lpuart32_write(struct uart_port *port, u32 val,
 397				  u32 off)
 398{
 399	switch (port->iotype) {
 400	case UPIO_MEM32:
 401		writel(val, port->membase + off);
 402		break;
 403	case UPIO_MEM32BE:
 404		iowrite32be(val, port->membase + off);
 405		break;
 406	default:
 407		break;
 408	}
 409}
 410
 411static int __lpuart_enable_clks(struct lpuart_port *sport, bool is_en)
 412{
 413	int ret = 0;
 414
 415	if (is_en) {
 416		ret = clk_prepare_enable(sport->ipg_clk);
 417		if (ret)
 418			return ret;
 419
 420		ret = clk_prepare_enable(sport->baud_clk);
 421		if (ret) {
 422			clk_disable_unprepare(sport->ipg_clk);
 423			return ret;
 424		}
 425	} else {
 426		clk_disable_unprepare(sport->baud_clk);
 427		clk_disable_unprepare(sport->ipg_clk);
 428	}
 429
 430	return 0;
 431}
 432
 433static unsigned int lpuart_get_baud_clk_rate(struct lpuart_port *sport)
 434{
 435	if (is_imx8qxp_lpuart(sport))
 436		return clk_get_rate(sport->baud_clk);
 437
 438	return clk_get_rate(sport->ipg_clk);
 439}
 440
 441#define lpuart_enable_clks(x)	__lpuart_enable_clks(x, true)
 442#define lpuart_disable_clks(x)	__lpuart_enable_clks(x, false)
 443
 444static void lpuart_stop_tx(struct uart_port *port)
 445{
 446	u8 cr2;
 447
 448	cr2 = readb(port->membase + UARTCR2);
 449	cr2 &= ~(UARTCR2_TIE | UARTCR2_TCIE);
 450	writeb(cr2, port->membase + UARTCR2);
 451}
 452
 453static void lpuart32_stop_tx(struct uart_port *port)
 454{
 455	u32 ctrl;
 456
 457	ctrl = lpuart32_read(port, UARTCTRL);
 458	ctrl &= ~(UARTCTRL_TIE | UARTCTRL_TCIE);
 459	lpuart32_write(port, ctrl, UARTCTRL);
 460}
 461
 462static void lpuart_stop_rx(struct uart_port *port)
 463{
 464	u8 cr2;
 465
 466	cr2 = readb(port->membase + UARTCR2);
 467	writeb(cr2 & ~UARTCR2_RE, port->membase + UARTCR2);
 468}
 469
 470static void lpuart32_stop_rx(struct uart_port *port)
 471{
 472	u32 ctrl;
 473
 474	ctrl = lpuart32_read(port, UARTCTRL);
 475	lpuart32_write(port, ctrl & ~UARTCTRL_RE, UARTCTRL);
 476}
 477
 478static void lpuart_dma_tx(struct lpuart_port *sport)
 479{
 480	struct tty_port *tport = &sport->port.state->port;
 481	struct scatterlist *sgl = sport->tx_sgl;
 482	struct device *dev = sport->port.dev;
 483	struct dma_chan *chan = sport->dma_tx_chan;
 484	int ret;
 485
 486	if (sport->dma_tx_in_progress)
 487		return;
 488
 489	sg_init_table(sgl, ARRAY_SIZE(sport->tx_sgl));
 490	sport->dma_tx_bytes = kfifo_len(&tport->xmit_fifo);
 491	sport->dma_tx_nents = kfifo_dma_out_prepare(&tport->xmit_fifo, sgl,
 492			ARRAY_SIZE(sport->tx_sgl), sport->dma_tx_bytes);
 493
 494	ret = dma_map_sg(chan->device->dev, sgl, sport->dma_tx_nents,
 495			 DMA_TO_DEVICE);
 496	if (!ret) {
 497		dev_err(dev, "DMA mapping error for TX.\n");
 498		return;
 499	}
 500
 501	sport->dma_tx_desc = dmaengine_prep_slave_sg(chan, sgl,
 502					ret, DMA_MEM_TO_DEV,
 503					DMA_PREP_INTERRUPT);
 504	if (!sport->dma_tx_desc) {
 505		dma_unmap_sg(chan->device->dev, sgl, sport->dma_tx_nents,
 506			      DMA_TO_DEVICE);
 507		dev_err(dev, "Cannot prepare TX slave DMA!\n");
 508		return;
 509	}
 510
 511	sport->dma_tx_desc->callback = lpuart_dma_tx_complete;
 512	sport->dma_tx_desc->callback_param = sport;
 513	sport->dma_tx_in_progress = true;
 514	sport->dma_tx_cookie = dmaengine_submit(sport->dma_tx_desc);
 515	dma_async_issue_pending(chan);
 516}
 517
 518static bool lpuart_stopped_or_empty(struct uart_port *port)
 519{
 520	return kfifo_is_empty(&port->state->port.xmit_fifo) ||
 521		uart_tx_stopped(port);
 522}
 523
 524static void lpuart_dma_tx_complete(void *arg)
 525{
 526	struct lpuart_port *sport = arg;
 527	struct scatterlist *sgl = &sport->tx_sgl[0];
 528	struct tty_port *tport = &sport->port.state->port;
 529	struct dma_chan *chan = sport->dma_tx_chan;
 530	unsigned long flags;
 531
 532	uart_port_lock_irqsave(&sport->port, &flags);
 533	if (!sport->dma_tx_in_progress) {
 534		uart_port_unlock_irqrestore(&sport->port, flags);
 535		return;
 536	}
 537
 538	dma_unmap_sg(chan->device->dev, sgl, sport->dma_tx_nents,
 539		     DMA_TO_DEVICE);
 540
 541	uart_xmit_advance(&sport->port, sport->dma_tx_bytes);
 542	sport->dma_tx_in_progress = false;
 543	uart_port_unlock_irqrestore(&sport->port, flags);
 544
 545	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
 546		uart_write_wakeup(&sport->port);
 547
 548	if (waitqueue_active(&sport->dma_wait)) {
 549		wake_up(&sport->dma_wait);
 550		return;
 551	}
 552
 553	uart_port_lock_irqsave(&sport->port, &flags);
 554
 555	if (!lpuart_stopped_or_empty(&sport->port))
 556		lpuart_dma_tx(sport);
 557
 558	uart_port_unlock_irqrestore(&sport->port, flags);
 559}
 560
 561static dma_addr_t lpuart_dma_datareg_addr(struct lpuart_port *sport)
 562{
 563	switch (sport->port.iotype) {
 564	case UPIO_MEM32:
 565		return sport->port.mapbase + UARTDATA;
 566	case UPIO_MEM32BE:
 567		return sport->port.mapbase + UARTDATA + sizeof(u32) - 1;
 568	default:
 569		return sport->port.mapbase + UARTDR;
 570	}
 571}
 572
 573static int lpuart_dma_tx_request(struct uart_port *port)
 574{
 575	struct lpuart_port *sport = container_of(port,
 576					struct lpuart_port, port);
 577	struct dma_slave_config dma_tx_sconfig = {};
 578	int ret;
 579
 580	dma_tx_sconfig.dst_addr = lpuart_dma_datareg_addr(sport);
 581	dma_tx_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 582	dma_tx_sconfig.dst_maxburst = 1;
 583	dma_tx_sconfig.direction = DMA_MEM_TO_DEV;
 584	ret = dmaengine_slave_config(sport->dma_tx_chan, &dma_tx_sconfig);
 585
 586	if (ret) {
 587		dev_err(port->dev,
 588				"DMA slave config failed, err = %d\n", ret);
 589		return ret;
 590	}
 591
 592	return 0;
 593}
 594
 595static bool lpuart_is_32(struct lpuart_port *sport)
 596{
 597	return sport->port.iotype == UPIO_MEM32 ||
 598	       sport->port.iotype ==  UPIO_MEM32BE;
 599}
 600
 601static void lpuart_flush_buffer(struct uart_port *port)
 602{
 603	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
 604	struct dma_chan *chan = sport->dma_tx_chan;
 605	u32 fifo;
 606
 607	if (sport->lpuart_dma_tx_use) {
 608		if (sport->dma_tx_in_progress) {
 609			dma_unmap_sg(chan->device->dev, &sport->tx_sgl[0],
 610				sport->dma_tx_nents, DMA_TO_DEVICE);
 611			sport->dma_tx_in_progress = false;
 612		}
 613		dmaengine_terminate_async(chan);
 614	}
 615
 616	if (lpuart_is_32(sport)) {
 617		fifo = lpuart32_read(port, UARTFIFO);
 618		fifo |= UARTFIFO_TXFLUSH | UARTFIFO_RXFLUSH;
 619		lpuart32_write(port, fifo, UARTFIFO);
 620	} else {
 621		fifo = readb(port->membase + UARTCFIFO);
 622		fifo |= UARTCFIFO_TXFLUSH | UARTCFIFO_RXFLUSH;
 623		writeb(fifo, port->membase + UARTCFIFO);
 624	}
 625}
 626
 627static void lpuart_wait_bit_set(struct uart_port *port, unsigned int offset,
 628				u8 bit)
 629{
 630	while (!(readb(port->membase + offset) & bit))
 631		cpu_relax();
 632}
 633
 634static void lpuart32_wait_bit_set(struct uart_port *port, unsigned int offset,
 635				  u32 bit)
 636{
 637	while (!(lpuart32_read(port, offset) & bit))
 638		cpu_relax();
 639}
 640
 641#if defined(CONFIG_CONSOLE_POLL)
 642
 643static int lpuart_poll_init(struct uart_port *port)
 644{
 645	unsigned long flags;
 646	u8 fifo;
 647
 648	port->fifosize = 0;
 649
 650	uart_port_lock_irqsave(port, &flags);
 651	/* Disable Rx & Tx */
 652	writeb(0, port->membase + UARTCR2);
 653
 654	fifo = readb(port->membase + UARTPFIFO);
 655	/* Enable Rx and Tx FIFO */
 656	writeb(fifo | UARTPFIFO_RXFE | UARTPFIFO_TXFE,
 657			port->membase + UARTPFIFO);
 658
 659	/* flush Tx and Rx FIFO */
 660	writeb(UARTCFIFO_TXFLUSH | UARTCFIFO_RXFLUSH,
 661			port->membase + UARTCFIFO);
 662
 663	/* explicitly clear RDRF */
 664	if (readb(port->membase + UARTSR1) & UARTSR1_RDRF) {
 665		readb(port->membase + UARTDR);
 666		writeb(UARTSFIFO_RXUF, port->membase + UARTSFIFO);
 667	}
 668
 669	writeb(0, port->membase + UARTTWFIFO);
 670	writeb(1, port->membase + UARTRWFIFO);
 671
 672	/* Enable Rx and Tx */
 673	writeb(UARTCR2_RE | UARTCR2_TE, port->membase + UARTCR2);
 674	uart_port_unlock_irqrestore(port, flags);
 675
 676	return 0;
 677}
 678
 679static void lpuart_poll_put_char(struct uart_port *port, unsigned char c)
 680{
 681	/* drain */
 682	lpuart_wait_bit_set(port, UARTSR1, UARTSR1_TDRE);
 683	writeb(c, port->membase + UARTDR);
 684}
 685
 686static int lpuart_poll_get_char(struct uart_port *port)
 687{
 688	if (!(readb(port->membase + UARTSR1) & UARTSR1_RDRF))
 689		return NO_POLL_CHAR;
 690
 691	return readb(port->membase + UARTDR);
 692}
 693
 694static int lpuart32_poll_init(struct uart_port *port)
 695{
 696	unsigned long flags;
 697	u32 fifo;
 698
 699	port->fifosize = 0;
 700
 701	uart_port_lock_irqsave(port, &flags);
 702
 703	/* Disable Rx & Tx */
 704	lpuart32_write(port, 0, UARTCTRL);
 705
 706	fifo = lpuart32_read(port, UARTFIFO);
 707
 708	/* Enable Rx and Tx FIFO */
 709	lpuart32_write(port, fifo | UARTFIFO_RXFE | UARTFIFO_TXFE, UARTFIFO);
 710
 711	/* flush Tx and Rx FIFO */
 712	lpuart32_write(port, UARTFIFO_TXFLUSH | UARTFIFO_RXFLUSH, UARTFIFO);
 713
 714	/* explicitly clear RDRF */
 715	if (lpuart32_read(port, UARTSTAT) & UARTSTAT_RDRF) {
 716		lpuart32_read(port, UARTDATA);
 717		lpuart32_write(port, UARTFIFO_RXUF, UARTFIFO);
 718	}
 719
 720	/* Enable Rx and Tx */
 721	lpuart32_write(port, UARTCTRL_RE | UARTCTRL_TE, UARTCTRL);
 722	uart_port_unlock_irqrestore(port, flags);
 723
 724	return 0;
 725}
 726
 727static void lpuart32_poll_put_char(struct uart_port *port, unsigned char c)
 728{
 729	lpuart32_wait_bit_set(port, UARTSTAT, UARTSTAT_TDRE);
 730	lpuart32_write(port, c, UARTDATA);
 731}
 732
 733static int lpuart32_poll_get_char(struct uart_port *port)
 734{
 735	if (!(lpuart32_read(port, UARTWATER) >> UARTWATER_RXCNT_OFF))
 736		return NO_POLL_CHAR;
 737
 738	return lpuart32_read(port, UARTDATA);
 739}
 740#endif
 741
 742static inline void lpuart_transmit_buffer(struct lpuart_port *sport)
 743{
 744	struct uart_port *port = &sport->port;
 745	u8 ch;
 746
 747	uart_port_tx(port, ch,
 748		readb(port->membase + UARTTCFIFO) < sport->txfifo_size,
 749		writeb(ch, port->membase + UARTDR));
 750}
 751
 752static inline void lpuart32_transmit_buffer(struct lpuart_port *sport)
 753{
 754	struct tty_port *tport = &sport->port.state->port;
 755	u32 txcnt;
 756	unsigned char c;
 757
 758	if (sport->port.x_char) {
 759		lpuart32_write(&sport->port, sport->port.x_char, UARTDATA);
 760		sport->port.icount.tx++;
 761		sport->port.x_char = 0;
 762		return;
 763	}
 764
 765	if (lpuart_stopped_or_empty(&sport->port)) {
 766		lpuart32_stop_tx(&sport->port);
 767		return;
 768	}
 769
 770	txcnt = lpuart32_read(&sport->port, UARTWATER);
 771	txcnt = txcnt >> UARTWATER_TXCNT_OFF;
 772	txcnt &= UARTWATER_COUNT_MASK;
 773	while (txcnt < sport->txfifo_size &&
 774			uart_fifo_get(&sport->port, &c)) {
 775		lpuart32_write(&sport->port, c, UARTDATA);
 776		txcnt = lpuart32_read(&sport->port, UARTWATER);
 777		txcnt = txcnt >> UARTWATER_TXCNT_OFF;
 778		txcnt &= UARTWATER_COUNT_MASK;
 779	}
 780
 781	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
 782		uart_write_wakeup(&sport->port);
 783
 784	if (kfifo_is_empty(&tport->xmit_fifo))
 785		lpuart32_stop_tx(&sport->port);
 786}
 787
 788static void lpuart_start_tx(struct uart_port *port)
 789{
 790	struct lpuart_port *sport = container_of(port,
 791			struct lpuart_port, port);
 792	u8 cr2;
 793
 794	cr2 = readb(port->membase + UARTCR2);
 795	writeb(cr2 | UARTCR2_TIE, port->membase + UARTCR2);
 796
 797	if (sport->lpuart_dma_tx_use) {
 798		if (!lpuart_stopped_or_empty(port))
 799			lpuart_dma_tx(sport);
 800	} else {
 801		if (readb(port->membase + UARTSR1) & UARTSR1_TDRE)
 802			lpuart_transmit_buffer(sport);
 803	}
 804}
 805
 806static void lpuart32_start_tx(struct uart_port *port)
 807{
 808	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
 809	u32 ctrl;
 810
 811	if (sport->lpuart_dma_tx_use) {
 812		if (!lpuart_stopped_or_empty(port))
 813			lpuart_dma_tx(sport);
 814	} else {
 815		ctrl = lpuart32_read(port, UARTCTRL);
 816		lpuart32_write(port, ctrl | UARTCTRL_TIE, UARTCTRL);
 817
 818		if (lpuart32_read(port, UARTSTAT) & UARTSTAT_TDRE)
 819			lpuart32_transmit_buffer(sport);
 820	}
 821}
 822
 823static void
 824lpuart_uart_pm(struct uart_port *port, unsigned int state, unsigned int oldstate)
 825{
 826	switch (state) {
 827	case UART_PM_STATE_OFF:
 828		pm_runtime_mark_last_busy(port->dev);
 829		pm_runtime_put_autosuspend(port->dev);
 830		break;
 831	default:
 832		pm_runtime_get_sync(port->dev);
 833		break;
 834	}
 835}
 836
 837/* return TIOCSER_TEMT when transmitter is not busy */
 838static unsigned int lpuart_tx_empty(struct uart_port *port)
 839{
 840	struct lpuart_port *sport = container_of(port,
 841			struct lpuart_port, port);
 842	u8 sr1 = readb(port->membase + UARTSR1);
 843	u8 sfifo = readb(port->membase + UARTSFIFO);
 844
 845	if (sport->dma_tx_in_progress)
 846		return 0;
 847
 848	if (sr1 & UARTSR1_TC && sfifo & UARTSFIFO_TXEMPT)
 849		return TIOCSER_TEMT;
 850
 851	return 0;
 852}
 853
 854static unsigned int lpuart32_tx_empty(struct uart_port *port)
 855{
 856	struct lpuart_port *sport = container_of(port,
 857			struct lpuart_port, port);
 858	u32 stat = lpuart32_read(port, UARTSTAT);
 859	u32 sfifo = lpuart32_read(port, UARTFIFO);
 860	u32 ctrl = lpuart32_read(port, UARTCTRL);
 861
 862	if (sport->dma_tx_in_progress)
 863		return 0;
 864
 865	/*
 866	 * LPUART Transmission Complete Flag may never be set while queuing a break
 867	 * character, so avoid checking for transmission complete when UARTCTRL_SBK
 868	 * is asserted.
 869	 */
 870	if ((stat & UARTSTAT_TC && sfifo & UARTFIFO_TXEMPT) || ctrl & UARTCTRL_SBK)
 871		return TIOCSER_TEMT;
 872
 873	return 0;
 874}
 875
 876static void lpuart_txint(struct lpuart_port *sport)
 877{
 878	uart_port_lock(&sport->port);
 879	lpuart_transmit_buffer(sport);
 880	uart_port_unlock(&sport->port);
 881}
 882
 883static void lpuart_rxint(struct lpuart_port *sport)
 884{
 885	unsigned int flg, ignored = 0, overrun = 0;
 886	struct tty_port *port = &sport->port.state->port;
 887	u8 rx, sr;
 888
 889	uart_port_lock(&sport->port);
 890
 891	while (!(readb(sport->port.membase + UARTSFIFO) & UARTSFIFO_RXEMPT)) {
 892		flg = TTY_NORMAL;
 893		sport->port.icount.rx++;
 894		/*
 895		 * to clear the FE, OR, NF, FE, PE flags,
 896		 * read SR1 then read DR
 897		 */
 898		sr = readb(sport->port.membase + UARTSR1);
 899		rx = readb(sport->port.membase + UARTDR);
 900
 901		if (uart_prepare_sysrq_char(&sport->port, rx))
 902			continue;
 903
 904		if (sr & (UARTSR1_PE | UARTSR1_OR | UARTSR1_FE)) {
 905			if (sr & UARTSR1_PE)
 906				sport->port.icount.parity++;
 907			else if (sr & UARTSR1_FE)
 908				sport->port.icount.frame++;
 909
 910			if (sr & UARTSR1_OR)
 911				overrun++;
 912
 913			if (sr & sport->port.ignore_status_mask) {
 914				if (++ignored > 100)
 915					goto out;
 916				continue;
 917			}
 918
 919			sr &= sport->port.read_status_mask;
 920
 921			if (sr & UARTSR1_PE)
 922				flg = TTY_PARITY;
 923			else if (sr & UARTSR1_FE)
 924				flg = TTY_FRAME;
 925
 926			if (sr & UARTSR1_OR)
 927				flg = TTY_OVERRUN;
 928
 929			sport->port.sysrq = 0;
 930		}
 931
 932		if (tty_insert_flip_char(port, rx, flg) == 0)
 933			sport->port.icount.buf_overrun++;
 934	}
 935
 936out:
 937	if (overrun) {
 938		sport->port.icount.overrun += overrun;
 939
 940		/*
 941		 * Overruns cause FIFO pointers to become missaligned.
 942		 * Flushing the receive FIFO reinitializes the pointers.
 943		 */
 944		writeb(UARTCFIFO_RXFLUSH, sport->port.membase + UARTCFIFO);
 945		writeb(UARTSFIFO_RXOF, sport->port.membase + UARTSFIFO);
 946	}
 947
 948	uart_unlock_and_check_sysrq(&sport->port);
 949
 950	tty_flip_buffer_push(port);
 951}
 952
 953static void lpuart32_txint(struct lpuart_port *sport)
 954{
 955	uart_port_lock(&sport->port);
 956	lpuart32_transmit_buffer(sport);
 957	uart_port_unlock(&sport->port);
 958}
 959
 960static void lpuart32_rxint(struct lpuart_port *sport)
 961{
 962	unsigned int flg, ignored = 0;
 963	struct tty_port *port = &sport->port.state->port;
 964	u32 rx, sr;
 965	bool is_break;
 966
 967	uart_port_lock(&sport->port);
 968
 969	while (!(lpuart32_read(&sport->port, UARTFIFO) & UARTFIFO_RXEMPT)) {
 970		flg = TTY_NORMAL;
 971		sport->port.icount.rx++;
 972		/*
 973		 * to clear the FE, OR, NF, FE, PE flags,
 974		 * read STAT then read DATA reg
 975		 */
 976		sr = lpuart32_read(&sport->port, UARTSTAT);
 977		rx = lpuart32_read(&sport->port, UARTDATA);
 978		rx &= UARTDATA_MASK;
 979
 980		/*
 981		 * The LPUART can't distinguish between a break and a framing error,
 982		 * thus we assume it is a break if the received data is zero.
 983		 */
 984		is_break = (sr & UARTSTAT_FE) && !rx;
 985
 986		if (is_break && uart_handle_break(&sport->port))
 987			continue;
 988
 989		if (uart_prepare_sysrq_char(&sport->port, rx))
 990			continue;
 991
 992		if (sr & (UARTSTAT_PE | UARTSTAT_OR | UARTSTAT_FE)) {
 993			if (sr & UARTSTAT_PE) {
 994				sport->port.icount.parity++;
 995			} else if (sr & UARTSTAT_FE) {
 996				if (is_break)
 997					sport->port.icount.brk++;
 998				else
 999					sport->port.icount.frame++;
1000			}
1001
1002			if (sr & UARTSTAT_OR)
1003				sport->port.icount.overrun++;
1004
1005			if (sr & sport->port.ignore_status_mask) {
1006				if (++ignored > 100)
1007					goto out;
1008				continue;
1009			}
1010
1011			sr &= sport->port.read_status_mask;
1012
1013			if (sr & UARTSTAT_PE) {
1014				flg = TTY_PARITY;
1015			} else if (sr & UARTSTAT_FE) {
1016				if (is_break)
1017					flg = TTY_BREAK;
1018				else
1019					flg = TTY_FRAME;
1020			}
1021
1022			if (sr & UARTSTAT_OR)
1023				flg = TTY_OVERRUN;
1024		}
1025
1026		if (sport->is_cs7)
1027			rx &= 0x7F;
1028
1029		if (tty_insert_flip_char(port, rx, flg) == 0)
1030			sport->port.icount.buf_overrun++;
1031	}
1032
1033out:
1034	uart_unlock_and_check_sysrq(&sport->port);
1035
1036	tty_flip_buffer_push(port);
1037}
1038
1039static irqreturn_t lpuart_int(int irq, void *dev_id)
1040{
1041	struct lpuart_port *sport = dev_id;
1042	u8 sts;
1043
1044	sts = readb(sport->port.membase + UARTSR1);
1045
1046	/* SysRq, using dma, check for linebreak by framing err. */
1047	if (sts & UARTSR1_FE && sport->lpuart_dma_rx_use) {
1048		readb(sport->port.membase + UARTDR);
1049		uart_handle_break(&sport->port);
1050		/* linebreak produces some garbage, removing it */
1051		writeb(UARTCFIFO_RXFLUSH, sport->port.membase + UARTCFIFO);
1052		return IRQ_HANDLED;
1053	}
1054
1055	if (sts & UARTSR1_RDRF && !sport->lpuart_dma_rx_use)
1056		lpuart_rxint(sport);
1057
1058	if (sts & UARTSR1_TDRE && !sport->lpuart_dma_tx_use)
1059		lpuart_txint(sport);
1060
1061	return IRQ_HANDLED;
1062}
1063
1064static inline void lpuart_handle_sysrq_chars(struct uart_port *port,
1065					     unsigned char *p, int count)
1066{
1067	while (count--) {
1068		if (*p && uart_handle_sysrq_char(port, *p))
1069			return;
1070		p++;
1071	}
1072}
1073
1074static void lpuart_handle_sysrq(struct lpuart_port *sport)
1075{
1076	struct circ_buf *ring = &sport->rx_ring;
1077	int count;
1078
1079	if (ring->head < ring->tail) {
1080		count = sport->rx_sgl.length - ring->tail;
1081		lpuart_handle_sysrq_chars(&sport->port,
1082					  ring->buf + ring->tail, count);
1083		ring->tail = 0;
1084	}
1085
1086	if (ring->head > ring->tail) {
1087		count = ring->head - ring->tail;
1088		lpuart_handle_sysrq_chars(&sport->port,
1089					  ring->buf + ring->tail, count);
1090		ring->tail = ring->head;
1091	}
1092}
1093
1094static int lpuart_tty_insert_flip_string(struct tty_port *port,
1095	unsigned char *chars, size_t size, bool is_cs7)
1096{
1097	int i;
1098
1099	if (is_cs7)
1100		for (i = 0; i < size; i++)
1101			chars[i] &= 0x7F;
1102	return tty_insert_flip_string(port, chars, size);
1103}
1104
1105static void lpuart_copy_rx_to_tty(struct lpuart_port *sport)
1106{
1107	struct tty_port *port = &sport->port.state->port;
1108	struct dma_tx_state state;
1109	enum dma_status dmastat;
1110	struct dma_chan *chan = sport->dma_rx_chan;
1111	struct circ_buf *ring = &sport->rx_ring;
1112	unsigned long flags;
1113	int count, copied;
1114
1115	if (lpuart_is_32(sport)) {
1116		u32 sr = lpuart32_read(&sport->port, UARTSTAT);
1117
1118		if (sr & (UARTSTAT_PE | UARTSTAT_FE)) {
1119			/* Clear the error flags */
1120			lpuart32_write(&sport->port, sr, UARTSTAT);
1121
1122			if (sr & UARTSTAT_PE)
1123				sport->port.icount.parity++;
1124			else if (sr & UARTSTAT_FE)
1125				sport->port.icount.frame++;
1126		}
1127	} else {
1128		u8 sr = readb(sport->port.membase + UARTSR1);
1129
1130		if (sr & (UARTSR1_PE | UARTSR1_FE)) {
1131			u8 cr2;
1132
1133			/* Disable receiver during this operation... */
1134			cr2 = readb(sport->port.membase + UARTCR2);
1135			cr2 &= ~UARTCR2_RE;
1136			writeb(cr2, sport->port.membase + UARTCR2);
1137
1138			/* Read DR to clear the error flags */
1139			readb(sport->port.membase + UARTDR);
1140
1141			if (sr & UARTSR1_PE)
1142				sport->port.icount.parity++;
1143			else if (sr & UARTSR1_FE)
1144				sport->port.icount.frame++;
1145			/*
1146			 * At this point parity/framing error is
1147			 * cleared However, since the DMA already read
1148			 * the data register and we had to read it
1149			 * again after reading the status register to
1150			 * properly clear the flags, the FIFO actually
1151			 * underflowed... This requires a clearing of
1152			 * the FIFO...
1153			 */
1154			if (readb(sport->port.membase + UARTSFIFO) &
1155			    UARTSFIFO_RXUF) {
1156				writeb(UARTSFIFO_RXUF,
1157				       sport->port.membase + UARTSFIFO);
1158				writeb(UARTCFIFO_RXFLUSH,
1159				       sport->port.membase + UARTCFIFO);
1160			}
1161
1162			cr2 |= UARTCR2_RE;
1163			writeb(cr2, sport->port.membase + UARTCR2);
1164		}
1165	}
1166
1167	async_tx_ack(sport->dma_rx_desc);
1168
1169	uart_port_lock_irqsave(&sport->port, &flags);
1170
1171	dmastat = dmaengine_tx_status(chan, sport->dma_rx_cookie, &state);
1172	if (dmastat == DMA_ERROR) {
1173		dev_err(sport->port.dev, "Rx DMA transfer failed!\n");
1174		uart_port_unlock_irqrestore(&sport->port, flags);
1175		return;
1176	}
1177
1178	/* CPU claims ownership of RX DMA buffer */
1179	dma_sync_sg_for_cpu(chan->device->dev, &sport->rx_sgl, 1,
1180			    DMA_FROM_DEVICE);
1181
1182	/*
1183	 * ring->head points to the end of data already written by the DMA.
1184	 * ring->tail points to the beginning of data to be read by the
1185	 * framework.
1186	 * The current transfer size should not be larger than the dma buffer
1187	 * length.
1188	 */
1189	ring->head = sport->rx_sgl.length - state.residue;
1190	BUG_ON(ring->head > sport->rx_sgl.length);
1191
1192	/*
1193	 * Silent handling of keys pressed in the sysrq timeframe
1194	 */
1195	if (sport->port.sysrq) {
1196		lpuart_handle_sysrq(sport);
1197		goto exit;
1198	}
1199
1200	/*
1201	 * At this point ring->head may point to the first byte right after the
1202	 * last byte of the dma buffer:
1203	 * 0 <= ring->head <= sport->rx_sgl.length
1204	 *
1205	 * However ring->tail must always points inside the dma buffer:
1206	 * 0 <= ring->tail <= sport->rx_sgl.length - 1
1207	 *
1208	 * Since we use a ring buffer, we have to handle the case
1209	 * where head is lower than tail. In such a case, we first read from
1210	 * tail to the end of the buffer then reset tail.
1211	 */
1212	if (ring->head < ring->tail) {
1213		count = sport->rx_sgl.length - ring->tail;
1214
1215		copied = lpuart_tty_insert_flip_string(port, ring->buf + ring->tail,
1216					count, sport->is_cs7);
1217		if (copied != count)
1218			sport->port.icount.buf_overrun++;
1219		ring->tail = 0;
1220		sport->port.icount.rx += copied;
1221	}
1222
1223	/* Finally we read data from tail to head */
1224	if (ring->tail < ring->head) {
1225		count = ring->head - ring->tail;
1226		copied = lpuart_tty_insert_flip_string(port, ring->buf + ring->tail,
1227					count, sport->is_cs7);
1228		if (copied != count)
1229			sport->port.icount.buf_overrun++;
1230		/* Wrap ring->head if needed */
1231		if (ring->head >= sport->rx_sgl.length)
1232			ring->head = 0;
1233		ring->tail = ring->head;
1234		sport->port.icount.rx += copied;
1235	}
1236
1237	sport->last_residue = state.residue;
1238
1239exit:
1240	dma_sync_sg_for_device(chan->device->dev, &sport->rx_sgl, 1,
1241			       DMA_FROM_DEVICE);
1242
1243	uart_port_unlock_irqrestore(&sport->port, flags);
1244
1245	tty_flip_buffer_push(port);
1246	if (!sport->dma_idle_int)
1247		mod_timer(&sport->lpuart_timer, jiffies + sport->dma_rx_timeout);
1248}
1249
1250static void lpuart_dma_rx_complete(void *arg)
1251{
1252	struct lpuart_port *sport = arg;
1253
1254	lpuart_copy_rx_to_tty(sport);
1255}
1256
1257static void lpuart32_dma_idleint(struct lpuart_port *sport)
1258{
1259	enum dma_status dmastat;
1260	struct dma_chan *chan = sport->dma_rx_chan;
1261	struct circ_buf *ring = &sport->rx_ring;
1262	struct dma_tx_state state;
1263	int count = 0;
1264
1265	dmastat = dmaengine_tx_status(chan, sport->dma_rx_cookie, &state);
1266	if (dmastat == DMA_ERROR) {
1267		dev_err(sport->port.dev, "Rx DMA transfer failed!\n");
1268		return;
1269	}
1270
1271	ring->head = sport->rx_sgl.length - state.residue;
1272	count = CIRC_CNT(ring->head, ring->tail, sport->rx_sgl.length);
1273
1274	/* Check if new data received before copying */
1275	if (count)
1276		lpuart_copy_rx_to_tty(sport);
1277}
1278
1279static irqreturn_t lpuart32_int(int irq, void *dev_id)
1280{
1281	struct lpuart_port *sport = dev_id;
1282	u32 sts, rxcount;
1283
1284	sts = lpuart32_read(&sport->port, UARTSTAT);
1285	rxcount = lpuart32_read(&sport->port, UARTWATER);
1286	rxcount = rxcount >> UARTWATER_RXCNT_OFF;
1287
1288	if ((sts & UARTSTAT_RDRF || rxcount > 0) && !sport->lpuart_dma_rx_use)
1289		lpuart32_rxint(sport);
1290
1291	if ((sts & UARTSTAT_TDRE) && !sport->lpuart_dma_tx_use)
1292		lpuart32_txint(sport);
1293
1294	if ((sts & UARTSTAT_IDLE) && sport->lpuart_dma_rx_use && sport->dma_idle_int)
1295		lpuart32_dma_idleint(sport);
1296
1297	lpuart32_write(&sport->port, sts, UARTSTAT);
1298	return IRQ_HANDLED;
1299}
1300
1301/*
1302 * Timer function to simulate the hardware EOP (End Of Package) event.
1303 * The timer callback is to check for new RX data and copy to TTY buffer.
1304 * If no new data are received since last interval, the EOP condition is
1305 * met, complete the DMA transfer by copying the data. Otherwise, just
1306 * restart timer.
1307 */
1308static void lpuart_timer_func(struct timer_list *t)
1309{
1310	struct lpuart_port *sport = timer_container_of(sport, t, lpuart_timer);
1311	enum dma_status dmastat;
1312	struct dma_chan *chan = sport->dma_rx_chan;
1313	struct circ_buf *ring = &sport->rx_ring;
1314	struct dma_tx_state state;
1315	unsigned long flags;
1316	int count;
1317
1318	dmastat = dmaengine_tx_status(chan, sport->dma_rx_cookie, &state);
1319	if (dmastat == DMA_ERROR) {
1320		dev_err(sport->port.dev, "Rx DMA transfer failed!\n");
1321		return;
1322	}
1323
1324	ring->head = sport->rx_sgl.length - state.residue;
1325	count = CIRC_CNT(ring->head, ring->tail, sport->rx_sgl.length);
1326
1327	/* Check if new data received before copying */
1328	if ((count != 0) && (sport->last_residue == state.residue))
1329		lpuart_copy_rx_to_tty(sport);
1330	else
1331		mod_timer(&sport->lpuart_timer,
1332			  jiffies + sport->dma_rx_timeout);
1333
1334	if (uart_port_trylock_irqsave(&sport->port, &flags)) {
1335		sport->last_residue = state.residue;
1336		uart_port_unlock_irqrestore(&sport->port, flags);
1337	}
1338}
1339
1340static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
1341{
1342	struct dma_slave_config dma_rx_sconfig = {};
1343	struct circ_buf *ring = &sport->rx_ring;
1344	int ret, nent;
1345	struct tty_port *port = &sport->port.state->port;
1346	struct tty_struct *tty = port->tty;
1347	struct ktermios *termios = &tty->termios;
1348	struct dma_chan *chan = sport->dma_rx_chan;
1349	unsigned int bits = tty_get_frame_size(termios->c_cflag);
1350	unsigned int baud = tty_get_baud_rate(tty);
1351
1352	/*
1353	 * Calculate length of one DMA buffer size to keep latency below
1354	 * 10ms at any baud rate.
1355	 */
1356	sport->rx_dma_rng_buf_len = (DMA_RX_TIMEOUT * baud /  bits / 1000) * 2;
1357	sport->rx_dma_rng_buf_len = (1 << fls(sport->rx_dma_rng_buf_len));
1358	sport->rx_dma_rng_buf_len = max_t(int,
1359					  sport->rxfifo_size * 2,
1360					  sport->rx_dma_rng_buf_len);
1361	/*
1362	 * Keep this condition check in case rxfifo_size is unavailable
1363	 * for some SoCs.
1364	 */
1365	if (sport->rx_dma_rng_buf_len < 16)
1366		sport->rx_dma_rng_buf_len = 16;
1367
1368	sport->last_residue = 0;
1369	sport->dma_rx_timeout = max(nsecs_to_jiffies(
1370		sport->port.frame_time * DMA_RX_IDLE_CHARS), 1UL);
1371
1372	ring->buf = kzalloc(sport->rx_dma_rng_buf_len, GFP_ATOMIC);
1373	if (!ring->buf)
1374		return -ENOMEM;
1375
1376	sg_init_one(&sport->rx_sgl, ring->buf, sport->rx_dma_rng_buf_len);
1377	nent = dma_map_sg(chan->device->dev, &sport->rx_sgl, 1,
1378			  DMA_FROM_DEVICE);
1379
1380	if (!nent) {
1381		dev_err(sport->port.dev, "DMA Rx mapping error\n");
1382		return -EINVAL;
1383	}
1384
1385	dma_rx_sconfig.src_addr = lpuart_dma_datareg_addr(sport);
1386	dma_rx_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
1387	dma_rx_sconfig.src_maxburst = 1;
1388	dma_rx_sconfig.direction = DMA_DEV_TO_MEM;
1389	ret = dmaengine_slave_config(chan, &dma_rx_sconfig);
1390
1391	if (ret < 0) {
1392		dev_err(sport->port.dev,
1393				"DMA Rx slave config failed, err = %d\n", ret);
1394		return ret;
1395	}
1396
1397	sport->dma_rx_desc = dmaengine_prep_dma_cyclic(chan,
1398				 sg_dma_address(&sport->rx_sgl),
1399				 sport->rx_sgl.length,
1400				 sport->rx_sgl.length / 2,
1401				 DMA_DEV_TO_MEM,
1402				 DMA_PREP_INTERRUPT);
1403	if (!sport->dma_rx_desc) {
1404		dev_err(sport->port.dev, "Cannot prepare cyclic DMA\n");
1405		return -EFAULT;
1406	}
1407
1408	sport->dma_rx_desc->callback = lpuart_dma_rx_complete;
1409	sport->dma_rx_desc->callback_param = sport;
1410	sport->dma_rx_cookie = dmaengine_submit(sport->dma_rx_desc);
1411	dma_async_issue_pending(chan);
1412
1413	if (lpuart_is_32(sport)) {
1414		u32 baud = lpuart32_read(&sport->port, UARTBAUD);
1415
1416		lpuart32_write(&sport->port, baud | UARTBAUD_RDMAE, UARTBAUD);
1417
1418		if (sport->dma_idle_int) {
1419			u32 ctrl = lpuart32_read(&sport->port, UARTCTRL);
1420
1421			lpuart32_write(&sport->port, ctrl | UARTCTRL_ILIE, UARTCTRL);
1422		}
1423	} else {
1424		writeb(readb(sport->port.membase + UARTCR5) | UARTCR5_RDMAS,
1425		       sport->port.membase + UARTCR5);
1426	}
1427
1428	return 0;
1429}
1430
1431static void lpuart_dma_rx_free(struct uart_port *port)
1432{
1433	struct lpuart_port *sport = container_of(port,
1434					struct lpuart_port, port);
1435	struct dma_chan *chan = sport->dma_rx_chan;
1436
1437	dmaengine_terminate_sync(chan);
1438	if (!sport->dma_idle_int)
1439		timer_delete_sync(&sport->lpuart_timer);
1440
1441	dma_unmap_sg(chan->device->dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
1442	kfree(sport->rx_ring.buf);
1443	sport->rx_ring.tail = 0;
1444	sport->rx_ring.head = 0;
1445	sport->dma_rx_desc = NULL;
1446	sport->dma_rx_cookie = -EINVAL;
1447}
1448
1449static int lpuart_config_rs485(struct uart_port *port, struct ktermios *termios,
1450			struct serial_rs485 *rs485)
1451{
1452	u8 modem = readb(port->membase + UARTMODEM) &
1453		~(UARTMODEM_TXRTSPOL | UARTMODEM_TXRTSE);
1454	writeb(modem, port->membase + UARTMODEM);
1455
1456	if (rs485->flags & SER_RS485_ENABLED) {
1457		/* Enable auto RS-485 RTS mode */
1458		modem |= UARTMODEM_TXRTSE;
1459
1460		/*
1461		 * The hardware defaults to RTS logic HIGH while transfer.
1462		 * Switch polarity in case RTS shall be logic HIGH
1463		 * after transfer.
1464		 * Note: UART is assumed to be active high.
1465		 */
1466		if (rs485->flags & SER_RS485_RTS_ON_SEND)
1467			modem |= UARTMODEM_TXRTSPOL;
1468		else if (rs485->flags & SER_RS485_RTS_AFTER_SEND)
1469			modem &= ~UARTMODEM_TXRTSPOL;
1470	}
1471
1472	writeb(modem, port->membase + UARTMODEM);
1473	return 0;
1474}
1475
1476static int lpuart32_config_rs485(struct uart_port *port, struct ktermios *termios,
1477			struct serial_rs485 *rs485)
1478{
1479	u32 modem = lpuart32_read(port, UARTMODIR)
1480				& ~(UARTMODIR_TXRTSPOL | UARTMODIR_TXRTSE);
1481	u32 ctrl;
1482
1483	/* TXRTSE and TXRTSPOL only can be changed when transmitter is disabled. */
1484	ctrl = lpuart32_read(port, UARTCTRL);
1485	if (ctrl & UARTCTRL_TE) {
1486		/* wait for the transmit engine to complete */
1487		lpuart32_wait_bit_set(port, UARTSTAT, UARTSTAT_TC);
1488		lpuart32_write(port, ctrl & ~UARTCTRL_TE, UARTCTRL);
1489
1490		while (lpuart32_read(port, UARTCTRL) & UARTCTRL_TE)
1491			cpu_relax();
1492	}
1493
1494	lpuart32_write(port, modem, UARTMODIR);
1495
1496	if (rs485->flags & SER_RS485_ENABLED) {
1497		/* Enable auto RS-485 RTS mode */
1498		modem |= UARTMODIR_TXRTSE;
1499
1500		/*
1501		 * The hardware defaults to RTS logic HIGH while transfer.
1502		 * Switch polarity in case RTS shall be logic HIGH
1503		 * after transfer.
1504		 * Note: UART is assumed to be active high.
1505		 */
1506		if (rs485->flags & SER_RS485_RTS_ON_SEND)
1507			modem |= UARTMODIR_TXRTSPOL;
1508		else if (rs485->flags & SER_RS485_RTS_AFTER_SEND)
1509			modem &= ~UARTMODIR_TXRTSPOL;
1510	}
1511
1512	lpuart32_write(port, modem, UARTMODIR);
1513
1514	if (ctrl & UARTCTRL_TE)
1515		lpuart32_write(port, ctrl, UARTCTRL);
1516
1517	return 0;
1518}
1519
1520static unsigned int lpuart_get_mctrl(struct uart_port *port)
1521{
1522	unsigned int mctrl = 0;
1523	u8 cr1;
1524
1525	cr1 = readb(port->membase + UARTCR1);
1526	if (cr1 & UARTCR1_LOOPS)
1527		mctrl |= TIOCM_LOOP;
1528
1529	return mctrl;
1530}
1531
1532static unsigned int lpuart32_get_mctrl(struct uart_port *port)
1533{
1534	unsigned int mctrl = TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
1535	u32 ctrl;
1536
1537	ctrl = lpuart32_read(port, UARTCTRL);
1538	if (ctrl & UARTCTRL_LOOPS)
1539		mctrl |= TIOCM_LOOP;
1540
1541	return mctrl;
1542}
1543
1544static void lpuart_set_mctrl(struct uart_port *port, unsigned int mctrl)
1545{
1546	u8 cr1;
1547
1548	cr1 = readb(port->membase + UARTCR1);
1549
1550	/* for internal loopback we need LOOPS=1 and RSRC=0 */
1551	cr1 &= ~(UARTCR1_LOOPS | UARTCR1_RSRC);
1552	if (mctrl & TIOCM_LOOP)
1553		cr1 |= UARTCR1_LOOPS;
1554
1555	writeb(cr1, port->membase + UARTCR1);
1556}
1557
1558static void lpuart32_set_mctrl(struct uart_port *port, unsigned int mctrl)
1559{
1560	u32 ctrl;
1561
1562	ctrl = lpuart32_read(port, UARTCTRL);
1563
1564	/* for internal loopback we need LOOPS=1 and RSRC=0 */
1565	ctrl &= ~(UARTCTRL_LOOPS | UARTCTRL_RSRC);
1566	if (mctrl & TIOCM_LOOP)
1567		ctrl |= UARTCTRL_LOOPS;
1568
1569	lpuart32_write(port, ctrl, UARTCTRL);
1570}
1571
1572static void lpuart_break_ctl(struct uart_port *port, int break_state)
1573{
1574	u8 cr2;
1575
1576	cr2 = readb(port->membase + UARTCR2) & ~UARTCR2_SBK;
1577
1578	if (break_state != 0)
1579		cr2 |= UARTCR2_SBK;
1580
1581	writeb(cr2, port->membase + UARTCR2);
1582}
1583
1584static void lpuart32_break_ctl(struct uart_port *port, int break_state)
1585{
1586	u32 ctrl;
1587
1588	ctrl = lpuart32_read(port, UARTCTRL);
1589
1590	/*
1591	 * LPUART IP now has two known bugs, one is CTS has higher priority than the
1592	 * break signal, which causes the break signal sending through UARTCTRL_SBK
1593	 * may impacted by the CTS input if the HW flow control is enabled. It
1594	 * exists on all platforms we support in this driver.
1595	 * Another bug is i.MX8QM LPUART may have an additional break character
1596	 * being sent after SBK was cleared.
1597	 * To avoid above two bugs, we use Transmit Data Inversion function to send
1598	 * the break signal instead of UARTCTRL_SBK.
1599	 */
1600	if (break_state != 0) {
1601		/*
1602		 * Disable the transmitter to prevent any data from being sent out
1603		 * during break, then invert the TX line to send break.
1604		 */
1605		ctrl &= ~UARTCTRL_TE;
1606		lpuart32_write(port, ctrl, UARTCTRL);
1607		ctrl |= UARTCTRL_TXINV;
1608		lpuart32_write(port, ctrl, UARTCTRL);
1609	} else {
1610		/* Disable the TXINV to turn off break and re-enable transmitter. */
1611		ctrl &= ~UARTCTRL_TXINV;
1612		lpuart32_write(port, ctrl, UARTCTRL);
1613		ctrl |= UARTCTRL_TE;
1614		lpuart32_write(port, ctrl, UARTCTRL);
1615	}
1616}
1617
1618static void lpuart_setup_watermark(struct lpuart_port *sport)
1619{
1620	u8 fifo, cr2, cr2_saved;
1621
1622	cr2 = readb(sport->port.membase + UARTCR2);
1623	cr2_saved = cr2;
1624	cr2 &= ~(UARTCR2_TIE | UARTCR2_TCIE | UARTCR2_TE |
1625			UARTCR2_RIE | UARTCR2_RE);
1626	writeb(cr2, sport->port.membase + UARTCR2);
1627
1628	fifo = readb(sport->port.membase + UARTPFIFO);
1629	writeb(fifo | UARTPFIFO_TXFE | UARTPFIFO_RXFE,
1630			sport->port.membase + UARTPFIFO);
1631
1632	/* flush Tx and Rx FIFO */
1633	writeb(UARTCFIFO_TXFLUSH | UARTCFIFO_RXFLUSH,
1634			sport->port.membase + UARTCFIFO);
1635
1636	/* explicitly clear RDRF */
1637	if (readb(sport->port.membase + UARTSR1) & UARTSR1_RDRF) {
1638		readb(sport->port.membase + UARTDR);
1639		writeb(UARTSFIFO_RXUF, sport->port.membase + UARTSFIFO);
1640	}
1641
1642	if (uart_console(&sport->port))
1643		sport->rx_watermark = 1;
1644	writeb(0, sport->port.membase + UARTTWFIFO);
1645	writeb(sport->rx_watermark, sport->port.membase + UARTRWFIFO);
1646
1647	/* Restore cr2 */
1648	writeb(cr2_saved, sport->port.membase + UARTCR2);
1649}
1650
1651static void lpuart_setup_watermark_enable(struct lpuart_port *sport)
1652{
1653	u8 cr2;
1654
1655	lpuart_setup_watermark(sport);
1656
1657	cr2 = readb(sport->port.membase + UARTCR2);
1658	cr2 |= UARTCR2_RIE | UARTCR2_RE | UARTCR2_TE;
1659	writeb(cr2, sport->port.membase + UARTCR2);
1660}
1661
1662static void lpuart32_setup_watermark(struct lpuart_port *sport)
1663{
1664	u32 val, ctrl, ctrl_saved;
1665
1666	ctrl = lpuart32_read(&sport->port, UARTCTRL);
1667	ctrl_saved = ctrl;
1668	ctrl &= ~(UARTCTRL_TIE | UARTCTRL_TCIE | UARTCTRL_TE |
1669			UARTCTRL_RIE | UARTCTRL_RE | UARTCTRL_ILIE);
1670	lpuart32_write(&sport->port, ctrl, UARTCTRL);
1671
1672	/* enable FIFO mode */
1673	val = lpuart32_read(&sport->port, UARTFIFO);
1674	val |= UARTFIFO_TXFE | UARTFIFO_RXFE;
1675	val |= UARTFIFO_TXFLUSH | UARTFIFO_RXFLUSH;
1676	val |= FIELD_PREP(UARTFIFO_RXIDEN, 0x3);
1677	lpuart32_write(&sport->port, val, UARTFIFO);
1678
1679	/* set the watermark */
1680	if (uart_console(&sport->port))
1681		sport->rx_watermark = 1;
1682	val = (sport->rx_watermark << UARTWATER_RXWATER_OFF) |
1683	      (0x0 << UARTWATER_TXWATER_OFF);
1684	lpuart32_write(&sport->port, val, UARTWATER);
1685
1686	/* set RTS watermark */
1687	if (!uart_console(&sport->port)) {
1688		val = lpuart32_read(&sport->port, UARTMODIR);
1689		val |= FIELD_PREP(UARTMODIR_RTSWATER, sport->rxfifo_size >> 1);
1690		lpuart32_write(&sport->port, val, UARTMODIR);
1691	}
1692
1693	/* Restore cr2 */
1694	lpuart32_write(&sport->port, ctrl_saved, UARTCTRL);
1695}
1696
1697static void lpuart32_setup_watermark_enable(struct lpuart_port *sport)
1698{
1699	u32 ctrl;
1700
1701	lpuart32_setup_watermark(sport);
1702
1703	ctrl = lpuart32_read(&sport->port, UARTCTRL);
1704	ctrl |= UARTCTRL_RE | UARTCTRL_TE;
1705	ctrl |= FIELD_PREP(UARTCTRL_IDLECFG, 0x7);
1706	lpuart32_write(&sport->port, ctrl, UARTCTRL);
1707}
1708
1709static void rx_dma_timer_init(struct lpuart_port *sport)
1710{
1711	if (sport->dma_idle_int)
1712		return;
1713
1714	timer_setup(&sport->lpuart_timer, lpuart_timer_func, 0);
1715	sport->lpuart_timer.expires = jiffies + sport->dma_rx_timeout;
1716	add_timer(&sport->lpuart_timer);
1717}
1718
1719static void lpuart_request_dma(struct lpuart_port *sport)
1720{
1721	sport->dma_tx_chan = dma_request_chan(sport->port.dev, "tx");
1722	if (IS_ERR(sport->dma_tx_chan)) {
1723		dev_dbg_once(sport->port.dev,
1724			     "DMA tx channel request failed, operating without tx DMA (%ld)\n",
1725			     PTR_ERR(sport->dma_tx_chan));
1726		sport->dma_tx_chan = NULL;
1727	}
1728
1729	sport->dma_rx_chan = dma_request_chan(sport->port.dev, "rx");
1730	if (IS_ERR(sport->dma_rx_chan)) {
1731		dev_dbg_once(sport->port.dev,
1732			     "DMA rx channel request failed, operating without rx DMA (%ld)\n",
1733			     PTR_ERR(sport->dma_rx_chan));
1734		sport->dma_rx_chan = NULL;
1735	}
1736}
1737
1738static void lpuart_tx_dma_startup(struct lpuart_port *sport)
1739{
1740	u32 uartbaud;
1741	int ret;
1742
1743	if (uart_console(&sport->port))
1744		goto err;
1745
1746	if (!sport->dma_tx_chan)
1747		goto err;
1748
1749	ret = lpuart_dma_tx_request(&sport->port);
1750	if (ret)
1751		goto err;
1752
1753	init_waitqueue_head(&sport->dma_wait);
1754	sport->lpuart_dma_tx_use = true;
1755	if (lpuart_is_32(sport)) {
1756		uartbaud = lpuart32_read(&sport->port, UARTBAUD);
1757		lpuart32_write(&sport->port,
1758			       uartbaud | UARTBAUD_TDMAE, UARTBAUD);
1759	} else {
1760		writeb(readb(sport->port.membase + UARTCR5) |
1761		       UARTCR5_TDMAS, sport->port.membase + UARTCR5);
1762	}
1763
1764	return;
1765
1766err:
1767	sport->lpuart_dma_tx_use = false;
1768}
1769
1770static void lpuart_rx_dma_startup(struct lpuart_port *sport)
1771{
1772	int ret;
1773	u8 cr3;
1774
1775	if (uart_console(&sport->port))
1776		goto err;
1777
1778	if (!sport->dma_rx_chan)
1779		goto err;
1780
1781	/* set default Rx DMA timeout */
1782	sport->dma_rx_timeout = msecs_to_jiffies(DMA_RX_TIMEOUT);
1783
1784	ret = lpuart_start_rx_dma(sport);
1785	if (ret)
1786		goto err;
1787
1788	if (!sport->dma_rx_timeout)
1789		sport->dma_rx_timeout = 1;
1790
1791	sport->lpuart_dma_rx_use = true;
1792	rx_dma_timer_init(sport);
1793
1794	if (sport->port.has_sysrq && !lpuart_is_32(sport)) {
1795		cr3 = readb(sport->port.membase + UARTCR3);
1796		cr3 |= UARTCR3_FEIE;
1797		writeb(cr3, sport->port.membase + UARTCR3);
1798	}
1799
1800	return;
1801
1802err:
1803	sport->lpuart_dma_rx_use = false;
1804}
1805
1806static void lpuart_hw_setup(struct lpuart_port *sport)
1807{
1808	unsigned long flags;
1809
1810	uart_port_lock_irqsave(&sport->port, &flags);
1811
1812	lpuart_setup_watermark_enable(sport);
1813
1814	lpuart_rx_dma_startup(sport);
1815	lpuart_tx_dma_startup(sport);
1816
1817	uart_port_unlock_irqrestore(&sport->port, flags);
1818}
1819
1820static int lpuart_startup(struct uart_port *port)
1821{
1822	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
1823	u8 fifo;
1824
1825	/* determine FIFO size and enable FIFO mode */
1826	fifo = readb(port->membase + UARTPFIFO);
1827
1828	sport->txfifo_size = UARTFIFO_DEPTH((fifo >> UARTPFIFO_TXSIZE_OFF) &
1829					    UARTPFIFO_FIFOSIZE_MASK);
1830	port->fifosize = sport->txfifo_size;
1831
1832	sport->rxfifo_size = UARTFIFO_DEPTH((fifo >> UARTPFIFO_RXSIZE_OFF) &
1833					    UARTPFIFO_FIFOSIZE_MASK);
1834
1835	lpuart_request_dma(sport);
1836	lpuart_hw_setup(sport);
1837
1838	return 0;
1839}
1840
1841static void lpuart32_hw_disable(struct lpuart_port *sport)
1842{
1843	u32 ctrl;
1844
1845	ctrl = lpuart32_read(&sport->port, UARTCTRL);
1846	ctrl &= ~(UARTCTRL_RIE | UARTCTRL_ILIE | UARTCTRL_RE |
1847		  UARTCTRL_TIE | UARTCTRL_TE);
1848	lpuart32_write(&sport->port, ctrl, UARTCTRL);
1849}
1850
1851static void lpuart32_configure(struct lpuart_port *sport)
1852{
1853	u32 ctrl;
1854
1855	ctrl = lpuart32_read(&sport->port, UARTCTRL);
1856	if (!sport->lpuart_dma_rx_use)
1857		ctrl |= UARTCTRL_RIE | UARTCTRL_ILIE;
1858	if (!sport->lpuart_dma_tx_use)
1859		ctrl |= UARTCTRL_TIE;
1860	lpuart32_write(&sport->port, ctrl, UARTCTRL);
1861}
1862
1863static void lpuart32_hw_setup(struct lpuart_port *sport)
1864{
1865	unsigned long flags;
1866
1867	uart_port_lock_irqsave(&sport->port, &flags);
1868
1869	lpuart32_hw_disable(sport);
1870
1871	lpuart_rx_dma_startup(sport);
1872	lpuart_tx_dma_startup(sport);
1873
1874	lpuart32_setup_watermark_enable(sport);
1875	lpuart32_configure(sport);
1876
1877	uart_port_unlock_irqrestore(&sport->port, flags);
1878}
1879
1880static int lpuart32_startup(struct uart_port *port)
1881{
1882	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
1883	u32 fifo;
1884
1885	/* determine FIFO size */
1886	fifo = lpuart32_read(port, UARTFIFO);
1887
1888	sport->txfifo_size = UARTFIFO_DEPTH((fifo >> UARTFIFO_TXSIZE_OFF) &
1889					    UARTFIFO_FIFOSIZE_MASK);
1890	port->fifosize = sport->txfifo_size;
1891
1892	sport->rxfifo_size = UARTFIFO_DEPTH((fifo >> UARTFIFO_RXSIZE_OFF) &
1893					    UARTFIFO_FIFOSIZE_MASK);
1894
1895	/*
1896	 * The LS1021A and LS1028A have a fixed FIFO depth of 16 words.
1897	 * Although they support the RX/TXSIZE fields, their encoding is
1898	 * different. Eg the reference manual states 0b101 is 16 words.
1899	 */
1900	if (is_layerscape_lpuart(sport)) {
1901		sport->rxfifo_size = 16;
1902		sport->txfifo_size = 16;
1903		port->fifosize = sport->txfifo_size;
1904	}
1905
1906	lpuart_request_dma(sport);
1907	lpuart32_hw_setup(sport);
1908
1909	return 0;
1910}
1911
1912static void lpuart_dma_shutdown(struct lpuart_port *sport)
1913{
1914	if (sport->lpuart_dma_rx_use) {
1915		lpuart_dma_rx_free(&sport->port);
1916		sport->lpuart_dma_rx_use = false;
1917	}
1918
1919	if (sport->lpuart_dma_tx_use) {
1920		if (wait_event_interruptible_timeout(sport->dma_wait,
1921			!sport->dma_tx_in_progress, msecs_to_jiffies(300)) <= 0) {
1922			sport->dma_tx_in_progress = false;
1923			dmaengine_terminate_sync(sport->dma_tx_chan);
1924		}
1925		sport->lpuart_dma_tx_use = false;
1926	}
1927
1928	if (sport->dma_tx_chan)
1929		dma_release_channel(sport->dma_tx_chan);
1930	if (sport->dma_rx_chan)
1931		dma_release_channel(sport->dma_rx_chan);
1932}
1933
1934static void lpuart_shutdown(struct uart_port *port)
1935{
1936	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
1937	u8 cr2;
1938	unsigned long flags;
1939
1940	uart_port_lock_irqsave(port, &flags);
1941
1942	/* disable Rx/Tx and interrupts */
1943	cr2 = readb(port->membase + UARTCR2);
1944	cr2 &= ~(UARTCR2_TE | UARTCR2_RE |
1945			UARTCR2_TIE | UARTCR2_TCIE | UARTCR2_RIE);
1946	writeb(cr2, port->membase + UARTCR2);
1947
1948	uart_port_unlock_irqrestore(port, flags);
1949
1950	lpuart_dma_shutdown(sport);
1951}
1952
1953static void lpuart32_shutdown(struct uart_port *port)
1954{
1955	struct lpuart_port *sport =
1956		container_of(port, struct lpuart_port, port);
1957	u32 temp;
1958	unsigned long flags;
1959
1960	uart_port_lock_irqsave(port, &flags);
1961
1962	/* clear status */
1963	temp = lpuart32_read(port, UARTSTAT);
1964	lpuart32_write(port, temp, UARTSTAT);
1965
1966	/* disable Rx/Tx DMA */
1967	temp = lpuart32_read(port, UARTBAUD);
1968	temp &= ~(UARTBAUD_TDMAE | UARTBAUD_RDMAE);
1969	lpuart32_write(port, temp, UARTBAUD);
1970
1971	/* disable Rx/Tx and interrupts and break condition */
1972	temp = lpuart32_read(port, UARTCTRL);
1973	temp &= ~(UARTCTRL_TE | UARTCTRL_RE | UARTCTRL_ILIE |
1974			UARTCTRL_TIE | UARTCTRL_TCIE | UARTCTRL_RIE | UARTCTRL_SBK);
1975	lpuart32_write(port, temp, UARTCTRL);
1976
1977	/* flush Rx/Tx FIFO */
1978	temp = lpuart32_read(port, UARTFIFO);
1979	temp |= UARTFIFO_TXFLUSH | UARTFIFO_RXFLUSH;
1980	lpuart32_write(port, temp, UARTFIFO);
1981
1982	uart_port_unlock_irqrestore(port, flags);
1983
1984	lpuart_dma_shutdown(sport);
1985}
1986
1987static void
1988lpuart_set_termios(struct uart_port *port, struct ktermios *termios,
1989		   const struct ktermios *old)
1990{
1991	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
1992	unsigned long flags;
1993	u8 cr1, old_cr1, old_cr2, cr3, cr4, bdh, modem;
1994	unsigned int  baud;
1995	unsigned int old_csize = old ? old->c_cflag & CSIZE : CS8;
1996	unsigned int sbr, brfa;
1997
1998	cr1 = old_cr1 = readb(port->membase + UARTCR1);
1999	old_cr2 = readb(port->membase + UARTCR2);
2000	cr3 = readb(port->membase + UARTCR3);
2001	cr4 = readb(port->membase + UARTCR4);
2002	bdh = readb(port->membase + UARTBDH);
2003	modem = readb(port->membase + UARTMODEM);
2004	/*
2005	 * only support CS8 and CS7, and for CS7 must enable PE.
2006	 * supported mode:
2007	 *  - (7,e/o,1)
2008	 *  - (8,n,1)
2009	 *  - (8,m/s,1)
2010	 *  - (8,e/o,1)
2011	 */
2012	while ((termios->c_cflag & CSIZE) != CS8 &&
2013		(termios->c_cflag & CSIZE) != CS7) {
2014		termios->c_cflag &= ~CSIZE;
2015		termios->c_cflag |= old_csize;
2016		old_csize = CS8;
2017	}
2018
2019	if ((termios->c_cflag & CSIZE) == CS8 ||
2020		(termios->c_cflag & CSIZE) == CS7)
2021		cr1 = old_cr1 & ~UARTCR1_M;
2022
2023	if (termios->c_cflag & CMSPAR) {
2024		if ((termios->c_cflag & CSIZE) != CS8) {
2025			termios->c_cflag &= ~CSIZE;
2026			termios->c_cflag |= CS8;
2027		}
2028		cr1 |= UARTCR1_M;
2029	}
2030
2031	/*
2032	 * When auto RS-485 RTS mode is enabled,
2033	 * hardware flow control need to be disabled.
2034	 */
2035	if (port->rs485.flags & SER_RS485_ENABLED)
2036		termios->c_cflag &= ~CRTSCTS;
2037
2038	if (termios->c_cflag & CRTSCTS)
2039		modem |= UARTMODEM_RXRTSE | UARTMODEM_TXCTSE;
2040	else
2041		modem &= ~(UARTMODEM_RXRTSE | UARTMODEM_TXCTSE);
2042
2043	termios->c_cflag &= ~CSTOPB;
2044
2045	/* parity must be enabled when CS7 to match 8-bits format */
2046	if ((termios->c_cflag & CSIZE) == CS7)
2047		termios->c_cflag |= PARENB;
2048
2049	if (termios->c_cflag & PARENB) {
2050		if (termios->c_cflag & CMSPAR) {
2051			cr1 &= ~UARTCR1_PE;
2052			if (termios->c_cflag & PARODD)
2053				cr3 |= UARTCR3_T8;
2054			else
2055				cr3 &= ~UARTCR3_T8;
2056		} else {
2057			cr1 |= UARTCR1_PE;
2058			if ((termios->c_cflag & CSIZE) == CS8)
2059				cr1 |= UARTCR1_M;
2060			if (termios->c_cflag & PARODD)
2061				cr1 |= UARTCR1_PT;
2062			else
2063				cr1 &= ~UARTCR1_PT;
2064		}
2065	} else {
2066		cr1 &= ~UARTCR1_PE;
2067	}
2068
2069	/* ask the core to calculate the divisor */
2070	baud = uart_get_baud_rate(port, termios, old, 50, port->uartclk / 16);
2071
2072	/*
2073	 * Need to update the Ring buffer length according to the selected
2074	 * baud rate and restart Rx DMA path.
2075	 *
2076	 * Since timer function acqures port->lock, need to stop before
2077	 * acquring same lock because otherwise timer_delete_sync() can deadlock.
2078	 */
2079	if (old && sport->lpuart_dma_rx_use)
2080		lpuart_dma_rx_free(port);
2081
2082	uart_port_lock_irqsave(port, &flags);
2083
2084	port->read_status_mask = 0;
2085	if (termios->c_iflag & INPCK)
2086		port->read_status_mask |= UARTSR1_FE | UARTSR1_PE;
2087	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
2088		port->read_status_mask |= UARTSR1_FE;
2089
2090	/* characters to ignore */
2091	port->ignore_status_mask = 0;
2092	if (termios->c_iflag & IGNPAR)
2093		port->ignore_status_mask |= UARTSR1_PE;
2094	if (termios->c_iflag & IGNBRK) {
2095		port->ignore_status_mask |= UARTSR1_FE;
2096		/*
2097		 * if we're ignoring parity and break indicators,
2098		 * ignore overruns too (for real raw support).
2099		 */
2100		if (termios->c_iflag & IGNPAR)
2101			port->ignore_status_mask |= UARTSR1_OR;
2102	}
2103
2104	/* update the per-port timeout */
2105	uart_update_timeout(port, termios->c_cflag, baud);
2106
2107	/* wait transmit engin complete */
2108	lpuart_wait_bit_set(port, UARTSR1, UARTSR1_TC);
2109
2110	/* disable transmit and receive */
2111	writeb(old_cr2 & ~(UARTCR2_TE | UARTCR2_RE),
2112			port->membase + UARTCR2);
2113
2114	sbr = port->uartclk / (16 * baud);
2115	brfa = ((port->uartclk - (16 * sbr * baud)) * 2) / baud;
2116	bdh &= ~UARTBDH_SBR_MASK;
2117	bdh |= (sbr >> 8) & 0x1F;
2118	cr4 &= ~UARTCR4_BRFA_MASK;
2119	brfa &= UARTCR4_BRFA_MASK;
2120	writeb(cr4 | brfa, port->membase + UARTCR4);
2121	writeb(bdh, port->membase + UARTBDH);
2122	writeb(sbr & 0xFF, port->membase + UARTBDL);
2123	writeb(cr3, port->membase + UARTCR3);
2124	writeb(cr1, port->membase + UARTCR1);
2125	writeb(modem, port->membase + UARTMODEM);
2126
2127	/* restore control register */
2128	writeb(old_cr2, port->membase + UARTCR2);
2129
2130	if (old && sport->lpuart_dma_rx_use) {
2131		if (!lpuart_start_rx_dma(sport))
2132			rx_dma_timer_init(sport);
2133		else
2134			sport->lpuart_dma_rx_use = false;
2135	}
2136
2137	uart_port_unlock_irqrestore(port, flags);
2138}
2139
2140static void __lpuart32_serial_setbrg(struct uart_port *port,
2141				     unsigned int baudrate, bool use_rx_dma,
2142				     bool use_tx_dma)
2143{
2144	u32 sbr, osr, baud_diff, tmp_osr, tmp_sbr, tmp_diff, baud;
2145	u32 clk = port->uartclk;
2146
2147	/*
2148	 * The idea is to use the best OSR (over-sampling rate) possible.
2149	 * Note, OSR is typically hard-set to 16 in other LPUART instantiations.
2150	 * Loop to find the best OSR value possible, one that generates minimum
2151	 * baud_diff iterate through the rest of the supported values of OSR.
2152	 *
2153	 * Calculation Formula:
2154	 *  Baud Rate = baud clock / ((OSR+1) × SBR)
2155	 */
2156	baud_diff = baudrate;
2157	osr = 0;
2158	sbr = 0;
2159
2160	for (tmp_osr = 4; tmp_osr <= 32; tmp_osr++) {
2161		/* calculate the temporary sbr value  */
2162		tmp_sbr = (clk / (baudrate * tmp_osr));
2163		if (tmp_sbr == 0)
2164			tmp_sbr = 1;
2165
2166		/*
2167		 * calculate the baud rate difference based on the temporary
2168		 * osr and sbr values
2169		 */
2170		tmp_diff = clk / (tmp_osr * tmp_sbr) - baudrate;
2171
2172		/* select best values between sbr and sbr+1 */
2173		baud = clk / (tmp_osr * (tmp_sbr + 1));
2174		if (tmp_diff > (baudrate - baud)) {
2175			tmp_diff = baudrate - baud;
2176			tmp_sbr++;
2177		}
2178
2179		if (tmp_sbr > UARTBAUD_SBR_MASK)
2180			continue;
2181
2182		if (tmp_diff <= baud_diff) {
2183			baud_diff = tmp_diff;
2184			osr = tmp_osr;
2185			sbr = tmp_sbr;
2186
2187			if (!baud_diff)
2188				break;
2189		}
2190	}
2191
2192	/* handle buadrate outside acceptable rate */
2193	if (baud_diff > ((baudrate / 100) * 3))
2194		dev_warn(port->dev,
2195			 "unacceptable baud rate difference of more than 3%%\n");
2196
2197	baud = lpuart32_read(port, UARTBAUD);
2198
2199	if ((osr > 3) && (osr < 8))
2200		baud |= UARTBAUD_BOTHEDGE;
2201
2202	baud &= ~(UARTBAUD_OSR_MASK << UARTBAUD_OSR_SHIFT);
2203	baud |= ((osr-1) & UARTBAUD_OSR_MASK) << UARTBAUD_OSR_SHIFT;
2204
2205	baud &= ~UARTBAUD_SBR_MASK;
2206	baud |= sbr & UARTBAUD_SBR_MASK;
2207
2208	if (!use_rx_dma)
2209		baud &= ~UARTBAUD_RDMAE;
2210	if (!use_tx_dma)
2211		baud &= ~UARTBAUD_TDMAE;
2212
2213	lpuart32_write(port, baud, UARTBAUD);
2214}
2215
2216static void lpuart32_serial_setbrg(struct lpuart_port *sport,
2217				   unsigned int baudrate)
2218{
2219	__lpuart32_serial_setbrg(&sport->port, baudrate,
2220				 sport->lpuart_dma_rx_use,
2221				 sport->lpuart_dma_tx_use);
2222}
2223
2224
2225static void
2226lpuart32_set_termios(struct uart_port *port, struct ktermios *termios,
2227		     const struct ktermios *old)
2228{
2229	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
2230	unsigned long flags;
2231	u32 ctrl, old_ctrl, bd, modem;
2232	unsigned int  baud;
2233	unsigned int old_csize = old ? old->c_cflag & CSIZE : CS8;
2234
2235	ctrl = old_ctrl = lpuart32_read(port, UARTCTRL);
2236	bd = lpuart32_read(port, UARTBAUD);
2237	modem = lpuart32_read(port, UARTMODIR);
2238	sport->is_cs7 = false;
2239	/*
2240	 * only support CS8 and CS7
2241	 * supported mode:
2242	 *  - (7,n,1) (imx only)
2243	 *  - (7,e/o,1)
2244	 *  - (8,n,1)
2245	 *  - (8,m/s,1)
2246	 *  - (8,e/o,1)
2247	 */
2248	while ((termios->c_cflag & CSIZE) != CS8 &&
2249		(termios->c_cflag & CSIZE) != CS7) {
2250		termios->c_cflag &= ~CSIZE;
2251		termios->c_cflag |= old_csize;
2252		old_csize = CS8;
2253	}
2254
2255	if ((termios->c_cflag & CSIZE) == CS8 ||
2256		(termios->c_cflag & CSIZE) == CS7)
2257		ctrl = old_ctrl & ~(UARTCTRL_M | UARTCTRL_M7);
2258
2259	if (termios->c_cflag & CMSPAR) {
2260		if ((termios->c_cflag & CSIZE) != CS8) {
2261			termios->c_cflag &= ~CSIZE;
2262			termios->c_cflag |= CS8;
2263		}
2264		ctrl |= UARTCTRL_M;
2265	}
2266
2267	/*
2268	 * When auto RS-485 RTS mode is enabled,
2269	 * hardware flow control need to be disabled.
2270	 */
2271	if (port->rs485.flags & SER_RS485_ENABLED)
2272		termios->c_cflag &= ~CRTSCTS;
2273
2274	if (termios->c_cflag & CRTSCTS)
2275		modem |= UARTMODIR_RXRTSE | UARTMODIR_TXCTSE;
2276	else
2277		modem &= ~(UARTMODIR_RXRTSE | UARTMODIR_TXCTSE);
2278
2279	if (termios->c_cflag & CSTOPB)
2280		bd |= UARTBAUD_SBNS;
2281	else
2282		bd &= ~UARTBAUD_SBNS;
2283
2284	/*
2285	 * imx support 7-bits format, no limitation on parity when CS7
2286	 * for layerscape, parity must be enabled when CS7 to match 8-bits format
2287	 */
2288	if ((termios->c_cflag & CSIZE) == CS7 && !(termios->c_cflag & PARENB)) {
2289		if (is_imx7ulp_lpuart(sport) ||
2290		    is_imx8ulp_lpuart(sport) ||
2291		    is_imx8qxp_lpuart(sport))
2292			ctrl |= UARTCTRL_M7;
2293		else
2294			termios->c_cflag |= PARENB;
2295	}
2296
2297	if ((termios->c_cflag & PARENB)) {
2298		if (termios->c_cflag & CMSPAR) {
2299			ctrl &= ~UARTCTRL_PE;
2300			ctrl |= UARTCTRL_M;
2301		} else {
2302			ctrl |= UARTCTRL_PE;
2303			if ((termios->c_cflag & CSIZE) == CS8)
2304				ctrl |= UARTCTRL_M;
2305			if (termios->c_cflag & PARODD)
2306				ctrl |= UARTCTRL_PT;
2307			else
2308				ctrl &= ~UARTCTRL_PT;
2309		}
2310	} else {
2311		ctrl &= ~UARTCTRL_PE;
2312	}
2313
2314	/* ask the core to calculate the divisor */
2315	baud = uart_get_baud_rate(port, termios, old, 50, port->uartclk / 4);
2316
2317	/*
2318	 * Need to update the Ring buffer length according to the selected
2319	 * baud rate and restart Rx DMA path.
2320	 *
2321	 * Since timer function acqures port->lock, need to stop before
2322	 * acquring same lock because otherwise timer_delete_sync() can deadlock.
2323	 */
2324	if (old && sport->lpuart_dma_rx_use)
2325		lpuart_dma_rx_free(port);
2326
2327	uart_port_lock_irqsave(port, &flags);
2328
2329	port->read_status_mask = 0;
2330	if (termios->c_iflag & INPCK)
2331		port->read_status_mask |= UARTSTAT_FE | UARTSTAT_PE;
2332	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
2333		port->read_status_mask |= UARTSTAT_FE;
2334
2335	/* characters to ignore */
2336	port->ignore_status_mask = 0;
2337	if (termios->c_iflag & IGNPAR)
2338		port->ignore_status_mask |= UARTSTAT_PE;
2339	if (termios->c_iflag & IGNBRK) {
2340		port->ignore_status_mask |= UARTSTAT_FE;
2341		/*
2342		 * if we're ignoring parity and break indicators,
2343		 * ignore overruns too (for real raw support).
2344		 */
2345		if (termios->c_iflag & IGNPAR)
2346			port->ignore_status_mask |= UARTSTAT_OR;
2347	}
2348
2349	/* update the per-port timeout */
2350	uart_update_timeout(port, termios->c_cflag, baud);
2351
2352	/*
2353	 * disable CTS to ensure the transmit engine is not blocked by the flow
2354	 * control when there is dirty data in TX FIFO
2355	 */
2356	lpuart32_write(port, modem & ~UARTMODIR_TXCTSE, UARTMODIR);
2357
2358	/*
2359	 * LPUART Transmission Complete Flag may never be set while queuing a break
2360	 * character, so skip waiting for transmission complete when UARTCTRL_SBK is
2361	 * asserted.
2362	 */
2363	if (!(old_ctrl & UARTCTRL_SBK))
2364		lpuart32_wait_bit_set(port, UARTSTAT, UARTSTAT_TC);
2365
2366	/* disable transmit and receive */
2367	lpuart32_write(port, old_ctrl & ~(UARTCTRL_TE | UARTCTRL_RE),
2368		       UARTCTRL);
2369
2370	lpuart32_write(port, bd, UARTBAUD);
2371	lpuart32_serial_setbrg(sport, baud);
2372	/* restore control register */
2373	lpuart32_write(port, ctrl, UARTCTRL);
2374	/* re-enable the CTS if needed */
2375	lpuart32_write(port, modem, UARTMODIR);
2376
2377	if ((ctrl & (UARTCTRL_PE | UARTCTRL_M)) == UARTCTRL_PE)
2378		sport->is_cs7 = true;
2379
2380	if (old && sport->lpuart_dma_rx_use) {
2381		if (!lpuart_start_rx_dma(sport))
2382			rx_dma_timer_init(sport);
2383		else
2384			sport->lpuart_dma_rx_use = false;
2385	}
2386
2387	uart_port_unlock_irqrestore(port, flags);
2388}
2389
2390static const char *lpuart_type(struct uart_port *port)
2391{
2392	return "FSL_LPUART";
2393}
2394
2395static void lpuart_release_port(struct uart_port *port)
2396{
2397	/* nothing to do */
2398}
2399
2400static int lpuart_request_port(struct uart_port *port)
2401{
2402	return  0;
2403}
2404
2405/* configure/autoconfigure the port */
2406static void lpuart_config_port(struct uart_port *port, int flags)
2407{
2408	if (flags & UART_CONFIG_TYPE)
2409		port->type = PORT_LPUART;
2410}
2411
2412static int lpuart_verify_port(struct uart_port *port, struct serial_struct *ser)
2413{
2414	int ret = 0;
2415
2416	if (ser->type != PORT_UNKNOWN && ser->type != PORT_LPUART)
2417		ret = -EINVAL;
2418	if (port->irq != ser->irq)
2419		ret = -EINVAL;
2420	if (ser->io_type != UPIO_MEM)
2421		ret = -EINVAL;
2422	if (port->uartclk / 16 != ser->baud_base)
2423		ret = -EINVAL;
2424	if (port->iobase != ser->port)
2425		ret = -EINVAL;
2426	if (ser->hub6 != 0)
2427		ret = -EINVAL;
2428	return ret;
2429}
2430
2431static const struct uart_ops lpuart_pops = {
2432	.tx_empty	= lpuart_tx_empty,
2433	.set_mctrl	= lpuart_set_mctrl,
2434	.get_mctrl	= lpuart_get_mctrl,
2435	.stop_tx	= lpuart_stop_tx,
2436	.start_tx	= lpuart_start_tx,
2437	.stop_rx	= lpuart_stop_rx,
2438	.break_ctl	= lpuart_break_ctl,
2439	.startup	= lpuart_startup,
2440	.shutdown	= lpuart_shutdown,
2441	.set_termios	= lpuart_set_termios,
2442	.pm		= lpuart_uart_pm,
2443	.type		= lpuart_type,
2444	.request_port	= lpuart_request_port,
2445	.release_port	= lpuart_release_port,
2446	.config_port	= lpuart_config_port,
2447	.verify_port	= lpuart_verify_port,
2448	.flush_buffer	= lpuart_flush_buffer,
2449#if defined(CONFIG_CONSOLE_POLL)
2450	.poll_init	= lpuart_poll_init,
2451	.poll_get_char	= lpuart_poll_get_char,
2452	.poll_put_char	= lpuart_poll_put_char,
2453#endif
2454};
2455
2456static const struct uart_ops lpuart32_pops = {
2457	.tx_empty	= lpuart32_tx_empty,
2458	.set_mctrl	= lpuart32_set_mctrl,
2459	.get_mctrl	= lpuart32_get_mctrl,
2460	.stop_tx	= lpuart32_stop_tx,
2461	.start_tx	= lpuart32_start_tx,
2462	.stop_rx	= lpuart32_stop_rx,
2463	.break_ctl	= lpuart32_break_ctl,
2464	.startup	= lpuart32_startup,
2465	.shutdown	= lpuart32_shutdown,
2466	.set_termios	= lpuart32_set_termios,
2467	.pm		= lpuart_uart_pm,
2468	.type		= lpuart_type,
2469	.request_port	= lpuart_request_port,
2470	.release_port	= lpuart_release_port,
2471	.config_port	= lpuart_config_port,
2472	.verify_port	= lpuart_verify_port,
2473	.flush_buffer	= lpuart_flush_buffer,
2474#if defined(CONFIG_CONSOLE_POLL)
2475	.poll_init	= lpuart32_poll_init,
2476	.poll_get_char	= lpuart32_poll_get_char,
2477	.poll_put_char	= lpuart32_poll_put_char,
2478#endif
2479};
2480
2481static struct lpuart_port *lpuart_ports[UART_NR];
2482
2483#ifdef CONFIG_SERIAL_FSL_LPUART_CONSOLE
2484static void lpuart_console_putchar(struct uart_port *port, unsigned char ch)
2485{
2486	lpuart_wait_bit_set(port, UARTSR1, UARTSR1_TDRE);
2487	writeb(ch, port->membase + UARTDR);
2488}
2489
2490static void lpuart32_console_putchar(struct uart_port *port, unsigned char ch)
2491{
2492	lpuart32_wait_bit_set(port, UARTSTAT, UARTSTAT_TDRE);
2493	lpuart32_write(port, ch, UARTDATA);
2494}
2495
2496static void
2497lpuart_console_write(struct console *co, const char *s, unsigned int count)
2498{
2499	struct lpuart_port *sport = lpuart_ports[co->index];
2500	u8  old_cr2, cr2;
2501	unsigned long flags;
2502	int locked = 1;
2503
2504	if (oops_in_progress)
2505		locked = uart_port_trylock_irqsave(&sport->port, &flags);
2506	else
2507		uart_port_lock_irqsave(&sport->port, &flags);
2508
2509	/* first save CR2 and then disable interrupts */
2510	cr2 = old_cr2 = readb(sport->port.membase + UARTCR2);
2511	cr2 |= UARTCR2_TE | UARTCR2_RE;
2512	cr2 &= ~(UARTCR2_TIE | UARTCR2_TCIE | UARTCR2_RIE);
2513	writeb(cr2, sport->port.membase + UARTCR2);
2514
2515	uart_console_write(&sport->port, s, count, lpuart_console_putchar);
2516
2517	/* wait for transmitter finish complete and restore CR2 */
2518	lpuart_wait_bit_set(&sport->port, UARTSR1, UARTSR1_TC);
2519
2520	writeb(old_cr2, sport->port.membase + UARTCR2);
2521
2522	if (locked)
2523		uart_port_unlock_irqrestore(&sport->port, flags);
2524}
2525
2526static void
2527lpuart32_console_write(struct console *co, const char *s, unsigned int count)
2528{
2529	struct lpuart_port *sport = lpuart_ports[co->index];
2530	u32 old_cr, cr;
2531	unsigned long flags;
2532	int locked = 1;
2533
2534	if (oops_in_progress)
2535		locked = uart_port_trylock_irqsave(&sport->port, &flags);
2536	else
2537		uart_port_lock_irqsave(&sport->port, &flags);
2538
2539	/* first save CR2 and then disable interrupts */
2540	cr = old_cr = lpuart32_read(&sport->port, UARTCTRL);
2541	cr |= UARTCTRL_TE | UARTCTRL_RE;
2542	cr &= ~(UARTCTRL_TIE | UARTCTRL_TCIE | UARTCTRL_RIE);
2543	lpuart32_write(&sport->port, cr, UARTCTRL);
2544
2545	uart_console_write(&sport->port, s, count, lpuart32_console_putchar);
2546
2547	/* wait for transmitter finish complete and restore CR2 */
2548	lpuart32_wait_bit_set(&sport->port, UARTSTAT, UARTSTAT_TC);
2549
2550	lpuart32_write(&sport->port, old_cr, UARTCTRL);
2551
2552	if (locked)
2553		uart_port_unlock_irqrestore(&sport->port, flags);
2554}
2555
2556/*
2557 * if the port was already initialised (eg, by a boot loader),
2558 * try to determine the current setup.
2559 */
2560static void __init
2561lpuart_console_get_options(struct lpuart_port *sport, int *baud,
2562			   int *parity, int *bits)
2563{
2564	u8 cr, bdh, bdl, brfa;
2565	unsigned int sbr, uartclk, baud_raw;
2566
2567	cr = readb(sport->port.membase + UARTCR2);
2568	cr &= UARTCR2_TE | UARTCR2_RE;
2569	if (!cr)
2570		return;
2571
2572	/* ok, the port was enabled */
2573
2574	cr = readb(sport->port.membase + UARTCR1);
2575
2576	*parity = 'n';
2577	if (cr & UARTCR1_PE) {
2578		if (cr & UARTCR1_PT)
2579			*parity = 'o';
2580		else
2581			*parity = 'e';
2582	}
2583
2584	if (cr & UARTCR1_M)
2585		*bits = 9;
2586	else
2587		*bits = 8;
2588
2589	bdh = readb(sport->port.membase + UARTBDH);
2590	bdh &= UARTBDH_SBR_MASK;
2591	bdl = readb(sport->port.membase + UARTBDL);
2592	sbr = bdh;
2593	sbr <<= 8;
2594	sbr |= bdl;
2595	brfa = readb(sport->port.membase + UARTCR4);
2596	brfa &= UARTCR4_BRFA_MASK;
2597
2598	uartclk = lpuart_get_baud_clk_rate(sport);
2599	/*
2600	 * baud = mod_clk/(16*(sbr[13]+(brfa)/32)
2601	 */
2602	baud_raw = uartclk / (16 * (sbr + brfa / 32));
2603
2604	if (*baud != baud_raw)
2605		dev_info(sport->port.dev, "Serial: Console lpuart rounded baud rate"
2606				"from %d to %d\n", baud_raw, *baud);
2607}
2608
2609static void __init
2610lpuart32_console_get_options(struct lpuart_port *sport, int *baud,
2611			   int *parity, int *bits)
2612{
2613	u32 cr, bd;
2614	unsigned int sbr, uartclk, baud_raw;
2615
2616	cr = lpuart32_read(&sport->port, UARTCTRL);
2617	cr &= UARTCTRL_TE | UARTCTRL_RE;
2618	if (!cr)
2619		return;
2620
2621	/* ok, the port was enabled */
2622
2623	cr = lpuart32_read(&sport->port, UARTCTRL);
2624
2625	*parity = 'n';
2626	if (cr & UARTCTRL_PE) {
2627		if (cr & UARTCTRL_PT)
2628			*parity = 'o';
2629		else
2630			*parity = 'e';
2631	}
2632
2633	if (cr & UARTCTRL_M)
2634		*bits = 9;
2635	else
2636		*bits = 8;
2637
2638	bd = lpuart32_read(&sport->port, UARTBAUD);
2639	bd &= UARTBAUD_SBR_MASK;
2640	if (!bd)
2641		return;
2642
2643	sbr = bd;
2644	uartclk = lpuart_get_baud_clk_rate(sport);
2645	/*
2646	 * baud = mod_clk/(16*(sbr[13]+(brfa)/32)
2647	 */
2648	baud_raw = uartclk / (16 * sbr);
2649
2650	if (*baud != baud_raw)
2651		dev_info(sport->port.dev, "Serial: Console lpuart rounded baud rate"
2652				"from %d to %d\n", baud_raw, *baud);
2653}
2654
2655static int __init lpuart_console_setup(struct console *co, char *options)
2656{
2657	struct lpuart_port *sport;
2658	int baud = 115200;
2659	int bits = 8;
2660	int parity = 'n';
2661	int flow = 'n';
2662
2663	/*
2664	 * check whether an invalid uart number has been specified, and
2665	 * if so, search for the first available port that does have
2666	 * console support.
2667	 */
2668	if (co->index == -1 || co->index >= ARRAY_SIZE(lpuart_ports))
2669		co->index = 0;
2670
2671	sport = lpuart_ports[co->index];
2672	if (sport == NULL)
2673		return -ENODEV;
2674
2675	if (options)
2676		uart_parse_options(options, &baud, &parity, &bits, &flow);
2677	else
2678		if (lpuart_is_32(sport))
2679			lpuart32_console_get_options(sport, &baud, &parity, &bits);
2680		else
2681			lpuart_console_get_options(sport, &baud, &parity, &bits);
2682
2683	if (lpuart_is_32(sport))
2684		lpuart32_setup_watermark(sport);
2685	else
2686		lpuart_setup_watermark(sport);
2687
2688	return uart_set_options(&sport->port, co, baud, parity, bits, flow);
2689}
2690
2691static struct uart_driver lpuart_reg;
2692static struct console lpuart_console = {
2693	.name		= DEV_NAME,
2694	.write		= lpuart_console_write,
2695	.device		= uart_console_device,
2696	.setup		= lpuart_console_setup,
2697	.flags		= CON_PRINTBUFFER,
2698	.index		= -1,
2699	.data		= &lpuart_reg,
2700};
2701
2702static struct console lpuart32_console = {
2703	.name		= DEV_NAME,
2704	.write		= lpuart32_console_write,
2705	.device		= uart_console_device,
2706	.setup		= lpuart_console_setup,
2707	.flags		= CON_PRINTBUFFER,
2708	.index		= -1,
2709	.data		= &lpuart_reg,
2710};
2711
2712static void lpuart_early_write(struct console *con, const char *s, unsigned n)
2713{
2714	struct earlycon_device *dev = con->data;
2715
2716	uart_console_write(&dev->port, s, n, lpuart_console_putchar);
2717}
2718
2719static void lpuart32_early_write(struct console *con, const char *s, unsigned n)
2720{
2721	struct earlycon_device *dev = con->data;
2722
2723	uart_console_write(&dev->port, s, n, lpuart32_console_putchar);
2724}
2725
2726static int __init lpuart_early_console_setup(struct earlycon_device *device,
2727					  const char *opt)
2728{
2729	if (!device->port.membase)
2730		return -ENODEV;
2731
2732	device->con->write = lpuart_early_write;
2733	return 0;
2734}
2735
2736static int __init lpuart32_early_console_setup(struct earlycon_device *device,
2737					  const char *opt)
2738{
2739	if (!device->port.membase)
2740		return -ENODEV;
2741
2742	if (device->port.iotype != UPIO_MEM32)
2743		device->port.iotype = UPIO_MEM32BE;
2744
2745	device->con->write = lpuart32_early_write;
2746	return 0;
2747}
2748
2749static int __init ls1028a_early_console_setup(struct earlycon_device *device,
2750					      const char *opt)
2751{
2752	u32 cr;
2753
2754	if (!device->port.membase)
2755		return -ENODEV;
2756
2757	device->port.iotype = UPIO_MEM32;
2758	device->con->write = lpuart32_early_write;
2759
2760	/* set the baudrate */
2761	if (device->port.uartclk && device->baud)
2762		__lpuart32_serial_setbrg(&device->port, device->baud,
2763					 false, false);
2764
2765	/* enable transmitter */
2766	cr = lpuart32_read(&device->port, UARTCTRL);
2767	cr |= UARTCTRL_TE;
2768	lpuart32_write(&device->port, cr, UARTCTRL);
2769
2770	return 0;
2771}
2772
2773static int __init lpuart32_imx_early_console_setup(struct earlycon_device *device,
2774						   const char *opt)
2775{
2776	if (!device->port.membase)
2777		return -ENODEV;
2778
2779	device->port.iotype = UPIO_MEM32;
2780	device->port.membase += IMX_REG_OFF;
2781	device->con->write = lpuart32_early_write;
2782
2783	return 0;
2784}
2785OF_EARLYCON_DECLARE(lpuart, "fsl,vf610-lpuart", lpuart_early_console_setup);
2786OF_EARLYCON_DECLARE(lpuart32, "fsl,ls1021a-lpuart", lpuart32_early_console_setup);
2787OF_EARLYCON_DECLARE(lpuart32, "fsl,ls1028a-lpuart", ls1028a_early_console_setup);
2788OF_EARLYCON_DECLARE(lpuart32, "fsl,imx7ulp-lpuart", lpuart32_imx_early_console_setup);
2789OF_EARLYCON_DECLARE(lpuart32, "fsl,imx8ulp-lpuart", lpuart32_imx_early_console_setup);
2790OF_EARLYCON_DECLARE(lpuart32, "fsl,imx8qxp-lpuart", lpuart32_imx_early_console_setup);
2791OF_EARLYCON_DECLARE(lpuart32, "fsl,imxrt1050-lpuart", lpuart32_imx_early_console_setup);
2792EARLYCON_DECLARE(lpuart, lpuart_early_console_setup);
2793EARLYCON_DECLARE(lpuart32, lpuart32_early_console_setup);
2794
2795#define LPUART_CONSOLE	(&lpuart_console)
2796#define LPUART32_CONSOLE	(&lpuart32_console)
2797#else
2798#define LPUART_CONSOLE	NULL
2799#define LPUART32_CONSOLE	NULL
2800#endif
2801
2802static struct uart_driver lpuart_reg = {
2803	.owner		= THIS_MODULE,
2804	.driver_name	= DRIVER_NAME,
2805	.dev_name	= DEV_NAME,
2806	.nr		= ARRAY_SIZE(lpuart_ports),
2807	.cons		= LPUART_CONSOLE,
2808};
2809
2810static const struct serial_rs485 lpuart_rs485_supported = {
2811	.flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND | SER_RS485_RTS_AFTER_SEND,
2812	/* delay_rts_* and RX_DURING_TX are not supported */
2813};
2814
2815static int lpuart_global_reset(struct lpuart_port *sport)
2816{
2817	struct uart_port *port = &sport->port;
2818	void __iomem *global_addr;
2819	u32 ctrl, bd;
2820	unsigned int val = 0;
2821	int ret;
2822
2823	ret = clk_prepare_enable(sport->ipg_clk);
2824	if (ret) {
2825		dev_err(port->dev, "failed to enable uart ipg clk: %d\n", ret);
2826		return ret;
2827	}
2828
2829	if (is_imx7ulp_lpuart(sport) || is_imx8ulp_lpuart(sport) || is_imx8qxp_lpuart(sport)) {
2830		/*
2831		 * If the transmitter is used by earlycon, wait for transmit engine to
2832		 * complete and then reset.
2833		 */
2834		ctrl = lpuart32_read(port, UARTCTRL);
2835		if (ctrl & UARTCTRL_TE) {
2836			bd = lpuart32_read(port, UARTBAUD);
2837			if (read_poll_timeout(lpuart32_tx_empty, val, val, 1, 100000, false,
2838					      port)) {
2839				dev_warn(port->dev,
2840					 "timeout waiting for transmit engine to complete\n");
2841				clk_disable_unprepare(sport->ipg_clk);
2842				return 0;
2843			}
2844		}
2845
2846		global_addr = port->membase + UART_GLOBAL - IMX_REG_OFF;
2847		writel(UART_GLOBAL_RST, global_addr);
2848		usleep_range(GLOBAL_RST_MIN_US, GLOBAL_RST_MAX_US);
2849		writel(0, global_addr);
2850		usleep_range(GLOBAL_RST_MIN_US, GLOBAL_RST_MAX_US);
2851
2852		/* Recover the transmitter for earlycon. */
2853		if (ctrl & UARTCTRL_TE) {
2854			lpuart32_write(port, bd, UARTBAUD);
2855			lpuart32_write(port, ctrl, UARTCTRL);
2856		}
2857	}
2858
2859	clk_disable_unprepare(sport->ipg_clk);
2860	return 0;
2861}
2862
2863static int lpuart_probe(struct platform_device *pdev)
2864{
2865	const struct lpuart_soc_data *sdata = of_device_get_match_data(&pdev->dev);
2866	struct device_node *np = pdev->dev.of_node;
2867	struct lpuart_port *sport;
2868	struct resource *res;
2869	irq_handler_t handler;
2870	int ret;
2871
2872	sport = devm_kzalloc(&pdev->dev, sizeof(*sport), GFP_KERNEL);
2873	if (!sport)
2874		return -ENOMEM;
2875
2876	sport->port.membase = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2877	if (IS_ERR(sport->port.membase))
2878		return PTR_ERR(sport->port.membase);
2879
2880	sport->port.membase += sdata->reg_off;
2881	sport->port.mapbase = res->start + sdata->reg_off;
2882	sport->port.dev = &pdev->dev;
2883	sport->port.type = PORT_LPUART;
2884	sport->devtype = sdata->devtype;
2885	sport->rx_watermark = sdata->rx_watermark;
2886	sport->dma_idle_int = is_imx7ulp_lpuart(sport) || is_imx8ulp_lpuart(sport) ||
2887			      is_imx8qxp_lpuart(sport);
2888	ret = platform_get_irq(pdev, 0);
2889	if (ret < 0)
2890		return ret;
2891	sport->port.irq = ret;
2892	sport->port.iotype = sdata->iotype;
2893	if (lpuart_is_32(sport))
2894		sport->port.ops = &lpuart32_pops;
2895	else
2896		sport->port.ops = &lpuart_pops;
2897	sport->port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_FSL_LPUART_CONSOLE);
2898	sport->port.flags = UPF_BOOT_AUTOCONF;
2899
2900	if (lpuart_is_32(sport))
2901		sport->port.rs485_config = lpuart32_config_rs485;
2902	else
2903		sport->port.rs485_config = lpuart_config_rs485;
2904	sport->port.rs485_supported = lpuart_rs485_supported;
2905
2906	sport->ipg_clk = devm_clk_get(&pdev->dev, "ipg");
2907	if (IS_ERR(sport->ipg_clk)) {
2908		ret = PTR_ERR(sport->ipg_clk);
2909		return dev_err_probe(&pdev->dev, ret, "failed to get uart ipg clk\n");
2910	}
2911
2912	sport->baud_clk = NULL;
2913	if (is_imx8qxp_lpuart(sport)) {
2914		sport->baud_clk = devm_clk_get(&pdev->dev, "baud");
2915		if (IS_ERR(sport->baud_clk)) {
2916			ret = PTR_ERR(sport->baud_clk);
2917			return dev_err_probe(&pdev->dev, ret, "failed to get uart baud clk\n");
2918		}
2919	}
2920
2921	ret = of_alias_get_id(np, "serial");
2922	if (ret < 0) {
2923		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", ret);
2924		return ret;
2925	}
2926	if (ret >= ARRAY_SIZE(lpuart_ports)) {
2927		dev_err(&pdev->dev, "serial%d out of range\n", ret);
2928		return -EINVAL;
2929	}
2930	sport->port.line = ret;
2931
2932	ret = lpuart_enable_clks(sport);
2933	if (ret)
2934		return ret;
2935	sport->port.uartclk = lpuart_get_baud_clk_rate(sport);
2936
2937	lpuart_ports[sport->port.line] = sport;
2938
2939	platform_set_drvdata(pdev, &sport->port);
2940
2941	if (lpuart_is_32(sport)) {
2942		lpuart_reg.cons = LPUART32_CONSOLE;
2943		handler = lpuart32_int;
2944	} else {
2945		lpuart_reg.cons = LPUART_CONSOLE;
2946		handler = lpuart_int;
2947	}
2948
2949	pm_runtime_use_autosuspend(&pdev->dev);
2950	pm_runtime_set_autosuspend_delay(&pdev->dev, UART_AUTOSUSPEND_TIMEOUT);
2951	pm_runtime_set_active(&pdev->dev);
2952	pm_runtime_enable(&pdev->dev);
2953	pm_runtime_mark_last_busy(&pdev->dev);
2954
2955	ret = lpuart_global_reset(sport);
2956	if (ret)
2957		goto failed_reset;
2958
2959	ret = uart_get_rs485_mode(&sport->port);
2960	if (ret)
2961		goto failed_get_rs485;
2962
2963	ret = uart_add_one_port(&lpuart_reg, &sport->port);
2964	if (ret)
2965		goto failed_attach_port;
2966
2967	ret = devm_request_irq(&pdev->dev, sport->port.irq, handler, 0,
2968			       dev_name(&pdev->dev), sport);
2969	if (ret)
2970		goto failed_irq_request;
2971
2972	return 0;
2973
2974failed_irq_request:
2975	uart_remove_one_port(&lpuart_reg, &sport->port);
2976failed_attach_port:
2977failed_get_rs485:
2978failed_reset:
2979	pm_runtime_disable(&pdev->dev);
2980	pm_runtime_set_suspended(&pdev->dev);
2981	pm_runtime_dont_use_autosuspend(&pdev->dev);
2982	lpuart_disable_clks(sport);
2983	return ret;
2984}
2985
2986static void lpuart_remove(struct platform_device *pdev)
2987{
2988	struct lpuart_port *sport = platform_get_drvdata(pdev);
2989
2990	uart_remove_one_port(&lpuart_reg, &sport->port);
2991
2992	lpuart_disable_clks(sport);
2993
2994	if (sport->dma_tx_chan)
2995		dma_release_channel(sport->dma_tx_chan);
2996
2997	if (sport->dma_rx_chan)
2998		dma_release_channel(sport->dma_rx_chan);
2999
3000	pm_runtime_disable(&pdev->dev);
3001	pm_runtime_set_suspended(&pdev->dev);
3002	pm_runtime_dont_use_autosuspend(&pdev->dev);
3003}
3004
3005static int lpuart_runtime_suspend(struct device *dev)
3006{
3007	struct platform_device *pdev = to_platform_device(dev);
3008	struct lpuart_port *sport = platform_get_drvdata(pdev);
3009
3010	lpuart_disable_clks(sport);
3011
3012	return 0;
3013};
3014
3015static int lpuart_runtime_resume(struct device *dev)
3016{
3017	struct platform_device *pdev = to_platform_device(dev);
3018	struct lpuart_port *sport = platform_get_drvdata(pdev);
3019
3020	return lpuart_enable_clks(sport);
3021};
3022
3023static void serial_lpuart_enable_wakeup(struct lpuart_port *sport, bool on)
3024{
3025	u32 val, baud;
3026
3027	if (lpuart_is_32(sport)) {
3028		val = lpuart32_read(&sport->port, UARTCTRL);
3029		baud = lpuart32_read(&sport->port, UARTBAUD);
3030		if (on) {
3031			/* set rx_watermark to 0 in wakeup source mode */
3032			lpuart32_write(&sport->port, 0, UARTWATER);
3033			val |= UARTCTRL_RIE;
3034			/* clear RXEDGIF flag before enable RXEDGIE interrupt */
3035			lpuart32_write(&sport->port, UARTSTAT_RXEDGIF, UARTSTAT);
3036			baud |= UARTBAUD_RXEDGIE;
3037		} else {
3038			val &= ~UARTCTRL_RIE;
3039			baud &= ~UARTBAUD_RXEDGIE;
3040		}
3041		lpuart32_write(&sport->port, val, UARTCTRL);
3042		lpuart32_write(&sport->port, baud, UARTBAUD);
3043	} else {
3044		val = readb(sport->port.membase + UARTCR2);
3045		if (on)
3046			val |= UARTCR2_RIE;
3047		else
3048			val &= ~UARTCR2_RIE;
3049		writeb(val, sport->port.membase + UARTCR2);
3050	}
3051}
3052
3053static bool lpuart_uport_is_active(struct lpuart_port *sport)
3054{
3055	struct tty_port *port = &sport->port.state->port;
3056	struct tty_struct *tty;
3057	struct device *tty_dev;
3058	int may_wake = 0;
3059
3060	tty = tty_port_tty_get(port);
3061	if (tty) {
3062		tty_dev = tty->dev;
3063		may_wake = tty_dev && device_may_wakeup(tty_dev);
3064		tty_kref_put(tty);
3065	}
3066
3067	if ((tty_port_initialized(port) && may_wake) ||
3068	    (!console_suspend_enabled && uart_console(&sport->port)))
3069		return true;
3070
3071	return false;
3072}
3073
3074static int lpuart_suspend_noirq(struct device *dev)
3075{
3076	struct lpuart_port *sport = dev_get_drvdata(dev);
3077	bool irq_wake = irqd_is_wakeup_set(irq_get_irq_data(sport->port.irq));
3078
3079	if (lpuart_uport_is_active(sport))
3080		serial_lpuart_enable_wakeup(sport, !!irq_wake);
3081
3082	pinctrl_pm_select_sleep_state(dev);
3083
3084	return 0;
3085}
3086
3087static int lpuart_resume_noirq(struct device *dev)
3088{
3089	struct lpuart_port *sport = dev_get_drvdata(dev);
3090	struct tty_port *port = &sport->port.state->port;
3091	bool wake_active;
3092	u32 stat;
3093
3094	pinctrl_pm_select_default_state(dev);
3095
3096	if (lpuart_uport_is_active(sport)) {
3097		serial_lpuart_enable_wakeup(sport, false);
3098
3099		/* clear the wakeup flags */
3100		if (lpuart_is_32(sport)) {
3101			stat = lpuart32_read(&sport->port, UARTSTAT);
3102			lpuart32_write(&sport->port, stat, UARTSTAT);
3103
3104			/* check whether lpuart wakeup was triggered */
3105			wake_active = stat & (UARTSTAT_RDRF | UARTSTAT_RXEDGIF);
3106
3107			if (wake_active && irqd_is_wakeup_set(irq_get_irq_data(sport->port.irq)))
3108				pm_wakeup_event(tty_port_tty_get(port)->dev, 0);
3109		}
3110	}
3111
3112	return 0;
3113}
3114
3115static int lpuart_suspend(struct device *dev)
3116{
3117	struct lpuart_port *sport = dev_get_drvdata(dev);
3118	u32 temp;
3119	unsigned long flags;
3120
3121	uart_suspend_port(&lpuart_reg, &sport->port);
3122
3123	if (lpuart_uport_is_active(sport)) {
3124		uart_port_lock_irqsave(&sport->port, &flags);
3125		if (lpuart_is_32(sport)) {
3126			/* disable Rx/Tx and interrupts */
3127			temp = lpuart32_read(&sport->port, UARTCTRL);
3128			temp &= ~(UARTCTRL_TE | UARTCTRL_TIE | UARTCTRL_TCIE);
3129			lpuart32_write(&sport->port, temp, UARTCTRL);
3130		} else {
3131			/* disable Rx/Tx and interrupts */
3132			temp = readb(sport->port.membase + UARTCR2);
3133			temp &= ~(UARTCR2_TE | UARTCR2_TIE | UARTCR2_TCIE);
3134			writeb(temp, sport->port.membase + UARTCR2);
3135		}
3136		uart_port_unlock_irqrestore(&sport->port, flags);
3137
3138		if (sport->lpuart_dma_rx_use) {
3139			/*
3140			 * EDMA driver during suspend will forcefully release any
3141			 * non-idle DMA channels. If port wakeup is enabled or if port
3142			 * is console port or 'no_console_suspend' is set the Rx DMA
3143			 * cannot resume as expected, hence gracefully release the
3144			 * Rx DMA path before suspend and start Rx DMA path on resume.
3145			 */
3146			lpuart_dma_rx_free(&sport->port);
3147
3148			/* Disable Rx DMA to use UART port as wakeup source */
3149			uart_port_lock_irqsave(&sport->port, &flags);
3150			if (lpuart_is_32(sport)) {
3151				temp = lpuart32_read(&sport->port, UARTBAUD);
3152				lpuart32_write(&sport->port, temp & ~UARTBAUD_RDMAE,
3153					       UARTBAUD);
3154			} else {
3155				writeb(readb(sport->port.membase + UARTCR5) &
3156				       ~UARTCR5_RDMAS, sport->port.membase + UARTCR5);
3157			}
3158			uart_port_unlock_irqrestore(&sport->port, flags);
3159		}
3160
3161		if (sport->lpuart_dma_tx_use) {
3162			uart_port_lock_irqsave(&sport->port, &flags);
3163			if (lpuart_is_32(sport)) {
3164				temp = lpuart32_read(&sport->port, UARTBAUD);
3165				temp &= ~UARTBAUD_TDMAE;
3166				lpuart32_write(&sport->port, temp, UARTBAUD);
3167			} else {
3168				temp = readb(sport->port.membase + UARTCR5);
3169				temp &= ~UARTCR5_TDMAS;
3170				writeb(temp, sport->port.membase + UARTCR5);
3171			}
3172			uart_port_unlock_irqrestore(&sport->port, flags);
3173			sport->dma_tx_in_progress = false;
3174			dmaengine_terminate_sync(sport->dma_tx_chan);
3175		}
3176	} else if (pm_runtime_active(sport->port.dev)) {
3177		lpuart_disable_clks(sport);
3178		pm_runtime_disable(sport->port.dev);
3179		pm_runtime_set_suspended(sport->port.dev);
3180	}
3181
3182	return 0;
3183}
3184
3185static void lpuart_console_fixup(struct lpuart_port *sport)
3186{
3187	struct tty_port *port = &sport->port.state->port;
3188	struct uart_port *uport = &sport->port;
3189	struct ktermios termios;
3190
3191	/* i.MX7ULP enter VLLS mode that lpuart module power off and registers
3192	 * all lost no matter the port is wakeup source.
3193	 * For console port, console baud rate setting lost and print messy
3194	 * log when enable the console port as wakeup source. To avoid the
3195	 * issue happen, user should not enable uart port as wakeup source
3196	 * in VLLS mode, or restore console setting here.
3197	 */
3198	if (is_imx7ulp_lpuart(sport) && lpuart_uport_is_active(sport) &&
3199	    console_suspend_enabled && uart_console(uport)) {
3200
3201		mutex_lock(&port->mutex);
3202		memset(&termios, 0, sizeof(struct ktermios));
3203		termios.c_cflag = uport->cons->cflag;
3204		if (port->tty && termios.c_cflag == 0)
3205			termios = port->tty->termios;
3206		uport->ops->set_termios(uport, &termios, NULL);
3207		mutex_unlock(&port->mutex);
3208	}
3209}
3210
3211static int lpuart_resume(struct device *dev)
3212{
3213	struct lpuart_port *sport = dev_get_drvdata(dev);
3214	int ret;
3215
3216	if (lpuart_uport_is_active(sport)) {
3217		if (lpuart_is_32(sport))
3218			lpuart32_hw_setup(sport);
3219		else
3220			lpuart_hw_setup(sport);
3221	} else if (pm_runtime_active(sport->port.dev)) {
3222		ret = lpuart_enable_clks(sport);
3223		if (ret)
3224			return ret;
3225		pm_runtime_set_active(sport->port.dev);
3226		pm_runtime_enable(sport->port.dev);
3227	}
3228
3229	lpuart_console_fixup(sport);
3230	uart_resume_port(&lpuart_reg, &sport->port);
3231
3232	return 0;
3233}
3234
3235static const struct dev_pm_ops lpuart_pm_ops = {
3236	RUNTIME_PM_OPS(lpuart_runtime_suspend,
3237			   lpuart_runtime_resume, NULL)
3238	NOIRQ_SYSTEM_SLEEP_PM_OPS(lpuart_suspend_noirq,
3239				      lpuart_resume_noirq)
3240	SYSTEM_SLEEP_PM_OPS(lpuart_suspend, lpuart_resume)
3241};
3242
3243static struct platform_driver lpuart_driver = {
3244	.probe		= lpuart_probe,
3245	.remove		= lpuart_remove,
3246	.driver		= {
3247		.name	= "fsl-lpuart",
3248		.of_match_table = lpuart_dt_ids,
3249		.pm	= pm_ptr(&lpuart_pm_ops),
3250	},
3251};
3252
3253static int __init lpuart_serial_init(void)
3254{
3255	int ret = uart_register_driver(&lpuart_reg);
3256
3257	if (ret)
3258		return ret;
3259
3260	ret = platform_driver_register(&lpuart_driver);
3261	if (ret)
3262		uart_unregister_driver(&lpuart_reg);
3263
3264	return ret;
3265}
3266
3267static void __exit lpuart_serial_exit(void)
3268{
3269	platform_driver_unregister(&lpuart_driver);
3270	uart_unregister_driver(&lpuart_reg);
3271}
3272
3273module_init(lpuart_serial_init);
3274module_exit(lpuart_serial_exit);
3275
3276MODULE_DESCRIPTION("Freescale lpuart serial port driver");
3277MODULE_LICENSE("GPL v2");