qcom_geni_serial.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
   4 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
   5 */
   6
   7/* Disable MMIO tracing to prevent excessive logging of unwanted MMIO traces */
   8#define __DISABLE_TRACE_MMIO__
   9
  10#include <linux/clk.h>
  11#include <linux/console.h>
  12#include <linux/io.h>
  13#include <linux/iopoll.h>
  14#include <linux/irq.h>
  15#include <linux/module.h>
  16#include <linux/of.h>
  17#include <linux/pm_domain.h>
  18#include <linux/pm_opp.h>
  19#include <linux/platform_device.h>
  20#include <linux/pm_runtime.h>
  21#include <linux/pm_wakeirq.h>
  22#include <linux/soc/qcom/geni-se.h>
  23#include <linux/serial.h>
  24#include <linux/serial_core.h>
  25#include <linux/slab.h>
  26#include <linux/tty.h>
  27#include <linux/tty_flip.h>
  28#include <dt-bindings/interconnect/qcom,icc.h>
  29
  30/* UART specific GENI registers */
  31#define SE_UART_LOOPBACK_CFG		0x22c
  32#define SE_UART_IO_MACRO_CTRL		0x240
  33#define SE_UART_TX_TRANS_CFG		0x25c
  34#define SE_UART_TX_WORD_LEN		0x268
  35#define SE_UART_TX_STOP_BIT_LEN		0x26c
  36#define SE_UART_TX_TRANS_LEN		0x270
  37#define SE_UART_RX_TRANS_CFG		0x280
  38#define SE_UART_RX_WORD_LEN		0x28c
  39#define SE_UART_RX_STALE_CNT		0x294
  40#define SE_UART_TX_PARITY_CFG		0x2a4
  41#define SE_UART_RX_PARITY_CFG		0x2a8
  42#define SE_UART_MANUAL_RFR		0x2ac
  43
  44/* SE_UART_TRANS_CFG */
  45#define UART_TX_PAR_EN			BIT(0)
  46#define UART_CTS_MASK			BIT(1)
  47
  48/* SE_UART_TX_STOP_BIT_LEN */
  49#define TX_STOP_BIT_LEN_1		0
  50#define TX_STOP_BIT_LEN_2		2
  51
  52/* SE_UART_RX_TRANS_CFG */
  53#define UART_RX_PAR_EN			BIT(3)
  54
  55/* SE_UART_RX_WORD_LEN */
  56#define RX_WORD_LEN_MASK		GENMASK(9, 0)
  57
  58/* SE_UART_RX_STALE_CNT */
  59#define RX_STALE_CNT			GENMASK(23, 0)
  60
  61/* SE_UART_TX_PARITY_CFG/RX_PARITY_CFG */
  62#define PAR_CALC_EN			BIT(0)
  63#define PAR_EVEN			0x00
  64#define PAR_ODD				0x01
  65#define PAR_SPACE			0x10
  66
  67/* SE_UART_MANUAL_RFR register fields */
  68#define UART_MANUAL_RFR_EN		BIT(31)
  69#define UART_RFR_NOT_READY		BIT(1)
  70#define UART_RFR_READY			BIT(0)
  71
  72/* UART M_CMD OP codes */
  73#define UART_START_TX			0x1
  74/* UART S_CMD OP codes */
  75#define UART_START_READ			0x1
  76#define UART_PARAM			0x1
  77#define UART_PARAM_RFR_OPEN		BIT(7)
  78
  79#define UART_OVERSAMPLING		32
  80#define STALE_TIMEOUT			16
  81#define DEFAULT_BITS_PER_CHAR		10
  82#define GENI_UART_CONS_PORTS		1
  83#define DEF_FIFO_DEPTH_WORDS		16
  84#define DEF_TX_WM			2
  85#define DEF_FIFO_WIDTH_BITS		32
  86#define UART_RX_WM			2
  87
  88/* SE_UART_LOOPBACK_CFG */
  89#define RX_TX_SORTED			BIT(0)
  90#define CTS_RTS_SORTED			BIT(1)
  91#define RX_TX_CTS_RTS_SORTED		(RX_TX_SORTED | CTS_RTS_SORTED)
  92
  93/* UART pin swap value */
  94#define DEFAULT_IO_MACRO_IO0_IO1_MASK	GENMASK(3, 0)
  95#define IO_MACRO_IO0_SEL		0x3
  96#define DEFAULT_IO_MACRO_IO2_IO3_MASK	GENMASK(15, 4)
  97#define IO_MACRO_IO2_IO3_SWAP		0x4640
  98
  99/* We always configure 4 bytes per FIFO word */
 100#define BYTES_PER_FIFO_WORD		4U
 101
 102#define DMA_RX_BUF_SIZE		2048
 103
 104static DEFINE_IDA(port_ida);
 105#define DOMAIN_IDX_POWER	0
 106#define DOMAIN_IDX_PERF		1
 107
 108struct qcom_geni_device_data {
 109	bool console;
 110	enum geni_se_xfer_mode mode;
 111	struct dev_pm_domain_attach_data pd_data;
 112	int (*resources_init)(struct uart_port *uport);
 113	int (*set_rate)(struct uart_port *uport, unsigned int baud);
 114	int (*power_state)(struct uart_port *uport, bool state);
 115};
 116
 117struct qcom_geni_private_data {
 118	/* NOTE: earlycon port will have NULL here */
 119	struct uart_driver *drv;
 120
 121	u32 poll_cached_bytes;
 122	unsigned int poll_cached_bytes_cnt;
 123
 124	u32 write_cached_bytes;
 125	unsigned int write_cached_bytes_cnt;
 126};
 127
 128struct qcom_geni_serial_port {
 129	struct uart_port uport;
 130	struct geni_se se;
 131	const char *name;
 132	u32 tx_fifo_depth;
 133	u32 tx_fifo_width;
 134	u32 rx_fifo_depth;
 135	dma_addr_t tx_dma_addr;
 136	dma_addr_t rx_dma_addr;
 137	bool setup;
 138	unsigned long poll_timeout_us;
 139	unsigned long clk_rate;
 140	void *rx_buf;
 141	u32 loopback;
 142	bool brk;
 143
 144	unsigned int tx_remaining;
 145	unsigned int tx_queued;
 146	int wakeup_irq;
 147	bool rx_tx_swap;
 148	bool cts_rts_swap;
 149
 150	struct qcom_geni_private_data private_data;
 151	const struct qcom_geni_device_data *dev_data;
 152	struct dev_pm_domain_list *pd_list;
 153};
 154
 155static const struct uart_ops qcom_geni_console_pops;
 156static const struct uart_ops qcom_geni_uart_pops;
 157static struct uart_driver qcom_geni_console_driver;
 158static struct uart_driver qcom_geni_uart_driver;
 159
 160static void __qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport);
 161static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport);
 162static int qcom_geni_serial_port_setup(struct uart_port *uport);
 163
 164static inline struct qcom_geni_serial_port *to_dev_port(struct uart_port *uport)
 165{
 166	return container_of(uport, struct qcom_geni_serial_port, uport);
 167}
 168
 169static struct qcom_geni_serial_port qcom_geni_console_port = {
 170	.uport = {
 171		.iotype = UPIO_MEM,
 172		.ops = &qcom_geni_console_pops,
 173		.flags = UPF_BOOT_AUTOCONF,
 174		.line = 0,
 175	},
 176};
 177
 178static const struct serial_rs485 qcom_geni_rs485_supported = {
 179	.flags = SER_RS485_ENABLED | SER_RS485_RTS_AFTER_SEND | SER_RS485_RTS_ON_SEND,
 180};
 181
 182/**
 183 * qcom_geni_set_rs485_mode - Set RTS pin state for RS485 mode
 184 * @uport: UART port
 185 * @flag: RS485 flag to determine RTS polarity
 186 *
 187 * Enables manual RTS control for RS485. Sets RTS to READY or NOT_READY
 188 * based on the specified flag if RS485 mode is enabled.
 189 */
 190static void qcom_geni_set_rs485_mode(struct uart_port *uport, u32 flag)
 191{
 192	if (!(uport->rs485.flags & SER_RS485_ENABLED))
 193		return;
 194
 195	u32 rfr = UART_MANUAL_RFR_EN;
 196
 197	if (uport->rs485.flags & flag)
 198		rfr |= UART_RFR_NOT_READY;
 199	else
 200		rfr |= UART_RFR_READY;
 201
 202	writel(rfr, uport->membase + SE_UART_MANUAL_RFR);
 203}
 204
 205static int qcom_geni_serial_request_port(struct uart_port *uport)
 206{
 207	struct platform_device *pdev = to_platform_device(uport->dev);
 208	struct qcom_geni_serial_port *port = to_dev_port(uport);
 209
 210	uport->membase = devm_platform_ioremap_resource(pdev, 0);
 211	if (IS_ERR(uport->membase))
 212		return PTR_ERR(uport->membase);
 213	port->se.base = uport->membase;
 214	return 0;
 215}
 216
 217static void qcom_geni_serial_config_port(struct uart_port *uport, int cfg_flags)
 218{
 219	if (cfg_flags & UART_CONFIG_TYPE) {
 220		uport->type = PORT_MSM;
 221		qcom_geni_serial_request_port(uport);
 222	}
 223}
 224
 225static unsigned int qcom_geni_serial_get_mctrl(struct uart_port *uport)
 226{
 227	unsigned int mctrl = TIOCM_DSR | TIOCM_CAR;
 228	u32 geni_ios;
 229
 230	if (uart_console(uport)) {
 231		mctrl |= TIOCM_CTS;
 232	} else {
 233		geni_ios = readl(uport->membase + SE_GENI_IOS);
 234		if (!(geni_ios & IO2_DATA_IN))
 235			mctrl |= TIOCM_CTS;
 236	}
 237
 238	return mctrl;
 239}
 240
 241static void qcom_geni_serial_set_mctrl(struct uart_port *uport,
 242							unsigned int mctrl)
 243{
 244	u32 uart_manual_rfr = 0;
 245	struct qcom_geni_serial_port *port = to_dev_port(uport);
 246
 247	if (uart_console(uport))
 248		return;
 249
 250	if (mctrl & TIOCM_LOOP)
 251		port->loopback = RX_TX_CTS_RTS_SORTED;
 252
 253	if (!(mctrl & TIOCM_RTS) && !uport->suspended)
 254		uart_manual_rfr = UART_MANUAL_RFR_EN | UART_RFR_NOT_READY;
 255	writel(uart_manual_rfr, uport->membase + SE_UART_MANUAL_RFR);
 256}
 257
 258static const char *qcom_geni_serial_get_type(struct uart_port *uport)
 259{
 260	return "MSM";
 261}
 262
 263static struct qcom_geni_serial_port *get_port_from_line(int line, bool console, struct device *dev)
 264{
 265	struct qcom_geni_serial_port *port;
 266	int nr_ports = console ? GENI_UART_CONS_PORTS : CONFIG_SERIAL_QCOM_GENI_UART_PORTS;
 267
 268	if (console) {
 269		if (line < 0 || line >= nr_ports)
 270			return ERR_PTR(-ENXIO);
 271
 272		port = &qcom_geni_console_port;
 273	} else {
 274		int max_alias_num = of_alias_get_highest_id("serial");
 275
 276		if (line < 0 || line >= nr_ports)
 277			line = ida_alloc_range(&port_ida, max_alias_num + 1,
 278					       nr_ports - 1, GFP_KERNEL);
 279		else
 280			line = ida_alloc_range(&port_ida, line,
 281					       nr_ports - 1, GFP_KERNEL);
 282
 283		if (line < 0)
 284			return ERR_PTR(-ENXIO);
 285
 286		port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
 287		if (!port)
 288			return ERR_PTR(-ENOMEM);
 289
 290		port->uport.iotype = UPIO_MEM;
 291		port->uport.ops = &qcom_geni_uart_pops;
 292		port->uport.flags = UPF_BOOT_AUTOCONF;
 293		port->uport.line = line;
 294	}
 295	return port;
 296}
 297
 298static bool qcom_geni_serial_main_active(struct uart_port *uport)
 299{
 300	return readl(uport->membase + SE_GENI_STATUS) & M_GENI_CMD_ACTIVE;
 301}
 302
 303static bool qcom_geni_serial_secondary_active(struct uart_port *uport)
 304{
 305	return readl(uport->membase + SE_GENI_STATUS) & S_GENI_CMD_ACTIVE;
 306}
 307
 308static bool qcom_geni_serial_poll_bitfield(struct uart_port *uport,
 309					   unsigned int offset, u32 field, u32 val)
 310{
 311	u32 reg;
 312	struct qcom_geni_serial_port *port;
 313	unsigned long timeout_us = 20000;
 314	struct qcom_geni_private_data *private_data = uport->private_data;
 315
 316	if (private_data->drv) {
 317		port = to_dev_port(uport);
 318		if (port->poll_timeout_us)
 319			timeout_us = port->poll_timeout_us;
 320	}
 321
 322	/*
 323	 * Use custom implementation instead of readl_poll_atomic since ktimer
 324	 * is not ready at the time of early console.
 325	 */
 326	timeout_us = DIV_ROUND_UP(timeout_us, 10) * 10;
 327	while (timeout_us) {
 328		reg = readl(uport->membase + offset);
 329		if ((reg & field) == val)
 330			return true;
 331		udelay(10);
 332		timeout_us -= 10;
 333	}
 334	return false;
 335}
 336
 337static bool qcom_geni_serial_poll_bit(struct uart_port *uport,
 338				      unsigned int offset, u32 field, bool set)
 339{
 340	return qcom_geni_serial_poll_bitfield(uport, offset, field, set ? field : 0);
 341}
 342
 343static void qcom_geni_serial_setup_tx(struct uart_port *uport, u32 xmit_size)
 344{
 345	u32 m_cmd;
 346
 347	writel(xmit_size, uport->membase + SE_UART_TX_TRANS_LEN);
 348	m_cmd = UART_START_TX << M_OPCODE_SHFT;
 349	writel(m_cmd, uport->membase + SE_GENI_M_CMD0);
 350}
 351
 352static void qcom_geni_serial_poll_tx_done(struct uart_port *uport)
 353{
 354	int done;
 355
 356	done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 357						M_CMD_DONE_EN, true);
 358	if (!done) {
 359		writel(M_GENI_CMD_ABORT, uport->membase +
 360						SE_GENI_M_CMD_CTRL_REG);
 361		qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 362							M_CMD_ABORT_EN, true);
 363		writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 364	}
 365}
 366
 367static void qcom_geni_serial_abort_rx(struct uart_port *uport)
 368{
 369	u32 irq_clear = S_CMD_DONE_EN | S_CMD_ABORT_EN;
 370
 371	writel(S_GENI_CMD_ABORT, uport->membase + SE_GENI_S_CMD_CTRL_REG);
 372	qcom_geni_serial_poll_bit(uport, SE_GENI_S_CMD_CTRL_REG,
 373					S_GENI_CMD_ABORT, false);
 374	writel(irq_clear, uport->membase + SE_GENI_S_IRQ_CLEAR);
 375	writel(FORCE_DEFAULT, uport->membase + GENI_FORCE_DEFAULT_REG);
 376}
 377
 378#ifdef CONFIG_CONSOLE_POLL
 379static int qcom_geni_serial_get_char(struct uart_port *uport)
 380{
 381	struct qcom_geni_private_data *private_data = uport->private_data;
 382	u32 status;
 383	u32 word_cnt;
 384	int ret;
 385
 386	if (!private_data->poll_cached_bytes_cnt) {
 387		status = readl(uport->membase + SE_GENI_M_IRQ_STATUS);
 388		writel(status, uport->membase + SE_GENI_M_IRQ_CLEAR);
 389
 390		status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 391		writel(status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 392
 393		status = readl(uport->membase + SE_GENI_RX_FIFO_STATUS);
 394		word_cnt = status & RX_FIFO_WC_MSK;
 395		if (!word_cnt)
 396			return NO_POLL_CHAR;
 397
 398		if (word_cnt == 1 && (status & RX_LAST))
 399			/*
 400			 * NOTE: If RX_LAST_BYTE_VALID is 0 it needs to be
 401			 * treated as if it was BYTES_PER_FIFO_WORD.
 402			 */
 403			private_data->poll_cached_bytes_cnt =
 404				(status & RX_LAST_BYTE_VALID_MSK) >>
 405				RX_LAST_BYTE_VALID_SHFT;
 406
 407		if (private_data->poll_cached_bytes_cnt == 0)
 408			private_data->poll_cached_bytes_cnt = BYTES_PER_FIFO_WORD;
 409
 410		private_data->poll_cached_bytes =
 411			readl(uport->membase + SE_GENI_RX_FIFOn);
 412	}
 413
 414	private_data->poll_cached_bytes_cnt--;
 415	ret = private_data->poll_cached_bytes & 0xff;
 416	private_data->poll_cached_bytes >>= 8;
 417
 418	return ret;
 419}
 420
 421static void qcom_geni_serial_poll_put_char(struct uart_port *uport,
 422							unsigned char c)
 423{
 424	if (qcom_geni_serial_main_active(uport)) {
 425		qcom_geni_serial_poll_tx_done(uport);
 426		__qcom_geni_serial_cancel_tx_cmd(uport);
 427	}
 428
 429	writel(M_CMD_DONE_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 430	qcom_geni_serial_setup_tx(uport, 1);
 431	writel(c, uport->membase + SE_GENI_TX_FIFOn);
 432	qcom_geni_serial_poll_tx_done(uport);
 433}
 434
 435static int qcom_geni_serial_poll_init(struct uart_port *uport)
 436{
 437	struct qcom_geni_serial_port *port = to_dev_port(uport);
 438	int ret;
 439
 440	if (!port->setup) {
 441		ret = qcom_geni_serial_port_setup(uport);
 442		if (ret)
 443			return ret;
 444	}
 445
 446	if (!qcom_geni_serial_secondary_active(uport))
 447		geni_se_setup_s_cmd(&port->se, UART_START_READ, 0);
 448
 449	return 0;
 450}
 451#endif
 452
 453#ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
 454static void qcom_geni_serial_drain_fifo(struct uart_port *uport)
 455{
 456	struct qcom_geni_serial_port *port = to_dev_port(uport);
 457
 458	qcom_geni_serial_poll_bitfield(uport, SE_GENI_M_GP_LENGTH, GP_LENGTH,
 459			port->tx_queued);
 460}
 461
 462static void qcom_geni_serial_wr_char(struct uart_port *uport, unsigned char ch)
 463{
 464	struct qcom_geni_private_data *private_data = uport->private_data;
 465
 466	private_data->write_cached_bytes =
 467		(private_data->write_cached_bytes >> 8) | (ch << 24);
 468	private_data->write_cached_bytes_cnt++;
 469
 470	if (private_data->write_cached_bytes_cnt == BYTES_PER_FIFO_WORD) {
 471		writel(private_data->write_cached_bytes,
 472		       uport->membase + SE_GENI_TX_FIFOn);
 473		private_data->write_cached_bytes_cnt = 0;
 474	}
 475}
 476
 477static void
 478__qcom_geni_serial_console_write(struct uart_port *uport, const char *s,
 479				 unsigned int count)
 480{
 481	struct qcom_geni_private_data *private_data = uport->private_data;
 482
 483	int i;
 484	u32 bytes_to_send = count;
 485
 486	for (i = 0; i < count; i++) {
 487		/*
 488		 * uart_console_write() adds a carriage return for each newline.
 489		 * Account for additional bytes to be written.
 490		 */
 491		if (s[i] == '\n')
 492			bytes_to_send++;
 493	}
 494
 495	writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 496	writel(M_CMD_DONE_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 497	qcom_geni_serial_setup_tx(uport, bytes_to_send);
 498	for (i = 0; i < count; ) {
 499		size_t chars_to_write = 0;
 500		size_t avail = DEF_FIFO_DEPTH_WORDS - DEF_TX_WM;
 501
 502		/*
 503		 * If the WM bit never set, then the Tx state machine is not
 504		 * in a valid state, so break, cancel/abort any existing
 505		 * command. Unfortunately the current data being written is
 506		 * lost.
 507		 */
 508		if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 509						M_TX_FIFO_WATERMARK_EN, true))
 510			break;
 511		chars_to_write = min_t(size_t, count - i, avail / 2);
 512		uart_console_write(uport, s + i, chars_to_write,
 513						qcom_geni_serial_wr_char);
 514		writel(M_TX_FIFO_WATERMARK_EN, uport->membase +
 515							SE_GENI_M_IRQ_CLEAR);
 516		i += chars_to_write;
 517	}
 518
 519	if (private_data->write_cached_bytes_cnt) {
 520		private_data->write_cached_bytes >>= BITS_PER_BYTE *
 521			(BYTES_PER_FIFO_WORD - private_data->write_cached_bytes_cnt);
 522		writel(private_data->write_cached_bytes,
 523		       uport->membase + SE_GENI_TX_FIFOn);
 524		private_data->write_cached_bytes_cnt = 0;
 525	}
 526
 527	qcom_geni_serial_poll_tx_done(uport);
 528}
 529
 530static void qcom_geni_serial_console_write(struct console *co, const char *s,
 531			      unsigned int count)
 532{
 533	struct uart_port *uport;
 534	struct qcom_geni_serial_port *port;
 535	u32 m_irq_en, s_irq_en;
 536	bool locked = true;
 537	unsigned long flags;
 538
 539	WARN_ON(co->index < 0 || co->index >= GENI_UART_CONS_PORTS);
 540
 541	port = get_port_from_line(co->index, true, NULL);
 542	if (IS_ERR(port))
 543		return;
 544
 545	uport = &port->uport;
 546	if (oops_in_progress)
 547		locked = uart_port_trylock_irqsave(uport, &flags);
 548	else
 549		uart_port_lock_irqsave(uport, &flags);
 550
 551	m_irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 552	s_irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 553	writel(0, uport->membase + SE_GENI_M_IRQ_EN);
 554	writel(0, uport->membase + SE_GENI_S_IRQ_EN);
 555
 556	if (qcom_geni_serial_main_active(uport)) {
 557		/* Wait for completion or drain FIFO */
 558		if (!locked || port->tx_remaining == 0)
 559			qcom_geni_serial_poll_tx_done(uport);
 560		else
 561			qcom_geni_serial_drain_fifo(uport);
 562
 563		qcom_geni_serial_cancel_tx_cmd(uport);
 564	}
 565
 566	__qcom_geni_serial_console_write(uport, s, count);
 567
 568	writel(m_irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 569	writel(s_irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 570
 571	if (locked)
 572		uart_port_unlock_irqrestore(uport, flags);
 573}
 574
 575static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
 576{
 577	u32 i;
 578	unsigned char buf[sizeof(u32)];
 579	struct tty_port *tport;
 580	struct qcom_geni_serial_port *port = to_dev_port(uport);
 581
 582	tport = &uport->state->port;
 583	for (i = 0; i < bytes; ) {
 584		int c;
 585		int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD);
 586
 587		ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1);
 588		i += chunk;
 589		if (drop)
 590			continue;
 591
 592		for (c = 0; c < chunk; c++) {
 593			int sysrq;
 594
 595			uport->icount.rx++;
 596			if (port->brk && buf[c] == 0) {
 597				port->brk = false;
 598				if (uart_handle_break(uport))
 599					continue;
 600			}
 601
 602			sysrq = uart_prepare_sysrq_char(uport, buf[c]);
 603
 604			if (!sysrq)
 605				tty_insert_flip_char(tport, buf[c], TTY_NORMAL);
 606		}
 607	}
 608	if (!drop)
 609		tty_flip_buffer_push(tport);
 610}
 611#else
 612static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
 613{
 614
 615}
 616#endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
 617
 618static void handle_rx_uart(struct uart_port *uport, u32 bytes)
 619{
 620	struct qcom_geni_serial_port *port = to_dev_port(uport);
 621	struct tty_port *tport = &uport->state->port;
 622	int ret;
 623
 624	ret = tty_insert_flip_string(tport, port->rx_buf, bytes);
 625	if (ret != bytes) {
 626		dev_err_ratelimited(uport->dev, "failed to push data (%d < %u)\n",
 627				ret, bytes);
 628	}
 629	uport->icount.rx += ret;
 630	tty_flip_buffer_push(tport);
 631}
 632
 633static unsigned int qcom_geni_serial_tx_empty(struct uart_port *uport)
 634{
 635	return !readl(uport->membase + SE_GENI_TX_FIFO_STATUS);
 636}
 637
 638static void qcom_geni_serial_stop_tx_dma(struct uart_port *uport)
 639{
 640	struct qcom_geni_serial_port *port = to_dev_port(uport);
 641	bool done;
 642
 643	if (!qcom_geni_serial_main_active(uport))
 644		return;
 645
 646	if (port->tx_dma_addr) {
 647		geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr,
 648				      port->tx_remaining);
 649		port->tx_dma_addr = 0;
 650		port->tx_remaining = 0;
 651	}
 652
 653	geni_se_cancel_m_cmd(&port->se);
 654
 655	done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 656					 M_CMD_CANCEL_EN, true);
 657	if (!done) {
 658		geni_se_abort_m_cmd(&port->se);
 659		done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 660						 M_CMD_ABORT_EN, true);
 661		if (!done)
 662			dev_err_ratelimited(uport->dev, "M_CMD_ABORT_EN not set");
 663		writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 664	}
 665
 666	writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 667}
 668
 669static void qcom_geni_serial_start_tx_dma(struct uart_port *uport)
 670{
 671	struct qcom_geni_serial_port *port = to_dev_port(uport);
 672	struct tty_port *tport = &uport->state->port;
 673	unsigned int xmit_size;
 674	u8 *tail;
 675	int ret;
 676
 677	if (port->tx_dma_addr)
 678		return;
 679
 680	if (kfifo_is_empty(&tport->xmit_fifo))
 681		return;
 682
 683	xmit_size = kfifo_out_linear_ptr(&tport->xmit_fifo, &tail,
 684			UART_XMIT_SIZE);
 685
 686	qcom_geni_set_rs485_mode(uport, SER_RS485_RTS_ON_SEND);
 687
 688	qcom_geni_serial_setup_tx(uport, xmit_size);
 689
 690	ret = geni_se_tx_dma_prep(&port->se, tail, xmit_size,
 691				  &port->tx_dma_addr);
 692	if (ret) {
 693		dev_err(uport->dev, "unable to start TX SE DMA: %d\n", ret);
 694		qcom_geni_serial_stop_tx_dma(uport);
 695		return;
 696	}
 697
 698	port->tx_remaining = xmit_size;
 699}
 700
 701static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport)
 702{
 703	unsigned char c;
 704	u32 irq_en;
 705
 706	/*
 707	 * Start a new transfer in case the previous command was cancelled and
 708	 * left data in the FIFO which may prevent the watermark interrupt
 709	 * from triggering. Note that the stale data is discarded.
 710	 */
 711	if (!qcom_geni_serial_main_active(uport) &&
 712	    !qcom_geni_serial_tx_empty(uport)) {
 713		if (uart_fifo_out(uport, &c, 1) == 1) {
 714			writel(M_CMD_DONE_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 715			qcom_geni_serial_setup_tx(uport, 1);
 716			writel(c, uport->membase + SE_GENI_TX_FIFOn);
 717		}
 718	}
 719
 720	irq_en = readl(uport->membase +	SE_GENI_M_IRQ_EN);
 721	irq_en |= M_TX_FIFO_WATERMARK_EN | M_CMD_DONE_EN;
 722	writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 723	writel(irq_en, uport->membase +	SE_GENI_M_IRQ_EN);
 724}
 725
 726static void qcom_geni_serial_stop_tx_fifo(struct uart_port *uport)
 727{
 728	u32 irq_en;
 729
 730	irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 731	irq_en &= ~(M_CMD_DONE_EN | M_TX_FIFO_WATERMARK_EN);
 732	writel(0, uport->membase + SE_GENI_TX_WATERMARK_REG);
 733	writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 734}
 735
 736static void __qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport)
 737{
 738	struct qcom_geni_serial_port *port = to_dev_port(uport);
 739
 740	geni_se_cancel_m_cmd(&port->se);
 741	if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 742						M_CMD_CANCEL_EN, true)) {
 743		geni_se_abort_m_cmd(&port->se);
 744		qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 745						M_CMD_ABORT_EN, true);
 746		writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 747	}
 748	writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 749}
 750
 751static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport)
 752{
 753	struct qcom_geni_serial_port *port = to_dev_port(uport);
 754
 755	if (!qcom_geni_serial_main_active(uport))
 756		return;
 757
 758	__qcom_geni_serial_cancel_tx_cmd(uport);
 759
 760	port->tx_remaining = 0;
 761	port->tx_queued = 0;
 762}
 763
 764static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop)
 765{
 766	u32 status;
 767	u32 word_cnt;
 768	u32 last_word_byte_cnt;
 769	u32 last_word_partial;
 770	u32 total_bytes;
 771
 772	status = readl(uport->membase +	SE_GENI_RX_FIFO_STATUS);
 773	word_cnt = status & RX_FIFO_WC_MSK;
 774	last_word_partial = status & RX_LAST;
 775	last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >>
 776						RX_LAST_BYTE_VALID_SHFT;
 777
 778	if (!word_cnt)
 779		return;
 780	total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1);
 781	if (last_word_partial && last_word_byte_cnt)
 782		total_bytes += last_word_byte_cnt;
 783	else
 784		total_bytes += BYTES_PER_FIFO_WORD;
 785	handle_rx_console(uport, total_bytes, drop);
 786}
 787
 788static void qcom_geni_serial_stop_rx_fifo(struct uart_port *uport)
 789{
 790	u32 irq_en;
 791	struct qcom_geni_serial_port *port = to_dev_port(uport);
 792	u32 s_irq_status;
 793
 794	irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 795	irq_en &= ~(S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN);
 796	writel(irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 797
 798	irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 799	irq_en &= ~(M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN);
 800	writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 801
 802	if (!qcom_geni_serial_secondary_active(uport))
 803		return;
 804
 805	geni_se_cancel_s_cmd(&port->se);
 806	qcom_geni_serial_poll_bit(uport, SE_GENI_S_IRQ_STATUS,
 807					S_CMD_CANCEL_EN, true);
 808	/*
 809	 * If timeout occurs secondary engine remains active
 810	 * and Abort sequence is executed.
 811	 */
 812	s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 813	/* Flush the Rx buffer */
 814	if (s_irq_status & S_RX_FIFO_LAST_EN)
 815		qcom_geni_serial_handle_rx_fifo(uport, true);
 816	writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 817
 818	if (qcom_geni_serial_secondary_active(uport))
 819		qcom_geni_serial_abort_rx(uport);
 820}
 821
 822static void qcom_geni_serial_start_rx_fifo(struct uart_port *uport)
 823{
 824	u32 irq_en;
 825	struct qcom_geni_serial_port *port = to_dev_port(uport);
 826
 827	if (qcom_geni_serial_secondary_active(uport))
 828		qcom_geni_serial_stop_rx_fifo(uport);
 829
 830	geni_se_setup_s_cmd(&port->se, UART_START_READ, 0);
 831
 832	irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 833	irq_en |= S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN;
 834	writel(irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 835
 836	irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 837	irq_en |= M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN;
 838	writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 839}
 840
 841static void qcom_geni_serial_stop_rx_dma(struct uart_port *uport)
 842{
 843	struct qcom_geni_serial_port *port = to_dev_port(uport);
 844	bool done;
 845
 846	if (!qcom_geni_serial_secondary_active(uport))
 847		return;
 848
 849	geni_se_cancel_s_cmd(&port->se);
 850	done = qcom_geni_serial_poll_bit(uport, SE_DMA_RX_IRQ_STAT,
 851			RX_EOT, true);
 852	if (done) {
 853		writel(RX_EOT | RX_DMA_DONE,
 854				uport->membase + SE_DMA_RX_IRQ_CLR);
 855	} else {
 856		qcom_geni_serial_abort_rx(uport);
 857
 858		writel(1, uport->membase + SE_DMA_RX_FSM_RST);
 859		qcom_geni_serial_poll_bit(uport, SE_DMA_RX_IRQ_STAT,
 860				RX_RESET_DONE, true);
 861		writel(RX_RESET_DONE | RX_DMA_DONE,
 862				uport->membase + SE_DMA_RX_IRQ_CLR);
 863	}
 864
 865	if (port->rx_dma_addr) {
 866		geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr,
 867				      DMA_RX_BUF_SIZE);
 868		port->rx_dma_addr = 0;
 869	}
 870}
 871
 872static void qcom_geni_serial_start_rx_dma(struct uart_port *uport)
 873{
 874	struct qcom_geni_serial_port *port = to_dev_port(uport);
 875	int ret;
 876
 877	if (qcom_geni_serial_secondary_active(uport))
 878		qcom_geni_serial_stop_rx_dma(uport);
 879
 880	geni_se_setup_s_cmd(&port->se, UART_START_READ, UART_PARAM_RFR_OPEN);
 881
 882	ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
 883				  DMA_RX_BUF_SIZE,
 884				  &port->rx_dma_addr);
 885	if (ret) {
 886		dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
 887		qcom_geni_serial_stop_rx_dma(uport);
 888	}
 889}
 890
 891static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop)
 892{
 893	struct qcom_geni_serial_port *port = to_dev_port(uport);
 894	u32 rx_in;
 895	int ret;
 896
 897	if (!qcom_geni_serial_secondary_active(uport))
 898		return;
 899
 900	if (!port->rx_dma_addr)
 901		return;
 902
 903	geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE);
 904	port->rx_dma_addr = 0;
 905
 906	rx_in = readl(uport->membase + SE_DMA_RX_LEN_IN);
 907	if (!rx_in) {
 908		dev_warn(uport->dev, "serial engine reports 0 RX bytes in!\n");
 909		return;
 910	}
 911
 912	if (!drop)
 913		handle_rx_uart(uport, rx_in);
 914
 915	ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
 916				  DMA_RX_BUF_SIZE,
 917				  &port->rx_dma_addr);
 918	if (ret) {
 919		dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
 920		qcom_geni_serial_stop_rx_dma(uport);
 921	}
 922}
 923
 924static void qcom_geni_serial_start_rx(struct uart_port *uport)
 925{
 926	uport->ops->start_rx(uport);
 927}
 928
 929static void qcom_geni_serial_stop_rx(struct uart_port *uport)
 930{
 931	uport->ops->stop_rx(uport);
 932}
 933
 934static void qcom_geni_serial_stop_tx(struct uart_port *uport)
 935{
 936	uport->ops->stop_tx(uport);
 937}
 938
 939static void qcom_geni_serial_send_chunk_fifo(struct uart_port *uport,
 940					     unsigned int chunk)
 941{
 942	struct qcom_geni_serial_port *port = to_dev_port(uport);
 943	unsigned int tx_bytes, remaining = chunk;
 944	u8 buf[BYTES_PER_FIFO_WORD];
 945
 946	while (remaining) {
 947		memset(buf, 0, sizeof(buf));
 948		tx_bytes = min(remaining, BYTES_PER_FIFO_WORD);
 949
 950		uart_fifo_out(uport, buf, tx_bytes);
 951
 952		iowrite32_rep(uport->membase + SE_GENI_TX_FIFOn, buf, 1);
 953
 954		remaining -= tx_bytes;
 955		port->tx_remaining -= tx_bytes;
 956	}
 957}
 958
 959static void qcom_geni_serial_handle_tx_fifo(struct uart_port *uport,
 960					    bool done, bool active)
 961{
 962	struct qcom_geni_serial_port *port = to_dev_port(uport);
 963	struct tty_port *tport = &uport->state->port;
 964	size_t avail;
 965	size_t pending;
 966	u32 status;
 967	u32 irq_en;
 968	unsigned int chunk;
 969
 970	status = readl(uport->membase + SE_GENI_TX_FIFO_STATUS);
 971
 972	/* Complete the current tx command before taking newly added data */
 973	if (active)
 974		pending = port->tx_remaining;
 975	else
 976		pending = kfifo_len(&tport->xmit_fifo);
 977
 978	/* All data has been transmitted or command has been cancelled */
 979	if (!pending && done) {
 980		qcom_geni_serial_stop_tx_fifo(uport);
 981		goto out_write_wakeup;
 982	}
 983
 984	if (active)
 985		avail = port->tx_fifo_depth - (status & TX_FIFO_WC);
 986	else
 987		avail = port->tx_fifo_depth;
 988
 989	avail *= BYTES_PER_FIFO_WORD;
 990
 991	chunk = min(avail, pending);
 992	if (!chunk)
 993		goto out_write_wakeup;
 994
 995	if (!active) {
 996		qcom_geni_serial_setup_tx(uport, pending);
 997		port->tx_remaining = pending;
 998		port->tx_queued = 0;
 999
1000		irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
1001		if (!(irq_en & M_TX_FIFO_WATERMARK_EN))
1002			writel(irq_en | M_TX_FIFO_WATERMARK_EN,
1003					uport->membase + SE_GENI_M_IRQ_EN);
1004	}
1005
1006	qcom_geni_serial_send_chunk_fifo(uport, chunk);
1007	port->tx_queued += chunk;
1008
1009	/*
1010	 * The tx fifo watermark is level triggered and latched. Though we had
1011	 * cleared it in qcom_geni_serial_isr it will have already reasserted
1012	 * so we must clear it again here after our writes.
1013	 */
1014	writel(M_TX_FIFO_WATERMARK_EN,
1015			uport->membase + SE_GENI_M_IRQ_CLEAR);
1016
1017out_write_wakeup:
1018	if (!port->tx_remaining) {
1019		irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
1020		if (irq_en & M_TX_FIFO_WATERMARK_EN)
1021			writel(irq_en & ~M_TX_FIFO_WATERMARK_EN,
1022					uport->membase + SE_GENI_M_IRQ_EN);
1023	}
1024
1025	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
1026		uart_write_wakeup(uport);
1027}
1028
1029static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
1030{
1031	struct qcom_geni_serial_port *port = to_dev_port(uport);
1032	struct tty_port *tport = &uport->state->port;
1033
1034	uart_xmit_advance(uport, port->tx_remaining);
1035	geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining);
1036	port->tx_dma_addr = 0;
1037	port->tx_remaining = 0;
1038
1039	if (!kfifo_is_empty(&tport->xmit_fifo))
1040		qcom_geni_serial_start_tx_dma(uport);
1041
1042	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
1043		uart_write_wakeup(uport);
1044}
1045
1046static irqreturn_t qcom_geni_serial_isr(int isr, void *dev)
1047{
1048	u32 m_irq_en;
1049	u32 m_irq_status;
1050	u32 s_irq_status;
1051	u32 geni_status;
1052	u32 dma;
1053	u32 dma_tx_status;
1054	u32 dma_rx_status;
1055	struct uart_port *uport = dev;
1056	bool drop_rx = false;
1057	struct tty_port *tport = &uport->state->port;
1058	struct qcom_geni_serial_port *port = to_dev_port(uport);
1059
1060	if (uport->suspended)
1061		return IRQ_NONE;
1062
1063	uart_port_lock(uport);
1064
1065	m_irq_status = readl(uport->membase + SE_GENI_M_IRQ_STATUS);
1066	s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
1067	dma_tx_status = readl(uport->membase + SE_DMA_TX_IRQ_STAT);
1068	dma_rx_status = readl(uport->membase + SE_DMA_RX_IRQ_STAT);
1069	geni_status = readl(uport->membase + SE_GENI_STATUS);
1070	dma = readl(uport->membase + SE_GENI_DMA_MODE_EN);
1071	m_irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
1072	writel(m_irq_status, uport->membase + SE_GENI_M_IRQ_CLEAR);
1073	writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
1074	writel(dma_tx_status, uport->membase + SE_DMA_TX_IRQ_CLR);
1075	writel(dma_rx_status, uport->membase + SE_DMA_RX_IRQ_CLR);
1076
1077	if (WARN_ON(m_irq_status & M_ILLEGAL_CMD_EN))
1078		goto out_unlock;
1079
1080	if (s_irq_status & S_RX_FIFO_WR_ERR_EN) {
1081		uport->icount.overrun++;
1082		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
1083	}
1084
1085	if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) {
1086		if (s_irq_status & S_GP_IRQ_0_EN)
1087			uport->icount.parity++;
1088		drop_rx = true;
1089	} else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) {
1090		uport->icount.brk++;
1091		port->brk = true;
1092	}
1093
1094	if (dma) {
1095		if (dma_tx_status & TX_DMA_DONE) {
1096			qcom_geni_serial_handle_tx_dma(uport);
1097			qcom_geni_set_rs485_mode(uport, SER_RS485_RTS_AFTER_SEND);
1098	}
1099
1100		if (dma_rx_status) {
1101			if (dma_rx_status & RX_RESET_DONE)
1102				goto out_unlock;
1103
1104			if (dma_rx_status & RX_DMA_PARITY_ERR) {
1105				uport->icount.parity++;
1106				drop_rx = true;
1107			}
1108
1109			if (dma_rx_status & RX_DMA_BREAK)
1110				uport->icount.brk++;
1111
1112			if (dma_rx_status & (RX_DMA_DONE | RX_EOT))
1113				qcom_geni_serial_handle_rx_dma(uport, drop_rx);
1114		}
1115	} else {
1116		if (m_irq_status & m_irq_en &
1117		    (M_TX_FIFO_WATERMARK_EN | M_CMD_DONE_EN))
1118			qcom_geni_serial_handle_tx_fifo(uport,
1119					m_irq_status & M_CMD_DONE_EN,
1120					geni_status & M_GENI_CMD_ACTIVE);
1121
1122		if (s_irq_status & (S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN))
1123			qcom_geni_serial_handle_rx_fifo(uport, drop_rx);
1124	}
1125
1126out_unlock:
1127	uart_unlock_and_check_sysrq(uport);
1128
1129	return IRQ_HANDLED;
1130}
1131
1132static int setup_fifos(struct qcom_geni_serial_port *port)
1133{
1134	struct uart_port *uport;
1135	u32 old_rx_fifo_depth = port->rx_fifo_depth;
1136
1137	uport = &port->uport;
1138	port->tx_fifo_depth = geni_se_get_tx_fifo_depth(&port->se);
1139	port->tx_fifo_width = geni_se_get_tx_fifo_width(&port->se);
1140	port->rx_fifo_depth = geni_se_get_rx_fifo_depth(&port->se);
1141	uport->fifosize =
1142		(port->tx_fifo_depth * port->tx_fifo_width) / BITS_PER_BYTE;
1143
1144	if (port->rx_buf && (old_rx_fifo_depth != port->rx_fifo_depth) && port->rx_fifo_depth) {
1145		/*
1146		 * Use krealloc rather than krealloc_array because rx_buf is
1147		 * accessed as 1 byte entries as well as 4 byte entries so it's
1148		 * not necessarily an array.
1149		 */
1150		port->rx_buf = devm_krealloc(uport->dev, port->rx_buf,
1151					     port->rx_fifo_depth * sizeof(u32),
1152					     GFP_KERNEL);
1153		if (!port->rx_buf)
1154			return -ENOMEM;
1155	}
1156
1157	return 0;
1158}
1159
1160
1161static void qcom_geni_serial_shutdown(struct uart_port *uport)
1162{
1163	disable_irq(uport->irq);
1164
1165	uart_port_lock_irq(uport);
1166	qcom_geni_serial_stop_tx(uport);
1167	qcom_geni_serial_stop_rx(uport);
1168
1169	qcom_geni_serial_cancel_tx_cmd(uport);
1170	uart_port_unlock_irq(uport);
1171}
1172
1173static void qcom_geni_serial_flush_buffer(struct uart_port *uport)
1174{
1175	qcom_geni_serial_cancel_tx_cmd(uport);
1176}
1177
1178static int qcom_geni_serial_port_setup(struct uart_port *uport)
1179{
1180	struct qcom_geni_serial_port *port = to_dev_port(uport);
1181	u32 rxstale = DEFAULT_BITS_PER_CHAR * STALE_TIMEOUT;
1182	u32 proto;
1183	u32 pin_swap;
1184	int ret;
1185
1186	proto = geni_se_read_proto(&port->se);
1187	if (proto == GENI_SE_INVALID_PROTO) {
1188		ret = geni_load_se_firmware(&port->se, GENI_SE_UART);
1189		if (ret) {
1190			dev_err(uport->dev, "UART firmware load failed ret: %d\n", ret);
1191			return ret;
1192		}
1193	} else if (proto != GENI_SE_UART) {
1194		dev_err(uport->dev, "Invalid FW loaded, proto: %d\n", proto);
1195		return -ENXIO;
1196	}
1197
1198	qcom_geni_serial_stop_rx(uport);
1199
1200	ret = setup_fifos(port);
1201	if (ret)
1202		return ret;
1203
1204	writel(rxstale, uport->membase + SE_UART_RX_STALE_CNT);
1205
1206	pin_swap = readl(uport->membase + SE_UART_IO_MACRO_CTRL);
1207	if (port->rx_tx_swap) {
1208		pin_swap &= ~DEFAULT_IO_MACRO_IO2_IO3_MASK;
1209		pin_swap |= IO_MACRO_IO2_IO3_SWAP;
1210	}
1211	if (port->cts_rts_swap) {
1212		pin_swap &= ~DEFAULT_IO_MACRO_IO0_IO1_MASK;
1213		pin_swap |= IO_MACRO_IO0_SEL;
1214	}
1215	/* Configure this register if RX-TX, CTS-RTS pins are swapped */
1216	if (port->rx_tx_swap || port->cts_rts_swap)
1217		writel(pin_swap, uport->membase + SE_UART_IO_MACRO_CTRL);
1218
1219	/*
1220	 * Make an unconditional cancel on the main sequencer to reset
1221	 * it else we could end up in data loss scenarios.
1222	 */
1223	if (uart_console(uport))
1224		qcom_geni_serial_poll_tx_done(uport);
1225	geni_se_config_packing(&port->se, BITS_PER_BYTE, BYTES_PER_FIFO_WORD,
1226			       false, true, true);
1227	geni_se_init(&port->se, UART_RX_WM, port->rx_fifo_depth - 2);
1228	geni_se_select_mode(&port->se, port->dev_data->mode);
1229	port->setup = true;
1230
1231	return 0;
1232}
1233
1234static int qcom_geni_serial_startup(struct uart_port *uport)
1235{
1236	int ret;
1237	struct qcom_geni_serial_port *port = to_dev_port(uport);
1238
1239	if (!port->setup) {
1240		ret = qcom_geni_serial_port_setup(uport);
1241		if (ret)
1242			return ret;
1243	}
1244
1245	uart_port_lock_irq(uport);
1246	qcom_geni_serial_start_rx(uport);
1247	uart_port_unlock_irq(uport);
1248
1249	enable_irq(uport->irq);
1250
1251	return 0;
1252}
1253
1254static int geni_serial_set_rate(struct uart_port *uport, unsigned int baud)
1255{
1256	struct qcom_geni_serial_port *port = to_dev_port(uport);
1257	unsigned long clk_rate;
1258	unsigned int avg_bw_core, clk_idx;
1259	unsigned int clk_div;
1260	u32 ver, sampling_rate;
1261	u32 ser_clk_cfg;
1262	int ret;
1263
1264	sampling_rate = UART_OVERSAMPLING;
1265	/* Sampling rate is halved for IP versions >= 2.5 */
1266	ver = geni_se_get_qup_hw_version(&port->se);
1267	if (ver >= QUP_SE_VERSION_2_5)
1268		sampling_rate /= 2;
1269
1270	ret = geni_se_clk_freq_match(&port->se, baud * sampling_rate, &clk_idx, &clk_rate, false);
1271	if (ret) {
1272		dev_err(port->se.dev, "Failed to find src clk for baud rate: %d ret: %d\n",
1273			baud, ret);
1274		return ret;
1275	}
1276
1277	clk_div = DIV_ROUND_UP(clk_rate, baud * sampling_rate);
1278	/* Check if calculated divider exceeds maximum allowed value */
1279	if (clk_div > (CLK_DIV_MSK >> CLK_DIV_SHFT)) {
1280		dev_err(port->se.dev, "Calculated clock divider %u exceeds maximum\n", clk_div);
1281		return -EINVAL;
1282	}
1283
1284	dev_dbg(port->se.dev, "desired_rate = %u, clk_rate = %lu, clk_div = %u\n, clk_idx = %u\n",
1285		baud * sampling_rate, clk_rate, clk_div, clk_idx);
1286
1287	uport->uartclk = clk_rate;
1288	port->clk_rate = clk_rate;
1289	dev_pm_opp_set_rate(uport->dev, clk_rate);
1290	ser_clk_cfg = SER_CLK_EN;
1291	ser_clk_cfg |= clk_div << CLK_DIV_SHFT;
1292
1293	/*
1294	 * Bump up BW vote on CPU and CORE path as driver supports FIFO mode
1295	 * only.
1296	 */
1297	avg_bw_core = (baud > 115200) ? Bps_to_icc(CORE_2X_50_MHZ)
1298						: GENI_DEFAULT_BW;
1299	port->se.icc_paths[GENI_TO_CORE].avg_bw = avg_bw_core;
1300	port->se.icc_paths[CPU_TO_GENI].avg_bw = Bps_to_icc(baud);
1301	geni_icc_set_bw(&port->se);
1302
1303	writel(ser_clk_cfg, uport->membase + GENI_SER_M_CLK_CFG);
1304	writel(ser_clk_cfg, uport->membase + GENI_SER_S_CLK_CFG);
1305	/* Configure clock selection register with the selected clock index */
1306	writel(clk_idx & CLK_SEL_MSK, uport->membase + SE_GENI_CLK_SEL);
1307	return 0;
1308}
1309
1310static int geni_serial_set_level(struct uart_port *uport, unsigned int baud)
1311{
1312	struct qcom_geni_serial_port *port = to_dev_port(uport);
1313	struct device *perf_dev = port->pd_list->pd_devs[DOMAIN_IDX_PERF];
1314
1315	/*
1316	 * The performance protocol sets UART communication
1317	 * speeds by selecting different performance levels
1318	 * through the OPP framework.
1319	 *
1320	 * Supported perf levels for baudrates in firmware are below
1321	 * +---------------------+--------------------+
1322	 * |  Perf level value   |  Baudrate values   |
1323	 * +---------------------+--------------------+
1324	 * |      300            |      300           |
1325	 * |      1200           |      1200          |
1326	 * |      2400           |      2400          |
1327	 * |      4800           |      4800          |
1328	 * |      9600           |      9600          |
1329	 * |      19200          |      19200         |
1330	 * |      38400          |      38400         |
1331	 * |      57600          |      57600         |
1332	 * |      115200         |      115200        |
1333	 * |      230400         |      230400        |
1334	 * |      460800         |      460800        |
1335	 * |      921600         |      921600        |
1336	 * |      2000000        |      2000000       |
1337	 * |      3000000        |      3000000       |
1338	 * |      3200000        |      3200000       |
1339	 * |      4000000        |      4000000       |
1340	 * +---------------------+--------------------+
1341	 */
1342
1343	return dev_pm_opp_set_level(perf_dev, baud);
1344}
1345
1346static void qcom_geni_serial_set_termios(struct uart_port *uport,
1347					 struct ktermios *termios,
1348					 const struct ktermios *old)
1349{
1350	struct qcom_geni_serial_port *port = to_dev_port(uport);
1351	unsigned int baud;
1352	unsigned long timeout;
1353	u32 bits_per_char;
1354	u32 tx_trans_cfg;
1355	u32 tx_parity_cfg;
1356	u32 rx_trans_cfg;
1357	u32 rx_parity_cfg;
1358	u32 stop_bit_len;
1359	int ret = 0;
1360
1361	/* baud rate */
1362	baud = uart_get_baud_rate(uport, termios, old, 300, 8000000);
1363
1364	ret = port->dev_data->set_rate(uport, baud);
1365	if (ret)
1366		return;
1367
1368	/* parity */
1369	tx_trans_cfg = readl(uport->membase + SE_UART_TX_TRANS_CFG);
1370	tx_parity_cfg = readl(uport->membase + SE_UART_TX_PARITY_CFG);
1371	rx_trans_cfg = readl(uport->membase + SE_UART_RX_TRANS_CFG);
1372	rx_parity_cfg = readl(uport->membase + SE_UART_RX_PARITY_CFG);
1373	if (termios->c_cflag & PARENB) {
1374		tx_trans_cfg |= UART_TX_PAR_EN;
1375		rx_trans_cfg |= UART_RX_PAR_EN;
1376		tx_parity_cfg |= PAR_CALC_EN;
1377		rx_parity_cfg |= PAR_CALC_EN;
1378		if (termios->c_cflag & PARODD) {
1379			tx_parity_cfg |= PAR_ODD;
1380			rx_parity_cfg |= PAR_ODD;
1381		} else if (termios->c_cflag & CMSPAR) {
1382			tx_parity_cfg |= PAR_SPACE;
1383			rx_parity_cfg |= PAR_SPACE;
1384		} else {
1385			tx_parity_cfg |= PAR_EVEN;
1386			rx_parity_cfg |= PAR_EVEN;
1387		}
1388	} else {
1389		tx_trans_cfg &= ~UART_TX_PAR_EN;
1390		rx_trans_cfg &= ~UART_RX_PAR_EN;
1391		tx_parity_cfg &= ~PAR_CALC_EN;
1392		rx_parity_cfg &= ~PAR_CALC_EN;
1393	}
1394
1395	/* bits per char */
1396	bits_per_char = tty_get_char_size(termios->c_cflag);
1397
1398	/* stop bits */
1399	if (termios->c_cflag & CSTOPB)
1400		stop_bit_len = TX_STOP_BIT_LEN_2;
1401	else
1402		stop_bit_len = TX_STOP_BIT_LEN_1;
1403
1404	/* flow control, clear the CTS_MASK bit if using flow control. */
1405	if (termios->c_cflag & CRTSCTS)
1406		tx_trans_cfg &= ~UART_CTS_MASK;
1407	else
1408		tx_trans_cfg |= UART_CTS_MASK;
1409
1410	if (baud) {
1411		uart_update_timeout(uport, termios->c_cflag, baud);
1412
1413		/*
1414		 * Make sure that qcom_geni_serial_poll_bitfield() waits for
1415		 * the FIFO, two-word intermediate transfer register and shift
1416		 * register to clear.
1417		 *
1418		 * Note that uart_fifo_timeout() also adds a 20 ms margin.
1419		 */
1420		timeout = jiffies_to_usecs(uart_fifo_timeout(uport));
1421		timeout += 3 * timeout / port->tx_fifo_depth;
1422		WRITE_ONCE(port->poll_timeout_us, timeout);
1423	}
1424
1425	if (!uart_console(uport))
1426		writel(port->loopback,
1427				uport->membase + SE_UART_LOOPBACK_CFG);
1428	writel(tx_trans_cfg, uport->membase + SE_UART_TX_TRANS_CFG);
1429	writel(tx_parity_cfg, uport->membase + SE_UART_TX_PARITY_CFG);
1430	writel(rx_trans_cfg, uport->membase + SE_UART_RX_TRANS_CFG);
1431	writel(rx_parity_cfg, uport->membase + SE_UART_RX_PARITY_CFG);
1432	writel(bits_per_char, uport->membase + SE_UART_TX_WORD_LEN);
1433	writel(bits_per_char, uport->membase + SE_UART_RX_WORD_LEN);
1434	writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN);
1435}
1436
1437#ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
1438static int qcom_geni_console_setup(struct console *co, char *options)
1439{
1440	struct uart_port *uport;
1441	struct qcom_geni_serial_port *port;
1442	int baud = 115200;
1443	int bits = 8;
1444	int parity = 'n';
1445	int flow = 'n';
1446	int ret;
1447
1448	if (co->index >= GENI_UART_CONS_PORTS  || co->index < 0)
1449		return -ENXIO;
1450
1451	port = get_port_from_line(co->index, true, NULL);
1452	if (IS_ERR(port)) {
1453		pr_err("Invalid line %d\n", co->index);
1454		return PTR_ERR(port);
1455	}
1456
1457	uport = &port->uport;
1458
1459	if (unlikely(!uport->membase))
1460		return -ENXIO;
1461
1462	if (!port->setup) {
1463		ret = qcom_geni_serial_port_setup(uport);
1464		if (ret)
1465			return ret;
1466	}
1467
1468	if (options)
1469		uart_parse_options(options, &baud, &parity, &bits, &flow);
1470
1471	return uart_set_options(uport, co, baud, parity, bits, flow);
1472}
1473
1474static void qcom_geni_serial_earlycon_write(struct console *con,
1475					const char *s, unsigned int n)
1476{
1477	struct earlycon_device *dev = con->data;
1478
1479	__qcom_geni_serial_console_write(&dev->port, s, n);
1480}
1481
1482#ifdef CONFIG_CONSOLE_POLL
1483static int qcom_geni_serial_earlycon_read(struct console *con,
1484					  char *s, unsigned int n)
1485{
1486	struct earlycon_device *dev = con->data;
1487	struct uart_port *uport = &dev->port;
1488	int num_read = 0;
1489	int ch;
1490
1491	while (num_read < n) {
1492		ch = qcom_geni_serial_get_char(uport);
1493		if (ch == NO_POLL_CHAR)
1494			break;
1495		s[num_read++] = ch;
1496	}
1497
1498	return num_read;
1499}
1500
1501static void __init qcom_geni_serial_enable_early_read(struct geni_se *se,
1502						      struct console *con)
1503{
1504	geni_se_setup_s_cmd(se, UART_START_READ, 0);
1505	con->read = qcom_geni_serial_earlycon_read;
1506}
1507#else
1508static inline void qcom_geni_serial_enable_early_read(struct geni_se *se,
1509						      struct console *con) { }
1510#endif
1511
1512static struct qcom_geni_private_data earlycon_private_data;
1513
1514static int __init qcom_geni_serial_earlycon_setup(struct earlycon_device *dev,
1515								const char *opt)
1516{
1517	struct uart_port *uport = &dev->port;
1518	u32 tx_trans_cfg;
1519	u32 tx_parity_cfg = 0;	/* Disable Tx Parity */
1520	u32 rx_trans_cfg = 0;
1521	u32 rx_parity_cfg = 0;	/* Disable Rx Parity */
1522	u32 stop_bit_len = 0;	/* Default stop bit length - 1 bit */
1523	u32 bits_per_char;
1524	struct geni_se se;
1525
1526	if (!uport->membase)
1527		return -EINVAL;
1528
1529	uport->private_data = &earlycon_private_data;
1530
1531	memset(&se, 0, sizeof(se));
1532	se.base = uport->membase;
1533	if (geni_se_read_proto(&se) != GENI_SE_UART)
1534		return -ENXIO;
1535	/*
1536	 * Ignore Flow control.
1537	 * n = 8.
1538	 */
1539	tx_trans_cfg = UART_CTS_MASK;
1540	bits_per_char = BITS_PER_BYTE;
1541
1542	/*
1543	 * Make an unconditional cancel on the main sequencer to reset
1544	 * it else we could end up in data loss scenarios.
1545	 */
1546	qcom_geni_serial_poll_tx_done(uport);
1547	qcom_geni_serial_abort_rx(uport);
1548	geni_se_config_packing(&se, BITS_PER_BYTE, BYTES_PER_FIFO_WORD,
1549			       false, true, true);
1550	geni_se_init(&se, DEF_FIFO_DEPTH_WORDS / 2, DEF_FIFO_DEPTH_WORDS - 2);
1551	geni_se_select_mode(&se, GENI_SE_FIFO);
1552
1553	writel(tx_trans_cfg, uport->membase + SE_UART_TX_TRANS_CFG);
1554	writel(tx_parity_cfg, uport->membase + SE_UART_TX_PARITY_CFG);
1555	writel(rx_trans_cfg, uport->membase + SE_UART_RX_TRANS_CFG);
1556	writel(rx_parity_cfg, uport->membase + SE_UART_RX_PARITY_CFG);
1557	writel(bits_per_char, uport->membase + SE_UART_TX_WORD_LEN);
1558	writel(bits_per_char, uport->membase + SE_UART_RX_WORD_LEN);
1559	writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN);
1560
1561	dev->con->write = qcom_geni_serial_earlycon_write;
1562	dev->con->setup = NULL;
1563	qcom_geni_serial_enable_early_read(&se, dev->con);
1564
1565	return 0;
1566}
1567OF_EARLYCON_DECLARE(qcom_geni, "qcom,geni-debug-uart",
1568				qcom_geni_serial_earlycon_setup);
1569
1570static int __init console_register(struct uart_driver *drv)
1571{
1572	return uart_register_driver(drv);
1573}
1574
1575static void console_unregister(struct uart_driver *drv)
1576{
1577	uart_unregister_driver(drv);
1578}
1579
1580static struct console cons_ops = {
1581	.name = "ttyMSM",
1582	.write = qcom_geni_serial_console_write,
1583	.device = uart_console_device,
1584	.setup = qcom_geni_console_setup,
1585	.flags = CON_PRINTBUFFER,
1586	.index = -1,
1587	.data = &qcom_geni_console_driver,
1588};
1589
1590static struct uart_driver qcom_geni_console_driver = {
1591	.owner = THIS_MODULE,
1592	.driver_name = "qcom_geni_console",
1593	.dev_name = "ttyMSM",
1594	.nr =  GENI_UART_CONS_PORTS,
1595	.cons = &cons_ops,
1596};
1597#else
1598static int console_register(struct uart_driver *drv)
1599{
1600	return 0;
1601}
1602
1603static void console_unregister(struct uart_driver *drv)
1604{
1605}
1606#endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
1607
1608static struct uart_driver qcom_geni_uart_driver = {
1609	.owner = THIS_MODULE,
1610	.driver_name = "qcom_geni_uart",
1611	.dev_name = "ttyHS",
1612	.nr = CONFIG_SERIAL_QCOM_GENI_UART_PORTS,
1613};
1614
1615static int geni_serial_resources_on(struct uart_port *uport)
1616{
1617	struct qcom_geni_serial_port *port = to_dev_port(uport);
1618	int ret;
1619
1620	ret = geni_icc_enable(&port->se);
1621	if (ret)
1622		return ret;
1623
1624	ret = geni_se_resources_on(&port->se);
1625	if (ret) {
1626		geni_icc_disable(&port->se);
1627		return ret;
1628	}
1629
1630	if (port->clk_rate)
1631		dev_pm_opp_set_rate(uport->dev, port->clk_rate);
1632
1633	return 0;
1634}
1635
1636static int geni_serial_resources_off(struct uart_port *uport)
1637{
1638	struct qcom_geni_serial_port *port = to_dev_port(uport);
1639	int ret;
1640
1641	dev_pm_opp_set_rate(uport->dev, 0);
1642	ret = geni_se_resources_off(&port->se);
1643	if (ret)
1644		return ret;
1645
1646	geni_icc_disable(&port->se);
1647
1648	return 0;
1649}
1650
1651static int geni_serial_resource_state(struct uart_port *uport, bool power_on)
1652{
1653	return power_on ? geni_serial_resources_on(uport) : geni_serial_resources_off(uport);
1654}
1655
1656static int geni_serial_pwr_init(struct uart_port *uport)
1657{
1658	struct qcom_geni_serial_port *port = to_dev_port(uport);
1659	int ret;
1660
1661	ret = dev_pm_domain_attach_list(port->se.dev,
1662					&port->dev_data->pd_data, &port->pd_list);
1663	if (ret <= 0)
1664		return -EINVAL;
1665
1666	return 0;
1667}
1668
1669static int geni_serial_resource_init(struct uart_port *uport)
1670{
1671	struct qcom_geni_serial_port *port = to_dev_port(uport);
1672	int ret;
1673
1674	port->se.clk = devm_clk_get(port->se.dev, "se");
1675	if (IS_ERR(port->se.clk)) {
1676		ret = PTR_ERR(port->se.clk);
1677		dev_err(port->se.dev, "Err getting SE Core clk %d\n", ret);
1678		return ret;
1679	}
1680
1681	ret = geni_icc_get(&port->se, NULL);
1682	if (ret)
1683		return ret;
1684
1685	port->se.icc_paths[GENI_TO_CORE].avg_bw = GENI_DEFAULT_BW;
1686	port->se.icc_paths[CPU_TO_GENI].avg_bw = GENI_DEFAULT_BW;
1687
1688	/* Set BW for register access */
1689	ret = geni_icc_set_bw(&port->se);
1690	if (ret)
1691		return ret;
1692
1693	ret = devm_pm_opp_set_clkname(port->se.dev, "se");
1694	if (ret)
1695		return ret;
1696
1697	/* OPP table is optional */
1698	ret = devm_pm_opp_of_add_table(port->se.dev);
1699	if (ret && ret != -ENODEV) {
1700		dev_err(port->se.dev, "invalid OPP table in device tree\n");
1701		return ret;
1702	}
1703
1704	return 0;
1705}
1706
1707static void qcom_geni_serial_pm(struct uart_port *uport,
1708		unsigned int new_state, unsigned int old_state)
1709{
1710
1711	/* If we've never been called, treat it as off */
1712	if (old_state == UART_PM_STATE_UNDEFINED)
1713		old_state = UART_PM_STATE_OFF;
1714
1715	if (new_state == UART_PM_STATE_ON && old_state == UART_PM_STATE_OFF)
1716		pm_runtime_resume_and_get(uport->dev);
1717	else if (new_state == UART_PM_STATE_OFF &&
1718		 old_state == UART_PM_STATE_ON)
1719		pm_runtime_put_sync(uport->dev);
1720
1721}
1722
1723/**
1724 * qcom_geni_rs485_config - Configure RS485 settings for the UART port
1725 * @uport: Pointer to the UART port structure
1726 * @termios: Pointer to the termios structure
1727 * @rs485: Pointer to the RS485 configuration structure
1728 * This function configures the RTS (Request to Send) pin behavior for RS485 mode.
1729 * When RS485 mode is enabled, the RTS pin is kept in default ACTIVE HIGH state.
1730 * Return: Always returns 0.
1731 */
1732
1733static int qcom_geni_rs485_config(struct uart_port *uport,
1734				  struct ktermios *termios, struct serial_rs485 *rs485)
1735{
1736	qcom_geni_set_rs485_mode(uport, SER_RS485_ENABLED);
1737
1738	return 0;
1739}
1740
1741static const struct uart_ops qcom_geni_console_pops = {
1742	.tx_empty = qcom_geni_serial_tx_empty,
1743	.stop_tx = qcom_geni_serial_stop_tx_fifo,
1744	.start_tx = qcom_geni_serial_start_tx_fifo,
1745	.stop_rx = qcom_geni_serial_stop_rx_fifo,
1746	.start_rx = qcom_geni_serial_start_rx_fifo,
1747	.set_termios = qcom_geni_serial_set_termios,
1748	.startup = qcom_geni_serial_startup,
1749	.request_port = qcom_geni_serial_request_port,
1750	.config_port = qcom_geni_serial_config_port,
1751	.shutdown = qcom_geni_serial_shutdown,
1752	.flush_buffer = qcom_geni_serial_flush_buffer,
1753	.type = qcom_geni_serial_get_type,
1754	.set_mctrl = qcom_geni_serial_set_mctrl,
1755	.get_mctrl = qcom_geni_serial_get_mctrl,
1756#ifdef CONFIG_CONSOLE_POLL
1757	.poll_get_char	= qcom_geni_serial_get_char,
1758	.poll_put_char	= qcom_geni_serial_poll_put_char,
1759	.poll_init = qcom_geni_serial_poll_init,
1760#endif
1761	.pm = qcom_geni_serial_pm,
1762};
1763
1764static const struct uart_ops qcom_geni_uart_pops = {
1765	.tx_empty = qcom_geni_serial_tx_empty,
1766	.stop_tx = qcom_geni_serial_stop_tx_dma,
1767	.start_tx = qcom_geni_serial_start_tx_dma,
1768	.start_rx = qcom_geni_serial_start_rx_dma,
1769	.stop_rx = qcom_geni_serial_stop_rx_dma,
1770	.set_termios = qcom_geni_serial_set_termios,
1771	.startup = qcom_geni_serial_startup,
1772	.request_port = qcom_geni_serial_request_port,
1773	.config_port = qcom_geni_serial_config_port,
1774	.shutdown = qcom_geni_serial_shutdown,
1775	.type = qcom_geni_serial_get_type,
1776	.set_mctrl = qcom_geni_serial_set_mctrl,
1777	.get_mctrl = qcom_geni_serial_get_mctrl,
1778	.pm = qcom_geni_serial_pm,
1779};
1780
1781static int qcom_geni_serial_probe(struct platform_device *pdev)
1782{
1783	int ret = 0;
1784	int line;
1785	struct qcom_geni_serial_port *port;
1786	struct uart_port *uport;
1787	struct resource *res;
1788	int irq;
1789	struct uart_driver *drv;
1790	const struct qcom_geni_device_data *data;
1791
1792	data = of_device_get_match_data(&pdev->dev);
1793	if (!data)
1794		return -EINVAL;
1795
1796	if (data->console) {
1797		drv = &qcom_geni_console_driver;
1798		line = of_alias_get_id(pdev->dev.of_node, "serial");
1799	} else {
1800		drv = &qcom_geni_uart_driver;
1801		line = of_alias_get_id(pdev->dev.of_node, "serial");
1802		if (line == -ENODEV) /* compat with non-standard aliases */
1803			line = of_alias_get_id(pdev->dev.of_node, "hsuart");
1804	}
1805
1806	port = get_port_from_line(line, data->console, &pdev->dev);
1807	if (IS_ERR(port)) {
1808		dev_err(&pdev->dev, "Invalid line %d\n", line);
1809		return PTR_ERR(port);
1810	}
1811
1812	uport = &port->uport;
1813	/* Don't allow 2 drivers to access the same port */
1814	if (uport->private_data)
1815		return -ENODEV;
1816
1817	uport->dev = &pdev->dev;
1818	port->dev_data = data;
1819	port->se.dev = &pdev->dev;
1820	port->se.wrapper = dev_get_drvdata(pdev->dev.parent);
1821
1822	ret = port->dev_data->resources_init(uport);
1823	if (ret)
1824		return ret;
1825
1826	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1827	if (!res) {
1828		ret = -EINVAL;
1829		goto error;
1830	}
1831
1832	uport->mapbase = res->start;
1833
1834	uport->rs485_config = qcom_geni_rs485_config;
1835	uport->rs485_supported = qcom_geni_rs485_supported;
1836	port->tx_fifo_depth = DEF_FIFO_DEPTH_WORDS;
1837	port->rx_fifo_depth = DEF_FIFO_DEPTH_WORDS;
1838	port->tx_fifo_width = DEF_FIFO_WIDTH_BITS;
1839
1840	if (!data->console) {
1841		port->rx_buf = devm_kzalloc(uport->dev,
1842					    DMA_RX_BUF_SIZE, GFP_KERNEL);
1843		if (!port->rx_buf) {
1844			ret = -ENOMEM;
1845			goto error;
1846		}
1847	}
1848
1849	port->name = devm_kasprintf(uport->dev, GFP_KERNEL,
1850			"qcom_geni_serial_%s%d",
1851			uart_console(uport) ? "console" : "uart", uport->line);
1852	if (!port->name) {
1853		ret = -ENOMEM;
1854		goto error;
1855	}
1856
1857	irq = platform_get_irq(pdev, 0);
1858	if (irq < 0) {
1859		ret = irq;
1860		goto error;
1861	}
1862
1863	uport->irq = irq;
1864	uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_QCOM_GENI_CONSOLE);
1865
1866	if (!data->console)
1867		port->wakeup_irq = platform_get_irq_optional(pdev, 1);
1868
1869	if (of_property_read_bool(pdev->dev.of_node, "rx-tx-swap"))
1870		port->rx_tx_swap = true;
1871
1872	if (of_property_read_bool(pdev->dev.of_node, "cts-rts-swap"))
1873		port->cts_rts_swap = true;
1874
1875	port->private_data.drv = drv;
1876	uport->private_data = &port->private_data;
1877	platform_set_drvdata(pdev, port);
1878
1879	irq_set_status_flags(uport->irq, IRQ_NOAUTOEN);
1880	ret = devm_request_irq(uport->dev, uport->irq, qcom_geni_serial_isr,
1881			IRQF_TRIGGER_HIGH, port->name, uport);
1882	if (ret) {
1883		dev_err(uport->dev, "Failed to get IRQ ret %d\n", ret);
1884		goto error;
1885	}
1886
1887	ret = uart_get_rs485_mode(uport);
1888	if (ret)
1889		goto error;
1890
1891	if (port->wakeup_irq > 0) {
1892		device_init_wakeup(&pdev->dev, true);
1893		ret = dev_pm_set_dedicated_wake_irq(&pdev->dev,
1894						port->wakeup_irq);
1895		if (ret) {
1896			device_init_wakeup(&pdev->dev, false);
1897			ida_free(&port_ida, uport->line);
1898			goto error;
1899		}
1900	}
1901
1902	devm_pm_runtime_enable(port->se.dev);
1903
1904	ret = uart_add_one_port(drv, uport);
1905	if (ret)
1906		goto error;
1907
1908	return 0;
1909
1910error:
1911	dev_pm_domain_detach_list(port->pd_list);
1912	return ret;
1913}
1914
1915static void qcom_geni_serial_remove(struct platform_device *pdev)
1916{
1917	struct qcom_geni_serial_port *port = platform_get_drvdata(pdev);
1918	struct uart_port *uport = &port->uport;
1919	struct uart_driver *drv = port->private_data.drv;
1920
1921	dev_pm_clear_wake_irq(&pdev->dev);
1922	device_init_wakeup(&pdev->dev, false);
1923	ida_free(&port_ida, uport->line);
1924	uart_remove_one_port(drv, &port->uport);
1925	dev_pm_domain_detach_list(port->pd_list);
1926}
1927
1928static int __maybe_unused qcom_geni_serial_runtime_suspend(struct device *dev)
1929{
1930	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1931	struct uart_port *uport = &port->uport;
1932	int ret = 0;
1933
1934	if (port->dev_data->power_state)
1935		ret = port->dev_data->power_state(uport, false);
1936
1937	return ret;
1938}
1939
1940static int __maybe_unused qcom_geni_serial_runtime_resume(struct device *dev)
1941{
1942	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1943	struct uart_port *uport = &port->uport;
1944	int ret = 0;
1945
1946	if (port->dev_data->power_state)
1947		ret = port->dev_data->power_state(uport, true);
1948
1949	return ret;
1950}
1951
1952static int qcom_geni_serial_suspend(struct device *dev)
1953{
1954	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1955	struct uart_port *uport = &port->uport;
1956	struct qcom_geni_private_data *private_data = uport->private_data;
1957
1958	/*
1959	 * This is done so we can hit the lowest possible state in suspend
1960	 * even with no_console_suspend
1961	 */
1962	if (uart_console(uport)) {
1963		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ACTIVE_ONLY);
1964		geni_icc_set_bw(&port->se);
1965	}
1966	return uart_suspend_port(private_data->drv, uport);
1967}
1968
1969static int qcom_geni_serial_resume(struct device *dev)
1970{
1971	int ret;
1972	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1973	struct uart_port *uport = &port->uport;
1974	struct qcom_geni_private_data *private_data = uport->private_data;
1975
1976	ret = uart_resume_port(private_data->drv, uport);
1977	if (uart_console(uport)) {
1978		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS);
1979		geni_icc_set_bw(&port->se);
1980	}
1981	return ret;
1982}
1983
1984static const struct qcom_geni_device_data qcom_geni_console_data = {
1985	.console = true,
1986	.mode = GENI_SE_FIFO,
1987	.resources_init = geni_serial_resource_init,
1988	.set_rate = geni_serial_set_rate,
1989	.power_state = geni_serial_resource_state,
1990};
1991
1992static const struct qcom_geni_device_data qcom_geni_uart_data = {
1993	.console = false,
1994	.mode = GENI_SE_DMA,
1995	.resources_init = geni_serial_resource_init,
1996	.set_rate = geni_serial_set_rate,
1997	.power_state = geni_serial_resource_state,
1998};
1999
2000static const struct qcom_geni_device_data sa8255p_qcom_geni_console_data = {
2001	.console = true,
2002	.mode = GENI_SE_FIFO,
2003	.pd_data = {
2004		.pd_flags = PD_FLAG_DEV_LINK_ON,
2005		.pd_names = (const char*[]) { "power", "perf" },
2006		.num_pd_names = 2,
2007	},
2008	.resources_init = geni_serial_pwr_init,
2009	.set_rate = geni_serial_set_level,
2010};
2011
2012static const struct qcom_geni_device_data sa8255p_qcom_geni_uart_data = {
2013	.console = false,
2014	.mode = GENI_SE_DMA,
2015	.pd_data = {
2016		.pd_flags = PD_FLAG_DEV_LINK_ON,
2017		.pd_names = (const char*[]) { "power", "perf" },
2018		.num_pd_names = 2,
2019	},
2020	.resources_init = geni_serial_pwr_init,
2021	.set_rate = geni_serial_set_level,
2022};
2023
2024static const struct dev_pm_ops qcom_geni_serial_pm_ops = {
2025	SET_RUNTIME_PM_OPS(qcom_geni_serial_runtime_suspend,
2026			   qcom_geni_serial_runtime_resume, NULL)
2027	SYSTEM_SLEEP_PM_OPS(qcom_geni_serial_suspend, qcom_geni_serial_resume)
2028};
2029
2030static const struct of_device_id qcom_geni_serial_match_table[] = {
2031	{
2032		.compatible = "qcom,geni-debug-uart",
2033		.data = &qcom_geni_console_data,
2034	},
2035	{
2036		.compatible = "qcom,sa8255p-geni-debug-uart",
2037		.data = &sa8255p_qcom_geni_console_data,
2038	},
2039	{
2040		.compatible = "qcom,geni-uart",
2041		.data = &qcom_geni_uart_data,
2042	},
2043	{
2044		.compatible = "qcom,sa8255p-geni-uart",
2045		.data = &sa8255p_qcom_geni_uart_data,
2046	},
2047	{}
2048};
2049MODULE_DEVICE_TABLE(of, qcom_geni_serial_match_table);
2050
2051static struct platform_driver qcom_geni_serial_platform_driver = {
2052	.remove = qcom_geni_serial_remove,
2053	.probe = qcom_geni_serial_probe,
2054	.driver = {
2055		.name = "qcom_geni_serial",
2056		.of_match_table = qcom_geni_serial_match_table,
2057		.pm = &qcom_geni_serial_pm_ops,
2058	},
2059};
2060
2061static int __init qcom_geni_serial_init(void)
2062{
2063	int ret;
2064
2065	ret = console_register(&qcom_geni_console_driver);
2066	if (ret)
2067		return ret;
2068
2069	ret = uart_register_driver(&qcom_geni_uart_driver);
2070	if (ret) {
2071		console_unregister(&qcom_geni_console_driver);
2072		return ret;
2073	}
2074
2075	ret = platform_driver_register(&qcom_geni_serial_platform_driver);
2076	if (ret) {
2077		console_unregister(&qcom_geni_console_driver);
2078		uart_unregister_driver(&qcom_geni_uart_driver);
2079	}
2080	return ret;
2081}
2082module_init(qcom_geni_serial_init);
2083
2084static void __exit qcom_geni_serial_exit(void)
2085{
2086	platform_driver_unregister(&qcom_geni_serial_platform_driver);
2087	console_unregister(&qcom_geni_console_driver);
2088	uart_unregister_driver(&qcom_geni_uart_driver);
2089}
2090module_exit(qcom_geni_serial_exit);
2091
2092MODULE_DESCRIPTION("Serial driver for GENI based QUP cores");
2093MODULE_LICENSE("GPL v2");