File Index
Files and directories
Sign in
FILE

gpi.c

File preview with syntax highlighting.

linux-6.19.14/drivers/dma/qcom/gpi.c C 
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
   4 * Copyright (c) 2020, Linaro Limited
   5 */
   6
   7#include <dt-bindings/dma/qcom-gpi.h>
   8#include <linux/bitfield.h>
   9#include <linux/dma-mapping.h>
  10#include <linux/dmaengine.h>
  11#include <linux/module.h>
  12#include <linux/of_dma.h>
  13#include <linux/platform_device.h>
  14#include <linux/dma/qcom-gpi-dma.h>
  15#include <linux/scatterlist.h>
  16#include <linux/slab.h>
  17#include "../dmaengine.h"
  18#include "../virt-dma.h"
  19
  20#define TRE_TYPE_DMA		0x10
  21#define TRE_TYPE_IMMEDIATE_DMA	0x11
  22#define TRE_TYPE_GO		0x20
  23#define TRE_TYPE_CONFIG0	0x22
  24
  25/* TRE flags */
  26#define TRE_FLAGS_CHAIN		BIT(0)
  27#define TRE_FLAGS_IEOB		BIT(8)
  28#define TRE_FLAGS_IEOT		BIT(9)
  29#define TRE_FLAGS_BEI		BIT(10)
  30#define TRE_FLAGS_LINK		BIT(11)
  31#define TRE_FLAGS_TYPE		GENMASK(23, 16)
  32
  33/* SPI CONFIG0 WD0 */
  34#define TRE_SPI_C0_WORD_SZ	GENMASK(4, 0)
  35#define TRE_SPI_C0_LOOPBACK	BIT(8)
  36#define TRE_SPI_C0_CS		BIT(11)
  37#define TRE_SPI_C0_CPHA		BIT(12)
  38#define TRE_SPI_C0_CPOL		BIT(13)
  39#define TRE_SPI_C0_TX_PACK	BIT(24)
  40#define TRE_SPI_C0_RX_PACK	BIT(25)
  41
  42/* CONFIG0 WD2 */
  43#define TRE_C0_CLK_DIV		GENMASK(11, 0)
  44#define TRE_C0_CLK_SRC		GENMASK(19, 16)
  45
  46/* SPI GO WD0 */
  47#define TRE_SPI_GO_CMD		GENMASK(4, 0)
  48#define TRE_SPI_GO_CS		GENMASK(10, 8)
  49#define TRE_SPI_GO_FRAG		BIT(26)
  50
  51/* GO WD2 */
  52#define TRE_RX_LEN		GENMASK(23, 0)
  53
  54/* I2C Config0 WD0 */
  55#define TRE_I2C_C0_TLOW		GENMASK(7, 0)
  56#define TRE_I2C_C0_THIGH	GENMASK(15, 8)
  57#define TRE_I2C_C0_TCYL		GENMASK(23, 16)
  58#define TRE_I2C_C0_TX_PACK	BIT(24)
  59#define TRE_I2C_C0_RX_PACK      BIT(25)
  60
  61/* I2C GO WD0 */
  62#define TRE_I2C_GO_CMD          GENMASK(4, 0)
  63#define TRE_I2C_GO_ADDR		GENMASK(14, 8)
  64#define TRE_I2C_GO_STRETCH	BIT(26)
  65
  66/* DMA TRE */
  67#define TRE_DMA_LEN		GENMASK(23, 0)
  68#define TRE_DMA_IMMEDIATE_LEN	GENMASK(3, 0)
  69
  70/* Register offsets from gpi-top */
  71#define GPII_n_CH_k_CNTXT_0_OFFS(n, k)	(0x20000 + (0x4000 * (n)) + (0x80 * (k)))
  72#define GPII_n_CH_k_CNTXT_0_EL_SIZE	GENMASK(31, 24)
  73#define GPII_n_CH_k_CNTXT_0_CHSTATE	GENMASK(23, 20)
  74#define GPII_n_CH_k_CNTXT_0_ERIDX	GENMASK(18, 14)
  75#define GPII_n_CH_k_CNTXT_0_DIR		BIT(3)
  76#define GPII_n_CH_k_CNTXT_0_PROTO	GENMASK(2, 0)
  77
  78#define GPII_n_CH_k_CNTXT_0(el_size, erindex, dir, chtype_proto)  \
  79	(FIELD_PREP(GPII_n_CH_k_CNTXT_0_EL_SIZE, el_size)	| \
  80	 FIELD_PREP(GPII_n_CH_k_CNTXT_0_ERIDX, erindex)		| \
  81	 FIELD_PREP(GPII_n_CH_k_CNTXT_0_DIR, dir)		| \
  82	 FIELD_PREP(GPII_n_CH_k_CNTXT_0_PROTO, chtype_proto))
  83
  84#define GPI_CHTYPE_DIR_IN	(0)
  85#define GPI_CHTYPE_DIR_OUT	(1)
  86
  87#define GPI_CHTYPE_PROTO_GPI	(0x2)
  88
  89#define GPII_n_CH_k_DOORBELL_0_OFFS(n, k)	(0x22000 + (0x4000 * (n)) + (0x8 * (k)))
  90#define GPII_n_CH_CMD_OFFS(n)			(0x23008 + (0x4000 * (n)))
  91#define GPII_n_CH_CMD_OPCODE			GENMASK(31, 24)
  92#define GPII_n_CH_CMD_CHID			GENMASK(7, 0)
  93#define GPII_n_CH_CMD(opcode, chid)				 \
  94		     (FIELD_PREP(GPII_n_CH_CMD_OPCODE, opcode) | \
  95		      FIELD_PREP(GPII_n_CH_CMD_CHID, chid))
  96
  97#define GPII_n_CH_CMD_ALLOCATE		(0)
  98#define GPII_n_CH_CMD_START		(1)
  99#define GPII_n_CH_CMD_STOP		(2)
 100#define GPII_n_CH_CMD_RESET		(9)
 101#define GPII_n_CH_CMD_DE_ALLOC		(10)
 102#define GPII_n_CH_CMD_UART_SW_STALE	(32)
 103#define GPII_n_CH_CMD_UART_RFR_READY	(33)
 104#define GPII_n_CH_CMD_UART_RFR_NOT_READY (34)
 105
 106/* EV Context Array */
 107#define GPII_n_EV_CH_k_CNTXT_0_OFFS(n, k) (0x21000 + (0x4000 * (n)) + (0x80 * (k)))
 108#define GPII_n_EV_k_CNTXT_0_EL_SIZE	GENMASK(31, 24)
 109#define GPII_n_EV_k_CNTXT_0_CHSTATE	GENMASK(23, 20)
 110#define GPII_n_EV_k_CNTXT_0_INTYPE	BIT(16)
 111#define GPII_n_EV_k_CNTXT_0_CHTYPE	GENMASK(3, 0)
 112
 113#define GPII_n_EV_k_CNTXT_0(el_size, inttype, chtype)		\
 114	(FIELD_PREP(GPII_n_EV_k_CNTXT_0_EL_SIZE, el_size) |	\
 115	 FIELD_PREP(GPII_n_EV_k_CNTXT_0_INTYPE, inttype)  |	\
 116	 FIELD_PREP(GPII_n_EV_k_CNTXT_0_CHTYPE, chtype))
 117
 118#define GPI_INTTYPE_IRQ		(1)
 119#define GPI_CHTYPE_GPI_EV	(0x2)
 120
 121enum CNTXT_OFFS {
 122	CNTXT_0_CONFIG = 0x0,
 123	CNTXT_1_R_LENGTH = 0x4,
 124	CNTXT_2_RING_BASE_LSB = 0x8,
 125	CNTXT_3_RING_BASE_MSB = 0xC,
 126	CNTXT_4_RING_RP_LSB = 0x10,
 127	CNTXT_5_RING_RP_MSB = 0x14,
 128	CNTXT_6_RING_WP_LSB = 0x18,
 129	CNTXT_7_RING_WP_MSB = 0x1C,
 130	CNTXT_8_RING_INT_MOD = 0x20,
 131	CNTXT_9_RING_INTVEC = 0x24,
 132	CNTXT_10_RING_MSI_LSB = 0x28,
 133	CNTXT_11_RING_MSI_MSB = 0x2C,
 134	CNTXT_12_RING_RP_UPDATE_LSB = 0x30,
 135	CNTXT_13_RING_RP_UPDATE_MSB = 0x34,
 136};
 137
 138#define GPII_n_EV_CH_k_DOORBELL_0_OFFS(n, k)	(0x22100 + (0x4000 * (n)) + (0x8 * (k)))
 139#define GPII_n_EV_CH_CMD_OFFS(n)		(0x23010 + (0x4000 * (n)))
 140#define GPII_n_EV_CMD_OPCODE			GENMASK(31, 24)
 141#define GPII_n_EV_CMD_CHID			GENMASK(7, 0)
 142#define GPII_n_EV_CMD(opcode, chid)				 \
 143		     (FIELD_PREP(GPII_n_EV_CMD_OPCODE, opcode) | \
 144		      FIELD_PREP(GPII_n_EV_CMD_CHID, chid))
 145
 146#define GPII_n_EV_CH_CMD_ALLOCATE		(0x00)
 147#define GPII_n_EV_CH_CMD_RESET			(0x09)
 148#define GPII_n_EV_CH_CMD_DE_ALLOC		(0x0A)
 149
 150#define GPII_n_CNTXT_TYPE_IRQ_OFFS(n)		(0x23080 + (0x4000 * (n)))
 151
 152/* mask type register */
 153#define GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(n)	(0x23088 + (0x4000 * (n)))
 154#define GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK		GENMASK(6, 0)
 155#define GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL	BIT(6)
 156#define GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB		BIT(3)
 157#define GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB		BIT(2)
 158#define GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL	BIT(1)
 159#define GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL	BIT(0)
 160
 161#define GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(n)	(0x23090 + (0x4000 * (n)))
 162#define GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n)	(0x23094 + (0x4000 * (n)))
 163
 164/* Mask channel control interrupt register */
 165#define GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(n)	(0x23098 + (0x4000 * (n)))
 166#define GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK	GENMASK(1, 0)
 167
 168/* Mask event control interrupt register */
 169#define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n)	(0x2309C + (0x4000 * (n)))
 170#define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK	BIT(0)
 171
 172#define GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(n)	(0x230A0 + (0x4000 * (n)))
 173#define GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n)	(0x230A4 + (0x4000 * (n)))
 174
 175/* Mask event interrupt register */
 176#define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n)	(0x230B8 + (0x4000 * (n)))
 177#define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK	BIT(0)
 178
 179#define GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n)	(0x230C0 + (0x4000 * (n)))
 180#define GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(n)	(0x23100 + (0x4000 * (n)))
 181#define GPI_GLOB_IRQ_ERROR_INT_MSK		BIT(0)
 182
 183/* GPII specific Global - Enable bit register */
 184#define GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(n)	(0x23108 + (0x4000 * (n)))
 185#define GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(n)	(0x23110 + (0x4000 * (n)))
 186#define GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(n)	(0x23118 + (0x4000 * (n)))
 187
 188/* GPII general interrupt - Enable bit register */
 189#define GPII_n_CNTXT_GPII_IRQ_EN_OFFS(n)	(0x23120 + (0x4000 * (n)))
 190#define GPII_n_CNTXT_GPII_IRQ_EN_BMSK		GENMASK(3, 0)
 191
 192#define GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(n)	(0x23128 + (0x4000 * (n)))
 193
 194/* GPII Interrupt Type register */
 195#define GPII_n_CNTXT_INTSET_OFFS(n)		(0x23180 + (0x4000 * (n)))
 196#define GPII_n_CNTXT_INTSET_BMSK		BIT(0)
 197
 198#define GPII_n_CNTXT_MSI_BASE_LSB_OFFS(n)	(0x23188 + (0x4000 * (n)))
 199#define GPII_n_CNTXT_MSI_BASE_MSB_OFFS(n)	(0x2318C + (0x4000 * (n)))
 200#define GPII_n_CNTXT_SCRATCH_0_OFFS(n)		(0x23400 + (0x4000 * (n)))
 201#define GPII_n_CNTXT_SCRATCH_1_OFFS(n)		(0x23404 + (0x4000 * (n)))
 202
 203#define GPII_n_ERROR_LOG_OFFS(n)		(0x23200 + (0x4000 * (n)))
 204
 205/* QOS Registers */
 206#define GPII_n_CH_k_QOS_OFFS(n, k)		(0x2005C + (0x4000 * (n)) + (0x80 * (k)))
 207
 208/* Scratch registers */
 209#define GPII_n_CH_k_SCRATCH_0_OFFS(n, k)	(0x20060 + (0x4000 * (n)) + (0x80 * (k)))
 210#define GPII_n_CH_k_SCRATCH_0_SEID		GENMASK(2, 0)
 211#define GPII_n_CH_k_SCRATCH_0_PROTO		GENMASK(7, 4)
 212#define GPII_n_CH_k_SCRATCH_0_PAIR		GENMASK(20, 16)
 213#define GPII_n_CH_k_SCRATCH_0(pair, proto, seid)		\
 214			     (FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PAIR, pair)	| \
 215			      FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PROTO, proto)	| \
 216			      FIELD_PREP(GPII_n_CH_k_SCRATCH_0_SEID, seid))
 217#define GPII_n_CH_k_SCRATCH_1_OFFS(n, k)	(0x20064 + (0x4000 * (n)) + (0x80 * (k)))
 218#define GPII_n_CH_k_SCRATCH_2_OFFS(n, k)	(0x20068 + (0x4000 * (n)) + (0x80 * (k)))
 219#define GPII_n_CH_k_SCRATCH_3_OFFS(n, k)	(0x2006C + (0x4000 * (n)) + (0x80 * (k)))
 220
 221struct __packed gpi_tre {
 222	u32 dword[4];
 223};
 224
 225enum msm_gpi_tce_code {
 226	MSM_GPI_TCE_SUCCESS = 1,
 227	MSM_GPI_TCE_EOT = 2,
 228	MSM_GPI_TCE_EOB = 4,
 229	MSM_GPI_TCE_UNEXP_ERR = 16,
 230};
 231
 232#define CMD_TIMEOUT_MS		(250)
 233
 234#define MAX_CHANNELS_PER_GPII	(2)
 235#define GPI_TX_CHAN		(0)
 236#define GPI_RX_CHAN		(1)
 237#define STATE_IGNORE		(U32_MAX)
 238#define EV_FACTOR		(2)
 239#define REQ_OF_DMA_ARGS		(5) /* # of arguments required from client */
 240#define CHAN_TRES		64
 241
 242struct __packed xfer_compl_event {
 243	u64 ptr;
 244	u32 length:24;
 245	u8 code;
 246	u16 status;
 247	u8 type;
 248	u8 chid;
 249};
 250
 251struct __packed immediate_data_event {
 252	u8 data_bytes[8];
 253	u8 length:4;
 254	u8 resvd:4;
 255	u16 tre_index;
 256	u8 code;
 257	u16 status;
 258	u8 type;
 259	u8 chid;
 260};
 261
 262struct __packed qup_notif_event {
 263	u32 status;
 264	u32 time;
 265	u32 count:24;
 266	u8 resvd;
 267	u16 resvd1;
 268	u8 type;
 269	u8 chid;
 270};
 271
 272struct __packed gpi_ere {
 273	u32 dword[4];
 274};
 275
 276enum GPI_EV_TYPE {
 277	XFER_COMPLETE_EV_TYPE = 0x22,
 278	IMMEDIATE_DATA_EV_TYPE = 0x30,
 279	QUP_NOTIF_EV_TYPE = 0x31,
 280	STALE_EV_TYPE = 0xFF,
 281};
 282
 283union __packed gpi_event {
 284	struct __packed xfer_compl_event xfer_compl_event;
 285	struct __packed immediate_data_event immediate_data_event;
 286	struct __packed qup_notif_event qup_notif_event;
 287	struct __packed gpi_ere gpi_ere;
 288};
 289
 290enum gpii_irq_settings {
 291	DEFAULT_IRQ_SETTINGS,
 292	MASK_IEOB_SETTINGS,
 293};
 294
 295enum gpi_ev_state {
 296	DEFAULT_EV_CH_STATE = 0,
 297	EV_STATE_NOT_ALLOCATED = DEFAULT_EV_CH_STATE,
 298	EV_STATE_ALLOCATED,
 299	MAX_EV_STATES
 300};
 301
 302static const char *const gpi_ev_state_str[MAX_EV_STATES] = {
 303	[EV_STATE_NOT_ALLOCATED] = "NOT ALLOCATED",
 304	[EV_STATE_ALLOCATED] = "ALLOCATED",
 305};
 306
 307#define TO_GPI_EV_STATE_STR(_state) (((_state) >= MAX_EV_STATES) ? \
 308				    "INVALID" : gpi_ev_state_str[(_state)])
 309
 310enum gpi_ch_state {
 311	DEFAULT_CH_STATE = 0x0,
 312	CH_STATE_NOT_ALLOCATED = DEFAULT_CH_STATE,
 313	CH_STATE_ALLOCATED = 0x1,
 314	CH_STATE_STARTED = 0x2,
 315	CH_STATE_STOPPED = 0x3,
 316	CH_STATE_STOP_IN_PROC = 0x4,
 317	CH_STATE_ERROR = 0xf,
 318	MAX_CH_STATES
 319};
 320
 321enum gpi_cmd {
 322	GPI_CH_CMD_BEGIN,
 323	GPI_CH_CMD_ALLOCATE = GPI_CH_CMD_BEGIN,
 324	GPI_CH_CMD_START,
 325	GPI_CH_CMD_STOP,
 326	GPI_CH_CMD_RESET,
 327	GPI_CH_CMD_DE_ALLOC,
 328	GPI_CH_CMD_UART_SW_STALE,
 329	GPI_CH_CMD_UART_RFR_READY,
 330	GPI_CH_CMD_UART_RFR_NOT_READY,
 331	GPI_CH_CMD_END = GPI_CH_CMD_UART_RFR_NOT_READY,
 332	GPI_EV_CMD_BEGIN,
 333	GPI_EV_CMD_ALLOCATE = GPI_EV_CMD_BEGIN,
 334	GPI_EV_CMD_RESET,
 335	GPI_EV_CMD_DEALLOC,
 336	GPI_EV_CMD_END = GPI_EV_CMD_DEALLOC,
 337	GPI_MAX_CMD,
 338};
 339
 340#define IS_CHAN_CMD(_cmd) ((_cmd) <= GPI_CH_CMD_END)
 341
 342static const char *const gpi_cmd_str[GPI_MAX_CMD] = {
 343	[GPI_CH_CMD_ALLOCATE] = "CH ALLOCATE",
 344	[GPI_CH_CMD_START] = "CH START",
 345	[GPI_CH_CMD_STOP] = "CH STOP",
 346	[GPI_CH_CMD_RESET] = "CH_RESET",
 347	[GPI_CH_CMD_DE_ALLOC] = "DE ALLOC",
 348	[GPI_CH_CMD_UART_SW_STALE] = "UART SW STALE",
 349	[GPI_CH_CMD_UART_RFR_READY] = "UART RFR READY",
 350	[GPI_CH_CMD_UART_RFR_NOT_READY] = "UART RFR NOT READY",
 351	[GPI_EV_CMD_ALLOCATE] = "EV ALLOCATE",
 352	[GPI_EV_CMD_RESET] = "EV RESET",
 353	[GPI_EV_CMD_DEALLOC] = "EV DEALLOC",
 354};
 355
 356#define TO_GPI_CMD_STR(_cmd) (((_cmd) >= GPI_MAX_CMD) ? "INVALID" : \
 357				  gpi_cmd_str[(_cmd)])
 358
 359/*
 360 * @DISABLE_STATE: no register access allowed
 361 * @CONFIG_STATE:  client has configured the channel
 362 * @PREP_HARDWARE: register access is allowed
 363 *		   however, no processing EVENTS
 364 * @ACTIVE_STATE: channels are fully operational
 365 * @PREPARE_TERMINATE: graceful termination of channels
 366 *		       register access is allowed
 367 * @PAUSE_STATE: channels are active, but not processing any events
 368 */
 369enum gpi_pm_state {
 370	DISABLE_STATE,
 371	CONFIG_STATE,
 372	PREPARE_HARDWARE,
 373	ACTIVE_STATE,
 374	PREPARE_TERMINATE,
 375	PAUSE_STATE,
 376	MAX_PM_STATE
 377};
 378
 379#define REG_ACCESS_VALID(_pm_state) ((_pm_state) >= PREPARE_HARDWARE)
 380
 381static const char *const gpi_pm_state_str[MAX_PM_STATE] = {
 382	[DISABLE_STATE] = "DISABLE",
 383	[CONFIG_STATE] = "CONFIG",
 384	[PREPARE_HARDWARE] = "PREPARE HARDWARE",
 385	[ACTIVE_STATE] = "ACTIVE",
 386	[PREPARE_TERMINATE] = "PREPARE TERMINATE",
 387	[PAUSE_STATE] = "PAUSE",
 388};
 389
 390#define TO_GPI_PM_STR(_state) (((_state) >= MAX_PM_STATE) ? \
 391			      "INVALID" : gpi_pm_state_str[(_state)])
 392
 393static const struct {
 394	enum gpi_cmd gpi_cmd;
 395	u32 opcode;
 396	u32 state;
 397} gpi_cmd_info[GPI_MAX_CMD] = {
 398	{
 399		GPI_CH_CMD_ALLOCATE,
 400		GPII_n_CH_CMD_ALLOCATE,
 401		CH_STATE_ALLOCATED,
 402	},
 403	{
 404		GPI_CH_CMD_START,
 405		GPII_n_CH_CMD_START,
 406		CH_STATE_STARTED,
 407	},
 408	{
 409		GPI_CH_CMD_STOP,
 410		GPII_n_CH_CMD_STOP,
 411		CH_STATE_STOPPED,
 412	},
 413	{
 414		GPI_CH_CMD_RESET,
 415		GPII_n_CH_CMD_RESET,
 416		CH_STATE_ALLOCATED,
 417	},
 418	{
 419		GPI_CH_CMD_DE_ALLOC,
 420		GPII_n_CH_CMD_DE_ALLOC,
 421		CH_STATE_NOT_ALLOCATED,
 422	},
 423	{
 424		GPI_CH_CMD_UART_SW_STALE,
 425		GPII_n_CH_CMD_UART_SW_STALE,
 426		STATE_IGNORE,
 427	},
 428	{
 429		GPI_CH_CMD_UART_RFR_READY,
 430		GPII_n_CH_CMD_UART_RFR_READY,
 431		STATE_IGNORE,
 432	},
 433	{
 434		GPI_CH_CMD_UART_RFR_NOT_READY,
 435		GPII_n_CH_CMD_UART_RFR_NOT_READY,
 436		STATE_IGNORE,
 437	},
 438	{
 439		GPI_EV_CMD_ALLOCATE,
 440		GPII_n_EV_CH_CMD_ALLOCATE,
 441		EV_STATE_ALLOCATED,
 442	},
 443	{
 444		GPI_EV_CMD_RESET,
 445		GPII_n_EV_CH_CMD_RESET,
 446		EV_STATE_ALLOCATED,
 447	},
 448	{
 449		GPI_EV_CMD_DEALLOC,
 450		GPII_n_EV_CH_CMD_DE_ALLOC,
 451		EV_STATE_NOT_ALLOCATED,
 452	},
 453};
 454
 455struct gpi_ring {
 456	void *pre_aligned;
 457	size_t alloc_size;
 458	phys_addr_t phys_addr;
 459	dma_addr_t dma_handle;
 460	void *base;
 461	void *wp;
 462	void *rp;
 463	u32 len;
 464	u32 el_size;
 465	u32 elements;
 466	bool configured;
 467};
 468
 469struct gpi_dev {
 470	struct dma_device dma_device;
 471	struct device *dev;
 472	struct resource *res;
 473	void __iomem *regs;
 474	void __iomem *ee_base; /*ee register base address*/
 475	u32 max_gpii; /* maximum # of gpii instances available per gpi block */
 476	u32 gpii_mask; /* gpii instances available for apps */
 477	u32 ev_factor; /* ev ring length factor */
 478	struct gpii *gpiis;
 479};
 480
 481struct gchan {
 482	struct virt_dma_chan vc;
 483	u32 chid;
 484	u32 seid;
 485	u32 protocol;
 486	struct gpii *gpii;
 487	enum gpi_ch_state ch_state;
 488	enum gpi_pm_state pm_state;
 489	void __iomem *ch_cntxt_base_reg;
 490	void __iomem *ch_cntxt_db_reg;
 491	void __iomem *ch_cmd_reg;
 492	u32 dir;
 493	struct gpi_ring ch_ring;
 494	void *config;
 495};
 496
 497struct gpii {
 498	u32 gpii_id;
 499	struct gchan gchan[MAX_CHANNELS_PER_GPII];
 500	struct gpi_dev *gpi_dev;
 501	int irq;
 502	void __iomem *regs; /* points to gpi top */
 503	void __iomem *ev_cntxt_base_reg;
 504	void __iomem *ev_cntxt_db_reg;
 505	void __iomem *ev_ring_rp_lsb_reg;
 506	void __iomem *ev_cmd_reg;
 507	void __iomem *ieob_clr_reg;
 508	struct mutex ctrl_lock;
 509	enum gpi_ev_state ev_state;
 510	bool configured_irq;
 511	enum gpi_pm_state pm_state;
 512	rwlock_t pm_lock;
 513	struct gpi_ring ev_ring;
 514	struct tasklet_struct ev_task; /* event processing tasklet */
 515	struct completion cmd_completion;
 516	enum gpi_cmd gpi_cmd;
 517	u32 cntxt_type_irq_msk;
 518	bool ieob_set;
 519};
 520
 521#define MAX_TRE 3
 522
 523struct gpi_desc {
 524	struct virt_dma_desc vd;
 525	size_t len;
 526	void *db; /* DB register to program */
 527	struct gchan *gchan;
 528	struct gpi_tre tre[MAX_TRE];
 529	u32 num_tre;
 530};
 531
 532static const u32 GPII_CHAN_DIR[MAX_CHANNELS_PER_GPII] = {
 533	GPI_CHTYPE_DIR_OUT, GPI_CHTYPE_DIR_IN
 534};
 535
 536static irqreturn_t gpi_handle_irq(int irq, void *data);
 537static void gpi_ring_recycle_ev_element(struct gpi_ring *ring);
 538static int gpi_ring_add_element(struct gpi_ring *ring, void **wp);
 539static void gpi_process_events(struct gpii *gpii);
 540
 541static inline struct gchan *to_gchan(struct dma_chan *dma_chan)
 542{
 543	return container_of(dma_chan, struct gchan, vc.chan);
 544}
 545
 546static inline struct gpi_desc *to_gpi_desc(struct virt_dma_desc *vd)
 547{
 548	return container_of(vd, struct gpi_desc, vd);
 549}
 550
 551static inline phys_addr_t to_physical(const struct gpi_ring *const ring,
 552				      void *addr)
 553{
 554	return ring->phys_addr + (addr - ring->base);
 555}
 556
 557static inline void *to_virtual(const struct gpi_ring *const ring, phys_addr_t addr)
 558{
 559	return ring->base + (addr - ring->phys_addr);
 560}
 561
 562static inline u32 gpi_read_reg(struct gpii *gpii, void __iomem *addr)
 563{
 564	return readl_relaxed(addr);
 565}
 566
 567static inline void gpi_write_reg(struct gpii *gpii, void __iomem *addr, u32 val)
 568{
 569	writel_relaxed(val, addr);
 570}
 571
 572static __always_inline void
 573gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val)
 574{
 575	void __iomem *addr = gpii->regs + offset;
 576	u32 tmp = gpi_read_reg(gpii, addr);
 577
 578	tmp &= ~mask;
 579	tmp |= u32_encode_bits(val, mask);
 580
 581	gpi_write_reg(gpii, addr, tmp);
 582}
 583
 584static void gpi_disable_interrupts(struct gpii *gpii)
 585{
 586	gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
 587		       GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, 0);
 588	gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),
 589		       GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK, 0);
 590	gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),
 591		       GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK, 0);
 592	gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),
 593		       GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK, 0);
 594	gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),
 595		       GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0);
 596	gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),
 597		       GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0);
 598	gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),
 599		       GPII_n_CNTXT_INTSET_BMSK, 0);
 600
 601	gpii->cntxt_type_irq_msk = 0;
 602	devm_free_irq(gpii->gpi_dev->dev, gpii->irq, gpii);
 603	gpii->configured_irq = false;
 604}
 605
 606/* configure and enable interrupts */
 607static int gpi_config_interrupts(struct gpii *gpii, enum gpii_irq_settings settings, bool mask)
 608{
 609	const u32 enable = (GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL |
 610			      GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB |
 611			      GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB |
 612			      GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL |
 613			      GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL);
 614	int ret;
 615
 616	if (!gpii->configured_irq) {
 617		ret = devm_request_irq(gpii->gpi_dev->dev, gpii->irq,
 618				       gpi_handle_irq, IRQF_TRIGGER_HIGH,
 619				       "gpi-dma", gpii);
 620		if (ret < 0) {
 621			dev_err(gpii->gpi_dev->dev, "error request irq:%d ret:%d\n",
 622				gpii->irq, ret);
 623			return ret;
 624		}
 625	}
 626
 627	if (settings == MASK_IEOB_SETTINGS) {
 628		/*
 629		 * GPII only uses one EV ring per gpii so we can globally
 630		 * enable/disable IEOB interrupt
 631		 */
 632		if (mask)
 633			gpii->cntxt_type_irq_msk |= GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;
 634		else
 635			gpii->cntxt_type_irq_msk &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB);
 636		gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
 637			       GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, gpii->cntxt_type_irq_msk);
 638	} else {
 639		gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
 640			       GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, enable);
 641		gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),
 642			       GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK,
 643			       GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK);
 644		gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),
 645			       GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK,
 646			       GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK);
 647		gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),
 648			       GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK,
 649			       GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK);
 650		gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),
 651			       GPII_n_CNTXT_GPII_IRQ_EN_BMSK,
 652			       GPII_n_CNTXT_GPII_IRQ_EN_BMSK);
 653		gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),
 654			       GPII_n_CNTXT_GPII_IRQ_EN_BMSK, GPII_n_CNTXT_GPII_IRQ_EN_BMSK);
 655		gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_LSB_OFFS(gpii->gpii_id), U32_MAX, 0);
 656		gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_MSB_OFFS(gpii->gpii_id), U32_MAX, 0);
 657		gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_0_OFFS(gpii->gpii_id), U32_MAX, 0);
 658		gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_1_OFFS(gpii->gpii_id), U32_MAX, 0);
 659		gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),
 660			       GPII_n_CNTXT_INTSET_BMSK, 1);
 661		gpi_update_reg(gpii, GPII_n_ERROR_LOG_OFFS(gpii->gpii_id), U32_MAX, 0);
 662
 663		gpii->cntxt_type_irq_msk = enable;
 664	}
 665
 666	gpii->configured_irq = true;
 667	return 0;
 668}
 669
 670/* Sends gpii event or channel command */
 671static int gpi_send_cmd(struct gpii *gpii, struct gchan *gchan,
 672			enum gpi_cmd gpi_cmd)
 673{
 674	u32 chid = MAX_CHANNELS_PER_GPII;
 675	unsigned long timeout;
 676	void __iomem *cmd_reg;
 677	u32 cmd;
 678
 679	if (gpi_cmd >= GPI_MAX_CMD)
 680		return -EINVAL;
 681	if (IS_CHAN_CMD(gpi_cmd))
 682		chid = gchan->chid;
 683
 684	dev_dbg(gpii->gpi_dev->dev,
 685		"sending cmd: %s:%u\n", TO_GPI_CMD_STR(gpi_cmd), chid);
 686
 687	/* send opcode and wait for completion */
 688	reinit_completion(&gpii->cmd_completion);
 689	gpii->gpi_cmd = gpi_cmd;
 690
 691	cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg;
 692	cmd = IS_CHAN_CMD(gpi_cmd) ? GPII_n_CH_CMD(gpi_cmd_info[gpi_cmd].opcode, chid) :
 693				     GPII_n_EV_CMD(gpi_cmd_info[gpi_cmd].opcode, 0);
 694	gpi_write_reg(gpii, cmd_reg, cmd);
 695	timeout = wait_for_completion_timeout(&gpii->cmd_completion,
 696					      msecs_to_jiffies(CMD_TIMEOUT_MS));
 697	if (!timeout) {
 698		dev_err(gpii->gpi_dev->dev, "cmd: %s completion timeout:%u\n",
 699			TO_GPI_CMD_STR(gpi_cmd), chid);
 700		return -EIO;
 701	}
 702
 703	/* confirm new ch state is correct , if the cmd is a state change cmd */
 704	if (gpi_cmd_info[gpi_cmd].state == STATE_IGNORE)
 705		return 0;
 706
 707	if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state)
 708		return 0;
 709
 710	if (!IS_CHAN_CMD(gpi_cmd) && gpii->ev_state == gpi_cmd_info[gpi_cmd].state)
 711		return 0;
 712
 713	return -EIO;
 714}
 715
 716/* program transfer ring DB register */
 717static inline void gpi_write_ch_db(struct gchan *gchan,
 718				   struct gpi_ring *ring, void *wp)
 719{
 720	struct gpii *gpii = gchan->gpii;
 721	phys_addr_t p_wp;
 722
 723	p_wp = to_physical(ring, wp);
 724	gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp);
 725}
 726
 727/* program event ring DB register */
 728static inline void gpi_write_ev_db(struct gpii *gpii,
 729				   struct gpi_ring *ring, void *wp)
 730{
 731	phys_addr_t p_wp;
 732
 733	p_wp = ring->phys_addr + (wp - ring->base);
 734	gpi_write_reg(gpii, gpii->ev_cntxt_db_reg, p_wp);
 735}
 736
 737/* process transfer completion interrupt */
 738static void gpi_process_ieob(struct gpii *gpii)
 739{
 740	gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));
 741
 742	gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 0);
 743	tasklet_hi_schedule(&gpii->ev_task);
 744}
 745
 746/* process channel control interrupt */
 747static void gpi_process_ch_ctrl_irq(struct gpii *gpii)
 748{
 749	u32 gpii_id = gpii->gpii_id;
 750	u32 offset = GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(gpii_id);
 751	u32 ch_irq = gpi_read_reg(gpii, gpii->regs + offset);
 752	struct gchan *gchan;
 753	u32 chid, state;
 754
 755	/* clear the status */
 756	offset = GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(gpii_id);
 757	gpi_write_reg(gpii, gpii->regs + offset, (u32)ch_irq);
 758
 759	for (chid = 0; chid < MAX_CHANNELS_PER_GPII; chid++) {
 760		if (!(BIT(chid) & ch_irq))
 761			continue;
 762
 763		gchan = &gpii->gchan[chid];
 764		state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg +
 765				     CNTXT_0_CONFIG);
 766		state = FIELD_GET(GPII_n_CH_k_CNTXT_0_CHSTATE, state);
 767
 768		/*
 769		 * CH_CMD_DEALLOC cmd always successful. However cmd does
 770		 * not change hardware status. So overwriting software state
 771		 * to default state.
 772		 */
 773		if (gpii->gpi_cmd == GPI_CH_CMD_DE_ALLOC)
 774			state = DEFAULT_CH_STATE;
 775		gchan->ch_state = state;
 776
 777		/*
 778		 * Triggering complete all if ch_state is not a stop in process.
 779		 * Stop in process is a transition state and we will wait for
 780		 * stop interrupt before notifying.
 781		 */
 782		if (gchan->ch_state != CH_STATE_STOP_IN_PROC)
 783			complete_all(&gpii->cmd_completion);
 784	}
 785}
 786
 787/* processing gpi general error interrupts */
 788static void gpi_process_gen_err_irq(struct gpii *gpii)
 789{
 790	u32 gpii_id = gpii->gpii_id;
 791	u32 offset = GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(gpii_id);
 792	u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);
 793
 794	/* clear the status */
 795	dev_dbg(gpii->gpi_dev->dev, "irq_stts:0x%x\n", irq_stts);
 796
 797	/* Clear the register */
 798	offset = GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(gpii_id);
 799	gpi_write_reg(gpii, gpii->regs + offset, irq_stts);
 800}
 801
 802/* processing gpi level error interrupts */
 803static void gpi_process_glob_err_irq(struct gpii *gpii)
 804{
 805	u32 gpii_id = gpii->gpii_id;
 806	u32 offset = GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(gpii_id);
 807	u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);
 808
 809	offset = GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(gpii_id);
 810	gpi_write_reg(gpii, gpii->regs + offset, irq_stts);
 811
 812	/* only error interrupt should be set */
 813	if (irq_stts & ~GPI_GLOB_IRQ_ERROR_INT_MSK) {
 814		dev_err(gpii->gpi_dev->dev, "invalid error status:0x%x\n", irq_stts);
 815		return;
 816	}
 817
 818	offset = GPII_n_ERROR_LOG_OFFS(gpii_id);
 819	gpi_write_reg(gpii, gpii->regs + offset, 0);
 820}
 821
 822/* gpii interrupt handler */
 823static irqreturn_t gpi_handle_irq(int irq, void *data)
 824{
 825	struct gpii *gpii = data;
 826	u32 gpii_id = gpii->gpii_id;
 827	u32 type, offset;
 828	unsigned long flags;
 829
 830	read_lock_irqsave(&gpii->pm_lock, flags);
 831
 832	/*
 833	 * States are out of sync to receive interrupt
 834	 * while software state is in DISABLE state, bailing out.
 835	 */
 836	if (!REG_ACCESS_VALID(gpii->pm_state)) {
 837		dev_err(gpii->gpi_dev->dev, "receive interrupt while in %s state\n",
 838			TO_GPI_PM_STR(gpii->pm_state));
 839		goto exit_irq;
 840	}
 841
 842	offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);
 843	type = gpi_read_reg(gpii, gpii->regs + offset);
 844
 845	do {
 846		/* global gpii error */
 847		if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB) {
 848			gpi_process_glob_err_irq(gpii);
 849			type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB);
 850		}
 851
 852		/* transfer complete interrupt */
 853		if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB) {
 854			gpi_process_ieob(gpii);
 855			type &= ~GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;
 856		}
 857
 858		/* event control irq */
 859		if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL) {
 860			u32 ev_state;
 861			u32 ev_ch_irq;
 862
 863			dev_dbg(gpii->gpi_dev->dev,
 864				"processing EV CTRL interrupt\n");
 865			offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(gpii_id);
 866			ev_ch_irq = gpi_read_reg(gpii, gpii->regs + offset);
 867
 868			offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS
 869				(gpii_id);
 870			gpi_write_reg(gpii, gpii->regs + offset, ev_ch_irq);
 871			ev_state = gpi_read_reg(gpii, gpii->ev_cntxt_base_reg +
 872						CNTXT_0_CONFIG);
 873			ev_state = FIELD_GET(GPII_n_EV_k_CNTXT_0_CHSTATE, ev_state);
 874
 875			/*
 876			 * CMD EV_CMD_DEALLOC is always successful. However
 877			 * cmd does not change hardware status. So overwriting
 878			 * software state to default state.
 879			 */
 880			if (gpii->gpi_cmd == GPI_EV_CMD_DEALLOC)
 881				ev_state = DEFAULT_EV_CH_STATE;
 882
 883			gpii->ev_state = ev_state;
 884			dev_dbg(gpii->gpi_dev->dev, "setting EV state to %s\n",
 885				TO_GPI_EV_STATE_STR(gpii->ev_state));
 886			complete_all(&gpii->cmd_completion);
 887			type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL);
 888		}
 889
 890		/* channel control irq */
 891		if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL) {
 892			dev_dbg(gpii->gpi_dev->dev, "process CH CTRL interrupts\n");
 893			gpi_process_ch_ctrl_irq(gpii);
 894			type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL);
 895		}
 896
 897		if (type) {
 898			dev_err(gpii->gpi_dev->dev, "Unhandled interrupt status:0x%x\n", type);
 899			gpi_process_gen_err_irq(gpii);
 900			goto exit_irq;
 901		}
 902
 903		offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);
 904		type = gpi_read_reg(gpii, gpii->regs + offset);
 905	} while (type);
 906
 907exit_irq:
 908	read_unlock_irqrestore(&gpii->pm_lock, flags);
 909
 910	return IRQ_HANDLED;
 911}
 912
 913/* process DMA Immediate completion data events */
 914static void gpi_process_imed_data_event(struct gchan *gchan,
 915					struct immediate_data_event *imed_event)
 916{
 917	struct gpii *gpii = gchan->gpii;
 918	struct gpi_ring *ch_ring = &gchan->ch_ring;
 919	void *tre = ch_ring->base + (ch_ring->el_size * imed_event->tre_index);
 920	struct dmaengine_result result;
 921	struct gpi_desc *gpi_desc;
 922	struct virt_dma_desc *vd;
 923	unsigned long flags;
 924	u32 chid;
 925
 926	/*
 927	 * If channel not active don't process event
 928	 */
 929	if (gchan->pm_state != ACTIVE_STATE) {
 930		dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",
 931			TO_GPI_PM_STR(gchan->pm_state));
 932		return;
 933	}
 934
 935	spin_lock_irqsave(&gchan->vc.lock, flags);
 936	vd = vchan_next_desc(&gchan->vc);
 937	if (!vd) {
 938		struct gpi_ere *gpi_ere;
 939		struct gpi_tre *gpi_tre;
 940
 941		spin_unlock_irqrestore(&gchan->vc.lock, flags);
 942		dev_dbg(gpii->gpi_dev->dev, "event without a pending descriptor!\n");
 943		gpi_ere = (struct gpi_ere *)imed_event;
 944		dev_dbg(gpii->gpi_dev->dev,
 945			"Event: %08x %08x %08x %08x\n",
 946			gpi_ere->dword[0], gpi_ere->dword[1],
 947			gpi_ere->dword[2], gpi_ere->dword[3]);
 948		gpi_tre = tre;
 949		dev_dbg(gpii->gpi_dev->dev,
 950			"Pending TRE: %08x %08x %08x %08x\n",
 951			gpi_tre->dword[0], gpi_tre->dword[1],
 952			gpi_tre->dword[2], gpi_tre->dword[3]);
 953		return;
 954	}
 955	gpi_desc = to_gpi_desc(vd);
 956	spin_unlock_irqrestore(&gchan->vc.lock, flags);
 957
 958	/*
 959	 * RP pointed by Event is to last TRE processed,
 960	 * we need to update ring rp to tre + 1
 961	 */
 962	tre += ch_ring->el_size;
 963	if (tre >= (ch_ring->base + ch_ring->len))
 964		tre = ch_ring->base;
 965	ch_ring->rp = tre;
 966
 967	/* make sure rp updates are immediately visible to all cores */
 968	smp_wmb();
 969
 970	chid = imed_event->chid;
 971	if (imed_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {
 972		if (chid == GPI_RX_CHAN)
 973			goto gpi_free_desc;
 974		else
 975			return;
 976	}
 977
 978	if (imed_event->code == MSM_GPI_TCE_UNEXP_ERR)
 979		result.result = DMA_TRANS_ABORTED;
 980	else
 981		result.result = DMA_TRANS_NOERROR;
 982	result.residue = gpi_desc->len - imed_event->length;
 983
 984	dma_cookie_complete(&vd->tx);
 985	dmaengine_desc_get_callback_invoke(&vd->tx, &result);
 986
 987gpi_free_desc:
 988	spin_lock_irqsave(&gchan->vc.lock, flags);
 989	list_del(&vd->node);
 990	spin_unlock_irqrestore(&gchan->vc.lock, flags);
 991	kfree(gpi_desc);
 992	gpi_desc = NULL;
 993}
 994
 995/* processing transfer completion events */
 996static void gpi_process_xfer_compl_event(struct gchan *gchan,
 997					 struct xfer_compl_event *compl_event)
 998{
 999	struct gpii *gpii = gchan->gpii;
1000	struct gpi_ring *ch_ring = &gchan->ch_ring;
1001	void *ev_rp = to_virtual(ch_ring, compl_event->ptr);
1002	struct virt_dma_desc *vd;
1003	struct gpi_desc *gpi_desc;
1004	struct dmaengine_result result;
1005	unsigned long flags;
1006	u32 chid;
1007
1008	/* only process events on active channel */
1009	if (unlikely(gchan->pm_state != ACTIVE_STATE)) {
1010		dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",
1011			TO_GPI_PM_STR(gchan->pm_state));
1012		return;
1013	}
1014
1015	spin_lock_irqsave(&gchan->vc.lock, flags);
1016	vd = vchan_next_desc(&gchan->vc);
1017	if (!vd) {
1018		struct gpi_ere *gpi_ere;
1019
1020		spin_unlock_irqrestore(&gchan->vc.lock, flags);
1021		dev_err(gpii->gpi_dev->dev, "Event without a pending descriptor!\n");
1022		gpi_ere = (struct gpi_ere *)compl_event;
1023		dev_err(gpii->gpi_dev->dev,
1024			"Event: %08x %08x %08x %08x\n",
1025			gpi_ere->dword[0], gpi_ere->dword[1],
1026			gpi_ere->dword[2], gpi_ere->dword[3]);
1027		return;
1028	}
1029
1030	gpi_desc = to_gpi_desc(vd);
1031	spin_unlock_irqrestore(&gchan->vc.lock, flags);
1032
1033	/*
1034	 * RP pointed by Event is to last TRE processed,
1035	 * we need to update ring rp to ev_rp + 1
1036	 */
1037	ev_rp += ch_ring->el_size;
1038	if (ev_rp >= (ch_ring->base + ch_ring->len))
1039		ev_rp = ch_ring->base;
1040	ch_ring->rp = ev_rp;
1041
1042	/* update must be visible to other cores */
1043	smp_wmb();
1044
1045	chid = compl_event->chid;
1046	if (compl_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {
1047		if (chid == GPI_RX_CHAN)
1048			goto gpi_free_desc;
1049		else
1050			return;
1051	}
1052
1053	if (compl_event->code == MSM_GPI_TCE_UNEXP_ERR) {
1054		dev_err(gpii->gpi_dev->dev, "Error in Transaction\n");
1055		result.result = DMA_TRANS_ABORTED;
1056	} else {
1057		dev_dbg(gpii->gpi_dev->dev, "Transaction Success\n");
1058		result.result = DMA_TRANS_NOERROR;
1059	}
1060	result.residue = gpi_desc->len - compl_event->length;
1061	dev_dbg(gpii->gpi_dev->dev, "Residue %d\n", result.residue);
1062
1063	dma_cookie_complete(&vd->tx);
1064	dmaengine_desc_get_callback_invoke(&vd->tx, &result);
1065
1066gpi_free_desc:
1067	spin_lock_irqsave(&gchan->vc.lock, flags);
1068	list_del(&vd->node);
1069	spin_unlock_irqrestore(&gchan->vc.lock, flags);
1070	kfree(gpi_desc);
1071	gpi_desc = NULL;
1072}
1073
1074/* process all events */
1075static void gpi_process_events(struct gpii *gpii)
1076{
1077	struct gpi_ring *ev_ring = &gpii->ev_ring;
1078	phys_addr_t cntxt_rp;
1079	void *rp;
1080	union gpi_event *gpi_event;
1081	struct gchan *gchan;
1082	u32 chid, type;
1083
1084	cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1085	rp = to_virtual(ev_ring, cntxt_rp);
1086
1087	do {
1088		while (rp != ev_ring->rp) {
1089			gpi_event = ev_ring->rp;
1090			chid = gpi_event->xfer_compl_event.chid;
1091			type = gpi_event->xfer_compl_event.type;
1092
1093			dev_dbg(gpii->gpi_dev->dev,
1094				"Event: CHID:%u, type:%x %08x %08x %08x %08x\n",
1095				chid, type, gpi_event->gpi_ere.dword[0],
1096				gpi_event->gpi_ere.dword[1], gpi_event->gpi_ere.dword[2],
1097				gpi_event->gpi_ere.dword[3]);
1098
1099			switch (type) {
1100			case XFER_COMPLETE_EV_TYPE:
1101				gchan = &gpii->gchan[chid];
1102				gpi_process_xfer_compl_event(gchan,
1103							     &gpi_event->xfer_compl_event);
1104				break;
1105			case STALE_EV_TYPE:
1106				dev_dbg(gpii->gpi_dev->dev, "stale event, not processing\n");
1107				break;
1108			case IMMEDIATE_DATA_EV_TYPE:
1109				gchan = &gpii->gchan[chid];
1110				gpi_process_imed_data_event(gchan,
1111							    &gpi_event->immediate_data_event);
1112				break;
1113			case QUP_NOTIF_EV_TYPE:
1114				dev_dbg(gpii->gpi_dev->dev, "QUP_NOTIF_EV_TYPE\n");
1115				break;
1116			default:
1117				dev_dbg(gpii->gpi_dev->dev,
1118					"not supported event type:0x%x\n", type);
1119			}
1120			gpi_ring_recycle_ev_element(ev_ring);
1121		}
1122		gpi_write_ev_db(gpii, ev_ring, ev_ring->wp);
1123
1124		/* clear pending IEOB events */
1125		gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));
1126
1127		cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1128		rp = to_virtual(ev_ring, cntxt_rp);
1129
1130	} while (rp != ev_ring->rp);
1131}
1132
1133/* processing events using tasklet */
1134static void gpi_ev_tasklet(unsigned long data)
1135{
1136	struct gpii *gpii = (struct gpii *)data;
1137
1138	read_lock(&gpii->pm_lock);
1139	if (!REG_ACCESS_VALID(gpii->pm_state)) {
1140		read_unlock(&gpii->pm_lock);
1141		dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n",
1142			TO_GPI_PM_STR(gpii->pm_state));
1143		return;
1144	}
1145
1146	/* process the events */
1147	gpi_process_events(gpii);
1148
1149	/* enable IEOB, switching back to interrupts */
1150	gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1);
1151	read_unlock(&gpii->pm_lock);
1152}
1153
1154/* marks all pending events for the channel as stale */
1155static void gpi_mark_stale_events(struct gchan *gchan)
1156{
1157	struct gpii *gpii = gchan->gpii;
1158	struct gpi_ring *ev_ring = &gpii->ev_ring;
1159	u32 cntxt_rp, local_rp;
1160	void *ev_rp;
1161
1162	cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1163
1164	ev_rp = ev_ring->rp;
1165	local_rp = (u32)to_physical(ev_ring, ev_rp);
1166	while (local_rp != cntxt_rp) {
1167		union gpi_event *gpi_event = ev_rp;
1168		u32 chid = gpi_event->xfer_compl_event.chid;
1169
1170		if (chid == gchan->chid)
1171			gpi_event->xfer_compl_event.type = STALE_EV_TYPE;
1172		ev_rp += ev_ring->el_size;
1173		if (ev_rp >= (ev_ring->base + ev_ring->len))
1174			ev_rp = ev_ring->base;
1175		cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1176		local_rp = (u32)to_physical(ev_ring, ev_rp);
1177	}
1178}
1179
1180/* reset sw state and issue channel reset or de-alloc */
1181static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd)
1182{
1183	struct gpii *gpii = gchan->gpii;
1184	struct gpi_ring *ch_ring = &gchan->ch_ring;
1185	LIST_HEAD(list);
1186	int ret;
1187
1188	ret = gpi_send_cmd(gpii, gchan, gpi_cmd);
1189	if (ret) {
1190		dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1191			TO_GPI_CMD_STR(gpi_cmd), ret);
1192		return ret;
1193	}
1194
1195	/* initialize the local ring ptrs */
1196	ch_ring->rp = ch_ring->base;
1197	ch_ring->wp = ch_ring->base;
1198
1199	/* visible to other cores */
1200	smp_wmb();
1201
1202	/* check event ring for any stale events */
1203	write_lock_irq(&gpii->pm_lock);
1204	gpi_mark_stale_events(gchan);
1205
1206	/* remove all async descriptors */
1207	spin_lock(&gchan->vc.lock);
1208	vchan_get_all_descriptors(&gchan->vc, &list);
1209	spin_unlock(&gchan->vc.lock);
1210	write_unlock_irq(&gpii->pm_lock);
1211	vchan_dma_desc_free_list(&gchan->vc, &list);
1212
1213	return 0;
1214}
1215
1216static int gpi_start_chan(struct gchan *gchan)
1217{
1218	struct gpii *gpii = gchan->gpii;
1219	int ret;
1220
1221	ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_START);
1222	if (ret) {
1223		dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1224			TO_GPI_CMD_STR(GPI_CH_CMD_START), ret);
1225		return ret;
1226	}
1227
1228	/* gpii CH is active now */
1229	write_lock_irq(&gpii->pm_lock);
1230	gchan->pm_state = ACTIVE_STATE;
1231	write_unlock_irq(&gpii->pm_lock);
1232
1233	return 0;
1234}
1235
1236static int gpi_stop_chan(struct gchan *gchan)
1237{
1238	struct gpii *gpii = gchan->gpii;
1239	int ret;
1240
1241	ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_STOP);
1242	if (ret) {
1243		dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1244			TO_GPI_CMD_STR(GPI_CH_CMD_STOP), ret);
1245		return ret;
1246	}
1247
1248	return 0;
1249}
1250
1251/* allocate and configure the transfer channel */
1252static int gpi_alloc_chan(struct gchan *chan, bool send_alloc_cmd)
1253{
1254	struct gpii *gpii = chan->gpii;
1255	struct gpi_ring *ring = &chan->ch_ring;
1256	int ret;
1257	u32 id = gpii->gpii_id;
1258	u32 chid = chan->chid;
1259	u32 pair_chid = !chid;
1260
1261	if (send_alloc_cmd) {
1262		ret = gpi_send_cmd(gpii, chan, GPI_CH_CMD_ALLOCATE);
1263		if (ret) {
1264			dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1265				TO_GPI_CMD_STR(GPI_CH_CMD_ALLOCATE), ret);
1266			return ret;
1267		}
1268	}
1269
1270	gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_0_CONFIG,
1271		      GPII_n_CH_k_CNTXT_0(ring->el_size, 0, chan->dir, GPI_CHTYPE_PROTO_GPI));
1272	gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_1_R_LENGTH, ring->len);
1273	gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_2_RING_BASE_LSB, ring->phys_addr);
1274	gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_3_RING_BASE_MSB,
1275		      upper_32_bits(ring->phys_addr));
1276	gpi_write_reg(gpii, chan->ch_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,
1277		      upper_32_bits(ring->phys_addr));
1278	gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_0_OFFS(id, chid),
1279		      GPII_n_CH_k_SCRATCH_0(pair_chid, chan->protocol, chan->seid));
1280	gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_1_OFFS(id, chid), 0);
1281	gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_2_OFFS(id, chid), 0);
1282	gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_3_OFFS(id, chid), 0);
1283	gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_QOS_OFFS(id, chid), 1);
1284
1285	/* flush all the writes */
1286	wmb();
1287	return 0;
1288}
1289
1290/* allocate and configure event ring */
1291static int gpi_alloc_ev_chan(struct gpii *gpii)
1292{
1293	struct gpi_ring *ring = &gpii->ev_ring;
1294	void __iomem *base = gpii->ev_cntxt_base_reg;
1295	int ret;
1296
1297	ret = gpi_send_cmd(gpii, NULL, GPI_EV_CMD_ALLOCATE);
1298	if (ret) {
1299		dev_err(gpii->gpi_dev->dev, "error with cmd:%s ret:%d\n",
1300			TO_GPI_CMD_STR(GPI_EV_CMD_ALLOCATE), ret);
1301		return ret;
1302	}
1303
1304	/* program event context */
1305	gpi_write_reg(gpii, base + CNTXT_0_CONFIG,
1306		      GPII_n_EV_k_CNTXT_0(ring->el_size, GPI_INTTYPE_IRQ, GPI_CHTYPE_GPI_EV));
1307	gpi_write_reg(gpii, base + CNTXT_1_R_LENGTH, ring->len);
1308	gpi_write_reg(gpii, base + CNTXT_2_RING_BASE_LSB, lower_32_bits(ring->phys_addr));
1309	gpi_write_reg(gpii, base + CNTXT_3_RING_BASE_MSB, upper_32_bits(ring->phys_addr));
1310	gpi_write_reg(gpii, gpii->ev_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,
1311		      upper_32_bits(ring->phys_addr));
1312	gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0);
1313	gpi_write_reg(gpii, base + CNTXT_10_RING_MSI_LSB, 0);
1314	gpi_write_reg(gpii, base + CNTXT_11_RING_MSI_MSB, 0);
1315	gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0);
1316	gpi_write_reg(gpii, base + CNTXT_12_RING_RP_UPDATE_LSB, 0);
1317	gpi_write_reg(gpii, base + CNTXT_13_RING_RP_UPDATE_MSB, 0);
1318
1319	/* add events to ring */
1320	ring->wp = (ring->base + ring->len - ring->el_size);
1321
1322	/* flush all the writes */
1323	wmb();
1324
1325	/* gpii is active now */
1326	write_lock_irq(&gpii->pm_lock);
1327	gpii->pm_state = ACTIVE_STATE;
1328	write_unlock_irq(&gpii->pm_lock);
1329	gpi_write_ev_db(gpii, ring, ring->wp);
1330
1331	return 0;
1332}
1333
1334/* calculate # of ERE/TRE available to queue */
1335static int gpi_ring_num_elements_avail(const struct gpi_ring * const ring)
1336{
1337	int elements = 0;
1338
1339	if (ring->wp < ring->rp) {
1340		elements = ((ring->rp - ring->wp) / ring->el_size) - 1;
1341	} else {
1342		elements = (ring->rp - ring->base) / ring->el_size;
1343		elements += ((ring->base + ring->len - ring->wp) / ring->el_size) - 1;
1344	}
1345
1346	return elements;
1347}
1348
1349static int gpi_ring_add_element(struct gpi_ring *ring, void **wp)
1350{
1351	if (gpi_ring_num_elements_avail(ring) <= 0)
1352		return -ENOMEM;
1353
1354	*wp = ring->wp;
1355	ring->wp += ring->el_size;
1356	if (ring->wp  >= (ring->base + ring->len))
1357		ring->wp = ring->base;
1358
1359	/* visible to other cores */
1360	smp_wmb();
1361
1362	return 0;
1363}
1364
1365static void gpi_ring_recycle_ev_element(struct gpi_ring *ring)
1366{
1367	/* Update the WP */
1368	ring->wp += ring->el_size;
1369	if (ring->wp  >= (ring->base + ring->len))
1370		ring->wp = ring->base;
1371
1372	/* Update the RP */
1373	ring->rp += ring->el_size;
1374	if (ring->rp  >= (ring->base + ring->len))
1375		ring->rp = ring->base;
1376
1377	/* visible to other cores */
1378	smp_wmb();
1379}
1380
1381static void gpi_free_ring(struct gpi_ring *ring,
1382			  struct gpii *gpii)
1383{
1384	dma_free_coherent(gpii->gpi_dev->dev, ring->alloc_size,
1385			  ring->pre_aligned, ring->dma_handle);
1386	memset(ring, 0, sizeof(*ring));
1387}
1388
1389/* allocate memory for transfer and event rings */
1390static int gpi_alloc_ring(struct gpi_ring *ring, u32 elements,
1391			  u32 el_size, struct gpii *gpii)
1392{
1393	u64 len = elements * el_size;
1394	int bit;
1395
1396	/* ring len must be power of 2 */
1397	bit = find_last_bit((unsigned long *)&len, 32);
1398	if (((1 << bit) - 1) & len)
1399		bit++;
1400	len = 1 << bit;
1401	ring->alloc_size = (len + (len - 1));
1402	dev_dbg(gpii->gpi_dev->dev,
1403		"#el:%u el_size:%u len:%u actual_len:%llu alloc_size:%zu\n",
1404		  elements, el_size, (elements * el_size), len,
1405		  ring->alloc_size);
1406
1407	ring->pre_aligned = dma_alloc_coherent(gpii->gpi_dev->dev,
1408					       ring->alloc_size,
1409					       &ring->dma_handle, GFP_KERNEL);
1410	if (!ring->pre_aligned) {
1411		dev_err(gpii->gpi_dev->dev, "could not alloc size:%zu mem for ring\n",
1412			ring->alloc_size);
1413		return -ENOMEM;
1414	}
1415
1416	/* align the physical mem */
1417	ring->phys_addr = (ring->dma_handle + (len - 1)) & ~(len - 1);
1418	ring->base = ring->pre_aligned + (ring->phys_addr - ring->dma_handle);
1419	ring->rp = ring->base;
1420	ring->wp = ring->base;
1421	ring->len = len;
1422	ring->el_size = el_size;
1423	ring->elements = ring->len / ring->el_size;
1424	memset(ring->base, 0, ring->len);
1425	ring->configured = true;
1426
1427	/* update to other cores */
1428	smp_wmb();
1429
1430	dev_dbg(gpii->gpi_dev->dev,
1431		"phy_pre:%pad phy_alig:%pa len:%u el_size:%u elements:%u\n",
1432		&ring->dma_handle, &ring->phys_addr, ring->len,
1433		ring->el_size, ring->elements);
1434
1435	return 0;
1436}
1437
1438/* copy tre into transfer ring */
1439static void gpi_queue_xfer(struct gpii *gpii, struct gchan *gchan,
1440			   struct gpi_tre *gpi_tre, void **wp)
1441{
1442	struct gpi_tre *ch_tre;
1443	int ret;
1444
1445	/* get next tre location we can copy */
1446	ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre);
1447	if (unlikely(ret)) {
1448		dev_err(gpii->gpi_dev->dev, "Error adding ring element to xfer ring\n");
1449		return;
1450	}
1451
1452	/* copy the tre info */
1453	memcpy(ch_tre, gpi_tre, sizeof(*ch_tre));
1454	*wp = ch_tre;
1455}
1456
1457/* reset and restart transfer channel */
1458static int gpi_terminate_all(struct dma_chan *chan)
1459{
1460	struct gchan *gchan = to_gchan(chan);
1461	struct gpii *gpii = gchan->gpii;
1462	int schid, echid, i;
1463	int ret = 0;
1464
1465	mutex_lock(&gpii->ctrl_lock);
1466
1467	/*
1468	 * treat both channels as a group if its protocol is not UART
1469	 * STOP, RESET, or START needs to be in lockstep
1470	 */
1471	schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0;
1472	echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII;
1473
1474	/* stop the channel */
1475	for (i = schid; i < echid; i++) {
1476		gchan = &gpii->gchan[i];
1477
1478		/* disable ch state so no more TRE processing */
1479		write_lock_irq(&gpii->pm_lock);
1480		gchan->pm_state = PREPARE_TERMINATE;
1481		write_unlock_irq(&gpii->pm_lock);
1482
1483		/* send command to Stop the channel */
1484		ret = gpi_stop_chan(gchan);
1485	}
1486
1487	/* reset the channels (clears any pending tre) */
1488	for (i = schid; i < echid; i++) {
1489		gchan = &gpii->gchan[i];
1490
1491		ret = gpi_reset_chan(gchan, GPI_CH_CMD_RESET);
1492		if (ret) {
1493			dev_err(gpii->gpi_dev->dev, "Error resetting channel ret:%d\n", ret);
1494			goto terminate_exit;
1495		}
1496
1497		/* reprogram channel CNTXT */
1498		ret = gpi_alloc_chan(gchan, false);
1499		if (ret) {
1500			dev_err(gpii->gpi_dev->dev, "Error alloc_channel ret:%d\n", ret);
1501			goto terminate_exit;
1502		}
1503	}
1504
1505	/* restart the channels */
1506	for (i = schid; i < echid; i++) {
1507		gchan = &gpii->gchan[i];
1508
1509		ret = gpi_start_chan(gchan);
1510		if (ret) {
1511			dev_err(gpii->gpi_dev->dev, "Error Starting Channel ret:%d\n", ret);
1512			goto terminate_exit;
1513		}
1514	}
1515
1516terminate_exit:
1517	mutex_unlock(&gpii->ctrl_lock);
1518	return ret;
1519}
1520
1521/* pause dma transfer for all channels */
1522static int gpi_pause(struct dma_chan *chan)
1523{
1524	struct gchan *gchan = to_gchan(chan);
1525	struct gpii *gpii = gchan->gpii;
1526	int i, ret;
1527
1528	mutex_lock(&gpii->ctrl_lock);
1529
1530	/*
1531	 * pause/resume are per gpii not per channel, so
1532	 * client needs to call pause only once
1533	 */
1534	if (gpii->pm_state == PAUSE_STATE) {
1535		dev_dbg(gpii->gpi_dev->dev, "channel is already paused\n");
1536		mutex_unlock(&gpii->ctrl_lock);
1537		return 0;
1538	}
1539
1540	/* send stop command to stop the channels */
1541	for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1542		ret = gpi_stop_chan(&gpii->gchan[i]);
1543		if (ret) {
1544			mutex_unlock(&gpii->ctrl_lock);
1545			return ret;
1546		}
1547	}
1548
1549	disable_irq(gpii->irq);
1550
1551	/* Wait for threads to complete out */
1552	tasklet_kill(&gpii->ev_task);
1553
1554	write_lock_irq(&gpii->pm_lock);
1555	gpii->pm_state = PAUSE_STATE;
1556	write_unlock_irq(&gpii->pm_lock);
1557	mutex_unlock(&gpii->ctrl_lock);
1558
1559	return 0;
1560}
1561
1562/* resume dma transfer */
1563static int gpi_resume(struct dma_chan *chan)
1564{
1565	struct gchan *gchan = to_gchan(chan);
1566	struct gpii *gpii = gchan->gpii;
1567	int i, ret;
1568
1569	mutex_lock(&gpii->ctrl_lock);
1570	if (gpii->pm_state == ACTIVE_STATE) {
1571		dev_dbg(gpii->gpi_dev->dev, "channel is already active\n");
1572		mutex_unlock(&gpii->ctrl_lock);
1573		return 0;
1574	}
1575
1576	enable_irq(gpii->irq);
1577
1578	/* send start command to start the channels */
1579	for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1580		ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START);
1581		if (ret) {
1582			dev_err(gpii->gpi_dev->dev, "Error starting chan, ret:%d\n", ret);
1583			mutex_unlock(&gpii->ctrl_lock);
1584			return ret;
1585		}
1586	}
1587
1588	write_lock_irq(&gpii->pm_lock);
1589	gpii->pm_state = ACTIVE_STATE;
1590	write_unlock_irq(&gpii->pm_lock);
1591	mutex_unlock(&gpii->ctrl_lock);
1592
1593	return 0;
1594}
1595
1596static void gpi_desc_free(struct virt_dma_desc *vd)
1597{
1598	struct gpi_desc *gpi_desc = to_gpi_desc(vd);
1599
1600	kfree(gpi_desc);
1601	gpi_desc = NULL;
1602}
1603
1604static int
1605gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config)
1606{
1607	struct gchan *gchan = to_gchan(chan);
1608	void *new_config;
1609
1610	if (!config->peripheral_config)
1611		return -EINVAL;
1612
1613	new_config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT);
1614	if (!new_config)
1615		return -ENOMEM;
1616
1617	gchan->config = new_config;
1618	memcpy(gchan->config, config->peripheral_config, config->peripheral_size);
1619
1620	return 0;
1621}
1622
1623static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc,
1624			      struct scatterlist *sgl, enum dma_transfer_direction direction,
1625			      unsigned long flags)
1626{
1627	struct gpi_i2c_config *i2c = chan->config;
1628	struct device *dev = chan->gpii->gpi_dev->dev;
1629	unsigned int tre_idx = 0;
1630	dma_addr_t address;
1631	struct gpi_tre *tre;
1632	unsigned int i;
1633
1634	/* first create config tre if applicable */
1635	if (i2c->set_config) {
1636		tre = &desc->tre[tre_idx];
1637		tre_idx++;
1638
1639		tre->dword[0] = u32_encode_bits(i2c->low_count, TRE_I2C_C0_TLOW);
1640		tre->dword[0] |= u32_encode_bits(i2c->high_count, TRE_I2C_C0_THIGH);
1641		tre->dword[0] |= u32_encode_bits(i2c->cycle_count, TRE_I2C_C0_TCYL);
1642		tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_TX_PACK);
1643		tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_RX_PACK);
1644
1645		tre->dword[1] = 0;
1646
1647		tre->dword[2] = u32_encode_bits(i2c->clk_div, TRE_C0_CLK_DIV);
1648
1649		tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);
1650		tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1651	}
1652
1653	/* create the GO tre for Tx */
1654	if (i2c->op == I2C_WRITE) {
1655		tre = &desc->tre[tre_idx];
1656		tre_idx++;
1657
1658		if (i2c->multi_msg)
1659			tre->dword[0] = u32_encode_bits(I2C_READ, TRE_I2C_GO_CMD);
1660		else
1661			tre->dword[0] = u32_encode_bits(i2c->op, TRE_I2C_GO_CMD);
1662
1663		tre->dword[0] |= u32_encode_bits(i2c->addr, TRE_I2C_GO_ADDR);
1664		tre->dword[0] |= u32_encode_bits(i2c->stretch, TRE_I2C_GO_STRETCH);
1665
1666		tre->dword[1] = 0;
1667		tre->dword[2] = u32_encode_bits(i2c->rx_len, TRE_RX_LEN);
1668
1669		tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1670
1671		if (i2c->multi_msg)
1672			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1673		else
1674			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1675	}
1676
1677	if (i2c->op == I2C_READ || i2c->multi_msg == false) {
1678		/* create the DMA TRE */
1679		tre = &desc->tre[tre_idx];
1680		tre_idx++;
1681
1682		address = sg_dma_address(sgl);
1683		tre->dword[0] = lower_32_bits(address);
1684		tre->dword[1] = upper_32_bits(address);
1685
1686		tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN);
1687
1688		tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);
1689		tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT);
1690
1691		if (!(flags & DMA_PREP_INTERRUPT))
1692			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_BEI);
1693	}
1694
1695	for (i = 0; i < tre_idx; i++)
1696		dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],
1697			desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);
1698
1699	return tre_idx;
1700}
1701
1702static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc,
1703			      struct scatterlist *sgl, enum dma_transfer_direction direction)
1704{
1705	struct gpi_spi_config *spi = chan->config;
1706	struct device *dev = chan->gpii->gpi_dev->dev;
1707	unsigned int tre_idx = 0;
1708	dma_addr_t address;
1709	struct gpi_tre *tre;
1710	unsigned int i;
1711	int len;
1712
1713	/* first create config tre if applicable */
1714	if (direction == DMA_MEM_TO_DEV && spi->set_config) {
1715		tre = &desc->tre[tre_idx];
1716		tre_idx++;
1717
1718		tre->dword[0] = u32_encode_bits(spi->word_len, TRE_SPI_C0_WORD_SZ);
1719		tre->dword[0] |= u32_encode_bits(spi->loopback_en, TRE_SPI_C0_LOOPBACK);
1720		tre->dword[0] |= u32_encode_bits(spi->clock_pol_high, TRE_SPI_C0_CPOL);
1721		tre->dword[0] |= u32_encode_bits(spi->data_pol_high, TRE_SPI_C0_CPHA);
1722		tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_TX_PACK);
1723		tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_RX_PACK);
1724
1725		tre->dword[1] = 0;
1726
1727		tre->dword[2] = u32_encode_bits(spi->clk_div, TRE_C0_CLK_DIV);
1728		tre->dword[2] |= u32_encode_bits(spi->clk_src, TRE_C0_CLK_SRC);
1729
1730		tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);
1731		tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1732	}
1733
1734	/* create the GO tre for Tx */
1735	if (direction == DMA_MEM_TO_DEV) {
1736		tre = &desc->tre[tre_idx];
1737		tre_idx++;
1738
1739		tre->dword[0] = u32_encode_bits(spi->fragmentation, TRE_SPI_GO_FRAG);
1740		tre->dword[0] |= u32_encode_bits(spi->cs, TRE_SPI_GO_CS);
1741		tre->dword[0] |= u32_encode_bits(spi->cmd, TRE_SPI_GO_CMD);
1742
1743		tre->dword[1] = 0;
1744
1745		tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN);
1746
1747		tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1748		if (spi->cmd == SPI_RX) {
1749			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB);
1750			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1751		} else if (spi->cmd == SPI_TX) {
1752			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1753		} else { /* SPI_DUPLEX */
1754			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1755			tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1756		}
1757	}
1758
1759	/* create the dma tre */
1760	tre = &desc->tre[tre_idx];
1761	tre_idx++;
1762
1763	address = sg_dma_address(sgl);
1764	len = sg_dma_len(sgl);
1765
1766	/* Support Immediate dma for write transfers for data length up to 8 bytes */
1767	if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) {
1768		/*
1769		 * For Immediate dma, data length may not always be length of 8 bytes,
1770		 * it can be length less than 8, hence initialize both dword's with 0
1771		 */
1772		tre->dword[0] = 0;
1773		tre->dword[1] = 0;
1774		memcpy(&tre->dword[0], sg_virt(sgl), len);
1775
1776		tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN);
1777		tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE);
1778	} else {
1779		tre->dword[0] = lower_32_bits(address);
1780		tre->dword[1] = upper_32_bits(address);
1781
1782		tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN);
1783		tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);
1784	}
1785
1786	tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV,
1787					 TRE_FLAGS_IEOT);
1788
1789	for (i = 0; i < tre_idx; i++)
1790		dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],
1791			desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);
1792
1793	return tre_idx;
1794}
1795
1796/* copy tre into transfer ring */
1797static struct dma_async_tx_descriptor *
1798gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1799		  unsigned int sg_len, enum dma_transfer_direction direction,
1800		  unsigned long flags, void *context)
1801{
1802	struct gchan *gchan = to_gchan(chan);
1803	struct gpii *gpii = gchan->gpii;
1804	struct device *dev = gpii->gpi_dev->dev;
1805	struct gpi_ring *ch_ring = &gchan->ch_ring;
1806	struct gpi_desc *gpi_desc;
1807	u32 nr, nr_tre = 0;
1808	u8 set_config;
1809	int i;
1810
1811	gpii->ieob_set = false;
1812	if (!is_slave_direction(direction)) {
1813		dev_err(gpii->gpi_dev->dev, "invalid dma direction: %d\n", direction);
1814		return NULL;
1815	}
1816
1817	if (sg_len > 1) {
1818		dev_err(dev, "Multi sg sent, we support only one atm: %d\n", sg_len);
1819		return NULL;
1820	}
1821
1822	nr_tre = 3;
1823	set_config = *(u32 *)gchan->config;
1824	if (!set_config)
1825		nr_tre = 2;
1826	if (direction == DMA_DEV_TO_MEM) /* rx */
1827		nr_tre = 1;
1828
1829	/* calculate # of elements required & available */
1830	nr = gpi_ring_num_elements_avail(ch_ring);
1831	if (nr < nr_tre) {
1832		dev_err(dev, "not enough space in ring, avail:%u required:%u\n", nr, nr_tre);
1833		return NULL;
1834	}
1835
1836	if (!(flags & DMA_PREP_INTERRUPT) && (nr - nr_tre < 2))
1837		return NULL;
1838
1839	gpi_desc = kzalloc(sizeof(*gpi_desc), GFP_NOWAIT);
1840	if (!gpi_desc)
1841		return NULL;
1842
1843	/* create TREs for xfer */
1844	if (gchan->protocol == QCOM_GPI_SPI) {
1845		i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction);
1846	} else if (gchan->protocol == QCOM_GPI_I2C) {
1847		i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction, flags);
1848	} else {
1849		dev_err(dev, "invalid peripheral: %d\n", gchan->protocol);
1850		kfree(gpi_desc);
1851		return NULL;
1852	}
1853
1854	/* set up the descriptor */
1855	gpi_desc->gchan = gchan;
1856	gpi_desc->len = sg_dma_len(sgl);
1857	gpi_desc->num_tre  = i;
1858
1859	return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags);
1860}
1861
1862/* rings transfer ring db to being transfer */
1863static void gpi_issue_pending(struct dma_chan *chan)
1864{
1865	struct gchan *gchan = to_gchan(chan);
1866	struct gpii *gpii = gchan->gpii;
1867	unsigned long flags, pm_lock_flags;
1868	struct virt_dma_desc *vd = NULL;
1869	struct gpi_desc *gpi_desc;
1870	struct gpi_ring *ch_ring = &gchan->ch_ring;
1871	void *tre, *wp = NULL;
1872	int i;
1873
1874	read_lock_irqsave(&gpii->pm_lock, pm_lock_flags);
1875
1876	/* move all submitted descriptors to issued list */
1877	spin_lock_irqsave(&gchan->vc.lock, flags);
1878	if (vchan_issue_pending(&gchan->vc))
1879		vd = list_last_entry(&gchan->vc.desc_issued,
1880				     struct virt_dma_desc, node);
1881	spin_unlock_irqrestore(&gchan->vc.lock, flags);
1882
1883	/* nothing to do list is empty */
1884	if (!vd) {
1885		read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);
1886		return;
1887	}
1888
1889	gpi_desc = to_gpi_desc(vd);
1890	for (i = 0; i < gpi_desc->num_tre; i++) {
1891		tre = &gpi_desc->tre[i];
1892		gpi_queue_xfer(gpii, gchan, tre, &wp);
1893	}
1894
1895	gpi_desc->db = ch_ring->wp;
1896	gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db);
1897	read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);
1898}
1899
1900static int gpi_ch_init(struct gchan *gchan)
1901{
1902	struct gpii *gpii = gchan->gpii;
1903	const int ev_factor = gpii->gpi_dev->ev_factor;
1904	u32 elements;
1905	int i = 0, ret = 0;
1906
1907	gchan->pm_state = CONFIG_STATE;
1908
1909	/* check if both channels are configured before continue */
1910	for (i = 0; i < MAX_CHANNELS_PER_GPII; i++)
1911		if (gpii->gchan[i].pm_state != CONFIG_STATE)
1912			goto exit_gpi_init;
1913
1914	/* protocol must be same for both channels */
1915	if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) {
1916		dev_err(gpii->gpi_dev->dev, "protocol did not match protocol %u != %u\n",
1917			gpii->gchan[0].protocol, gpii->gchan[1].protocol);
1918		ret = -EINVAL;
1919		goto exit_gpi_init;
1920	}
1921
1922	/* allocate memory for event ring */
1923	elements = CHAN_TRES << ev_factor;
1924	ret = gpi_alloc_ring(&gpii->ev_ring, elements,
1925			     sizeof(union gpi_event), gpii);
1926	if (ret)
1927		goto exit_gpi_init;
1928
1929	/* configure interrupts */
1930	write_lock_irq(&gpii->pm_lock);
1931	gpii->pm_state = PREPARE_HARDWARE;
1932	write_unlock_irq(&gpii->pm_lock);
1933	ret = gpi_config_interrupts(gpii, DEFAULT_IRQ_SETTINGS, 0);
1934	if (ret) {
1935		dev_err(gpii->gpi_dev->dev, "error config. interrupts, ret:%d\n", ret);
1936		goto error_config_int;
1937	}
1938
1939	/* allocate event rings */
1940	ret = gpi_alloc_ev_chan(gpii);
1941	if (ret) {
1942		dev_err(gpii->gpi_dev->dev, "error alloc_ev_chan:%d\n", ret);
1943		goto error_alloc_ev_ring;
1944	}
1945
1946	/* Allocate all channels */
1947	for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1948		ret = gpi_alloc_chan(&gpii->gchan[i], true);
1949		if (ret) {
1950			dev_err(gpii->gpi_dev->dev, "Error allocating chan:%d\n", ret);
1951			goto error_alloc_chan;
1952		}
1953	}
1954
1955	/* start channels  */
1956	for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {
1957		ret = gpi_start_chan(&gpii->gchan[i]);
1958		if (ret) {
1959			dev_err(gpii->gpi_dev->dev, "Error start chan:%d\n", ret);
1960			goto error_start_chan;
1961		}
1962	}
1963	return ret;
1964
1965error_start_chan:
1966	for (i = i - 1; i >= 0; i--) {
1967		gpi_stop_chan(&gpii->gchan[i]);
1968		gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);
1969	}
1970	i = 2;
1971error_alloc_chan:
1972	for (i = i - 1; i >= 0; i--)
1973		gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);
1974error_alloc_ev_ring:
1975	gpi_disable_interrupts(gpii);
1976error_config_int:
1977	gpi_free_ring(&gpii->ev_ring, gpii);
1978exit_gpi_init:
1979	return ret;
1980}
1981
1982/* release all channel resources */
1983static void gpi_free_chan_resources(struct dma_chan *chan)
1984{
1985	struct gchan *gchan = to_gchan(chan);
1986	struct gpii *gpii = gchan->gpii;
1987	enum gpi_pm_state cur_state;
1988	int ret, i;
1989
1990	mutex_lock(&gpii->ctrl_lock);
1991
1992	cur_state = gchan->pm_state;
1993
1994	/* disable ch state so no more TRE processing for this channel */
1995	write_lock_irq(&gpii->pm_lock);
1996	gchan->pm_state = PREPARE_TERMINATE;
1997	write_unlock_irq(&gpii->pm_lock);
1998
1999	/* attempt to do graceful hardware shutdown */
2000	if (cur_state == ACTIVE_STATE) {
2001		gpi_stop_chan(gchan);
2002
2003		ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);
2004		if (ret)
2005			dev_err(gpii->gpi_dev->dev, "error resetting channel:%d\n", ret);
2006
2007		gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);
2008	}
2009
2010	/* free all allocated memory */
2011	gpi_free_ring(&gchan->ch_ring, gpii);
2012	vchan_free_chan_resources(&gchan->vc);
2013	kfree(gchan->config);
2014
2015	write_lock_irq(&gpii->pm_lock);
2016	gchan->pm_state = DISABLE_STATE;
2017	write_unlock_irq(&gpii->pm_lock);
2018
2019	/* if other rings are still active exit */
2020	for (i = 0; i < MAX_CHANNELS_PER_GPII; i++)
2021		if (gpii->gchan[i].ch_ring.configured)
2022			goto exit_free;
2023
2024	/* deallocate EV Ring */
2025	cur_state = gpii->pm_state;
2026	write_lock_irq(&gpii->pm_lock);
2027	gpii->pm_state = PREPARE_TERMINATE;
2028	write_unlock_irq(&gpii->pm_lock);
2029
2030	/* wait for threads to complete out */
2031	tasklet_kill(&gpii->ev_task);
2032
2033	/* send command to de allocate event ring */
2034	if (cur_state == ACTIVE_STATE)
2035		gpi_send_cmd(gpii, NULL, GPI_EV_CMD_DEALLOC);
2036
2037	gpi_free_ring(&gpii->ev_ring, gpii);
2038
2039	/* disable interrupts */
2040	if (cur_state == ACTIVE_STATE)
2041		gpi_disable_interrupts(gpii);
2042
2043	/* set final state to disable */
2044	write_lock_irq(&gpii->pm_lock);
2045	gpii->pm_state = DISABLE_STATE;
2046	write_unlock_irq(&gpii->pm_lock);
2047
2048exit_free:
2049	mutex_unlock(&gpii->ctrl_lock);
2050}
2051
2052/* allocate channel resources */
2053static int gpi_alloc_chan_resources(struct dma_chan *chan)
2054{
2055	struct gchan *gchan = to_gchan(chan);
2056	struct gpii *gpii = gchan->gpii;
2057	int ret;
2058
2059	mutex_lock(&gpii->ctrl_lock);
2060
2061	/* allocate memory for transfer ring */
2062	ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES,
2063			     sizeof(struct gpi_tre), gpii);
2064	if (ret)
2065		goto xfer_alloc_err;
2066
2067	ret = gpi_ch_init(gchan);
2068
2069	mutex_unlock(&gpii->ctrl_lock);
2070
2071	return ret;
2072xfer_alloc_err:
2073	mutex_unlock(&gpii->ctrl_lock);
2074
2075	return ret;
2076}
2077
2078static int gpi_find_avail_gpii(struct gpi_dev *gpi_dev, u32 seid)
2079{
2080	struct gchan *tx_chan, *rx_chan;
2081	unsigned int gpii;
2082
2083	/* check if same seid is already configured for another chid */
2084	for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {
2085		if (!((1 << gpii) & gpi_dev->gpii_mask))
2086			continue;
2087
2088		tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];
2089		rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];
2090
2091		if (rx_chan->vc.chan.client_count && rx_chan->seid == seid)
2092			return gpii;
2093		if (tx_chan->vc.chan.client_count && tx_chan->seid == seid)
2094			return gpii;
2095	}
2096
2097	/* no channels configured with same seid, return next avail gpii */
2098	for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {
2099		if (!((1 << gpii) & gpi_dev->gpii_mask))
2100			continue;
2101
2102		tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];
2103		rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];
2104
2105		/* check if gpii is configured */
2106		if (tx_chan->vc.chan.client_count ||
2107		    rx_chan->vc.chan.client_count)
2108			continue;
2109
2110		/* found a free gpii */
2111		return gpii;
2112	}
2113
2114	/* no gpii instance available to use */
2115	return -EIO;
2116}
2117
2118/* gpi_of_dma_xlate: open client requested channel */
2119static struct dma_chan *gpi_of_dma_xlate(struct of_phandle_args *args,
2120					 struct of_dma *of_dma)
2121{
2122	struct gpi_dev *gpi_dev = (struct gpi_dev *)of_dma->of_dma_data;
2123	u32 seid, chid;
2124	int gpii;
2125	struct gchan *gchan;
2126
2127	if (args->args_count < 3) {
2128		dev_err(gpi_dev->dev, "gpii require minimum 2 args, client passed:%d args\n",
2129			args->args_count);
2130		return NULL;
2131	}
2132
2133	chid = args->args[0];
2134	if (chid >= MAX_CHANNELS_PER_GPII) {
2135		dev_err(gpi_dev->dev, "gpii channel:%d not valid\n", chid);
2136		return NULL;
2137	}
2138
2139	seid = args->args[1];
2140
2141	/* find next available gpii to use */
2142	gpii = gpi_find_avail_gpii(gpi_dev, seid);
2143	if (gpii < 0) {
2144		dev_err(gpi_dev->dev, "no available gpii instances\n");
2145		return NULL;
2146	}
2147
2148	gchan = &gpi_dev->gpiis[gpii].gchan[chid];
2149	if (gchan->vc.chan.client_count) {
2150		dev_err(gpi_dev->dev, "gpii:%d chid:%d seid:%d already configured\n",
2151			gpii, chid, gchan->seid);
2152		return NULL;
2153	}
2154
2155	gchan->seid = seid;
2156	gchan->protocol = args->args[2];
2157
2158	return dma_get_slave_channel(&gchan->vc.chan);
2159}
2160
2161static int gpi_probe(struct platform_device *pdev)
2162{
2163	struct gpi_dev *gpi_dev;
2164	unsigned int i;
2165	u32 ee_offset;
2166	int ret;
2167
2168	gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL);
2169	if (!gpi_dev)
2170		return -ENOMEM;
2171
2172	gpi_dev->dev = &pdev->dev;
2173	gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res);
2174	if (IS_ERR(gpi_dev->regs))
2175		return PTR_ERR(gpi_dev->regs);
2176	gpi_dev->ee_base = gpi_dev->regs;
2177
2178	ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channels",
2179				   &gpi_dev->max_gpii);
2180	if (ret) {
2181		dev_err(gpi_dev->dev, "missing 'max-no-gpii' DT node\n");
2182		return ret;
2183	}
2184
2185	ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channel-mask",
2186				   &gpi_dev->gpii_mask);
2187	if (ret) {
2188		dev_err(gpi_dev->dev, "missing 'gpii-mask' DT node\n");
2189		return ret;
2190	}
2191
2192	ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev);
2193	gpi_dev->ee_base = gpi_dev->ee_base - ee_offset;
2194
2195	gpi_dev->ev_factor = EV_FACTOR;
2196
2197	ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64));
2198	if (ret) {
2199		dev_err(gpi_dev->dev, "Error setting dma_mask to 64, ret:%d\n", ret);
2200		return ret;
2201	}
2202
2203	gpi_dev->gpiis = devm_kzalloc(gpi_dev->dev, sizeof(*gpi_dev->gpiis) *
2204				      gpi_dev->max_gpii, GFP_KERNEL);
2205	if (!gpi_dev->gpiis)
2206		return -ENOMEM;
2207
2208	/* setup all the supported gpii */
2209	INIT_LIST_HEAD(&gpi_dev->dma_device.channels);
2210	for (i = 0; i < gpi_dev->max_gpii; i++) {
2211		struct gpii *gpii = &gpi_dev->gpiis[i];
2212		int chan;
2213
2214		if (!((1 << i) & gpi_dev->gpii_mask))
2215			continue;
2216
2217		/* set up ev cntxt register map */
2218		gpii->ev_cntxt_base_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_CNTXT_0_OFFS(i, 0);
2219		gpii->ev_cntxt_db_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_DOORBELL_0_OFFS(i, 0);
2220		gpii->ev_ring_rp_lsb_reg = gpii->ev_cntxt_base_reg + CNTXT_4_RING_RP_LSB;
2221		gpii->ev_cmd_reg = gpi_dev->ee_base + GPII_n_EV_CH_CMD_OFFS(i);
2222		gpii->ieob_clr_reg = gpi_dev->ee_base + GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(i);
2223
2224		/* set up irq */
2225		ret = platform_get_irq(pdev, i);
2226		if (ret < 0)
2227			return ret;
2228		gpii->irq = ret;
2229
2230		/* set up channel specific register info */
2231		for (chan = 0; chan < MAX_CHANNELS_PER_GPII; chan++) {
2232			struct gchan *gchan = &gpii->gchan[chan];
2233
2234			/* set up ch cntxt register map */
2235			gchan->ch_cntxt_base_reg = gpi_dev->ee_base +
2236				GPII_n_CH_k_CNTXT_0_OFFS(i, chan);
2237			gchan->ch_cntxt_db_reg = gpi_dev->ee_base +
2238				GPII_n_CH_k_DOORBELL_0_OFFS(i, chan);
2239			gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i);
2240
2241			/* vchan setup */
2242			vchan_init(&gchan->vc, &gpi_dev->dma_device);
2243			gchan->vc.desc_free = gpi_desc_free;
2244			gchan->chid = chan;
2245			gchan->gpii = gpii;
2246			gchan->dir = GPII_CHAN_DIR[chan];
2247		}
2248		mutex_init(&gpii->ctrl_lock);
2249		rwlock_init(&gpii->pm_lock);
2250		tasklet_init(&gpii->ev_task, gpi_ev_tasklet,
2251			     (unsigned long)gpii);
2252		init_completion(&gpii->cmd_completion);
2253		gpii->gpii_id = i;
2254		gpii->regs = gpi_dev->ee_base;
2255		gpii->gpi_dev = gpi_dev;
2256	}
2257
2258	platform_set_drvdata(pdev, gpi_dev);
2259
2260	/* clear and Set capabilities */
2261	dma_cap_zero(gpi_dev->dma_device.cap_mask);
2262	dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask);
2263
2264	/* configure dmaengine apis */
2265	gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2266	gpi_dev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
2267	gpi_dev->dma_device.src_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;
2268	gpi_dev->dma_device.dst_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;
2269	gpi_dev->dma_device.device_alloc_chan_resources = gpi_alloc_chan_resources;
2270	gpi_dev->dma_device.device_free_chan_resources = gpi_free_chan_resources;
2271	gpi_dev->dma_device.device_tx_status = dma_cookie_status;
2272	gpi_dev->dma_device.device_issue_pending = gpi_issue_pending;
2273	gpi_dev->dma_device.device_prep_slave_sg = gpi_prep_slave_sg;
2274	gpi_dev->dma_device.device_config = gpi_peripheral_config;
2275	gpi_dev->dma_device.device_terminate_all = gpi_terminate_all;
2276	gpi_dev->dma_device.dev = gpi_dev->dev;
2277	gpi_dev->dma_device.device_pause = gpi_pause;
2278	gpi_dev->dma_device.device_resume = gpi_resume;
2279
2280	/* register with dmaengine framework */
2281	ret = dma_async_device_register(&gpi_dev->dma_device);
2282	if (ret) {
2283		dev_err(gpi_dev->dev, "async_device_register failed ret:%d", ret);
2284		return ret;
2285	}
2286
2287	ret = of_dma_controller_register(gpi_dev->dev->of_node,
2288					 gpi_of_dma_xlate, gpi_dev);
2289	if (ret) {
2290		dev_err(gpi_dev->dev, "of_dma_controller_reg failed ret:%d", ret);
2291		return ret;
2292	}
2293
2294	return ret;
2295}
2296
2297static const struct of_device_id gpi_of_match[] = {
2298	{ .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },
2299	{ .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 },
2300	/*
2301	 * Do not grow the list for compatible devices. Instead use
2302	 * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma
2303	 * (for ee_offset = 0x10000).
2304	 */
2305	{ .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },
2306	{ .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },
2307	{ .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },
2308	{ .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },
2309	{ .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 },
2310	{ },
2311};
2312MODULE_DEVICE_TABLE(of, gpi_of_match);
2313
2314static struct platform_driver gpi_driver = {
2315	.probe = gpi_probe,
2316	.driver = {
2317		.name = KBUILD_MODNAME,
2318		.of_match_table = gpi_of_match,
2319	},
2320};
2321
2322static int __init gpi_init(void)
2323{
2324	return platform_driver_register(&gpi_driver);
2325}
2326subsys_initcall(gpi_init)
2327
2328MODULE_DESCRIPTION("QCOM GPI DMA engine driver");
2329MODULE_LICENSE("GPL v2");
~/ Read-only file preview.