fman.c

   1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
   2/*
   3 * Copyright 2008 - 2015 Freescale Semiconductor Inc.
   4 * Copyright 2020 NXP
   5 */
   6
   7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9#include <linux/fsl/guts.h>
  10#include <linux/platform_device.h>
  11#include <linux/slab.h>
  12#include <linux/delay.h>
  13#include <linux/module.h>
  14#include <linux/of_platform.h>
  15#include <linux/clk.h>
  16#include <linux/of_address.h>
  17#include <linux/of_irq.h>
  18#include <linux/interrupt.h>
  19#include <linux/libfdt_env.h>
  20
  21#include "fman.h"
  22#include "fman_muram.h"
  23#include "fman_keygen.h"
  24
  25/* General defines */
  26#define FMAN_LIODN_TBL			64	/* size of LIODN table */
  27#define FM_NUM_OF_FMAN_CTRL_EVENT_REGS	4
  28#define BASE_RX_PORTID			0x08
  29#define BASE_TX_PORTID			0x28
  30
  31/* Modules registers offsets */
  32#define BMI_OFFSET		0x00080000
  33#define QMI_OFFSET		0x00080400
  34#define KG_OFFSET		0x000C1000
  35#define DMA_OFFSET		0x000C2000
  36#define FPM_OFFSET		0x000C3000
  37#define IMEM_OFFSET		0x000C4000
  38#define HWP_OFFSET		0x000C7000
  39#define CGP_OFFSET		0x000DB000
  40
  41/* Exceptions bit map */
  42#define EX_DMA_BUS_ERROR		0x80000000
  43#define EX_DMA_READ_ECC			0x40000000
  44#define EX_DMA_SYSTEM_WRITE_ECC	0x20000000
  45#define EX_DMA_FM_WRITE_ECC		0x10000000
  46#define EX_FPM_STALL_ON_TASKS		0x08000000
  47#define EX_FPM_SINGLE_ECC		0x04000000
  48#define EX_FPM_DOUBLE_ECC		0x02000000
  49#define EX_QMI_SINGLE_ECC		0x01000000
  50#define EX_QMI_DEQ_FROM_UNKNOWN_PORTID	0x00800000
  51#define EX_QMI_DOUBLE_ECC		0x00400000
  52#define EX_BMI_LIST_RAM_ECC		0x00200000
  53#define EX_BMI_STORAGE_PROFILE_ECC	0x00100000
  54#define EX_BMI_STATISTICS_RAM_ECC	0x00080000
  55#define EX_IRAM_ECC			0x00040000
  56#define EX_MURAM_ECC			0x00020000
  57#define EX_BMI_DISPATCH_RAM_ECC	0x00010000
  58#define EX_DMA_SINGLE_PORT_ECC		0x00008000
  59
  60/* DMA defines */
  61/* masks */
  62#define DMA_MODE_BER			0x00200000
  63#define DMA_MODE_ECC			0x00000020
  64#define DMA_MODE_SECURE_PROT		0x00000800
  65#define DMA_MODE_AXI_DBG_MASK		0x0F000000
  66
  67#define DMA_TRANSFER_PORTID_MASK	0xFF000000
  68#define DMA_TRANSFER_TNUM_MASK		0x00FF0000
  69#define DMA_TRANSFER_LIODN_MASK	0x00000FFF
  70
  71#define DMA_STATUS_BUS_ERR		0x08000000
  72#define DMA_STATUS_READ_ECC		0x04000000
  73#define DMA_STATUS_SYSTEM_WRITE_ECC	0x02000000
  74#define DMA_STATUS_FM_WRITE_ECC	0x01000000
  75#define DMA_STATUS_FM_SPDAT_ECC	0x00080000
  76
  77#define DMA_MODE_CACHE_OR_SHIFT		30
  78#define DMA_MODE_AXI_DBG_SHIFT			24
  79#define DMA_MODE_CEN_SHIFT			13
  80#define DMA_MODE_CEN_MASK			0x00000007
  81#define DMA_MODE_DBG_SHIFT			7
  82#define DMA_MODE_AID_MODE_SHIFT		4
  83
  84#define DMA_THRESH_COMMQ_SHIFT			24
  85#define DMA_THRESH_READ_INT_BUF_SHIFT		16
  86#define DMA_THRESH_READ_INT_BUF_MASK		0x0000003f
  87#define DMA_THRESH_WRITE_INT_BUF_MASK		0x0000003f
  88
  89#define DMA_TRANSFER_PORTID_SHIFT		24
  90#define DMA_TRANSFER_TNUM_SHIFT		16
  91
  92#define DMA_CAM_SIZEOF_ENTRY			0x40
  93#define DMA_CAM_UNITS				8
  94
  95#define DMA_LIODN_SHIFT		16
  96#define DMA_LIODN_BASE_MASK	0x00000FFF
  97
  98/* FPM defines */
  99#define FPM_EV_MASK_DOUBLE_ECC		0x80000000
 100#define FPM_EV_MASK_STALL		0x40000000
 101#define FPM_EV_MASK_SINGLE_ECC		0x20000000
 102#define FPM_EV_MASK_RELEASE_FM		0x00010000
 103#define FPM_EV_MASK_DOUBLE_ECC_EN	0x00008000
 104#define FPM_EV_MASK_STALL_EN		0x00004000
 105#define FPM_EV_MASK_SINGLE_ECC_EN	0x00002000
 106#define FPM_EV_MASK_EXTERNAL_HALT	0x00000008
 107#define FPM_EV_MASK_ECC_ERR_HALT	0x00000004
 108
 109#define FPM_RAM_MURAM_ECC		0x00008000
 110#define FPM_RAM_IRAM_ECC		0x00004000
 111#define FPM_IRAM_ECC_ERR_EX_EN		0x00020000
 112#define FPM_MURAM_ECC_ERR_EX_EN	0x00040000
 113#define FPM_RAM_IRAM_ECC_EN		0x40000000
 114#define FPM_RAM_RAMS_ECC_EN		0x80000000
 115#define FPM_RAM_RAMS_ECC_EN_SRC_SEL	0x08000000
 116
 117#define FPM_REV1_MAJOR_MASK		0x0000FF00
 118#define FPM_REV1_MINOR_MASK		0x000000FF
 119
 120#define FPM_DISP_LIMIT_SHIFT		24
 121
 122#define FPM_PRT_FM_CTL1			0x00000001
 123#define FPM_PRT_FM_CTL2			0x00000002
 124#define FPM_PORT_FM_CTL_PORTID_SHIFT	24
 125#define FPM_PRC_ORA_FM_CTL_SEL_SHIFT	16
 126
 127#define FPM_THR1_PRS_SHIFT		24
 128#define FPM_THR1_KG_SHIFT		16
 129#define FPM_THR1_PLCR_SHIFT		8
 130#define FPM_THR1_BMI_SHIFT		0
 131
 132#define FPM_THR2_QMI_ENQ_SHIFT		24
 133#define FPM_THR2_QMI_DEQ_SHIFT		0
 134#define FPM_THR2_FM_CTL1_SHIFT		16
 135#define FPM_THR2_FM_CTL2_SHIFT		8
 136
 137#define FPM_EV_MASK_CAT_ERR_SHIFT	1
 138#define FPM_EV_MASK_DMA_ERR_SHIFT	0
 139
 140#define FPM_REV1_MAJOR_SHIFT		8
 141
 142#define FPM_RSTC_FM_RESET		0x80000000
 143#define FPM_RSTC_MAC0_RESET		0x40000000
 144#define FPM_RSTC_MAC1_RESET		0x20000000
 145#define FPM_RSTC_MAC2_RESET		0x10000000
 146#define FPM_RSTC_MAC3_RESET		0x08000000
 147#define FPM_RSTC_MAC8_RESET		0x04000000
 148#define FPM_RSTC_MAC4_RESET		0x02000000
 149#define FPM_RSTC_MAC5_RESET		0x01000000
 150#define FPM_RSTC_MAC6_RESET		0x00800000
 151#define FPM_RSTC_MAC7_RESET		0x00400000
 152#define FPM_RSTC_MAC9_RESET		0x00200000
 153
 154#define FPM_TS_INT_SHIFT		16
 155#define FPM_TS_CTL_EN			0x80000000
 156
 157/* BMI defines */
 158#define BMI_INIT_START				0x80000000
 159#define BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC	0x80000000
 160#define BMI_ERR_INTR_EN_LIST_RAM_ECC		0x40000000
 161#define BMI_ERR_INTR_EN_STATISTICS_RAM_ECC	0x20000000
 162#define BMI_ERR_INTR_EN_DISPATCH_RAM_ECC	0x10000000
 163#define BMI_NUM_OF_TASKS_MASK			0x3F000000
 164#define BMI_NUM_OF_EXTRA_TASKS_MASK		0x000F0000
 165#define BMI_NUM_OF_DMAS_MASK			0x00000F00
 166#define BMI_NUM_OF_EXTRA_DMAS_MASK		0x0000000F
 167#define BMI_FIFO_SIZE_MASK			0x000003FF
 168#define BMI_EXTRA_FIFO_SIZE_MASK		0x03FF0000
 169#define BMI_CFG2_DMAS_MASK			0x0000003F
 170#define BMI_CFG2_TASKS_MASK			0x0000003F
 171
 172#define BMI_CFG2_TASKS_SHIFT		16
 173#define BMI_CFG2_DMAS_SHIFT		0
 174#define BMI_CFG1_FIFO_SIZE_SHIFT	16
 175#define BMI_NUM_OF_TASKS_SHIFT		24
 176#define BMI_EXTRA_NUM_OF_TASKS_SHIFT	16
 177#define BMI_NUM_OF_DMAS_SHIFT		8
 178#define BMI_EXTRA_NUM_OF_DMAS_SHIFT	0
 179
 180#define BMI_FIFO_ALIGN			0x100
 181
 182#define BMI_EXTRA_FIFO_SIZE_SHIFT	16
 183
 184/* QMI defines */
 185#define QMI_CFG_ENQ_EN			0x80000000
 186#define QMI_CFG_DEQ_EN			0x40000000
 187#define QMI_CFG_EN_COUNTERS		0x10000000
 188#define QMI_CFG_DEQ_MASK		0x0000003F
 189#define QMI_CFG_ENQ_MASK		0x00003F00
 190#define QMI_CFG_ENQ_SHIFT		8
 191
 192#define QMI_ERR_INTR_EN_DOUBLE_ECC	0x80000000
 193#define QMI_ERR_INTR_EN_DEQ_FROM_DEF	0x40000000
 194#define QMI_INTR_EN_SINGLE_ECC		0x80000000
 195
 196#define QMI_GS_HALT_NOT_BUSY		0x00000002
 197
 198/* HWP defines */
 199#define HWP_RPIMAC_PEN			0x00000001
 200
 201/* IRAM defines */
 202#define IRAM_IADD_AIE			0x80000000
 203#define IRAM_READY			0x80000000
 204
 205/* Default values */
 206#define DEFAULT_CATASTROPHIC_ERR		0
 207#define DEFAULT_DMA_ERR				0
 208#define DEFAULT_AID_MODE			FMAN_DMA_AID_OUT_TNUM
 209#define DEFAULT_DMA_COMM_Q_LOW			0x2A
 210#define DEFAULT_DMA_COMM_Q_HIGH		0x3F
 211#define DEFAULT_CACHE_OVERRIDE			0
 212#define DEFAULT_DMA_CAM_NUM_OF_ENTRIES		64
 213#define DEFAULT_DMA_DBG_CNT_MODE		0
 214#define DEFAULT_DMA_SOS_EMERGENCY		0
 215#define DEFAULT_DMA_WATCHDOG			0
 216#define DEFAULT_DISP_LIMIT			0
 217#define DEFAULT_PRS_DISP_TH			16
 218#define DEFAULT_PLCR_DISP_TH			16
 219#define DEFAULT_KG_DISP_TH			16
 220#define DEFAULT_BMI_DISP_TH			16
 221#define DEFAULT_QMI_ENQ_DISP_TH		16
 222#define DEFAULT_QMI_DEQ_DISP_TH		16
 223#define DEFAULT_FM_CTL1_DISP_TH		16
 224#define DEFAULT_FM_CTL2_DISP_TH		16
 225
 226#define DFLT_AXI_DBG_NUM_OF_BEATS		1
 227
 228#define DFLT_DMA_READ_INT_BUF_LOW(dma_thresh_max_buf)	\
 229	((dma_thresh_max_buf + 1) / 2)
 230#define DFLT_DMA_READ_INT_BUF_HIGH(dma_thresh_max_buf)	\
 231	((dma_thresh_max_buf + 1) * 3 / 4)
 232#define DFLT_DMA_WRITE_INT_BUF_LOW(dma_thresh_max_buf)	\
 233	((dma_thresh_max_buf + 1) / 2)
 234#define DFLT_DMA_WRITE_INT_BUF_HIGH(dma_thresh_max_buf)\
 235	((dma_thresh_max_buf + 1) * 3 / 4)
 236
 237#define DMA_COMM_Q_LOW_FMAN_V3		0x2A
 238#define DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq)		\
 239	((dma_thresh_max_commq + 1) / 2)
 240#define DFLT_DMA_COMM_Q_LOW(major, dma_thresh_max_commq)	\
 241	((major == 6) ? DMA_COMM_Q_LOW_FMAN_V3 :		\
 242	DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq))
 243
 244#define DMA_COMM_Q_HIGH_FMAN_V3	0x3f
 245#define DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq)		\
 246	((dma_thresh_max_commq + 1) * 3 / 4)
 247#define DFLT_DMA_COMM_Q_HIGH(major, dma_thresh_max_commq)	\
 248	((major == 6) ? DMA_COMM_Q_HIGH_FMAN_V3 :		\
 249	DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq))
 250
 251#define TOTAL_NUM_OF_TASKS_FMAN_V3L	59
 252#define TOTAL_NUM_OF_TASKS_FMAN_V3H	124
 253#define DFLT_TOTAL_NUM_OF_TASKS(major, minor, bmi_max_num_of_tasks)	\
 254	((major == 6) ? ((minor == 1 || minor == 4) ?			\
 255	TOTAL_NUM_OF_TASKS_FMAN_V3L : TOTAL_NUM_OF_TASKS_FMAN_V3H) :	\
 256	bmi_max_num_of_tasks)
 257
 258#define DMA_CAM_NUM_OF_ENTRIES_FMAN_V3		64
 259#define DMA_CAM_NUM_OF_ENTRIES_FMAN_V2		32
 260#define DFLT_DMA_CAM_NUM_OF_ENTRIES(major)			\
 261	(major == 6 ? DMA_CAM_NUM_OF_ENTRIES_FMAN_V3 :		\
 262	DMA_CAM_NUM_OF_ENTRIES_FMAN_V2)
 263
 264#define FM_TIMESTAMP_1_USEC_BIT             8
 265
 266/* Defines used for enabling/disabling FMan interrupts */
 267#define ERR_INTR_EN_DMA         0x00010000
 268#define ERR_INTR_EN_FPM         0x80000000
 269#define ERR_INTR_EN_BMI         0x00800000
 270#define ERR_INTR_EN_QMI         0x00400000
 271#define ERR_INTR_EN_MURAM       0x00040000
 272#define ERR_INTR_EN_MAC0        0x00004000
 273#define ERR_INTR_EN_MAC1        0x00002000
 274#define ERR_INTR_EN_MAC2        0x00001000
 275#define ERR_INTR_EN_MAC3        0x00000800
 276#define ERR_INTR_EN_MAC4        0x00000400
 277#define ERR_INTR_EN_MAC5        0x00000200
 278#define ERR_INTR_EN_MAC6        0x00000100
 279#define ERR_INTR_EN_MAC7        0x00000080
 280#define ERR_INTR_EN_MAC8        0x00008000
 281#define ERR_INTR_EN_MAC9        0x00000040
 282
 283#define INTR_EN_QMI             0x40000000
 284#define INTR_EN_MAC0            0x00080000
 285#define INTR_EN_MAC1            0x00040000
 286#define INTR_EN_MAC2            0x00020000
 287#define INTR_EN_MAC3            0x00010000
 288#define INTR_EN_MAC4            0x00000040
 289#define INTR_EN_MAC5            0x00000020
 290#define INTR_EN_MAC6            0x00000008
 291#define INTR_EN_MAC7            0x00000002
 292#define INTR_EN_MAC8            0x00200000
 293#define INTR_EN_MAC9            0x00100000
 294#define INTR_EN_REV0            0x00008000
 295#define INTR_EN_REV1            0x00004000
 296#define INTR_EN_REV2            0x00002000
 297#define INTR_EN_REV3            0x00001000
 298#define INTR_EN_TMR             0x01000000
 299
 300enum fman_dma_aid_mode {
 301	FMAN_DMA_AID_OUT_PORT_ID = 0,		  /* 4 LSB of PORT_ID */
 302	FMAN_DMA_AID_OUT_TNUM			  /* 4 LSB of TNUM */
 303};
 304
 305struct fman_iram_regs {
 306	u32 iadd;	/* FM IRAM instruction address register */
 307	u32 idata;	/* FM IRAM instruction data register */
 308	u32 itcfg;	/* FM IRAM timing config register */
 309	u32 iready;	/* FM IRAM ready register */
 310};
 311
 312struct fman_fpm_regs {
 313	u32 fmfp_tnc;		/* FPM TNUM Control 0x00 */
 314	u32 fmfp_prc;		/* FPM Port_ID FmCtl Association 0x04 */
 315	u32 fmfp_brkc;		/* FPM Breakpoint Control 0x08 */
 316	u32 fmfp_mxd;		/* FPM Flush Control 0x0c */
 317	u32 fmfp_dist1;		/* FPM Dispatch Thresholds1 0x10 */
 318	u32 fmfp_dist2;		/* FPM Dispatch Thresholds2 0x14 */
 319	u32 fm_epi;		/* FM Error Pending Interrupts 0x18 */
 320	u32 fm_rie;		/* FM Error Interrupt Enable 0x1c */
 321	u32 fmfp_fcev[4];	/* FPM FMan-Controller Event 1-4 0x20-0x2f */
 322	u32 res0030[4];		/* res 0x30 - 0x3f */
 323	u32 fmfp_cee[4];	/* PM FMan-Controller Event 1-4 0x40-0x4f */
 324	u32 res0050[4];		/* res 0x50-0x5f */
 325	u32 fmfp_tsc1;		/* FPM TimeStamp Control1 0x60 */
 326	u32 fmfp_tsc2;		/* FPM TimeStamp Control2 0x64 */
 327	u32 fmfp_tsp;		/* FPM Time Stamp 0x68 */
 328	u32 fmfp_tsf;		/* FPM Time Stamp Fraction 0x6c */
 329	u32 fm_rcr;		/* FM Rams Control 0x70 */
 330	u32 fmfp_extc;		/* FPM External Requests Control 0x74 */
 331	u32 fmfp_ext1;		/* FPM External Requests Config1 0x78 */
 332	u32 fmfp_ext2;		/* FPM External Requests Config2 0x7c */
 333	u32 fmfp_drd[16];	/* FPM Data_Ram Data 0-15 0x80 - 0xbf */
 334	u32 fmfp_dra;		/* FPM Data Ram Access 0xc0 */
 335	u32 fm_ip_rev_1;	/* FM IP Block Revision 1 0xc4 */
 336	u32 fm_ip_rev_2;	/* FM IP Block Revision 2 0xc8 */
 337	u32 fm_rstc;		/* FM Reset Command 0xcc */
 338	u32 fm_cld;		/* FM Classifier Debug 0xd0 */
 339	u32 fm_npi;		/* FM Normal Pending Interrupts 0xd4 */
 340	u32 fmfp_exte;		/* FPM External Requests Enable 0xd8 */
 341	u32 fmfp_ee;		/* FPM Event&Mask 0xdc */
 342	u32 fmfp_cev[4];	/* FPM CPU Event 1-4 0xe0-0xef */
 343	u32 res00f0[4];		/* res 0xf0-0xff */
 344	u32 fmfp_ps[50];	/* FPM Port Status 0x100-0x1c7 */
 345	u32 res01c8[14];	/* res 0x1c8-0x1ff */
 346	u32 fmfp_clfabc;	/* FPM CLFABC 0x200 */
 347	u32 fmfp_clfcc;		/* FPM CLFCC 0x204 */
 348	u32 fmfp_clfaval;	/* FPM CLFAVAL 0x208 */
 349	u32 fmfp_clfbval;	/* FPM CLFBVAL 0x20c */
 350	u32 fmfp_clfcval;	/* FPM CLFCVAL 0x210 */
 351	u32 fmfp_clfamsk;	/* FPM CLFAMSK 0x214 */
 352	u32 fmfp_clfbmsk;	/* FPM CLFBMSK 0x218 */
 353	u32 fmfp_clfcmsk;	/* FPM CLFCMSK 0x21c */
 354	u32 fmfp_clfamc;	/* FPM CLFAMC 0x220 */
 355	u32 fmfp_clfbmc;	/* FPM CLFBMC 0x224 */
 356	u32 fmfp_clfcmc;	/* FPM CLFCMC 0x228 */
 357	u32 fmfp_decceh;	/* FPM DECCEH 0x22c */
 358	u32 res0230[116];	/* res 0x230 - 0x3ff */
 359	u32 fmfp_ts[128];	/* 0x400: FPM Task Status 0x400 - 0x5ff */
 360	u32 res0600[0x400 - 384];
 361};
 362
 363struct fman_bmi_regs {
 364	u32 fmbm_init;		/* BMI Initialization 0x00 */
 365	u32 fmbm_cfg1;		/* BMI Configuration 1 0x04 */
 366	u32 fmbm_cfg2;		/* BMI Configuration 2 0x08 */
 367	u32 res000c[5];		/* 0x0c - 0x1f */
 368	u32 fmbm_ievr;		/* Interrupt Event Register 0x20 */
 369	u32 fmbm_ier;		/* Interrupt Enable Register 0x24 */
 370	u32 fmbm_ifr;		/* Interrupt Force Register 0x28 */
 371	u32 res002c[5];		/* 0x2c - 0x3f */
 372	u32 fmbm_arb[8];	/* BMI Arbitration 0x40 - 0x5f */
 373	u32 res0060[12];	/* 0x60 - 0x8f */
 374	u32 fmbm_dtc[3];	/* Debug Trap Counter 0x90 - 0x9b */
 375	u32 res009c;		/* 0x9c */
 376	u32 fmbm_dcv[3][4];	/* Debug Compare val 0xa0-0xcf */
 377	u32 fmbm_dcm[3][4];	/* Debug Compare Mask 0xd0-0xff */
 378	u32 fmbm_gde;		/* BMI Global Debug Enable 0x100 */
 379	u32 fmbm_pp[63];	/* BMI Port Parameters 0x104 - 0x1ff */
 380	u32 res0200;		/* 0x200 */
 381	u32 fmbm_pfs[63];	/* BMI Port FIFO Size 0x204 - 0x2ff */
 382	u32 res0300;		/* 0x300 */
 383	u32 fmbm_spliodn[63];	/* Port Partition ID 0x304 - 0x3ff */
 384};
 385
 386struct fman_qmi_regs {
 387	u32 fmqm_gc;		/* General Configuration Register 0x00 */
 388	u32 res0004;		/* 0x04 */
 389	u32 fmqm_eie;		/* Error Interrupt Event Register 0x08 */
 390	u32 fmqm_eien;		/* Error Interrupt Enable Register 0x0c */
 391	u32 fmqm_eif;		/* Error Interrupt Force Register 0x10 */
 392	u32 fmqm_ie;		/* Interrupt Event Register 0x14 */
 393	u32 fmqm_ien;		/* Interrupt Enable Register 0x18 */
 394	u32 fmqm_if;		/* Interrupt Force Register 0x1c */
 395	u32 fmqm_gs;		/* Global Status Register 0x20 */
 396	u32 fmqm_ts;		/* Task Status Register 0x24 */
 397	u32 fmqm_etfc;		/* Enqueue Total Frame Counter 0x28 */
 398	u32 fmqm_dtfc;		/* Dequeue Total Frame Counter 0x2c */
 399	u32 fmqm_dc0;		/* Dequeue Counter 0 0x30 */
 400	u32 fmqm_dc1;		/* Dequeue Counter 1 0x34 */
 401	u32 fmqm_dc2;		/* Dequeue Counter 2 0x38 */
 402	u32 fmqm_dc3;		/* Dequeue Counter 3 0x3c */
 403	u32 fmqm_dfdc;		/* Dequeue FQID from Default Counter 0x40 */
 404	u32 fmqm_dfcc;		/* Dequeue FQID from Context Counter 0x44 */
 405	u32 fmqm_dffc;		/* Dequeue FQID from FD Counter 0x48 */
 406	u32 fmqm_dcc;		/* Dequeue Confirm Counter 0x4c */
 407	u32 res0050[7];		/* 0x50 - 0x6b */
 408	u32 fmqm_tapc;		/* Tnum Aging Period Control 0x6c */
 409	u32 fmqm_dmcvc;		/* Dequeue MAC Command Valid Counter 0x70 */
 410	u32 fmqm_difdcc;	/* Dequeue Invalid FD Command Counter 0x74 */
 411	u32 fmqm_da1v;		/* Dequeue A1 Valid Counter 0x78 */
 412	u32 res007c;		/* 0x7c */
 413	u32 fmqm_dtc;		/* 0x80 Debug Trap Counter 0x80 */
 414	u32 fmqm_efddd;		/* 0x84 Enqueue Frame desc Dynamic dbg 0x84 */
 415	u32 res0088[2];		/* 0x88 - 0x8f */
 416	struct {
 417		u32 fmqm_dtcfg1;	/* 0x90 dbg trap cfg 1 Register 0x00 */
 418		u32 fmqm_dtval1;	/* Debug Trap Value 1 Register 0x04 */
 419		u32 fmqm_dtm1;		/* Debug Trap Mask 1 Register 0x08 */
 420		u32 fmqm_dtc1;		/* Debug Trap Counter 1 Register 0x0c */
 421		u32 fmqm_dtcfg2;	/* dbg Trap cfg 2 Register 0x10 */
 422		u32 fmqm_dtval2;	/* Debug Trap Value 2 Register 0x14 */
 423		u32 fmqm_dtm2;		/* Debug Trap Mask 2 Register 0x18 */
 424		u32 res001c;		/* 0x1c */
 425	} dbg_traps[3];			/* 0x90 - 0xef */
 426	u8 res00f0[0x400 - 0xf0];	/* 0xf0 - 0x3ff */
 427};
 428
 429struct fman_dma_regs {
 430	u32 fmdmsr;	/* FM DMA status register 0x00 */
 431	u32 fmdmmr;	/* FM DMA mode register 0x04 */
 432	u32 fmdmtr;	/* FM DMA bus threshold register 0x08 */
 433	u32 fmdmhy;	/* FM DMA bus hysteresis register 0x0c */
 434	u32 fmdmsetr;	/* FM DMA SOS emergency Threshold Register 0x10 */
 435	u32 fmdmtah;	/* FM DMA transfer bus address high reg 0x14 */
 436	u32 fmdmtal;	/* FM DMA transfer bus address low reg 0x18 */
 437	u32 fmdmtcid;	/* FM DMA transfer bus communication ID reg 0x1c */
 438	u32 fmdmra;	/* FM DMA bus internal ram address register 0x20 */
 439	u32 fmdmrd;	/* FM DMA bus internal ram data register 0x24 */
 440	u32 fmdmwcr;	/* FM DMA CAM watchdog counter value 0x28 */
 441	u32 fmdmebcr;	/* FM DMA CAM base in MURAM register 0x2c */
 442	u32 fmdmccqdr;	/* FM DMA CAM and CMD Queue Debug reg 0x30 */
 443	u32 fmdmccqvr1;	/* FM DMA CAM and CMD Queue Value reg #1 0x34 */
 444	u32 fmdmccqvr2;	/* FM DMA CAM and CMD Queue Value reg #2 0x38 */
 445	u32 fmdmcqvr3;	/* FM DMA CMD Queue Value register #3 0x3c */
 446	u32 fmdmcqvr4;	/* FM DMA CMD Queue Value register #4 0x40 */
 447	u32 fmdmcqvr5;	/* FM DMA CMD Queue Value register #5 0x44 */
 448	u32 fmdmsefrc;	/* FM DMA Semaphore Entry Full Reject Cntr 0x48 */
 449	u32 fmdmsqfrc;	/* FM DMA Semaphore Queue Full Reject Cntr 0x4c */
 450	u32 fmdmssrc;	/* FM DMA Semaphore SYNC Reject Counter 0x50 */
 451	u32 fmdmdcr;	/* FM DMA Debug Counter 0x54 */
 452	u32 fmdmemsr;	/* FM DMA Emergency Smoother Register 0x58 */
 453	u32 res005c;	/* 0x5c */
 454	u32 fmdmplr[FMAN_LIODN_TBL / 2];	/* DMA LIODN regs 0x60-0xdf */
 455	u32 res00e0[0x400 - 56];
 456};
 457
 458struct fman_hwp_regs {
 459	u32 res0000[0x844 / 4];		/* 0x000..0x843 */
 460	u32 fmprrpimac;	/* FM Parser Internal memory access control */
 461	u32 res[(0x1000 - 0x848) / 4];	/* 0x848..0xFFF */
 462};
 463
 464/* Structure that holds current FMan state.
 465 * Used for saving run time information.
 466 */
 467struct fman_state_struct {
 468	u8 fm_id;
 469	u16 fm_clk_freq;
 470	struct fman_rev_info rev_info;
 471	bool enabled_time_stamp;
 472	u8 count1_micro_bit;
 473	u8 total_num_of_tasks;
 474	u8 accumulated_num_of_tasks;
 475	u32 accumulated_fifo_size;
 476	u8 accumulated_num_of_open_dmas;
 477	u8 accumulated_num_of_deq_tnums;
 478	u32 exceptions;
 479	u32 extra_fifo_pool_size;
 480	u8 extra_tasks_pool_size;
 481	u8 extra_open_dmas_pool_size;
 482	u16 port_mfl[MAX_NUM_OF_MACS];
 483	u16 mac_mfl[MAX_NUM_OF_MACS];
 484
 485	/* SOC specific */
 486	u32 fm_iram_size;
 487	/* DMA */
 488	u32 dma_thresh_max_commq;
 489	u32 dma_thresh_max_buf;
 490	u32 max_num_of_open_dmas;
 491	/* QMI */
 492	u32 qmi_max_num_of_tnums;
 493	u32 qmi_def_tnums_thresh;
 494	/* BMI */
 495	u32 bmi_max_num_of_tasks;
 496	u32 bmi_max_fifo_size;
 497	/* General */
 498	u32 fm_port_num_of_cg;
 499	u32 num_of_rx_ports;
 500	u32 total_fifo_size;
 501
 502	u32 qman_channel_base;
 503	u32 num_of_qman_channels;
 504
 505	struct resource *res;
 506};
 507
 508/* Structure that holds FMan initial configuration */
 509struct fman_cfg {
 510	u8 disp_limit_tsh;
 511	u8 prs_disp_tsh;
 512	u8 plcr_disp_tsh;
 513	u8 kg_disp_tsh;
 514	u8 bmi_disp_tsh;
 515	u8 qmi_enq_disp_tsh;
 516	u8 qmi_deq_disp_tsh;
 517	u8 fm_ctl1_disp_tsh;
 518	u8 fm_ctl2_disp_tsh;
 519	int dma_cache_override;
 520	enum fman_dma_aid_mode dma_aid_mode;
 521	u32 dma_axi_dbg_num_of_beats;
 522	u32 dma_cam_num_of_entries;
 523	u32 dma_watchdog;
 524	u8 dma_comm_qtsh_asrt_emer;
 525	u32 dma_write_buf_tsh_asrt_emer;
 526	u32 dma_read_buf_tsh_asrt_emer;
 527	u8 dma_comm_qtsh_clr_emer;
 528	u32 dma_write_buf_tsh_clr_emer;
 529	u32 dma_read_buf_tsh_clr_emer;
 530	u32 dma_sos_emergency;
 531	int dma_dbg_cnt_mode;
 532	int catastrophic_err;
 533	int dma_err;
 534	u32 exceptions;
 535	u16 clk_freq;
 536	u32 cam_base_addr;
 537	u32 fifo_base_addr;
 538	u32 total_fifo_size;
 539	u32 total_num_of_tasks;
 540	u32 qmi_def_tnums_thresh;
 541};
 542
 543#ifdef CONFIG_DPAA_ERRATUM_A050385
 544static bool fman_has_err_a050385;
 545#endif
 546
 547static irqreturn_t fman_exceptions(struct fman *fman,
 548				   enum fman_exceptions exception)
 549{
 550	dev_dbg(fman->dev, "%s: FMan[%d] exception %d\n",
 551		__func__, fman->state->fm_id, exception);
 552
 553	return IRQ_HANDLED;
 554}
 555
 556static irqreturn_t fman_bus_error(struct fman *fman, u8 __maybe_unused port_id,
 557				  u64 __maybe_unused addr,
 558				  u8 __maybe_unused tnum,
 559				  u16 __maybe_unused liodn)
 560{
 561	dev_dbg(fman->dev, "%s: FMan[%d] bus error: port_id[%d]\n",
 562		__func__, fman->state->fm_id, port_id);
 563
 564	return IRQ_HANDLED;
 565}
 566
 567static inline irqreturn_t call_mac_isr(struct fman *fman, u8 id)
 568{
 569	if (fman->intr_mng[id].isr_cb) {
 570		fman->intr_mng[id].isr_cb(fman->intr_mng[id].src_handle);
 571
 572		return IRQ_HANDLED;
 573	}
 574
 575	return IRQ_NONE;
 576}
 577
 578static inline u8 hw_port_id_to_sw_port_id(u8 major, u8 hw_port_id)
 579{
 580	u8 sw_port_id = 0;
 581
 582	if (hw_port_id >= BASE_TX_PORTID)
 583		sw_port_id = hw_port_id - BASE_TX_PORTID;
 584	else if (hw_port_id >= BASE_RX_PORTID)
 585		sw_port_id = hw_port_id - BASE_RX_PORTID;
 586	else
 587		sw_port_id = 0;
 588
 589	return sw_port_id;
 590}
 591
 592static void set_port_order_restoration(struct fman_fpm_regs __iomem *fpm_rg,
 593				       u8 port_id)
 594{
 595	u32 tmp = 0;
 596
 597	tmp = port_id << FPM_PORT_FM_CTL_PORTID_SHIFT;
 598
 599	tmp |= FPM_PRT_FM_CTL2 | FPM_PRT_FM_CTL1;
 600
 601	/* order restoration */
 602	if (port_id % 2)
 603		tmp |= FPM_PRT_FM_CTL1 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 604	else
 605		tmp |= FPM_PRT_FM_CTL2 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 606
 607	iowrite32be(tmp, &fpm_rg->fmfp_prc);
 608}
 609
 610static void set_port_liodn(struct fman *fman, u8 port_id,
 611			   u32 liodn_base, u32 liodn_ofst)
 612{
 613	u32 tmp;
 614
 615	iowrite32be(liodn_ofst, &fman->bmi_regs->fmbm_spliodn[port_id - 1]);
 616	if (!IS_ENABLED(CONFIG_FSL_PAMU))
 617		return;
 618	/* set LIODN base for this port */
 619	tmp = ioread32be(&fman->dma_regs->fmdmplr[port_id / 2]);
 620	if (port_id % 2) {
 621		tmp &= ~DMA_LIODN_BASE_MASK;
 622		tmp |= liodn_base;
 623	} else {
 624		tmp &= ~(DMA_LIODN_BASE_MASK << DMA_LIODN_SHIFT);
 625		tmp |= liodn_base << DMA_LIODN_SHIFT;
 626	}
 627	iowrite32be(tmp, &fman->dma_regs->fmdmplr[port_id / 2]);
 628}
 629
 630static void enable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 631{
 632	u32 tmp;
 633
 634	tmp = ioread32be(&fpm_rg->fm_rcr);
 635	if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 636		iowrite32be(tmp | FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 637	else
 638		iowrite32be(tmp | FPM_RAM_RAMS_ECC_EN |
 639			    FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 640}
 641
 642static void disable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 643{
 644	u32 tmp;
 645
 646	tmp = ioread32be(&fpm_rg->fm_rcr);
 647	if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 648		iowrite32be(tmp & ~FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 649	else
 650		iowrite32be(tmp & ~(FPM_RAM_RAMS_ECC_EN | FPM_RAM_IRAM_ECC_EN),
 651			    &fpm_rg->fm_rcr);
 652}
 653
 654static void fman_defconfig(struct fman_cfg *cfg)
 655{
 656	memset(cfg, 0, sizeof(struct fman_cfg));
 657
 658	cfg->catastrophic_err = DEFAULT_CATASTROPHIC_ERR;
 659	cfg->dma_err = DEFAULT_DMA_ERR;
 660	cfg->dma_aid_mode = DEFAULT_AID_MODE;
 661	cfg->dma_comm_qtsh_clr_emer = DEFAULT_DMA_COMM_Q_LOW;
 662	cfg->dma_comm_qtsh_asrt_emer = DEFAULT_DMA_COMM_Q_HIGH;
 663	cfg->dma_cache_override = DEFAULT_CACHE_OVERRIDE;
 664	cfg->dma_cam_num_of_entries = DEFAULT_DMA_CAM_NUM_OF_ENTRIES;
 665	cfg->dma_dbg_cnt_mode = DEFAULT_DMA_DBG_CNT_MODE;
 666	cfg->dma_sos_emergency = DEFAULT_DMA_SOS_EMERGENCY;
 667	cfg->dma_watchdog = DEFAULT_DMA_WATCHDOG;
 668	cfg->disp_limit_tsh = DEFAULT_DISP_LIMIT;
 669	cfg->prs_disp_tsh = DEFAULT_PRS_DISP_TH;
 670	cfg->plcr_disp_tsh = DEFAULT_PLCR_DISP_TH;
 671	cfg->kg_disp_tsh = DEFAULT_KG_DISP_TH;
 672	cfg->bmi_disp_tsh = DEFAULT_BMI_DISP_TH;
 673	cfg->qmi_enq_disp_tsh = DEFAULT_QMI_ENQ_DISP_TH;
 674	cfg->qmi_deq_disp_tsh = DEFAULT_QMI_DEQ_DISP_TH;
 675	cfg->fm_ctl1_disp_tsh = DEFAULT_FM_CTL1_DISP_TH;
 676	cfg->fm_ctl2_disp_tsh = DEFAULT_FM_CTL2_DISP_TH;
 677}
 678
 679static int dma_init(struct fman *fman)
 680{
 681	struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
 682	struct fman_cfg *cfg = fman->cfg;
 683	u32 tmp_reg;
 684
 685	/* Init DMA Registers */
 686
 687	/* clear status reg events */
 688	tmp_reg = (DMA_STATUS_BUS_ERR | DMA_STATUS_READ_ECC |
 689		   DMA_STATUS_SYSTEM_WRITE_ECC | DMA_STATUS_FM_WRITE_ECC);
 690	iowrite32be(ioread32be(&dma_rg->fmdmsr) | tmp_reg, &dma_rg->fmdmsr);
 691
 692	/* configure mode register */
 693	tmp_reg = 0;
 694	tmp_reg |= cfg->dma_cache_override << DMA_MODE_CACHE_OR_SHIFT;
 695	if (cfg->exceptions & EX_DMA_BUS_ERROR)
 696		tmp_reg |= DMA_MODE_BER;
 697	if ((cfg->exceptions & EX_DMA_SYSTEM_WRITE_ECC) |
 698	    (cfg->exceptions & EX_DMA_READ_ECC) |
 699	    (cfg->exceptions & EX_DMA_FM_WRITE_ECC))
 700		tmp_reg |= DMA_MODE_ECC;
 701	if (cfg->dma_axi_dbg_num_of_beats)
 702		tmp_reg |= (DMA_MODE_AXI_DBG_MASK &
 703			((cfg->dma_axi_dbg_num_of_beats - 1)
 704			<< DMA_MODE_AXI_DBG_SHIFT));
 705
 706	tmp_reg |= (((cfg->dma_cam_num_of_entries / DMA_CAM_UNITS) - 1) &
 707		DMA_MODE_CEN_MASK) << DMA_MODE_CEN_SHIFT;
 708	tmp_reg |= DMA_MODE_SECURE_PROT;
 709	tmp_reg |= cfg->dma_dbg_cnt_mode << DMA_MODE_DBG_SHIFT;
 710	tmp_reg |= cfg->dma_aid_mode << DMA_MODE_AID_MODE_SHIFT;
 711
 712	iowrite32be(tmp_reg, &dma_rg->fmdmmr);
 713
 714	/* configure thresholds register */
 715	tmp_reg = ((u32)cfg->dma_comm_qtsh_asrt_emer <<
 716		DMA_THRESH_COMMQ_SHIFT);
 717	tmp_reg |= (cfg->dma_read_buf_tsh_asrt_emer &
 718		DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 719	tmp_reg |= cfg->dma_write_buf_tsh_asrt_emer &
 720		DMA_THRESH_WRITE_INT_BUF_MASK;
 721
 722	iowrite32be(tmp_reg, &dma_rg->fmdmtr);
 723
 724	/* configure hysteresis register */
 725	tmp_reg = ((u32)cfg->dma_comm_qtsh_clr_emer <<
 726		DMA_THRESH_COMMQ_SHIFT);
 727	tmp_reg |= (cfg->dma_read_buf_tsh_clr_emer &
 728		DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 729	tmp_reg |= cfg->dma_write_buf_tsh_clr_emer &
 730		DMA_THRESH_WRITE_INT_BUF_MASK;
 731
 732	iowrite32be(tmp_reg, &dma_rg->fmdmhy);
 733
 734	/* configure emergency threshold */
 735	iowrite32be(cfg->dma_sos_emergency, &dma_rg->fmdmsetr);
 736
 737	/* configure Watchdog */
 738	iowrite32be((cfg->dma_watchdog * cfg->clk_freq), &dma_rg->fmdmwcr);
 739
 740	iowrite32be(cfg->cam_base_addr, &dma_rg->fmdmebcr);
 741
 742	/* Allocate MURAM for CAM */
 743	fman->cam_size =
 744		(u32)(fman->cfg->dma_cam_num_of_entries * DMA_CAM_SIZEOF_ENTRY);
 745	fman->cam_offset = fman_muram_alloc(fman->muram, fman->cam_size);
 746	if (IS_ERR_VALUE(fman->cam_offset)) {
 747		dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 748			__func__);
 749		return -ENOMEM;
 750	}
 751
 752	if (fman->state->rev_info.major == 2) {
 753		u32 __iomem *cam_base_addr;
 754
 755		fman_muram_free_mem(fman->muram, fman->cam_offset,
 756				    fman->cam_size);
 757
 758		fman->cam_size = fman->cfg->dma_cam_num_of_entries * 72 + 128;
 759		fman->cam_offset = fman_muram_alloc(fman->muram,
 760						    fman->cam_size);
 761		if (IS_ERR_VALUE(fman->cam_offset)) {
 762			dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 763				__func__);
 764			return -ENOMEM;
 765		}
 766
 767		if (fman->cfg->dma_cam_num_of_entries % 8 ||
 768		    fman->cfg->dma_cam_num_of_entries > 32) {
 769			dev_err(fman->dev, "%s: wrong dma_cam_num_of_entries\n",
 770				__func__);
 771			return -EINVAL;
 772		}
 773
 774		cam_base_addr = (u32 __iomem *)
 775			fman_muram_offset_to_vbase(fman->muram,
 776						   fman->cam_offset);
 777		iowrite32be(~((1 <<
 778			    (32 - fman->cfg->dma_cam_num_of_entries)) - 1),
 779			    cam_base_addr);
 780	}
 781
 782	fman->cfg->cam_base_addr = fman->cam_offset;
 783
 784	return 0;
 785}
 786
 787static void fpm_init(struct fman_fpm_regs __iomem *fpm_rg, struct fman_cfg *cfg)
 788{
 789	u32 tmp_reg;
 790	int i;
 791
 792	/* Init FPM Registers */
 793
 794	tmp_reg = (u32)(cfg->disp_limit_tsh << FPM_DISP_LIMIT_SHIFT);
 795	iowrite32be(tmp_reg, &fpm_rg->fmfp_mxd);
 796
 797	tmp_reg = (((u32)cfg->prs_disp_tsh << FPM_THR1_PRS_SHIFT) |
 798		   ((u32)cfg->kg_disp_tsh << FPM_THR1_KG_SHIFT) |
 799		   ((u32)cfg->plcr_disp_tsh << FPM_THR1_PLCR_SHIFT) |
 800		   ((u32)cfg->bmi_disp_tsh << FPM_THR1_BMI_SHIFT));
 801	iowrite32be(tmp_reg, &fpm_rg->fmfp_dist1);
 802
 803	tmp_reg =
 804		(((u32)cfg->qmi_enq_disp_tsh << FPM_THR2_QMI_ENQ_SHIFT) |
 805		 ((u32)cfg->qmi_deq_disp_tsh << FPM_THR2_QMI_DEQ_SHIFT) |
 806		 ((u32)cfg->fm_ctl1_disp_tsh << FPM_THR2_FM_CTL1_SHIFT) |
 807		 ((u32)cfg->fm_ctl2_disp_tsh << FPM_THR2_FM_CTL2_SHIFT));
 808	iowrite32be(tmp_reg, &fpm_rg->fmfp_dist2);
 809
 810	/* define exceptions and error behavior */
 811	tmp_reg = 0;
 812	/* Clear events */
 813	tmp_reg |= (FPM_EV_MASK_STALL | FPM_EV_MASK_DOUBLE_ECC |
 814		    FPM_EV_MASK_SINGLE_ECC);
 815	/* enable interrupts */
 816	if (cfg->exceptions & EX_FPM_STALL_ON_TASKS)
 817		tmp_reg |= FPM_EV_MASK_STALL_EN;
 818	if (cfg->exceptions & EX_FPM_SINGLE_ECC)
 819		tmp_reg |= FPM_EV_MASK_SINGLE_ECC_EN;
 820	if (cfg->exceptions & EX_FPM_DOUBLE_ECC)
 821		tmp_reg |= FPM_EV_MASK_DOUBLE_ECC_EN;
 822	tmp_reg |= (cfg->catastrophic_err << FPM_EV_MASK_CAT_ERR_SHIFT);
 823	tmp_reg |= (cfg->dma_err << FPM_EV_MASK_DMA_ERR_SHIFT);
 824	/* FMan is not halted upon external halt activation */
 825	tmp_reg |= FPM_EV_MASK_EXTERNAL_HALT;
 826	/* Man is not halted upon  Unrecoverable ECC error behavior */
 827	tmp_reg |= FPM_EV_MASK_ECC_ERR_HALT;
 828	iowrite32be(tmp_reg, &fpm_rg->fmfp_ee);
 829
 830	/* clear all fmCtls event registers */
 831	for (i = 0; i < FM_NUM_OF_FMAN_CTRL_EVENT_REGS; i++)
 832		iowrite32be(0xFFFFFFFF, &fpm_rg->fmfp_cev[i]);
 833
 834	/* RAM ECC -  enable and clear events */
 835	/* first we need to clear all parser memory,
 836	 * as it is uninitialized and may cause ECC errors
 837	 */
 838	/* event bits */
 839	tmp_reg = (FPM_RAM_MURAM_ECC | FPM_RAM_IRAM_ECC);
 840
 841	iowrite32be(tmp_reg, &fpm_rg->fm_rcr);
 842
 843	tmp_reg = 0;
 844	if (cfg->exceptions & EX_IRAM_ECC) {
 845		tmp_reg |= FPM_IRAM_ECC_ERR_EX_EN;
 846		enable_rams_ecc(fpm_rg);
 847	}
 848	if (cfg->exceptions & EX_MURAM_ECC) {
 849		tmp_reg |= FPM_MURAM_ECC_ERR_EX_EN;
 850		enable_rams_ecc(fpm_rg);
 851	}
 852	iowrite32be(tmp_reg, &fpm_rg->fm_rie);
 853}
 854
 855static void bmi_init(struct fman_bmi_regs __iomem *bmi_rg,
 856		     struct fman_cfg *cfg)
 857{
 858	u32 tmp_reg;
 859
 860	/* Init BMI Registers */
 861
 862	/* define common resources */
 863	tmp_reg = cfg->fifo_base_addr;
 864	tmp_reg = tmp_reg / BMI_FIFO_ALIGN;
 865
 866	tmp_reg |= ((cfg->total_fifo_size / FMAN_BMI_FIFO_UNITS - 1) <<
 867		    BMI_CFG1_FIFO_SIZE_SHIFT);
 868	iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg1);
 869
 870	tmp_reg = ((cfg->total_num_of_tasks - 1) & BMI_CFG2_TASKS_MASK) <<
 871		   BMI_CFG2_TASKS_SHIFT;
 872	/* num of DMA's will be dynamically updated when each port is set */
 873	iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg2);
 874
 875	/* define unmaskable exceptions, enable and clear events */
 876	tmp_reg = 0;
 877	iowrite32be(BMI_ERR_INTR_EN_LIST_RAM_ECC |
 878		    BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC |
 879		    BMI_ERR_INTR_EN_STATISTICS_RAM_ECC |
 880		    BMI_ERR_INTR_EN_DISPATCH_RAM_ECC, &bmi_rg->fmbm_ievr);
 881
 882	if (cfg->exceptions & EX_BMI_LIST_RAM_ECC)
 883		tmp_reg |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
 884	if (cfg->exceptions & EX_BMI_STORAGE_PROFILE_ECC)
 885		tmp_reg |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
 886	if (cfg->exceptions & EX_BMI_STATISTICS_RAM_ECC)
 887		tmp_reg |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
 888	if (cfg->exceptions & EX_BMI_DISPATCH_RAM_ECC)
 889		tmp_reg |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
 890	iowrite32be(tmp_reg, &bmi_rg->fmbm_ier);
 891}
 892
 893static void qmi_init(struct fman_qmi_regs __iomem *qmi_rg,
 894		     struct fman_cfg *cfg)
 895{
 896	u32 tmp_reg;
 897
 898	/* Init QMI Registers */
 899
 900	/* Clear error interrupt events */
 901
 902	iowrite32be(QMI_ERR_INTR_EN_DOUBLE_ECC | QMI_ERR_INTR_EN_DEQ_FROM_DEF,
 903		    &qmi_rg->fmqm_eie);
 904	tmp_reg = 0;
 905	if (cfg->exceptions & EX_QMI_DEQ_FROM_UNKNOWN_PORTID)
 906		tmp_reg |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
 907	if (cfg->exceptions & EX_QMI_DOUBLE_ECC)
 908		tmp_reg |= QMI_ERR_INTR_EN_DOUBLE_ECC;
 909	/* enable events */
 910	iowrite32be(tmp_reg, &qmi_rg->fmqm_eien);
 911
 912	tmp_reg = 0;
 913	/* Clear interrupt events */
 914	iowrite32be(QMI_INTR_EN_SINGLE_ECC, &qmi_rg->fmqm_ie);
 915	if (cfg->exceptions & EX_QMI_SINGLE_ECC)
 916		tmp_reg |= QMI_INTR_EN_SINGLE_ECC;
 917	/* enable events */
 918	iowrite32be(tmp_reg, &qmi_rg->fmqm_ien);
 919}
 920
 921static void hwp_init(struct fman_hwp_regs __iomem *hwp_rg)
 922{
 923	/* enable HW Parser */
 924	iowrite32be(HWP_RPIMAC_PEN, &hwp_rg->fmprrpimac);
 925}
 926
 927static int enable(struct fman *fman, struct fman_cfg *cfg)
 928{
 929	u32 cfg_reg = 0;
 930
 931	/* Enable all modules */
 932
 933	/* clear&enable global counters - calculate reg and save for later,
 934	 * because it's the same reg for QMI enable
 935	 */
 936	cfg_reg = QMI_CFG_EN_COUNTERS;
 937
 938	/* Set enqueue and dequeue thresholds */
 939	cfg_reg |= (cfg->qmi_def_tnums_thresh << 8) | cfg->qmi_def_tnums_thresh;
 940
 941	iowrite32be(BMI_INIT_START, &fman->bmi_regs->fmbm_init);
 942	iowrite32be(cfg_reg | QMI_CFG_ENQ_EN | QMI_CFG_DEQ_EN,
 943		    &fman->qmi_regs->fmqm_gc);
 944
 945	return 0;
 946}
 947
 948static int set_exception(struct fman *fman,
 949			 enum fman_exceptions exception, bool enable)
 950{
 951	u32 tmp;
 952
 953	switch (exception) {
 954	case FMAN_EX_DMA_BUS_ERROR:
 955		tmp = ioread32be(&fman->dma_regs->fmdmmr);
 956		if (enable)
 957			tmp |= DMA_MODE_BER;
 958		else
 959			tmp &= ~DMA_MODE_BER;
 960		/* disable bus error */
 961		iowrite32be(tmp, &fman->dma_regs->fmdmmr);
 962		break;
 963	case FMAN_EX_DMA_READ_ECC:
 964	case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
 965	case FMAN_EX_DMA_FM_WRITE_ECC:
 966		tmp = ioread32be(&fman->dma_regs->fmdmmr);
 967		if (enable)
 968			tmp |= DMA_MODE_ECC;
 969		else
 970			tmp &= ~DMA_MODE_ECC;
 971		iowrite32be(tmp, &fman->dma_regs->fmdmmr);
 972		break;
 973	case FMAN_EX_FPM_STALL_ON_TASKS:
 974		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
 975		if (enable)
 976			tmp |= FPM_EV_MASK_STALL_EN;
 977		else
 978			tmp &= ~FPM_EV_MASK_STALL_EN;
 979		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
 980		break;
 981	case FMAN_EX_FPM_SINGLE_ECC:
 982		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
 983		if (enable)
 984			tmp |= FPM_EV_MASK_SINGLE_ECC_EN;
 985		else
 986			tmp &= ~FPM_EV_MASK_SINGLE_ECC_EN;
 987		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
 988		break;
 989	case FMAN_EX_FPM_DOUBLE_ECC:
 990		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
 991		if (enable)
 992			tmp |= FPM_EV_MASK_DOUBLE_ECC_EN;
 993		else
 994			tmp &= ~FPM_EV_MASK_DOUBLE_ECC_EN;
 995		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
 996		break;
 997	case FMAN_EX_QMI_SINGLE_ECC:
 998		tmp = ioread32be(&fman->qmi_regs->fmqm_ien);
 999		if (enable)
1000			tmp |= QMI_INTR_EN_SINGLE_ECC;
1001		else
1002			tmp &= ~QMI_INTR_EN_SINGLE_ECC;
1003		iowrite32be(tmp, &fman->qmi_regs->fmqm_ien);
1004		break;
1005	case FMAN_EX_QMI_DOUBLE_ECC:
1006		tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1007		if (enable)
1008			tmp |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1009		else
1010			tmp &= ~QMI_ERR_INTR_EN_DOUBLE_ECC;
1011		iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1012		break;
1013	case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1014		tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1015		if (enable)
1016			tmp |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1017		else
1018			tmp &= ~QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1019		iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1020		break;
1021	case FMAN_EX_BMI_LIST_RAM_ECC:
1022		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1023		if (enable)
1024			tmp |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
1025		else
1026			tmp &= ~BMI_ERR_INTR_EN_LIST_RAM_ECC;
1027		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1028		break;
1029	case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1030		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1031		if (enable)
1032			tmp |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1033		else
1034			tmp &= ~BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1035		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1036		break;
1037	case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1038		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1039		if (enable)
1040			tmp |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1041		else
1042			tmp &= ~BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1043		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1044		break;
1045	case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1046		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1047		if (enable)
1048			tmp |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1049		else
1050			tmp &= ~BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1051		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1052		break;
1053	case FMAN_EX_IRAM_ECC:
1054		tmp = ioread32be(&fman->fpm_regs->fm_rie);
1055		if (enable) {
1056			/* enable ECC if not enabled */
1057			enable_rams_ecc(fman->fpm_regs);
1058			/* enable ECC interrupts */
1059			tmp |= FPM_IRAM_ECC_ERR_EX_EN;
1060		} else {
1061			/* ECC mechanism may be disabled,
1062			 * depending on driver status
1063			 */
1064			disable_rams_ecc(fman->fpm_regs);
1065			tmp &= ~FPM_IRAM_ECC_ERR_EX_EN;
1066		}
1067		iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1068		break;
1069	case FMAN_EX_MURAM_ECC:
1070		tmp = ioread32be(&fman->fpm_regs->fm_rie);
1071		if (enable) {
1072			/* enable ECC if not enabled */
1073			enable_rams_ecc(fman->fpm_regs);
1074			/* enable ECC interrupts */
1075			tmp |= FPM_MURAM_ECC_ERR_EX_EN;
1076		} else {
1077			/* ECC mechanism may be disabled,
1078			 * depending on driver status
1079			 */
1080			disable_rams_ecc(fman->fpm_regs);
1081			tmp &= ~FPM_MURAM_ECC_ERR_EX_EN;
1082		}
1083		iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1084		break;
1085	default:
1086		return -EINVAL;
1087	}
1088	return 0;
1089}
1090
1091static void resume(struct fman_fpm_regs __iomem *fpm_rg)
1092{
1093	u32 tmp;
1094
1095	tmp = ioread32be(&fpm_rg->fmfp_ee);
1096	/* clear tmp_reg event bits in order not to clear standing events */
1097	tmp &= ~(FPM_EV_MASK_DOUBLE_ECC |
1098		 FPM_EV_MASK_STALL | FPM_EV_MASK_SINGLE_ECC);
1099	tmp |= FPM_EV_MASK_RELEASE_FM;
1100
1101	iowrite32be(tmp, &fpm_rg->fmfp_ee);
1102}
1103
1104static int fill_soc_specific_params(struct fman_state_struct *state)
1105{
1106	u8 minor = state->rev_info.minor;
1107	/* P4080 - Major 2
1108	 * P2041/P3041/P5020/P5040 - Major 3
1109	 * Tx/Bx - Major 6
1110	 */
1111	switch (state->rev_info.major) {
1112	case 3:
1113		state->bmi_max_fifo_size	= 160 * 1024;
1114		state->fm_iram_size		= 64 * 1024;
1115		state->dma_thresh_max_commq	= 31;
1116		state->dma_thresh_max_buf	= 127;
1117		state->qmi_max_num_of_tnums	= 64;
1118		state->qmi_def_tnums_thresh	= 48;
1119		state->bmi_max_num_of_tasks	= 128;
1120		state->max_num_of_open_dmas	= 32;
1121		state->fm_port_num_of_cg	= 256;
1122		state->num_of_rx_ports	= 6;
1123		state->total_fifo_size	= 136 * 1024;
1124		break;
1125
1126	case 2:
1127		state->bmi_max_fifo_size	= 160 * 1024;
1128		state->fm_iram_size		= 64 * 1024;
1129		state->dma_thresh_max_commq	= 31;
1130		state->dma_thresh_max_buf	= 127;
1131		state->qmi_max_num_of_tnums	= 64;
1132		state->qmi_def_tnums_thresh	= 48;
1133		state->bmi_max_num_of_tasks	= 128;
1134		state->max_num_of_open_dmas	= 32;
1135		state->fm_port_num_of_cg	= 256;
1136		state->num_of_rx_ports	= 5;
1137		state->total_fifo_size	= 100 * 1024;
1138		break;
1139
1140	case 6:
1141		state->dma_thresh_max_commq	= 83;
1142		state->dma_thresh_max_buf	= 127;
1143		state->qmi_max_num_of_tnums	= 64;
1144		state->qmi_def_tnums_thresh	= 32;
1145		state->fm_port_num_of_cg	= 256;
1146
1147		/* FManV3L */
1148		if (minor == 1 || minor == 4) {
1149			state->bmi_max_fifo_size	= 192 * 1024;
1150			state->bmi_max_num_of_tasks	= 64;
1151			state->max_num_of_open_dmas	= 32;
1152			state->num_of_rx_ports		= 5;
1153			if (minor == 1)
1154				state->fm_iram_size	= 32 * 1024;
1155			else
1156				state->fm_iram_size	= 64 * 1024;
1157			state->total_fifo_size		= 156 * 1024;
1158		}
1159		/* FManV3H */
1160		else if (minor == 0 || minor == 2 || minor == 3) {
1161			state->bmi_max_fifo_size	= 384 * 1024;
1162			state->fm_iram_size		= 64 * 1024;
1163			state->bmi_max_num_of_tasks	= 128;
1164			state->max_num_of_open_dmas	= 84;
1165			state->num_of_rx_ports		= 8;
1166			state->total_fifo_size		= 295 * 1024;
1167		} else {
1168			pr_err("Unsupported FManv3 version\n");
1169			return -EINVAL;
1170		}
1171
1172		break;
1173	default:
1174		pr_err("Unsupported FMan version\n");
1175		return -EINVAL;
1176	}
1177
1178	return 0;
1179}
1180
1181static bool is_init_done(struct fman_cfg *cfg)
1182{
1183	/* Checks if FMan driver parameters were initialized */
1184	if (!cfg)
1185		return true;
1186
1187	return false;
1188}
1189
1190static void free_init_resources(struct fman *fman)
1191{
1192	if (fman->cam_offset)
1193		fman_muram_free_mem(fman->muram, fman->cam_offset,
1194				    fman->cam_size);
1195	if (fman->fifo_offset)
1196		fman_muram_free_mem(fman->muram, fman->fifo_offset,
1197				    fman->fifo_size);
1198}
1199
1200static irqreturn_t bmi_err_event(struct fman *fman)
1201{
1202	u32 event, mask, force;
1203	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1204	irqreturn_t ret = IRQ_NONE;
1205
1206	event = ioread32be(&bmi_rg->fmbm_ievr);
1207	mask = ioread32be(&bmi_rg->fmbm_ier);
1208	event &= mask;
1209	/* clear the forced events */
1210	force = ioread32be(&bmi_rg->fmbm_ifr);
1211	if (force & event)
1212		iowrite32be(force & ~event, &bmi_rg->fmbm_ifr);
1213	/* clear the acknowledged events */
1214	iowrite32be(event, &bmi_rg->fmbm_ievr);
1215
1216	if (event & BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC)
1217		ret = fman->exception_cb(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC);
1218	if (event & BMI_ERR_INTR_EN_LIST_RAM_ECC)
1219		ret = fman->exception_cb(fman, FMAN_EX_BMI_LIST_RAM_ECC);
1220	if (event & BMI_ERR_INTR_EN_STATISTICS_RAM_ECC)
1221		ret = fman->exception_cb(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC);
1222	if (event & BMI_ERR_INTR_EN_DISPATCH_RAM_ECC)
1223		ret = fman->exception_cb(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC);
1224
1225	return ret;
1226}
1227
1228static irqreturn_t qmi_err_event(struct fman *fman)
1229{
1230	u32 event, mask, force;
1231	struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1232	irqreturn_t ret = IRQ_NONE;
1233
1234	event = ioread32be(&qmi_rg->fmqm_eie);
1235	mask = ioread32be(&qmi_rg->fmqm_eien);
1236	event &= mask;
1237
1238	/* clear the forced events */
1239	force = ioread32be(&qmi_rg->fmqm_eif);
1240	if (force & event)
1241		iowrite32be(force & ~event, &qmi_rg->fmqm_eif);
1242	/* clear the acknowledged events */
1243	iowrite32be(event, &qmi_rg->fmqm_eie);
1244
1245	if (event & QMI_ERR_INTR_EN_DOUBLE_ECC)
1246		ret = fman->exception_cb(fman, FMAN_EX_QMI_DOUBLE_ECC);
1247	if (event & QMI_ERR_INTR_EN_DEQ_FROM_DEF)
1248		ret = fman->exception_cb(fman,
1249					 FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID);
1250
1251	return ret;
1252}
1253
1254static irqreturn_t dma_err_event(struct fman *fman)
1255{
1256	u32 status, mask, com_id;
1257	u8 tnum, port_id, relative_port_id;
1258	u16 liodn;
1259	struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
1260	irqreturn_t ret = IRQ_NONE;
1261
1262	status = ioread32be(&dma_rg->fmdmsr);
1263	mask = ioread32be(&dma_rg->fmdmmr);
1264
1265	/* clear DMA_STATUS_BUS_ERR if mask has no DMA_MODE_BER */
1266	if ((mask & DMA_MODE_BER) != DMA_MODE_BER)
1267		status &= ~DMA_STATUS_BUS_ERR;
1268
1269	/* clear relevant bits if mask has no DMA_MODE_ECC */
1270	if ((mask & DMA_MODE_ECC) != DMA_MODE_ECC)
1271		status &= ~(DMA_STATUS_FM_SPDAT_ECC |
1272			    DMA_STATUS_READ_ECC |
1273			    DMA_STATUS_SYSTEM_WRITE_ECC |
1274			    DMA_STATUS_FM_WRITE_ECC);
1275
1276	/* clear set events */
1277	iowrite32be(status, &dma_rg->fmdmsr);
1278
1279	if (status & DMA_STATUS_BUS_ERR) {
1280		u64 addr;
1281
1282		addr = (u64)ioread32be(&dma_rg->fmdmtal);
1283		addr |= ((u64)(ioread32be(&dma_rg->fmdmtah)) << 32);
1284
1285		com_id = ioread32be(&dma_rg->fmdmtcid);
1286		port_id = (u8)(((com_id & DMA_TRANSFER_PORTID_MASK) >>
1287			       DMA_TRANSFER_PORTID_SHIFT));
1288		relative_port_id =
1289		hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
1290		tnum = (u8)((com_id & DMA_TRANSFER_TNUM_MASK) >>
1291			    DMA_TRANSFER_TNUM_SHIFT);
1292		liodn = (u16)(com_id & DMA_TRANSFER_LIODN_MASK);
1293		ret = fman->bus_error_cb(fman, relative_port_id, addr, tnum,
1294					 liodn);
1295	}
1296	if (status & DMA_STATUS_FM_SPDAT_ECC)
1297		ret = fman->exception_cb(fman, FMAN_EX_DMA_SINGLE_PORT_ECC);
1298	if (status & DMA_STATUS_READ_ECC)
1299		ret = fman->exception_cb(fman, FMAN_EX_DMA_READ_ECC);
1300	if (status & DMA_STATUS_SYSTEM_WRITE_ECC)
1301		ret = fman->exception_cb(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC);
1302	if (status & DMA_STATUS_FM_WRITE_ECC)
1303		ret = fman->exception_cb(fman, FMAN_EX_DMA_FM_WRITE_ECC);
1304
1305	return ret;
1306}
1307
1308static irqreturn_t fpm_err_event(struct fman *fman)
1309{
1310	u32 event;
1311	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1312	irqreturn_t ret = IRQ_NONE;
1313
1314	event = ioread32be(&fpm_rg->fmfp_ee);
1315	/* clear the all occurred events */
1316	iowrite32be(event, &fpm_rg->fmfp_ee);
1317
1318	if ((event & FPM_EV_MASK_DOUBLE_ECC) &&
1319	    (event & FPM_EV_MASK_DOUBLE_ECC_EN))
1320		ret = fman->exception_cb(fman, FMAN_EX_FPM_DOUBLE_ECC);
1321	if ((event & FPM_EV_MASK_STALL) && (event & FPM_EV_MASK_STALL_EN))
1322		ret = fman->exception_cb(fman, FMAN_EX_FPM_STALL_ON_TASKS);
1323	if ((event & FPM_EV_MASK_SINGLE_ECC) &&
1324	    (event & FPM_EV_MASK_SINGLE_ECC_EN))
1325		ret = fman->exception_cb(fman, FMAN_EX_FPM_SINGLE_ECC);
1326
1327	return ret;
1328}
1329
1330static irqreturn_t muram_err_intr(struct fman *fman)
1331{
1332	u32 event, mask;
1333	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1334	irqreturn_t ret = IRQ_NONE;
1335
1336	event = ioread32be(&fpm_rg->fm_rcr);
1337	mask = ioread32be(&fpm_rg->fm_rie);
1338
1339	/* clear MURAM event bit (do not clear IRAM event) */
1340	iowrite32be(event & ~FPM_RAM_IRAM_ECC, &fpm_rg->fm_rcr);
1341
1342	if ((mask & FPM_MURAM_ECC_ERR_EX_EN) && (event & FPM_RAM_MURAM_ECC))
1343		ret = fman->exception_cb(fman, FMAN_EX_MURAM_ECC);
1344
1345	return ret;
1346}
1347
1348static irqreturn_t qmi_event(struct fman *fman)
1349{
1350	u32 event, mask, force;
1351	struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1352	irqreturn_t ret = IRQ_NONE;
1353
1354	event = ioread32be(&qmi_rg->fmqm_ie);
1355	mask = ioread32be(&qmi_rg->fmqm_ien);
1356	event &= mask;
1357	/* clear the forced events */
1358	force = ioread32be(&qmi_rg->fmqm_if);
1359	if (force & event)
1360		iowrite32be(force & ~event, &qmi_rg->fmqm_if);
1361	/* clear the acknowledged events */
1362	iowrite32be(event, &qmi_rg->fmqm_ie);
1363
1364	if (event & QMI_INTR_EN_SINGLE_ECC)
1365		ret = fman->exception_cb(fman, FMAN_EX_QMI_SINGLE_ECC);
1366
1367	return ret;
1368}
1369
1370static void enable_time_stamp(struct fman *fman)
1371{
1372	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1373	u16 fm_clk_freq = fman->state->fm_clk_freq;
1374	u32 tmp, intgr, ts_freq, frac;
1375
1376	ts_freq = (u32)(1 << fman->state->count1_micro_bit);
1377	/* configure timestamp so that bit 8 will count 1 microsecond
1378	 * Find effective count rate at TIMESTAMP least significant bits:
1379	 * Effective_Count_Rate = 1MHz x 2^8 = 256MHz
1380	 * Find frequency ratio between effective count rate and the clock:
1381	 * Effective_Count_Rate / CLK e.g. for 600 MHz clock:
1382	 * 256/600 = 0.4266666...
1383	 */
1384
1385	intgr = ts_freq / fm_clk_freq;
1386	/* we multiply by 2^16 to keep the fraction of the division
1387	 * we do not div back, since we write this value as a fraction
1388	 * see spec
1389	 */
1390
1391	frac = ((ts_freq << 16) - (intgr << 16) * fm_clk_freq) / fm_clk_freq;
1392	/* we check remainder of the division in order to round up if not int */
1393	if (((ts_freq << 16) - (intgr << 16) * fm_clk_freq) % fm_clk_freq)
1394		frac++;
1395
1396	tmp = (intgr << FPM_TS_INT_SHIFT) | (u16)frac;
1397	iowrite32be(tmp, &fpm_rg->fmfp_tsc2);
1398
1399	/* enable timestamp with original clock */
1400	iowrite32be(FPM_TS_CTL_EN, &fpm_rg->fmfp_tsc1);
1401	fman->state->enabled_time_stamp = true;
1402}
1403
1404static int clear_iram(struct fman *fman)
1405{
1406	struct fman_iram_regs __iomem *iram;
1407	int i, count;
1408
1409	iram = fman->base_addr + IMEM_OFFSET;
1410
1411	/* Enable the auto-increment */
1412	iowrite32be(IRAM_IADD_AIE, &iram->iadd);
1413	count = 100;
1414	do {
1415		udelay(1);
1416	} while ((ioread32be(&iram->iadd) != IRAM_IADD_AIE) && --count);
1417	if (count == 0)
1418		return -EBUSY;
1419
1420	for (i = 0; i < (fman->state->fm_iram_size / 4); i++)
1421		iowrite32be(0xffffffff, &iram->idata);
1422
1423	iowrite32be(fman->state->fm_iram_size - 4, &iram->iadd);
1424	count = 100;
1425	do {
1426		udelay(1);
1427	} while ((ioread32be(&iram->idata) != 0xffffffff) && --count);
1428	if (count == 0)
1429		return -EBUSY;
1430
1431	return 0;
1432}
1433
1434static u32 get_exception_flag(enum fman_exceptions exception)
1435{
1436	u32 bit_mask;
1437
1438	switch (exception) {
1439	case FMAN_EX_DMA_BUS_ERROR:
1440		bit_mask = EX_DMA_BUS_ERROR;
1441		break;
1442	case FMAN_EX_DMA_SINGLE_PORT_ECC:
1443		bit_mask = EX_DMA_SINGLE_PORT_ECC;
1444		break;
1445	case FMAN_EX_DMA_READ_ECC:
1446		bit_mask = EX_DMA_READ_ECC;
1447		break;
1448	case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1449		bit_mask = EX_DMA_SYSTEM_WRITE_ECC;
1450		break;
1451	case FMAN_EX_DMA_FM_WRITE_ECC:
1452		bit_mask = EX_DMA_FM_WRITE_ECC;
1453		break;
1454	case FMAN_EX_FPM_STALL_ON_TASKS:
1455		bit_mask = EX_FPM_STALL_ON_TASKS;
1456		break;
1457	case FMAN_EX_FPM_SINGLE_ECC:
1458		bit_mask = EX_FPM_SINGLE_ECC;
1459		break;
1460	case FMAN_EX_FPM_DOUBLE_ECC:
1461		bit_mask = EX_FPM_DOUBLE_ECC;
1462		break;
1463	case FMAN_EX_QMI_SINGLE_ECC:
1464		bit_mask = EX_QMI_SINGLE_ECC;
1465		break;
1466	case FMAN_EX_QMI_DOUBLE_ECC:
1467		bit_mask = EX_QMI_DOUBLE_ECC;
1468		break;
1469	case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1470		bit_mask = EX_QMI_DEQ_FROM_UNKNOWN_PORTID;
1471		break;
1472	case FMAN_EX_BMI_LIST_RAM_ECC:
1473		bit_mask = EX_BMI_LIST_RAM_ECC;
1474		break;
1475	case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1476		bit_mask = EX_BMI_STORAGE_PROFILE_ECC;
1477		break;
1478	case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1479		bit_mask = EX_BMI_STATISTICS_RAM_ECC;
1480		break;
1481	case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1482		bit_mask = EX_BMI_DISPATCH_RAM_ECC;
1483		break;
1484	case FMAN_EX_MURAM_ECC:
1485		bit_mask = EX_MURAM_ECC;
1486		break;
1487	default:
1488		bit_mask = 0;
1489		break;
1490	}
1491
1492	return bit_mask;
1493}
1494
1495static int get_module_event(enum fman_event_modules module, u8 mod_id,
1496			    enum fman_intr_type intr_type)
1497{
1498	int event;
1499
1500	switch (module) {
1501	case FMAN_MOD_MAC:
1502		if (intr_type == FMAN_INTR_TYPE_ERR)
1503			event = FMAN_EV_ERR_MAC0 + mod_id;
1504		else
1505			event = FMAN_EV_MAC0 + mod_id;
1506		break;
1507	case FMAN_MOD_FMAN_CTRL:
1508		if (intr_type == FMAN_INTR_TYPE_ERR)
1509			event = FMAN_EV_CNT;
1510		else
1511			event = (FMAN_EV_FMAN_CTRL_0 + mod_id);
1512		break;
1513	case FMAN_MOD_DUMMY_LAST:
1514		event = FMAN_EV_CNT;
1515		break;
1516	default:
1517		event = FMAN_EV_CNT;
1518		break;
1519	}
1520
1521	return event;
1522}
1523
1524static int set_size_of_fifo(struct fman *fman, u8 port_id, u32 *size_of_fifo,
1525			    u32 *extra_size_of_fifo)
1526{
1527	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1528	u32 fifo = *size_of_fifo;
1529	u32 extra_fifo = *extra_size_of_fifo;
1530	u32 tmp;
1531
1532	/* if this is the first time a port requires extra_fifo_pool_size,
1533	 * the total extra_fifo_pool_size must be initialized to 1 buffer per
1534	 * port
1535	 */
1536	if (extra_fifo && !fman->state->extra_fifo_pool_size)
1537		fman->state->extra_fifo_pool_size =
1538			fman->state->num_of_rx_ports * FMAN_BMI_FIFO_UNITS;
1539
1540	fman->state->extra_fifo_pool_size =
1541		max(fman->state->extra_fifo_pool_size, extra_fifo);
1542
1543	/* check that there are enough uncommitted fifo size */
1544	if ((fman->state->accumulated_fifo_size + fifo) >
1545	    (fman->state->total_fifo_size -
1546	    fman->state->extra_fifo_pool_size)) {
1547		dev_err(fman->dev, "%s: Requested fifo size and extra size exceed total FIFO size.\n",
1548			__func__);
1549		return -EAGAIN;
1550	}
1551
1552	/* Read, modify and write to HW */
1553	tmp = (fifo / FMAN_BMI_FIFO_UNITS - 1) |
1554	       ((extra_fifo / FMAN_BMI_FIFO_UNITS) <<
1555	       BMI_EXTRA_FIFO_SIZE_SHIFT);
1556	iowrite32be(tmp, &bmi_rg->fmbm_pfs[port_id - 1]);
1557
1558	/* update accumulated */
1559	fman->state->accumulated_fifo_size += fifo;
1560
1561	return 0;
1562}
1563
1564static int set_num_of_tasks(struct fman *fman, u8 port_id, u8 *num_of_tasks,
1565			    u8 *num_of_extra_tasks)
1566{
1567	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1568	u8 tasks = *num_of_tasks;
1569	u8 extra_tasks = *num_of_extra_tasks;
1570	u32 tmp;
1571
1572	if (extra_tasks)
1573		fman->state->extra_tasks_pool_size =
1574		max(fman->state->extra_tasks_pool_size, extra_tasks);
1575
1576	/* check that there are enough uncommitted tasks */
1577	if ((fman->state->accumulated_num_of_tasks + tasks) >
1578	    (fman->state->total_num_of_tasks -
1579	     fman->state->extra_tasks_pool_size)) {
1580		dev_err(fman->dev, "%s: Requested num_of_tasks and extra tasks pool for fm%d exceed total num_of_tasks.\n",
1581			__func__, fman->state->fm_id);
1582		return -EAGAIN;
1583	}
1584	/* update accumulated */
1585	fman->state->accumulated_num_of_tasks += tasks;
1586
1587	/* Write to HW */
1588	tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1589	    ~(BMI_NUM_OF_TASKS_MASK | BMI_NUM_OF_EXTRA_TASKS_MASK);
1590	tmp |= ((u32)((tasks - 1) << BMI_NUM_OF_TASKS_SHIFT) |
1591		(u32)(extra_tasks << BMI_EXTRA_NUM_OF_TASKS_SHIFT));
1592	iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1593
1594	return 0;
1595}
1596
1597static int set_num_of_open_dmas(struct fman *fman, u8 port_id,
1598				u8 *num_of_open_dmas,
1599				u8 *num_of_extra_open_dmas)
1600{
1601	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1602	u8 open_dmas = *num_of_open_dmas;
1603	u8 extra_open_dmas = *num_of_extra_open_dmas;
1604	u8 total_num_dmas = 0, current_val = 0, current_extra_val = 0;
1605	u32 tmp;
1606
1607	if (!open_dmas) {
1608		/* Configuration according to values in the HW.
1609		 * read the current number of open Dma's
1610		 */
1611		tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1612		current_extra_val = (u8)((tmp & BMI_NUM_OF_EXTRA_DMAS_MASK) >>
1613					 BMI_EXTRA_NUM_OF_DMAS_SHIFT);
1614
1615		tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1616		current_val = (u8)(((tmp & BMI_NUM_OF_DMAS_MASK) >>
1617				   BMI_NUM_OF_DMAS_SHIFT) + 1);
1618
1619		/* This is the first configuration and user did not
1620		 * specify value (!open_dmas), reset values will be used
1621		 * and we just save these values for resource management
1622		 */
1623		fman->state->extra_open_dmas_pool_size =
1624			(u8)max(fman->state->extra_open_dmas_pool_size,
1625				current_extra_val);
1626		fman->state->accumulated_num_of_open_dmas += current_val;
1627		*num_of_open_dmas = current_val;
1628		*num_of_extra_open_dmas = current_extra_val;
1629		return 0;
1630	}
1631
1632	if (extra_open_dmas > current_extra_val)
1633		fman->state->extra_open_dmas_pool_size =
1634		    (u8)max(fman->state->extra_open_dmas_pool_size,
1635			    extra_open_dmas);
1636
1637	if ((fman->state->rev_info.major < 6) &&
1638	    (fman->state->accumulated_num_of_open_dmas - current_val +
1639	     open_dmas > fman->state->max_num_of_open_dmas)) {
1640		dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds total num_of_open_dmas.\n",
1641			__func__, fman->state->fm_id);
1642		return -EAGAIN;
1643	} else if ((fman->state->rev_info.major >= 6) &&
1644		   !((fman->state->rev_info.major == 6) &&
1645		   (fman->state->rev_info.minor == 0)) &&
1646		   (fman->state->accumulated_num_of_open_dmas -
1647		   current_val + open_dmas >
1648		   fman->state->dma_thresh_max_commq + 1)) {
1649		dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds DMA Command queue (%d)\n",
1650			__func__, fman->state->fm_id,
1651		       fman->state->dma_thresh_max_commq + 1);
1652		return -EAGAIN;
1653	}
1654
1655	WARN_ON(fman->state->accumulated_num_of_open_dmas < current_val);
1656	/* update acummulated */
1657	fman->state->accumulated_num_of_open_dmas -= current_val;
1658	fman->state->accumulated_num_of_open_dmas += open_dmas;
1659
1660	if (fman->state->rev_info.major < 6)
1661		total_num_dmas =
1662		    (u8)(fman->state->accumulated_num_of_open_dmas +
1663		    fman->state->extra_open_dmas_pool_size);
1664
1665	/* calculate reg */
1666	tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1667	    ~(BMI_NUM_OF_DMAS_MASK | BMI_NUM_OF_EXTRA_DMAS_MASK);
1668	tmp |= (u32)(((open_dmas - 1) << BMI_NUM_OF_DMAS_SHIFT) |
1669			   (extra_open_dmas << BMI_EXTRA_NUM_OF_DMAS_SHIFT));
1670	iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1671
1672	/* update total num of DMA's with committed number of open DMAS,
1673	 * and max uncommitted pool.
1674	 */
1675	if (total_num_dmas) {
1676		tmp = ioread32be(&bmi_rg->fmbm_cfg2) & ~BMI_CFG2_DMAS_MASK;
1677		tmp |= (u32)(total_num_dmas - 1) << BMI_CFG2_DMAS_SHIFT;
1678		iowrite32be(tmp, &bmi_rg->fmbm_cfg2);
1679	}
1680
1681	return 0;
1682}
1683
1684static int fman_config(struct fman *fman)
1685{
1686	void __iomem *base_addr;
1687	int err;
1688
1689	base_addr = fman->dts_params.base_addr;
1690
1691	fman->state = kzalloc(sizeof(*fman->state), GFP_KERNEL);
1692	if (!fman->state)
1693		goto err_fm_state;
1694
1695	/* Allocate the FM driver's parameters structure */
1696	fman->cfg = kzalloc(sizeof(*fman->cfg), GFP_KERNEL);
1697	if (!fman->cfg)
1698		goto err_fm_drv;
1699
1700	/* Initialize MURAM block */
1701	fman->muram =
1702		fman_muram_init(fman->dts_params.muram_res.start,
1703				resource_size(&fman->dts_params.muram_res));
1704	if (!fman->muram)
1705		goto err_fm_soc_specific;
1706
1707	/* Initialize FM parameters which will be kept by the driver */
1708	fman->state->fm_id = fman->dts_params.id;
1709	fman->state->fm_clk_freq = fman->dts_params.clk_freq;
1710	fman->state->qman_channel_base = fman->dts_params.qman_channel_base;
1711	fman->state->num_of_qman_channels =
1712		fman->dts_params.num_of_qman_channels;
1713	fman->state->res = fman->dts_params.res;
1714	fman->exception_cb = fman_exceptions;
1715	fman->bus_error_cb = fman_bus_error;
1716	fman->fpm_regs = base_addr + FPM_OFFSET;
1717	fman->bmi_regs = base_addr + BMI_OFFSET;
1718	fman->qmi_regs = base_addr + QMI_OFFSET;
1719	fman->dma_regs = base_addr + DMA_OFFSET;
1720	fman->hwp_regs = base_addr + HWP_OFFSET;
1721	fman->kg_regs = base_addr + KG_OFFSET;
1722	fman->base_addr = base_addr;
1723
1724	spin_lock_init(&fman->spinlock);
1725	fman_defconfig(fman->cfg);
1726
1727	fman->state->extra_fifo_pool_size = 0;
1728	fman->state->exceptions = (EX_DMA_BUS_ERROR                 |
1729					EX_DMA_READ_ECC              |
1730					EX_DMA_SYSTEM_WRITE_ECC      |
1731					EX_DMA_FM_WRITE_ECC          |
1732					EX_FPM_STALL_ON_TASKS        |
1733					EX_FPM_SINGLE_ECC            |
1734					EX_FPM_DOUBLE_ECC            |
1735					EX_QMI_DEQ_FROM_UNKNOWN_PORTID |
1736					EX_BMI_LIST_RAM_ECC          |
1737					EX_BMI_STORAGE_PROFILE_ECC   |
1738					EX_BMI_STATISTICS_RAM_ECC    |
1739					EX_MURAM_ECC                 |
1740					EX_BMI_DISPATCH_RAM_ECC      |
1741					EX_QMI_DOUBLE_ECC            |
1742					EX_QMI_SINGLE_ECC);
1743
1744	/* Read FMan revision for future use*/
1745	fman_get_revision(fman, &fman->state->rev_info);
1746
1747	err = fill_soc_specific_params(fman->state);
1748	if (err)
1749		goto err_fm_soc_specific;
1750
1751	/* FM_AID_MODE_NO_TNUM_SW005 Errata workaround */
1752	if (fman->state->rev_info.major >= 6)
1753		fman->cfg->dma_aid_mode = FMAN_DMA_AID_OUT_PORT_ID;
1754
1755	fman->cfg->qmi_def_tnums_thresh = fman->state->qmi_def_tnums_thresh;
1756
1757	fman->state->total_num_of_tasks =
1758	(u8)DFLT_TOTAL_NUM_OF_TASKS(fman->state->rev_info.major,
1759				    fman->state->rev_info.minor,
1760				    fman->state->bmi_max_num_of_tasks);
1761
1762	if (fman->state->rev_info.major < 6) {
1763		fman->cfg->dma_comm_qtsh_clr_emer =
1764		(u8)DFLT_DMA_COMM_Q_LOW(fman->state->rev_info.major,
1765					fman->state->dma_thresh_max_commq);
1766
1767		fman->cfg->dma_comm_qtsh_asrt_emer =
1768		(u8)DFLT_DMA_COMM_Q_HIGH(fman->state->rev_info.major,
1769					 fman->state->dma_thresh_max_commq);
1770
1771		fman->cfg->dma_cam_num_of_entries =
1772		DFLT_DMA_CAM_NUM_OF_ENTRIES(fman->state->rev_info.major);
1773
1774		fman->cfg->dma_read_buf_tsh_clr_emer =
1775		DFLT_DMA_READ_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1776
1777		fman->cfg->dma_read_buf_tsh_asrt_emer =
1778		DFLT_DMA_READ_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1779
1780		fman->cfg->dma_write_buf_tsh_clr_emer =
1781		DFLT_DMA_WRITE_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1782
1783		fman->cfg->dma_write_buf_tsh_asrt_emer =
1784		DFLT_DMA_WRITE_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1785
1786		fman->cfg->dma_axi_dbg_num_of_beats =
1787		DFLT_AXI_DBG_NUM_OF_BEATS;
1788	}
1789
1790	return 0;
1791
1792err_fm_soc_specific:
1793	kfree(fman->cfg);
1794err_fm_drv:
1795	kfree(fman->state);
1796err_fm_state:
1797	kfree(fman);
1798	return -EINVAL;
1799}
1800
1801static int fman_reset(struct fman *fman)
1802{
1803	u32 count;
1804	int err = 0;
1805
1806	if (fman->state->rev_info.major < 6) {
1807		iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1808		/* Wait for reset completion */
1809		count = 100;
1810		do {
1811			udelay(1);
1812		} while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1813			 FPM_RSTC_FM_RESET) && --count);
1814		if (count == 0)
1815			err = -EBUSY;
1816
1817		goto _return;
1818	} else {
1819#ifdef CONFIG_PPC
1820		struct device_node *guts_node;
1821		struct ccsr_guts __iomem *guts_regs;
1822		u32 devdisr2, reg;
1823
1824		/* Errata A007273 */
1825		guts_node =
1826			of_find_compatible_node(NULL, NULL,
1827						"fsl,qoriq-device-config-2.0");
1828		if (!guts_node) {
1829			dev_err(fman->dev, "%s: Couldn't find guts node\n",
1830				__func__);
1831			goto guts_node;
1832		}
1833
1834		guts_regs = of_iomap(guts_node, 0);
1835		if (!guts_regs) {
1836			dev_err(fman->dev, "%s: Couldn't map %pOF regs\n",
1837				__func__, guts_node);
1838			goto guts_regs;
1839		}
1840#define FMAN1_ALL_MACS_MASK	0xFCC00000
1841#define FMAN2_ALL_MACS_MASK	0x000FCC00
1842		/* Read current state */
1843		devdisr2 = ioread32be(&guts_regs->devdisr2);
1844		if (fman->dts_params.id == 0)
1845			reg = devdisr2 & ~FMAN1_ALL_MACS_MASK;
1846		else
1847			reg = devdisr2 & ~FMAN2_ALL_MACS_MASK;
1848
1849		/* Enable all MACs */
1850		iowrite32be(reg, &guts_regs->devdisr2);
1851#endif
1852
1853		/* Perform FMan reset */
1854		iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1855
1856		/* Wait for reset completion */
1857		count = 100;
1858		do {
1859			udelay(1);
1860		} while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1861			 FPM_RSTC_FM_RESET) && --count);
1862		if (count == 0) {
1863#ifdef CONFIG_PPC
1864			iounmap(guts_regs);
1865			of_node_put(guts_node);
1866#endif
1867			err = -EBUSY;
1868			goto _return;
1869		}
1870#ifdef CONFIG_PPC
1871
1872		/* Restore devdisr2 value */
1873		iowrite32be(devdisr2, &guts_regs->devdisr2);
1874
1875		iounmap(guts_regs);
1876		of_node_put(guts_node);
1877#endif
1878
1879		goto _return;
1880
1881#ifdef CONFIG_PPC
1882guts_regs:
1883		of_node_put(guts_node);
1884guts_node:
1885		dev_dbg(fman->dev, "%s: Didn't perform FManV3 reset due to Errata A007273!\n",
1886			__func__);
1887#endif
1888	}
1889_return:
1890	return err;
1891}
1892
1893static int fman_init(struct fman *fman)
1894{
1895	struct fman_cfg *cfg = NULL;
1896	int err = 0, i, count;
1897
1898	if (is_init_done(fman->cfg))
1899		return -EINVAL;
1900
1901	fman->state->count1_micro_bit = FM_TIMESTAMP_1_USEC_BIT;
1902
1903	cfg = fman->cfg;
1904
1905	/* clear revision-dependent non existing exception */
1906	if (fman->state->rev_info.major < 6)
1907		fman->state->exceptions &= ~FMAN_EX_BMI_DISPATCH_RAM_ECC;
1908
1909	if (fman->state->rev_info.major >= 6)
1910		fman->state->exceptions &= ~FMAN_EX_QMI_SINGLE_ECC;
1911
1912	/* clear CPG */
1913	memset_io((void __iomem *)(fman->base_addr + CGP_OFFSET), 0,
1914		  fman->state->fm_port_num_of_cg);
1915
1916	/* Save LIODN info before FMan reset
1917	 * Skipping non-existent port 0 (i = 1)
1918	 */
1919	for (i = 1; i < FMAN_LIODN_TBL; i++) {
1920		u32 liodn_base;
1921
1922		fman->liodn_offset[i] =
1923			ioread32be(&fman->bmi_regs->fmbm_spliodn[i - 1]);
1924		if (!IS_ENABLED(CONFIG_FSL_PAMU))
1925			continue;
1926		liodn_base = ioread32be(&fman->dma_regs->fmdmplr[i / 2]);
1927		if (i % 2) {
1928			/* FMDM_PLR LSB holds LIODN base for odd ports */
1929			liodn_base &= DMA_LIODN_BASE_MASK;
1930		} else {
1931			/* FMDM_PLR MSB holds LIODN base for even ports */
1932			liodn_base >>= DMA_LIODN_SHIFT;
1933			liodn_base &= DMA_LIODN_BASE_MASK;
1934		}
1935		fman->liodn_base[i] = liodn_base;
1936	}
1937
1938	err = fman_reset(fman);
1939	if (err)
1940		return err;
1941
1942	if (ioread32be(&fman->qmi_regs->fmqm_gs) & QMI_GS_HALT_NOT_BUSY) {
1943		resume(fman->fpm_regs);
1944		/* Wait until QMI is not in halt not busy state */
1945		count = 100;
1946		do {
1947			udelay(1);
1948		} while (((ioread32be(&fman->qmi_regs->fmqm_gs)) &
1949			 QMI_GS_HALT_NOT_BUSY) && --count);
1950		if (count == 0)
1951			dev_warn(fman->dev, "%s: QMI is in halt not busy state\n",
1952				 __func__);
1953	}
1954
1955	if (clear_iram(fman) != 0)
1956		return -EINVAL;
1957
1958	cfg->exceptions = fman->state->exceptions;
1959
1960	/* Init DMA Registers */
1961
1962	err = dma_init(fman);
1963	if (err != 0) {
1964		free_init_resources(fman);
1965		return err;
1966	}
1967
1968	/* Init FPM Registers */
1969	fpm_init(fman->fpm_regs, fman->cfg);
1970
1971	/* define common resources */
1972	/* allocate MURAM for FIFO according to total size */
1973	fman->fifo_offset = fman_muram_alloc(fman->muram,
1974					     fman->state->total_fifo_size);
1975	if (IS_ERR_VALUE(fman->fifo_offset)) {
1976		free_init_resources(fman);
1977		dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
1978			__func__);
1979		return -ENOMEM;
1980	}
1981
1982	cfg->fifo_base_addr = fman->fifo_offset;
1983	cfg->total_fifo_size = fman->state->total_fifo_size;
1984	cfg->total_num_of_tasks = fman->state->total_num_of_tasks;
1985	cfg->clk_freq = fman->state->fm_clk_freq;
1986
1987	/* Init BMI Registers */
1988	bmi_init(fman->bmi_regs, fman->cfg);
1989
1990	/* Init QMI Registers */
1991	qmi_init(fman->qmi_regs, fman->cfg);
1992
1993	/* Init HW Parser */
1994	hwp_init(fman->hwp_regs);
1995
1996	/* Init KeyGen */
1997	fman->keygen = keygen_init(fman->kg_regs);
1998	if (!fman->keygen)
1999		return -EINVAL;
2000
2001	err = enable(fman, cfg);
2002	if (err != 0)
2003		return err;
2004
2005	enable_time_stamp(fman);
2006
2007	kfree(fman->cfg);
2008	fman->cfg = NULL;
2009
2010	return 0;
2011}
2012
2013static int fman_set_exception(struct fman *fman,
2014			      enum fman_exceptions exception, bool enable)
2015{
2016	u32 bit_mask = 0;
2017
2018	if (!is_init_done(fman->cfg))
2019		return -EINVAL;
2020
2021	bit_mask = get_exception_flag(exception);
2022	if (bit_mask) {
2023		if (enable)
2024			fman->state->exceptions |= bit_mask;
2025		else
2026			fman->state->exceptions &= ~bit_mask;
2027	} else {
2028		dev_err(fman->dev, "%s: Undefined exception (%d)\n",
2029			__func__, exception);
2030		return -EINVAL;
2031	}
2032
2033	return set_exception(fman, exception, enable);
2034}
2035
2036/**
2037 * fman_register_intr
2038 * @fman:	A Pointer to FMan device
2039 * @module:	Calling module
2040 * @mod_id:	Module id (if more than 1 exists, '0' if not)
2041 * @intr_type:	Interrupt type (error/normal) selection.
2042 * @isr_cb:	The interrupt service routine.
2043 * @src_arg:	Argument to be passed to isr_cb.
2044 *
2045 * Used to register an event handler to be processed by FMan
2046 *
2047 * Return: 0 on success; Error code otherwise.
2048 */
2049void fman_register_intr(struct fman *fman, enum fman_event_modules module,
2050			u8 mod_id, enum fman_intr_type intr_type,
2051			void (*isr_cb)(void *src_arg), void *src_arg)
2052{
2053	int event = 0;
2054
2055	event = get_module_event(module, mod_id, intr_type);
2056	WARN_ON(event >= FMAN_EV_CNT);
2057
2058	/* register in local FM structure */
2059	fman->intr_mng[event].isr_cb = isr_cb;
2060	fman->intr_mng[event].src_handle = src_arg;
2061}
2062EXPORT_SYMBOL(fman_register_intr);
2063
2064/**
2065 * fman_unregister_intr
2066 * @fman:	A Pointer to FMan device
2067 * @module:	Calling module
2068 * @mod_id:	Module id (if more than 1 exists, '0' if not)
2069 * @intr_type:	Interrupt type (error/normal) selection.
2070 *
2071 * Used to unregister an event handler to be processed by FMan
2072 *
2073 * Return: 0 on success; Error code otherwise.
2074 */
2075void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
2076			  u8 mod_id, enum fman_intr_type intr_type)
2077{
2078	int event = 0;
2079
2080	event = get_module_event(module, mod_id, intr_type);
2081	WARN_ON(event >= FMAN_EV_CNT);
2082
2083	fman->intr_mng[event].isr_cb = NULL;
2084	fman->intr_mng[event].src_handle = NULL;
2085}
2086EXPORT_SYMBOL(fman_unregister_intr);
2087
2088/**
2089 * fman_set_port_params
2090 * @fman:		A Pointer to FMan device
2091 * @port_params:	Port parameters
2092 *
2093 * Used by FMan Port to pass parameters to the FMan
2094 *
2095 * Return: 0 on success; Error code otherwise.
2096 */
2097int fman_set_port_params(struct fman *fman,
2098			 struct fman_port_init_params *port_params)
2099{
2100	int err;
2101	unsigned long flags;
2102	u8 port_id = port_params->port_id, mac_id;
2103
2104	spin_lock_irqsave(&fman->spinlock, flags);
2105
2106	err = set_num_of_tasks(fman, port_params->port_id,
2107			       &port_params->num_of_tasks,
2108			       &port_params->num_of_extra_tasks);
2109	if (err)
2110		goto return_err;
2111
2112	/* TX Ports */
2113	if (port_params->port_type != FMAN_PORT_TYPE_RX) {
2114		u32 enq_th, deq_th, reg;
2115
2116		/* update qmi ENQ/DEQ threshold */
2117		fman->state->accumulated_num_of_deq_tnums +=
2118			port_params->deq_pipeline_depth;
2119		enq_th = (ioread32be(&fman->qmi_regs->fmqm_gc) &
2120			  QMI_CFG_ENQ_MASK) >> QMI_CFG_ENQ_SHIFT;
2121		/* if enq_th is too big, we reduce it to the max value
2122		 * that is still 0
2123		 */
2124		if (enq_th >= (fman->state->qmi_max_num_of_tnums -
2125		    fman->state->accumulated_num_of_deq_tnums)) {
2126			enq_th =
2127			fman->state->qmi_max_num_of_tnums -
2128			fman->state->accumulated_num_of_deq_tnums - 1;
2129
2130			reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2131			reg &= ~QMI_CFG_ENQ_MASK;
2132			reg |= (enq_th << QMI_CFG_ENQ_SHIFT);
2133			iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2134		}
2135
2136		deq_th = ioread32be(&fman->qmi_regs->fmqm_gc) &
2137				    QMI_CFG_DEQ_MASK;
2138		/* if deq_th is too small, we enlarge it to the min
2139		 * value that is still 0.
2140		 * depTh may not be larger than 63
2141		 * (fman->state->qmi_max_num_of_tnums-1).
2142		 */
2143		if ((deq_th <= fman->state->accumulated_num_of_deq_tnums) &&
2144		    (deq_th < fman->state->qmi_max_num_of_tnums - 1)) {
2145			deq_th = fman->state->accumulated_num_of_deq_tnums + 1;
2146			reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2147			reg &= ~QMI_CFG_DEQ_MASK;
2148			reg |= deq_th;
2149			iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2150		}
2151	}
2152
2153	err = set_size_of_fifo(fman, port_params->port_id,
2154			       &port_params->size_of_fifo,
2155			       &port_params->extra_size_of_fifo);
2156	if (err)
2157		goto return_err;
2158
2159	err = set_num_of_open_dmas(fman, port_params->port_id,
2160				   &port_params->num_of_open_dmas,
2161				   &port_params->num_of_extra_open_dmas);
2162	if (err)
2163		goto return_err;
2164
2165	set_port_liodn(fman, port_id, fman->liodn_base[port_id],
2166		       fman->liodn_offset[port_id]);
2167
2168	if (fman->state->rev_info.major < 6)
2169		set_port_order_restoration(fman->fpm_regs, port_id);
2170
2171	mac_id = hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
2172
2173	if (port_params->max_frame_length >= fman->state->mac_mfl[mac_id]) {
2174		fman->state->port_mfl[mac_id] = port_params->max_frame_length;
2175	} else {
2176		dev_warn(fman->dev, "%s: Port (%d) max_frame_length is smaller than MAC (%d) current MTU\n",
2177			 __func__, port_id, mac_id);
2178		err = -EINVAL;
2179		goto return_err;
2180	}
2181
2182	spin_unlock_irqrestore(&fman->spinlock, flags);
2183
2184	return 0;
2185
2186return_err:
2187	spin_unlock_irqrestore(&fman->spinlock, flags);
2188	return err;
2189}
2190EXPORT_SYMBOL(fman_set_port_params);
2191
2192/**
2193 * fman_reset_mac
2194 * @fman:	A Pointer to FMan device
2195 * @mac_id:	MAC id to be reset
2196 *
2197 * Reset a specific MAC
2198 *
2199 * Return: 0 on success; Error code otherwise.
2200 */
2201int fman_reset_mac(struct fman *fman, u8 mac_id)
2202{
2203	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
2204	u32 msk, timeout = 100;
2205
2206	if (fman->state->rev_info.major >= 6) {
2207		dev_err(fman->dev, "%s: FMan MAC reset no available for FMan V3!\n",
2208			__func__);
2209		return -EINVAL;
2210	}
2211
2212	/* Get the relevant bit mask */
2213	switch (mac_id) {
2214	case 0:
2215		msk = FPM_RSTC_MAC0_RESET;
2216		break;
2217	case 1:
2218		msk = FPM_RSTC_MAC1_RESET;
2219		break;
2220	case 2:
2221		msk = FPM_RSTC_MAC2_RESET;
2222		break;
2223	case 3:
2224		msk = FPM_RSTC_MAC3_RESET;
2225		break;
2226	case 4:
2227		msk = FPM_RSTC_MAC4_RESET;
2228		break;
2229	case 5:
2230		msk = FPM_RSTC_MAC5_RESET;
2231		break;
2232	case 6:
2233		msk = FPM_RSTC_MAC6_RESET;
2234		break;
2235	case 7:
2236		msk = FPM_RSTC_MAC7_RESET;
2237		break;
2238	case 8:
2239		msk = FPM_RSTC_MAC8_RESET;
2240		break;
2241	case 9:
2242		msk = FPM_RSTC_MAC9_RESET;
2243		break;
2244	default:
2245		dev_warn(fman->dev, "%s: Illegal MAC Id [%d]\n",
2246			 __func__, mac_id);
2247		return -EINVAL;
2248	}
2249
2250	/* reset */
2251	iowrite32be(msk, &fpm_rg->fm_rstc);
2252	while ((ioread32be(&fpm_rg->fm_rstc) & msk) && --timeout)
2253		udelay(10);
2254
2255	if (!timeout)
2256		return -EIO;
2257
2258	return 0;
2259}
2260EXPORT_SYMBOL(fman_reset_mac);
2261
2262/**
2263 * fman_set_mac_max_frame
2264 * @fman:	A Pointer to FMan device
2265 * @mac_id:	MAC id
2266 * @mfl:	Maximum frame length
2267 *
2268 * Set maximum frame length of specific MAC in FMan driver
2269 *
2270 * Return: 0 on success; Error code otherwise.
2271 */
2272int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
2273{
2274	/* if port is already initialized, check that MaxFrameLength is smaller
2275	 * or equal to the port's max
2276	 */
2277	if ((!fman->state->port_mfl[mac_id]) ||
2278	    (mfl <= fman->state->port_mfl[mac_id])) {
2279		fman->state->mac_mfl[mac_id] = mfl;
2280	} else {
2281		dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
2282			 __func__);
2283		return -EINVAL;
2284	}
2285	return 0;
2286}
2287EXPORT_SYMBOL(fman_set_mac_max_frame);
2288
2289/**
2290 * fman_get_clock_freq
2291 * @fman:	A Pointer to FMan device
2292 *
2293 * Get FMan clock frequency
2294 *
2295 * Return: FMan clock frequency
2296 */
2297u16 fman_get_clock_freq(struct fman *fman)
2298{
2299	return fman->state->fm_clk_freq;
2300}
2301
2302/**
2303 * fman_get_bmi_max_fifo_size
2304 * @fman:	A Pointer to FMan device
2305 *
2306 * Get FMan maximum FIFO size
2307 *
2308 * Return: FMan Maximum FIFO size
2309 */
2310u32 fman_get_bmi_max_fifo_size(struct fman *fman)
2311{
2312	return fman->state->bmi_max_fifo_size;
2313}
2314EXPORT_SYMBOL(fman_get_bmi_max_fifo_size);
2315
2316/**
2317 * fman_get_revision
2318 * @fman:		- Pointer to the FMan module
2319 * @rev_info:		- A structure of revision information parameters.
2320 *
2321 * Returns the FM revision
2322 *
2323 * Allowed only following fman_init().
2324 *
2325 * Return: 0 on success; Error code otherwise.
2326 */
2327void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
2328{
2329	u32 tmp;
2330
2331	tmp = ioread32be(&fman->fpm_regs->fm_ip_rev_1);
2332	rev_info->major = (u8)((tmp & FPM_REV1_MAJOR_MASK) >>
2333				FPM_REV1_MAJOR_SHIFT);
2334	rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
2335}
2336EXPORT_SYMBOL(fman_get_revision);
2337
2338/**
2339 * fman_get_qman_channel_id
2340 * @fman:	A Pointer to FMan device
2341 * @port_id:	Port id
2342 *
2343 * Get QMan channel ID associated to the Port id
2344 *
2345 * Return: QMan channel ID
2346 */
2347u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
2348{
2349	int i;
2350
2351	if (fman->state->rev_info.major >= 6) {
2352		static const u32 port_ids[] = {
2353			0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
2354			0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2355		};
2356
2357		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2358			if (port_ids[i] == port_id)
2359				break;
2360		}
2361	} else {
2362		static const u32 port_ids[] = {
2363			0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
2364			0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2365		};
2366
2367		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2368			if (port_ids[i] == port_id)
2369				break;
2370		}
2371	}
2372
2373	if (i == fman->state->num_of_qman_channels)
2374		return 0;
2375
2376	return fman->state->qman_channel_base + i;
2377}
2378EXPORT_SYMBOL(fman_get_qman_channel_id);
2379
2380/**
2381 * fman_get_mem_region
2382 * @fman:	A Pointer to FMan device
2383 *
2384 * Get FMan memory region
2385 *
2386 * Return: A structure with FMan memory region information
2387 */
2388struct resource *fman_get_mem_region(struct fman *fman)
2389{
2390	return fman->state->res;
2391}
2392EXPORT_SYMBOL(fman_get_mem_region);
2393
2394/* Bootargs defines */
2395/* Extra headroom for RX buffers - Default, min and max */
2396#define FSL_FM_RX_EXTRA_HEADROOM	64
2397#define FSL_FM_RX_EXTRA_HEADROOM_MIN	16
2398#define FSL_FM_RX_EXTRA_HEADROOM_MAX	384
2399
2400/* Maximum frame length */
2401#define FSL_FM_MAX_FRAME_SIZE			1522
2402#define FSL_FM_MAX_POSSIBLE_FRAME_SIZE		9600
2403#define FSL_FM_MIN_POSSIBLE_FRAME_SIZE		64
2404
2405/* Extra headroom for Rx buffers.
2406 * FMan is instructed to allocate, on the Rx path, this amount of
2407 * space at the beginning of a data buffer, beside the DPA private
2408 * data area and the IC fields.
2409 * Does not impact Tx buffer layout.
2410 * Configurable from bootargs. 64 by default, it's needed on
2411 * particular forwarding scenarios that add extra headers to the
2412 * forwarded frame.
2413 */
2414static int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2415module_param(fsl_fm_rx_extra_headroom, int, 0);
2416MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
2417
2418/* Max frame size, across all interfaces.
2419 * Configurable from bootargs, to avoid allocating oversized (socket)
2420 * buffers when not using jumbo frames.
2421 * Must be large enough to accommodate the network MTU, but small enough
2422 * to avoid wasting skb memory.
2423 */
2424static int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2425module_param(fsl_fm_max_frm, int, 0);
2426MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
2427
2428/**
2429 * fman_get_max_frm
2430 *
2431 * Return: Max frame length configured in the FM driver
2432 */
2433u16 fman_get_max_frm(void)
2434{
2435	static bool fm_check_mfl;
2436
2437	if (!fm_check_mfl) {
2438		if (fsl_fm_max_frm > FSL_FM_MAX_POSSIBLE_FRAME_SIZE ||
2439		    fsl_fm_max_frm < FSL_FM_MIN_POSSIBLE_FRAME_SIZE) {
2440			pr_warn("Invalid fsl_fm_max_frm value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2441				fsl_fm_max_frm,
2442				FSL_FM_MIN_POSSIBLE_FRAME_SIZE,
2443				FSL_FM_MAX_POSSIBLE_FRAME_SIZE,
2444				FSL_FM_MAX_FRAME_SIZE);
2445			fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2446		}
2447		fm_check_mfl = true;
2448	}
2449
2450	return fsl_fm_max_frm;
2451}
2452EXPORT_SYMBOL(fman_get_max_frm);
2453
2454/**
2455 * fman_get_rx_extra_headroom
2456 *
2457 * Return: Extra headroom size configured in the FM driver
2458 */
2459int fman_get_rx_extra_headroom(void)
2460{
2461	static bool fm_check_rx_extra_headroom;
2462
2463	if (!fm_check_rx_extra_headroom) {
2464		if (fsl_fm_rx_extra_headroom > FSL_FM_RX_EXTRA_HEADROOM_MAX ||
2465		    fsl_fm_rx_extra_headroom < FSL_FM_RX_EXTRA_HEADROOM_MIN) {
2466			pr_warn("Invalid fsl_fm_rx_extra_headroom value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2467				fsl_fm_rx_extra_headroom,
2468				FSL_FM_RX_EXTRA_HEADROOM_MIN,
2469				FSL_FM_RX_EXTRA_HEADROOM_MAX,
2470				FSL_FM_RX_EXTRA_HEADROOM);
2471			fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2472		}
2473
2474		fm_check_rx_extra_headroom = true;
2475		fsl_fm_rx_extra_headroom = ALIGN(fsl_fm_rx_extra_headroom, 16);
2476	}
2477
2478	return fsl_fm_rx_extra_headroom;
2479}
2480EXPORT_SYMBOL(fman_get_rx_extra_headroom);
2481
2482/**
2483 * fman_bind
2484 * @fm_dev:	FMan OF device pointer
2485 *
2486 * Bind to a specific FMan device.
2487 *
2488 * Allowed only after the port was created.
2489 *
2490 * Return: A pointer to the FMan device
2491 */
2492struct fman *fman_bind(struct device *fm_dev)
2493{
2494	return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
2495}
2496EXPORT_SYMBOL(fman_bind);
2497
2498#ifdef CONFIG_DPAA_ERRATUM_A050385
2499bool fman_has_errata_a050385(void)
2500{
2501	return fman_has_err_a050385;
2502}
2503EXPORT_SYMBOL(fman_has_errata_a050385);
2504#endif
2505
2506static irqreturn_t fman_err_irq(int irq, void *handle)
2507{
2508	struct fman *fman = (struct fman *)handle;
2509	u32 pending;
2510	struct fman_fpm_regs __iomem *fpm_rg;
2511	irqreturn_t single_ret, ret = IRQ_NONE;
2512
2513	if (!is_init_done(fman->cfg))
2514		return IRQ_NONE;
2515
2516	fpm_rg = fman->fpm_regs;
2517
2518	/* error interrupts */
2519	pending = ioread32be(&fpm_rg->fm_epi);
2520	if (!pending)
2521		return IRQ_NONE;
2522
2523	if (pending & ERR_INTR_EN_BMI) {
2524		single_ret = bmi_err_event(fman);
2525		if (single_ret == IRQ_HANDLED)
2526			ret = IRQ_HANDLED;
2527	}
2528	if (pending & ERR_INTR_EN_QMI) {
2529		single_ret = qmi_err_event(fman);
2530		if (single_ret == IRQ_HANDLED)
2531			ret = IRQ_HANDLED;
2532	}
2533	if (pending & ERR_INTR_EN_FPM) {
2534		single_ret = fpm_err_event(fman);
2535		if (single_ret == IRQ_HANDLED)
2536			ret = IRQ_HANDLED;
2537	}
2538	if (pending & ERR_INTR_EN_DMA) {
2539		single_ret = dma_err_event(fman);
2540		if (single_ret == IRQ_HANDLED)
2541			ret = IRQ_HANDLED;
2542	}
2543	if (pending & ERR_INTR_EN_MURAM) {
2544		single_ret = muram_err_intr(fman);
2545		if (single_ret == IRQ_HANDLED)
2546			ret = IRQ_HANDLED;
2547	}
2548
2549	/* MAC error interrupts */
2550	if (pending & ERR_INTR_EN_MAC0) {
2551		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 0);
2552		if (single_ret == IRQ_HANDLED)
2553			ret = IRQ_HANDLED;
2554	}
2555	if (pending & ERR_INTR_EN_MAC1) {
2556		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 1);
2557		if (single_ret == IRQ_HANDLED)
2558			ret = IRQ_HANDLED;
2559	}
2560	if (pending & ERR_INTR_EN_MAC2) {
2561		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 2);
2562		if (single_ret == IRQ_HANDLED)
2563			ret = IRQ_HANDLED;
2564	}
2565	if (pending & ERR_INTR_EN_MAC3) {
2566		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 3);
2567		if (single_ret == IRQ_HANDLED)
2568			ret = IRQ_HANDLED;
2569	}
2570	if (pending & ERR_INTR_EN_MAC4) {
2571		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 4);
2572		if (single_ret == IRQ_HANDLED)
2573			ret = IRQ_HANDLED;
2574	}
2575	if (pending & ERR_INTR_EN_MAC5) {
2576		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 5);
2577		if (single_ret == IRQ_HANDLED)
2578			ret = IRQ_HANDLED;
2579	}
2580	if (pending & ERR_INTR_EN_MAC6) {
2581		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 6);
2582		if (single_ret == IRQ_HANDLED)
2583			ret = IRQ_HANDLED;
2584	}
2585	if (pending & ERR_INTR_EN_MAC7) {
2586		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 7);
2587		if (single_ret == IRQ_HANDLED)
2588			ret = IRQ_HANDLED;
2589	}
2590	if (pending & ERR_INTR_EN_MAC8) {
2591		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 8);
2592		if (single_ret == IRQ_HANDLED)
2593			ret = IRQ_HANDLED;
2594	}
2595	if (pending & ERR_INTR_EN_MAC9) {
2596		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 9);
2597		if (single_ret == IRQ_HANDLED)
2598			ret = IRQ_HANDLED;
2599	}
2600
2601	return ret;
2602}
2603
2604static irqreturn_t fman_irq(int irq, void *handle)
2605{
2606	struct fman *fman = (struct fman *)handle;
2607	u32 pending;
2608	struct fman_fpm_regs __iomem *fpm_rg;
2609	irqreturn_t single_ret, ret = IRQ_NONE;
2610
2611	if (!is_init_done(fman->cfg))
2612		return IRQ_NONE;
2613
2614	fpm_rg = fman->fpm_regs;
2615
2616	/* normal interrupts */
2617	pending = ioread32be(&fpm_rg->fm_npi);
2618	if (!pending)
2619		return IRQ_NONE;
2620
2621	if (pending & INTR_EN_QMI) {
2622		single_ret = qmi_event(fman);
2623		if (single_ret == IRQ_HANDLED)
2624			ret = IRQ_HANDLED;
2625	}
2626
2627	/* MAC interrupts */
2628	if (pending & INTR_EN_MAC0) {
2629		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 0);
2630		if (single_ret == IRQ_HANDLED)
2631			ret = IRQ_HANDLED;
2632	}
2633	if (pending & INTR_EN_MAC1) {
2634		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 1);
2635		if (single_ret == IRQ_HANDLED)
2636			ret = IRQ_HANDLED;
2637	}
2638	if (pending & INTR_EN_MAC2) {
2639		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 2);
2640		if (single_ret == IRQ_HANDLED)
2641			ret = IRQ_HANDLED;
2642	}
2643	if (pending & INTR_EN_MAC3) {
2644		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 3);
2645		if (single_ret == IRQ_HANDLED)
2646			ret = IRQ_HANDLED;
2647	}
2648	if (pending & INTR_EN_MAC4) {
2649		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 4);
2650		if (single_ret == IRQ_HANDLED)
2651			ret = IRQ_HANDLED;
2652	}
2653	if (pending & INTR_EN_MAC5) {
2654		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 5);
2655		if (single_ret == IRQ_HANDLED)
2656			ret = IRQ_HANDLED;
2657	}
2658	if (pending & INTR_EN_MAC6) {
2659		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 6);
2660		if (single_ret == IRQ_HANDLED)
2661			ret = IRQ_HANDLED;
2662	}
2663	if (pending & INTR_EN_MAC7) {
2664		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 7);
2665		if (single_ret == IRQ_HANDLED)
2666			ret = IRQ_HANDLED;
2667	}
2668	if (pending & INTR_EN_MAC8) {
2669		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 8);
2670		if (single_ret == IRQ_HANDLED)
2671			ret = IRQ_HANDLED;
2672	}
2673	if (pending & INTR_EN_MAC9) {
2674		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 9);
2675		if (single_ret == IRQ_HANDLED)
2676			ret = IRQ_HANDLED;
2677	}
2678
2679	return ret;
2680}
2681
2682static const struct of_device_id fman_muram_match[] = {
2683	{
2684		.compatible = "fsl,fman-muram"},
2685	{}
2686};
2687MODULE_DEVICE_TABLE(of, fman_muram_match);
2688
2689static struct fman *read_dts_node(struct platform_device *of_dev)
2690{
2691	struct fman *fman;
2692	struct device_node *fm_node, *muram_node;
2693	void __iomem *base_addr;
2694	struct resource *res;
2695	u32 val, range[2];
2696	int err, irq;
2697	struct clk *clk;
2698	u32 clk_rate;
2699
2700	fman = kzalloc(sizeof(*fman), GFP_KERNEL);
2701	if (!fman)
2702		return ERR_PTR(-ENOMEM);
2703
2704	fm_node = of_node_get(of_dev->dev.of_node);
2705
2706	err = of_property_read_u32(fm_node, "cell-index", &val);
2707	if (err) {
2708		dev_err(&of_dev->dev, "%s: failed to read cell-index for %pOF\n",
2709			__func__, fm_node);
2710		goto fman_node_put;
2711	}
2712	fman->dts_params.id = (u8)val;
2713
2714	/* Get the FM interrupt */
2715	err = platform_get_irq(of_dev, 0);
2716	if (err < 0)
2717		goto fman_node_put;
2718	irq = err;
2719
2720	/* Get the FM error interrupt */
2721	err = platform_get_irq(of_dev, 1);
2722	if (err < 0)
2723		goto fman_node_put;
2724	fman->dts_params.err_irq = err;
2725
2726	clk = of_clk_get(fm_node, 0);
2727	if (IS_ERR(clk)) {
2728		err = PTR_ERR(clk);
2729		dev_err(&of_dev->dev, "%s: Failed to get FM%d clock structure\n",
2730			__func__, fman->dts_params.id);
2731		goto fman_node_put;
2732	}
2733
2734	clk_rate = clk_get_rate(clk);
2735	if (!clk_rate) {
2736		err = -EINVAL;
2737		dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
2738			__func__, fman->dts_params.id);
2739		goto fman_node_put;
2740	}
2741	/* Rounding to MHz */
2742	fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
2743
2744	err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
2745					 &range[0], 2);
2746	if (err) {
2747		dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %pOF\n",
2748			__func__, fm_node);
2749		goto fman_node_put;
2750	}
2751	fman->dts_params.qman_channel_base = range[0];
2752	fman->dts_params.num_of_qman_channels = range[1];
2753
2754	/* Get the MURAM base address and size */
2755	muram_node = of_find_matching_node(fm_node, fman_muram_match);
2756	if (!muram_node) {
2757		err = -EINVAL;
2758		dev_err(&of_dev->dev, "%s: could not find MURAM node\n",
2759			__func__);
2760		goto fman_free;
2761	}
2762
2763	err = of_address_to_resource(muram_node, 0,
2764				     &fman->dts_params.muram_res);
2765	if (err) {
2766		of_node_put(muram_node);
2767		dev_err(&of_dev->dev, "%s: of_address_to_resource() = %d\n",
2768			__func__, err);
2769		goto fman_free;
2770	}
2771
2772	of_node_put(muram_node);
2773
2774	err = devm_request_irq(&of_dev->dev, irq, fman_irq, IRQF_SHARED,
2775			       "fman", fman);
2776	if (err < 0) {
2777		dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2778			__func__, irq, err);
2779		goto fman_free;
2780	}
2781
2782	if (fman->dts_params.err_irq != 0) {
2783		err = devm_request_irq(&of_dev->dev, fman->dts_params.err_irq,
2784				       fman_err_irq, IRQF_SHARED,
2785				       "fman-err", fman);
2786		if (err < 0) {
2787			dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2788				__func__, fman->dts_params.err_irq, err);
2789			goto fman_free;
2790		}
2791	}
2792
2793	base_addr = devm_platform_get_and_ioremap_resource(of_dev, 0, &res);
2794	if (IS_ERR(base_addr)) {
2795		err = PTR_ERR(base_addr);
2796		dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
2797		goto fman_free;
2798	}
2799
2800	fman->dts_params.base_addr = base_addr;
2801	fman->dts_params.res = res;
2802
2803	fman->dev = &of_dev->dev;
2804
2805	err = of_platform_populate(fm_node, NULL, NULL, &of_dev->dev);
2806	if (err) {
2807		dev_err(&of_dev->dev, "%s: of_platform_populate() failed\n",
2808			__func__);
2809		goto fman_free;
2810	}
2811
2812#ifdef CONFIG_DPAA_ERRATUM_A050385
2813	fman_has_err_a050385 =
2814		of_property_read_bool(fm_node, "fsl,erratum-a050385");
2815#endif
2816
2817	return fman;
2818
2819fman_node_put:
2820	of_node_put(fm_node);
2821fman_free:
2822	kfree(fman);
2823	return ERR_PTR(err);
2824}
2825
2826static int fman_probe(struct platform_device *of_dev)
2827{
2828	struct fman *fman;
2829	struct device *dev;
2830	int err;
2831
2832	dev = &of_dev->dev;
2833
2834	fman = read_dts_node(of_dev);
2835	if (IS_ERR(fman))
2836		return PTR_ERR(fman);
2837
2838	err = fman_config(fman);
2839	if (err) {
2840		dev_err(dev, "%s: FMan config failed\n", __func__);
2841		return -EINVAL;
2842	}
2843
2844	if (fman_init(fman) != 0) {
2845		dev_err(dev, "%s: FMan init failed\n", __func__);
2846		return -EINVAL;
2847	}
2848
2849	if (fman->dts_params.err_irq == 0) {
2850		fman_set_exception(fman, FMAN_EX_DMA_BUS_ERROR, false);
2851		fman_set_exception(fman, FMAN_EX_DMA_READ_ECC, false);
2852		fman_set_exception(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC, false);
2853		fman_set_exception(fman, FMAN_EX_DMA_FM_WRITE_ECC, false);
2854		fman_set_exception(fman, FMAN_EX_DMA_SINGLE_PORT_ECC, false);
2855		fman_set_exception(fman, FMAN_EX_FPM_STALL_ON_TASKS, false);
2856		fman_set_exception(fman, FMAN_EX_FPM_SINGLE_ECC, false);
2857		fman_set_exception(fman, FMAN_EX_FPM_DOUBLE_ECC, false);
2858		fman_set_exception(fman, FMAN_EX_QMI_SINGLE_ECC, false);
2859		fman_set_exception(fman, FMAN_EX_QMI_DOUBLE_ECC, false);
2860		fman_set_exception(fman,
2861				   FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID, false);
2862		fman_set_exception(fman, FMAN_EX_BMI_LIST_RAM_ECC, false);
2863		fman_set_exception(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC,
2864				   false);
2865		fman_set_exception(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC, false);
2866		fman_set_exception(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC, false);
2867	}
2868
2869	dev_set_drvdata(dev, fman);
2870
2871	dev_dbg(dev, "FMan%d probed\n", fman->dts_params.id);
2872
2873	return 0;
2874}
2875
2876static const struct of_device_id fman_match[] = {
2877	{
2878		.compatible = "fsl,fman"},
2879	{}
2880};
2881
2882MODULE_DEVICE_TABLE(of, fman_match);
2883
2884static struct platform_driver fman_driver = {
2885	.driver = {
2886		.name = "fsl-fman",
2887		.of_match_table = fman_match,
2888	},
2889	.probe = fman_probe,
2890};
2891
2892static int __init fman_load(void)
2893{
2894	int err;
2895
2896	pr_debug("FSL DPAA FMan driver\n");
2897
2898	err = platform_driver_register(&fman_driver);
2899	if (err < 0)
2900		pr_err("Error, platform_driver_register() = %d\n", err);
2901
2902	return err;
2903}
2904module_init(fman_load);
2905
2906static void __exit fman_unload(void)
2907{
2908	platform_driver_unregister(&fman_driver);
2909}
2910module_exit(fman_unload);
2911
2912MODULE_LICENSE("Dual BSD/GPL");
2913MODULE_DESCRIPTION("Freescale DPAA Frame Manager driver");