type.h

   1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2/* Copyright (c) 2015 - 2021 Intel Corporation */
   3#ifndef IRDMA_TYPE_H
   4#define IRDMA_TYPE_H
   5#include "osdep.h"
   6#include "irdma.h"
   7#include "user.h"
   8#include "hmc.h"
   9#include "uda.h"
  10#include "ws.h"
  11#include "virtchnl.h"
  12
  13#define IRDMA_DEBUG_ERR		"ERR"
  14#define IRDMA_DEBUG_INIT	"INIT"
  15#define IRDMA_DEBUG_DEV		"DEV"
  16#define IRDMA_DEBUG_CM		"CM"
  17#define IRDMA_DEBUG_VERBS	"VERBS"
  18#define IRDMA_DEBUG_PUDA	"PUDA"
  19#define IRDMA_DEBUG_ILQ		"ILQ"
  20#define IRDMA_DEBUG_IEQ		"IEQ"
  21#define IRDMA_DEBUG_QP		"QP"
  22#define IRDMA_DEBUG_CQ		"CQ"
  23#define IRDMA_DEBUG_MR		"MR"
  24#define IRDMA_DEBUG_PBLE	"PBLE"
  25#define IRDMA_DEBUG_WQE		"WQE"
  26#define IRDMA_DEBUG_AEQ		"AEQ"
  27#define IRDMA_DEBUG_CQP		"CQP"
  28#define IRDMA_DEBUG_HMC		"HMC"
  29#define IRDMA_DEBUG_USER	"USER"
  30#define IRDMA_DEBUG_VIRT	"VIRT"
  31#define IRDMA_DEBUG_DCB		"DCB"
  32#define	IRDMA_DEBUG_CQE		"CQE"
  33#define IRDMA_DEBUG_CLNT	"CLNT"
  34#define IRDMA_DEBUG_WS		"WS"
  35#define IRDMA_DEBUG_STATS	"STATS"
  36
  37enum irdma_page_size {
  38	IRDMA_PAGE_SIZE_4K = 0,
  39	IRDMA_PAGE_SIZE_2M,
  40	IRDMA_PAGE_SIZE_1G,
  41};
  42
  43enum irdma_hdrct_flags {
  44	DDP_LEN_FLAG  = 0x80,
  45	DDP_HDR_FLAG  = 0x40,
  46	RDMA_HDR_FLAG = 0x20,
  47};
  48
  49enum irdma_term_layers {
  50	LAYER_RDMA = 0,
  51	LAYER_DDP  = 1,
  52	LAYER_MPA  = 2,
  53};
  54
  55enum irdma_term_error_types {
  56	RDMAP_REMOTE_PROT = 1,
  57	RDMAP_REMOTE_OP   = 2,
  58	DDP_CATASTROPHIC  = 0,
  59	DDP_TAGGED_BUF    = 1,
  60	DDP_UNTAGGED_BUF  = 2,
  61	DDP_LLP		  = 3,
  62};
  63
  64enum irdma_term_rdma_errors {
  65	RDMAP_INV_STAG		  = 0x00,
  66	RDMAP_INV_BOUNDS	  = 0x01,
  67	RDMAP_ACCESS		  = 0x02,
  68	RDMAP_UNASSOC_STAG	  = 0x03,
  69	RDMAP_TO_WRAP		  = 0x04,
  70	RDMAP_INV_RDMAP_VER       = 0x05,
  71	RDMAP_UNEXPECTED_OP       = 0x06,
  72	RDMAP_CATASTROPHIC_LOCAL  = 0x07,
  73	RDMAP_CATASTROPHIC_GLOBAL = 0x08,
  74	RDMAP_CANT_INV_STAG       = 0x09,
  75	RDMAP_UNSPECIFIED	  = 0xff,
  76};
  77
  78enum irdma_term_ddp_errors {
  79	DDP_CATASTROPHIC_LOCAL      = 0x00,
  80	DDP_TAGGED_INV_STAG	    = 0x00,
  81	DDP_TAGGED_BOUNDS	    = 0x01,
  82	DDP_TAGGED_UNASSOC_STAG     = 0x02,
  83	DDP_TAGGED_TO_WRAP	    = 0x03,
  84	DDP_TAGGED_INV_DDP_VER      = 0x04,
  85	DDP_UNTAGGED_INV_QN	    = 0x01,
  86	DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
  87	DDP_UNTAGGED_INV_MSN_RANGE  = 0x03,
  88	DDP_UNTAGGED_INV_MO	    = 0x04,
  89	DDP_UNTAGGED_INV_TOO_LONG   = 0x05,
  90	DDP_UNTAGGED_INV_DDP_VER    = 0x06,
  91};
  92
  93enum irdma_term_mpa_errors {
  94	MPA_CLOSED  = 0x01,
  95	MPA_CRC     = 0x02,
  96	MPA_MARKER  = 0x03,
  97	MPA_REQ_RSP = 0x04,
  98};
  99
 100enum irdma_hw_stats_index {
 101	/* gen1 - 32-bit */
 102	IRDMA_HW_STAT_INDEX_IP4RXDISCARD	= 0,
 103	IRDMA_HW_STAT_INDEX_IP4RXTRUNC		= 1,
 104	IRDMA_HW_STAT_INDEX_IP4TXNOROUTE	= 2,
 105	IRDMA_HW_STAT_INDEX_IP6RXDISCARD	= 3,
 106	IRDMA_HW_STAT_INDEX_IP6RXTRUNC		= 4,
 107	IRDMA_HW_STAT_INDEX_IP6TXNOROUTE	= 5,
 108	IRDMA_HW_STAT_INDEX_TCPRTXSEG		= 6,
 109	IRDMA_HW_STAT_INDEX_TCPRXOPTERR		= 7,
 110	IRDMA_HW_STAT_INDEX_TCPRXPROTOERR	= 8,
 111	IRDMA_HW_STAT_INDEX_RXVLANERR		= 9,
 112		/* gen1 - 64-bit */
 113	IRDMA_HW_STAT_INDEX_IP4RXOCTS		= 10,
 114	IRDMA_HW_STAT_INDEX_IP4RXPKTS		= 11,
 115	IRDMA_HW_STAT_INDEX_IP4RXFRAGS		= 12,
 116	IRDMA_HW_STAT_INDEX_IP4RXMCPKTS		= 13,
 117	IRDMA_HW_STAT_INDEX_IP4TXOCTS		= 14,
 118	IRDMA_HW_STAT_INDEX_IP4TXPKTS		= 15,
 119	IRDMA_HW_STAT_INDEX_IP4TXFRAGS		= 16,
 120	IRDMA_HW_STAT_INDEX_IP4TXMCPKTS		= 17,
 121	IRDMA_HW_STAT_INDEX_IP6RXOCTS		= 18,
 122	IRDMA_HW_STAT_INDEX_IP6RXPKTS		= 19,
 123	IRDMA_HW_STAT_INDEX_IP6RXFRAGS		= 20,
 124	IRDMA_HW_STAT_INDEX_IP6RXMCPKTS		= 21,
 125	IRDMA_HW_STAT_INDEX_IP6TXOCTS		= 22,
 126	IRDMA_HW_STAT_INDEX_IP6TXPKTS		= 23,
 127	IRDMA_HW_STAT_INDEX_IP6TXFRAGS		= 24,
 128	IRDMA_HW_STAT_INDEX_IP6TXMCPKTS		= 25,
 129	IRDMA_HW_STAT_INDEX_TCPRXSEGS		= 26,
 130	IRDMA_HW_STAT_INDEX_TCPTXSEG		= 27,
 131	IRDMA_HW_STAT_INDEX_RDMARXRDS		= 28,
 132	IRDMA_HW_STAT_INDEX_RDMARXSNDS		= 29,
 133	IRDMA_HW_STAT_INDEX_RDMARXWRS		= 30,
 134	IRDMA_HW_STAT_INDEX_RDMATXRDS		= 31,
 135	IRDMA_HW_STAT_INDEX_RDMATXSNDS		= 32,
 136	IRDMA_HW_STAT_INDEX_RDMATXWRS		= 33,
 137	IRDMA_HW_STAT_INDEX_RDMAVBND		= 34,
 138	IRDMA_HW_STAT_INDEX_RDMAVINV		= 35,
 139	IRDMA_HW_STAT_INDEX_IP4RXMCOCTS         = 36,
 140	IRDMA_HW_STAT_INDEX_IP4TXMCOCTS         = 37,
 141	IRDMA_HW_STAT_INDEX_IP6RXMCOCTS         = 38,
 142	IRDMA_HW_STAT_INDEX_IP6TXMCOCTS         = 39,
 143	IRDMA_HW_STAT_INDEX_UDPRXPKTS           = 40,
 144	IRDMA_HW_STAT_INDEX_UDPTXPKTS           = 41,
 145	IRDMA_HW_STAT_INDEX_MAX_GEN_1           = 42, /* Must be same value as next entry */
 146	/* gen2 - 64-bit */
 147	IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS   = 42,
 148	/* gen2 - 32-bit */
 149	IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED      = 43,
 150	IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED      = 44,
 151	IRDMA_HW_STAT_INDEX_TXNPCNPSENT         = 45,
 152	IRDMA_HW_STAT_INDEX_MAX_GEN_2		= 46,
 153
 154	/* gen3 */
 155	IRDMA_HW_STAT_INDEX_RNR_SENT		= 46,
 156	IRDMA_HW_STAT_INDEX_RNR_RCVD		= 47,
 157	IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT	= 48,
 158	IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT	= 49,
 159	IRDMA_HW_STAT_INDEX_RDMARXATS		= 50,
 160	IRDMA_HW_STAT_INDEX_RDMATXATS		= 51,
 161	IRDMA_HW_STAT_INDEX_NAKSEQERR		= 52,
 162	IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED	= 53,
 163	IRDMA_HW_STAT_INDEX_RTO			= 54,
 164	IRDMA_HW_STAT_INDEX_RXOOOPKTS		= 55,
 165	IRDMA_HW_STAT_INDEX_ICRCERR		= 56,
 166
 167	IRDMA_HW_STAT_INDEX_MAX_GEN_3		= 57,
 168};
 169
 170enum irdma_feature_type {
 171	IRDMA_FEATURE_FW_INFO = 0,
 172	IRDMA_HW_VERSION_INFO = 1,
 173	IRDMA_QP_MAX_INCR     = 2,
 174	IRDMA_CQ_MAX_INCR     = 3,
 175	IRDMA_CEQ_MAX_INCR    = 4,
 176	IRDMA_SD_MAX_INCR     = 5,
 177	IRDMA_MR_MAX_INCR     = 6,
 178	IRDMA_Q1_MAX_INCR     = 7,
 179	IRDMA_AH_MAX_INCR     = 8,
 180	IRDMA_SRQ_MAX_INCR    = 9,
 181	IRDMA_TIMER_MAX_INCR  = 10,
 182	IRDMA_XF_MAX_INCR     = 11,
 183	IRDMA_RRF_MAX_INCR    = 12,
 184	IRDMA_PBLE_MAX_INCR   = 13,
 185	IRDMA_OBJ_1           = 22,
 186	IRDMA_OBJ_2           = 23,
 187	IRDMA_ENDPT_TRK       = 24,
 188	IRDMA_FTN_INLINE_MAX  = 25,
 189	IRDMA_QSETS_MAX       = 26,
 190	IRDMA_ASO	      = 27,
 191	IRDMA_FTN_FLAGS	      = 32,
 192	IRDMA_FTN_NOP         = 33,
 193	IRDMA_MAX_FEATURES, /* Must be last entry */
 194};
 195
 196enum irdma_sched_prio_type {
 197	IRDMA_PRIO_WEIGHTED_RR     = 1,
 198	IRDMA_PRIO_STRICT	   = 2,
 199	IRDMA_PRIO_WEIGHTED_STRICT = 3,
 200};
 201
 202enum irdma_vm_vf_type {
 203	IRDMA_VF_TYPE = 0,
 204	IRDMA_VM_TYPE,
 205	IRDMA_PF_TYPE,
 206};
 207
 208enum irdma_cqp_hmc_profile {
 209	IRDMA_HMC_PROFILE_DEFAULT  = 1,
 210	IRDMA_HMC_PROFILE_FAVOR_VF = 2,
 211	IRDMA_HMC_PROFILE_EQUAL    = 3,
 212};
 213
 214enum irdma_quad_entry_type {
 215	IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
 216	IRDMA_QHASH_TYPE_TCP_SYN,
 217	IRDMA_QHASH_TYPE_UDP_UNICAST,
 218	IRDMA_QHASH_TYPE_UDP_MCAST,
 219	IRDMA_QHASH_TYPE_ROCE_MCAST,
 220	IRDMA_QHASH_TYPE_ROCEV2_HW,
 221};
 222
 223enum irdma_quad_hash_manage_type {
 224	IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
 225	IRDMA_QHASH_MANAGE_TYPE_ADD,
 226	IRDMA_QHASH_MANAGE_TYPE_MODIFY,
 227};
 228
 229enum irdma_syn_rst_handling {
 230	IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,
 231	IRDMA_SYN_RST_HANDLING_HW_TCP,
 232	IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,
 233	IRDMA_SYN_RST_HANDLING_FW_TCP,
 234};
 235
 236enum irdma_queue_type {
 237	IRDMA_QUEUE_TYPE_SQ_RQ = 0,
 238	IRDMA_QUEUE_TYPE_CQP,
 239	IRDMA_QUEUE_TYPE_SRQ,
 240};
 241
 242struct irdma_sc_dev;
 243struct irdma_vsi_pestat;
 244
 245struct irdma_dcqcn_cc_params {
 246	u8 cc_cfg_valid;
 247	u8 min_dec_factor;
 248	u8 min_rate;
 249	u8 dcqcn_f;
 250	u16 rai_factor;
 251	u16 hai_factor;
 252	u16 dcqcn_t;
 253	u32 dcqcn_b;
 254	u32 rreduce_mperiod;
 255};
 256
 257struct irdma_cqp_init_info {
 258	u64 cqp_compl_ctx;
 259	u64 host_ctx_pa;
 260	u64 sq_pa;
 261	struct irdma_sc_dev *dev;
 262	struct irdma_cqp_quanta *sq;
 263	struct irdma_dcqcn_cc_params dcqcn_params;
 264	__le64 *host_ctx;
 265	u64 *scratch_array;
 266	u32 sq_size;
 267	struct irdma_ooo_cqp_op *ooo_op_array;
 268	u32 pe_en_vf_cnt;
 269	u16 hw_maj_ver;
 270	u16 hw_min_ver;
 271	u8 struct_ver;
 272	u8 hmc_profile;
 273	u8 ena_vf_count;
 274	u8 ceqs_per_vf;
 275	u8 ooisc_blksize;
 276	u8 rrsp_blksize;
 277	u8 q1_blksize;
 278	u8 xmit_blksize;
 279	u8 ts_override;
 280	u8 ts_shift;
 281	u8 en_fine_grained_timers;
 282	u8 blksizes_valid;
 283	bool en_datacenter_tcp:1;
 284	bool disable_packed:1;
 285	bool rocev2_rto_policy:1;
 286	enum irdma_protocol_used protocol_used;
 287};
 288
 289struct irdma_terminate_hdr {
 290	u8 layer_etype;
 291	u8 error_code;
 292	u8 hdrct;
 293	u8 rsvd;
 294};
 295
 296struct irdma_cqp_sq_wqe {
 297	__le64 buf[IRDMA_CQP_WQE_SIZE];
 298};
 299
 300struct irdma_sc_aeqe {
 301	__le64 buf[IRDMA_AEQE_SIZE];
 302};
 303
 304struct irdma_ceqe {
 305	__le64 buf[IRDMA_CEQE_SIZE];
 306};
 307
 308struct irdma_cqp_ctx {
 309	__le64 buf[IRDMA_CQP_CTX_SIZE];
 310};
 311
 312struct irdma_cq_shadow_area {
 313	__le64 buf[IRDMA_SHADOW_AREA_SIZE];
 314};
 315
 316struct irdma_dev_hw_stats_offsets {
 317	u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
 318};
 319
 320struct irdma_dev_hw_stats {
 321	u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
 322};
 323
 324struct irdma_gather_stats {
 325	u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
 326};
 327
 328struct irdma_hw_stat_map {
 329	u16 byteoff;
 330	u8 bitoff;
 331	u64 bitmask;
 332};
 333
 334struct irdma_stats_gather_info {
 335	bool use_hmc_fcn_index:1;
 336	bool use_stats_inst:1;
 337	u8 hmc_fcn_index;
 338	u8 stats_inst_index;
 339	struct irdma_dma_mem stats_buff_mem;
 340	void *gather_stats_va;
 341	void *last_gather_stats_va;
 342};
 343
 344struct irdma_vsi_pestat {
 345	struct irdma_hw *hw;
 346	struct irdma_dev_hw_stats hw_stats;
 347	struct irdma_stats_gather_info gather_info;
 348	struct timer_list stats_timer;
 349	struct irdma_sc_vsi *vsi;
 350	struct irdma_dev_hw_stats last_hw_stats;
 351	spinlock_t lock; /* rdma stats lock */
 352};
 353
 354struct irdma_mmio_region {
 355	u8 __iomem *addr;
 356	resource_size_t len;
 357	resource_size_t offset;
 358};
 359
 360struct irdma_hw {
 361	union {
 362		u8 __iomem *hw_addr;
 363		struct {
 364			struct irdma_mmio_region rdma_reg; /* RDMA region */
 365			struct irdma_mmio_region *io_regs; /* Non-RDMA MMIO regions */
 366			u16 num_io_regions; /* Number of Non-RDMA MMIO regions */
 367		};
 368	};
 369	struct device *device;
 370	struct irdma_hmc_info hmc;
 371};
 372
 373struct irdma_pfpdu {
 374	struct list_head rxlist;
 375	u32 rcv_nxt;
 376	u32 fps;
 377	u32 max_fpdu_data;
 378	u32 nextseqnum;
 379	u32 rcv_start_seq;
 380	bool mode:1;
 381	bool mpa_crc_err:1;
 382	u8  marker_len;
 383	u64 total_ieq_bufs;
 384	u64 fpdu_processed;
 385	u64 bad_seq_num;
 386	u64 crc_err;
 387	u64 no_tx_bufs;
 388	u64 tx_err;
 389	u64 out_of_order;
 390	u64 pmode_count;
 391	struct irdma_sc_ah *ah;
 392	struct irdma_puda_buf *ah_buf;
 393	spinlock_t lock; /* fpdu processing lock */
 394	struct irdma_puda_buf *lastrcv_buf;
 395};
 396
 397struct irdma_sc_pd {
 398	struct irdma_sc_dev *dev;
 399	u32 pd_id;
 400	int abi_ver;
 401};
 402
 403struct irdma_cqp_quanta {
 404	__le64 elem[IRDMA_CQP_WQE_SIZE];
 405};
 406
 407struct irdma_ooo_cqp_op {
 408	struct list_head list_entry;
 409	u64 scratch;
 410	u32 def_info;
 411	u32 sw_def_info;
 412	u32 wqe_idx;
 413	bool deferred:1;
 414};
 415
 416struct irdma_sc_cqp {
 417	spinlock_t ooo_list_lock; /* protects list of pending completions */
 418	struct list_head ooo_avail;
 419	struct list_head ooo_pnd;
 420	u32 last_def_cmpl_ticket;
 421	u32 sw_def_cmpl_ticket;
 422	u32 size;
 423	u64 sq_pa;
 424	u64 host_ctx_pa;
 425	void *back_cqp;
 426	struct irdma_sc_dev *dev;
 427	int (*process_cqp_sds)(struct irdma_sc_dev *dev,
 428			       struct irdma_update_sds_info *info);
 429	struct irdma_dma_mem sdbuf;
 430	struct irdma_ring sq_ring;
 431	struct irdma_cqp_quanta *sq_base;
 432	struct irdma_dcqcn_cc_params dcqcn_params;
 433	__le64 *host_ctx;
 434	u64 *scratch_array;
 435	u64 requested_ops;
 436	atomic64_t completed_ops;
 437	struct irdma_ooo_cqp_op *ooo_op_array;
 438	u32 cqp_id;
 439	u32 sq_size;
 440	u32 pe_en_vf_cnt;
 441	u32 hw_sq_size;
 442	u16 hw_maj_ver;
 443	u16 hw_min_ver;
 444	u8 struct_ver;
 445	u8 polarity;
 446	u8 hmc_profile;
 447	u8 ena_vf_count;
 448	u8 timeout_count;
 449	u8 ceqs_per_vf;
 450	u8 ooisc_blksize;
 451	u8 rrsp_blksize;
 452	u8 q1_blksize;
 453	u8 xmit_blksize;
 454	u8 ts_override;
 455	u8 ts_shift;
 456	u8 en_fine_grained_timers;
 457	u8 blksizes_valid;
 458	bool en_datacenter_tcp:1;
 459	bool disable_packed:1;
 460	bool rocev2_rto_policy:1;
 461	enum irdma_protocol_used protocol_used;
 462};
 463
 464struct irdma_sc_aeq {
 465	u32 size;
 466	u64 aeq_elem_pa;
 467	struct irdma_sc_dev *dev;
 468	struct irdma_sc_aeqe *aeqe_base;
 469	void *pbl_list;
 470	u32 elem_cnt;
 471	struct irdma_ring aeq_ring;
 472	u8 pbl_chunk_size;
 473	u32 first_pm_pbl_idx;
 474	u32 msix_idx;
 475	u8 polarity;
 476	bool virtual_map:1;
 477	bool pasid_valid:1;
 478	u32 pasid;
 479};
 480
 481struct irdma_sc_ceq {
 482	u32 size;
 483	u64 ceq_elem_pa;
 484	struct irdma_sc_dev *dev;
 485	struct irdma_ceqe *ceqe_base;
 486	void *pbl_list;
 487	u32 ceq_id;
 488	u32 elem_cnt;
 489	struct irdma_ring ceq_ring;
 490	u8 pbl_chunk_size;
 491	u8 tph_val;
 492	u32 first_pm_pbl_idx;
 493	u8 polarity;
 494	u16 vsi_idx;
 495	bool virtual_map:1;
 496	bool tph_en:1;
 497	bool itr_no_expire:1;
 498	bool pasid_valid:1;
 499	u32 pasid;
 500};
 501
 502struct irdma_sc_cq {
 503	struct irdma_cq_uk cq_uk;
 504	u64 cq_pa;
 505	u64 shadow_area_pa;
 506	struct irdma_sc_dev *dev;
 507	u16 vsi_idx;
 508	struct irdma_sc_vsi *vsi;
 509	void *pbl_list;
 510	void *back_cq;
 511	u32 ceq_id;
 512	u32 shadow_read_threshold;
 513	u8 pbl_chunk_size;
 514	u8 cq_type;
 515	u8 tph_val;
 516	u32 first_pm_pbl_idx;
 517	bool ceqe_mask:1;
 518	bool virtual_map:1;
 519	bool check_overflow:1;
 520	bool ceq_id_valid:1;
 521	bool tph_en;
 522};
 523
 524struct irdma_sc_qp {
 525	struct irdma_qp_uk qp_uk;
 526	u64 sq_pa;
 527	u64 rq_pa;
 528	u64 hw_host_ctx_pa;
 529	u64 shadow_area_pa;
 530	u64 q2_pa;
 531	struct irdma_sc_dev *dev;
 532	struct irdma_sc_vsi *vsi;
 533	struct irdma_sc_pd *pd;
 534	__le64 *hw_host_ctx;
 535	void *llp_stream_handle;
 536	struct irdma_pfpdu pfpdu;
 537	u32 ieq_qp;
 538	u8 *q2_buf;
 539	u64 qp_compl_ctx;
 540	u32 push_idx;
 541	u16 qs_handle;
 542	u16 push_offset;
 543	u8 flush_wqes_count;
 544	u8 sq_tph_val;
 545	u8 rq_tph_val;
 546	u8 qp_state;
 547	u8 hw_sq_size;
 548	u8 hw_rq_size;
 549	u8 src_mac_addr_idx;
 550	bool on_qoslist:1;
 551	bool ieq_pass_thru:1;
 552	bool sq_tph_en:1;
 553	bool rq_tph_en:1;
 554	bool rcv_tph_en:1;
 555	bool xmit_tph_en:1;
 556	bool virtual_map:1;
 557	bool flush_sq:1;
 558	bool flush_rq:1;
 559	bool err_sq_idx_valid:1;
 560	bool err_rq_idx_valid:1;
 561	u32 err_sq_idx;
 562	u32 err_rq_idx;
 563	bool sq_flush_code:1;
 564	bool rq_flush_code:1;
 565	u32 pkt_limit;
 566	enum irdma_flush_opcode flush_code;
 567	enum irdma_qp_event_type event_type;
 568	u8 term_flags;
 569	u8 user_pri;
 570	struct list_head list;
 571};
 572
 573struct irdma_stats_inst_info {
 574	bool use_hmc_fcn_index;
 575	u8 hmc_fn_id;
 576	u16 stats_idx;
 577};
 578
 579struct irdma_up_info {
 580	u8 map[8];
 581	u8 cnp_up_override;
 582	u16 hmc_fcn_idx;
 583	bool use_vlan:1;
 584	bool use_cnp_up_override:1;
 585};
 586
 587#define IRDMA_MAX_WS_NODES	0x3FF
 588#define IRDMA_WS_NODE_INVALID	0xFFFF
 589
 590struct irdma_ws_node_info {
 591	u16 id;
 592	u16 vsi;
 593	u16 parent_id;
 594	u16 qs_handle;
 595	bool type_leaf:1;
 596	bool enable:1;
 597	u8 prio_type;
 598	u8 tc;
 599	u8 weight;
 600};
 601
 602struct irdma_hmc_fpm_misc {
 603	u32 max_ceqs;
 604	u32 max_sds;
 605	u32 loc_mem_pages;
 606	u8 ird;
 607	u32 xf_block_size;
 608	u32 q1_block_size;
 609	u32 ht_multiplier;
 610	u32 timer_bucket;
 611	u32 rrf_block_size;
 612	u32 ooiscf_block_size;
 613};
 614
 615#define IRDMA_VCHNL_MAX_MSG_SIZE 512
 616#define IRDMA_LEAF_DEFAULT_REL_BW		64
 617#define IRDMA_PARENT_DEFAULT_REL_BW		1
 618
 619struct irdma_qos {
 620	struct list_head qplist;
 621	struct mutex qos_mutex; /* protect QoS attributes per QoS level */
 622	u64 lan_qos_handle;
 623	u32 l2_sched_node_id;
 624	u16 qs_handle;
 625	u8 traffic_class;
 626	u8 rel_bw;
 627	u8 prio_type;
 628	bool valid;
 629};
 630
 631#define IRDMA_INVALID_STATS_IDX 0xff
 632struct irdma_sc_vsi {
 633	u16 vsi_idx;
 634	struct irdma_sc_dev *dev;
 635	void *back_vsi;
 636	u32 ilq_count;
 637	struct irdma_virt_mem ilq_mem;
 638	struct irdma_puda_rsrc *ilq;
 639	u32 ieq_count;
 640	struct irdma_virt_mem ieq_mem;
 641	struct irdma_puda_rsrc *ieq;
 642	u32 exception_lan_q;
 643	u16 mtu;
 644	u16 vm_id;
 645	enum irdma_vm_vf_type vm_vf_type;
 646	bool stats_inst_alloc:1;
 647	bool tc_change_pending:1;
 648	struct irdma_vsi_pestat *pestat;
 649	atomic_t qp_suspend_reqs;
 650	int (*register_qset)(struct irdma_sc_vsi *vsi,
 651			     struct irdma_ws_node *tc_node);
 652	void (*unregister_qset)(struct irdma_sc_vsi *vsi,
 653				struct irdma_ws_node *tc_node);
 654	u8 qos_rel_bw;
 655	u8 qos_prio_type;
 656	u8 stats_idx;
 657	u8 dscp_map[DSCP_MAX];
 658	struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
 659	u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
 660	bool dscp_mode:1;
 661};
 662
 663struct irdma_sc_dev {
 664	struct list_head cqp_cmd_head; /* head of the CQP command list */
 665	spinlock_t cqp_lock; /* protect CQP list access */
 666	bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];
 667	struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
 668	u64 fpm_query_buf_pa;
 669	u64 fpm_commit_buf_pa;
 670	__le64 *fpm_query_buf;
 671	__le64 *fpm_commit_buf;
 672	struct irdma_hw *hw;
 673	u8 __iomem *db_addr;
 674	u32 __iomem *wqe_alloc_db;
 675	u32 __iomem *cq_arm_db;
 676	u32 __iomem *aeq_alloc_db;
 677	u32 __iomem *cqp_db;
 678	u32 __iomem *cq_ack_db;
 679	u32 __iomem *ceq_itr_mask_db;
 680	u32 __iomem *aeq_itr_mask_db;
 681	u32 __iomem *hw_regs[IRDMA_MAX_REGS];
 682	u32 ceq_itr;   /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
 683	u64 hw_masks[IRDMA_MAX_MASKS];
 684	u64 hw_shifts[IRDMA_MAX_SHIFTS];
 685	const struct irdma_hw_stat_map *hw_stats_map;
 686	u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
 687	u64 feature_info[IRDMA_MAX_FEATURES];
 688	u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
 689	struct irdma_hw_attrs hw_attrs;
 690	struct irdma_hmc_info *hmc_info;
 691	struct irdma_vchnl_rdma_caps vc_caps;
 692	u8 vc_recv_buf[IRDMA_VCHNL_MAX_MSG_SIZE];
 693	u16 vc_recv_len;
 694	struct irdma_sc_cqp *cqp;
 695	struct irdma_sc_aeq *aeq;
 696	struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
 697	struct irdma_sc_cq *ccq;
 698	const struct irdma_irq_ops *irq_ops;
 699	struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
 700	struct irdma_hmc_fpm_misc hmc_fpm_misc;
 701	struct irdma_ws_node *ws_tree_root;
 702	struct mutex ws_mutex; /* ws tree mutex */
 703	u32 vchnl_ver;
 704	u16 num_vfs;
 705	u16 hmc_fn_id;
 706	u16 vf_id;
 707	bool privileged:1;
 708	bool vchnl_up:1;
 709	bool ceq_valid:1;
 710	bool is_pf:1;
 711	u8 protocol_used;
 712	struct mutex vchnl_mutex; /* mutex to synchronize RDMA virtual channel messages */
 713	u8 pci_rev;
 714	int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
 715	void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
 716	void (*ws_reset)(struct irdma_sc_vsi *vsi);
 717};
 718
 719struct irdma_modify_cq_info {
 720	u64 cq_pa;
 721	struct irdma_cqe *cq_base;
 722	u32 cq_size;
 723	u32 shadow_read_threshold;
 724	u8 pbl_chunk_size;
 725	u32 first_pm_pbl_idx;
 726	bool virtual_map:1;
 727	bool check_overflow;
 728	bool cq_resize:1;
 729};
 730
 731struct irdma_srq_init_info {
 732	struct irdma_sc_pd *pd;
 733	struct irdma_sc_vsi *vsi;
 734	u64 srq_pa;
 735	u64 shadow_area_pa;
 736	u32 first_pm_pbl_idx;
 737	u32 pasid;
 738	u32 srq_size;
 739	u16 srq_limit;
 740	u8 pasid_valid;
 741	u8 wqe_size;
 742	u8 leaf_pbl_size;
 743	u8 virtual_map;
 744	u8 tph_en;
 745	u8 arm_limit_event;
 746	u8 tph_value;
 747	u8 pbl_chunk_size;
 748	struct irdma_srq_uk_init_info srq_uk_init_info;
 749};
 750
 751struct irdma_sc_srq {
 752	struct irdma_sc_dev *dev;
 753	struct irdma_sc_vsi *vsi;
 754	struct irdma_sc_pd *pd;
 755	struct irdma_srq_uk srq_uk;
 756	void *back_srq;
 757	u64 srq_pa;
 758	u64 shadow_area_pa;
 759	u32 first_pm_pbl_idx;
 760	u32 pasid;
 761	u32 hw_srq_size;
 762	u16 srq_limit;
 763	u8 pasid_valid;
 764	u8 leaf_pbl_size;
 765	u8 virtual_map;
 766	u8 tph_en;
 767	u8 arm_limit_event;
 768	u8 tph_val;
 769};
 770
 771struct irdma_modify_srq_info {
 772	u16 srq_limit;
 773	u8 arm_limit_event;
 774};
 775
 776struct irdma_create_qp_info {
 777	bool ord_valid:1;
 778	bool tcp_ctx_valid:1;
 779	bool cq_num_valid:1;
 780	bool arp_cache_idx_valid:1;
 781	bool mac_valid:1;
 782	bool force_lpb;
 783	u8 next_iwarp_state;
 784};
 785
 786struct irdma_modify_qp_info {
 787	u64 rx_win0;
 788	u64 rx_win1;
 789	u16 new_mss;
 790	u8 next_iwarp_state;
 791	u8 curr_iwarp_state;
 792	u8 termlen;
 793	bool ord_valid:1;
 794	bool tcp_ctx_valid:1;
 795	bool udp_ctx_valid:1;
 796	bool cq_num_valid:1;
 797	bool arp_cache_idx_valid:1;
 798	bool reset_tcp_conn:1;
 799	bool remove_hash_idx:1;
 800	bool dont_send_term:1;
 801	bool dont_send_fin:1;
 802	bool cached_var_valid:1;
 803	bool mss_change:1;
 804	bool force_lpb:1;
 805	bool mac_valid:1;
 806};
 807
 808struct irdma_ccq_cqe_info {
 809	struct irdma_sc_cqp *cqp;
 810	u64 scratch;
 811	u32 op_ret_val;
 812	u16 maj_err_code;
 813	u16 min_err_code;
 814	u8 op_code;
 815	bool error:1;
 816	bool pending:1;
 817};
 818
 819struct irdma_dcb_app_info {
 820	u8 priority;
 821	u8 selector;
 822	u16 prot_id;
 823};
 824
 825struct irdma_qos_tc_info {
 826	u64 tc_ctx;
 827	u8 rel_bw;
 828	u8 prio_type;
 829	u8 egress_virt_up;
 830	u8 ingress_virt_up;
 831};
 832
 833struct irdma_l2params {
 834	struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
 835	struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
 836	u32 num_apps;
 837	u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
 838	u16 mtu;
 839	u8 up2tc[IRDMA_MAX_USER_PRIORITY];
 840	u8 dscp_map[DSCP_MAX];
 841	u8 num_tc;
 842	u8 vsi_rel_bw;
 843	u8 vsi_prio_type;
 844	bool mtu_changed:1;
 845	bool tc_changed:1;
 846	bool dscp_mode:1;
 847};
 848
 849struct irdma_vsi_init_info {
 850	struct irdma_sc_dev *dev;
 851	void *back_vsi;
 852	struct irdma_l2params *params;
 853	u16 exception_lan_q;
 854	u16 pf_data_vsi_num;
 855	enum irdma_vm_vf_type vm_vf_type;
 856	u16 vm_id;
 857	int (*register_qset)(struct irdma_sc_vsi *vsi,
 858			     struct irdma_ws_node *tc_node);
 859	void (*unregister_qset)(struct irdma_sc_vsi *vsi,
 860				struct irdma_ws_node *tc_node);
 861};
 862
 863struct irdma_vsi_stats_info {
 864	struct irdma_vsi_pestat *pestat;
 865	u16 fcn_id;
 866	bool alloc_stats_inst;
 867};
 868
 869struct irdma_device_init_info {
 870	u64 fpm_query_buf_pa;
 871	u64 fpm_commit_buf_pa;
 872	__le64 *fpm_query_buf;
 873	__le64 *fpm_commit_buf;
 874	struct irdma_hw *hw;
 875	void __iomem *bar0;
 876	enum irdma_protocol_used protocol_used;
 877	u16 hmc_fn_id;
 878};
 879
 880struct irdma_ceq_init_info {
 881	u64 ceqe_pa;
 882	struct irdma_sc_dev *dev;
 883	u64 *ceqe_base;
 884	void *pbl_list;
 885	u32 elem_cnt;
 886	u32 ceq_id;
 887	bool virtual_map:1;
 888	bool tph_en:1;
 889	bool itr_no_expire:1;
 890	u8 pbl_chunk_size;
 891	u8 tph_val;
 892	u16 vsi_idx;
 893	u32 first_pm_pbl_idx;
 894};
 895
 896struct irdma_aeq_init_info {
 897	u64 aeq_elem_pa;
 898	struct irdma_sc_dev *dev;
 899	u32 *aeqe_base;
 900	void *pbl_list;
 901	u32 elem_cnt;
 902	bool virtual_map;
 903	u8 pbl_chunk_size;
 904	u32 first_pm_pbl_idx;
 905	u32 msix_idx;
 906};
 907
 908struct irdma_ccq_init_info {
 909	u64 cq_pa;
 910	u64 shadow_area_pa;
 911	struct irdma_sc_dev *dev;
 912	struct irdma_cqe *cq_base;
 913	__le64 *shadow_area;
 914	void *pbl_list;
 915	u32 num_elem;
 916	u32 ceq_id;
 917	u32 shadow_read_threshold;
 918	bool ceqe_mask:1;
 919	bool ceq_id_valid:1;
 920	bool avoid_mem_cflct:1;
 921	bool virtual_map:1;
 922	bool tph_en:1;
 923	u8 tph_val;
 924	u8 pbl_chunk_size;
 925	u32 first_pm_pbl_idx;
 926	struct irdma_sc_vsi *vsi;
 927};
 928
 929struct irdma_udp_offload_info {
 930	bool ipv4:1;
 931	bool insert_vlan_tag:1;
 932	u8 ttl;
 933	u8 tos;
 934	u16 src_port;
 935	u16 dst_port;
 936	u32 dest_ip_addr[4];
 937	u32 snd_mss;
 938	u16 vlan_tag;
 939	u16 arp_idx;
 940	u32 flow_label;
 941	u8 udp_state;
 942	u32 psn_nxt;
 943	u32 lsn;
 944	u32 epsn;
 945	u32 psn_max;
 946	u32 psn_una;
 947	u32 local_ipaddr[4];
 948	u32 cwnd;
 949	u8 rexmit_thresh;
 950	u8 rnr_nak_thresh;
 951	u8 rnr_nak_tmr;
 952	u8 min_rnr_timer;
 953};
 954
 955struct irdma_roce_offload_info {
 956	u16 p_key;
 957	u16 err_rq_idx;
 958	u32 qkey;
 959	u32 dest_qp;
 960	u8 roce_tver;
 961	u8 ack_credits;
 962	u8 err_rq_idx_valid;
 963	u32 pd_id;
 964	u16 ord_size;
 965	u16 ird_size;
 966	bool is_qp1:1;
 967	bool udprivcq_en:1;
 968	bool dcqcn_en:1;
 969	bool rcv_no_icrc:1;
 970	bool wr_rdresp_en:1;
 971	bool bind_en:1;
 972	bool fast_reg_en:1;
 973	bool priv_mode_en:1;
 974	bool rd_en:1;
 975	bool timely_en:1;
 976	bool dctcp_en:1;
 977	bool fw_cc_enable:1;
 978	bool use_stats_inst:1;
 979	u8 local_ack_timeout;
 980	u16 t_high;
 981	u16 t_low;
 982	u8 last_byte_sent;
 983	u8 mac_addr[ETH_ALEN];
 984	u8 rtomin;
 985};
 986
 987struct irdma_iwarp_offload_info {
 988	u16 rcv_mark_offset;
 989	u16 snd_mark_offset;
 990	u8 ddp_ver;
 991	u8 rdmap_ver;
 992	u8 iwarp_mode;
 993	u16 err_rq_idx;
 994	u32 pd_id;
 995	u16 ord_size;
 996	u16 ird_size;
 997	bool ib_rd_en:1;
 998	bool align_hdrs:1;
 999	bool rcv_no_mpa_crc:1;
1000	bool err_rq_idx_valid:1;
1001	bool snd_mark_en:1;
1002	bool rcv_mark_en:1;
1003	bool wr_rdresp_en:1;
1004	bool bind_en:1;
1005	bool fast_reg_en:1;
1006	bool priv_mode_en:1;
1007	bool rd_en:1;
1008	bool timely_en:1;
1009	bool use_stats_inst:1;
1010	bool ecn_en:1;
1011	bool dctcp_en:1;
1012	u16 t_high;
1013	u16 t_low;
1014	u8 last_byte_sent;
1015	u8 mac_addr[ETH_ALEN];
1016	u8 rtomin;
1017};
1018
1019struct irdma_tcp_offload_info {
1020	bool ipv4:1;
1021	bool no_nagle:1;
1022	bool insert_vlan_tag:1;
1023	bool time_stamp:1;
1024	bool drop_ooo_seg:1;
1025	bool avoid_stretch_ack:1;
1026	bool wscale:1;
1027	bool ignore_tcp_opt:1;
1028	bool ignore_tcp_uns_opt:1;
1029	u8 cwnd_inc_limit;
1030	u8 dup_ack_thresh;
1031	u8 ttl;
1032	u8 src_mac_addr_idx;
1033	u8 tos;
1034	u16 src_port;
1035	u16 dst_port;
1036	u32 dest_ip_addr[4];
1037	//u32 dest_ip_addr0;
1038	//u32 dest_ip_addr1;
1039	//u32 dest_ip_addr2;
1040	//u32 dest_ip_addr3;
1041	u32 snd_mss;
1042	u16 syn_rst_handling;
1043	u16 vlan_tag;
1044	u16 arp_idx;
1045	u32 flow_label;
1046	u8 tcp_state;
1047	u8 snd_wscale;
1048	u8 rcv_wscale;
1049	u32 time_stamp_recent;
1050	u32 time_stamp_age;
1051	u32 snd_nxt;
1052	u32 snd_wnd;
1053	u32 rcv_nxt;
1054	u32 rcv_wnd;
1055	u32 snd_max;
1056	u32 snd_una;
1057	u32 srtt;
1058	u32 rtt_var;
1059	u32 ss_thresh;
1060	u32 cwnd;
1061	u32 snd_wl1;
1062	u32 snd_wl2;
1063	u32 max_snd_window;
1064	u8 rexmit_thresh;
1065	u32 local_ipaddr[4];
1066};
1067
1068struct irdma_qp_host_ctx_info {
1069	u64 qp_compl_ctx;
1070	union {
1071		struct irdma_tcp_offload_info *tcp_info;
1072		struct irdma_udp_offload_info *udp_info;
1073	};
1074	union {
1075		struct irdma_iwarp_offload_info *iwarp_info;
1076		struct irdma_roce_offload_info *roce_info;
1077	};
1078	u32 send_cq_num;
1079	u32 rcv_cq_num;
1080	u32 srq_id;
1081	u32 rem_endpoint_idx;
1082	u16 stats_idx;
1083	bool remote_atomics_en:1;
1084	bool srq_valid:1;
1085	bool tcp_info_valid:1;
1086	bool iwarp_info_valid:1;
1087	bool stats_idx_valid:1;
1088	u8 user_pri;
1089};
1090
1091struct irdma_aeqe_info {
1092	u64 compl_ctx;
1093	u32 qp_cq_id;
1094	u32 def_info;	/* only valid for DEF_CMPL */
1095	u16 ae_id;
1096	u16 wqe_idx;
1097	u8 tcp_state;
1098	u8 iwarp_state;
1099	bool qp:1;
1100	bool cq:1;
1101	bool sq:1;
1102	bool rq:1;
1103	bool srq:1;
1104	bool in_rdrsp_wr:1;
1105	bool out_rdrsp:1;
1106	bool aeqe_overflow:1;
1107	bool err_rq_idx_valid:1;
1108	u8 q2_data_written;
1109	u8 ae_src;
1110};
1111
1112struct irdma_allocate_stag_info {
1113	u64 total_len;
1114	u64 first_pm_pbl_idx;
1115	u32 chunk_size;
1116	u32 stag_idx;
1117	u32 page_size;
1118	u32 pd_id;
1119	u16 access_rights;
1120	bool remote_access:1;
1121	bool use_hmc_fcn_index:1;
1122	bool use_pf_rid:1;
1123	bool all_memory:1;
1124	bool remote_atomics_en:1;
1125	u16 hmc_fcn_index;
1126};
1127
1128struct irdma_mw_alloc_info {
1129	u32 mw_stag_index;
1130	u32 page_size;
1131	u32 pd_id;
1132	bool remote_access:1;
1133	bool mw_wide:1;
1134	bool mw1_bind_dont_vldt_key:1;
1135};
1136
1137struct irdma_reg_ns_stag_info {
1138	u64 reg_addr_pa;
1139	u64 va;
1140	u64 total_len;
1141	u32 page_size;
1142	u32 chunk_size;
1143	u32 first_pm_pbl_index;
1144	enum irdma_addressing_type addr_type;
1145	irdma_stag_index stag_idx;
1146	u16 access_rights;
1147	u32 pd_id;
1148	irdma_stag_key stag_key;
1149	bool use_hmc_fcn_index:1;
1150	u8 hmc_fcn_index;
1151	bool use_pf_rid:1;
1152	bool all_memory:1;
1153	bool remote_atomics_en:1;
1154};
1155
1156struct irdma_fast_reg_stag_info {
1157	u64 wr_id;
1158	u64 reg_addr_pa;
1159	u64 fbo;
1160	void *va;
1161	u64 total_len;
1162	u32 page_size;
1163	u32 chunk_size;
1164	u32 first_pm_pbl_index;
1165	enum irdma_addressing_type addr_type;
1166	irdma_stag_index stag_idx;
1167	u16 access_rights;
1168	u32 pd_id;
1169	irdma_stag_key stag_key;
1170	bool local_fence:1;
1171	bool read_fence:1;
1172	bool signaled:1;
1173	bool use_hmc_fcn_index:1;
1174	u8 hmc_fcn_index;
1175	bool use_pf_rid:1;
1176	bool defer_flag:1;
1177	bool remote_atomics_en:1;
1178};
1179
1180struct irdma_dealloc_stag_info {
1181	u32 stag_idx;
1182	u32 pd_id;
1183	bool mr:1;
1184	bool dealloc_pbl:1;
1185};
1186
1187struct irdma_register_shared_stag {
1188	u64 va;
1189	enum irdma_addressing_type addr_type;
1190	irdma_stag_index new_stag_idx;
1191	irdma_stag_index parent_stag_idx;
1192	u32 access_rights;
1193	u32 pd_id;
1194	u32 page_size;
1195	irdma_stag_key new_stag_key;
1196};
1197
1198struct irdma_qp_init_info {
1199	struct irdma_qp_uk_init_info qp_uk_init_info;
1200	struct irdma_sc_pd *pd;
1201	struct irdma_sc_vsi *vsi;
1202	__le64 *host_ctx;
1203	u8 *q2;
1204	u64 sq_pa;
1205	u64 rq_pa;
1206	u64 host_ctx_pa;
1207	u64 q2_pa;
1208	u64 shadow_area_pa;
1209	u8 sq_tph_val;
1210	u8 rq_tph_val;
1211	bool sq_tph_en:1;
1212	bool rq_tph_en:1;
1213	bool rcv_tph_en:1;
1214	bool xmit_tph_en:1;
1215	bool virtual_map:1;
1216};
1217
1218struct irdma_cq_init_info {
1219	struct irdma_sc_dev *dev;
1220	u64 cq_base_pa;
1221	u64 shadow_area_pa;
1222	u32 ceq_id;
1223	u32 shadow_read_threshold;
1224	u8 pbl_chunk_size;
1225	u32 first_pm_pbl_idx;
1226	bool virtual_map:1;
1227	bool ceqe_mask:1;
1228	bool ceq_id_valid:1;
1229	bool tph_en:1;
1230	u8 tph_val;
1231	u8 type;
1232	struct irdma_cq_uk_init_info cq_uk_init_info;
1233	struct irdma_sc_vsi *vsi;
1234};
1235
1236struct irdma_upload_context_info {
1237	u64 buf_pa;
1238	u32 qp_id;
1239	u8 qp_type;
1240	bool freeze_qp:1;
1241	bool raw_format:1;
1242};
1243
1244struct irdma_local_mac_entry_info {
1245	u8 mac_addr[6];
1246	u16 entry_idx;
1247};
1248
1249struct irdma_add_arp_cache_entry_info {
1250	u8 mac_addr[ETH_ALEN];
1251	u32 reach_max;
1252	u16 arp_index;
1253	bool permanent;
1254};
1255
1256struct irdma_apbvt_info {
1257	u16 port;
1258	bool add;
1259};
1260
1261struct irdma_qhash_table_info {
1262	struct irdma_sc_vsi *vsi;
1263	enum irdma_quad_hash_manage_type manage;
1264	enum irdma_quad_entry_type entry_type;
1265	bool vlan_valid:1;
1266	bool ipv4_valid:1;
1267	u8 mac_addr[ETH_ALEN];
1268	u16 vlan_id;
1269	u8 user_pri;
1270	u32 qp_num;
1271	u32 dest_ip[4];
1272	u32 src_ip[4];
1273	u16 dest_port;
1274	u16 src_port;
1275};
1276
1277struct irdma_cqp_manage_push_page_info {
1278	u32 push_idx;
1279	u16 qs_handle;
1280	u8 free_page;
1281	u8 push_page_type;
1282};
1283
1284struct irdma_qp_flush_info {
1285	u32 err_sq_idx;
1286	u32 err_rq_idx;
1287	u16 sq_minor_code;
1288	u16 sq_major_code;
1289	u16 rq_minor_code;
1290	u16 rq_major_code;
1291	u16 ae_code;
1292	u8 ae_src;
1293	bool sq:1;
1294	bool rq:1;
1295	bool userflushcode:1;
1296	bool generate_ae:1;
1297	bool err_sq_idx_valid:1;
1298	bool err_rq_idx_valid:1;
1299};
1300
1301struct irdma_gen_ae_info {
1302	u16 ae_code;
1303	u8 ae_src;
1304};
1305
1306struct irdma_cqp_timeout {
1307	u64 compl_cqp_cmds;
1308	u32 count;
1309};
1310
1311struct irdma_irq_ops {
1312	void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);
1313	void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
1314			      bool enable);
1315	void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
1316	void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
1317};
1318
1319void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
1320int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
1321			bool check_overflow, bool post_sq);
1322int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
1323int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
1324			      struct irdma_ccq_cqe_info *info);
1325int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
1326		      struct irdma_ccq_init_info *info);
1327
1328int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
1329int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
1330
1331int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
1332int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
1333		      struct irdma_ceq_init_info *info);
1334void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
1335void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
1336
1337int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
1338		      struct irdma_aeq_init_info *info);
1339int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
1340			   struct irdma_aeqe_info *info);
1341void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
1342
1343void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
1344		      int abi_ver);
1345void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
1346void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
1347			      struct irdma_sc_dev *dev);
1348void irdma_sc_cqp_def_cmpl_ae_handler(struct irdma_sc_dev *dev,
1349				      struct irdma_aeqe_info *info,
1350				      bool first, u64 *scratch,
1351				      u32 *sw_def_info);
1352u64 irdma_sc_cqp_cleanup_handler(struct irdma_sc_dev *dev);
1353int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
1354int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
1355int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
1356		      struct irdma_cqp_init_info *info);
1357void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
1358int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
1359				  struct irdma_ccq_cqe_info *cmpl_info);
1360int irdma_sc_fast_register(struct irdma_sc_qp *qp,
1361			   struct irdma_fast_reg_stag_info *info, bool post_sq);
1362int irdma_sc_qp_create(struct irdma_sc_qp *qp,
1363		       struct irdma_create_qp_info *info, u64 scratch,
1364		       bool post_sq);
1365int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
1366			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
1367int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
1368			   struct irdma_qp_flush_info *info, u64 scratch,
1369			   bool post_sq);
1370int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
1371int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
1372		       struct irdma_modify_qp_info *info, u64 scratch,
1373		       bool post_sq);
1374void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1375			irdma_stag stag);
1376
1377void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
1378void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1379			struct irdma_qp_host_ctx_info *info);
1380void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1381			     struct irdma_qp_host_ctx_info *info);
1382int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
1383int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
1384void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
1385int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1386					u8 hmc_fn_id, bool post_sq,
1387					bool poll_registers);
1388int irdma_sc_srq_init(struct irdma_sc_srq *srq,
1389		      struct irdma_srq_init_info *info);
1390
1391void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1392struct cqp_info {
1393	union {
1394		struct {
1395			struct irdma_sc_qp *qp;
1396			struct irdma_create_qp_info info;
1397			u64 scratch;
1398		} qp_create;
1399
1400		struct {
1401			struct irdma_sc_qp *qp;
1402			struct irdma_modify_qp_info info;
1403			u64 scratch;
1404		} qp_modify;
1405
1406		struct {
1407			struct irdma_sc_qp *qp;
1408			u64 scratch;
1409			bool remove_hash_idx;
1410			bool ignore_mw_bnd;
1411		} qp_destroy;
1412
1413		struct {
1414			struct irdma_sc_cq *cq;
1415			u64 scratch;
1416			bool check_overflow;
1417		} cq_create;
1418
1419		struct {
1420			struct irdma_sc_cq *cq;
1421			struct irdma_modify_cq_info info;
1422			u64 scratch;
1423		} cq_modify;
1424
1425		struct {
1426			struct irdma_sc_cq *cq;
1427			u64 scratch;
1428		} cq_destroy;
1429
1430		struct {
1431			struct irdma_sc_dev *dev;
1432			struct irdma_allocate_stag_info info;
1433			u64 scratch;
1434		} alloc_stag;
1435
1436		struct {
1437			struct irdma_sc_dev *dev;
1438			struct irdma_mw_alloc_info info;
1439			u64 scratch;
1440		} mw_alloc;
1441
1442		struct {
1443			struct irdma_sc_dev *dev;
1444			struct irdma_reg_ns_stag_info info;
1445			u64 scratch;
1446		} mr_reg_non_shared;
1447
1448		struct {
1449			struct irdma_sc_dev *dev;
1450			struct irdma_dealloc_stag_info info;
1451			u64 scratch;
1452		} dealloc_stag;
1453
1454		struct {
1455			struct irdma_sc_cqp *cqp;
1456			struct irdma_add_arp_cache_entry_info info;
1457			u64 scratch;
1458		} add_arp_cache_entry;
1459
1460		struct {
1461			struct irdma_sc_cqp *cqp;
1462			u64 scratch;
1463			u16 arp_index;
1464		} del_arp_cache_entry;
1465
1466		struct {
1467			struct irdma_sc_cqp *cqp;
1468			struct irdma_local_mac_entry_info info;
1469			u64 scratch;
1470		} add_local_mac_entry;
1471
1472		struct {
1473			struct irdma_sc_cqp *cqp;
1474			u64 scratch;
1475			u8 entry_idx;
1476			u8 ignore_ref_count;
1477		} del_local_mac_entry;
1478
1479		struct {
1480			struct irdma_sc_cqp *cqp;
1481			u64 scratch;
1482		} alloc_local_mac_entry;
1483
1484		struct {
1485			struct irdma_sc_cqp *cqp;
1486			struct irdma_cqp_manage_push_page_info info;
1487			u64 scratch;
1488		} manage_push_page;
1489
1490		struct {
1491			struct irdma_sc_dev *dev;
1492			struct irdma_upload_context_info info;
1493			u64 scratch;
1494		} qp_upload_context;
1495
1496		struct {
1497			struct irdma_sc_dev *dev;
1498			struct irdma_hmc_fcn_info info;
1499			u64 scratch;
1500		} manage_hmc_pm;
1501
1502		struct {
1503			struct irdma_sc_ceq *ceq;
1504			u64 scratch;
1505		} ceq_create;
1506
1507		struct {
1508			struct irdma_sc_ceq *ceq;
1509			u64 scratch;
1510		} ceq_destroy;
1511
1512		struct {
1513			struct irdma_sc_aeq *aeq;
1514			u64 scratch;
1515		} aeq_create;
1516
1517		struct {
1518			struct irdma_sc_aeq *aeq;
1519			u64 scratch;
1520		} aeq_destroy;
1521
1522		struct {
1523			struct irdma_sc_qp *qp;
1524			struct irdma_qp_flush_info info;
1525			u64 scratch;
1526		} qp_flush_wqes;
1527
1528		struct {
1529			struct irdma_sc_qp *qp;
1530			struct irdma_gen_ae_info info;
1531			u64 scratch;
1532		} gen_ae;
1533
1534		struct {
1535			struct irdma_sc_cqp *cqp;
1536			void *fpm_val_va;
1537			u64 fpm_val_pa;
1538			u8 hmc_fn_id;
1539			u64 scratch;
1540		} query_fpm_val;
1541
1542		struct {
1543			struct irdma_sc_cqp *cqp;
1544			void *fpm_val_va;
1545			u64 fpm_val_pa;
1546			u8 hmc_fn_id;
1547			u64 scratch;
1548		} commit_fpm_val;
1549
1550		struct {
1551			struct irdma_sc_cqp *cqp;
1552			struct irdma_apbvt_info info;
1553			u64 scratch;
1554		} manage_apbvt_entry;
1555
1556		struct {
1557			struct irdma_sc_cqp *cqp;
1558			struct irdma_qhash_table_info info;
1559			u64 scratch;
1560		} manage_qhash_table_entry;
1561
1562		struct {
1563			struct irdma_sc_dev *dev;
1564			struct irdma_update_sds_info info;
1565			u64 scratch;
1566		} update_pe_sds;
1567
1568		struct {
1569			struct irdma_sc_cqp *cqp;
1570			struct irdma_sc_qp *qp;
1571			u64 scratch;
1572		} suspend_resume;
1573
1574		struct {
1575			struct irdma_sc_cqp *cqp;
1576			struct irdma_ah_info info;
1577			u64 scratch;
1578		} ah_create;
1579
1580		struct {
1581			struct irdma_sc_cqp *cqp;
1582			struct irdma_ah_info info;
1583			u64 scratch;
1584		} ah_destroy;
1585
1586		struct {
1587			struct irdma_sc_cqp *cqp;
1588			struct irdma_mcast_grp_info info;
1589			u64 scratch;
1590		} mc_create;
1591
1592		struct {
1593			struct irdma_sc_cqp *cqp;
1594			struct irdma_mcast_grp_info info;
1595			u64 scratch;
1596		} mc_destroy;
1597
1598		struct {
1599			struct irdma_sc_cqp *cqp;
1600			struct irdma_mcast_grp_info info;
1601			u64 scratch;
1602		} mc_modify;
1603
1604		struct {
1605			struct irdma_sc_cqp *cqp;
1606			struct irdma_stats_inst_info info;
1607			u64 scratch;
1608		} stats_manage;
1609
1610		struct {
1611			struct irdma_sc_cqp *cqp;
1612			struct irdma_stats_gather_info info;
1613			u64 scratch;
1614		} stats_gather;
1615
1616		struct {
1617			struct irdma_sc_cqp *cqp;
1618			struct irdma_ws_node_info info;
1619			u64 scratch;
1620		} ws_node;
1621
1622		struct {
1623			struct irdma_sc_cqp *cqp;
1624			struct irdma_up_info info;
1625			u64 scratch;
1626		} up_map;
1627
1628		struct {
1629			struct irdma_sc_cqp *cqp;
1630			struct irdma_dma_mem query_buff_mem;
1631			u64 scratch;
1632		} query_rdma;
1633
1634		struct {
1635			struct irdma_sc_srq *srq;
1636			u64 scratch;
1637		} srq_create;
1638
1639		struct {
1640			struct irdma_sc_srq *srq;
1641			struct irdma_modify_srq_info info;
1642			u64 scratch;
1643		} srq_modify;
1644
1645		struct {
1646			struct irdma_sc_srq *srq;
1647			u64 scratch;
1648		} srq_destroy;
1649
1650	} u;
1651};
1652
1653struct cqp_cmds_info {
1654	struct list_head cqp_cmd_entry;
1655	u8 cqp_cmd;
1656	u8 post_sq;
1657	struct cqp_info in;
1658};
1659
1660__le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
1661					   u32 *wqe_idx);
1662
1663/**
1664 * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
1665 * @cqp: struct for cqp hw
1666 * @scratch: private data for CQP WQE
1667 */
1668static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
1669{
1670	u32 wqe_idx;
1671
1672	return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
1673}
1674#endif /* IRDMA_TYPE_H */