user.h

  1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2/* Copyright (c) 2015 - 2020 Intel Corporation */
  3#ifndef IRDMA_USER_H
  4#define IRDMA_USER_H
  5
  6#define irdma_handle void *
  7#define irdma_adapter_handle irdma_handle
  8#define irdma_qp_handle irdma_handle
  9#define irdma_cq_handle irdma_handle
 10#define irdma_pd_id irdma_handle
 11#define irdma_stag_handle irdma_handle
 12#define irdma_stag_index u32
 13#define irdma_stag u32
 14#define irdma_stag_key u8
 15#define irdma_tagged_offset u64
 16#define irdma_access_privileges u32
 17#define irdma_physical_fragment u64
 18#define irdma_address_list u64 *
 19
 20#define	IRDMA_MAX_MR_SIZE       0x200000000000ULL
 21
 22#define IRDMA_ACCESS_FLAGS_LOCALREAD		0x01
 23#define IRDMA_ACCESS_FLAGS_LOCALWRITE		0x02
 24#define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY	0x04
 25#define IRDMA_ACCESS_FLAGS_REMOTEREAD		0x05
 26#define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY	0x08
 27#define IRDMA_ACCESS_FLAGS_REMOTEWRITE		0x0a
 28#define IRDMA_ACCESS_FLAGS_BIND_WINDOW		0x10
 29#define IRDMA_ACCESS_FLAGS_ZERO_BASED		0x20
 30#define IRDMA_ACCESS_FLAGS_ALL			0x3f
 31
 32#define IRDMA_OP_TYPE_RDMA_WRITE		0x00
 33#define IRDMA_OP_TYPE_RDMA_READ			0x01
 34#define IRDMA_OP_TYPE_SEND			0x03
 35#define IRDMA_OP_TYPE_SEND_INV			0x04
 36#define IRDMA_OP_TYPE_SEND_SOL			0x05
 37#define IRDMA_OP_TYPE_SEND_SOL_INV		0x06
 38#define IRDMA_OP_TYPE_RDMA_WRITE_SOL		0x0d
 39#define IRDMA_OP_TYPE_BIND_MW			0x08
 40#define IRDMA_OP_TYPE_FAST_REG_NSMR		0x09
 41#define IRDMA_OP_TYPE_INV_STAG			0x0a
 42#define IRDMA_OP_TYPE_RDMA_READ_INV_STAG	0x0b
 43#define IRDMA_OP_TYPE_NOP			0x0c
 44#define IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD	0x0f
 45#define IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP	0x11
 46#define IRDMA_OP_TYPE_REC	0x3e
 47#define IRDMA_OP_TYPE_REC_IMM	0x3f
 48
 49#define IRDMA_FLUSH_MAJOR_ERR 1
 50#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
 51
 52/* Async Events codes */
 53#define IRDMA_AE_AMP_UNALLOCATED_STAG					0x0102
 54#define IRDMA_AE_AMP_INVALID_STAG					0x0103
 55#define IRDMA_AE_AMP_BAD_QP						0x0104
 56#define IRDMA_AE_AMP_BAD_PD						0x0105
 57#define IRDMA_AE_AMP_BAD_STAG_KEY					0x0106
 58#define IRDMA_AE_AMP_BAD_STAG_INDEX					0x0107
 59#define IRDMA_AE_AMP_BOUNDS_VIOLATION					0x0108
 60#define IRDMA_AE_AMP_RIGHTS_VIOLATION					0x0109
 61#define IRDMA_AE_AMP_TO_WRAP						0x010a
 62#define IRDMA_AE_AMP_FASTREG_VALID_STAG					0x010c
 63#define IRDMA_AE_AMP_FASTREG_MW_STAG					0x010d
 64#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS				0x010e
 65#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH				0x0110
 66#define IRDMA_AE_AMP_INVALIDATE_SHARED					0x0111
 67#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS			0x0112
 68#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS			0x0113
 69#define IRDMA_AE_AMP_MWBIND_VALID_STAG					0x0114
 70#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG					0x0115
 71#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG				0x0116
 72#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG					0x0117
 73#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS				0x0118
 74#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS				0x0119
 75#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT				0x011a
 76#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED				0x011b
 77#define IRDMA_AE_PRIV_OPERATION_DENIED					0x011c
 78#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW				0x011d
 79#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW				0x011e
 80#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG			0x011f
 81#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE					0x0120
 82#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH				0x0121
 83#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG				0x0132
 84#define IRDMA_AE_UDA_XMIT_BAD_PD					0x0133
 85#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT				0x0134
 86#define IRDMA_AE_UDA_L4LEN_INVALID					0x0135
 87#define IRDMA_AE_BAD_CLOSE						0x0201
 88#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE				0x0202
 89#define IRDMA_AE_CQ_OPERATION_ERROR					0x0203
 90#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO				0x0205
 91#define IRDMA_AE_STAG_ZERO_INVALID					0x0206
 92#define IRDMA_AE_IB_RREQ_AND_Q1_FULL					0x0207
 93#define IRDMA_AE_IB_INVALID_REQUEST					0x0208
 94#define IRDMA_AE_SRQ_LIMIT						0x0209
 95#define IRDMA_AE_WQE_UNEXPECTED_OPCODE					0x020a
 96#define IRDMA_AE_WQE_INVALID_PARAMETER					0x020b
 97#define IRDMA_AE_WQE_INVALID_FRAG_DATA					0x020c
 98#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR					0x020d
 99#define IRDMA_AE_IB_REMOTE_OP_ERROR					0x020e
100#define IRDMA_AE_SRQ_CATASTROPHIC_ERROR					0x020f
101#define IRDMA_AE_WQE_LSMM_TOO_LONG					0x0220
102#define IRDMA_AE_ATOMIC_ALIGNMENT					0x0221
103#define IRDMA_AE_ATOMIC_MASK						0x0222
104#define IRDMA_AE_INVALID_REQUEST					0x0223
105#define IRDMA_AE_PCIE_ATOMIC_DISABLE					0x0224
106#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN				0x0301
107#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER	0x0303
108#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION				0x0304
109#define IRDMA_AE_DDP_UBE_INVALID_MO					0x0305
110#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE		0x0306
111#define IRDMA_AE_DDP_UBE_INVALID_QN					0x0307
112#define IRDMA_AE_DDP_NO_L_BIT						0x0308
113#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION			0x0311
114#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE				0x0312
115#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST				0x0313
116#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP			0x0314
117#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR					0x0316
118#define IRDMA_AE_ROCE_EMPTY_MCG						0x0380
119#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR					0x0381
120#define IRDMA_AE_ROCE_BAD_MC_QPID					0x0382
121#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH				0x0383
122#define IRDMA_AE_INVALID_ARP_ENTRY					0x0401
123#define IRDMA_AE_INVALID_TCP_OPTION_RCVD				0x0402
124#define IRDMA_AE_STALE_ARP_ENTRY					0x0403
125#define IRDMA_AE_INVALID_AH_ENTRY					0x0406
126#define IRDMA_AE_LLP_CLOSE_COMPLETE					0x0501
127#define IRDMA_AE_LLP_CONNECTION_RESET					0x0502
128#define IRDMA_AE_LLP_FIN_RECEIVED					0x0503
129#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH	0x0504
130#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR				0x0505
131#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL					0x0507
132#define IRDMA_AE_LLP_SYN_RECEIVED					0x0508
133#define IRDMA_AE_LLP_TERMINATE_RECEIVED					0x0509
134#define IRDMA_AE_LLP_TOO_MANY_RETRIES					0x050a
135#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES				0x050b
136#define IRDMA_AE_LLP_DOUBT_REACHABILITY					0x050c
137#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED				0x050e
138#define IRDMA_AE_LLP_TOO_MANY_RNRS					0x050f
139#define IRDMA_AE_RESOURCE_EXHAUSTION					0x0520
140#define IRDMA_AE_RESET_SENT						0x0601
141#define IRDMA_AE_TERMINATE_SENT						0x0602
142#define IRDMA_AE_RESET_NOT_SENT						0x0603
143#define IRDMA_AE_LCE_QP_CATASTROPHIC					0x0700
144#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC				0x0701
145#define IRDMA_AE_LCE_CQ_CATASTROPHIC					0x0702
146#define IRDMA_AE_REMOTE_QP_CATASTROPHIC					0x0703
147#define IRDMA_AE_LOCAL_QP_CATASTROPHIC					0x0704
148#define IRDMA_AE_RCE_QP_CATASTROPHIC					0x0705
149#define IRDMA_AE_QP_SUSPEND_COMPLETE					0x0900
150#define IRDMA_AE_CQP_DEFERRED_COMPLETE					0x0901
151#define IRDMA_AE_ADAPTER_CATASTROPHIC					0x0B0B
152
153enum irdma_device_caps_const {
154	IRDMA_WQE_SIZE =			4,
155	IRDMA_CQP_WQE_SIZE =			8,
156	IRDMA_CQE_SIZE =			4,
157	IRDMA_EXTENDED_CQE_SIZE =		8,
158	IRDMA_AEQE_SIZE =			2,
159	IRDMA_CEQE_SIZE =			1,
160	IRDMA_CQP_CTX_SIZE =			8,
161	IRDMA_SHADOW_AREA_SIZE =		8,
162	IRDMA_QUERY_FPM_BUF_SIZE =		192,
163	IRDMA_COMMIT_FPM_BUF_SIZE =		192,
164	IRDMA_GATHER_STATS_BUF_SIZE =		1024,
165	IRDMA_MIN_IW_QP_ID =			0,
166	IRDMA_MAX_IW_QP_ID =			262143,
167	IRDMA_MIN_IW_SRQ_ID =			0,
168	IRDMA_MIN_CEQID =			0,
169	IRDMA_MAX_CEQID =			1023,
170	IRDMA_CEQ_MAX_COUNT =			IRDMA_MAX_CEQID + 1,
171	IRDMA_MIN_CQID =			0,
172	IRDMA_MAX_CQID =			524287,
173	IRDMA_MIN_AEQ_ENTRIES =			1,
174	IRDMA_MAX_AEQ_ENTRIES =			524287,
175	IRDMA_MAX_AEQ_ENTRIES_GEN_3 =           262144,
176	IRDMA_MIN_CEQ_ENTRIES =			1,
177	IRDMA_MAX_CEQ_ENTRIES =			262143,
178	IRDMA_MIN_CQ_SIZE =			1,
179	IRDMA_MAX_CQ_SIZE =			1048575,
180	IRDMA_DB_ID_ZERO =			0,
181	IRDMA_MAX_WQ_FRAGMENT_COUNT =		13,
182	IRDMA_MAX_SGE_RD =			13,
183	IRDMA_MAX_OUTBOUND_MSG_SIZE =		2147483647,
184	IRDMA_MAX_INBOUND_MSG_SIZE =		2147483647,
185	IRDMA_MAX_PUSH_PAGE_COUNT =		1024,
186	IRDMA_MAX_PE_ENA_VF_COUNT =		32,
187	IRDMA_MAX_VF_FPM_ID =			47,
188	IRDMA_MAX_SQ_PAYLOAD_SIZE =		2145386496,
189	IRDMA_MAX_INLINE_DATA_SIZE =		101,
190	IRDMA_MAX_WQ_ENTRIES =			32768,
191	IRDMA_Q2_BUF_SIZE =			256,
192	IRDMA_QP_CTX_SIZE =			256,
193	IRDMA_MAX_PDS =				262144,
194	IRDMA_MIN_WQ_SIZE_GEN2 =                8,
195};
196
197enum irdma_addressing_type {
198	IRDMA_ADDR_TYPE_ZERO_BASED = 0,
199	IRDMA_ADDR_TYPE_VA_BASED   = 1,
200};
201
202enum irdma_flush_opcode {
203	FLUSH_INVALID = 0,
204	FLUSH_GENERAL_ERR,
205	FLUSH_PROT_ERR,
206	FLUSH_REM_ACCESS_ERR,
207	FLUSH_LOC_QP_OP_ERR,
208	FLUSH_REM_OP_ERR,
209	FLUSH_LOC_LEN_ERR,
210	FLUSH_FATAL_ERR,
211	FLUSH_RETRY_EXC_ERR,
212	FLUSH_MW_BIND_ERR,
213	FLUSH_REM_INV_REQ_ERR,
214	FLUSH_RNR_RETRY_EXC_ERR,
215};
216
217enum irdma_qp_event_type {
218	IRDMA_QP_EVENT_CATASTROPHIC,
219	IRDMA_QP_EVENT_ACCESS_ERR,
220	IRDMA_QP_EVENT_REQ_ERR,
221};
222
223enum irdma_cmpl_status {
224	IRDMA_COMPL_STATUS_SUCCESS = 0,
225	IRDMA_COMPL_STATUS_FLUSHED,
226	IRDMA_COMPL_STATUS_INVALID_WQE,
227	IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
228	IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
229	IRDMA_COMPL_STATUS_INVALID_STAG,
230	IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
231	IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
232	IRDMA_COMPL_STATUS_INVALID_PD_ID,
233	IRDMA_COMPL_STATUS_WRAP_ERROR,
234	IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
235	IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
236	IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
237	IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
238	IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
239	IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
240	IRDMA_COMPL_STATUS_INVALID_FBO,
241	IRDMA_COMPL_STATUS_INVALID_LEN,
242	IRDMA_COMPL_STATUS_INVALID_ACCESS,
243	IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
244	IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
245	IRDMA_COMPL_STATUS_INVALID_REGION,
246	IRDMA_COMPL_STATUS_INVALID_WINDOW,
247	IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
248	IRDMA_COMPL_STATUS_UNKNOWN,
249};
250
251enum irdma_cmpl_notify {
252	IRDMA_CQ_COMPL_EVENT     = 0,
253	IRDMA_CQ_COMPL_SOLICITED = 1,
254};
255
256enum irdma_qp_caps {
257	IRDMA_WRITE_WITH_IMM = 1,
258	IRDMA_SEND_WITH_IMM  = 2,
259	IRDMA_ROCE	     = 4,
260	IRDMA_PUSH_MODE      = 8,
261};
262
263struct irdma_srq_uk;
264struct irdma_srq_uk_init_info;
265struct irdma_qp_uk;
266struct irdma_cq_uk;
267struct irdma_qp_uk_init_info;
268struct irdma_cq_uk_init_info;
269
270struct irdma_ring {
271	u32 head;
272	u32 tail;
273	u32 size;
274};
275
276struct irdma_cqe {
277	__le64 buf[IRDMA_CQE_SIZE];
278};
279
280struct irdma_extended_cqe {
281	__le64 buf[IRDMA_EXTENDED_CQE_SIZE];
282};
283
284struct irdma_post_send {
285	struct ib_sge *sg_list;
286	u32 num_sges;
287	u32 qkey;
288	u32 dest_qp;
289	u32 ah_id;
290};
291
292struct irdma_post_rq_info {
293	u64 wr_id;
294	struct ib_sge *sg_list;
295	u32 num_sges;
296};
297
298struct irdma_rdma_write {
299	struct ib_sge *lo_sg_list;
300	u32 num_lo_sges;
301	struct ib_sge rem_addr;
302};
303
304struct irdma_rdma_read {
305	struct ib_sge *lo_sg_list;
306	u32 num_lo_sges;
307	struct ib_sge rem_addr;
308};
309
310struct irdma_bind_window {
311	irdma_stag mr_stag;
312	u64 bind_len;
313	void *va;
314	enum irdma_addressing_type addressing_type;
315	bool ena_reads:1;
316	bool ena_writes:1;
317	irdma_stag mw_stag;
318	bool mem_window_type_1:1;
319	bool remote_atomics_en:1;
320};
321
322struct irdma_atomic_fetch_add {
323	u64 tagged_offset;
324	u64 remote_tagged_offset;
325	u64 fetch_add_data_bytes;
326	u32 stag;
327	u32 remote_stag;
328};
329
330struct irdma_atomic_compare_swap {
331	u64 tagged_offset;
332	u64 remote_tagged_offset;
333	u64 swap_data_bytes;
334	u64 compare_data_bytes;
335	u32 stag;
336	u32 remote_stag;
337};
338
339struct irdma_inv_local_stag {
340	irdma_stag target_stag;
341};
342
343struct irdma_post_sq_info {
344	u64 wr_id;
345	u8 op_type;
346	u8 l4len;
347	bool signaled:1;
348	bool read_fence:1;
349	bool local_fence:1;
350	bool inline_data:1;
351	bool imm_data_valid:1;
352	bool report_rtt:1;
353	bool udp_hdr:1;
354	bool defer_flag:1;
355	bool remote_atomic_en:1;
356	u32 imm_data;
357	u32 stag_to_inv;
358	union {
359		struct irdma_post_send send;
360		struct irdma_rdma_write rdma_write;
361		struct irdma_rdma_read rdma_read;
362		struct irdma_bind_window bind_window;
363		struct irdma_inv_local_stag inv_local_stag;
364		struct irdma_atomic_fetch_add atomic_fetch_add;
365		struct irdma_atomic_compare_swap atomic_compare_swap;
366	} op;
367};
368
369struct irdma_cq_poll_info {
370	u64 wr_id;
371	irdma_qp_handle qp_handle;
372	u32 bytes_xfered;
373	u32 tcp_seq_num_rtt;
374	u32 qp_id;
375	u32 ud_src_qpn;
376	u32 imm_data;
377	irdma_stag inv_stag; /* or L_R_Key */
378	enum irdma_cmpl_status comp_status;
379	u16 major_err;
380	u16 minor_err;
381	u16 ud_vlan;
382	u8 ud_smac[6];
383	u8 op_type;
384	u8 q_type;
385	bool stag_invalid_set:1; /* or L_R_Key set */
386	bool error:1;
387	bool solicited_event:1;
388	bool ipv4:1;
389	bool ud_vlan_valid:1;
390	bool ud_smac_valid:1;
391	bool imm_valid:1;
392};
393
394struct qp_err_code {
395	enum irdma_flush_opcode flush_code;
396	enum irdma_qp_event_type event_type;
397};
398
399int irdma_uk_atomic_compare_swap(struct irdma_qp_uk *qp,
400				 struct irdma_post_sq_info *info, bool post_sq);
401int irdma_uk_atomic_fetch_add(struct irdma_qp_uk *qp,
402			      struct irdma_post_sq_info *info, bool post_sq);
403int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
404			       struct irdma_post_sq_info *info, bool post_sq);
405int irdma_uk_inline_send(struct irdma_qp_uk *qp,
406			 struct irdma_post_sq_info *info, bool post_sq);
407int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,
408		      bool post_sq);
409int irdma_uk_post_receive(struct irdma_qp_uk *qp,
410			  struct irdma_post_rq_info *info);
411void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
412int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
413		       bool inv_stag, bool post_sq);
414int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
415			bool post_sq);
416int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
417		  bool post_sq);
418int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
419				   struct irdma_post_sq_info *info,
420				   bool post_sq);
421
422struct irdma_wqe_uk_ops {
423	void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list,
424				    u32 num_sges, u8 polarity);
425	u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
426	void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
427				u8 valid);
428	void (*iw_set_mw_bind_wqe)(__le64 *wqe,
429				   struct irdma_bind_window *op_info);
430};
431
432bool irdma_uk_cq_empty(struct irdma_cq_uk *cq);
433int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
434			  struct irdma_cq_poll_info *info);
435void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
436				      enum irdma_cmpl_notify cq_notify);
437void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
438void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
439void irdma_uk_cq_init(struct irdma_cq_uk *cq,
440		      struct irdma_cq_uk_init_info *info);
441int irdma_uk_qp_init(struct irdma_qp_uk *qp,
442		     struct irdma_qp_uk_init_info *info);
443void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
444			    u8 *rq_shift);
445int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
446				 u32 *sq_depth, u8 *sq_shift);
447int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
448				 u32 *rq_depth, u8 *rq_shift);
449int irdma_uk_srq_init(struct irdma_srq_uk *srq,
450		      struct irdma_srq_uk_init_info *info);
451int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,
452			      struct irdma_post_rq_info *info);
453
454struct irdma_srq_uk {
455	u32 srq_caps;
456	struct irdma_qp_quanta *srq_base;
457	struct irdma_uk_attrs *uk_attrs;
458	__le64 *shadow_area;
459	struct irdma_ring srq_ring;
460	u32 srq_id;
461	u32 srq_size;
462	u32 max_srq_frag_cnt;
463	struct irdma_wqe_uk_ops wqe_ops;
464	u8 srwqe_polarity;
465	u8 wqe_size;
466	u8 wqe_size_multiplier;
467	u8 deferred_flag;
468	spinlock_t *lock;
469};
470
471struct irdma_srq_uk_init_info {
472	struct irdma_qp_quanta *srq;
473	struct irdma_uk_attrs *uk_attrs;
474	__le64 *shadow_area;
475	u64 *srq_wrid_array;
476	u32 srq_id;
477	u32 srq_caps;
478	u32 srq_size;
479	u32 max_srq_frag_cnt;
480};
481
482struct irdma_sq_uk_wr_trk_info {
483	u64 wrid;
484	u32 wr_len;
485	u16 quanta;
486	u8 signaled;
487	u8 reserved[1];
488};
489
490struct irdma_qp_quanta {
491	__le64 elem[IRDMA_WQE_SIZE];
492};
493
494struct irdma_qp_uk {
495	struct irdma_qp_quanta *sq_base;
496	struct irdma_qp_quanta *rq_base;
497	struct irdma_uk_attrs *uk_attrs;
498	u32 __iomem *wqe_alloc_db;
499	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
500	u64 *rq_wrid_array;
501	__le64 *shadow_area;
502	struct irdma_ring sq_ring;
503	struct irdma_ring rq_ring;
504	struct irdma_ring initial_ring;
505	u32 qp_id;
506	u32 qp_caps;
507	u32 sq_size;
508	u32 rq_size;
509	u32 max_sq_frag_cnt;
510	u32 max_rq_frag_cnt;
511	u32 max_inline_data;
512	struct irdma_wqe_uk_ops wqe_ops;
513	u16 conn_wqes;
514	u8 qp_type;
515	u8 swqe_polarity;
516	u8 swqe_polarity_deferred;
517	u8 rwqe_polarity;
518	u8 rq_wqe_size;
519	u8 rq_wqe_size_multiplier;
520	bool deferred_flag:1;
521	bool first_sq_wq:1;
522	bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */
523	bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */
524	bool destroy_pending:1; /* Indicates the QP is being destroyed */
525	void *back_qp;
526	u8 dbg_rq_flushed;
527	struct irdma_srq_uk *srq_uk;
528	u8 sq_flush_seen;
529	u8 rq_flush_seen;
530};
531
532struct irdma_cq_uk {
533	struct irdma_cqe *cq_base;
534	u32 __iomem *cqe_alloc_db;
535	u32 __iomem *cq_ack_db;
536	__le64 *shadow_area;
537	u32 cq_id;
538	u32 cq_size;
539	struct irdma_ring cq_ring;
540	u8 polarity;
541	bool avoid_mem_cflct:1;
542};
543
544struct irdma_qp_uk_init_info {
545	struct irdma_qp_quanta *sq;
546	struct irdma_qp_quanta *rq;
547	struct irdma_uk_attrs *uk_attrs;
548	u32 __iomem *wqe_alloc_db;
549	__le64 *shadow_area;
550	struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
551	u64 *rq_wrid_array;
552	u32 qp_id;
553	u32 qp_caps;
554	u32 sq_size;
555	u32 rq_size;
556	u32 max_sq_frag_cnt;
557	u32 max_rq_frag_cnt;
558	u32 max_inline_data;
559	u32 sq_depth;
560	u32 rq_depth;
561	u8 first_sq_wq;
562	u8 type;
563	u8 sq_shift;
564	u8 rq_shift;
565	int abi_ver;
566	bool legacy_mode;
567	struct irdma_srq_uk *srq_uk;
568};
569
570struct irdma_cq_uk_init_info {
571	u32 __iomem *cqe_alloc_db;
572	u32 __iomem *cq_ack_db;
573	struct irdma_cqe *cq_base;
574	__le64 *shadow_area;
575	u32 cq_size;
576	u32 cq_id;
577	bool avoid_mem_cflct;
578};
579
580__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
581				   u16 quanta, u32 total_size,
582				   struct irdma_post_sq_info *info);
583__le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx);
584__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
585void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
586int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
587int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
588int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
589void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
590			 u32 inline_data, u8 *shift);
591int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
592		      u32 *wqdepth);
593int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
594		      u32 *wqdepth);
595int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,
596		       u32 *srqdepth);
597void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
598
599static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
600{
601	struct qp_err_code qp_err = {};
602
603	switch (ae_id) {
604	case IRDMA_AE_AMP_BOUNDS_VIOLATION:
605	case IRDMA_AE_AMP_INVALID_STAG:
606	case IRDMA_AE_AMP_RIGHTS_VIOLATION:
607	case IRDMA_AE_AMP_UNALLOCATED_STAG:
608	case IRDMA_AE_AMP_BAD_PD:
609	case IRDMA_AE_AMP_BAD_QP:
610	case IRDMA_AE_AMP_BAD_STAG_KEY:
611	case IRDMA_AE_AMP_BAD_STAG_INDEX:
612	case IRDMA_AE_AMP_TO_WRAP:
613	case IRDMA_AE_PRIV_OPERATION_DENIED:
614		qp_err.flush_code = FLUSH_PROT_ERR;
615		qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
616		break;
617	case IRDMA_AE_UDA_XMIT_BAD_PD:
618	case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
619		qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
620		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
621		break;
622	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
623	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
624	case IRDMA_AE_UDA_L4LEN_INVALID:
625	case IRDMA_AE_DDP_UBE_INVALID_MO:
626	case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
627		qp_err.flush_code = FLUSH_LOC_LEN_ERR;
628		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
629		break;
630	case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
631	case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
632		qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
633		qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
634		break;
635	case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
636	case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
637	case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
638	case IRDMA_AE_AMP_MWBIND_VALID_STAG:
639		qp_err.flush_code = FLUSH_MW_BIND_ERR;
640		qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
641		break;
642	case IRDMA_AE_LLP_TOO_MANY_RETRIES:
643		qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
644		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
645		break;
646	case IRDMA_AE_IB_INVALID_REQUEST:
647		qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
648		qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
649		break;
650	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
651	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
652	case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
653	case IRDMA_AE_IB_REMOTE_OP_ERROR:
654		qp_err.flush_code = FLUSH_REM_OP_ERR;
655		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
656		break;
657	case IRDMA_AE_LLP_TOO_MANY_RNRS:
658		qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;
659		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
660		break;
661	case IRDMA_AE_LCE_QP_CATASTROPHIC:
662	case IRDMA_AE_REMOTE_QP_CATASTROPHIC:
663	case IRDMA_AE_LOCAL_QP_CATASTROPHIC:
664	case IRDMA_AE_RCE_QP_CATASTROPHIC:
665		qp_err.flush_code = FLUSH_FATAL_ERR;
666		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
667		break;
668	default:
669		qp_err.flush_code = FLUSH_GENERAL_ERR;
670		qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
671		break;
672	}
673
674	return qp_err;
675}
676#endif /* IRDMA_USER_H */