erdma_hw.h

  1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
  2
  3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
  4/*          Kai Shen <kaishen@linux.alibaba.com> */
  5/* Copyright (c) 2020-2022, Alibaba Group. */
  6
  7#ifndef __ERDMA_HW_H__
  8#define __ERDMA_HW_H__
  9
 10#include <linux/kernel.h>
 11#include <linux/types.h>
 12#include <linux/if_ether.h>
 13
 14/* PCIe device related definition. */
 15#define ERDMA_PCI_WIDTH 64
 16#define ERDMA_FUNC_BAR 0
 17#define ERDMA_MISX_BAR 2
 18
 19#define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))
 20
 21/* MSI-X related. */
 22#define ERDMA_NUM_MSIX_VEC 32U
 23#define ERDMA_MSIX_VECTOR_CMDQ 0
 24
 25/* RoCEv2 related */
 26#define ERDMA_ROCEV2_GID_SIZE 16
 27#define ERDMA_MAX_PKEYS 1
 28#define ERDMA_DEFAULT_PKEY 0xFFFF
 29
 30/* erdma device protocol type */
 31enum erdma_proto_type {
 32	ERDMA_PROTO_IWARP = 0,
 33	ERDMA_PROTO_ROCEV2 = 1,
 34	ERDMA_PROTO_COUNT = 2,
 35};
 36
 37/* PCIe Bar0 Registers. */
 38#define ERDMA_REGS_VERSION_REG 0x0
 39#define ERDMA_REGS_DEV_PROTO_REG 0xC
 40#define ERDMA_REGS_DEV_CTRL_REG 0x10
 41#define ERDMA_REGS_DEV_ST_REG 0x14
 42#define ERDMA_REGS_NETDEV_MAC_L_REG 0x18
 43#define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C
 44#define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20
 45#define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24
 46#define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28
 47#define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C
 48#define ERDMA_REGS_CMDQ_DEPTH_REG 0x30
 49#define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34
 50#define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38
 51#define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C
 52#define ERDMA_REGS_AEQ_ADDR_L_REG 0x40
 53#define ERDMA_REGS_AEQ_ADDR_H_REG 0x44
 54#define ERDMA_REGS_AEQ_DEPTH_REG 0x48
 55#define ERDMA_REGS_GRP_NUM_REG 0x4c
 56#define ERDMA_REGS_AEQ_DB_REG 0x50
 57#define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60
 58#define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68
 59#define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70
 60#define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78
 61#define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80
 62#define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88
 63#define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90
 64#define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98
 65#define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0
 66#define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8
 67#define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0
 68#define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8
 69#define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0
 70#define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8
 71#define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0
 72#define ERDMA_REGS_CEQ_DB_BASE_REG 0x100
 73#define ERDMA_CMDQ_SQDB_REG 0x200
 74#define ERDMA_CMDQ_CQDB_REG 0x300
 75
 76/* DEV_CTRL_REG details. */
 77#define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001
 78#define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002
 79
 80/* DEV_ST_REG details. */
 81#define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U
 82#define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U
 83
 84/* eRDMA PCIe DBs definition. */
 85#define ERDMA_BAR_DB_SPACE_BASE 4096
 86
 87#define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE
 88#define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024)
 89
 90#define ERDMA_BAR_RQDB_SPACE_OFFSET \
 91	(ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)
 92#define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024)
 93
 94#define ERDMA_BAR_CQDB_SPACE_OFFSET \
 95	(ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)
 96
 97#define ERDMA_SDB_SHARED_PAGE_INDEX 95
 98
 99/* Doorbell related. */
100#define ERDMA_DB_SIZE 8
101
102#define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56)
103#define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)
104#define ERDMA_CQDB_ARM_MASK BIT_ULL(31)
105#define ERDMA_CQDB_SOL_MASK BIT_ULL(30)
106#define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)
107#define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)
108
109#define ERDMA_EQDB_ARM_MASK BIT(31)
110#define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0)
111
112#define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
113
114/* Hardware page size definition */
115#define ERDMA_HW_PAGE_SHIFT 12
116#define ERDMA_HW_PAGE_SIZE 4096
117
118/* WQE related. */
119#define EQE_SIZE 16
120#define EQE_SHIFT 4
121#define RQE_SIZE 32
122#define RQE_SHIFT 5
123#define CQE_SIZE 32
124#define CQE_SHIFT 5
125#define SQEBB_SIZE 32
126#define SQEBB_SHIFT 5
127#define SQEBB_MASK (~(SQEBB_SIZE - 1))
128#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
129#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
130
131#define ERDMA_MAX_SQE_SIZE 128
132#define ERDMA_MAX_WQEBB_PER_SQE 4
133
134/* CMDQ related. */
135#define ERDMA_CMDQ_MAX_OUTSTANDING 128
136#define ERDMA_CMDQ_SQE_SIZE 128
137
138/* cmdq sub module definition. */
139enum CMDQ_WQE_SUB_MOD {
140	CMDQ_SUBMOD_RDMA = 0,
141	CMDQ_SUBMOD_COMMON = 1
142};
143
144enum CMDQ_RDMA_OPCODE {
145	CMDQ_OPCODE_QUERY_DEVICE = 0,
146	CMDQ_OPCODE_CREATE_QP = 1,
147	CMDQ_OPCODE_DESTROY_QP = 2,
148	CMDQ_OPCODE_MODIFY_QP = 3,
149	CMDQ_OPCODE_CREATE_CQ = 4,
150	CMDQ_OPCODE_DESTROY_CQ = 5,
151	CMDQ_OPCODE_REFLUSH = 6,
152	CMDQ_OPCODE_REG_MR = 8,
153	CMDQ_OPCODE_DEREG_MR = 9,
154	CMDQ_OPCODE_SET_GID = 14,
155	CMDQ_OPCODE_CREATE_AH = 15,
156	CMDQ_OPCODE_DESTROY_AH = 16,
157	CMDQ_OPCODE_QUERY_QP = 17,
158};
159
160enum CMDQ_COMMON_OPCODE {
161	CMDQ_OPCODE_CREATE_EQ = 0,
162	CMDQ_OPCODE_DESTROY_EQ = 1,
163	CMDQ_OPCODE_QUERY_FW_INFO = 2,
164	CMDQ_OPCODE_CONF_MTU = 3,
165	CMDQ_OPCODE_GET_STATS = 4,
166	CMDQ_OPCODE_CONF_DEVICE = 5,
167	CMDQ_OPCODE_ALLOC_DB = 8,
168	CMDQ_OPCODE_FREE_DB = 9,
169};
170
171/* cmdq-SQE HDR */
172#define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
173#define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32)
174#define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24)
175#define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16)
176#define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
177
178struct erdma_cmdq_destroy_cq_req {
179	u64 hdr;
180	u32 cqn;
181};
182
183#define ERDMA_EQ_TYPE_AEQ 0
184#define ERDMA_EQ_TYPE_CEQ 1
185
186struct erdma_cmdq_create_eq_req {
187	u64 hdr;
188	u64 qbuf_addr;
189	u8 vector_idx;
190	u8 eqn;
191	u8 depth;
192	u8 qtype;
193	u32 db_dma_addr_l;
194	u32 db_dma_addr_h;
195};
196
197struct erdma_cmdq_destroy_eq_req {
198	u64 hdr;
199	u64 rsvd0;
200	u8 vector_idx;
201	u8 eqn;
202	u8 rsvd1;
203	u8 qtype;
204};
205
206/* config device cfg */
207#define ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK BIT(31)
208#define ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK GENMASK(4, 0)
209
210struct erdma_cmdq_config_device_req {
211	u64 hdr;
212	u32 cfg;
213	u32 rsvd[5];
214};
215
216struct erdma_cmdq_config_mtu_req {
217	u64 hdr;
218	u32 mtu;
219};
220
221/* ext db requests(alloc and free) cfg */
222#define ERDMA_CMD_EXT_DB_CQ_EN_MASK BIT(2)
223#define ERDMA_CMD_EXT_DB_RQ_EN_MASK BIT(1)
224#define ERDMA_CMD_EXT_DB_SQ_EN_MASK BIT(0)
225
226struct erdma_cmdq_ext_db_req {
227	u64 hdr;
228	u32 cfg;
229	u16 rdb_off;
230	u16 sdb_off;
231	u16 rsvd0;
232	u16 cdb_off;
233	u32 rsvd1[3];
234};
235
236/* alloc db response qword 0 definition */
237#define ERDMA_CMD_ALLOC_DB_RESP_RDB_MASK GENMASK_ULL(63, 48)
238#define ERDMA_CMD_ALLOC_DB_RESP_CDB_MASK GENMASK_ULL(47, 32)
239#define ERDMA_CMD_ALLOC_DB_RESP_SDB_MASK GENMASK_ULL(15, 0)
240
241/* create_cq cfg0 */
242#define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24)
243#define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20)
244#define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0)
245
246/* create_cq cfg1 */
247#define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)
248#define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15)
249#define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)
250#define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)
251
252/* create_cq cfg2 */
253#define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0)
254
255struct erdma_cmdq_create_cq_req {
256	u64 hdr;
257	u32 cfg0;
258	u32 qbuf_addr_l;
259	u32 qbuf_addr_h;
260	u32 cfg1;
261	u64 cq_dbrec_dma;
262	u32 first_page_offset;
263	u32 cfg2;
264};
265
266/* regmr/deregmr cfg0 */
267#define ERDMA_CMD_MR_VALID_MASK BIT(31)
268#define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28)
269#define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)
270#define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)
271
272/* regmr cfg1 */
273#define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
274#define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
275#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
276
277/* regmr cfg2 */
278#define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
279#define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24)
280#define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)
281#define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)
282
283struct erdma_cmdq_reg_mr_req {
284	u64 hdr;
285	u32 cfg0;
286	u32 cfg1;
287	u64 start_va;
288	u32 size;
289	u32 cfg2;
290	union {
291		u64 phy_addr[4];
292		struct {
293			u64 rsvd;
294			u32 size_h;
295			u32 mtt_cnt_h;
296		};
297	};
298};
299
300struct erdma_cmdq_dereg_mr_req {
301	u64 hdr;
302	u32 cfg;
303};
304
305/* create_av cfg0 */
306#define ERDMA_CMD_CREATE_AV_FL_MASK GENMASK(19, 0)
307#define ERDMA_CMD_CREATE_AV_NTYPE_MASK BIT(20)
308
309struct erdma_av_cfg {
310	u32 cfg0;
311	u8 traffic_class;
312	u8 hop_limit;
313	u8 sl;
314	u8 rsvd;
315	u16 udp_sport;
316	u16 sgid_index;
317	u8 dmac[ETH_ALEN];
318	u8 padding[2];
319	u8 dgid[ERDMA_ROCEV2_GID_SIZE];
320};
321
322struct erdma_cmdq_create_ah_req {
323	u64 hdr;
324	u32 pdn;
325	u32 ahn;
326	struct erdma_av_cfg av_cfg;
327};
328
329struct erdma_cmdq_destroy_ah_req {
330	u64 hdr;
331	u32 pdn;
332	u32 ahn;
333};
334
335/* modify qp cfg */
336#define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24)
337#define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20)
338#define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0)
339
340struct erdma_cmdq_modify_qp_req {
341	u64 hdr;
342	u32 cfg;
343	u32 cookie;
344	__be32 dip;
345	__be32 sip;
346	__be16 sport;
347	__be16 dport;
348	u32 send_nxt;
349	u32 recv_nxt;
350};
351
352/* modify qp cfg1 for roce device */
353#define ERDMA_CMD_MODIFY_QP_DQPN_MASK GENMASK(19, 0)
354
355struct erdma_cmdq_mod_qp_req_rocev2 {
356	u64 hdr;
357	u32 cfg0;
358	u32 cfg1;
359	u32 attr_mask;
360	u32 qkey;
361	u32 rq_psn;
362	u32 sq_psn;
363	struct erdma_av_cfg av_cfg;
364};
365
366/* query qp response mask */
367#define ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK GENMASK_ULL(23, 0)
368#define ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK GENMASK_ULL(47, 24)
369#define ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK GENMASK_ULL(55, 48)
370#define ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK GENMASK_ULL(56, 56)
371
372struct erdma_cmdq_query_qp_req_rocev2 {
373	u64 hdr;
374	u32 qpn;
375};
376
377enum erdma_qp_type {
378	ERDMA_QPT_RC = 0,
379	ERDMA_QPT_UD = 1,
380};
381
382/* create qp cfg0 */
383#define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20)
384#define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0)
385
386/* create qp cfg1 */
387#define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20)
388#define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0)
389
390/* create qp cfg2 */
391#define ERDMA_CMD_CREATE_QP_TYPE_MASK GENMASK(3, 0)
392
393/* create qp cqn_mtt_cfg */
394#define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)
395#define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25)
396#define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)
397
398/* create qp mtt_cfg */
399#define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)
400#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)
401#define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0)
402
403/* create qp db cfg */
404#define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)
405#define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0)
406
407#define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)
408
409struct erdma_cmdq_create_qp_req {
410	u64 hdr;
411	u32 cfg0;
412	u32 cfg1;
413	u32 sq_cqn_mtt_cfg;
414	u32 rq_cqn_mtt_cfg;
415	u64 sq_buf_addr;
416	u64 rq_buf_addr;
417	u32 sq_mtt_cfg;
418	u32 rq_mtt_cfg;
419	u64 sq_dbrec_dma;
420	u64 rq_dbrec_dma;
421
422	u64 sq_mtt_entry[3];
423	u64 rq_mtt_entry[3];
424
425	u32 db_cfg;
426	u32 cfg2;
427};
428
429struct erdma_cmdq_destroy_qp_req {
430	u64 hdr;
431	u32 qpn;
432};
433
434struct erdma_cmdq_reflush_req {
435	u64 hdr;
436	u32 qpn;
437	u32 sq_pi;
438	u32 rq_pi;
439};
440
441#define ERDMA_HW_RESP_SIZE 256
442
443struct erdma_cmdq_query_req {
444	u64 hdr;
445	u32 rsvd;
446	u32 index;
447
448	u64 target_addr;
449	u32 target_length;
450};
451
452#define ERDMA_HW_RESP_MAGIC 0x5566
453
454struct erdma_cmdq_query_resp_hdr {
455	u16 magic;
456	u8 ver;
457	u8 length;
458
459	u32 index;
460	u32 rsvd[2];
461};
462
463struct erdma_cmdq_query_stats_resp {
464	struct erdma_cmdq_query_resp_hdr hdr;
465
466	u64 tx_req_cnt;
467	u64 tx_packets_cnt;
468	u64 tx_bytes_cnt;
469	u64 tx_drop_packets_cnt;
470	u64 tx_bps_meter_drop_packets_cnt;
471	u64 tx_pps_meter_drop_packets_cnt;
472	u64 rx_packets_cnt;
473	u64 rx_bytes_cnt;
474	u64 rx_drop_packets_cnt;
475	u64 rx_bps_meter_drop_packets_cnt;
476	u64 rx_pps_meter_drop_packets_cnt;
477};
478
479enum erdma_network_type {
480	ERDMA_NETWORK_TYPE_IPV4 = 0,
481	ERDMA_NETWORK_TYPE_IPV6 = 1,
482};
483
484enum erdma_set_gid_op {
485	ERDMA_SET_GID_OP_ADD = 0,
486	ERDMA_SET_GID_OP_DEL = 1,
487};
488
489/* set gid cfg */
490#define ERDMA_CMD_SET_GID_SGID_IDX_MASK GENMASK(15, 0)
491#define ERDMA_CMD_SET_GID_NTYPE_MASK BIT(16)
492#define ERDMA_CMD_SET_GID_OP_MASK BIT(31)
493
494struct erdma_cmdq_set_gid_req {
495	u64 hdr;
496	u32 cfg;
497	u8 gid[ERDMA_ROCEV2_GID_SIZE];
498};
499
500/* cap qword 0 definition */
501#define ERDMA_CMD_DEV_CAP_MAX_GID_MASK GENMASK_ULL(51, 48)
502#define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
503#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
504#define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
505#define ERDMA_CMD_DEV_CAP_MAX_AH_MASK GENMASK_ULL(15, 8)
506#define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
507
508/* cap qword 1 definition */
509#define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)
510#define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28)
511#define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16)
512#define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0)
513
514#define ERDMA_NQP_PER_QBLOCK 1024
515
516enum {
517	ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
518	ERDMA_DEV_CAP_FLAGS_MTT_VA = 1 << 5,
519	ERDMA_DEV_CAP_FLAGS_EXTEND_DB = 1 << 3,
520};
521
522#define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
523
524/* CQE hdr */
525#define ERDMA_CQE_HDR_OWNER_MASK BIT(31)
526#define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)
527#define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)
528#define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)
529
530#define ERDMA_CQE_QTYPE_SQ 0
531#define ERDMA_CQE_QTYPE_RQ 1
532#define ERDMA_CQE_QTYPE_CMDQ 2
533
534#define ERDMA_CQE_NTYPE_MASK BIT(31)
535#define ERDMA_CQE_SL_MASK GENMASK(27, 20)
536#define ERDMA_CQE_SQPN_MASK GENMASK(19, 0)
537
538struct erdma_cqe {
539	__be32 hdr;
540	__be32 qe_idx;
541	__be32 qpn;
542	union {
543		__le32 imm_data;
544		__be32 inv_rkey;
545	};
546	__be32 size;
547	union {
548		struct {
549			__be32 rsvd[3];
550		} rc;
551
552		struct {
553			__be32 rsvd[2];
554			__be32 info;
555		} ud;
556	};
557};
558
559struct erdma_sge {
560	__aligned_le64 addr;
561	__le32 length;
562	__le32 key;
563};
564
565/* Receive Queue Element */
566struct erdma_rqe {
567	__le16 qe_idx;
568	__le16 rsvd0;
569	__le32 qpn;
570	__le32 rsvd1;
571	__le32 rsvd2;
572	__le64 to;
573	__le32 length;
574	__le32 stag;
575};
576
577/* SQE */
578#define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)
579#define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
580#define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)
581#define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)
582#define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)
583#define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)
584#define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)
585#define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)
586#define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)
587#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
588
589/* REG MR attrs */
590#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
591#define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
592#define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
593
594struct erdma_write_sqe {
595	__le64 hdr;
596	__be32 imm_data;
597	__le32 length;
598
599	__le32 sink_stag;
600	__le32 sink_to_l;
601	__le32 sink_to_h;
602
603	__le32 rsvd;
604
605	struct erdma_sge sgl[];
606};
607
608struct erdma_send_sqe_rc {
609	__le64 hdr;
610	union {
611		__be32 imm_data;
612		__le32 invalid_stag;
613	};
614
615	__le32 length;
616	struct erdma_sge sgl[];
617};
618
619struct erdma_send_sqe_ud {
620	__le64 hdr;
621	__be32 imm_data;
622	__le32 length;
623	__le32 qkey;
624	__le32 dst_qpn;
625	__le32 ahn;
626	__le32 rsvd;
627	struct erdma_sge sgl[];
628};
629
630struct erdma_readreq_sqe {
631	__le64 hdr;
632	__le32 invalid_stag;
633	__le32 length;
634	__le32 sink_stag;
635	__le32 sink_to_l;
636	__le32 sink_to_h;
637	__le32 rsvd;
638};
639
640struct erdma_atomic_sqe {
641	__le64 hdr;
642	__le64 rsvd;
643	__le64 fetchadd_swap_data;
644	__le64 cmp_data;
645
646	struct erdma_sge remote;
647	struct erdma_sge sgl;
648};
649
650struct erdma_reg_mr_sqe {
651	__le64 hdr;
652	__le64 addr;
653	__le32 length;
654	__le32 stag;
655	__le32 attrs;
656	__le32 rsvd;
657};
658
659/* EQ related. */
660#define ERDMA_DEFAULT_EQ_DEPTH 4096
661
662/* ceqe */
663#define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)
664#define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32)
665#define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31)
666#define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)
667
668/* aeqe */
669#define ERDMA_AEQE_HDR_O_MASK BIT(31)
670#define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16)
671#define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)
672
673#define ERDMA_AE_TYPE_QP_FATAL_EVENT 0
674#define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1
675#define ERDMA_AE_TYPE_ACC_ERR_EVENT 2
676#define ERDMA_AE_TYPE_CQ_ERR 3
677#define ERDMA_AE_TYPE_OTHER_ERROR 4
678
679struct erdma_aeqe {
680	__le32 hdr;
681	__le32 event_data0;
682	__le32 event_data1;
683	__le32 rsvd;
684};
685
686enum erdma_opcode {
687	ERDMA_OP_WRITE = 0,
688	ERDMA_OP_READ = 1,
689	ERDMA_OP_SEND = 2,
690	ERDMA_OP_SEND_WITH_IMM = 3,
691
692	ERDMA_OP_RECEIVE = 4,
693	ERDMA_OP_RECV_IMM = 5,
694	ERDMA_OP_RECV_INV = 6,
695
696	ERDMA_OP_RSVD0 = 7,
697	ERDMA_OP_RSVD1 = 8,
698	ERDMA_OP_WRITE_WITH_IMM = 9,
699
700	ERDMA_OP_RSVD2 = 10,
701	ERDMA_OP_RSVD3 = 11,
702
703	ERDMA_OP_RSP_SEND_IMM = 12,
704	ERDMA_OP_SEND_WITH_INV = 13,
705
706	ERDMA_OP_REG_MR = 14,
707	ERDMA_OP_LOCAL_INV = 15,
708	ERDMA_OP_READ_WITH_INV = 16,
709	ERDMA_OP_ATOMIC_CAS = 17,
710	ERDMA_OP_ATOMIC_FAA = 18,
711	ERDMA_NUM_OPCODES = 19,
712	ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
713};
714
715enum erdma_wc_status {
716	ERDMA_WC_SUCCESS = 0,
717	ERDMA_WC_GENERAL_ERR = 1,
718	ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
719	ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
720	ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
721	ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
722	ERDMA_WC_RECV_PDID_ERR = 6,
723	ERDMA_WC_RECV_WARRPING_ERR = 7,
724	ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
725	ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
726	ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
727	ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
728	ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
729	ERDMA_WC_SEND_PDID_ERR = 13,
730	ERDMA_WC_SEND_WARRPING_ERR = 14,
731	ERDMA_WC_FLUSH_ERR = 15,
732	ERDMA_WC_RETRY_EXC_ERR = 16,
733	ERDMA_NUM_WC_STATUS
734};
735
736enum erdma_vendor_err {
737	ERDMA_WC_VENDOR_NO_ERR = 0,
738	ERDMA_WC_VENDOR_INVALID_RQE = 1,
739	ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
740	ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
741	ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
742	ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
743	ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
744	ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
745	ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
746	ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
747	ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
748	ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
749	ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
750	ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
751};
752
753#endif