efa_io_defs.h

  1/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
  2/*
  3 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  4 */
  5
  6#ifndef _EFA_IO_H_
  7#define _EFA_IO_H_
  8
  9#define EFA_IO_TX_DESC_NUM_BUFS              2
 10#define EFA_IO_TX_DESC_NUM_RDMA_BUFS         1
 11#define EFA_IO_TX_DESC_INLINE_MAX_SIZE       32
 12#define EFA_IO_TX_DESC_IMM_DATA_SIZE         4
 13#define EFA_IO_TX_DESC_INLINE_PBL_SIZE       1
 14
 15enum efa_io_queue_type {
 16	/* send queue (of a QP) */
 17	EFA_IO_SEND_QUEUE                           = 1,
 18	/* recv queue (of a QP) */
 19	EFA_IO_RECV_QUEUE                           = 2,
 20};
 21
 22enum efa_io_send_op_type {
 23	/* send message */
 24	EFA_IO_SEND                                 = 0,
 25	/* RDMA read */
 26	EFA_IO_RDMA_READ                            = 1,
 27	/* RDMA write */
 28	EFA_IO_RDMA_WRITE                           = 2,
 29	/* Fast MR registration */
 30	EFA_IO_FAST_REG                             = 3,
 31	/* Fast MR invalidation */
 32	EFA_IO_FAST_INV                             = 4,
 33};
 34
 35enum efa_io_comp_status {
 36	/* Successful completion */
 37	EFA_IO_COMP_STATUS_OK                       = 0,
 38	/* Flushed during QP destroy */
 39	EFA_IO_COMP_STATUS_FLUSHED                  = 1,
 40	/* Internal QP error */
 41	EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,
 42	/* Unsupported operation */
 43	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP = 3,
 44	/* Bad AH */
 45	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH   = 4,
 46	/* LKEY not registered or does not match IOVA */
 47	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,
 48	/* Message too long */
 49	EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH   = 6,
 50	/* RKEY not registered or does not match remote IOVA */
 51	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,
 52	/* Connection was reset by remote side */
 53	EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT       = 8,
 54	/* Bad dest QP number (QP does not exist or is in error state) */
 55	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,
 56	/* Destination resource not ready (no WQEs posted on RQ) */
 57	EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR         = 10,
 58	/* Receiver SGL too short */
 59	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH  = 11,
 60	/* Unexpected status returned by responder */
 61	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS  = 12,
 62	/* Unresponsive remote - was previously responsive */
 63	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,
 64	/* No valid AH at remote side (required for RDMA operations) */
 65	EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER = 14,
 66	/* Unreachable remote - never received a response */
 67	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE = 15,
 68};
 69
 70enum efa_io_frwr_pbl_mode {
 71	EFA_IO_FRWR_INLINE_PBL                      = 0,
 72	EFA_IO_FRWR_DIRECT_PBL                      = 1,
 73};
 74
 75struct efa_io_tx_meta_desc {
 76	/* Verbs-generated Request ID */
 77	u16 req_id;
 78
 79	/*
 80	 * control flags
 81	 * 3:0 : op_type - enum efa_io_send_op_type
 82	 * 4 : has_imm - immediate_data field carries valid
 83	 *    data.
 84	 * 5 : inline_msg - inline mode - inline message data
 85	 *    follows this descriptor (no buffer descriptors).
 86	 *    Note that it is different from immediate data
 87	 * 6 : meta_extension - Extended metadata. MBZ
 88	 * 7 : meta_desc - Indicates metadata descriptor.
 89	 *    Must be set.
 90	 */
 91	u8 ctrl1;
 92
 93	/*
 94	 * control flags
 95	 * 0 : phase
 96	 * 1 : reserved25 - MBZ
 97	 * 2 : first - Indicates first descriptor in
 98	 *    transaction. Must be set.
 99	 * 3 : last - Indicates last descriptor in
100	 *    transaction. Must be set.
101	 * 4 : comp_req - Indicates whether completion should
102	 *    be posted, after packet is transmitted. Valid only
103	 *    for the first descriptor
104	 * 7:5 : reserved29 - MBZ
105	 */
106	u8 ctrl2;
107
108	u16 dest_qp_num;
109
110	/*
111	 * If inline_msg bit is set, length of inline message in bytes,
112	 * otherwise length of SGL (number of buffers).
113	 */
114	u16 length;
115
116	/*
117	 * immediate data: if has_imm is set, then this field is included within
118	 * Tx message and reported in remote Rx completion.
119	 */
120	u32 immediate_data;
121
122	u16 ah;
123
124	u16 reserved;
125
126	/* Queue key */
127	u32 qkey;
128
129	u8 reserved2[12];
130};
131
132/*
133 * Tx queue buffer descriptor, for any transport type. Preceded by metadata
134 * descriptor.
135 */
136struct efa_io_tx_buf_desc {
137	/* length in bytes */
138	u32 length;
139
140	/*
141	 * 23:0 : lkey - local memory translation key
142	 * 31:24 : reserved - MBZ
143	 */
144	u32 lkey;
145
146	/* Buffer address bits[31:0] */
147	u32 buf_addr_lo;
148
149	/* Buffer address bits[63:32] */
150	u32 buf_addr_hi;
151};
152
153struct efa_io_remote_mem_addr {
154	/* length in bytes */
155	u32 length;
156
157	/* remote memory translation key */
158	u32 rkey;
159
160	/* Buffer address bits[31:0] */
161	u32 buf_addr_lo;
162
163	/* Buffer address bits[63:32] */
164	u32 buf_addr_hi;
165};
166
167struct efa_io_rdma_req {
168	/* Remote memory address */
169	struct efa_io_remote_mem_addr remote_mem;
170
171	/* Local memory address */
172	struct efa_io_tx_buf_desc local_mem[1];
173};
174
175struct efa_io_fast_mr_reg_req {
176	/* Updated local key of the MR after lkey/rkey increment */
177	u32 lkey;
178
179	/*
180	 * permissions
181	 * 0 : local_write_enable - Local write permissions:
182	 *    must be set for RQ buffers and buffers posted for
183	 *    RDMA Read requests
184	 * 1 : remote_write_enable - Remote write
185	 *    permissions: must be set to enable RDMA write to
186	 *    the region
187	 * 2 : remote_read_enable - Remote read permissions:
188	 *    must be set to enable RDMA read from the region
189	 * 7:3 : reserved2 - MBZ
190	 */
191	u8 permissions;
192
193	/*
194	 * control flags
195	 * 4:0 : phys_page_size_shift - page size is (1 <<
196	 *    phys_page_size_shift)
197	 * 6:5 : pbl_mode - enum efa_io_frwr_pbl_mode
198	 * 7 : reserved - MBZ
199	 */
200	u8 flags;
201
202	/* MBZ */
203	u8 reserved[2];
204
205	/* IO Virtual Address associated with this MR */
206	u64 iova;
207
208	/* Memory region length, in bytes */
209	u64 mr_length;
210
211	/* Physical Buffer List, each element is page-aligned. */
212	union {
213		/*
214		 * Inline array of physical page addresses (optimization
215		 * for short region activation).
216		 */
217		u64 inline_array[1];
218
219		/* points to PBL (Currently only direct) */
220		u64 dma_addr;
221	} pbl;
222};
223
224struct efa_io_fast_mr_inv_req {
225	/* Local key of the MR to invalidate */
226	u32 lkey;
227
228	/* MBZ */
229	u8 reserved[28];
230};
231
232/*
233 * Tx WQE, composed of tx meta descriptors followed by either tx buffer
234 * descriptors or inline data
235 */
236struct efa_io_tx_wqe {
237	/* TX meta */
238	struct efa_io_tx_meta_desc meta;
239
240	union {
241		/* Send buffer descriptors */
242		struct efa_io_tx_buf_desc sgl[2];
243
244		u8 inline_data[32];
245
246		/* RDMA local and remote memory addresses */
247		struct efa_io_rdma_req rdma_req;
248
249		/* Fast registration */
250		struct efa_io_fast_mr_reg_req reg_mr_req;
251
252		/* Fast invalidation */
253		struct efa_io_fast_mr_inv_req inv_mr_req;
254	} data;
255};
256
257/*
258 * Rx buffer descriptor; RX WQE is composed of one or more RX buffer
259 * descriptors.
260 */
261struct efa_io_rx_desc {
262	/* Buffer address bits[31:0] */
263	u32 buf_addr_lo;
264
265	/* Buffer Pointer[63:32] */
266	u32 buf_addr_hi;
267
268	/* Verbs-generated request id. */
269	u16 req_id;
270
271	/* Length in bytes. */
272	u16 length;
273
274	/*
275	 * LKey and control flags
276	 * 23:0 : lkey
277	 * 29:24 : reserved - MBZ
278	 * 30 : first - Indicates first descriptor in WQE
279	 * 31 : last - Indicates last descriptor in WQE
280	 */
281	u32 lkey_ctrl;
282};
283
284/* Common IO completion descriptor */
285struct efa_io_cdesc_common {
286	/*
287	 * verbs-generated request ID, as provided in the completed tx or rx
288	 * descriptor.
289	 */
290	u16 req_id;
291
292	u8 status;
293
294	/*
295	 * flags
296	 * 0 : phase - Phase bit
297	 * 2:1 : q_type - enum efa_io_queue_type: send/recv
298	 * 3 : has_imm - indicates that immediate data is
299	 *    present - for RX completions only
300	 * 6:4 : op_type - enum efa_io_send_op_type
301	 * 7 : unsolicited - indicates that there is no
302	 *    matching request - for RDMA with imm. RX only
303	 */
304	u8 flags;
305
306	/* local QP number */
307	u16 qp_num;
308};
309
310/* Tx completion descriptor */
311struct efa_io_tx_cdesc {
312	/* Common completion info */
313	struct efa_io_cdesc_common common;
314
315	/* MBZ */
316	u16 reserved16;
317};
318
319/* Rx Completion Descriptor */
320struct efa_io_rx_cdesc {
321	/* Common completion info */
322	struct efa_io_cdesc_common common;
323
324	/* Transferred length bits[15:0] */
325	u16 length;
326
327	/* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
328	u16 ah;
329
330	u16 src_qp_num;
331
332	/* Immediate data */
333	u32 imm;
334};
335
336/* Rx Completion Descriptor RDMA write info */
337struct efa_io_rx_cdesc_rdma_write {
338	/* Transferred length bits[31:16] */
339	u16 length_hi;
340};
341
342/* Extended Rx Completion Descriptor */
343struct efa_io_rx_cdesc_ex {
344	/* Base RX completion info */
345	struct efa_io_rx_cdesc base;
346
347	union {
348		struct efa_io_rx_cdesc_rdma_write rdma_write;
349
350		/*
351		 * Valid only in case of unknown AH (0xFFFF) and CQ
352		 * set_src_addr is enabled.
353		 */
354		u8 src_addr[16];
355	} u;
356};
357
358/* tx_meta_desc */
359#define EFA_IO_TX_META_DESC_OP_TYPE_MASK                    GENMASK(3, 0)
360#define EFA_IO_TX_META_DESC_HAS_IMM_MASK                    BIT(4)
361#define EFA_IO_TX_META_DESC_INLINE_MSG_MASK                 BIT(5)
362#define EFA_IO_TX_META_DESC_META_EXTENSION_MASK             BIT(6)
363#define EFA_IO_TX_META_DESC_META_DESC_MASK                  BIT(7)
364#define EFA_IO_TX_META_DESC_PHASE_MASK                      BIT(0)
365#define EFA_IO_TX_META_DESC_FIRST_MASK                      BIT(2)
366#define EFA_IO_TX_META_DESC_LAST_MASK                       BIT(3)
367#define EFA_IO_TX_META_DESC_COMP_REQ_MASK                   BIT(4)
368
369/* tx_buf_desc */
370#define EFA_IO_TX_BUF_DESC_LKEY_MASK                        GENMASK(23, 0)
371
372/* fast_mr_reg_req */
373#define EFA_IO_FAST_MR_REG_REQ_LOCAL_WRITE_ENABLE_MASK      BIT(0)
374#define EFA_IO_FAST_MR_REG_REQ_REMOTE_WRITE_ENABLE_MASK     BIT(1)
375#define EFA_IO_FAST_MR_REG_REQ_REMOTE_READ_ENABLE_MASK      BIT(2)
376#define EFA_IO_FAST_MR_REG_REQ_PHYS_PAGE_SIZE_SHIFT_MASK    GENMASK(4, 0)
377#define EFA_IO_FAST_MR_REG_REQ_PBL_MODE_MASK                GENMASK(6, 5)
378
379/* rx_desc */
380#define EFA_IO_RX_DESC_LKEY_MASK                            GENMASK(23, 0)
381#define EFA_IO_RX_DESC_FIRST_MASK                           BIT(30)
382#define EFA_IO_RX_DESC_LAST_MASK                            BIT(31)
383
384/* cdesc_common */
385#define EFA_IO_CDESC_COMMON_PHASE_MASK                      BIT(0)
386#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK                     GENMASK(2, 1)
387#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK                    BIT(3)
388#define EFA_IO_CDESC_COMMON_OP_TYPE_MASK                    GENMASK(6, 4)
389#define EFA_IO_CDESC_COMMON_UNSOLICITED_MASK                BIT(7)
390
391#endif /* _EFA_IO_H_ */