erdma_verbs.h

  1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
  2
  3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
  4/*          Kai Shen <kaishen@linux.alibaba.com> */
  5/* Copyright (c) 2020-2022, Alibaba Group. */
  6
  7#ifndef __ERDMA_VERBS_H__
  8#define __ERDMA_VERBS_H__
  9
 10#include "erdma.h"
 11
 12/* RDMA Capability. */
 13#define ERDMA_MAX_PD (128 * 1024)
 14#define ERDMA_MAX_SEND_WR 8192
 15#define ERDMA_MAX_ORD 128
 16#define ERDMA_MAX_IRD 128
 17#define ERDMA_MAX_SGE_RD 1
 18#define ERDMA_MAX_CONTEXT (128 * 1024)
 19#define ERDMA_MAX_SEND_SGE 6
 20#define ERDMA_MAX_RECV_SGE 1
 21#define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))
 22#define ERDMA_MAX_FRMR_PA 512
 23
 24enum {
 25	ERDMA_MMAP_IO_NC = 0, /* no cache */
 26};
 27
 28struct erdma_user_mmap_entry {
 29	struct rdma_user_mmap_entry rdma_entry;
 30	u64 address;
 31	u8 mmap_flag;
 32};
 33
 34struct erdma_ext_db_info {
 35	bool enable;
 36	u16 sdb_off;
 37	u16 rdb_off;
 38	u16 cdb_off;
 39};
 40
 41struct erdma_ucontext {
 42	struct ib_ucontext ibucontext;
 43
 44	struct erdma_ext_db_info ext_db;
 45
 46	u64 sdb;
 47	u64 rdb;
 48	u64 cdb;
 49
 50	struct rdma_user_mmap_entry *sq_db_mmap_entry;
 51	struct rdma_user_mmap_entry *rq_db_mmap_entry;
 52	struct rdma_user_mmap_entry *cq_db_mmap_entry;
 53
 54	/* doorbell records */
 55	struct list_head dbrecords_page_list;
 56	struct mutex dbrecords_page_mutex;
 57};
 58
 59struct erdma_pd {
 60	struct ib_pd ibpd;
 61	u32 pdn;
 62};
 63
 64/*
 65 * MemoryRegion definition.
 66 */
 67#define ERDMA_MAX_INLINE_MTT_ENTRIES 4
 68#define MTT_SIZE(mtt_cnt) ((mtt_cnt) << 3) /* per mtt entry takes 8 Bytes. */
 69#define ERDMA_MR_MAX_MTT_CNT 524288
 70#define ERDMA_MTT_ENTRY_SIZE 8
 71
 72#define ERDMA_MR_TYPE_NORMAL 0
 73#define ERDMA_MR_TYPE_FRMR 1
 74#define ERDMA_MR_TYPE_DMA 2
 75
 76#define ERDMA_MR_MTT_0LEVEL 0
 77#define ERDMA_MR_MTT_1LEVEL 1
 78
 79#define ERDMA_MR_ACC_RA BIT(0)
 80#define ERDMA_MR_ACC_LR BIT(1)
 81#define ERDMA_MR_ACC_LW BIT(2)
 82#define ERDMA_MR_ACC_RR BIT(3)
 83#define ERDMA_MR_ACC_RW BIT(4)
 84
 85static inline u8 to_erdma_access_flags(int access)
 86{
 87	return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |
 88	       (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |
 89	       (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) |
 90	       (access & IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0);
 91}
 92
 93/* Hierarchical storage structure for MTT entries */
 94struct erdma_mtt {
 95	u64 *buf;
 96	size_t size;
 97
 98	bool continuous;
 99	union {
100		dma_addr_t buf_dma;
101		struct {
102			dma_addr_t *dma_addrs;
103			u32 npages;
104			u32 level;
105		};
106	};
107
108	struct erdma_mtt *low_level;
109};
110
111struct erdma_mem {
112	struct ib_umem *umem;
113	struct erdma_mtt *mtt;
114
115	u32 page_size;
116	u32 page_offset;
117	u32 page_cnt;
118	u32 mtt_nents;
119
120	u64 va;
121	u64 len;
122};
123
124struct erdma_mr {
125	struct ib_mr ibmr;
126	struct erdma_mem mem;
127	u8 type;
128	u8 access;
129	u8 valid;
130};
131
132struct erdma_user_dbrecords_page {
133	struct list_head list;
134	struct ib_umem *umem;
135	u64 va;
136	int refcnt;
137};
138
139struct erdma_av {
140	u8 port;
141	u8 hop_limit;
142	u8 traffic_class;
143	u8 sl;
144	u8 sgid_index;
145	u16 udp_sport;
146	u32 flow_label;
147	u8 dmac[ETH_ALEN];
148	u8 dgid[ERDMA_ROCEV2_GID_SIZE];
149	enum erdma_network_type ntype;
150};
151
152struct erdma_ah {
153	struct ib_ah ibah;
154	struct erdma_av av;
155	u32 ahn;
156};
157
158struct erdma_uqp {
159	struct erdma_mem sq_mem;
160	struct erdma_mem rq_mem;
161
162	dma_addr_t sq_dbrec_dma;
163	dma_addr_t rq_dbrec_dma;
164
165	struct erdma_user_dbrecords_page *user_dbr_page;
166
167	u32 rq_offset;
168};
169
170struct erdma_kqp {
171	u16 sq_pi;
172	u16 sq_ci;
173
174	u16 rq_pi;
175	u16 rq_ci;
176
177	u64 *swr_tbl;
178	u64 *rwr_tbl;
179
180	void __iomem *hw_sq_db;
181	void __iomem *hw_rq_db;
182
183	void *sq_buf;
184	dma_addr_t sq_buf_dma_addr;
185
186	void *rq_buf;
187	dma_addr_t rq_buf_dma_addr;
188
189	void *sq_dbrec;
190	void *rq_dbrec;
191
192	dma_addr_t sq_dbrec_dma;
193	dma_addr_t rq_dbrec_dma;
194
195	u8 sig_all;
196};
197
198enum erdma_qps_iwarp {
199	ERDMA_QPS_IWARP_IDLE = 0,
200	ERDMA_QPS_IWARP_RTR = 1,
201	ERDMA_QPS_IWARP_RTS = 2,
202	ERDMA_QPS_IWARP_CLOSING = 3,
203	ERDMA_QPS_IWARP_TERMINATE = 4,
204	ERDMA_QPS_IWARP_ERROR = 5,
205	ERDMA_QPS_IWARP_UNDEF = 6,
206	ERDMA_QPS_IWARP_COUNT = 7,
207};
208
209enum erdma_qpa_mask_iwarp {
210	ERDMA_QPA_IWARP_STATE = (1 << 0),
211	ERDMA_QPA_IWARP_LLP_HANDLE = (1 << 2),
212	ERDMA_QPA_IWARP_ORD = (1 << 3),
213	ERDMA_QPA_IWARP_IRD = (1 << 4),
214	ERDMA_QPA_IWARP_SQ_SIZE = (1 << 5),
215	ERDMA_QPA_IWARP_RQ_SIZE = (1 << 6),
216	ERDMA_QPA_IWARP_MPA = (1 << 7),
217	ERDMA_QPA_IWARP_CC = (1 << 8),
218};
219
220enum erdma_qps_rocev2 {
221	ERDMA_QPS_ROCEV2_RESET = 0,
222	ERDMA_QPS_ROCEV2_INIT = 1,
223	ERDMA_QPS_ROCEV2_RTR = 2,
224	ERDMA_QPS_ROCEV2_RTS = 3,
225	ERDMA_QPS_ROCEV2_SQD = 4,
226	ERDMA_QPS_ROCEV2_SQE = 5,
227	ERDMA_QPS_ROCEV2_ERROR = 6,
228	ERDMA_QPS_ROCEV2_COUNT = 7,
229};
230
231enum erdma_qpa_mask_rocev2 {
232	ERDMA_QPA_ROCEV2_STATE = (1 << 0),
233	ERDMA_QPA_ROCEV2_QKEY = (1 << 1),
234	ERDMA_QPA_ROCEV2_AV = (1 << 2),
235	ERDMA_QPA_ROCEV2_SQ_PSN = (1 << 3),
236	ERDMA_QPA_ROCEV2_RQ_PSN = (1 << 4),
237	ERDMA_QPA_ROCEV2_DST_QPN = (1 << 5),
238};
239
240enum erdma_qp_flags {
241	ERDMA_QP_IN_FLUSHING = (1 << 0),
242};
243
244#define ERDMA_QP_ACTIVE 0
245#define ERDMA_QP_PASSIVE 1
246
247struct erdma_mod_qp_params_iwarp {
248	enum erdma_qps_iwarp state;
249	enum erdma_cc_alg cc;
250	u8 qp_type;
251	u8 pd_len;
252	u32 irq_size;
253	u32 orq_size;
254};
255
256struct erdma_qp_attrs_iwarp {
257	enum erdma_qps_iwarp state;
258	u32 cookie;
259};
260
261struct erdma_mod_qp_params_rocev2 {
262	enum erdma_qps_rocev2 state;
263	u32 qkey;
264	u32 sq_psn;
265	u32 rq_psn;
266	u32 dst_qpn;
267	struct erdma_av av;
268};
269
270union erdma_mod_qp_params {
271	struct erdma_mod_qp_params_iwarp iwarp;
272	struct erdma_mod_qp_params_rocev2 rocev2;
273};
274
275struct erdma_qp_attrs_rocev2 {
276	enum erdma_qps_rocev2 state;
277	u32 qkey;
278	u32 dst_qpn;
279	struct erdma_av av;
280};
281
282struct erdma_qp_attrs {
283	enum erdma_cc_alg cc; /* Congestion control algorithm */
284	u32 sq_size;
285	u32 rq_size;
286	u32 orq_size;
287	u32 irq_size;
288	u32 max_send_sge;
289	u32 max_recv_sge;
290	union {
291		struct erdma_qp_attrs_iwarp iwarp;
292		struct erdma_qp_attrs_rocev2 rocev2;
293	};
294};
295
296struct erdma_qp {
297	struct ib_qp ibqp;
298	struct kref ref;
299	struct completion safe_free;
300	struct erdma_dev *dev;
301	struct erdma_cep *cep;
302	struct rw_semaphore state_lock;
303
304	unsigned long flags;
305	struct delayed_work reflush_dwork;
306
307	union {
308		struct erdma_kqp kern_qp;
309		struct erdma_uqp user_qp;
310	};
311
312	struct erdma_cq *scq;
313	struct erdma_cq *rcq;
314
315	struct erdma_qp_attrs attrs;
316	spinlock_t lock;
317};
318
319struct erdma_kcq_info {
320	void *qbuf;
321	dma_addr_t qbuf_dma_addr;
322	u32 ci;
323	u32 cmdsn;
324	u32 notify_cnt;
325
326	spinlock_t lock;
327	u8 __iomem *db;
328	u64 *dbrec;
329	dma_addr_t dbrec_dma;
330};
331
332struct erdma_ucq_info {
333	struct erdma_mem qbuf_mem;
334	struct erdma_user_dbrecords_page *user_dbr_page;
335	dma_addr_t dbrec_dma;
336};
337
338struct erdma_cq {
339	struct ib_cq ibcq;
340	u32 cqn;
341
342	u32 depth;
343	u32 assoc_eqn;
344
345	union {
346		struct erdma_kcq_info kern_cq;
347		struct erdma_ucq_info user_cq;
348	};
349};
350
351#define QP_ID(qp) ((qp)->ibqp.qp_num)
352
353static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)
354{
355	return (struct erdma_qp *)xa_load(&dev->qp_xa, id);
356}
357
358static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)
359{
360	return (struct erdma_cq *)xa_load(&dev->cq_xa, id);
361}
362
363void erdma_qp_get(struct erdma_qp *qp);
364void erdma_qp_put(struct erdma_qp *qp);
365int erdma_modify_qp_state_iwarp(struct erdma_qp *qp,
366				struct erdma_mod_qp_params_iwarp *params,
367				int mask);
368int erdma_modify_qp_state_rocev2(struct erdma_qp *qp,
369				 struct erdma_mod_qp_params_rocev2 *params,
370				 int attr_mask);
371void erdma_qp_llp_close(struct erdma_qp *qp);
372void erdma_qp_cm_drop(struct erdma_qp *qp);
373
374static inline bool erdma_device_iwarp(struct erdma_dev *dev)
375{
376	return dev->proto == ERDMA_PROTO_IWARP;
377}
378
379static inline bool erdma_device_rocev2(struct erdma_dev *dev)
380{
381	return dev->proto == ERDMA_PROTO_ROCEV2;
382}
383
384static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
385{
386	return container_of(ibctx, struct erdma_ucontext, ibucontext);
387}
388
389static inline struct erdma_pd *to_epd(struct ib_pd *pd)
390{
391	return container_of(pd, struct erdma_pd, ibpd);
392}
393
394static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)
395{
396	return container_of(ibmr, struct erdma_mr, ibmr);
397}
398
399static inline struct erdma_qp *to_eqp(struct ib_qp *qp)
400{
401	return container_of(qp, struct erdma_qp, ibqp);
402}
403
404static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)
405{
406	return container_of(ibcq, struct erdma_cq, ibcq);
407}
408
409static inline struct erdma_ah *to_eah(struct ib_ah *ibah)
410{
411	return container_of(ibah, struct erdma_ah, ibah);
412}
413
414static inline int erdma_check_gid_attr(const struct ib_gid_attr *attr)
415{
416	u8 ntype = rdma_gid_attr_network_type(attr);
417
418	if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6)
419		return -EINVAL;
420
421	return 0;
422}
423
424static inline struct erdma_user_mmap_entry *
425to_emmap(struct rdma_user_mmap_entry *ibmmap)
426{
427	return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);
428}
429
430int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data);
431void erdma_dealloc_ucontext(struct ib_ucontext *ibctx);
432int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,
433		       struct ib_udata *data);
434int erdma_get_port_immutable(struct ib_device *dev, u32 port,
435			     struct ib_port_immutable *ib_port_immutable);
436int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
437		    struct uverbs_attr_bundle *attrs);
438int erdma_query_port(struct ib_device *dev, u32 port,
439		     struct ib_port_attr *attr);
440int erdma_query_gid(struct ib_device *dev, u32 port, int idx,
441		    union ib_gid *gid);
442int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data);
443int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
444int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
445		    struct ib_udata *data);
446int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
447		   struct ib_qp_init_attr *init_attr);
448int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
449		    struct ib_udata *data);
450int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
451int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
452void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext);
453int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
454struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
455				u64 virt, int access, struct ib_dmah *dmah,
456				struct ib_udata *udata);
457struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
458int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
459int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
460void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
461void erdma_qp_get_ref(struct ib_qp *ibqp);
462void erdma_qp_put_ref(struct ib_qp *ibqp);
463struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);
464int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
465		    const struct ib_send_wr **bad_send_wr);
466int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
467		    const struct ib_recv_wr **bad_recv_wr);
468int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
469void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn);
470struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
471				u32 max_num_sg);
472int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
473		    unsigned int *sg_offset);
474void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);
475void erdma_set_mtu(struct erdma_dev *dev, u32 mtu);
476struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
477						u32 port_num);
478int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
479		       u32 port, int index);
480enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev,
481					  u32 port_num);
482int erdma_add_gid(const struct ib_gid_attr *attr, void **context);
483int erdma_del_gid(const struct ib_gid_attr *attr, void **context);
484int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey);
485void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av);
486int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
487		    struct ib_udata *udata);
488int erdma_destroy_ah(struct ib_ah *ibah, u32 flags);
489int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
490
491#endif