rxe_verbs.h

  1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2/*
  3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
  4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  5 */
  6
  7#ifndef RXE_VERBS_H
  8#define RXE_VERBS_H
  9
 10#include <linux/interrupt.h>
 11#include <linux/workqueue.h>
 12#include "rxe_pool.h"
 13#include "rxe_task.h"
 14#include "rxe_hw_counters.h"
 15
 16static inline int pkey_match(u16 key1, u16 key2)
 17{
 18	return (((key1 & 0x7fff) != 0) &&
 19		((key1 & 0x7fff) == (key2 & 0x7fff)) &&
 20		((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
 21}
 22
 23/* Return >0 if psn_a > psn_b
 24 *	   0 if psn_a == psn_b
 25 *	  <0 if psn_a < psn_b
 26 */
 27static inline int psn_compare(u32 psn_a, u32 psn_b)
 28{
 29	s32 diff;
 30
 31	diff = (psn_a - psn_b) << 8;
 32	return diff;
 33}
 34
 35struct rxe_ucontext {
 36	struct ib_ucontext ibuc;
 37	struct rxe_pool_elem	elem;
 38};
 39
 40struct rxe_pd {
 41	struct ib_pd            ibpd;
 42	struct rxe_pool_elem	elem;
 43};
 44
 45struct rxe_ah {
 46	struct ib_ah		ibah;
 47	struct rxe_pool_elem	elem;
 48	struct rxe_av		av;
 49	bool			is_user;
 50	int			ah_num;
 51};
 52
 53struct rxe_cqe {
 54	union {
 55		struct ib_wc		ibwc;
 56		struct ib_uverbs_wc	uibwc;
 57	};
 58};
 59
 60struct rxe_cq {
 61	struct ib_cq		ibcq;
 62	struct rxe_pool_elem	elem;
 63	struct rxe_queue	*queue;
 64	spinlock_t		cq_lock;
 65	u8			notify;
 66	bool			is_user;
 67	atomic_t		num_wq;
 68};
 69
 70enum wqe_state {
 71	wqe_state_posted,
 72	wqe_state_processing,
 73	wqe_state_pending,
 74	wqe_state_done,
 75	wqe_state_error,
 76};
 77
 78struct rxe_sq {
 79	int			max_wr;
 80	int			max_sge;
 81	int			max_inline;
 82	spinlock_t		sq_lock; /* guard queue */
 83	struct rxe_queue	*queue;
 84};
 85
 86struct rxe_rq {
 87	int			max_wr;
 88	int			max_sge;
 89	spinlock_t		producer_lock; /* guard queue producer */
 90	spinlock_t		consumer_lock; /* guard queue consumer */
 91	struct rxe_queue	*queue;
 92};
 93
 94struct rxe_srq {
 95	struct ib_srq		ibsrq;
 96	struct rxe_pool_elem	elem;
 97	struct rxe_pd		*pd;
 98	struct rxe_rq		rq;
 99	u32			srq_num;
100
101	int			limit;
102	int			error;
103};
104
105struct rxe_req_info {
106	int			wqe_index;
107	u32			psn;
108	int			opcode;
109	atomic_t		rd_atomic;
110	int			wait_fence;
111	int			need_rd_atomic;
112	int			wait_psn;
113	int			need_retry;
114	int			wait_for_rnr_timer;
115	int			noack_pkts;
116	int			again;
117};
118
119struct rxe_comp_info {
120	u32			psn;
121	int			opcode;
122	int			timeout;
123	int			timeout_retry;
124	int			started_retry;
125	u32			retry_cnt;
126	u32			rnr_retry;
127};
128
129/* responder states */
130enum resp_states {
131	RESPST_NONE,
132	RESPST_GET_REQ,
133	RESPST_CHK_PSN,
134	RESPST_CHK_OP_SEQ,
135	RESPST_CHK_OP_VALID,
136	RESPST_CHK_RESOURCE,
137	RESPST_CHK_LENGTH,
138	RESPST_CHK_RKEY,
139	RESPST_EXECUTE,
140	RESPST_READ_REPLY,
141	RESPST_ATOMIC_REPLY,
142	RESPST_ATOMIC_WRITE_REPLY,
143	RESPST_PROCESS_FLUSH,
144	RESPST_COMPLETE,
145	RESPST_ACKNOWLEDGE,
146	RESPST_CLEANUP,
147	RESPST_DUPLICATE_REQUEST,
148	RESPST_ERR_MALFORMED_WQE,
149	RESPST_ERR_UNSUPPORTED_OPCODE,
150	RESPST_ERR_MISALIGNED_ATOMIC,
151	RESPST_ERR_PSN_OUT_OF_SEQ,
152	RESPST_ERR_MISSING_OPCODE_FIRST,
153	RESPST_ERR_MISSING_OPCODE_LAST_C,
154	RESPST_ERR_MISSING_OPCODE_LAST_D1E,
155	RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
156	RESPST_ERR_RNR,
157	RESPST_ERR_RKEY_VIOLATION,
158	RESPST_ERR_INVALIDATE_RKEY,
159	RESPST_ERR_LENGTH,
160	RESPST_ERR_CQ_OVERFLOW,
161	RESPST_ERROR,
162	RESPST_DONE,
163	RESPST_EXIT,
164};
165
166enum rdatm_res_state {
167	rdatm_res_state_next,
168	rdatm_res_state_new,
169	rdatm_res_state_replay,
170};
171
172struct resp_res {
173	int			type;
174	int			replay;
175	u32			first_psn;
176	u32			last_psn;
177	u32			cur_psn;
178	enum rdatm_res_state	state;
179
180	union {
181		struct {
182			u64		orig_val;
183		} atomic;
184		struct {
185			u64		va_org;
186			u32		rkey;
187			u32		length;
188			u64		va;
189			u32		resid;
190		} read;
191		struct {
192			u32		length;
193			u64		va;
194			u8		type;
195			u8		level;
196		} flush;
197	};
198};
199
200struct rxe_resp_info {
201	u32			msn;
202	u32			psn;
203	u32			ack_psn;
204	int			opcode;
205	int			drop_msg;
206	int			goto_error;
207	int			sent_psn_nak;
208	enum ib_wc_status	status;
209	u8			aeth_syndrome;
210
211	/* Receive only */
212	struct rxe_recv_wqe	*wqe;
213
214	/* RDMA read / atomic only */
215	u64			va;
216	u64			offset;
217	struct rxe_mr		*mr;
218	u32			resid;
219	u32			rkey;
220	u32			length;
221
222	/* SRQ only */
223	struct {
224		struct rxe_recv_wqe	wqe;
225		struct ib_sge		sge[RXE_MAX_SGE];
226	} srq_wqe;
227
228	/* Responder resources. It's a circular list where the oldest
229	 * resource is dropped first.
230	 */
231	struct resp_res		*resources;
232	unsigned int		res_head;
233	unsigned int		res_tail;
234	struct resp_res		*res;
235};
236
237struct rxe_qp {
238	struct ib_qp		ibqp;
239	struct rxe_pool_elem	elem;
240	struct ib_qp_attr	attr;
241	unsigned int		valid;
242	unsigned int		mtu;
243	bool			is_user;
244
245	struct rxe_pd		*pd;
246	struct rxe_srq		*srq;
247	struct rxe_cq		*scq;
248	struct rxe_cq		*rcq;
249
250	enum ib_sig_type	sq_sig_type;
251
252	struct rxe_sq		sq;
253	struct rxe_rq		rq;
254
255	struct socket		*sk;
256	u32			dst_cookie;
257	u16			src_port;
258
259	struct rxe_av		pri_av;
260	struct rxe_av		alt_av;
261
262	atomic_t		mcg_num;
263
264	struct sk_buff_head	req_pkts;
265	struct sk_buff_head	resp_pkts;
266
267	struct rxe_task		send_task;
268	struct rxe_task		recv_task;
269
270	struct rxe_req_info	req;
271	struct rxe_comp_info	comp;
272	struct rxe_resp_info	resp;
273
274	atomic_t		ssn;
275	atomic_t		skb_out;
276	int			need_req_skb;
277
278	/* Timer for retranmitting packet when ACKs have been lost. RC
279	 * only. The requester sets it when it is not already
280	 * started. The responder resets it whenever an ack is
281	 * received.
282	 */
283	struct timer_list retrans_timer;
284	u64 qp_timeout_jiffies;
285
286	/* Timer for handling RNR NAKS. */
287	struct timer_list rnr_nak_timer;
288
289	spinlock_t		state_lock; /* guard requester and completer */
290
291	struct execute_work	cleanup_work;
292};
293
294enum {
295	RXE_ACCESS_REMOTE	= IB_ACCESS_REMOTE_READ
296				| IB_ACCESS_REMOTE_WRITE
297				| IB_ACCESS_REMOTE_ATOMIC,
298	RXE_ACCESS_SUPPORTED_MR	= RXE_ACCESS_REMOTE
299				| IB_ACCESS_LOCAL_WRITE
300				| IB_ACCESS_MW_BIND
301				| IB_ACCESS_ON_DEMAND
302				| IB_ACCESS_FLUSH_GLOBAL
303				| IB_ACCESS_FLUSH_PERSISTENT
304				| IB_ACCESS_OPTIONAL,
305	RXE_ACCESS_SUPPORTED_QP	= RXE_ACCESS_SUPPORTED_MR,
306	RXE_ACCESS_SUPPORTED_MW	= RXE_ACCESS_SUPPORTED_MR
307				| IB_ZERO_BASED,
308};
309
310enum rxe_mr_state {
311	RXE_MR_STATE_INVALID,
312	RXE_MR_STATE_FREE,
313	RXE_MR_STATE_VALID,
314};
315
316enum rxe_mr_copy_dir {
317	RXE_TO_MR_OBJ,
318	RXE_FROM_MR_OBJ,
319};
320
321enum rxe_mr_lookup_type {
322	RXE_LOOKUP_LOCAL,
323	RXE_LOOKUP_REMOTE,
324};
325
326enum rxe_rereg {
327	RXE_MR_REREG_SUPPORTED	= IB_MR_REREG_PD
328				| IB_MR_REREG_ACCESS,
329};
330
331static inline int rkey_is_mw(u32 rkey)
332{
333	u32 index = rkey >> 8;
334
335	return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
336}
337
338struct rxe_mr_page {
339	struct page		*page;
340	unsigned int		offset; /* offset in system page */
341};
342
343struct rxe_mr {
344	struct rxe_pool_elem	elem;
345	struct ib_mr		ibmr;
346
347	struct ib_umem		*umem;
348
349	u32			lkey;
350	u32			rkey;
351	enum rxe_mr_state	state;
352	int			access;
353	atomic_t		num_mw;
354
355	unsigned int		page_offset;
356	unsigned int		page_shift;
357	u64			page_mask;
358
359	/* size of page_info when mr allocated */
360	u32			num_buf;
361	/* real size of page_info */
362	u32			max_allowed_buf;
363	u32			nbuf;
364
365	struct rxe_mr_page	*page_info;
366};
367
368static inline unsigned int mr_page_size(struct rxe_mr *mr)
369{
370	return mr ? mr->ibmr.page_size : PAGE_SIZE;
371}
372
373enum rxe_mw_state {
374	RXE_MW_STATE_INVALID	= RXE_MR_STATE_INVALID,
375	RXE_MW_STATE_FREE	= RXE_MR_STATE_FREE,
376	RXE_MW_STATE_VALID	= RXE_MR_STATE_VALID,
377};
378
379struct rxe_mw {
380	struct ib_mw		ibmw;
381	struct rxe_pool_elem	elem;
382	spinlock_t		lock;
383	enum rxe_mw_state	state;
384	struct rxe_qp		*qp; /* Type 2 only */
385	struct rxe_mr		*mr;
386	u32			rkey;
387	int			access;
388	u64			addr;
389	u64			length;
390};
391
392struct rxe_mcg {
393	struct rb_node		node;
394	struct kref		ref_cnt;
395	struct rxe_dev		*rxe;
396	struct list_head	qp_list;
397	union ib_gid		mgid;
398	atomic_t		qp_num;
399	u32			qkey;
400	u16			pkey;
401};
402
403struct rxe_mca {
404	struct list_head	qp_list;
405	struct rxe_qp		*qp;
406};
407
408struct rxe_port {
409	struct ib_port_attr	attr;
410	__be64			port_guid;
411	__be64			subnet_prefix;
412	spinlock_t		port_lock; /* guard port */
413	unsigned int		mtu_cap;
414	/* special QPs */
415	u32			qp_gsi_index;
416};
417
418#define	RXE_PORT	1
419struct rxe_dev {
420	struct ib_device	ib_dev;
421	struct ib_device_attr	attr;
422	int			max_ucontext;
423	int			max_inline_data;
424	struct mutex		usdev_lock;
425
426	char			raw_gid[ETH_ALEN];
427
428	struct rxe_pool		uc_pool;
429	struct rxe_pool		pd_pool;
430	struct rxe_pool		ah_pool;
431	struct rxe_pool		srq_pool;
432	struct rxe_pool		qp_pool;
433	struct rxe_pool		cq_pool;
434	struct rxe_pool		mr_pool;
435	struct rxe_pool		mw_pool;
436
437	/* multicast support */
438	spinlock_t		mcg_lock;
439	struct rb_root		mcg_tree;
440	atomic_t		mcg_num;
441	atomic_t		mcg_attach;
442
443	spinlock_t		pending_lock; /* guard pending_mmaps */
444	struct list_head	pending_mmaps;
445
446	spinlock_t		mmap_offset_lock; /* guard mmap_offset */
447	u64			mmap_offset;
448
449	atomic64_t		stats_counters[RXE_NUM_OF_COUNTERS];
450
451	struct rxe_port		port;
452};
453
454static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
455{
456	return ib_device_get_netdev(dev, RXE_PORT);
457}
458
459static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
460{
461	atomic64_inc(&rxe->stats_counters[index]);
462}
463
464static inline struct rxe_dev *to_rdev(struct ib_device *dev)
465{
466	return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
467}
468
469static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
470{
471	return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
472}
473
474static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
475{
476	return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
477}
478
479static inline struct rxe_ah *to_rah(struct ib_ah *ah)
480{
481	return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
482}
483
484static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
485{
486	return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
487}
488
489static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
490{
491	return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
492}
493
494static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
495{
496	return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
497}
498
499static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
500{
501	return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
502}
503
504static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
505{
506	return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
507}
508
509static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
510{
511	return to_rpd(ah->ibah.pd);
512}
513
514static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
515{
516	return to_rpd(mr->ibmr.pd);
517}
518
519static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
520{
521	return to_rpd(mw->ibmw.pd);
522}
523
524int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
525						struct net_device *ndev);
526
527#endif /* RXE_VERBS_H */