ionic_ibdev.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
  3
  4#ifndef _IONIC_IBDEV_H_
  5#define _IONIC_IBDEV_H_
  6
  7#include <rdma/ib_umem.h>
  8#include <rdma/ib_verbs.h>
  9#include <rdma/ib_pack.h>
 10#include <rdma/uverbs_ioctl.h>
 11
 12#include <rdma/ionic-abi.h>
 13#include <ionic_api.h>
 14#include <ionic_regs.h>
 15
 16#include "ionic_fw.h"
 17#include "ionic_queue.h"
 18#include "ionic_res.h"
 19
 20#include "ionic_lif_cfg.h"
 21
 22/* Config knobs */
 23#define IONIC_EQ_DEPTH 511
 24#define IONIC_EQ_COUNT 32
 25#define IONIC_AQ_DEPTH 63
 26#define IONIC_AQ_COUNT 4
 27#define IONIC_EQ_ISR_BUDGET 10
 28#define IONIC_EQ_WORK_BUDGET 1000
 29#define IONIC_MAX_RD_ATOM 16
 30#define IONIC_PKEY_TBL_LEN 1
 31#define IONIC_GID_TBL_LEN 256
 32
 33#define IONIC_MAX_QPID 0xffffff
 34#define IONIC_SPEC_HIGH 8
 35#define IONIC_MAX_PD 1024
 36#define IONIC_SPEC_HIGH 8
 37#define IONIC_SQCMB_ORDER 5
 38#define IONIC_RQCMB_ORDER 0
 39
 40#define IONIC_META_LAST		((void *)1ul)
 41#define IONIC_META_POSTED	((void *)2ul)
 42
 43#define IONIC_CQ_GRACE 100
 44
 45#define IONIC_ROCE_UDP_SPORT	28272
 46#define IONIC_DMA_LKEY		0
 47#define IONIC_DMA_RKEY		IONIC_DMA_LKEY
 48
 49#define IONIC_CMB_SUPPORTED \
 50	(IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
 51	 IONIC_CMB_WC | IONIC_CMB_UC)
 52
 53/* resource is not reserved on the device, indicated in tbl_order */
 54#define IONIC_RES_INVALID	-1
 55
 56struct ionic_aq;
 57struct ionic_cq;
 58struct ionic_eq;
 59struct ionic_vcq;
 60
 61enum ionic_admin_state {
 62	IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
 63	IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
 64	IONIC_ADMIN_KILLED, /* not submitting, locally completed */
 65};
 66
 67enum ionic_admin_flags {
 68	IONIC_ADMIN_F_BUSYWAIT  = BIT(0),	/* Don't sleep */
 69	IONIC_ADMIN_F_TEARDOWN  = BIT(1),	/* In destroy path */
 70	IONIC_ADMIN_F_INTERRUPT = BIT(2),	/* Interruptible w/timeout */
 71};
 72
 73enum ionic_mmap_flag {
 74	IONIC_MMAP_WC = BIT(0),
 75};
 76
 77struct ionic_mmap_entry {
 78	struct rdma_user_mmap_entry rdma_entry;
 79	unsigned long size;
 80	unsigned long pfn;
 81	u8 mmap_flags;
 82};
 83
 84struct ionic_ibdev {
 85	struct ib_device	ibdev;
 86
 87	struct ionic_lif_cfg	lif_cfg;
 88
 89	struct xarray		qp_tbl;
 90	struct xarray		cq_tbl;
 91
 92	struct ionic_resid_bits	inuse_dbid;
 93	struct ionic_resid_bits	inuse_pdid;
 94	struct ionic_resid_bits	inuse_ahid;
 95	struct ionic_resid_bits	inuse_mrid;
 96	struct ionic_resid_bits	inuse_qpid;
 97	struct ionic_resid_bits	inuse_cqid;
 98
 99	u8			half_cqid_udma_shift;
100	u8			half_qpid_udma_shift;
101	u8			next_qpid_udma_idx;
102	u8			next_mrkey;
103
104	struct work_struct	reset_work;
105	bool			reset_posted;
106	u32			reset_cnt;
107
108	struct delayed_work	admin_dwork;
109	struct ionic_aq		**aq_vec;
110	atomic_t		admin_state;
111
112	struct ionic_eq		**eq_vec;
113
114	struct ionic_v1_stat	*hw_stats;
115	void			*hw_stats_buf;
116	struct rdma_stat_desc	*hw_stats_hdrs;
117	struct ionic_counter_stats *counter_stats;
118	int			hw_stats_count;
119};
120
121struct ionic_eq {
122	struct ionic_ibdev	*dev;
123
124	u32			eqid;
125	u32			intr;
126
127	struct ionic_queue	q;
128
129	int			armed;
130	bool			enable;
131
132	struct work_struct	work;
133
134	int			irq;
135	char			name[32];
136};
137
138struct ionic_admin_wr {
139	struct completion		work;
140	struct list_head		aq_ent;
141	struct ionic_v1_admin_wqe	wqe;
142	struct ionic_v1_cqe		cqe;
143	struct ionic_aq			*aq;
144	int				status;
145};
146
147struct ionic_admin_wr_q {
148	struct ionic_admin_wr	*wr;
149	int			wqe_strides;
150};
151
152struct ionic_aq {
153	struct ionic_ibdev	*dev;
154	struct ionic_vcq	*vcq;
155
156	struct work_struct	work;
157
158	atomic_t		admin_state;
159	unsigned long		stamp;
160	bool			armed;
161
162	u32			aqid;
163	u32			cqid;
164
165	spinlock_t		lock; /* for posting */
166	struct ionic_queue	q;
167	struct ionic_admin_wr_q	*q_wr;
168	struct list_head	wr_prod;
169	struct list_head	wr_post;
170};
171
172struct ionic_ctx {
173	struct ib_ucontext	ibctx;
174	u32			dbid;
175	struct rdma_user_mmap_entry	*mmap_dbell;
176};
177
178struct ionic_tbl_buf {
179	u32		tbl_limit;
180	u32		tbl_pages;
181	size_t		tbl_size;
182	__le64		*tbl_buf;
183	dma_addr_t	tbl_dma;
184	u8		page_size_log2;
185};
186
187struct ionic_pd {
188	struct ib_pd		ibpd;
189
190	u32			pdid;
191	u32			flags;
192};
193
194struct ionic_cq {
195	struct ionic_vcq	*vcq;
196
197	u32			cqid;
198	u32			eqid;
199
200	spinlock_t		lock; /* for polling */
201	struct list_head	poll_sq;
202	bool			flush;
203	struct list_head	flush_sq;
204	struct list_head	flush_rq;
205	struct list_head	ibkill_flush_ent;
206
207	struct ionic_queue	q;
208	bool			color;
209	int			credit;
210	u16			arm_any_prod;
211	u16			arm_sol_prod;
212
213	struct kref		cq_kref;
214	struct completion	cq_rel_comp;
215
216	/* infrequently accessed, keep at end */
217	struct ib_umem		*umem;
218};
219
220struct ionic_vcq {
221	struct ib_cq		ibcq;
222	struct ionic_cq		cq[2];
223	u8			udma_mask;
224	u8			poll_idx;
225};
226
227struct ionic_sq_meta {
228	u64			wrid;
229	u32			len;
230	u16			seq;
231	u8			ibop;
232	u8			ibsts;
233	u8			remote:1;
234	u8			signal:1;
235	u8			local_comp:1;
236};
237
238struct ionic_rq_meta {
239	struct ionic_rq_meta	*next;
240	u64			wrid;
241};
242
243struct ionic_qp {
244	struct ib_qp		ibqp;
245	enum ib_qp_state	state;
246
247	u32			qpid;
248	u32			ahid;
249	u32			sq_cqid;
250	u32			rq_cqid;
251	u8			udma_idx;
252	u8			has_ah:1;
253	u8			has_sq:1;
254	u8			has_rq:1;
255	u8			sig_all:1;
256
257	struct list_head	qp_list_counter;
258
259	struct list_head	cq_poll_sq;
260	struct list_head	cq_flush_sq;
261	struct list_head	cq_flush_rq;
262	struct list_head	ibkill_flush_ent;
263
264	spinlock_t		sq_lock; /* for posting and polling */
265	struct ionic_queue	sq;
266	struct ionic_sq_meta	*sq_meta;
267	u16			*sq_msn_idx;
268	int			sq_spec;
269	u16			sq_old_prod;
270	u16			sq_msn_prod;
271	u16			sq_msn_cons;
272	u8			sq_cmb;
273	bool			sq_flush;
274	bool			sq_flush_rcvd;
275
276	spinlock_t		rq_lock; /* for posting and polling */
277	struct ionic_queue	rq;
278	struct ionic_rq_meta	*rq_meta;
279	struct ionic_rq_meta	*rq_meta_head;
280	int			rq_spec;
281	u16			rq_old_prod;
282	u8			rq_cmb;
283	bool			rq_flush;
284
285	struct kref		qp_kref;
286	struct completion	qp_rel_comp;
287
288	/* infrequently accessed, keep at end */
289	int			sgid_index;
290	int			sq_cmb_order;
291	u32			sq_cmb_pgid;
292	phys_addr_t		sq_cmb_addr;
293	struct rdma_user_mmap_entry *mmap_sq_cmb;
294
295	struct ib_umem		*sq_umem;
296
297	int			rq_cmb_order;
298	u32			rq_cmb_pgid;
299	phys_addr_t		rq_cmb_addr;
300	struct rdma_user_mmap_entry *mmap_rq_cmb;
301
302	struct ib_umem		*rq_umem;
303
304	int			dcqcn_profile;
305
306	struct ib_ud_header	*hdr;
307};
308
309struct ionic_ah {
310	struct ib_ah		ibah;
311	u32			ahid;
312	int			sgid_index;
313	struct ib_ud_header	hdr;
314};
315
316struct ionic_mr {
317	union {
318		struct ib_mr	ibmr;
319		struct ib_mw	ibmw;
320	};
321
322	u32			mrid;
323	int			flags;
324
325	struct ib_umem		*umem;
326	struct ionic_tbl_buf	buf;
327	bool			created;
328};
329
330struct ionic_counter_stats {
331	int queue_stats_count;
332	struct ionic_v1_stat *hdr;
333	struct rdma_stat_desc *stats_hdrs;
334	struct xarray xa_counters;
335};
336
337struct ionic_counter {
338	void *vals;
339	struct list_head qp_list;
340};
341
342static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
343{
344	return container_of(ibdev, struct ionic_ibdev, ibdev);
345}
346
347static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)
348{
349	return container_of(ibctx, struct ionic_ctx, ibctx);
350}
351
352static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)
353{
354	if (!uobj)
355		return NULL;
356
357	if (!uobj->context)
358		return NULL;
359
360	return to_ionic_ctx(uobj->context);
361}
362
363static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)
364{
365	return container_of(ibpd, struct ionic_pd, ibpd);
366}
367
368static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)
369{
370	return container_of(ibmr, struct ionic_mr, ibmr);
371}
372
373static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)
374{
375	return container_of(ibmw, struct ionic_mr, ibmw);
376}
377
378static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)
379{
380	return container_of(ibcq, struct ionic_vcq, ibcq);
381}
382
383static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,
384					       uint8_t udma_idx)
385{
386	return &to_ionic_vcq(ibcq)->cq[udma_idx];
387}
388
389static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
390{
391	return container_of(ibqp, struct ionic_qp, ibqp);
392}
393
394static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
395{
396	return container_of(ibah, struct ionic_ah, ibah);
397}
398
399static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,
400				 struct ionic_ctx *ctx)
401{
402	if (!ctx)
403		return dev->lif_cfg.dbid;
404
405	return ctx->dbid;
406}
407
408static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,
409				 struct ib_uobject *uobj)
410{
411	return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));
412}
413
414static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)
415{
416	return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;
417}
418
419static inline void ionic_qp_complete(struct kref *kref)
420{
421	struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);
422
423	complete(&qp->qp_rel_comp);
424}
425
426static inline void ionic_cq_complete(struct kref *kref)
427{
428	struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);
429
430	complete(&cq->cq_rel_comp);
431}
432
433/* ionic_admin.c */
434extern struct workqueue_struct *ionic_evt_workq;
435void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
436int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
437		     enum ionic_admin_flags);
438
439int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);
440
441int ionic_create_rdma_admin(struct ionic_ibdev *dev);
442void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);
443void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);
444
445/* ionic_controlpath.c */
446int ionic_create_cq_common(struct ionic_vcq *vcq,
447			   struct ionic_tbl_buf *buf,
448			   const struct ib_cq_init_attr *attr,
449			   struct ionic_ctx *ctx,
450			   struct ib_udata *udata,
451			   struct ionic_qdesc *req_cq,
452			   __u32 *resp_cqid,
453			   int udma_idx);
454void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
455void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
456void ionic_notify_flush_cq(struct ionic_cq *cq);
457
458int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);
459void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);
460int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);
461void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
462int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
463int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
464int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
465		    struct ib_udata *udata);
466int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
467int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);
468struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);
469struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
470				u64 addr, int access, struct ib_dmah *dmah,
471				struct ib_udata *udata);
472struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
473				       u64 length, u64 addr, int fd, int access,
474					   struct ib_dmah *dmah,
475				       struct uverbs_attr_bundle *attrs);
476int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
477struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
478			     u32 max_sg);
479int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
480		    unsigned int *sg_offset);
481int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
482int ionic_dealloc_mw(struct ib_mw *ibmw);
483int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
484		    struct uverbs_attr_bundle *attrs);
485int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
486int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
487		    struct ib_udata *udata);
488int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
489		    struct ib_udata *udata);
490int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
491		   struct ib_qp_init_attr *init_attr);
492int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
493
494/* ionic_datapath.c */
495int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
496		    const struct ib_send_wr **bad);
497int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
498		    const struct ib_recv_wr **bad);
499int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);
500int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
501
502/* ionic_hw_stats.c */
503void ionic_stats_init(struct ionic_ibdev *dev);
504void ionic_stats_cleanup(struct ionic_ibdev *dev);
505
506/* ionic_pgtbl.c */
507__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);
508__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);
509int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);
510int ionic_pgtbl_init(struct ionic_ibdev *dev,
511		     struct ionic_tbl_buf *buf,
512		     struct ib_umem *umem,
513		     dma_addr_t dma,
514		     int limit,
515		     u64 page_size);
516void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);
517#endif /* _IONIC_IBDEV_H_ */